QC Dashboards and Visualizations¶

%%bash
# Create a test schema
cat > /tmp/dashboard_schema.yaml << 'EOF'
id: https://example.org/dashboard-demo
name: dashboard_demo
prefixes:
  linkml: https://w3id.org/linkml/
imports:
  - linkml:types
default_range: string

classes:
  Dataset:
    attributes:
      name:
        required: true
      description:
        recommended: true
      samples:
        multivalued: true
        range: Sample

  Sample:
    attributes:
      id:
        required: true
        identifier: true
      name:
        recommended: true
      tissue_type:
        recommended: true
      ontology_term:
        description: Ontology mapping for this sample
        recommended: true
      collection_date:
        recommended: true
EOF
echo "Schema created!"

%%bash
# Create test data with varying compliance
cat > /tmp/dashboard_data.yaml << 'EOF'
name: Tissue Expression Study
description: Analysis of gene expression across tissue types
samples:
  - id: SAMP001
    name: Liver Sample 1
    tissue_type: liver
    ontology_term: UBERON:0002107
    collection_date: "2024-01-15"
  - id: SAMP002
    name: Heart Sample 1
    tissue_type: heart
    ontology_term: UBERON:0000948
    # Missing collection_date
  - id: SAMP003
    name: Brain Sample 1
    tissue_type: brain
    # Missing ontology_term and collection_date
  - id: SAMP004
    # Missing name, tissue_type
    ontology_term: UBERON:0000955
    collection_date: "2024-02-20"
  - id: SAMP005
    name: Kidney Sample 1
    # Missing tissue_type, ontology_term, collection_date
  - id: SAMP006
    name: Lung Sample 1
    tissue_type: lung
    ontology_term: UBERON:0002048
    collection_date: "2024-03-01"
  - id: SAMP007
    # All recommended fields missing except required id
  - id: SAMP008
    name: Spleen Sample 1
    tissue_type: spleen
    ontology_term: UBERON:0002106
    collection_date: "2024-03-15"
EOF
echo "Data created!"

%%bash
linkml-data-qc /tmp/dashboard_data.yaml \
    -s /tmp/dashboard_schema.yaml \
    -t Dataset \
    --dashboard /tmp/basic_dashboard.png

echo "\nDashboard saved:"
ls -lh /tmp/basic_dashboard.png

%%bash
# Create QC configuration with weights and thresholds
cat > /tmp/dashboard_config.yaml << 'EOF'
default_weight: 1.0

slots:
  ontology_term:
    weight: 3.0           # Ontology mappings are critical
    min_compliance: 80.0  # At least 80% should have mappings
  tissue_type:
    weight: 2.0           # Tissue type is important
    min_compliance: 90.0
  name:
    weight: 1.5
    min_compliance: 95.0
  collection_date:
    weight: 1.0
    min_compliance: 70.0
EOF
echo "Configuration created!"

%%bash
# Generate dashboard with configuration
linkml-data-qc /tmp/dashboard_data.yaml \
    -s /tmp/dashboard_schema.yaml \
    -t Dataset \
    -c /tmp/dashboard_config.yaml \
    --dashboard /tmp/configured_dashboard.png

echo "\nDashboard with thresholds saved:"
ls -lh /tmp/configured_dashboard.png

%%bash
linkml-data-qc /tmp/dashboard_data.yaml \
    -s /tmp/dashboard_schema.yaml \
    -t Dataset \
    -c /tmp/dashboard_config.yaml \
    -f json \
    -o /tmp/report.json \
    --dashboard /tmp/full_dashboard.png

echo "Files generated:"
ls -lh /tmp/report.json /tmp/full_dashboard.png

%%bash
# Create a second dataset with better compliance
cat > /tmp/dashboard_data2.yaml << 'EOF'
name: Improved Tissue Study
description: Updated dataset with better annotation
samples:
  - id: SAMP101
    name: Liver Sample A
    tissue_type: liver
    ontology_term: UBERON:0002107
    collection_date: "2024-04-01"
  - id: SAMP102
    name: Heart Sample A
    tissue_type: heart
    ontology_term: UBERON:0000948
    collection_date: "2024-04-02"
  - id: SAMP103
    name: Brain Sample A
    tissue_type: brain
    ontology_term: UBERON:0000955
    collection_date: "2024-04-03"
  - id: SAMP104
    name: Kidney Sample A
    tissue_type: kidney
    ontology_term: UBERON:0002113
    # Missing collection_date
EOF
echo "Second dataset created!"

%%bash
# Compare both datasets
linkml-data-qc /tmp/dashboard_data.yaml /tmp/dashboard_data2.yaml \
    -s /tmp/dashboard_schema.yaml \
    -t Dataset \
    --dashboard /tmp/comparison_dashboard.png

echo "\nComparison dashboard saved:"
ls -lh /tmp/comparison_dashboard.png

%%bash
# Generate dashboard AND fail if below threshold
# (using the good dataset so this won't fail)
linkml-data-qc /tmp/dashboard_data2.yaml \
    -s /tmp/dashboard_schema.yaml \
    -t Dataset \
    --min-compliance 70 \
    --dashboard /tmp/ci_dashboard.png

echo "\nCI check passed! Dashboard saved."

%%bash
# This would fail CI due to threshold violations
# (but we capture the exit code to show it)
linkml-data-qc /tmp/dashboard_data.yaml \
    -s /tmp/dashboard_schema.yaml \
    -t Dataset \
    -c /tmp/dashboard_config.yaml \
    --fail-on-violations \
    --dashboard /tmp/failed_ci_dashboard.png || echo "\nCI check failed (as expected due to violations)"

%%bash
# Create a directory with multiple data files
mkdir -p /tmp/datasets
cp /tmp/dashboard_data.yaml /tmp/datasets/study1.yaml
cp /tmp/dashboard_data2.yaml /tmp/datasets/study2.yaml

# Analyze directory
linkml-data-qc /tmp/datasets/ \
    -s /tmp/dashboard_schema.yaml \
    -t Dataset \
    --pattern "*.yaml" \
    --dashboard /tmp/directory_dashboard.png

echo "\nDirectory analysis dashboard saved:"
ls -lh /tmp/directory_dashboard.png

%%bash
# Generate HTML dashboard for directory
linkml-data-qc /tmp/datasets/ \
    -s /tmp/dashboard_schema.yaml \
    -t Dataset \
    --pattern "*.yaml" \
    --dashboard-dir /tmp/html_dashboard

echo "\nHTML dashboard generated:"
ls -la /tmp/html_dashboard/

%%bash
# View the JSON report for automation
cat /tmp/html_dashboard/reports.json

%%bash
echo "All generated dashboards:"
ls -lh /tmp/*dashboard*.png 2>/dev/null || echo "No dashboards found"