# core/management/commands/seed_data.py import csv import os from django.core.management.base import BaseCommand from django.conf import settings from tqdm import tqdm from tcgabcd.models import Cna, Cna_hg19, Gene, GeneExpression, Mutation, Patient, ClinicalPatient, ClinicalSample, SvData, TreatmentTimeline, HypoxiaScore,Dataset import pandas as pd import numpy as np class Command(BaseCommand): help = 'Seeds the database with TCGA breast cancer data' def handle(self, *args, **kwargs): self.stdout.write('Starting data seeding...') # Define data directory data_dir = os.path.join(settings.BASE_DIR, 'data') # Seed patients first (they're referenced by other models) # self.seed_patients(data_dir) self.seed_cases_all(data_dir) # Seed other data self.seed_clinical_patient_data(data_dir) self.seed_clinical_sample_data(data_dir) self.seed_treatment_timeline_data(data_dir) self.seed_hypoxia_data(data_dir) self.seed_gene_expression(data_dir) self.seed_mutation_data(data_dir) self.seed_cna_hg19(data_dir) self.seed_sv_data(data_dir) self.seed_cna(data_dir) self.stdout.write(self.style.SUCCESS('Data seeding completed successfully!')) def clean_value(self, value): """Helper function to clean values and handle NaN""" if pd.isna(value): return None return value def clean_numeric(self, value): """Helper function to clean numeric values""" if pd.isna(value) or value == '': return None try: return float(value) except (ValueError, TypeError): return None def seed_cases_all(self, data_dir): """Seed dataset and patient data from cases_all.txt""" self.stdout.write('Seeding dataset and patient data from cases_all.txt...') file_path = os.path.join(data_dir, 'cases_all.txt') # Read the file with open(file_path, 'r') as file: lines = file.readlines() # Extract case_list_ids for line in lines: if line.startswith("case_list_ids:"): case_ids = line.split(":")[1].strip().split("\t") break else: self.stdout.write(self.style.WARNING("No 'case_list_ids' found in cases_all.txt")) return # Create or get the Dataset dataset, created = Dataset.objects.get_or_create( name="Breast Cancer Dataset", description="All samples from TCGA breast cancer study", case_id_suffix="01" ) patients = [] # Process case IDs and derive patient IDs for case_id in case_ids: patient_id = "-".join(case_id.split("-")[:-1]) # Remove the suffix # Create or get the Patient patient, _ = Patient.objects.get_or_create( dataset=dataset, patient_id=patient_id, defaults={'case_id':case_id} ) patients.append(patient) # Link patient to dataset dataset.case_ids.add(patient) self.stdout.write(f'Processed {len(case_ids)} case IDs and {len(patients)} patients.') self.stdout.write(f"Dataset '{dataset.name}' now contains {dataset.case_ids.count()} patients.") def seed_clinical_patient_data(self, data_dir): """Seed clinical patient data with proper NaN handling""" self.stdout.write('Seeding clinical patient data...') file_path = os.path.join(data_dir, 'data_clinical_patient.csv') df = pd.read_csv(file_path, comment='#') clinical_patients = [] for _, row in df.iterrows(): try: patient_data = { 'patient_id': row['PATIENT_ID'], 'subtype': self.clean_value(row['SUBTYPE']), 'cancer_type_acronym': self.clean_value(row['CANCER_TYPE_ACRONYM']), 'age': self.clean_numeric(row['AGE']), 'sex': self.clean_value(row['SEX']), 'ajcc_pathologic_tumor_stage': self.clean_value(row['AJCC_PATHOLOGIC_TUMOR_STAGE']), 'ajcc_staging_edition': self.clean_value(row['AJCC_STAGING_EDITION']), 'days_last_followup': self.clean_numeric(row['DAYS_LAST_FOLLOWUP']), 'days_to_birth': self.clean_numeric(row['DAYS_TO_BIRTH']), 'ethnicity': self.clean_value(row['ETHNICITY']), 'race': self.clean_value(row['RACE']), 'weight': self.clean_numeric(row['WEIGHT']), 'icd_10': self.clean_value(row['ICD_10']), 'icd_o_3_histology': self.clean_value(row['ICD_O_3_HISTOLOGY']), 'icd_o_3_site': self.clean_value(row['ICD_O_3_SITE']), 'new_tumor_event_after_initial_treatment': self.clean_value(row['NEW_TUMOR_EVENT_AFTER_INITIAL_TREATMENT']), 'path_m_stage': self.clean_value(row['PATH_M_STAGE']), 'path_n_stage': self.clean_value(row['PATH_N_STAGE']), 'path_t_stage': self.clean_value(row['PATH_T_STAGE']), 'person_neoplasm_cancer_status': self.clean_value(row['PERSON_NEOPLASM_CANCER_STATUS']), 'os_status': self.clean_value(row['OS_STATUS']), 'os_months': self.clean_numeric(row['OS_MONTHS']), 'dss_status': self.clean_value(row['DSS_STATUS']), 'dss_months': self.clean_numeric(row['DSS_MONTHS']), 'dfs_status': self.clean_value(row['DFS_STATUS']), 'dfs_months': self.clean_numeric(row['DFS_MONTHS']), 'pfs_status': self.clean_value(row['PFS_STATUS']), 'pfs_months': self.clean_numeric(row['PFS_MONTHS']), 'genetic_ancestry_label': self.clean_value(row['GENETIC_ANCESTRY_LABEL']) } clinical_patients.append(ClinicalPatient(**patient_data)) except Exception as e: self.stdout.write(self.style.WARNING( f'Error processing patient {row["PATIENT_ID"]}: {str(e)}' )) continue ClinicalPatient.objects.bulk_create(clinical_patients, ignore_conflicts=True) self.stdout.write(f'Created {len(clinical_patients)} clinical patient records') def seed_clinical_sample_data(self, data_dir): """Seed clinical sample data""" self.stdout.write('Seeding clinical sample data...') file_path = os.path.join(data_dir, 'data_clinical_sample.csv') df = pd.read_csv(file_path, comment='#') clinical_samples = [] for _, row in df.iterrows(): try: clinical_samples.append( ClinicalSample( patient_id=row['PATIENT_ID'], sample_id=row['SAMPLE_ID'], oncotree_code=self.clean_value(row['ONCOTREE_CODE']), cancer_type=self.clean_value(row['CANCER_TYPE']), cancer_type_detailed=self.clean_value(row['CANCER_TYPE_DETAILED']), tumor_type=self.clean_value(row['TUMOR_TYPE']), tumor_tissue_site=self.clean_value(row['TUMOR_TISSUE_SITE']), sample_type=self.clean_value(row['SAMPLE_TYPE']), tmb_nonsynonymous=self.clean_numeric(row['TMB_NONSYNONYMOUS']), ) ) except Exception as e: self.stdout.write(self.style.WARNING( f'Error processing sample for patient {row["PATIENT_ID"]}: {str(e)}' )) continue ClinicalSample.objects.bulk_create(clinical_samples, ignore_conflicts=True) self.stdout.write(f'Created {len(clinical_samples)} clinical sample records') def seed_treatment_timeline_data(self, data_dir): """Seed treatment timeline data""" self.stdout.write('Seeding treatment timeline data...') file_path = os.path.join(data_dir, 'data_timeline_treatment.csv') df = pd.read_csv(file_path, comment='#') treatments = [] for _, row in df.iterrows(): try: treatments.append( TreatmentTimeline( patient_id=row['PATIENT_ID'], start_date=self.clean_numeric(row['START_DATE']), stop_date=self.clean_numeric(row['STOP_DATE']), event_type=self.clean_value(row['EVENT_TYPE']), treatment_type=self.clean_value(row['TREATMENT_TYPE']), treatment_subtype=self.clean_value(row['TREATMENT_SUBTYPE']), agent=self.clean_value(row['AGENT']), number_of_cycles=self.clean_value(row['NUMBER_OF_CYCLES']), radiation_dosage=self.clean_numeric(row['RADIATION_DOSAGE']), radiation_type=self.clean_value(row['RADIATION_TYPE']) ) ) except Exception as e: self.stdout.write(self.style.WARNING( f'Error processing treatment for patient {row["PATIENT_ID"]}: {str(e)}' )) continue TreatmentTimeline.objects.bulk_create(treatments, ignore_conflicts=True) self.stdout.write(f'Created {len(treatments)} treatment timeline records') def seed_hypoxia_data(self, data_dir): """Seed hypoxia score data""" self.stdout.write('Seeding hypoxia score data...') file_path = os.path.join(data_dir, 'data_clinical_supp_hypoxia.csv') df = pd.read_csv(file_path, comment='#') hypoxia_scores = [] for _, row in df.iterrows(): try: hypoxia_scores.append( HypoxiaScore( patient_id=row['PATIENT_ID'], buffa_score=int(row['BUFFA_HYPOXIA_SCORE']), winter_score=int(row['WINTER_HYPOXIA_SCORE']), ragnum_score=int(row['RAGNUM_HYPOXIA_SCORE']) ) ) except Exception as e: self.stdout.write(self.style.WARNING( f'Error processing hypoxia scores for patient {row["PATIENT_ID"]}: {str(e)}' )) continue HypoxiaScore.objects.bulk_create(hypoxia_scores, ignore_conflicts=True) self.stdout.write(f'Created {len(hypoxia_scores)} hypoxia score records') def seed_gene_expression(self, data_dir): """Seed gene expression data for specific genes (e.g., HIF1A and ESR1)""" self.stdout.write('Seeding gene expression data for selected genes...') # List of genes to process selected_genes = ["HIF1A", "ESR1"] # File path to gene expression data file_path = os.path.join(data_dir, 'data_mrna_seq_v2_rsem_zscores_ref_all_samples.csv') # Load the CSV file df = pd.read_csv(file_path) # Filter the rows for the selected genes df = df[df['Hugo_Symbol'].isin(selected_genes)] if df.empty: self.stdout.write(self.style.WARNING("No data found for the selected genes!")) return # Build patient cache to avoid repeated lookups self.stdout.write("Building patient cache...") patient_cache = {patient.case_id: patient for patient in Patient.objects.all()} # Collect data for bulk creation gene_expressions = [] # Total cells for progress tracking total_cells = df.shape[0] * (df.shape[1] - 2) # Total rows * number of patient columns progress_bar = tqdm(total=total_cells, desc="Processing cells", unit="cell") # Process the filtered gene expression data for _, row in df.iterrows(): try: hugo_symbol = row['Hugo_Symbol'] entrez_gene_id = int(row['Entrez_Gene_Id']) # Create or get the Gene gene, _ = Gene.objects.get_or_create( entrez_gene_id=entrez_gene_id, defaults={'hugo_symbol': hugo_symbol} ) # Process patient z-scores for case_id, z_score in row.iloc[2:].items(): progress_bar.update(1) if pd.isna(z_score): # Skip NaN values continue # Check if patient exists in the cache patient = patient_cache.get(case_id) if not patient: continue # Skip if patient does not exist # Collect gene expression data for bulk creation gene_expressions.append(GeneExpression( gene=gene, patient=patient, z_score=z_score )) # Batch insert to avoid memory overload if len(gene_expressions) >= 5000: # Batch size of 5000 GeneExpression.objects.bulk_create(gene_expressions, ignore_conflicts=True) gene_expressions = [] # Clear batch except Exception as e: self.stdout.write(self.style.WARNING( f"Error processing gene {row.get('Hugo_Symbol', 'Unknown')} ({row['Entrez_Gene_Id']}): {str(e)}" )) continue # Insert remaining data if gene_expressions: GeneExpression.objects.bulk_create(gene_expressions, ignore_conflicts=True) progress_bar.close() self.stdout.write('Gene expression data seeding completed for selected genes.') def seed_mutation_data(self, data_dir): """Seed mutation data""" self.stdout.write('Seeding mutation data...') file_path = os.path.join(data_dir, 'data_mutations.csv') df = pd.read_csv(file_path, comment='#') # Adjust comment if needed # Build patient cache for quick lookups self.stdout.write("Building patient cache for mutations...") patient_cache = {patient.patient_id: patient for patient in Patient.objects.all()} mutations = [] for _, row in tqdm(df.iterrows(), total=len(df), desc="Processing mutations"): try: # Extract relevant fields (modify as per your file structure) hugo_symbol = row['Hugo_Symbol'] chromosome = row['Chromosome'] start_position = self.clean_numeric(row['Start_Position']) end_position = self.clean_numeric(row['End_Position']) variant_classification = row['Variant_Classification'] variant_type = row['Variant_Type'] reference_allele= row['Reference_Allele'] tumor_seq_allele1 = row['Tumor_Seq_Allele1'] tumor_seq_allele2 = row['Tumor_Seq_Allele2'] tumor_sample = row['Tumor_Sample_Barcode'] # Extract patient ID without suffix patient_id_clean = "-".join(tumor_sample.split("-")[:-1]) # Check if patient exists in cache patient = patient_cache.get(patient_id_clean) if not patient: continue # Skip if patient does not exist # Create Mutation object mutation = Mutation( tumor_sample_barcode=patient, hugo_symbol=hugo_symbol, variant_classification=variant_classification, variant_type=variant_type, chromosome=chromosome, start_position=start_position, end_position=end_position, reference_allele=reference_allele, tumor_seq_allele1=tumor_seq_allele1, tumor_seq_allele2=tumor_seq_allele2 ) mutations.append(mutation) # Batch insert to avoid memory overload if len(mutations) >= 5000: # Batch size of 5000 Mutation.objects.bulk_create(mutations, ignore_conflicts=True) mutations = [] # Clear batch except Exception as e: self.stdout.write(self.style.WARNING( f"Error processing mutation for {row.get('Tumor_Sample_Barcode', 'Unknown')}: {str(e)}" )) continue # Insert remaining data if mutations: Mutation.objects.bulk_create(mutations, ignore_conflicts=True) self.stdout.write(f"Created mutation records for {len(df)} samples.") # Add these new methods for seeding CNA and SV data def seed_cna_hg19(self, data_dir): """Seed CNA (Copy Number Alteration) data from data_cna_hg19.csv""" self.stdout.write('Seeding CNA HG19 data...') file_path = os.path.join(data_dir, 'data_cna_hg19.csv') df = pd.read_csv(file_path) # Build patient cache for quick lookups self.stdout.write("Building patient cache for CNA data...") patient_cache = {patient.case_id: patient for patient in Patient.objects.all()} cna_records = [] for _, row in tqdm(df.iterrows(), total=len(df), desc="Processing CNA records"): # print(row['ID']) try: # print("tried") case_id = row['ID'] chromosome = row['chrom'] loc_start = int(row['loc.start']) loc_end = int(row['loc.end']) num_mark = float(row['num.mark']) seg_mean = float(row['seg.mean']) # Check if patient exists in cache patient = patient_cache.get(case_id) if not patient: continue # Skip if patient does not exist # Create Cna_hg19 object cna_record = Cna_hg19( patient=patient, chromosome=chromosome, loc_start=loc_start, loc_end=loc_end, num_mark=num_mark, seg_mean=seg_mean ) cna_records.append(cna_record) # print("worked") # Batch insert to avoid memory overload if len(cna_records) >= 5000: # Batch size of 5000 Cna_hg19.objects.bulk_create(cna_records, ignore_conflicts=True) cna_records = [] # Clear batch except Exception as e: print("FAILED",e) self.stdout.write(self.style.WARNING( f"Error processing CNA record for patient {row.get('ID', 'Unknown')}: {str(e)}" )) continue # Insert remaining data # print(len(cna_records)) if cna_records: Cna_hg19.objects.bulk_create(cna_records, ignore_conflicts=True) self.stdout.write(f"Created {len(df)} CNA HG19 records.") def seed_sv_data(self, data_dir): """Seed Structural Variant (SV) data from data_sv.csv""" self.stdout.write('Seeding Structural Variant (SV) data...') file_path = os.path.join(data_dir, 'data_sv.csv') df = pd.read_csv(file_path) # Build patient and gene caches for quick lookups self.stdout.write("Building patient and gene caches for SV data...") patient_cache = {patient.case_id: patient for patient in Patient.objects.all()} gene_cache = {gene.hugo_symbol: gene for gene in Gene.objects.all()} sv_records = [] for _, row in tqdm(df.iterrows(), total=len(df), desc="Processing SV records"): try: case_id = row['Sample_Id'] site1_hugo_symbol = row['Site1_Hugo_Symbol'] site2_hugo_symbol = row['Site2_Hugo_Symbol'] sv_status = row['SV_Status'] # Check if patient exists in cache patient = patient_cache.get(case_id) if not patient: continue # Skip if patient does not exist # Check if site1_gene exists in cache; if not, create it site1_gene = gene_cache.get(site1_hugo_symbol) if not site1_gene: site1_gene, created = Gene.objects.get_or_create( hugo_symbol=site1_hugo_symbol, defaults={'entrez_gene_id': None} # Set a default or handle missing Entrez ID ) gene_cache[site1_hugo_symbol] = site1_gene # Update cache # Check if site2_gene exists in cache; if not, create it site2_gene = gene_cache.get(site2_hugo_symbol) if not site2_gene: site2_gene, created = Gene.objects.get_or_create( hugo_symbol=site2_hugo_symbol, defaults={'entrez_gene_id': None} # Set a default or handle missing Entrez ID ) gene_cache[site2_hugo_symbol] = site2_gene # Update cache # Create SvData object sv_record = SvData( patient=patient, site1_gene=site1_gene, site2_gene=site2_gene, sv_status=sv_status ) sv_records.append(sv_record) # # Batch insert to avoid memory overload if len(sv_records) >= 5000: # Batch size of 5000 SvData.objects.bulk_create(sv_records, ignore_conflicts=True) sv_records = [] # Clear batch except Exception as e: self.stdout.write(self.style.WARNING( f"Error processing SV record for patient {row.get('Sample_Id', 'Unknown')}: {str(e)}" )) continue # Insert remaining data if sv_records: SvData.objects.bulk_create(sv_records, ignore_conflicts=True) self.stdout.write(f"Created {len(df)} Structural Variant (SV) records.") def seed_cna(self, data_dir): self.stdout.write('Seeding CNA expression data for selected genes...') # List of genes to process # selected_genes = ["HIF1A", "ESR1"] # File path to gene expression data file_path = os.path.join(data_dir, 'data_cna.csv') # Load the CSV file df = pd.read_csv(file_path) # Filter the rows for the selected genes # df = df[df['Hugo_Symbol'].isin(selected_genes)] if df.empty: self.stdout.write(self.style.WARNING("No data found for the selected genes!")) return # Build patient cache to avoid repeated lookups self.stdout.write("Building patient cache...") patient_cache = {patient.case_id: patient for patient in Patient.objects.all()} # Collect data for bulk creation cna_expressions = [] # Total cells for progress tracking total_cells = df.shape[0] * (df.shape[1] - 2) # Total rows * number of patient columns progress_bar = tqdm(total=total_cells, desc="Processing cells", unit="cell") # Process the filtered gene expression data for _, row in df.iterrows(): try: hugo_symbol = row['Hugo_Symbol'] entrez_gene_id = int(row['Entrez_Gene_Id']) # Create or get the Gene gene, _ = Gene.objects.get_or_create( entrez_gene_id=entrez_gene_id, defaults={'hugo_symbol': hugo_symbol} ) # Process patient z-scores for case_id, value in row.iloc[2:].items(): progress_bar.update(1) if pd.isna(value): # Skip NaN values continue # Check if patient exists in the cache patient = patient_cache.get(case_id) if not patient: continue # Skip if patient does not exist # Collect gene expression data for bulk creation cna_expressions.append(Cna( gene=gene, patient=patient, value=value )) # Batch insert to avoid memory overload if len(cna_expressions) >= 5000: # Batch size of 5000 Cna.objects.bulk_create(cna_expressions, ignore_conflicts=True) cna_expressions = [] # Clear batch except Exception as e: self.stdout.write(self.style.WARNING( f"Error processing gene {row.get('Hugo_Symbol', 'Unknown')} ({row['Entrez_Gene_Id']}): {str(e)}" )) continue # Insert remaining data if cna_expressions: Cna.objects.bulk_create(cna_expressions, ignore_conflicts=True) progress_bar.close() self.stdout.write('Gene expression data seeding completed for selected genes.')