#!/usr/local/bin/perl5.6.1

#######################################################################################
# A script to pull out 'sets' of data for analysis.
# 'Sets' are either destined for SSaHa SNP analysis or for ExoTrace
# analysis.#
# We need to be able to trap and warn system operators of sets which are never
# deemed as 'ready to process'.
# 
# NEW NEW NEW NEW NEW NEW NEW NEW NEW NEW NEW NEW NEW NEW
# (-) Steve wants to process by Gene rather than by Project
# (-) Need to implement other modes eg. make comparison groups for ExoTrace
# (-) Specify a loadingplate and change the status (eg. a test plate -do not analyse)
# (-) What course of action is taken when things either fail to enter an analysis group
#     or fail analysis (flag ?)
# (-) Human SNP finding vs. automatic system; comparison of results
# (-) Specifying candidate reads by say id_oligoprimer (as well as status)
# (-) 'Forcing' a particular set into an analysis group
# NEW NEW NEW NEW NEW NEW NEW NEW NEW NEW NEW NEW NEW NEW
# 
######################################################################################

use strict;
use Getopt::Long;
use sql_engine;
use Mail::Mailer;
use Data::Dumper;
use Date::Manip qw(ParseDate ParseDateString UnixDate);


my $this_program = "Exo_set_anaysis.pl_v1";
my $this_program_id = 1;

unless ($ENV{USER} eq 'dn1'){
    die "This script should only be run by ExoSeq group members !\nQuiting\n";
}

$| = 1;

#die @ARGV;
my $command = "Command-line options\n";
foreach my $com (@ARGV){
    $command = "$command" . "$com\n";
}


#############################
# Valid Command-line options
#############################
my ($help, $debug, $live_connect, $mode, $verbose, $commit);
GetOptions (
            "h"           =>    \$help,
            "d"           =>    \$debug,
	    "l"           =>    \$live_connect,
	    "mode=s"      =>    \$mode,
	    "v"           =>    \$verbose,
	    "c"           =>    \$commit
            );

if ($help){

    print STDERR "This program isolates and groups suitable reads into oligoprimer based comparison groups.\nNote that an analysis_run contains many comparison groups and that they are clustered by project.\n\n";

    print STDERR "(-) Isolates reads of a specified status, ordering them by projectname and id_oligoprimer\n";
    print STDERR "(-) From the above dataset, isolates good and bad comparison groups based on id_oligoprimer\n";
    print STDERR "(-) Dumps a file per analysis run set regarding comparison group contents into the relevant project directory\n";
    print STDERR "(-) Updates the database wrt.\n\tEXOSEQ_READINFO.ID_STATUS\n\tCG_STATUS\n\tCOMPARISON_GROUP\n\tANALYSIS_RUN\n\tGROUP_CONTENTS\n";
    print STDERR "(-) Mails results\n\n";

    print STDERR "Command line options follow...\n";

    print STDERR "-h\tDisplays this help\n";
    print STDERR "-d\tRun in debug mode\n\tActivates verbose mode\n\tAll files written to ./Exo_set_analysis_rundir/\n";
    print STDERR "-l\tRun on the live database (currently not implemented)\n";
    print STDERR "-mode=?\teg. mode=SH\n\tDetermines the status of reads to look for when making oligoprimer based sets\n\tCurrently implemented; SH = exoseq_readinfo.id_status = 1\n";
    print STDERR "-v\tRun in verbose mode. Prints useful info during processing\n";
    print STDERR "-c\tActivate a commit to the database after processing.\n\tNote that normal behaviour is to roll back changes\n";
    die "\n\nEnd of help";

}

if ($verbose){
    $verbose = 1;
}

if ($commit){
    $commit = 1;
} else {
    $commit = 0;
}

################################################################################################################
# Debug mode should 
# (-) Print processing info to STDOUT              (done)
# (-) Dump all files to local running directory    (done)
# (-) Rollback database updates                    (default operation is to rollback; use -c to force a commit)
################################################################################################################
if ($debug){
    $debug = 1;
    $verbose = 1;
    print STDOUT "\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\nProgram running in debug mode... \n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n";
    #sleep 5;
}


if ($live_connect){
    die "Live functionality not yet supported !\n";
    #$live_connect = 'helpdesk';
} else {
    $live_connect = 'snptest';
}


print STDOUT "MODE ---> $mode\n" if ($verbose);
unless ($mode eq 'SH'){
    die "Only SH mode supported !\n";
}


##################
# Database object
##################
my $statements;
eval {
    $statements = statements->new($live_connect);
};
if ($@){
    my $database_email = "dn1\@sanger.ac.uk, srl\@sanger.ac.uk";
    my $database_subject = "Database Connection Problem";
    my $database_message = "There is a problem with Exo_set_analysis.pl\nNo database connection could be established to WrapDBI user $live_connect\n";
    &send_mail($database_email, $database_subject, $database_message);
    exit;
}
$statements->default_sql_lib_path('./');
$statements->load_sql_lib(["Exo_set_analysis.sql_lib::all"]);


my $readinfo_status;
my $id_ana_program;
if ($mode eq 'SH'){
    ###########################################################
    # We need to pull out reads with a status of 1 - 'Entered'
    # Which are linked to ExoSeq individuals
    ###########################################################
    $readinfo_status = 1;
    $id_ana_program = 2; # Probably need to think a bit more about how we register the program
                         # version to use
}


##################################################
# Retrieve the analysis sets we are interested in
##################################################
my $results = &get_analysis_sets($readinfo_status, $statements);

############################################
# Get the data into a form we can work with
############################################
my $data = &create_datastructure($results);

#################################################################################
# Isolate sets to process and let people know what is not fit to process and why
#################################################################################
my ($bad_sets, $good_sets, $good_data) = &find_comparison_groups ($data);

##############################################################################################
# For sets deemed worthy of analysis (per project=analysis_run we need to;
# (-) Make some database entries (we need ANALYSIS.ID_ANARUN to name the file) 
# (-) Dump a file in the relevant project directory
##############################################################################################
my ($db_problems, $sink) = &update_database($good_data);

my ($file_problems, $dumped_files);
########################################
# Rollback if we encountered any errors
########################################
unless (@$db_problems[0] eq 'END'){
    print STDOUT "Datbase update problems follow ...\n\n";
    print STDOUT Data::Dumper::Dumper($db_problems);
    
    print STDOUT "Problems encountered... rolling back database updates !\n";
    $statements->{DBH}->rollback;
    die;
    
} else {

    ##################################
    # Dump our analysis files to disk
    ##################################
    ($file_problems, $dumped_files) = &dump_files($sink);


    unless (@$file_problems[0] eq 'END'){
	print STDOUT "File dump problems follow ...\n\n";
	print STDOUT Data::Dumper::Dumper($file_problems);
	
	print STDOUT "Problems encountered... rolling back database updates !\n";
	$statements->{DBH}->rollback;
	die;
	
    } else {

	###########################################
	# Only commit if the user has specified to
	###########################################
	if ( ($commit == 0) ){
	    print STDOUT "Rolling back database updates !\n" if ($verbose);
	    $statements->{DBH}->rollback;
	} else {
	    ##############################
	    # Commit the database changes
	    ##############################
	    print STDOUT "Commiting database updates !\n" if ($verbose);
	    $statements->{DBH}->commit;
	}
    }

    ########################################
    # Fire off an analysis job to the queue
    ########################################
    
}


###################################################
# Send a mail wrt sets found and files dumped
# - include a count of good vs a count of bad sets
###################################################
my $count_bad_sets = scalar @$bad_sets;
my $count_good_sets = scalar @$good_sets;
my $count_dumped = scalar @$dumped_files;
my $mail = "$command\n\nBad sets follow ($count_bad_sets)\n\n@$bad_sets\n\n\nGood sets follow ($count_good_sets)\n\n@$good_sets\n\n\nDumped files ($count_dumped)\n\n@$dumped_files";
&send_mail("dn1, srl", "Exo_set_analysis ran", "$mail");





# ------------------------------------
# Subroutines
# ------------------------------------
sub update_database {

    my ($data) = @_;

    if ($verbose) { warn "Inside update_database\n"; }

    my %sink; # This datatructure will be used to write out the ananlysis files later
    my @db_problems; # Somewhere to store database update problems

    ###############################################################################################
    # We need to make 
    # (-) ANALYSIS_RUN entries
    #     (-) COMPARISON_GROUP entries
    #     (-) CG_STATUS entries
    #         (-) GROUP_CONTENTS entries
    #
    # ID_DICT    INSTANCE	 ORACLE_USER     TABLE_NAME      TABLE_COLUMN    DB_LINK
    # ---------- --------------- --------------- --------------- --------------- ---------------
    #	 5       SNP	         SNPOWNER	 STS		 ID_STS
    #	 9       SNP	         SNPOWNER	 EXOSEQ_READINFO SEQREAD_NAME
    #         (-) Update EXOSEQ_READINFO.ID_STATUS
    ##############################################################################################

    ########################################
    # Go through the supplied datastructure
    ########################################

    foreach my $id_anarun (sort (keys %$data)){
	#print STDOUT "id_anarun $id_anarun\n";
	###########################
	# (-) ANALYSIS_RUN entries
	###########################
	
	##########################
	# What is our filename ?
	##########################
	my $this_path;
	my $store_path;
	if ($id_anarun =~ /(\S+)\s+(\S+)/){
	    $this_path = "$2" . "$1";
	    $store_path = "$1" . "_$mode" . "___$2";
	} else {
	    die "Could not get a path out $id_anarun !\n";
	}

	##########################################
	# Per analysis run we need to dump a file
	##########################################
	my $analysis_run_file = "$this_path"."_$mode";
	my $sink_file = $store_path;
	
	#print STDOUT " -----> sink_file $sink_file\n";
	$sink{$sink_file} = [];
	
	
	######################################
	# Remove the filename from $id_anarun
	######################################
	my ($parsed_id_anarun) = $id_anarun =~ /^(\d+)/;
	print STDOUT "\tEntering analysis_run id_anarun $parsed_id_anarun id_ana_program $id_ana_program this_path $analysis_run_file\n" if ($verbose);

	eval {

	    $statements->do('sickdesk.sql_lib::Create_analysis_run', [$parsed_id_anarun, $id_ana_program, $analysis_run_file]);
	};
	if ($@){
	    push @db_problems, "sickdesk.sql_lib::Create_analysis_run";
	    push @db_problems, "$@";
	}

	
	my $this_oligoprimer;
	foreach my $id_group (sort (keys %{$data->{$id_anarun}}) ){
	    
	    ###############################
	    # (-) COMPARISON_GROUP entries
	    ###############################
	    print STDOUT "\t\tEntering comparison_group id_group $id_group id_anarun $parsed_id_anarun\n" if ($verbose);

	    eval {
		$statements->do('sickdesk.sql_lib::Create_comparison_group', [$id_group, $parsed_id_anarun]);
	    };
	    if ($@){
		push @db_problems, "sickdesk.sql_lib::Create_comparison_group";
		push @db_problems, "$@";
	    }
	    
            ########################
	    # (-) CG_STATUS entries
	    ########################
	    my $id_status = 1;
#	    print STDOUT "\t\tEntering cg_status id_group $id_group id_status $id_status id_ana_program $this_program_id\n";
	    eval {
		$statements->do('sickdesk.sql_lib::Create_cg_status', [$id_group, $id_status, $this_program_id]);
	    };
	    if ($@){
		push @db_problems, "sickdesk.sql_lib::Create_cg_status";
		push @db_problems, "$@";
	    }

	    
	    my $this_pathname;
	    foreach my $read (@{$data->{$id_anarun}->{$id_group}}){
		#print STDOUT "\t\tread $read\n";

		########################################################
		# Only include reads in the group which have passed ASP
		########################################################
		
		if ($read =~ /ASP 1/){
		    #############################
		    # (-) GROUP_CONTENTS entries
		    #############################

		    #######################
		    # Determine a readname
		    #######################
		    my $this_readname;
		    if ($read =~ /PATH (\S+)/){
			$this_pathname = $1;
			# Remove the last /xxxxxxx/ from the path; this should be our readname
			if ($this_pathname =~ /\/([^\/]+)$/){
			    $this_readname = $1;
			} else {
			    die "Could not gleam a readname !\n";
			}
		    } else {
			die "Could not gleam a path !\n";
		    }
		    
		    my $this_id_content_type = 9;
#		    print STDOUT "\t\t\tEntering group_contents id_group $id_group id_content $this_readname id_content_type $this_id_content_type\n";
		    eval {
			$statements->do('sickdesk.sql_lib::Create_group_contents', [$id_group, $this_readname, $this_id_content_type]);
		    };
		    if ($@){
			push @db_problems, "sickdesk.sql_lib::Create_group_contents";
			push @db_problems, "$@";
		    }

		    #######################################
		    # (-) Update EXOSEQ_READINFO.ID_STATUS
		    #######################################
		    my $new_status = 3;
#		    print STDOUT "\t\t\tUpdating EXOSEQ_READINFO.ID_STATUS for seqread_name $this_readname to $new_status\n";
		    eval {
			$statements->do('sickdesk.sql_lib::Update_exoseq_readinfo_status', [$new_status, $this_readname]);
		    };
		    if ($@){
			push @db_problems, "sickdesk.sql_lib::Update_exoseq_readinfo_status";
			push @db_problems, "$@";
		    }


		    #############################################
		    # Grab the ID_OLIGOPRIMER on the way through
		    #############################################
		    $this_oligoprimer;
		    if ($read =~ /OP (\d+)/){
#			warn "this_oligoprimer $1\n";
			$this_oligoprimer = $1;
		    } else {
			die "Could not gleam id_oligoprimer !\n";
		    }

		    #########################################
		    # Per analysis run we need to dump a file
		    ##########################################
		    push @{$sink{$sink_file}}, "id_group $id_group   id_oligoprimer $this_oligoprimer   file $this_pathname";

		}
		
	    } # END OF      foreach my $read (@{$data->{$id_anarun}->{$id_group}}){
	    
	    #####################################################
	    # We also need to add an entry for the amplimer
	    # Use the oligoprimer value to determine an id_sts
	    #####################################################
	    my $sts = $statements->get('sickdesk.sql_lib::Get_STS_from_oligoprimer', [$this_oligoprimer]);

	    ####################################################
	    # Problems if we get more that 1x ID_STS returned !
	    ####################################################
	    if ($sts->[1]->[0]){
		die "ID_OLIGOPRIMER $this_oligoprimer has returned more than 1x ID_STS !\n";
	    }
	    my $sts = $sts->[0]->[0];

	    my $this_id_content_type = 5;
#	    print STDOUT "\t\t\tEntering group_contents (amplimer) id_group $id_group id_content $sts id_content_type $this_id_content_type\n";
	    eval {
		$statements->do('sickdesk.sql_lib::Create_group_contents', [$id_group, $sts, $this_id_content_type]);
	    };

	    push @{$sink{$sink_file}}, "id_group $id_group   id_sts $sts";

	    if ($@){
		push @db_problems, "sickdesk.sql_lib::Create_group_contents";
		push @db_problems, "$@";
	    }
	    
	}
    }
    
    push @db_problems, 'END';
    return (\@db_problems, \%sink);
    
}






sub dump_files {

    ################
    # File format ?
    ################
    # The name of the file should be ID_ANARUN_PROGRAM
    # With contents ... ID_GROUP ID_OLIGOPRIMER FILENAME

    my ($data) = @_;

    my @dumped_files; # Somewhere to store the names of the files we have dumped
    my @file_problems; # Somewhere to store file_dumping problems
    
    if ($verbose) { warn "Inside dump_files\n"; }
    
    print STDOUT "--------------------------------------------------------\n";
    print STDOUT Data::Dumper::Dumper($data);

    my $this_file;
    foreach my $file (sort (keys %$data)){
	print STDOUT "file ---> $file\n" if ($verbose);
	
	#########################################################
	# We need to swap the path back around
	#########################################################
	if ($file =~ /(\S+)___(\S+)/){
	    my $path = $2;
	    #########################################################
	    # .. if we are in debug mode then dump the files locally
	    #########################################################
	    if ($debug){
		$path = "./Exo_set_analysis_rundir/";
	    }
	    $this_file = "$path" . "$1";
	    
	    print STDOUT "Dumping file $this_file\n" if ($verbose);

	} else {
	    die "Problem rearranging path from $file !\n";
	}



	########################################
	# Try to open and write to a filehandle
	########################################
	eval {

	    #####################################
	    # Check that the file does not exist
	    #####################################
	    if ( -e $this_file ){
		die "File $this_file already exists !\n";
	    }
	    unless ( open (SINK, "> $this_file") ){
		die "Couldn't open a filehandle to $this_file ! $!";
	    }
	    foreach my $line (@{$data->{$file}}){
		print SINK "$line\n";
	    }
	    close SINK;
	    push @dumped_files, "$this_file";
	};

	if ($@){
	    push @file_problems, "dump_files Problems writing to file $this_file";
	    push @file_problems, "$@";
	}

    }

    push @file_problems, 'END';
    return (\@file_problems, \@dumped_files);
}




sub find_comparison_groups {

    my ($data) = @_;
    my @bad_sets;
    my @good_sets;
    my $good_data;

    if ($verbose) { warn "Inside find_comparison_groups\n"; }
    
    my %id_anarun_2_path;
    my ($meta_count, $meta_asp_count, $meta_cm_date);
    ###########################################################################################
    # We want a (COUNT) and (ASP_PASS count) and (newest CM_DATE) per ID_ANARUN+ID_GROUP combo
    ###########################################################################################
    foreach my $id_anarun (keys %$data){
	
	foreach my $id_group (keys %{$data->{$id_anarun}}){
	    
	    my $count = 0;
	    my $asp_count = 0;
	    my $epoch = 0;
	    foreach my $read (@{$data->{$id_anarun}->{$id_group}}){
		
		$count++;
		if ($read =~ /ASP 1/){
		    $asp_count++;
		    
		    #####################################################################
		    # Grab a path out of the read entry and populate %id_anarun_2_path
		    # eg. PATH /nfs/repository/p496/ExoSeq-99/8289/ExoSeq152992608g7.w2k
		    #####################################################################
		    if ($read =~ /PATH (\S+)/){
			my $this_path = $1;
			# Remove the last 2 /xxxxx/xxxxxxx/ from the path
			$this_path =~ s/([^\/]+)\/([^\/]+)$//;
			$id_anarun_2_path{$id_anarun} = $this_path; 
		    } else {
			die "Could not determine path from $read ! \n";
		    }
		    
		}

		if ($read =~ /EPO (\d+)/){
		    my ($this_epoch) = $1;
		    if ($this_epoch > $epoch){
			$epoch = $this_epoch;
		    } 
		} else {
		    die "Could not determine EPOCH from $read !\n";
		}

	    }

	    ###################################
	    # Store anarun/group specific meta
	    ###################################
	    $meta_count->{$id_anarun}->{$id_group} = $count;
	    $meta_asp_count->{$id_anarun}->{$id_group} = $asp_count;
	    $meta_cm_date->{$id_anarun}->{$id_group} = $epoch;

	}
    }

    print STDOUT "COUNT --------------------------------------------------------\n" if ($verbose);
    print STDOUT Data::Dumper::Dumper($meta_count) if ($verbose);
    
    print STDOUT "ASP PASS COUNT--------------------------------------------------------\n" if ($verbose);
    print STDOUT Data::Dumper::Dumper($meta_asp_count) if ($verbose);

    print STDOUT "LAST DATE--------------------------------------------------------\n" if ($verbose);
    print STDOUT Data::Dumper::Dumper($meta_cm_date) if ($verbose);
    
	

    ###########################################################################
    # We can now go through the meta info and recrete a new datatructure which
    # contains only things we are going to process
    ###########################################################################

    my ($to_process, $dont_process);

    foreach my $id_anarun (keys %$data){
	
	#################################################
	# Need a new ANALYSIS_RUN.ID_ANARUN from SEQ_RUN
	#################################################
	my $new_id_anarun = $statements->get('sickdesk.sql_lib::Get_seq_run');
	my $new_id_anarun = $new_id_anarun->[0]->[0];

	foreach my $id_group (keys %{$data->{$id_anarun}}){

	    ################################
	    # Good or Bad set logic follows
	    ################################

	    ############################################
	    # Convert back from epoch to day month year
	    ############################################
	    my $date = &ParseDate("epoch $meta_cm_date->{$id_anarun}->{$id_group}");
	    my ($year, $month, $day) = &UnixDate($date, "%Y", "%m", "%d");
	    chomp ($date =  "$day-$month-$year\n");

	    #########################################################
	    # Good Sets - Add to our new datastructure $good_data
	    # if ( ($meta_count->{$id_anarun}->{$id_group} == 48) &&
	    # Note that one individual is currently water
	    #########################################################
	    if ( ($meta_count->{$id_anarun}->{$id_group} == 47) &&
	    	 ($meta_asp_count->{$id_anarun}->{$id_group} >= 30) ){

#	    if (1) {
		## !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

		##########################################
		# Add to our new datastructure $good_data
                ##########################################

		######################################################
		# Need a new COMPARISON_GROUP.ID_GROUP from SEQ_COMGR
		######################################################
		my $new_id_group = $statements->get('sickdesk.sql_lib::Get_seq_comgr');
		my $new_id_group = $new_id_group->[0]->[0];
		my $c = 0;
		
		print STDOUT "GOOD SET ! Oligoprimer $id_group id_anarun $new_id_anarun id_group $new_id_group (Total $meta_count->{$id_anarun}->{$id_group} ASP_PASS $meta_asp_count->{$id_anarun}->{$id_group} Last read entered group on $date)\n" if ($verbose);

		push @good_sets, "Oligoprimer $id_group id_anarun $new_id_anarun id_group $new_id_group (Total $meta_count->{$id_anarun}->{$id_group} ASP_PASS $meta_asp_count->{$id_anarun}->{$id_group} Last read entered group on $date)\n";


		##################################################################
		# Can we include the destination path for the analysis_run here ?
		##################################################################
		my $new_id_anarun_plus_path = "$new_id_anarun" . "    $id_anarun_2_path{$id_anarun}";
		foreach my $read (@{$data->{$id_anarun}->{$id_group}}){
		    $good_data->{$new_id_anarun_plus_path}->{$new_id_group}->[$c] = $read;
		    $c++;
		}
		

	    } else {
		
		##############################
		# Bad Sets - Collect and mail
		##############################
		if ($meta_count->{$id_anarun}->{$id_group} > 47){
		    print STDOUT "BAD SET ! on oligoprimer $id_group\t> 47 reads seen\t(Total $meta_count->{$id_anarun}->{$id_group} ASP_PASS $meta_asp_count->{$id_anarun}->{$id_group} Last read entered group on $date)\n" if ($verbose);

		    push @bad_sets, "Oligoprimer $id_group > 47 reads seen (Total $meta_count->{$id_anarun}->{$id_group} ASP_PASS $meta_asp_count->{$id_anarun}->{$id_group} Last read entered group on $date)\n";
		}
		
		elsif ($meta_count->{$id_anarun}->{$id_group} < 47){
		    print STDOUT "BAD SET ! on oligoprimer $id_group\t< 47 reads seen\t(Total $meta_count->{$id_anarun}->{$id_group} ASP_PASS $meta_asp_count->{$id_anarun}->{$id_group} Last read entered group on $date)\n" if ($verbose);

		    push @bad_sets, "Oligoprimer $id_group < 47 reads seen (Total $meta_count->{$id_anarun}->{$id_group} ASP_PASS $meta_asp_count->{$id_anarun}->{$id_group} Last read entered group on $date)\n";
		}

		else {
		    # ($meta_asp_count->{$id_anarun}->{$id_group} < 30 ){
		    print STDOUT "BAD SET ! on oligoprimer $id_group\t< 30 reads ASP passed\t(Total $meta_count->{$id_anarun}->{$id_group} ASP_PASS $meta_asp_count->{$id_anarun}->{$id_group} Last read entered group on $date)\n" if ($verbose);

		    push @bad_sets, "Oligoprimer $id_group < 30 reads ASP passed (Total $meta_count->{$id_anarun}->{$id_group} ASP_PASS $meta_asp_count->{$id_anarun}->{$id_group} Last read entered group on $date)\n";
		}

	    }
	}
    }

    
    return (\@bad_sets, \@good_sets, $good_data);
}



sub create_datastructure {
    my ($results) = @_;
    my @results = @$results;

    if ($verbose) { warn "Inside create_datastructure\n"; }

    my $hash;
    foreach my $n (@results){
#######################################################################################################################
#id_anarun 31251 id_group 45943 count 1 ASP 1 OP 678470 PATH /nfs/repository/p580/ExoSeq-132/9170/ExoSeq146185545o16.w2k GE 9170 PR ExoSeq-132 DA 10-MAR-03
	
	if ($n =~ s/^id_anarun (\d+) id_group (\d+) count (\d+)//){
	    $hash->{$1}->{$2}->[$3-1] = $n;
	}
    }

    return ($hash);
}



sub get_analysis_sets {

    my ($readinfo_status, $statements) = @_;
    
    if ($verbose) { warn "Inside get_analysis_sets\n"; }

    ####################
    # Retrieve the sets
    ####################
    my $sets = $statements->get('sickdesk.sql_lib::Get_SH_sets_by_status', [$readinfo_status]);

    #################################################################
    # From this read list we need to create analysis run candidates.
    # Note that each analysis run candidate will contain oligoprimer
    # specific sets of reads specific to a particular project
    #################################################################
    my ($this_oligoprimer, $this_projectname); 
    my ($start_a_new_set, $start_a_new_run) = 0;
    my ($set_count) = 0;
    my @results;
    my $id_group;
    my $id_anarun = 0;

    foreach my $n (@$sets){
    
    #####################################
    # New set (ie projectname changed) ?
    #####################################
	if ($n->[4] ne $this_projectname){
	    $start_a_new_run = 1;
	    $this_projectname = $n->[4];
	} else {
	    $start_a_new_run = 0;
	}
	
	if ($start_a_new_run == 1){
	    ############################################################
	    # Eventually need a new ANALYSIS_RUN.ID_ANARUN from SEQ_RUN
	    ############################################################
	    $id_anarun++;
	    
	    push @results, "\tProject $this_projectname START A NEW RUN => ANALYSIS_RUN.ID_ANARUN = $id_anarun\n";
	    print STDOUT "\tProject $this_projectname START A NEW RUN => ANALYSIS_RUN.ID_ANARUN = $id_anarun\n" if ($verbose == 1);
	}
	
	
	#####################################
	# New set (ie oligoprimer changed) ?
	#####################################
	if ($n->[1] ne $this_oligoprimer){
	    $start_a_new_set = 1;
	    $this_oligoprimer = $n->[1];
	    $set_count++;

	} else {
	    $start_a_new_set = 0;
	    $set_count++;
	}
	
	if ($start_a_new_set == 1){
	    #################################################################
	    # Eventually need a new COMPARISON_GROUP.ID_GROUP from SEQ_COMGR
	    # - Set the group id to be that of the oligoprimer
	    #################################################################
	    $id_group = $n->[1];

	    my $last_set_count = $set_count-1;
	    push @results, "\t\tlast set count = $last_set_count  START A NEW SET => COMPARISON_GROUP.ID_GROUP = $id_group\n";
	    print STDOUT "\t\tlast set count = $last_set_count START A NEW SET => COMPARISON_GROUP.ID_GROUP = $id_group\n"
		if ($verbose == 1);
	    $set_count = 1;
	}
	
	############################################################################
	# Convert out Oracle retrieved date into seconds since epoch for comparison
	############################################################################
	my $secs = &UnixDate($n->[5],"%s");

	push @results, "id_anarun $id_anarun id_group $id_group count $set_count ASP $n->[0] OP $n->[1] PATH $n->[2] GE $n->[3] PR $n->[4] DA $n->[5] EPO $secs\n";
	print STDOUT "id_anarun $id_anarun id_group $id_group count $set_count ASP $n->[0] OP $n->[1] PATH $n->[2] GE $n->[3] PR $n->[4] DA $n->[5] EPO $secs\n" if ($verbose == 1);
	
    }
    ##################################
    # Need a summary for the last set
    ##################################
    my $last_set_count = $set_count;
    push @results, "\t\tlast set count = $last_set_count  START A NEW SET => NO\n";
    print STDOUT "\t\tlast set count = $last_set_count START A NEW SET => NO\n" if ($verbose == 1);

    return (\@results);
}





sub send_mail {
  my ($email, $subject, $message) = @_;
  my ($mailer);

  if ($verbose) { warn "Inside send_mail\n"; }

  eval 
  {
      my $mailer = Mail::Mailer->new("mail");
      
      if ($debug == 1){
	  $mailer->open({ From    => "Exo_set_analysis.pl",
			  To      => "$email",
			  Subject => "DEBUG MODE !PLEASE IGNORE! $subject"}) ||  die "Can't open: $!\n";
      } else {
	  
	  $mailer->open({ From    => "Exo_set_analysis.pl",
			  To      => "$email",
			  Subject => "$subject"}) ||  die "Can't open: $!\n";
      }
      
      print $mailer "$message";
      close($mailer);
  };
  
  if ($@){
      
      print "Error sending mail; $@\n\nFrom Exo_set_analysis.pl\nTo $email\nSubject $subject\n\n$message\n";
      
      my $ret = "Problems sending email: ". "$@";
      return ($ret);
  }
  
}




END {
    if ($statements){
	print STDOUT "Inside END block; rolling back\n";
	$statements->{DBH}->rollback;
	$statements->{DBH}->disconnect;
    }
};
