#!/usr/bin/env python
# mine_SUTs_and_merge_into_mito_transcripts_df.py
__author__ = "Wayne Decatur" #fomightez on GitHub
__license__ = "MIT"
__version__ = "0.1.0"


# mine_SUTs_and_merge_into_mito_transcripts_df.py by Wayne Decatur
# ver 0.1
#
#*******************************************************************************
# Compatible with both Python 2.7 and Python 3.6 (verified); written initially 
# in Python 2.7 to hopefully be convertible to Python 3.
#
# PURPOSE: Takes details of SUTs and a dataframe previously generated from a 
# transcriptome file and produces a dataframe where the SUTs data is merged.
# Intended to work with the dataframe generated by running 
# `mine_mito_features_from_transcriptome.py` on a file of the Ensembl cDNAs + 
# Ensembl ncRNAs (plus some custom RNAs I added) as the transcriptome.
#
# Returns a dataframe with the details for the mito features with the SUTS added.
#
#
#
#
#
#
# Dependencies beyond the mostly standard libraries/modules:
#
#
#
# VERSION HISTORY:
# v.0.1. basic working version
#
# To do:
# - see `POSSIBLE TO DO` about references when using a file as source of SUTs data.
#
#
#
# TO RUN:
# Example,
# Enter on the command line of your terminal, the line
#-----------------------------------
# python mine_SUTs_and_merge_into_mito_transcripts_df.py
#-----------------------------------
# Issue `python mine_SUTs_and_merge_into_mito_transcripts_df.py -h` for details.
# 
#
# To use this after pasting (or loading) into a cell in a Jupyter notebook, 
# define `input_SUTs` as the source of the SUTs data. It will either be the name 
# of the file to read or use `use_harcoded` to specify to use the SUTs data that
# was hardcoded into the script. Also define `save_as` the name of file to store
# the resulting dataframe. To not save anything, set `save_as= "no_output"`.  
# Then call the main function in a cell as shown below:
# input_SUTs = "use_hardcoded"
# save_as = "mito_transcripts_dataframeplusSUTS.pkl"
# df = mine_and_mergeSUTs()
# 
# Alternatively set `previous_pickled_df="no_pickle"`, before calling the main
# function in Jupyter to not supply a previous dataframe (useful for testing).
# Additionally, # add `pickle_df = False` like
# `df = mine_and_mergeSUTs(pickle_df = False)` to 
# not produce a pickled dataframe file, and then you don't need to supply a 
# `save_as` setting either.
#
#
#*******************************************************************************
#


#*******************************************************************************
##################################
#  USER ADJUSTABLE VALUES        #

##################################
#

SUT_data ='''

sys_gene_id     gene_symbol     start        end       strand
SUT2729.1       None        6229       6557         -1
SUT2730.1       None        13893      13957         -1
SUT2731.1       None        18277      18365         -1
SUT2733.1       None        31545      31608         1
SUT2732.1       None        41421      41541         -1
SUT2734.1       None        62601      62872         1

'''
# above entered by hand from copying and pasting from
# `Lardenois_2011_noncoding_RNAs_V64-chrmt-1..85146.gff3`
# At this point, I haven't scripted to mine from the gff3 format directly, but 
# CERTAINLY I should AS I FORESEE similar exported tracks being a source of 
# others. (Notes for that: `Name=` is descriptor for sys_id.) And if there are
# others, I'd need to adjust it to mine from multiple gffs and merge all into
# the `suts_df`.
# Details of SUT_data:
# The data for SUT2729.1 through SUT2734.1  is from 
# Lardenois_2011_noncoding_RNAs_V64      PMID: 21149693.  I
# found it by adding tracks in jbrowse and exported the gff3.
# JUST DEALING WITH THE MITOCHONDRIAL SUTs FOR NOW.

SUT_data_refs = [
"SUT2729.1 through SUT2734.1 from Lardenois et al PMID: 21149693.",
                ]



previous_pickled_df = "mito_transcripts_dataframe.pkl"
suffix_for_saving = "plusSUTs"

sort_on_midpoint = True # sort dataframe on gene midpoint location



#
#*******************************************************************************
#**********************END USER ADJUSTABLE VARIABLES****************************


















#*******************************************************************************
#*******************************************************************************
###DO NOT EDIT BELOW HERE - ENTER VALUES ABOVE###

import sys
import os
import pandas as pd
try:
    from StringIO import StringIO
except ImportError:
    from io import StringIO





###---------------------------HELPER FUNCTIONS---------------------------------###
def sort_on_midpt_and_reset_ind(df):
    '''
    Takes a dataframe and sorts it on midpoint and returns a dataframe
    '''
    df = df.sort_values('midpoint', ascending=True)
    return df.reset_index(drop=True) # res the index to reflect re-order


def mine_and_mergeSUTs(pickle_df = True):
    '''
    Takes data on SUTs and merges into a previously generated dataframe of
    transcripts, if one is provided.

    returns a dataframe. This will either be the merged dataframe with the 
    SUTs data added to the provided dataframe, or the SUTs data as a DataFrame
    alone.
    '''
    # verify the file to add data to exists
    #os.path.isfile(fname) 


    #SUT data
    # if file provided read in data from there, otherwise read in data
    # placed in 'user-defined variables '
    if input_SUTs == 'use_hardcoded':
        suts_df = pd.read_table(
            StringIO(SUT_data), header=0, delim_whitespace= True)
        sys.stderr.write("SUTs data included in the script will be used.")
    else:
        suts_df = pd.read_table(input_SUTs, header=0, delim_whitespace= True)
        sys.stderr.write( "SUTs data has been read in the file '{}'"
            .format(input_SUTs))
        # POSSIBLE TO DO: add a way to have the references in the file
        # & have them mined out of there and punt in list `SUT_data_refs`. Then
        # remove conditional `input_SUTs == 'use_hardcoded'` below when 
        # documenting references to console/Jupyter cell.
    # need columns at end to match, before merging, so...
    # need to generate midpoint column (unless, although unlikely, already there)
    if 'midpoint' not in suts_df.columns:
        suts_df['midpoint'] = suts_df[['start','end']].apply(midpoint, axis=1)
    # Easiest to be consistent with previous convention of column order. Plus 
    # makes it so won't have two orders going for each dataframe and still 
    # require setting column order after merge. So set order to match previous.
    columns = ['sys_gene_id','gene_symbol',
    'start','end','midpoint','strand']
    suts_df = suts_df[columns]
    # sort on midpoint, depending on `sort_on_midpoint` setting.
    if sort_on_midpoint:
        suts_df = sort_on_midpt_and_reset_ind(suts_df)
    # send the SUTs to the terminal or Jupyter notebook display in some manner 
    # so will be documenting what SUTs are being mined/added in the course of
    # running without the user needing to look at dataframe or output. 
    # Using `df.to_string()` because more universal than `print(df)` 
    # or Jupyter's `display(df)`.
    sys.stderr.write( "\nFor documenting purposes, the following is "
        "the SUTs data that was read in:\n")
    #with pd.option_context('display.max_rows', None, 'display.max_columns', None):
    #    display(df)
    sys.stderr.write(suts_df.to_string())
    if input_SUTs == 'use_hardcoded':
        refs = "\n".join(SUT_data_refs)
        sys.stderr.write( "\nFor documenting purposes, the following sources "
        "were used to complile the SUTs data:\n{}".format(refs))



    # Merging. 
    # Get previous pickled dataframe, if there is one. And merge
    if previous_pickled_df == 'no_pickle':
        # no pickled dataframe to be provided, so don't load pickled df and 
        # don't try to merge SUTS dataframe 
        df = suts_df
        sys.stderr.write("\n\nBE AWARE: No dataframe was read in for merging "
        "because the setting `no_pickle`\nwas specified. The final dataframe "
        "generated will just be the SUTs data.")
    else:
        # load the pickled dataframe and merge
        df = pd.read_pickle(previous_pickled_df)
        sys.stderr.write("\n\nThe SUTs will be merged into the provided "
        "dataframe, '{}'".format(previous_pickled_df))
        df = pd.concat([df,suts_df], ignore_index=True)
        sys.stderr.write("\nMerging completed...")
    # Sort dataframe, depending on `sort_on_midpoint` setting. 
    if sort_on_midpoint:
        df = sort_on_midpt_and_reset_ind(df)



    # Handle pickling
    if save_as == "no_output":
        sys.stderr.write("\n\nThe dataframe was not stored for use elsewhere "
        "because `no_output` was specified in place of the output file name.")
    else:
        df.to_pickle(save_as)
        # Let user know
        sys.stderr.write( "\n\nThe created dataframe has been saved as a file "
            "in a manner where other\nPython programs can access it "
            "(pickled form).\n"
            "RESULTING DATAFRAME is stored as ==> '{}'".format(save_as))

    return df





def midpoint(items):
    '''
    takes a iterable of items and returns the midpoint (integer) of the first 
    and second values
    '''
    return int((int(items[0])+int(items[1]))/2)

def generate_output_file_name(file_name):
    '''
    Takes a file name as an argument and returns string for the name of the
    output file. The generated name is based on the original file
    name.

    Specific example
    =================
    Calling function with
        ("mito_transcripts_dataframe.pk1")
    returns
        "mito_transcripts_dataframeplusSUTs.pkl"
    '''
    main_part_of_name, file_extension = os.path.splitext(
        file_name) #from http://stackoverflow.com/questions/541390/extracting-extension-from-filename-in-python
    if '.' in file_name:  #I don't know if this is needed with the os.path.splitext method but I had it before so left it
        return main_part_of_name + suffix_for_saving  + file_extension
    else:
        return file_name + suffix_for_saving + ".pkl"


###--------------------------END OF HELPER FUNCTIONS---------------------------###
###--------------------------END OF HELPER FUNCTIONS---------------------------###















#*******************************************************************************
###-----------------Actual Main portion of script---------------------------###

def main():
    """ Main entry point of the script """
    # placing actual main action in a 'helper'script so can call that easily
    # with a distinguishing name in Jupyter notebooks, where `main()` may get
    # assigned multiple times depending how many scripts imported/pasted in.
    if save_as == 'no_output':
        df = mine_and_mergeSUTs(pickle_df = False)
    else:
        df = mine_and_mergeSUTs()


        





if __name__ == "__main__" and '__file__' in globals():
    """ This is executed when run from the command line """
    # Code with just `if __name__ == "__main__":` alone will be run if pasted
    # into a notebook. The addition of ` and '__file__' in globals()` is based
    # on https://stackoverflow.com/a/22923872/8508004
    # See also https://stackoverflow.com/a/22424821/8508004 for an option to 
    # provide arguments when prototyping a full script in the notebook.
    ###-----------------for parsing command line arguments-----------------------###
    import argparse
    parser = argparse.ArgumentParser(prog='mine_SUTs_and_merge_into_mito_transcripts_df.py',
        description="mine_SUTs_and_merge_into_mito_transcripts_df.py  takes  \
        SUT data and merges it with a dataframe derived from yeast \
        transcriptome that has been stored in pickled form.      \
        **** Script by Wayne Decatur   \
        (fomightez @ github) ***")

    parser.add_argument("input_SUTs", nargs='?', help="**OPTIONAL**Name of file \
        to use as incoming data on SUTS. If none is provided, the data \
        hardcoded into this script will be used. The data should be lines of \
        data matching the order in the header \
        `sys_gene_id     gene_symbol     start        end       strand` with \
        rwos on separate lines and whitespace used to separate individual \
        columns. If the\
        name of dataframe from previous efforts is to be provided when calling \
        the script, something must be provided here, and you can provide \
        `use_hardcoded` without the quotes to use the SUTS data hard-coded in \
        the script.", default='use_hardcoded' , metavar="SUTS")
    parser.add_argument("pickled_df", nargs='?', help="**OPTIONAL**Name of \
        dataframe file from previous efforts. If none is provided, the name \
        '"+previous_pickled_df+"' will be used unless you enter `no_pickle` \
        without \
        quotes in place of the previous dataframe. `no_pickle` will mean no \
        previous dataframe will be accessed, and thus the resulting dataframe \
        will only consist of the data collated in this script (ATYPICAL).", 
        default=previous_pickled_df, metavar="DATAFRAME")
    parser.add_argument('-sa', '--save_as', action='store', type=str, 
        default= generate_output_file_name(previous_pickled_df), help="Use \
        this option to supply a name of \
        the file to save for storing produced dataframe. If none is provided, \
        the name \'"+generate_output_file_name(previous_pickled_df)+"' will be \
        used. To force nothing to be saved, enter \
        `-sa no_output` without quotes as output file (ATYPICAL).") 

    # Note see https://stackoverflow.com/questions/18862836/how-to-open-file-using-argparse#comment35222484_18863004
    # for why not using `argparse.FileType` approach here.
    # See
    # https://stackoverflow.com/questions/4480075/argparse-optional-positional-arguments 
    # and 
    # https://docs.python.org/2/library/argparse.html#nargs for use of `nargs='?'` 
    # to make input and output file names optional. Note that the square brackets
    # shown in the usage out signify optional according to 
    # https://stackoverflow.com/questions/4480075/argparse-optional-positional-arguments#comment40460395_4480202
    # , but because placed under positional I added clarifying text to help 
    # description.
    # IF MODIFYING THIS SCRIPT FOR USE ELSEWHERE AND DON'T NEED/WANT THE OUTPUT 
    # FILE TO BE OPTIONAL, remove `nargs` (& default?) BUT KEEP WHERE NOT
    # USING `argparse.FileType` AND USING `with open` AS CONISDERED MORE PYTHONIC.


    # Normally..
    #I would also like trigger help to display if no arguments provided because need at least one input file
    # BUT NOT HERE
    #if len(sys.argv)==1:    #from http://stackoverflow.com/questions/4042452/display-help-message-with-python-argparse-when-script-is-called-without-any-argu
    #   parser.print_help()
    #   sys.exit(1)
    args = parser.parse_args()
    # Note see https://stackoverflow.com/questions/18862836/how-to-open-file-using-argparse#comment35222484_18863004
    # for why not using `argparse.FileType` approach here.
    input_SUTs = args.input_SUTs
    previous_pickled_df = args.pickled_df
    save_as = args.save_as


    main()

#*******************************************************************************
###-***********************END MAIN PORTION OF SCRIPT***********************-###
#*******************************************************************************

3a111f12f41ef660bd6ccd4883c0fe380ff9a316
<mask token>


def sort_on_midpt_and_reset_ind(df):
    """
    Takes a dataframe and sorts it on midpoint and returns a dataframe
    """
    df = df.sort_values('midpoint', ascending=True)
    return df.reset_index(drop=True)


def mine_and_mergeSUTs(pickle_df=True):
    """
    Takes data on SUTs and merges into a previously generated dataframe of
    transcripts, if one is provided.

    returns a dataframe. This will either be the merged dataframe with the 
    SUTs data added to the provided dataframe, or the SUTs data as a DataFrame
    alone.
    """
    if input_SUTs == 'use_hardcoded':
        suts_df = pd.read_table(StringIO(SUT_data), header=0,
            delim_whitespace=True)
        sys.stderr.write('SUTs data included in the script will be used.')
    else:
        suts_df = pd.read_table(input_SUTs, header=0, delim_whitespace=True)
        sys.stderr.write("SUTs data has been read in the file '{}'".format(
            input_SUTs))
    if 'midpoint' not in suts_df.columns:
        suts_df['midpoint'] = suts_df[['start', 'end']].apply(midpoint, axis=1)
    columns = ['sys_gene_id', 'gene_symbol', 'start', 'end', 'midpoint',
        'strand']
    suts_df = suts_df[columns]
    if sort_on_midpoint:
        suts_df = sort_on_midpt_and_reset_ind(suts_df)
    sys.stderr.write(
        """
For documenting purposes, the following is the SUTs data that was read in:
"""
        )
    sys.stderr.write(suts_df.to_string())
    if input_SUTs == 'use_hardcoded':
        refs = '\n'.join(SUT_data_refs)
        sys.stderr.write(
            """
For documenting purposes, the following sources were used to complile the SUTs data:
{}"""
            .format(refs))
    if previous_pickled_df == 'no_pickle':
        df = suts_df
        sys.stderr.write(
            """

BE AWARE: No dataframe was read in for merging because the setting `no_pickle`
was specified. The final dataframe generated will just be the SUTs data."""
            )
    else:
        df = pd.read_pickle(previous_pickled_df)
        sys.stderr.write(
            "\n\nThe SUTs will be merged into the provided dataframe, '{}'"
            .format(previous_pickled_df))
        df = pd.concat([df, suts_df], ignore_index=True)
        sys.stderr.write('\nMerging completed...')
    if sort_on_midpoint:
        df = sort_on_midpt_and_reset_ind(df)
    if save_as == 'no_output':
        sys.stderr.write(
            """

The dataframe was not stored for use elsewhere because `no_output` was specified in place of the output file name."""
            )
    else:
        df.to_pickle(save_as)
        sys.stderr.write(
            """

The created dataframe has been saved as a file in a manner where other
Python programs can access it (pickled form).
RESULTING DATAFRAME is stored as ==> '{}'"""
            .format(save_as))
    return df


<mask token>


def generate_output_file_name(file_name):
    """
    Takes a file name as an argument and returns string for the name of the
    output file. The generated name is based on the original file
    name.

    Specific example
    =================
    Calling function with
        ("mito_transcripts_dataframe.pk1")
    returns
        "mito_transcripts_dataframeplusSUTs.pkl"
    """
    main_part_of_name, file_extension = os.path.splitext(file_name)
    if '.' in file_name:
        return main_part_of_name + suffix_for_saving + file_extension
    else:
        return file_name + suffix_for_saving + '.pkl'


def main():
    """ Main entry point of the script """
    if save_as == 'no_output':
        df = mine_and_mergeSUTs(pickle_df=False)
    else:
        df = mine_and_mergeSUTs()


<mask token>

<mask token>


def sort_on_midpt_and_reset_ind(df):
    """
    Takes a dataframe and sorts it on midpoint and returns a dataframe
    """
    df = df.sort_values('midpoint', ascending=True)
    return df.reset_index(drop=True)


def mine_and_mergeSUTs(pickle_df=True):
    """
    Takes data on SUTs and merges into a previously generated dataframe of
    transcripts, if one is provided.

    returns a dataframe. This will either be the merged dataframe with the 
    SUTs data added to the provided dataframe, or the SUTs data as a DataFrame
    alone.
    """
    if input_SUTs == 'use_hardcoded':
        suts_df = pd.read_table(StringIO(SUT_data), header=0,
            delim_whitespace=True)
        sys.stderr.write('SUTs data included in the script will be used.')
    else:
        suts_df = pd.read_table(input_SUTs, header=0, delim_whitespace=True)
        sys.stderr.write("SUTs data has been read in the file '{}'".format(
            input_SUTs))
    if 'midpoint' not in suts_df.columns:
        suts_df['midpoint'] = suts_df[['start', 'end']].apply(midpoint, axis=1)
    columns = ['sys_gene_id', 'gene_symbol', 'start', 'end', 'midpoint',
        'strand']
    suts_df = suts_df[columns]
    if sort_on_midpoint:
        suts_df = sort_on_midpt_and_reset_ind(suts_df)
    sys.stderr.write(
        """
For documenting purposes, the following is the SUTs data that was read in:
"""
        )
    sys.stderr.write(suts_df.to_string())
    if input_SUTs == 'use_hardcoded':
        refs = '\n'.join(SUT_data_refs)
        sys.stderr.write(
            """
For documenting purposes, the following sources were used to complile the SUTs data:
{}"""
            .format(refs))
    if previous_pickled_df == 'no_pickle':
        df = suts_df
        sys.stderr.write(
            """

BE AWARE: No dataframe was read in for merging because the setting `no_pickle`
was specified. The final dataframe generated will just be the SUTs data."""
            )
    else:
        df = pd.read_pickle(previous_pickled_df)
        sys.stderr.write(
            "\n\nThe SUTs will be merged into the provided dataframe, '{}'"
            .format(previous_pickled_df))
        df = pd.concat([df, suts_df], ignore_index=True)
        sys.stderr.write('\nMerging completed...')
    if sort_on_midpoint:
        df = sort_on_midpt_and_reset_ind(df)
    if save_as == 'no_output':
        sys.stderr.write(
            """

The dataframe was not stored for use elsewhere because `no_output` was specified in place of the output file name."""
            )
    else:
        df.to_pickle(save_as)
        sys.stderr.write(
            """

The created dataframe has been saved as a file in a manner where other
Python programs can access it (pickled form).
RESULTING DATAFRAME is stored as ==> '{}'"""
            .format(save_as))
    return df


def midpoint(items):
    """
    takes a iterable of items and returns the midpoint (integer) of the first 
    and second values
    """
    return int((int(items[0]) + int(items[1])) / 2)


def generate_output_file_name(file_name):
    """
    Takes a file name as an argument and returns string for the name of the
    output file. The generated name is based on the original file
    name.

    Specific example
    =================
    Calling function with
        ("mito_transcripts_dataframe.pk1")
    returns
        "mito_transcripts_dataframeplusSUTs.pkl"
    """
    main_part_of_name, file_extension = os.path.splitext(file_name)
    if '.' in file_name:
        return main_part_of_name + suffix_for_saving + file_extension
    else:
        return file_name + suffix_for_saving + '.pkl'


def main():
    """ Main entry point of the script """
    if save_as == 'no_output':
        df = mine_and_mergeSUTs(pickle_df=False)
    else:
        df = mine_and_mergeSUTs()


<mask token>

__author__ = 'Wayne Decatur'
__license__ = 'MIT'
__version__ = '0.1.0'
SUT_data = """

sys_gene_id     gene_symbol     start        end       strand
SUT2729.1       None        6229       6557         -1
SUT2730.1       None        13893      13957         -1
SUT2731.1       None        18277      18365         -1
SUT2733.1       None        31545      31608         1
SUT2732.1       None        41421      41541         -1
SUT2734.1       None        62601      62872         1

"""
SUT_data_refs = [
    'SUT2729.1 through SUT2734.1 from Lardenois et al PMID: 21149693.']
previous_pickled_df = 'mito_transcripts_dataframe.pkl'
suffix_for_saving = 'plusSUTs'
sort_on_midpoint = True
<mask token>
try:
    from StringIO import StringIO
except ImportError:
    from io import StringIO


def sort_on_midpt_and_reset_ind(df):
    """
    Takes a dataframe and sorts it on midpoint and returns a dataframe
    """
    df = df.sort_values('midpoint', ascending=True)
    return df.reset_index(drop=True)


def mine_and_mergeSUTs(pickle_df=True):
    """
    Takes data on SUTs and merges into a previously generated dataframe of
    transcripts, if one is provided.

    returns a dataframe. This will either be the merged dataframe with the 
    SUTs data added to the provided dataframe, or the SUTs data as a DataFrame
    alone.
    """
    if input_SUTs == 'use_hardcoded':
        suts_df = pd.read_table(StringIO(SUT_data), header=0,
            delim_whitespace=True)
        sys.stderr.write('SUTs data included in the script will be used.')
    else:
        suts_df = pd.read_table(input_SUTs, header=0, delim_whitespace=True)
        sys.stderr.write("SUTs data has been read in the file '{}'".format(
            input_SUTs))
    if 'midpoint' not in suts_df.columns:
        suts_df['midpoint'] = suts_df[['start', 'end']].apply(midpoint, axis=1)
    columns = ['sys_gene_id', 'gene_symbol', 'start', 'end', 'midpoint',
        'strand']
    suts_df = suts_df[columns]
    if sort_on_midpoint:
        suts_df = sort_on_midpt_and_reset_ind(suts_df)
    sys.stderr.write(
        """
For documenting purposes, the following is the SUTs data that was read in:
"""
        )
    sys.stderr.write(suts_df.to_string())
    if input_SUTs == 'use_hardcoded':
        refs = '\n'.join(SUT_data_refs)
        sys.stderr.write(
            """
For documenting purposes, the following sources were used to complile the SUTs data:
{}"""
            .format(refs))
    if previous_pickled_df == 'no_pickle':
        df = suts_df
        sys.stderr.write(
            """

BE AWARE: No dataframe was read in for merging because the setting `no_pickle`
was specified. The final dataframe generated will just be the SUTs data."""
            )
    else:
        df = pd.read_pickle(previous_pickled_df)
        sys.stderr.write(
            "\n\nThe SUTs will be merged into the provided dataframe, '{}'"
            .format(previous_pickled_df))
        df = pd.concat([df, suts_df], ignore_index=True)
        sys.stderr.write('\nMerging completed...')
    if sort_on_midpoint:
        df = sort_on_midpt_and_reset_ind(df)
    if save_as == 'no_output':
        sys.stderr.write(
            """

The dataframe was not stored for use elsewhere because `no_output` was specified in place of the output file name."""
            )
    else:
        df.to_pickle(save_as)
        sys.stderr.write(
            """

The created dataframe has been saved as a file in a manner where other
Python programs can access it (pickled form).
RESULTING DATAFRAME is stored as ==> '{}'"""
            .format(save_as))
    return df


def midpoint(items):
    """
    takes a iterable of items and returns the midpoint (integer) of the first 
    and second values
    """
    return int((int(items[0]) + int(items[1])) / 2)


def generate_output_file_name(file_name):
    """
    Takes a file name as an argument and returns string for the name of the
    output file. The generated name is based on the original file
    name.

    Specific example
    =================
    Calling function with
        ("mito_transcripts_dataframe.pk1")
    returns
        "mito_transcripts_dataframeplusSUTs.pkl"
    """
    main_part_of_name, file_extension = os.path.splitext(file_name)
    if '.' in file_name:
        return main_part_of_name + suffix_for_saving + file_extension
    else:
        return file_name + suffix_for_saving + '.pkl'


def main():
    """ Main entry point of the script """
    if save_as == 'no_output':
        df = mine_and_mergeSUTs(pickle_df=False)
    else:
        df = mine_and_mergeSUTs()


if __name__ == '__main__' and '__file__' in globals():
    """ This is executed when run from the command line """
    import argparse
    parser = argparse.ArgumentParser(prog=
        'mine_SUTs_and_merge_into_mito_transcripts_df.py', description=
        'mine_SUTs_and_merge_into_mito_transcripts_df.py  takes          SUT data and merges it with a dataframe derived from yeast         transcriptome that has been stored in pickled form.              **** Script by Wayne Decatur           (fomightez @ github) ***'
        )
    parser.add_argument('input_SUTs', nargs='?', help=
        '**OPTIONAL**Name of file         to use as incoming data on SUTS. If none is provided, the data         hardcoded into this script will be used. The data should be lines of         data matching the order in the header         `sys_gene_id     gene_symbol     start        end       strand` with         rwos on separate lines and whitespace used to separate individual         columns. If the        name of dataframe from previous efforts is to be provided when calling         the script, something must be provided here, and you can provide         `use_hardcoded` without the quotes to use the SUTS data hard-coded in         the script.'
        , default='use_hardcoded', metavar='SUTS')
    parser.add_argument('pickled_df', nargs='?', help=
        "**OPTIONAL**Name of         dataframe file from previous efforts. If none is provided, the name         '"
         + previous_pickled_df +
        "' will be used unless you enter `no_pickle`         without         quotes in place of the previous dataframe. `no_pickle` will mean no         previous dataframe will be accessed, and thus the resulting dataframe         will only consist of the data collated in this script (ATYPICAL)."
        , default=previous_pickled_df, metavar='DATAFRAME')
    parser.add_argument('-sa', '--save_as', action='store', type=str,
        default=generate_output_file_name(previous_pickled_df), help=
        "Use         this option to supply a name of         the file to save for storing produced dataframe. If none is provided,         the name '"
         + generate_output_file_name(previous_pickled_df) +
        "' will be         used. To force nothing to be saved, enter         `-sa no_output` without quotes as output file (ATYPICAL)."
        )
    args = parser.parse_args()
    input_SUTs = args.input_SUTs
    previous_pickled_df = args.pickled_df
    save_as = args.save_as
    main()

__author__ = 'Wayne Decatur'
__license__ = 'MIT'
__version__ = '0.1.0'
SUT_data = """

sys_gene_id     gene_symbol     start        end       strand
SUT2729.1       None        6229       6557         -1
SUT2730.1       None        13893      13957         -1
SUT2731.1       None        18277      18365         -1
SUT2733.1       None        31545      31608         1
SUT2732.1       None        41421      41541         -1
SUT2734.1       None        62601      62872         1

"""
SUT_data_refs = [
    'SUT2729.1 through SUT2734.1 from Lardenois et al PMID: 21149693.']
previous_pickled_df = 'mito_transcripts_dataframe.pkl'
suffix_for_saving = 'plusSUTs'
sort_on_midpoint = True
import sys
import os
import pandas as pd
try:
    from StringIO import StringIO
except ImportError:
    from io import StringIO


def sort_on_midpt_and_reset_ind(df):
    """
    Takes a dataframe and sorts it on midpoint and returns a dataframe
    """
    df = df.sort_values('midpoint', ascending=True)
    return df.reset_index(drop=True)


def mine_and_mergeSUTs(pickle_df=True):
    """
    Takes data on SUTs and merges into a previously generated dataframe of
    transcripts, if one is provided.

    returns a dataframe. This will either be the merged dataframe with the 
    SUTs data added to the provided dataframe, or the SUTs data as a DataFrame
    alone.
    """
    if input_SUTs == 'use_hardcoded':
        suts_df = pd.read_table(StringIO(SUT_data), header=0,
            delim_whitespace=True)
        sys.stderr.write('SUTs data included in the script will be used.')
    else:
        suts_df = pd.read_table(input_SUTs, header=0, delim_whitespace=True)
        sys.stderr.write("SUTs data has been read in the file '{}'".format(
            input_SUTs))
    if 'midpoint' not in suts_df.columns:
        suts_df['midpoint'] = suts_df[['start', 'end']].apply(midpoint, axis=1)
    columns = ['sys_gene_id', 'gene_symbol', 'start', 'end', 'midpoint',
        'strand']
    suts_df = suts_df[columns]
    if sort_on_midpoint:
        suts_df = sort_on_midpt_and_reset_ind(suts_df)
    sys.stderr.write(
        """
For documenting purposes, the following is the SUTs data that was read in:
"""
        )
    sys.stderr.write(suts_df.to_string())
    if input_SUTs == 'use_hardcoded':
        refs = '\n'.join(SUT_data_refs)
        sys.stderr.write(
            """
For documenting purposes, the following sources were used to complile the SUTs data:
{}"""
            .format(refs))
    if previous_pickled_df == 'no_pickle':
        df = suts_df
        sys.stderr.write(
            """

BE AWARE: No dataframe was read in for merging because the setting `no_pickle`
was specified. The final dataframe generated will just be the SUTs data."""
            )
    else:
        df = pd.read_pickle(previous_pickled_df)
        sys.stderr.write(
            "\n\nThe SUTs will be merged into the provided dataframe, '{}'"
            .format(previous_pickled_df))
        df = pd.concat([df, suts_df], ignore_index=True)
        sys.stderr.write('\nMerging completed...')
    if sort_on_midpoint:
        df = sort_on_midpt_and_reset_ind(df)
    if save_as == 'no_output':
        sys.stderr.write(
            """

The dataframe was not stored for use elsewhere because `no_output` was specified in place of the output file name."""
            )
    else:
        df.to_pickle(save_as)
        sys.stderr.write(
            """

The created dataframe has been saved as a file in a manner where other
Python programs can access it (pickled form).
RESULTING DATAFRAME is stored as ==> '{}'"""
            .format(save_as))
    return df


def midpoint(items):
    """
    takes a iterable of items and returns the midpoint (integer) of the first 
    and second values
    """
    return int((int(items[0]) + int(items[1])) / 2)


def generate_output_file_name(file_name):
    """
    Takes a file name as an argument and returns string for the name of the
    output file. The generated name is based on the original file
    name.

    Specific example
    =================
    Calling function with
        ("mito_transcripts_dataframe.pk1")
    returns
        "mito_transcripts_dataframeplusSUTs.pkl"
    """
    main_part_of_name, file_extension = os.path.splitext(file_name)
    if '.' in file_name:
        return main_part_of_name + suffix_for_saving + file_extension
    else:
        return file_name + suffix_for_saving + '.pkl'


def main():
    """ Main entry point of the script """
    if save_as == 'no_output':
        df = mine_and_mergeSUTs(pickle_df=False)
    else:
        df = mine_and_mergeSUTs()


if __name__ == '__main__' and '__file__' in globals():
    """ This is executed when run from the command line """
    import argparse
    parser = argparse.ArgumentParser(prog=
        'mine_SUTs_and_merge_into_mito_transcripts_df.py', description=
        'mine_SUTs_and_merge_into_mito_transcripts_df.py  takes          SUT data and merges it with a dataframe derived from yeast         transcriptome that has been stored in pickled form.              **** Script by Wayne Decatur           (fomightez @ github) ***'
        )
    parser.add_argument('input_SUTs', nargs='?', help=
        '**OPTIONAL**Name of file         to use as incoming data on SUTS. If none is provided, the data         hardcoded into this script will be used. The data should be lines of         data matching the order in the header         `sys_gene_id     gene_symbol     start        end       strand` with         rwos on separate lines and whitespace used to separate individual         columns. If the        name of dataframe from previous efforts is to be provided when calling         the script, something must be provided here, and you can provide         `use_hardcoded` without the quotes to use the SUTS data hard-coded in         the script.'
        , default='use_hardcoded', metavar='SUTS')
    parser.add_argument('pickled_df', nargs='?', help=
        "**OPTIONAL**Name of         dataframe file from previous efforts. If none is provided, the name         '"
         + previous_pickled_df +
        "' will be used unless you enter `no_pickle`         without         quotes in place of the previous dataframe. `no_pickle` will mean no         previous dataframe will be accessed, and thus the resulting dataframe         will only consist of the data collated in this script (ATYPICAL)."
        , default=previous_pickled_df, metavar='DATAFRAME')
    parser.add_argument('-sa', '--save_as', action='store', type=str,
        default=generate_output_file_name(previous_pickled_df), help=
        "Use         this option to supply a name of         the file to save for storing produced dataframe. If none is provided,         the name '"
         + generate_output_file_name(previous_pickled_df) +
        "' will be         used. To force nothing to be saved, enter         `-sa no_output` without quotes as output file (ATYPICAL)."
        )
    args = parser.parse_args()
    input_SUTs = args.input_SUTs
    previous_pickled_df = args.pickled_df
    save_as = args.save_as
    main()

#!/usr/bin/env python
# mine_SUTs_and_merge_into_mito_transcripts_df.py
__author__ = "Wayne Decatur" #fomightez on GitHub
__license__ = "MIT"
__version__ = "0.1.0"


# mine_SUTs_and_merge_into_mito_transcripts_df.py by Wayne Decatur
# ver 0.1
#
#*******************************************************************************
# Compatible with both Python 2.7 and Python 3.6 (verified); written initially 
# in Python 2.7 to hopefully be convertible to Python 3.
#
# PURPOSE: Takes details of SUTs and a dataframe previously generated from a 
# transcriptome file and produces a dataframe where the SUTs data is merged.
# Intended to work with the dataframe generated by running 
# `mine_mito_features_from_transcriptome.py` on a file of the Ensembl cDNAs + 
# Ensembl ncRNAs (plus some custom RNAs I added) as the transcriptome.
#
# Returns a dataframe with the details for the mito features with the SUTS added.
#
#
#
#
#
#
# Dependencies beyond the mostly standard libraries/modules:
#
#
#
# VERSION HISTORY:
# v.0.1. basic working version
#
# To do:
# - see `POSSIBLE TO DO` about references when using a file as source of SUTs data.
#
#
#
# TO RUN:
# Example,
# Enter on the command line of your terminal, the line
#-----------------------------------
# python mine_SUTs_and_merge_into_mito_transcripts_df.py
#-----------------------------------
# Issue `python mine_SUTs_and_merge_into_mito_transcripts_df.py -h` for details.
# 
#
# To use this after pasting (or loading) into a cell in a Jupyter notebook, 
# define `input_SUTs` as the source of the SUTs data. It will either be the name 
# of the file to read or use `use_harcoded` to specify to use the SUTs data that
# was hardcoded into the script. Also define `save_as` the name of file to store
# the resulting dataframe. To not save anything, set `save_as= "no_output"`.  
# Then call the main function in a cell as shown below:
# input_SUTs = "use_hardcoded"
# save_as = "mito_transcripts_dataframeplusSUTS.pkl"
# df = mine_and_mergeSUTs()
# 
# Alternatively set `previous_pickled_df="no_pickle"`, before calling the main
# function in Jupyter to not supply a previous dataframe (useful for testing).
# Additionally, # add `pickle_df = False` like
# `df = mine_and_mergeSUTs(pickle_df = False)` to 
# not produce a pickled dataframe file, and then you don't need to supply a 
# `save_as` setting either.
#
#
#*******************************************************************************
#


#*******************************************************************************
##################################
#  USER ADJUSTABLE VALUES        #

##################################
#

SUT_data ='''

sys_gene_id     gene_symbol     start        end       strand
SUT2729.1       None        6229       6557         -1
SUT2730.1       None        13893      13957         -1
SUT2731.1       None        18277      18365         -1
SUT2733.1       None        31545      31608         1
SUT2732.1       None        41421      41541         -1
SUT2734.1       None        62601      62872         1

'''
# above entered by hand from copying and pasting from
# `Lardenois_2011_noncoding_RNAs_V64-chrmt-1..85146.gff3`
# At this point, I haven't scripted to mine from the gff3 format directly, but 
# CERTAINLY I should AS I FORESEE similar exported tracks being a source of 
# others. (Notes for that: `Name=` is descriptor for sys_id.) And if there are
# others, I'd need to adjust it to mine from multiple gffs and merge all into
# the `suts_df`.
# Details of SUT_data:
# The data for SUT2729.1 through SUT2734.1  is from 
# Lardenois_2011_noncoding_RNAs_V64      PMID: 21149693.  I
# found it by adding tracks in jbrowse and exported the gff3.
# JUST DEALING WITH THE MITOCHONDRIAL SUTs FOR NOW.

SUT_data_refs = [
"SUT2729.1 through SUT2734.1 from Lardenois et al PMID: 21149693.",
                ]



previous_pickled_df = "mito_transcripts_dataframe.pkl"
suffix_for_saving = "plusSUTs"

sort_on_midpoint = True # sort dataframe on gene midpoint location



#
#*******************************************************************************
#**********************END USER ADJUSTABLE VARIABLES****************************


















#*******************************************************************************
#*******************************************************************************
###DO NOT EDIT BELOW HERE - ENTER VALUES ABOVE###

import sys
import os
import pandas as pd
try:
    from StringIO import StringIO
except ImportError:
    from io import StringIO





###---------------------------HELPER FUNCTIONS---------------------------------###
def sort_on_midpt_and_reset_ind(df):
    '''
    Takes a dataframe and sorts it on midpoint and returns a dataframe
    '''
    df = df.sort_values('midpoint', ascending=True)
    return df.reset_index(drop=True) # res the index to reflect re-order


def mine_and_mergeSUTs(pickle_df = True):
    '''
    Takes data on SUTs and merges into a previously generated dataframe of
    transcripts, if one is provided.

    returns a dataframe. This will either be the merged dataframe with the 
    SUTs data added to the provided dataframe, or the SUTs data as a DataFrame
    alone.
    '''
    # verify the file to add data to exists
    #os.path.isfile(fname) 


    #SUT data
    # if file provided read in data from there, otherwise read in data
    # placed in 'user-defined variables '
    if input_SUTs == 'use_hardcoded':
        suts_df = pd.read_table(
            StringIO(SUT_data), header=0, delim_whitespace= True)
        sys.stderr.write("SUTs data included in the script will be used.")
    else:
        suts_df = pd.read_table(input_SUTs, header=0, delim_whitespace= True)
        sys.stderr.write( "SUTs data has been read in the file '{}'"
            .format(input_SUTs))
        # POSSIBLE TO DO: add a way to have the references in the file
        # & have them mined out of there and punt in list `SUT_data_refs`. Then
        # remove conditional `input_SUTs == 'use_hardcoded'` below when 
        # documenting references to console/Jupyter cell.
    # need columns at end to match, before merging, so...
    # need to generate midpoint column (unless, although unlikely, already there)
    if 'midpoint' not in suts_df.columns:
        suts_df['midpoint'] = suts_df[['start','end']].apply(midpoint, axis=1)
    # Easiest to be consistent with previous convention of column order. Plus 
    # makes it so won't have two orders going for each dataframe and still 
    # require setting column order after merge. So set order to match previous.
    columns = ['sys_gene_id','gene_symbol',
    'start','end','midpoint','strand']
    suts_df = suts_df[columns]
    # sort on midpoint, depending on `sort_on_midpoint` setting.
    if sort_on_midpoint:
        suts_df = sort_on_midpt_and_reset_ind(suts_df)
    # send the SUTs to the terminal or Jupyter notebook display in some manner 
    # so will be documenting what SUTs are being mined/added in the course of
    # running without the user needing to look at dataframe or output. 
    # Using `df.to_string()` because more universal than `print(df)` 
    # or Jupyter's `display(df)`.
    sys.stderr.write( "\nFor documenting purposes, the following is "
        "the SUTs data that was read in:\n")
    #with pd.option_context('display.max_rows', None, 'display.max_columns', None):
    #    display(df)
    sys.stderr.write(suts_df.to_string())
    if input_SUTs == 'use_hardcoded':
        refs = "\n".join(SUT_data_refs)
        sys.stderr.write( "\nFor documenting purposes, the following sources "
        "were used to complile the SUTs data:\n{}".format(refs))



    # Merging. 
    # Get previous pickled dataframe, if there is one. And merge
    if previous_pickled_df == 'no_pickle':
        # no pickled dataframe to be provided, so don't load pickled df and 
        # don't try to merge SUTS dataframe 
        df = suts_df
        sys.stderr.write("\n\nBE AWARE: No dataframe was read in for merging "
        "because the setting `no_pickle`\nwas specified. The final dataframe "
        "generated will just be the SUTs data.")
    else:
        # load the pickled dataframe and merge
        df = pd.read_pickle(previous_pickled_df)
        sys.stderr.write("\n\nThe SUTs will be merged into the provided "
        "dataframe, '{}'".format(previous_pickled_df))
        df = pd.concat([df,suts_df], ignore_index=True)
        sys.stderr.write("\nMerging completed...")
    # Sort dataframe, depending on `sort_on_midpoint` setting. 
    if sort_on_midpoint:
        df = sort_on_midpt_and_reset_ind(df)



    # Handle pickling
    if save_as == "no_output":
        sys.stderr.write("\n\nThe dataframe was not stored for use elsewhere "
        "because `no_output` was specified in place of the output file name.")
    else:
        df.to_pickle(save_as)
        # Let user know
        sys.stderr.write( "\n\nThe created dataframe has been saved as a file "
            "in a manner where other\nPython programs can access it "
            "(pickled form).\n"
            "RESULTING DATAFRAME is stored as ==> '{}'".format(save_as))

    return df





def midpoint(items):
    '''
    takes a iterable of items and returns the midpoint (integer) of the first 
    and second values
    '''
    return int((int(items[0])+int(items[1]))/2)

def generate_output_file_name(file_name):
    '''
    Takes a file name as an argument and returns string for the name of the
    output file. The generated name is based on the original file
    name.

    Specific example
    =================
    Calling function with
        ("mito_transcripts_dataframe.pk1")
    returns
        "mito_transcripts_dataframeplusSUTs.pkl"
    '''
    main_part_of_name, file_extension = os.path.splitext(
        file_name) #from http://stackoverflow.com/questions/541390/extracting-extension-from-filename-in-python
    if '.' in file_name:  #I don't know if this is needed with the os.path.splitext method but I had it before so left it
        return main_part_of_name + suffix_for_saving  + file_extension
    else:
        return file_name + suffix_for_saving + ".pkl"


###--------------------------END OF HELPER FUNCTIONS---------------------------###
###--------------------------END OF HELPER FUNCTIONS---------------------------###















#*******************************************************************************
###-----------------Actual Main portion of script---------------------------###

def main():
    """ Main entry point of the script """
    # placing actual main action in a 'helper'script so can call that easily
    # with a distinguishing name in Jupyter notebooks, where `main()` may get
    # assigned multiple times depending how many scripts imported/pasted in.
    if save_as == 'no_output':
        df = mine_and_mergeSUTs(pickle_df = False)
    else:
        df = mine_and_mergeSUTs()


        





if __name__ == "__main__" and '__file__' in globals():
    """ This is executed when run from the command line """
    # Code with just `if __name__ == "__main__":` alone will be run if pasted
    # into a notebook. The addition of ` and '__file__' in globals()` is based
    # on https://stackoverflow.com/a/22923872/8508004
    # See also https://stackoverflow.com/a/22424821/8508004 for an option to 
    # provide arguments when prototyping a full script in the notebook.
    ###-----------------for parsing command line arguments-----------------------###
    import argparse
    parser = argparse.ArgumentParser(prog='mine_SUTs_and_merge_into_mito_transcripts_df.py',
        description="mine_SUTs_and_merge_into_mito_transcripts_df.py  takes  \
        SUT data and merges it with a dataframe derived from yeast \
        transcriptome that has been stored in pickled form.      \
        **** Script by Wayne Decatur   \
        (fomightez @ github) ***")

    parser.add_argument("input_SUTs", nargs='?', help="**OPTIONAL**Name of file \
        to use as incoming data on SUTS. If none is provided, the data \
        hardcoded into this script will be used. The data should be lines of \
        data matching the order in the header \
        `sys_gene_id     gene_symbol     start        end       strand` with \
        rwos on separate lines and whitespace used to separate individual \
        columns. If the\
        name of dataframe from previous efforts is to be provided when calling \
        the script, something must be provided here, and you can provide \
        `use_hardcoded` without the quotes to use the SUTS data hard-coded in \
        the script.", default='use_hardcoded' , metavar="SUTS")
    parser.add_argument("pickled_df", nargs='?', help="**OPTIONAL**Name of \
        dataframe file from previous efforts. If none is provided, the name \
        '"+previous_pickled_df+"' will be used unless you enter `no_pickle` \
        without \
        quotes in place of the previous dataframe. `no_pickle` will mean no \
        previous dataframe will be accessed, and thus the resulting dataframe \
        will only consist of the data collated in this script (ATYPICAL).", 
        default=previous_pickled_df, metavar="DATAFRAME")
    parser.add_argument('-sa', '--save_as', action='store', type=str, 
        default= generate_output_file_name(previous_pickled_df), help="Use \
        this option to supply a name of \
        the file to save for storing produced dataframe. If none is provided, \
        the name \'"+generate_output_file_name(previous_pickled_df)+"' will be \
        used. To force nothing to be saved, enter \
        `-sa no_output` without quotes as output file (ATYPICAL).") 

    # Note see https://stackoverflow.com/questions/18862836/how-to-open-file-using-argparse#comment35222484_18863004
    # for why not using `argparse.FileType` approach here.
    # See
    # https://stackoverflow.com/questions/4480075/argparse-optional-positional-arguments 
    # and 
    # https://docs.python.org/2/library/argparse.html#nargs for use of `nargs='?'` 
    # to make input and output file names optional. Note that the square brackets
    # shown in the usage out signify optional according to 
    # https://stackoverflow.com/questions/4480075/argparse-optional-positional-arguments#comment40460395_4480202
    # , but because placed under positional I added clarifying text to help 
    # description.
    # IF MODIFYING THIS SCRIPT FOR USE ELSEWHERE AND DON'T NEED/WANT THE OUTPUT 
    # FILE TO BE OPTIONAL, remove `nargs` (& default?) BUT KEEP WHERE NOT
    # USING `argparse.FileType` AND USING `with open` AS CONISDERED MORE PYTHONIC.


    # Normally..
    #I would also like trigger help to display if no arguments provided because need at least one input file
    # BUT NOT HERE
    #if len(sys.argv)==1:    #from http://stackoverflow.com/questions/4042452/display-help-message-with-python-argparse-when-script-is-called-without-any-argu
    #   parser.print_help()
    #   sys.exit(1)
    args = parser.parse_args()
    # Note see https://stackoverflow.com/questions/18862836/how-to-open-file-using-argparse#comment35222484_18863004
    # for why not using `argparse.FileType` approach here.
    input_SUTs = args.input_SUTs
    previous_pickled_df = args.pickled_df
    save_as = args.save_as


    main()

#*******************************************************************************
###-***********************END MAIN PORTION OF SCRIPT***********************-###
#*******************************************************************************
