import sys # ' sys ' is only needed to display our Pyhon version
print("This code is running on Python version " + sys.version[0:5])

        
        Copy
    

    	
            
import DatastreamDSWS as DSWS
 
# We can use our Refinitiv's Datastream Web Socket (DSWS) API keys that allows us to be identified by
# Refinitiv's back-end services and enables us to request (and fetch) data:
# Credentials are placed in a text file so that it may be used in this code without showing it itself.
(DSWS_username, DSWS_password) = (open("Datastream_username.txt","r"),
                                  open("Datastream_password.txt","r"))
 
ds = DSWS.Datastream(username = str(DSWS_username.read()),
                     password = str(DSWS_password.read()))
 
# It is best to close the files we opened in order to make sure that we don't stop any other
# services/programs from accessing them if they need to.
DSWS_username.close()
DSWS_password.close()
 
 
# # Alternatively one can use the following:
# import getpass
# dsusername = input()
# dspassword = getpass.getpass()
# ds = DSWS.Datastream(username = dsusername, password = dspassword)

        
        Copy
    

    	
            
import calendar # This library will be used to get the last day of any one month.
from datetime import date # ' datetime ' and ' date ' is crucial to manipulate dates.
import datetime # ' datetime ' and ' date ' is crucial to manipulate dates.

        
        Copy
    

    	
            
import dateutil # ' datetime ' and ' date ' is crucial to manipulate dates.
import numpy as np
import pandas as pd
for i,j in zip(["dateutil", "np","pd"],
               [ dateutil,   np,  pd]):
    print(f"The {i} library imported in this code is version: {j.__version__}")

        
        Copy
    

    	
            
list_of_fields = ['X(EVT1FD12)~USD', 'X(EBT1FD12)~USD', 'X(EBD1FD12)~USD', 'X(CPS1FD12)~USD*X(IBNOSH)',
                  'X(MV)~USD', 'PEFD12', 'X(SAL1FD12)~USD', 'X(BPS1FD12)~USD*X(IBNOSH)']

        
        Copy
    

    	
            
df = ds.get_data("LMSWRLD0313|L",
                 ['NAME',
                  list_of_fields[0]],
                 start = '2013-08-31',
                 kind = 0)

        
        Copy
    

    	
            
# ' df["Datatype"] == "X(EVT1FD12)~USD" ' here sieves through ' df ' and only
#    returns data where what is in the outer most square brackets is
#    true, i.e.: where values in the 'Datatype' column is 'X(EVT1FD12)~USD'.
df[df["Datatype"] == "X(EVT1FD12)~USD"] # Note that ' "X(EVT1FD12)~USD" ' is the same as ' list_of_fields[0] '

        
        Copy
    

    	
            df[df["Instrument"] == df["Instrument"].unique()[0]] # ' [0] ' here is needed to indicate that we want the 0th unique value in question.
        
        Copy
    

    	
            df[df["Datatype"] == list_of_fields[0]]
        
        Copy
    

    	
            
print(list_of_fields[0]) # Printing out the field of interest's name
df[df["Datatype"] == list_of_fields[0]][df["Value"] != 'NA']["Value"].median() # ' .median() ' gets us - you guessed it - the median value.

        
        Copy
    

    	
            
def DSWS_Median_Index_Data(indices = ["MSWRLD"],
                           list_of_fields = ['X(EVT1FD12)~USD', 'X(EBT1FD12)~USD', 'X(EBD1FD12)~USD', 'X(CPS1FD12)~USD*X(IBNOSH)',
                                             'X(MV)~USD', 'PEFD12', 'X(SAL1FD12)~USD', 'X(BPS1FD12)~USD*X(IBNOSH)'], # Don't include non numerical fields like 'GDSCN'.
                          start = '2013-01-01',
                          end = '2021-01-31',
                          track_progress = False,
                          add_to_global_DF = False):
    """
    ' DSWS_Median_Index_Data ' requests data defined in ' list_of_fields ' from Datastream's API - DataStream Web
    Services (DSWS) - for a series of indices of choice - defined in 'indices' - and returns three lists full of Pandas data-frames:
    1st - the raw output from the requests (with columns 'Instrument', 'Datatype', 'Value' and 'Dates');
    2nd - the monthly median from the raw output from the requests (with columns of the requested metrics and an index of dates);
    3rd - the raw data pulled from the DSWS request (and where issues are most likely to lie).
    
    One could re-write this basing themselves off datetime objects, it would be equivalent; its code may look neater too.
    
    indices (list): List of strings of the DSWS mnemonic for the index for which data is requested.
    Default: indices = ["MSWRLD"]
    
    list_of_fields (list): List of strings of the DSWS fields we are after. This has to be comprised of 49 elements or less.
    Don't include non numerical fields like 'GDSCN'. That will result in a printed error
    Default: list_of_fields = ['EVT1FD12', 'EBT1FD12', 'EBD1FD12', 'X(CPS1FD12)*X(IBNOSH)', 'X(MV)~IBCUR', 'PEFD12', 'SAL1FD12', 'X(BPS1FD12)*X(IBNOSH)', '(X(SAL1FD12)*X(IBUNIT))', '354E(X)', 'X(INC1FD12)*X(IBUNIT)', 'X(FCF1FD12)*X(IBNOSH)', 'X(FCF1FD12)*X(IBNOSH)*X(IBUNIT)', 'GDSCN']
    
    start (str): String of the start date from which we are asking for data in 'yyy-mm-dd' format.
    Default: start = '2013-01-01'
    
    end (str): String of the end date until which we are asking for data in 'yyy-mm-dd' format.
    Default: end = '2021-01-31'
    
    track_progress (Boolean): If set to True, the ' DSWS_Median_Index_Data ' function will print out each mnemonic requested of DSWS
    Default: track_progress = False
    
    
    track_progress (Boolean): If set to True, each string of the transformed index will be printed off - remember that there is one per month.
    Default: track_progress = False
    
    add_to_global_DF (Boolean): If set to True, on top of returning the 3 lists of Pandas data-frames described above, this function will append lists named DF, _DF, and debug_df.
    This does mean that you need to include a line such as 'DF, _DF, debug_df = [], [], []' beforehand as a result.
    Default: add_to_global_DF = False
    """
    
    # We would like to keep the name of the companies in question.
    if 'NAME' not in list_of_fields:
        list_of_fields = ['NAME'] + list_of_fields
    
    # We are often going to need to split our dates into more useful objects, that is what 'Proccess_Date' does: 
    def Proccess_Date(date):
        d = date.split("-")
        year, year_2, month = d[0], d[0][2:4], d[1]
        day = calendar.monthrange(int(year), int(month))[1] # gets last day for that month
        return(year, year_2, month, day)
    
    # Finding the number of months between ' start ' and ' end '.
    from_year, from_year_2, from_month, from_day = Proccess_Date(date = start)
    to_year, to_year_2, to_month, to_day = Proccess_Date(date = end)
    start_date = datetime.datetime(int(from_year), int(from_month), int(from_day))
    end_date = datetime.datetime(int(to_year), int(to_month), int(to_day))
    num_months = (end_date.year - start_date.year) * 12 + (end_date.month - start_date.month) + 1 # ' + 1 ' to include the last month.
    
    if add_to_global_DF == True:
        # If the user is afraid that the function could fail halfway through,
        #    (s)he can append an existing data-frame via this if statement.
        global DF
        global _DF
        global debug_df
    else:
        DF, _DF, debug_df = [], [], [] # These ate the lists of the data-frames of interest that will be populated in the following loop.
    
    for a in indices: # Iterating through the list of indices:
        df_list = [] # List to be appended/populated with the for loop below.
        # for d in range(num_months+2): # ' +1 ' since the ' range ' function is exclusive with respect to its upper limit, and an additional 1 for go over the month inclusive n the range of the argument.
        for d in range(num_months):
            _start_date = start_date + dateutil.relativedelta.relativedelta(months=+d)
            from_year, from_year_2, from_month, from_day = Proccess_Date(date = _start_date.strftime("%Y-%m-%d"))
 
            if track_progress == True:
                print(f"L{a}{from_month}{from_year_2}|L")
 
            df_list.append(
                ds.get_data(
                    tickers = f"L{a}{from_month}{from_year_2}|L",
                    fields = list_of_fields,
                    start = _start_date.strftime("%Y-%m-%d"),
                    kind = 0))
 
        df = pd.concat(df_list).reset_index(drop = True)
        
        date_range = df[df["Dates"].notna()]["Dates"].unique()
        
        debug_df.append(df)
        
        # This try loop is in case a field value requests non-numerical data that would break the code (since a median value could then not be computed).
        try:
            _df = pd.DataFrame(
                data = [[df[df["Value"] != "NA"][df["Datatype"] == i][df["Dates"] == j]["Value"].median()
                         for i in list_of_fields[1:]] for j in date_range], # ' list_of_fields[1:] ' as opposed to ' list_of_fields ' because we don't want 'NAME'.
                columns = pd.MultiIndex.from_product([[a], list_of_fields[1:]],
                                                     names = ['Index', 'Metrics']),
                index = date_range)
        except:
            for i in list_of_fields[1:]:
                for j in date_range:
                    try:
                        df[df["Value"] != "NA"][df["Datatype"] == i][df["Dates"] == j]["Value"].median()
                    except:
                        print(f"field '{i}' does not return numerical data; a median could thus not be computed for it.")
                        print("The program has stopped. Please run it again without this field.")
        
        DF.append(df)
        _DF.append(_df)
    return DF, _DF, debug_df

        
        Copy
    

    	
            
DF, _DF, debug_df = [], [], []
 
test1, test1_df, test1_deb_df = DSWS_Median_Index_Data(
    indices = ["MSWRLD", "S&PCOMP"],
    list_of_fields = ['X(EVT1FD12)~USD', 'X(EBT1FD12)~USD', 'X(EBD1FD12)~USD',
                      'X(CPS1FD12)~USD*X(IBNOSH)', 'X(MV)~USD', 'X(PEFD12)',
                      'X(SAL1FD12)~USD', 'X(BPS1FD12)~USD*X(IBNOSH)'],
    start = '2013-01-01',
    end = '2013-03-01',
    track_progress = True,
    add_to_global_DF = True)

        
        Copy
    

    	
            test1[0]
        
        Copy
    

    	
            test1_df[0]
        
        Copy
    

    	
            test1_df[1]
        
        Copy
    

    	
            test1_deb_df[0]
        
        Copy
    

    	
            DF[0]
        
        Copy
    

    	
            _DF[0]
        
        Copy
    

    	
            debug_df[0]
        
        Copy
    

    	
            help(DSWS_Median_Index_Data)
        
        Copy