#!/usr/bin/env python
# coding: utf-8

# In[1]:


# Import libraries
import numpy as np
import pandas as pd
import os
import geopy.distance as distance
import random
import pickle
import matplotlib.pyplot as plt
import warnings
warnings.filterwarnings('ignore')


# In[2]:


'''
TripStatsData: will help to find trips with desired time interval.
TripMainData : will help to provide positional coordinated for trips.
'''
# import trip stats
with (open('/scratch/skp454/Trajectory/TaxiDataSet/data/20190709_TaxiTripStatsData.pkl', "rb")) as openfile:
    df_tripStat = pickle.load(openfile, encoding = 'latin1')

# import trip main
with (open('/scratch/skp454/Trajectory/TaxiDataSet/data/20190709_TaxiTripMainData.pkl', "rb")) as openfile:
    df_tripMain = pickle.load(openfile, encoding = 'latin1')


# In[3]:


def findError(row, df_tripMain, toTime):
    try:
        df_subMain = df_tripMain[(df_tripMain['trip_id'] == row['trip_id'])*(df_tripMain['Stationary'] == False)]
        df_subMain['DistC'] = df_subMain['Dist'].cumsum()
        t = row['Dist'] - toTime
        index1 = list((df_subMain['DistC']<t).index)
        random.shuffle(index1)
        df_subMain['Time'] = (df_subMain.index>index1[0]) * df_subMain['TimeDiff']
        df_subMain['Time'] = df_subMain['Time'].cumsum()
        df_subMain['DistC'] = (df_subMain.index>index1[0]) * df_subMain['Dist']
        df_subMain['DistC'] = df_subMain['DistC'].cumsum()
        df_subMain = df_subMain.loc[index1[0]:(df_subMain['DistC']<toTime).index[-1]+1]
        df_subMain['predLAT'] = ((df_subMain.iloc[-1]['LAT'] - df_subMain.iloc[0]['LAT']) *                                   (df_subMain['Time']/df_subMain.iloc[-1]['Time'])) + df_subMain.iloc[0]['LAT']
        df_subMain['predLON'] = ((df_subMain.iloc[-1]['LON'] - df_subMain.iloc[0]['LON']) *                                   (df_subMain['Time']/df_subMain.iloc[-1]['Time'])) + df_subMain.iloc[0]['LON']
        df_subMain['Error'] = df_subMain.apply(lambda row: distance.geodesic((row['predLAT'],row['predLON'])
                                                                          ,(row['LAT'],row['LON'])).km, axis=1)
        return np.sum(df_subMain['Error'])/(len(df_subMain)-2)
    except:
        np.nan


# In[4]:


def errorSummary(fromTime, timeFrame, df_tripStat, df_tripMain):
    try:
        df_subStat = df_tripStat[(df_tripStat['Dist']>fromTime)]
        index = list(df_subStat.index)
        random.shuffle(index)
        df_subStat = df_subStat.loc[index[0:100],:]
        toTime = fromTime + timeFrame
        df_subStat['Error'] = df_subStat.apply(lambda row: findError(row, df_tripMain, toTime), axis =1)
        return df_subStat['Error'].describe()
    except:
        return "error"


# In[5]:


df_maskStat = {}
for dist in np.arange(0,10.1,0.5):
    df_maskStat[dist] = errorSummary(dist, 0.5, df_tripStat, df_tripMain)

# save the statistics for linear interpolation
outfile = open('/scratch/skp454/Trajectory/TaxiDataSet/data/20190709_TaxiLinearInterpolationDistShort.pkl','wb')
pickle.dump(df_maskStat,outfile)
outfile.close()

df_maskStat = {}
for dist in np.arange(0,61,3):
    df_maskStat[dist] = errorSummary(dist, 3, df_tripStat, df_tripMain)

# save the statistics for linear interpolation
outfile = open('/scratch/skp454/Trajectory/TaxiDataSet/data/20190709_TaxiLinearInterpolationDistLong.pkl','wb')
pickle.dump(df_maskStat,outfile)
outfile.close()


c1216efe702234f08c78b6a5a58797860f73ff1f
<mask token>


def errorSummary(fromTime, timeFrame, df_tripStat, df_tripMain):
    try:
        df_subStat = df_tripStat[df_tripStat['Dist'] > fromTime]
        index = list(df_subStat.index)
        random.shuffle(index)
        df_subStat = df_subStat.loc[index[0:100], :]
        toTime = fromTime + timeFrame
        df_subStat['Error'] = df_subStat.apply(lambda row: findError(row,
            df_tripMain, toTime), axis=1)
        return df_subStat['Error'].describe()
    except:
        return 'error'


<mask token>

<mask token>
warnings.filterwarnings('ignore')
<mask token>
with open(
    '/scratch/skp454/Trajectory/TaxiDataSet/data/20190709_TaxiTripStatsData.pkl'
    , 'rb') as openfile:
    df_tripStat = pickle.load(openfile, encoding='latin1')
with open(
    '/scratch/skp454/Trajectory/TaxiDataSet/data/20190709_TaxiTripMainData.pkl'
    , 'rb') as openfile:
    df_tripMain = pickle.load(openfile, encoding='latin1')


def findError(row, df_tripMain, toTime):
    try:
        df_subMain = df_tripMain[(df_tripMain['trip_id'] == row['trip_id']) *
            (df_tripMain['Stationary'] == False)]
        df_subMain['DistC'] = df_subMain['Dist'].cumsum()
        t = row['Dist'] - toTime
        index1 = list((df_subMain['DistC'] < t).index)
        random.shuffle(index1)
        df_subMain['Time'] = (df_subMain.index > index1[0]) * df_subMain[
            'TimeDiff']
        df_subMain['Time'] = df_subMain['Time'].cumsum()
        df_subMain['DistC'] = (df_subMain.index > index1[0]) * df_subMain[
            'Dist']
        df_subMain['DistC'] = df_subMain['DistC'].cumsum()
        df_subMain = df_subMain.loc[index1[0]:(df_subMain['DistC'] < toTime
            ).index[-1] + 1]
        df_subMain['predLAT'] = (df_subMain.iloc[-1]['LAT'] - df_subMain.
            iloc[0]['LAT']) * (df_subMain['Time'] / df_subMain.iloc[-1]['Time']
            ) + df_subMain.iloc[0]['LAT']
        df_subMain['predLON'] = (df_subMain.iloc[-1]['LON'] - df_subMain.
            iloc[0]['LON']) * (df_subMain['Time'] / df_subMain.iloc[-1]['Time']
            ) + df_subMain.iloc[0]['LON']
        df_subMain['Error'] = df_subMain.apply(lambda row: distance.
            geodesic((row['predLAT'], row['predLON']), (row['LAT'], row[
            'LON'])).km, axis=1)
        return np.sum(df_subMain['Error']) / (len(df_subMain) - 2)
    except:
        np.nan


def errorSummary(fromTime, timeFrame, df_tripStat, df_tripMain):
    try:
        df_subStat = df_tripStat[df_tripStat['Dist'] > fromTime]
        index = list(df_subStat.index)
        random.shuffle(index)
        df_subStat = df_subStat.loc[index[0:100], :]
        toTime = fromTime + timeFrame
        df_subStat['Error'] = df_subStat.apply(lambda row: findError(row,
            df_tripMain, toTime), axis=1)
        return df_subStat['Error'].describe()
    except:
        return 'error'


<mask token>
for dist in np.arange(0, 10.1, 0.5):
    df_maskStat[dist] = errorSummary(dist, 0.5, df_tripStat, df_tripMain)
<mask token>
pickle.dump(df_maskStat, outfile)
outfile.close()
<mask token>
for dist in np.arange(0, 61, 3):
    df_maskStat[dist] = errorSummary(dist, 3, df_tripStat, df_tripMain)
<mask token>
pickle.dump(df_maskStat, outfile)
outfile.close()

<mask token>
warnings.filterwarnings('ignore')
<mask token>
with open(
    '/scratch/skp454/Trajectory/TaxiDataSet/data/20190709_TaxiTripStatsData.pkl'
    , 'rb') as openfile:
    df_tripStat = pickle.load(openfile, encoding='latin1')
with open(
    '/scratch/skp454/Trajectory/TaxiDataSet/data/20190709_TaxiTripMainData.pkl'
    , 'rb') as openfile:
    df_tripMain = pickle.load(openfile, encoding='latin1')


def findError(row, df_tripMain, toTime):
    try:
        df_subMain = df_tripMain[(df_tripMain['trip_id'] == row['trip_id']) *
            (df_tripMain['Stationary'] == False)]
        df_subMain['DistC'] = df_subMain['Dist'].cumsum()
        t = row['Dist'] - toTime
        index1 = list((df_subMain['DistC'] < t).index)
        random.shuffle(index1)
        df_subMain['Time'] = (df_subMain.index > index1[0]) * df_subMain[
            'TimeDiff']
        df_subMain['Time'] = df_subMain['Time'].cumsum()
        df_subMain['DistC'] = (df_subMain.index > index1[0]) * df_subMain[
            'Dist']
        df_subMain['DistC'] = df_subMain['DistC'].cumsum()
        df_subMain = df_subMain.loc[index1[0]:(df_subMain['DistC'] < toTime
            ).index[-1] + 1]
        df_subMain['predLAT'] = (df_subMain.iloc[-1]['LAT'] - df_subMain.
            iloc[0]['LAT']) * (df_subMain['Time'] / df_subMain.iloc[-1]['Time']
            ) + df_subMain.iloc[0]['LAT']
        df_subMain['predLON'] = (df_subMain.iloc[-1]['LON'] - df_subMain.
            iloc[0]['LON']) * (df_subMain['Time'] / df_subMain.iloc[-1]['Time']
            ) + df_subMain.iloc[0]['LON']
        df_subMain['Error'] = df_subMain.apply(lambda row: distance.
            geodesic((row['predLAT'], row['predLON']), (row['LAT'], row[
            'LON'])).km, axis=1)
        return np.sum(df_subMain['Error']) / (len(df_subMain) - 2)
    except:
        np.nan


def errorSummary(fromTime, timeFrame, df_tripStat, df_tripMain):
    try:
        df_subStat = df_tripStat[df_tripStat['Dist'] > fromTime]
        index = list(df_subStat.index)
        random.shuffle(index)
        df_subStat = df_subStat.loc[index[0:100], :]
        toTime = fromTime + timeFrame
        df_subStat['Error'] = df_subStat.apply(lambda row: findError(row,
            df_tripMain, toTime), axis=1)
        return df_subStat['Error'].describe()
    except:
        return 'error'


df_maskStat = {}
for dist in np.arange(0, 10.1, 0.5):
    df_maskStat[dist] = errorSummary(dist, 0.5, df_tripStat, df_tripMain)
outfile = open(
    '/scratch/skp454/Trajectory/TaxiDataSet/data/20190709_TaxiLinearInterpolationDistShort.pkl'
    , 'wb')
pickle.dump(df_maskStat, outfile)
outfile.close()
df_maskStat = {}
for dist in np.arange(0, 61, 3):
    df_maskStat[dist] = errorSummary(dist, 3, df_tripStat, df_tripMain)
outfile = open(
    '/scratch/skp454/Trajectory/TaxiDataSet/data/20190709_TaxiLinearInterpolationDistLong.pkl'
    , 'wb')
pickle.dump(df_maskStat, outfile)
outfile.close()

import numpy as np
import pandas as pd
import os
import geopy.distance as distance
import random
import pickle
import matplotlib.pyplot as plt
import warnings
warnings.filterwarnings('ignore')
<mask token>
with open(
    '/scratch/skp454/Trajectory/TaxiDataSet/data/20190709_TaxiTripStatsData.pkl'
    , 'rb') as openfile:
    df_tripStat = pickle.load(openfile, encoding='latin1')
with open(
    '/scratch/skp454/Trajectory/TaxiDataSet/data/20190709_TaxiTripMainData.pkl'
    , 'rb') as openfile:
    df_tripMain = pickle.load(openfile, encoding='latin1')


def findError(row, df_tripMain, toTime):
    try:
        df_subMain = df_tripMain[(df_tripMain['trip_id'] == row['trip_id']) *
            (df_tripMain['Stationary'] == False)]
        df_subMain['DistC'] = df_subMain['Dist'].cumsum()
        t = row['Dist'] - toTime
        index1 = list((df_subMain['DistC'] < t).index)
        random.shuffle(index1)
        df_subMain['Time'] = (df_subMain.index > index1[0]) * df_subMain[
            'TimeDiff']
        df_subMain['Time'] = df_subMain['Time'].cumsum()
        df_subMain['DistC'] = (df_subMain.index > index1[0]) * df_subMain[
            'Dist']
        df_subMain['DistC'] = df_subMain['DistC'].cumsum()
        df_subMain = df_subMain.loc[index1[0]:(df_subMain['DistC'] < toTime
            ).index[-1] + 1]
        df_subMain['predLAT'] = (df_subMain.iloc[-1]['LAT'] - df_subMain.
            iloc[0]['LAT']) * (df_subMain['Time'] / df_subMain.iloc[-1]['Time']
            ) + df_subMain.iloc[0]['LAT']
        df_subMain['predLON'] = (df_subMain.iloc[-1]['LON'] - df_subMain.
            iloc[0]['LON']) * (df_subMain['Time'] / df_subMain.iloc[-1]['Time']
            ) + df_subMain.iloc[0]['LON']
        df_subMain['Error'] = df_subMain.apply(lambda row: distance.
            geodesic((row['predLAT'], row['predLON']), (row['LAT'], row[
            'LON'])).km, axis=1)
        return np.sum(df_subMain['Error']) / (len(df_subMain) - 2)
    except:
        np.nan


def errorSummary(fromTime, timeFrame, df_tripStat, df_tripMain):
    try:
        df_subStat = df_tripStat[df_tripStat['Dist'] > fromTime]
        index = list(df_subStat.index)
        random.shuffle(index)
        df_subStat = df_subStat.loc[index[0:100], :]
        toTime = fromTime + timeFrame
        df_subStat['Error'] = df_subStat.apply(lambda row: findError(row,
            df_tripMain, toTime), axis=1)
        return df_subStat['Error'].describe()
    except:
        return 'error'


df_maskStat = {}
for dist in np.arange(0, 10.1, 0.5):
    df_maskStat[dist] = errorSummary(dist, 0.5, df_tripStat, df_tripMain)
outfile = open(
    '/scratch/skp454/Trajectory/TaxiDataSet/data/20190709_TaxiLinearInterpolationDistShort.pkl'
    , 'wb')
pickle.dump(df_maskStat, outfile)
outfile.close()
df_maskStat = {}
for dist in np.arange(0, 61, 3):
    df_maskStat[dist] = errorSummary(dist, 3, df_tripStat, df_tripMain)
outfile = open(
    '/scratch/skp454/Trajectory/TaxiDataSet/data/20190709_TaxiLinearInterpolationDistLong.pkl'
    , 'wb')
pickle.dump(df_maskStat, outfile)
outfile.close()

#!/usr/bin/env python
# coding: utf-8

# In[1]:


# Import libraries
import numpy as np
import pandas as pd
import os
import geopy.distance as distance
import random
import pickle
import matplotlib.pyplot as plt
import warnings
warnings.filterwarnings('ignore')


# In[2]:


'''
TripStatsData: will help to find trips with desired time interval.
TripMainData : will help to provide positional coordinated for trips.
'''
# import trip stats
with (open('/scratch/skp454/Trajectory/TaxiDataSet/data/20190709_TaxiTripStatsData.pkl', "rb")) as openfile:
    df_tripStat = pickle.load(openfile, encoding = 'latin1')

# import trip main
with (open('/scratch/skp454/Trajectory/TaxiDataSet/data/20190709_TaxiTripMainData.pkl', "rb")) as openfile:
    df_tripMain = pickle.load(openfile, encoding = 'latin1')


# In[3]:


def findError(row, df_tripMain, toTime):
    try:
        df_subMain = df_tripMain[(df_tripMain['trip_id'] == row['trip_id'])*(df_tripMain['Stationary'] == False)]
        df_subMain['DistC'] = df_subMain['Dist'].cumsum()
        t = row['Dist'] - toTime
        index1 = list((df_subMain['DistC']<t).index)
        random.shuffle(index1)
        df_subMain['Time'] = (df_subMain.index>index1[0]) * df_subMain['TimeDiff']
        df_subMain['Time'] = df_subMain['Time'].cumsum()
        df_subMain['DistC'] = (df_subMain.index>index1[0]) * df_subMain['Dist']
        df_subMain['DistC'] = df_subMain['DistC'].cumsum()
        df_subMain = df_subMain.loc[index1[0]:(df_subMain['DistC']<toTime).index[-1]+1]
        df_subMain['predLAT'] = ((df_subMain.iloc[-1]['LAT'] - df_subMain.iloc[0]['LAT']) *                                   (df_subMain['Time']/df_subMain.iloc[-1]['Time'])) + df_subMain.iloc[0]['LAT']
        df_subMain['predLON'] = ((df_subMain.iloc[-1]['LON'] - df_subMain.iloc[0]['LON']) *                                   (df_subMain['Time']/df_subMain.iloc[-1]['Time'])) + df_subMain.iloc[0]['LON']
        df_subMain['Error'] = df_subMain.apply(lambda row: distance.geodesic((row['predLAT'],row['predLON'])
                                                                          ,(row['LAT'],row['LON'])).km, axis=1)
        return np.sum(df_subMain['Error'])/(len(df_subMain)-2)
    except:
        np.nan


# In[4]:


def errorSummary(fromTime, timeFrame, df_tripStat, df_tripMain):
    try:
        df_subStat = df_tripStat[(df_tripStat['Dist']>fromTime)]
        index = list(df_subStat.index)
        random.shuffle(index)
        df_subStat = df_subStat.loc[index[0:100],:]
        toTime = fromTime + timeFrame
        df_subStat['Error'] = df_subStat.apply(lambda row: findError(row, df_tripMain, toTime), axis =1)
        return df_subStat['Error'].describe()
    except:
        return "error"


# In[5]:


df_maskStat = {}
for dist in np.arange(0,10.1,0.5):
    df_maskStat[dist] = errorSummary(dist, 0.5, df_tripStat, df_tripMain)

# save the statistics for linear interpolation
outfile = open('/scratch/skp454/Trajectory/TaxiDataSet/data/20190709_TaxiLinearInterpolationDistShort.pkl','wb')
pickle.dump(df_maskStat,outfile)
outfile.close()

df_maskStat = {}
for dist in np.arange(0,61,3):
    df_maskStat[dist] = errorSummary(dist, 3, df_tripStat, df_tripMain)

# save the statistics for linear interpolation
outfile = open('/scratch/skp454/Trajectory/TaxiDataSet/data/20190709_TaxiLinearInterpolationDistLong.pkl','wb')
pickle.dump(df_maskStat,outfile)
outfile.close()

