Issue in converting Python code to Pyspark : Hello I have written python code which I need to convert to PySpark, But I am new to PySpark. I am trying to convert my Python code to PySpark but getting syntax error. Can someone assist me here for PySpark. Below is my Python code.

  

    import datetime
    import time
    import warnings
    warnings.filterwarnings('ignore')
    
    data = data_pit.copy(deep=True)
    
    # Begin preprocessing
    
    # Capture time metrics to support tuning
    start_time = time.time()
    
    # Enrich data with L2 Function and Source columns for deeper analysis
    '''
    pivoted = data.groupby([data.index, 'OrgUnit_Function_L2_NEWCO'], as_index=True)['Shortname'].count().unstack()
    data = pd.concat([data, pivoted], axis=1)
    
    pivoted = data.groupby([data.index, 'Source'], as_index=True)['Shortname'].count().unstack()
    data = pd.concat([data, pivoted], axis=1)
    '''
    
    data ['X_Work_Location_City'] = data['Work_Location_City']+data['work_location_state_desc'].fillna('')
    
    # Consolidate status for reference purposes
    data['Current Status'] = 'Involuntary Term.'
    data.loc[(data['Termination_Action']=='Voluntary'), 'Current Status'] = 'Voluntary Term.'
    data.loc[(data['Employee_Status_Alt'].isin(['Active','Inactive'])), 'Current Status'] = 'Active'
    data['target'] = data['Current Status']
    
    # Remove all with Unknown status
    #data = data[(data['target']=='Active') | (data['target']=='Involuntary Term.') | (data['target']=='Voluntary Term.')]
    columns_for_scaler = list(set(data.columns.values)-\
                              set(['Shortname','Current Status','PIT_Date','target',\
                                  'Termination_Date','employee_id']))
    
    # Normalize columns to categories and normal distribution
    from sklearn.preprocessing import MinMaxScaler
    import datetime as dt
    scaler = MinMaxScaler(feature_range=(0, 1))
    for column in columns_for_scaler:
        
    # 2018-01-10 begin
        print(column+' '+str(data[column].dtype))
        if str(data[column].dtype) == 'datetime64[ns]':
            data[column] = pd.to_datetime(data[column], errors='coerce').astype('int64')   
        # Convert all remaining columns to Categorical
        data[column] = pd.Categorical(data[column])
        data[column] = data[column].cat.codes
    # 2018-01-10 end
    
        # Scale All values
        data[column].fillna(value=0,inplace =True)
        col_values = data[column].values.reshape(data.shape[0], 1)
        data[column] = scaler.fit_transform(col_values)
    
    print('Data scaling successfully applied. Time: '+str(datetime.timedelta(seconds=(time.time() - start_time)))[:-7])
    data_scaled = data.copy(deep=True)
    pd.options.display.float_format = '{:.4f}'.format

Thanks in advance for your prompt assistance. 