#!/usr/bin/python
# -*- coding: utf-8 -*-
###############################################################################################################
#
#
#
#
##############################################################################################################
import os
import sys
import csv
import shutil
import string
import time
from xml.etree.cElementTree import iterparse
import datetime
#from datetime import timedelta
import re
#import logging
#doc = iterparse('/data6/splice/July_Files/DataWarehouse37/DataWarehouse37_FO_USA_M_201608034.xml', ('start', 'end'))
doc = iterparse('/home/kumarsu/sun-000000/DataWarehouse37_VA_USA_D_20171130.xml', ('start', 'end'))
path='CUSIP/CUSIP'
path_parts = path.split('/')
#next(doc)
#tag_stack = []
#elem_stack = []
st=datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d_%H_%M_%S')
#print(st)
path=[]
rec=[]
c_rec=[]
SubAccount_Ordered_key=['SubAccountName','SubAccountLegalName','UnderlyingFundId','PolicyId','PolicyName','PolicyShortName','PolicyType','PolicyType','PolicyInceptionDate','InsuranceCompanyId','InsuranceCompanyName','SubAccountInceptionDate','SubAccountObsoleteDate','SubAccountClosedToNewInvestors','SubAccountClosedToNewInvestorsDate','SubAccountAvailableInAnnuityStage','SubAccountAvailableInAccumStage','SubAccountMERisk','SubAccountInsuranceExpense','SubAccountFixedAccountName']
SubAccount={'SubAccountName':'','SubAccountLegalName':'','UnderlyingFundId':'','PolicyId':'','PolicyName':'','PolicyShortName':'','PolicyType':'','PolicyType':'','PolicyInceptionDate':'','InsuranceCompanyId':'','InsuranceCompanyName':'','SubAccountInceptionDate':'','SubAccountObsoleteDate':'','SubAccountClosedToNewInvestors':'','SubAccountClosedToNewInvestorsDate':'','SubAccountAvailableInAnnuityStage':'','SubAccountAvailableInAccumStage':'','SubAccountMERisk':'','SubAccountInsuranceExpense':'','SubAccountFixedAccountName':''}
for event, elem in doc:
#     nm=elem.find('PackageName')
#     nm=elem.find('InvestmentVehicle')
#     if nm is not None:
#         print(nm.get('_Id'))
#        print(nm.text)
    if event == 'start':
#        nm=elem.find('InvestmentVehicle')
         path.append(elem.tag)
         if elem.tag == 'InvestmentVehicle':
#            print(elem.attrib['_Id'])
            rec.append(elem.attrib['_Id'])
         if elem.tag == 'PolicyType':
            if 'SubAccount' in path:
               if elem.attrib['_Id'] is None:
                  attval=''
               else:
                  attval=elem.attrib['_Id']
               SubAccount[elem.tag]=attval
         if elem.tag =='SubAccountName' or elem.tag =='SubAccountLegalName' or elem.tag =='UnderlyingFundId' or elem.tag =='PolicyId' or elem.tag =='PolicyName' or elem.tag =='PolicyShortName' or elem.tag =='PolicyType' or elem.tag =='PolicyInceptionDate' or elem.tag =='InsuranceCompanyId' or elem.tag =='InsuranceCompanyName' or elem.tag =='SubAccountInceptionDate' or elem.tag =='SubAccountObsoleteDate' or elem.tag =='SubAccountClosedToNewInvestors' or elem.tag =='SubAccountClosedToNewInvestorsDate' or elem.tag =='SubAccountAvailableInAnnuityStage' or elem.tag =='SubAccountAvailableInAccumStage' or elem.tag =='SubAccountMERisk' or elem.tag =='SubAccountInsuranceExpense' or elem.tag =='SubAccountFixedAccountName':
#         or elem.tag == 'MarketValue':
            if 'SubAccount' in path:
               if elem.text is None:
                  str_val=''
               else:
                  str_val=elem.text
#            str_val_lst.append(str_val)
               SubAccount[elem.tag]=str_val
#            str_val_rec='|'.join(str_val_lst)
#            c_rec.append(str_val_rec)
    if event == 'end':
         if elem.tag == 'SubAccount':
            str_val_rec=u'|'.join(SubAccount[x] for x in SubAccount_Ordered_key).encode('utf-8').strip()
            c_rec.append(str_val_rec)
            SubAccount=dict.fromkeys(SubAccount,'')
         if elem.tag == 'InvestmentVehicle':
            rec_dump='|'.join(rec)
            for i in range(0,len(c_rec)):
                print(rec_dump + '|' + str(c_rec[i]))
            del rec[:]
            del c_rec[:]
         path.pop()               
    elem.clear()
#       tag_stack.append(elem.tag)
#       elem_stack.append(elem)
#    elif event == 'end':
#         if tag_stack == path_parts:
#            print(elem)
#            elem_stack[-2].remove(elem)
#         try:
#            tag_stack.pop()
#            elem_stack.pop()
#         except IndexError:
#            pass
ed=datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d_%H_%M_%S')
print(st)
print(ed)
sys.exit(0)

cbc1af0c5cc5f65eb7e8e0032f4dd7e09ec39c3b
<mask token>

<mask token>
for event, elem in doc:
    if event == 'start':
        path.append(elem.tag)
        if elem.tag == 'InvestmentVehicle':
            rec.append(elem.attrib['_Id'])
        if elem.tag == 'PolicyType':
            if 'SubAccount' in path:
                if elem.attrib['_Id'] is None:
                    attval = ''
                else:
                    attval = elem.attrib['_Id']
                SubAccount[elem.tag] = attval
        if (elem.tag == 'SubAccountName' or elem.tag ==
            'SubAccountLegalName' or elem.tag == 'UnderlyingFundId' or elem
            .tag == 'PolicyId' or elem.tag == 'PolicyName' or elem.tag ==
            'PolicyShortName' or elem.tag == 'PolicyType' or elem.tag ==
            'PolicyInceptionDate' or elem.tag == 'InsuranceCompanyId' or 
            elem.tag == 'InsuranceCompanyName' or elem.tag ==
            'SubAccountInceptionDate' or elem.tag ==
            'SubAccountObsoleteDate' or elem.tag ==
            'SubAccountClosedToNewInvestors' or elem.tag ==
            'SubAccountClosedToNewInvestorsDate' or elem.tag ==
            'SubAccountAvailableInAnnuityStage' or elem.tag ==
            'SubAccountAvailableInAccumStage' or elem.tag ==
            'SubAccountMERisk' or elem.tag == 'SubAccountInsuranceExpense' or
            elem.tag == 'SubAccountFixedAccountName'):
            if 'SubAccount' in path:
                if elem.text is None:
                    str_val = ''
                else:
                    str_val = elem.text
                SubAccount[elem.tag] = str_val
    if event == 'end':
        if elem.tag == 'SubAccount':
            str_val_rec = u'|'.join(SubAccount[x] for x in
                SubAccount_Ordered_key).encode('utf-8').strip()
            c_rec.append(str_val_rec)
            SubAccount = dict.fromkeys(SubAccount, '')
        if elem.tag == 'InvestmentVehicle':
            rec_dump = '|'.join(rec)
            for i in range(0, len(c_rec)):
                print(rec_dump + '|' + str(c_rec[i]))
            del rec[:]
            del c_rec[:]
        path.pop()
    elem.clear()
<mask token>
print(st)
print(ed)
sys.exit(0)

<mask token>
doc = iterparse(
    '/home/kumarsu/sun-000000/DataWarehouse37_VA_USA_D_20171130.xml', (
    'start', 'end'))
path = 'CUSIP/CUSIP'
path_parts = path.split('/')
st = datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d_%H_%M_%S')
path = []
rec = []
c_rec = []
SubAccount_Ordered_key = ['SubAccountName', 'SubAccountLegalName',
    'UnderlyingFundId', 'PolicyId', 'PolicyName', 'PolicyShortName',
    'PolicyType', 'PolicyType', 'PolicyInceptionDate', 'InsuranceCompanyId',
    'InsuranceCompanyName', 'SubAccountInceptionDate',
    'SubAccountObsoleteDate', 'SubAccountClosedToNewInvestors',
    'SubAccountClosedToNewInvestorsDate',
    'SubAccountAvailableInAnnuityStage', 'SubAccountAvailableInAccumStage',
    'SubAccountMERisk', 'SubAccountInsuranceExpense',
    'SubAccountFixedAccountName']
SubAccount = {'SubAccountName': '', 'SubAccountLegalName': '',
    'UnderlyingFundId': '', 'PolicyId': '', 'PolicyName': '',
    'PolicyShortName': '', 'PolicyType': '', 'PolicyType': '',
    'PolicyInceptionDate': '', 'InsuranceCompanyId': '',
    'InsuranceCompanyName': '', 'SubAccountInceptionDate': '',
    'SubAccountObsoleteDate': '', 'SubAccountClosedToNewInvestors': '',
    'SubAccountClosedToNewInvestorsDate': '',
    'SubAccountAvailableInAnnuityStage': '',
    'SubAccountAvailableInAccumStage': '', 'SubAccountMERisk': '',
    'SubAccountInsuranceExpense': '', 'SubAccountFixedAccountName': ''}
for event, elem in doc:
    if event == 'start':
        path.append(elem.tag)
        if elem.tag == 'InvestmentVehicle':
            rec.append(elem.attrib['_Id'])
        if elem.tag == 'PolicyType':
            if 'SubAccount' in path:
                if elem.attrib['_Id'] is None:
                    attval = ''
                else:
                    attval = elem.attrib['_Id']
                SubAccount[elem.tag] = attval
        if (elem.tag == 'SubAccountName' or elem.tag ==
            'SubAccountLegalName' or elem.tag == 'UnderlyingFundId' or elem
            .tag == 'PolicyId' or elem.tag == 'PolicyName' or elem.tag ==
            'PolicyShortName' or elem.tag == 'PolicyType' or elem.tag ==
            'PolicyInceptionDate' or elem.tag == 'InsuranceCompanyId' or 
            elem.tag == 'InsuranceCompanyName' or elem.tag ==
            'SubAccountInceptionDate' or elem.tag ==
            'SubAccountObsoleteDate' or elem.tag ==
            'SubAccountClosedToNewInvestors' or elem.tag ==
            'SubAccountClosedToNewInvestorsDate' or elem.tag ==
            'SubAccountAvailableInAnnuityStage' or elem.tag ==
            'SubAccountAvailableInAccumStage' or elem.tag ==
            'SubAccountMERisk' or elem.tag == 'SubAccountInsuranceExpense' or
            elem.tag == 'SubAccountFixedAccountName'):
            if 'SubAccount' in path:
                if elem.text is None:
                    str_val = ''
                else:
                    str_val = elem.text
                SubAccount[elem.tag] = str_val
    if event == 'end':
        if elem.tag == 'SubAccount':
            str_val_rec = u'|'.join(SubAccount[x] for x in
                SubAccount_Ordered_key).encode('utf-8').strip()
            c_rec.append(str_val_rec)
            SubAccount = dict.fromkeys(SubAccount, '')
        if elem.tag == 'InvestmentVehicle':
            rec_dump = '|'.join(rec)
            for i in range(0, len(c_rec)):
                print(rec_dump + '|' + str(c_rec[i]))
            del rec[:]
            del c_rec[:]
        path.pop()
    elem.clear()
ed = datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d_%H_%M_%S')
print(st)
print(ed)
sys.exit(0)

import os
import sys
import csv
import shutil
import string
import time
from xml.etree.cElementTree import iterparse
import datetime
import re
doc = iterparse(
    '/home/kumarsu/sun-000000/DataWarehouse37_VA_USA_D_20171130.xml', (
    'start', 'end'))
path = 'CUSIP/CUSIP'
path_parts = path.split('/')
st = datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d_%H_%M_%S')
path = []
rec = []
c_rec = []
SubAccount_Ordered_key = ['SubAccountName', 'SubAccountLegalName',
    'UnderlyingFundId', 'PolicyId', 'PolicyName', 'PolicyShortName',
    'PolicyType', 'PolicyType', 'PolicyInceptionDate', 'InsuranceCompanyId',
    'InsuranceCompanyName', 'SubAccountInceptionDate',
    'SubAccountObsoleteDate', 'SubAccountClosedToNewInvestors',
    'SubAccountClosedToNewInvestorsDate',
    'SubAccountAvailableInAnnuityStage', 'SubAccountAvailableInAccumStage',
    'SubAccountMERisk', 'SubAccountInsuranceExpense',
    'SubAccountFixedAccountName']
SubAccount = {'SubAccountName': '', 'SubAccountLegalName': '',
    'UnderlyingFundId': '', 'PolicyId': '', 'PolicyName': '',
    'PolicyShortName': '', 'PolicyType': '', 'PolicyType': '',
    'PolicyInceptionDate': '', 'InsuranceCompanyId': '',
    'InsuranceCompanyName': '', 'SubAccountInceptionDate': '',
    'SubAccountObsoleteDate': '', 'SubAccountClosedToNewInvestors': '',
    'SubAccountClosedToNewInvestorsDate': '',
    'SubAccountAvailableInAnnuityStage': '',
    'SubAccountAvailableInAccumStage': '', 'SubAccountMERisk': '',
    'SubAccountInsuranceExpense': '', 'SubAccountFixedAccountName': ''}
for event, elem in doc:
    if event == 'start':
        path.append(elem.tag)
        if elem.tag == 'InvestmentVehicle':
            rec.append(elem.attrib['_Id'])
        if elem.tag == 'PolicyType':
            if 'SubAccount' in path:
                if elem.attrib['_Id'] is None:
                    attval = ''
                else:
                    attval = elem.attrib['_Id']
                SubAccount[elem.tag] = attval
        if (elem.tag == 'SubAccountName' or elem.tag ==
            'SubAccountLegalName' or elem.tag == 'UnderlyingFundId' or elem
            .tag == 'PolicyId' or elem.tag == 'PolicyName' or elem.tag ==
            'PolicyShortName' or elem.tag == 'PolicyType' or elem.tag ==
            'PolicyInceptionDate' or elem.tag == 'InsuranceCompanyId' or 
            elem.tag == 'InsuranceCompanyName' or elem.tag ==
            'SubAccountInceptionDate' or elem.tag ==
            'SubAccountObsoleteDate' or elem.tag ==
            'SubAccountClosedToNewInvestors' or elem.tag ==
            'SubAccountClosedToNewInvestorsDate' or elem.tag ==
            'SubAccountAvailableInAnnuityStage' or elem.tag ==
            'SubAccountAvailableInAccumStage' or elem.tag ==
            'SubAccountMERisk' or elem.tag == 'SubAccountInsuranceExpense' or
            elem.tag == 'SubAccountFixedAccountName'):
            if 'SubAccount' in path:
                if elem.text is None:
                    str_val = ''
                else:
                    str_val = elem.text
                SubAccount[elem.tag] = str_val
    if event == 'end':
        if elem.tag == 'SubAccount':
            str_val_rec = u'|'.join(SubAccount[x] for x in
                SubAccount_Ordered_key).encode('utf-8').strip()
            c_rec.append(str_val_rec)
            SubAccount = dict.fromkeys(SubAccount, '')
        if elem.tag == 'InvestmentVehicle':
            rec_dump = '|'.join(rec)
            for i in range(0, len(c_rec)):
                print(rec_dump + '|' + str(c_rec[i]))
            del rec[:]
            del c_rec[:]
        path.pop()
    elem.clear()
ed = datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d_%H_%M_%S')
print(st)
print(ed)
sys.exit(0)

#!/usr/bin/python
# -*- coding: utf-8 -*-
###############################################################################################################
#
#
#
#
##############################################################################################################
import os
import sys
import csv
import shutil
import string
import time
from xml.etree.cElementTree import iterparse
import datetime
#from datetime import timedelta
import re
#import logging
#doc = iterparse('/data6/splice/July_Files/DataWarehouse37/DataWarehouse37_FO_USA_M_201608034.xml', ('start', 'end'))
doc = iterparse('/home/kumarsu/sun-000000/DataWarehouse37_VA_USA_D_20171130.xml', ('start', 'end'))
path='CUSIP/CUSIP'
path_parts = path.split('/')
#next(doc)
#tag_stack = []
#elem_stack = []
st=datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d_%H_%M_%S')
#print(st)
path=[]
rec=[]
c_rec=[]
SubAccount_Ordered_key=['SubAccountName','SubAccountLegalName','UnderlyingFundId','PolicyId','PolicyName','PolicyShortName','PolicyType','PolicyType','PolicyInceptionDate','InsuranceCompanyId','InsuranceCompanyName','SubAccountInceptionDate','SubAccountObsoleteDate','SubAccountClosedToNewInvestors','SubAccountClosedToNewInvestorsDate','SubAccountAvailableInAnnuityStage','SubAccountAvailableInAccumStage','SubAccountMERisk','SubAccountInsuranceExpense','SubAccountFixedAccountName']
SubAccount={'SubAccountName':'','SubAccountLegalName':'','UnderlyingFundId':'','PolicyId':'','PolicyName':'','PolicyShortName':'','PolicyType':'','PolicyType':'','PolicyInceptionDate':'','InsuranceCompanyId':'','InsuranceCompanyName':'','SubAccountInceptionDate':'','SubAccountObsoleteDate':'','SubAccountClosedToNewInvestors':'','SubAccountClosedToNewInvestorsDate':'','SubAccountAvailableInAnnuityStage':'','SubAccountAvailableInAccumStage':'','SubAccountMERisk':'','SubAccountInsuranceExpense':'','SubAccountFixedAccountName':''}
for event, elem in doc:
#     nm=elem.find('PackageName')
#     nm=elem.find('InvestmentVehicle')
#     if nm is not None:
#         print(nm.get('_Id'))
#        print(nm.text)
    if event == 'start':
#        nm=elem.find('InvestmentVehicle')
         path.append(elem.tag)
         if elem.tag == 'InvestmentVehicle':
#            print(elem.attrib['_Id'])
            rec.append(elem.attrib['_Id'])
         if elem.tag == 'PolicyType':
            if 'SubAccount' in path:
               if elem.attrib['_Id'] is None:
                  attval=''
               else:
                  attval=elem.attrib['_Id']
               SubAccount[elem.tag]=attval
         if elem.tag =='SubAccountName' or elem.tag =='SubAccountLegalName' or elem.tag =='UnderlyingFundId' or elem.tag =='PolicyId' or elem.tag =='PolicyName' or elem.tag =='PolicyShortName' or elem.tag =='PolicyType' or elem.tag =='PolicyInceptionDate' or elem.tag =='InsuranceCompanyId' or elem.tag =='InsuranceCompanyName' or elem.tag =='SubAccountInceptionDate' or elem.tag =='SubAccountObsoleteDate' or elem.tag =='SubAccountClosedToNewInvestors' or elem.tag =='SubAccountClosedToNewInvestorsDate' or elem.tag =='SubAccountAvailableInAnnuityStage' or elem.tag =='SubAccountAvailableInAccumStage' or elem.tag =='SubAccountMERisk' or elem.tag =='SubAccountInsuranceExpense' or elem.tag =='SubAccountFixedAccountName':
#         or elem.tag == 'MarketValue':
            if 'SubAccount' in path:
               if elem.text is None:
                  str_val=''
               else:
                  str_val=elem.text
#            str_val_lst.append(str_val)
               SubAccount[elem.tag]=str_val
#            str_val_rec='|'.join(str_val_lst)
#            c_rec.append(str_val_rec)
    if event == 'end':
         if elem.tag == 'SubAccount':
            str_val_rec=u'|'.join(SubAccount[x] for x in SubAccount_Ordered_key).encode('utf-8').strip()
            c_rec.append(str_val_rec)
            SubAccount=dict.fromkeys(SubAccount,'')
         if elem.tag == 'InvestmentVehicle':
            rec_dump='|'.join(rec)
            for i in range(0,len(c_rec)):
                print(rec_dump + '|' + str(c_rec[i]))
            del rec[:]
            del c_rec[:]
         path.pop()               
    elem.clear()
#       tag_stack.append(elem.tag)
#       elem_stack.append(elem)
#    elif event == 'end':
#         if tag_stack == path_parts:
#            print(elem)
#            elem_stack[-2].remove(elem)
#         try:
#            tag_stack.pop()
#            elem_stack.pop()
#         except IndexError:
#            pass
ed=datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d_%H_%M_%S')
print(st)
print(ed)
sys.exit(0)
