'''
Created on Feb 26, 2014

@author: igor, bgv, alexv
'''
# event handlers
NEW_TASK_HANDLER = 1

# Global HCE package version string
VERSION_STRING = "2.0.0-chaika"
# Global HCE project logger name
LOGGER_NAME = "hce"
# Global HCE project profiler logger name
LOGGER_NAME_PROFILER = "profiler"
# Global HCE project traceback logger name
LOGGER_NAME_TRACEBACK = "traceback"
# Start date mark application stats field name
START_DATE_NAME = "START_DATE"
# Constants exit codes
EXIT_SUCCESS = 0
EXIT_FAILURE = 1

# 1 - Crawler errors diapason
ERROR_OK = 0
ERROR_BAD_URL = 1
ERROR_REQUEST_TIMEOUT = 1 << 1
ERROR_HTTP_ERROR = 1 << 2
ERROR_EMPTY_RESPONSE = 1 << 3
ERROR_WRONG_MIME = 1 << 4
ERROR_CONNECTION_ERROR = 1 << 5
ERROR_PAGE_CONVERT_ERROR = 1 << 6
ERROR_MACRO = 1 << 7
ERROR_RESPONSE_SIZE_ERROR = 1 << 8
ERROR_AUTH_ERROR = 1 << 9
ERROR_WRITE_FILE_ERROR = 1 << 10
ERROR_ROBOTS_NOT_ALLOW = 1 << 11
ERROR_PARSE_ERROR = 1 << 12
ERROR_BAD_ENCODING = 1 << 13
ERROR_SITE_MAX_ERRORS = 1 << 14
ERROR_SYNCHRONIZE_URL_WITH_DB = 1 << 15
ERROR_CRAWLER_FILTERS_BREAK = 1 << 16
ERROR_MAX_ALLOW_HTTP_REDIRECTS = 1 << 17
ERROR_MAX_ALLOW_HTML_REDIRECTS = 1 << 18
ERROR_GENERAL_CRAWLER = 1 << 19
ERROR_DTD_INVALID = 1 << 20
ERROR_MACRO_DESERIALIZATION = 1 << 21
ERROR_FETCH_AMBIGUOUS_REQUEST = 1 << 22
ERROR_FETCH_CONNECTION_ERROR = 1 << 23
ERROR_FETCH_HTTP_ERROR = 1 << 24
ERROR_FETCH_INVALID_URL = 1 << 25
ERROR_FETCH_TOO_MANY_REDIRECTS = 1 << 26
ERROR_FETCH_CONNECTION_TIMEOUT = 1 << 27
ERROR_FETCH_READ_TIMEOUT = 1 << 28
ERROR_FETCH_TIMEOUT = 1 << 29
ERROR_FETCHER_INTERNAL = 1 << 30

# Processor/Scraper errors diapason
ERROR_MASK_SITE_MAX_RESOURCES_NUMBER = 1 << 31
ERROR_DATABASE_ERROR = 1 << 32
ERROR_MASK_SITE_MAX_RESOURCES_SIZE = 1 << 33
ERROR_MASK_SITE_UNSUPPORTED_CONTENT_TYPE = 1 << 34
ERROR_MASK_URL_ENCODING_ERROR = 1 << 35
ERROR_MASK_SCRAPER_ERROR = 1 << 36
ERROR_MASK_MISSED_RAW_CONTENT_ON_DISK = 1 << 37
ERROR_RE_ERROR = 1 << 38
ERROR_MANDATORY_TEMPLATE = 1 << 39
ERROR_PROCESSOR_FILTERS_BREAK = 1 << 40
ERROR_MASK_SITE_STATE = 1 << 41
ERROR_MAX_ITEMS = 1 << 42
ERROR_MAX_URLS_FROM_PAGE = 1 << 43
ERROR_TEMPLATE_SOURCE = 1 << 44

# 2 - Crawler errors diapason
ERROR_RSS_EMPTY = 1 << 45
ERROR_URLS_SCHEMA_EXTERNAL = 1 << 46
ERROR_NOT_EXIST_ANY_VALID_PROXY = 1 << 47
ERROR_FETCH_FORBIDDEN = 1 << 48
ERROR_NO_TIME_WINDOW = 1 << 49
ERROR_CRAWLER_FATAL_INITIALIZATION_PROJECT_ERROR = 1 << 50
ERROR_PROCESSOR_BATCH_ITEM_PROCESS = 1 << 51
ERROR_MAX_EXECUTION_TIME = 1 << 52

CONFIG_APPLICATION_SECTION_NAME = "Application"
CONFIG_PROFILER_SECTION_NAME = "Profiler"

LOGGER_DELIMITER_LINE = "============================================\n"

# Application class based on the cement usage forms
APP_USAGE_MODEL_PROCESS = 0
APP_USAGE_MODEL_MODULE = 1

# Application names used in modules
URLS_TO_BATCH_TASK_APP_NAME = "UrlsToBatchTask"
URLS_FETCH_JSON_TO_DBTASK_APP_NAME = "URLFetchToJsonDBTaskConvertor"
SOCIAL_TASK_APP_NAME = "SocialTask"
CONTENT_UPDATER_APP_NAME = "ContentUpdater"
POST_PROCESSOR_APP_NAME = "PostprocessorTask"
SOCIAL_PROFILES_VERIFIER_APP_NAME = "SocialProfilesVerifier"
SOCIAL_DATA_GET_API_APP_NAME = "SocialDataGetApi"
SOCIAL_DATA_UPDATER_APP_NAME = "SocialDataUpdater"

# Sites properties for work with source mask
PDATE_SOURCES_MASK_PROP_NAME = "PDATE_SOURCES_MASK"
PDATE_SOURCES_MASK_OVERWRITE_PROP_NAME = "PDATE_SOURCES_MASK_OVERWRITE"
PDATE_SOURCES_EXPRESSION_PROP_NAME = "PDATE_SOURCES_EXPRESSION"

# Sites properties for work with sql expression fields update
SQL_EXPRESSION_FIELDS_UPDATE_CRAWLER = 'SQL_EXPRESSION_FIELDS_UPDATE_CRAWLER'
SQL_EXPRESSION_FIELDS_UPDATE_PROCESSOR = 'SQL_EXPRESSION_FIELDS_UPDATE_PROCESSOR'
SQL_EXPRESSION_FIELDS_PDATE_TIME = 'PDATE_TIME'
# Site property for replacement content
REPLACEMENT_CONTENT_DATA = 'REPLACE'
# Site property for url normalization
URL_NORMALIZE = 'URL_NORMALIZE'
# Site property for collect urls limits
COLLECT_URLS_LIMITS = 'COLLECT_URLS_LIMITS'
# Site property for url limits
URL_LIMITS = 'URL_LIMITS'
# Site property for fetcher post
FETCHER_POST = 'FETCHER_POST'

# Bits constant values for source mask
# Used on crawler:
PDATE_SOURCES_MASK_URL_NAME = 1
PDATE_SOURCES_MASK_RSS_FEED = 1 << 1
PDATE_SOURCES_MASK_HTTP_DATE = 1 << 2
PDATE_SOURCES_MASK_HTTP_LAST_MODIFIED = 1 << 3
# Used on scraper:
PDATE_SOURCES_MASK_DC_DATE = 1 << 4
PDATE_SOURCES_MASK_PUBDATE = 1 << 5
PDATE_SOURCES_MASK_NOW = 1 << 6
PDATE_SOURCES_MASK_SQL_EXPRESSION = 1 << 7
# Default values for source mask
PDATE_SOURCES_MASK_BIT_DEFAULT = 255
PDATE_SOURCES_MASK_OVERWRITE_DEFAULT = 192

8d5442280adcf8754fbb250ac71642ca8d1bef1f
<mask token>

<mask token>
NEW_TASK_HANDLER = 1
VERSION_STRING = '2.0.0-chaika'
LOGGER_NAME = 'hce'
LOGGER_NAME_PROFILER = 'profiler'
LOGGER_NAME_TRACEBACK = 'traceback'
START_DATE_NAME = 'START_DATE'
EXIT_SUCCESS = 0
EXIT_FAILURE = 1
ERROR_OK = 0
ERROR_BAD_URL = 1
ERROR_REQUEST_TIMEOUT = 1 << 1
ERROR_HTTP_ERROR = 1 << 2
ERROR_EMPTY_RESPONSE = 1 << 3
ERROR_WRONG_MIME = 1 << 4
ERROR_CONNECTION_ERROR = 1 << 5
ERROR_PAGE_CONVERT_ERROR = 1 << 6
ERROR_MACRO = 1 << 7
ERROR_RESPONSE_SIZE_ERROR = 1 << 8
ERROR_AUTH_ERROR = 1 << 9
ERROR_WRITE_FILE_ERROR = 1 << 10
ERROR_ROBOTS_NOT_ALLOW = 1 << 11
ERROR_PARSE_ERROR = 1 << 12
ERROR_BAD_ENCODING = 1 << 13
ERROR_SITE_MAX_ERRORS = 1 << 14
ERROR_SYNCHRONIZE_URL_WITH_DB = 1 << 15
ERROR_CRAWLER_FILTERS_BREAK = 1 << 16
ERROR_MAX_ALLOW_HTTP_REDIRECTS = 1 << 17
ERROR_MAX_ALLOW_HTML_REDIRECTS = 1 << 18
ERROR_GENERAL_CRAWLER = 1 << 19
ERROR_DTD_INVALID = 1 << 20
ERROR_MACRO_DESERIALIZATION = 1 << 21
ERROR_FETCH_AMBIGUOUS_REQUEST = 1 << 22
ERROR_FETCH_CONNECTION_ERROR = 1 << 23
ERROR_FETCH_HTTP_ERROR = 1 << 24
ERROR_FETCH_INVALID_URL = 1 << 25
ERROR_FETCH_TOO_MANY_REDIRECTS = 1 << 26
ERROR_FETCH_CONNECTION_TIMEOUT = 1 << 27
ERROR_FETCH_READ_TIMEOUT = 1 << 28
ERROR_FETCH_TIMEOUT = 1 << 29
ERROR_FETCHER_INTERNAL = 1 << 30
ERROR_MASK_SITE_MAX_RESOURCES_NUMBER = 1 << 31
ERROR_DATABASE_ERROR = 1 << 32
ERROR_MASK_SITE_MAX_RESOURCES_SIZE = 1 << 33
ERROR_MASK_SITE_UNSUPPORTED_CONTENT_TYPE = 1 << 34
ERROR_MASK_URL_ENCODING_ERROR = 1 << 35
ERROR_MASK_SCRAPER_ERROR = 1 << 36
ERROR_MASK_MISSED_RAW_CONTENT_ON_DISK = 1 << 37
ERROR_RE_ERROR = 1 << 38
ERROR_MANDATORY_TEMPLATE = 1 << 39
ERROR_PROCESSOR_FILTERS_BREAK = 1 << 40
ERROR_MASK_SITE_STATE = 1 << 41
ERROR_MAX_ITEMS = 1 << 42
ERROR_MAX_URLS_FROM_PAGE = 1 << 43
ERROR_TEMPLATE_SOURCE = 1 << 44
ERROR_RSS_EMPTY = 1 << 45
ERROR_URLS_SCHEMA_EXTERNAL = 1 << 46
ERROR_NOT_EXIST_ANY_VALID_PROXY = 1 << 47
ERROR_FETCH_FORBIDDEN = 1 << 48
ERROR_NO_TIME_WINDOW = 1 << 49
ERROR_CRAWLER_FATAL_INITIALIZATION_PROJECT_ERROR = 1 << 50
ERROR_PROCESSOR_BATCH_ITEM_PROCESS = 1 << 51
ERROR_MAX_EXECUTION_TIME = 1 << 52
CONFIG_APPLICATION_SECTION_NAME = 'Application'
CONFIG_PROFILER_SECTION_NAME = 'Profiler'
LOGGER_DELIMITER_LINE = '============================================\n'
APP_USAGE_MODEL_PROCESS = 0
APP_USAGE_MODEL_MODULE = 1
URLS_TO_BATCH_TASK_APP_NAME = 'UrlsToBatchTask'
URLS_FETCH_JSON_TO_DBTASK_APP_NAME = 'URLFetchToJsonDBTaskConvertor'
SOCIAL_TASK_APP_NAME = 'SocialTask'
CONTENT_UPDATER_APP_NAME = 'ContentUpdater'
POST_PROCESSOR_APP_NAME = 'PostprocessorTask'
SOCIAL_PROFILES_VERIFIER_APP_NAME = 'SocialProfilesVerifier'
SOCIAL_DATA_GET_API_APP_NAME = 'SocialDataGetApi'
SOCIAL_DATA_UPDATER_APP_NAME = 'SocialDataUpdater'
PDATE_SOURCES_MASK_PROP_NAME = 'PDATE_SOURCES_MASK'
PDATE_SOURCES_MASK_OVERWRITE_PROP_NAME = 'PDATE_SOURCES_MASK_OVERWRITE'
PDATE_SOURCES_EXPRESSION_PROP_NAME = 'PDATE_SOURCES_EXPRESSION'
SQL_EXPRESSION_FIELDS_UPDATE_CRAWLER = 'SQL_EXPRESSION_FIELDS_UPDATE_CRAWLER'
SQL_EXPRESSION_FIELDS_UPDATE_PROCESSOR = (
    'SQL_EXPRESSION_FIELDS_UPDATE_PROCESSOR')
SQL_EXPRESSION_FIELDS_PDATE_TIME = 'PDATE_TIME'
REPLACEMENT_CONTENT_DATA = 'REPLACE'
URL_NORMALIZE = 'URL_NORMALIZE'
COLLECT_URLS_LIMITS = 'COLLECT_URLS_LIMITS'
URL_LIMITS = 'URL_LIMITS'
FETCHER_POST = 'FETCHER_POST'
PDATE_SOURCES_MASK_URL_NAME = 1
PDATE_SOURCES_MASK_RSS_FEED = 1 << 1
PDATE_SOURCES_MASK_HTTP_DATE = 1 << 2
PDATE_SOURCES_MASK_HTTP_LAST_MODIFIED = 1 << 3
PDATE_SOURCES_MASK_DC_DATE = 1 << 4
PDATE_SOURCES_MASK_PUBDATE = 1 << 5
PDATE_SOURCES_MASK_NOW = 1 << 6
PDATE_SOURCES_MASK_SQL_EXPRESSION = 1 << 7
PDATE_SOURCES_MASK_BIT_DEFAULT = 255
PDATE_SOURCES_MASK_OVERWRITE_DEFAULT = 192

'''
Created on Feb 26, 2014

@author: igor, bgv, alexv
'''
# event handlers
NEW_TASK_HANDLER = 1

# Global HCE package version string
VERSION_STRING = "2.0.0-chaika"
# Global HCE project logger name
LOGGER_NAME = "hce"
# Global HCE project profiler logger name
LOGGER_NAME_PROFILER = "profiler"
# Global HCE project traceback logger name
LOGGER_NAME_TRACEBACK = "traceback"
# Start date mark application stats field name
START_DATE_NAME = "START_DATE"
# Constants exit codes
EXIT_SUCCESS = 0
EXIT_FAILURE = 1

# 1 - Crawler errors diapason
ERROR_OK = 0
ERROR_BAD_URL = 1
ERROR_REQUEST_TIMEOUT = 1 << 1
ERROR_HTTP_ERROR = 1 << 2
ERROR_EMPTY_RESPONSE = 1 << 3
ERROR_WRONG_MIME = 1 << 4
ERROR_CONNECTION_ERROR = 1 << 5
ERROR_PAGE_CONVERT_ERROR = 1 << 6
ERROR_MACRO = 1 << 7
ERROR_RESPONSE_SIZE_ERROR = 1 << 8
ERROR_AUTH_ERROR = 1 << 9
ERROR_WRITE_FILE_ERROR = 1 << 10
ERROR_ROBOTS_NOT_ALLOW = 1 << 11
ERROR_PARSE_ERROR = 1 << 12
ERROR_BAD_ENCODING = 1 << 13
ERROR_SITE_MAX_ERRORS = 1 << 14
ERROR_SYNCHRONIZE_URL_WITH_DB = 1 << 15
ERROR_CRAWLER_FILTERS_BREAK = 1 << 16
ERROR_MAX_ALLOW_HTTP_REDIRECTS = 1 << 17
ERROR_MAX_ALLOW_HTML_REDIRECTS = 1 << 18
ERROR_GENERAL_CRAWLER = 1 << 19
ERROR_DTD_INVALID = 1 << 20
ERROR_MACRO_DESERIALIZATION = 1 << 21
ERROR_FETCH_AMBIGUOUS_REQUEST = 1 << 22
ERROR_FETCH_CONNECTION_ERROR = 1 << 23
ERROR_FETCH_HTTP_ERROR = 1 << 24
ERROR_FETCH_INVALID_URL = 1 << 25
ERROR_FETCH_TOO_MANY_REDIRECTS = 1 << 26
ERROR_FETCH_CONNECTION_TIMEOUT = 1 << 27
ERROR_FETCH_READ_TIMEOUT = 1 << 28
ERROR_FETCH_TIMEOUT = 1 << 29
ERROR_FETCHER_INTERNAL = 1 << 30

# Processor/Scraper errors diapason
ERROR_MASK_SITE_MAX_RESOURCES_NUMBER = 1 << 31
ERROR_DATABASE_ERROR = 1 << 32
ERROR_MASK_SITE_MAX_RESOURCES_SIZE = 1 << 33
ERROR_MASK_SITE_UNSUPPORTED_CONTENT_TYPE = 1 << 34
ERROR_MASK_URL_ENCODING_ERROR = 1 << 35
ERROR_MASK_SCRAPER_ERROR = 1 << 36
ERROR_MASK_MISSED_RAW_CONTENT_ON_DISK = 1 << 37
ERROR_RE_ERROR = 1 << 38
ERROR_MANDATORY_TEMPLATE = 1 << 39
ERROR_PROCESSOR_FILTERS_BREAK = 1 << 40
ERROR_MASK_SITE_STATE = 1 << 41
ERROR_MAX_ITEMS = 1 << 42
ERROR_MAX_URLS_FROM_PAGE = 1 << 43
ERROR_TEMPLATE_SOURCE = 1 << 44

# 2 - Crawler errors diapason
ERROR_RSS_EMPTY = 1 << 45
ERROR_URLS_SCHEMA_EXTERNAL = 1 << 46
ERROR_NOT_EXIST_ANY_VALID_PROXY = 1 << 47
ERROR_FETCH_FORBIDDEN = 1 << 48
ERROR_NO_TIME_WINDOW = 1 << 49
ERROR_CRAWLER_FATAL_INITIALIZATION_PROJECT_ERROR = 1 << 50
ERROR_PROCESSOR_BATCH_ITEM_PROCESS = 1 << 51
ERROR_MAX_EXECUTION_TIME = 1 << 52

CONFIG_APPLICATION_SECTION_NAME = "Application"
CONFIG_PROFILER_SECTION_NAME = "Profiler"

LOGGER_DELIMITER_LINE = "============================================\n"

# Application class based on the cement usage forms
APP_USAGE_MODEL_PROCESS = 0
APP_USAGE_MODEL_MODULE = 1

# Application names used in modules
URLS_TO_BATCH_TASK_APP_NAME = "UrlsToBatchTask"
URLS_FETCH_JSON_TO_DBTASK_APP_NAME = "URLFetchToJsonDBTaskConvertor"
SOCIAL_TASK_APP_NAME = "SocialTask"
CONTENT_UPDATER_APP_NAME = "ContentUpdater"
POST_PROCESSOR_APP_NAME = "PostprocessorTask"
SOCIAL_PROFILES_VERIFIER_APP_NAME = "SocialProfilesVerifier"
SOCIAL_DATA_GET_API_APP_NAME = "SocialDataGetApi"
SOCIAL_DATA_UPDATER_APP_NAME = "SocialDataUpdater"

# Sites properties for work with source mask
PDATE_SOURCES_MASK_PROP_NAME = "PDATE_SOURCES_MASK"
PDATE_SOURCES_MASK_OVERWRITE_PROP_NAME = "PDATE_SOURCES_MASK_OVERWRITE"
PDATE_SOURCES_EXPRESSION_PROP_NAME = "PDATE_SOURCES_EXPRESSION"

# Sites properties for work with sql expression fields update
SQL_EXPRESSION_FIELDS_UPDATE_CRAWLER = 'SQL_EXPRESSION_FIELDS_UPDATE_CRAWLER'
SQL_EXPRESSION_FIELDS_UPDATE_PROCESSOR = 'SQL_EXPRESSION_FIELDS_UPDATE_PROCESSOR'
SQL_EXPRESSION_FIELDS_PDATE_TIME = 'PDATE_TIME'
# Site property for replacement content
REPLACEMENT_CONTENT_DATA = 'REPLACE'
# Site property for url normalization
URL_NORMALIZE = 'URL_NORMALIZE'
# Site property for collect urls limits
COLLECT_URLS_LIMITS = 'COLLECT_URLS_LIMITS'
# Site property for url limits
URL_LIMITS = 'URL_LIMITS'
# Site property for fetcher post
FETCHER_POST = 'FETCHER_POST'

# Bits constant values for source mask
# Used on crawler:
PDATE_SOURCES_MASK_URL_NAME = 1
PDATE_SOURCES_MASK_RSS_FEED = 1 << 1
PDATE_SOURCES_MASK_HTTP_DATE = 1 << 2
PDATE_SOURCES_MASK_HTTP_LAST_MODIFIED = 1 << 3
# Used on scraper:
PDATE_SOURCES_MASK_DC_DATE = 1 << 4
PDATE_SOURCES_MASK_PUBDATE = 1 << 5
PDATE_SOURCES_MASK_NOW = 1 << 6
PDATE_SOURCES_MASK_SQL_EXPRESSION = 1 << 7
# Default values for source mask
PDATE_SOURCES_MASK_BIT_DEFAULT = 255
PDATE_SOURCES_MASK_OVERWRITE_DEFAULT = 192
