# import the modules
import decimal
import ply.lex as lex
from sys import exit

# declare tokens
tokens = [
"ID",
"COMMENT",
"NUMBER",
"STRING",
"EQU",
"BOE",
"SOE",
"NOE",
"BIG",
"SMA",
"SET",
"AAS",
"SAS",
"MAS",
"DAS",
"RAS",
"SQS",
]

reserved = {
"if" : "IF",
"else" : "ELSE",
"while" : "WHILE",
"until" : "UNTIL",
"for" : "FOR",
"break" : "BREAK",
"continue" : "CONTINUE",
"define" : "DEFINE",
"true" : "TRUE",
"false" : "FALSE",
}

literals = ['+','-','*','/', '%', '^', '&', '|', '!', '(', ')', '{', '}', '[', ']', ';', ',']

# tokens = tokens + reserved
tokens = tokens + list(reserved.values())

# define tokens
def t_NUMBER(t):
    r'\d+\.?\d*([eE]\d+)?'
    t.value = decimal.Decimal(t.value)
    return t

def t_STRING(t):
    r'("(\\"|[^"])*")|(\'(\\\'|[^\'])*\')'
    t.value = t.value[1:-1]
    return t

def t_EQU(t):
    r'=='
    return t

def t_BOE(t):
    r'>='
    return t

def t_SOE(t):
    r'<='
    return t

def t_NOE(t):
    r'!='
    return t

def t_BIG(t):
    r'>'
    return t

def t_SMA(t):
    r'<'
    return t

def t_SET(t):
    r'='
    return t

def t_AAS(t):
    r'\+='
    return t

def t_SAS(t):
    r'-='
    return t

def t_MAS(t):
    r'\*='
    return t

def t_DAS(t):
    r'/='
    return t

def t_RAS(t):
    r'%='
    return t

def t_SQS(t):
    r'\^='
    return t

# reserved, line, error, comment, ignore
def t_ID(t):
    r'[a-zA-Z_][a-zA-Z_0-9]*'
    t.type = reserved.get(t.value,'ID')
    return t

def t_COMMENT(t):
    r'\#.*'
    pass

def t_newline(t):
    r'\n+'
    t.lexer.lineno += len(t.value)

def t_error(t): t.lexer.skip(1)

t_ignore    = ' \t'

# build lexer
lexer = lex.lex()

# debug code
if __name__ == "__main__":
    lexer.input(input('@'))
    while True:
        tok = lexer.token()
        if not tok: break
        print(":", tok)

6bb7a7a563e91cab11e1069bcd9165cec5607a7c
<mask token>


def t_NUMBER(t):
    """\\d+\\.?\\d*([eE]\\d+)?"""
    t.value = decimal.Decimal(t.value)
    return t


def t_STRING(t):
    """("(\\\\"|[^"])*")|(\\'(\\\\\\'|[^\\'])*\\')"""
    t.value = t.value[1:-1]
    return t


def t_EQU(t):
    """=="""
    return t


def t_BOE(t):
    """>="""
    return t


def t_SOE(t):
    """<="""
    return t


def t_NOE(t):
    """!="""
    return t


def t_BIG(t):
    """>"""
    return t


<mask token>


def t_SET(t):
    """="""
    return t


def t_AAS(t):
    """\\+="""
    return t


def t_SAS(t):
    """-="""
    return t


<mask token>


def t_newline(t):
    """\\n+"""
    t.lexer.lineno += len(t.value)


<mask token>

<mask token>


def t_NUMBER(t):
    """\\d+\\.?\\d*([eE]\\d+)?"""
    t.value = decimal.Decimal(t.value)
    return t


def t_STRING(t):
    """("(\\\\"|[^"])*")|(\\'(\\\\\\'|[^\\'])*\\')"""
    t.value = t.value[1:-1]
    return t


def t_EQU(t):
    """=="""
    return t


def t_BOE(t):
    """>="""
    return t


def t_SOE(t):
    """<="""
    return t


def t_NOE(t):
    """!="""
    return t


def t_BIG(t):
    """>"""
    return t


def t_SMA(t):
    """<"""
    return t


def t_SET(t):
    """="""
    return t


def t_AAS(t):
    """\\+="""
    return t


def t_SAS(t):
    """-="""
    return t


def t_MAS(t):
    """\\*="""
    return t


def t_DAS(t):
    """/="""
    return t


def t_RAS(t):
    """%="""
    return t


def t_SQS(t):
    """\\^="""
    return t


def t_ID(t):
    """[a-zA-Z_][a-zA-Z_0-9]*"""
    t.type = reserved.get(t.value, 'ID')
    return t


def t_COMMENT(t):
    """\\#.*"""
    pass


def t_newline(t):
    """\\n+"""
    t.lexer.lineno += len(t.value)


def t_error(t):
    t.lexer.skip(1)


<mask token>
if __name__ == '__main__':
    lexer.input(input('@'))
    while True:
        tok = lexer.token()
        if not tok:
            break
        print(':', tok)

<mask token>
tokens = ['ID', 'COMMENT', 'NUMBER', 'STRING', 'EQU', 'BOE', 'SOE', 'NOE',
    'BIG', 'SMA', 'SET', 'AAS', 'SAS', 'MAS', 'DAS', 'RAS', 'SQS']
reserved = {'if': 'IF', 'else': 'ELSE', 'while': 'WHILE', 'until': 'UNTIL',
    'for': 'FOR', 'break': 'BREAK', 'continue': 'CONTINUE', 'define':
    'DEFINE', 'true': 'TRUE', 'false': 'FALSE'}
literals = ['+', '-', '*', '/', '%', '^', '&', '|', '!', '(', ')', '{', '}',
    '[', ']', ';', ',']
tokens = tokens + list(reserved.values())


def t_NUMBER(t):
    """\\d+\\.?\\d*([eE]\\d+)?"""
    t.value = decimal.Decimal(t.value)
    return t


def t_STRING(t):
    """("(\\\\"|[^"])*")|(\\'(\\\\\\'|[^\\'])*\\')"""
    t.value = t.value[1:-1]
    return t


def t_EQU(t):
    """=="""
    return t


def t_BOE(t):
    """>="""
    return t


def t_SOE(t):
    """<="""
    return t


def t_NOE(t):
    """!="""
    return t


def t_BIG(t):
    """>"""
    return t


def t_SMA(t):
    """<"""
    return t


def t_SET(t):
    """="""
    return t


def t_AAS(t):
    """\\+="""
    return t


def t_SAS(t):
    """-="""
    return t


def t_MAS(t):
    """\\*="""
    return t


def t_DAS(t):
    """/="""
    return t


def t_RAS(t):
    """%="""
    return t


def t_SQS(t):
    """\\^="""
    return t


def t_ID(t):
    """[a-zA-Z_][a-zA-Z_0-9]*"""
    t.type = reserved.get(t.value, 'ID')
    return t


def t_COMMENT(t):
    """\\#.*"""
    pass


def t_newline(t):
    """\\n+"""
    t.lexer.lineno += len(t.value)


def t_error(t):
    t.lexer.skip(1)


t_ignore = ' \t'
lexer = lex.lex()
if __name__ == '__main__':
    lexer.input(input('@'))
    while True:
        tok = lexer.token()
        if not tok:
            break
        print(':', tok)

import decimal
import ply.lex as lex
from sys import exit
tokens = ['ID', 'COMMENT', 'NUMBER', 'STRING', 'EQU', 'BOE', 'SOE', 'NOE',
    'BIG', 'SMA', 'SET', 'AAS', 'SAS', 'MAS', 'DAS', 'RAS', 'SQS']
reserved = {'if': 'IF', 'else': 'ELSE', 'while': 'WHILE', 'until': 'UNTIL',
    'for': 'FOR', 'break': 'BREAK', 'continue': 'CONTINUE', 'define':
    'DEFINE', 'true': 'TRUE', 'false': 'FALSE'}
literals = ['+', '-', '*', '/', '%', '^', '&', '|', '!', '(', ')', '{', '}',
    '[', ']', ';', ',']
tokens = tokens + list(reserved.values())


def t_NUMBER(t):
    """\\d+\\.?\\d*([eE]\\d+)?"""
    t.value = decimal.Decimal(t.value)
    return t


def t_STRING(t):
    """("(\\\\"|[^"])*")|(\\'(\\\\\\'|[^\\'])*\\')"""
    t.value = t.value[1:-1]
    return t


def t_EQU(t):
    """=="""
    return t


def t_BOE(t):
    """>="""
    return t


def t_SOE(t):
    """<="""
    return t


def t_NOE(t):
    """!="""
    return t


def t_BIG(t):
    """>"""
    return t


def t_SMA(t):
    """<"""
    return t


def t_SET(t):
    """="""
    return t


def t_AAS(t):
    """\\+="""
    return t


def t_SAS(t):
    """-="""
    return t


def t_MAS(t):
    """\\*="""
    return t


def t_DAS(t):
    """/="""
    return t


def t_RAS(t):
    """%="""
    return t


def t_SQS(t):
    """\\^="""
    return t


def t_ID(t):
    """[a-zA-Z_][a-zA-Z_0-9]*"""
    t.type = reserved.get(t.value, 'ID')
    return t


def t_COMMENT(t):
    """\\#.*"""
    pass


def t_newline(t):
    """\\n+"""
    t.lexer.lineno += len(t.value)


def t_error(t):
    t.lexer.skip(1)


t_ignore = ' \t'
lexer = lex.lex()
if __name__ == '__main__':
    lexer.input(input('@'))
    while True:
        tok = lexer.token()
        if not tok:
            break
        print(':', tok)

# import the modules
import decimal
import ply.lex as lex
from sys import exit

# declare tokens
tokens = [
"ID",
"COMMENT",
"NUMBER",
"STRING",
"EQU",
"BOE",
"SOE",
"NOE",
"BIG",
"SMA",
"SET",
"AAS",
"SAS",
"MAS",
"DAS",
"RAS",
"SQS",
]

reserved = {
"if" : "IF",
"else" : "ELSE",
"while" : "WHILE",
"until" : "UNTIL",
"for" : "FOR",
"break" : "BREAK",
"continue" : "CONTINUE",
"define" : "DEFINE",
"true" : "TRUE",
"false" : "FALSE",
}

literals = ['+','-','*','/', '%', '^', '&', '|', '!', '(', ')', '{', '}', '[', ']', ';', ',']

# tokens = tokens + reserved
tokens = tokens + list(reserved.values())

# define tokens
def t_NUMBER(t):
    r'\d+\.?\d*([eE]\d+)?'
    t.value = decimal.Decimal(t.value)
    return t

def t_STRING(t):
    r'("(\\"|[^"])*")|(\'(\\\'|[^\'])*\')'
    t.value = t.value[1:-1]
    return t

def t_EQU(t):
    r'=='
    return t

def t_BOE(t):
    r'>='
    return t

def t_SOE(t):
    r'<='
    return t

def t_NOE(t):
    r'!='
    return t

def t_BIG(t):
    r'>'
    return t

def t_SMA(t):
    r'<'
    return t

def t_SET(t):
    r'='
    return t

def t_AAS(t):
    r'\+='
    return t

def t_SAS(t):
    r'-='
    return t

def t_MAS(t):
    r'\*='
    return t

def t_DAS(t):
    r'/='
    return t

def t_RAS(t):
    r'%='
    return t

def t_SQS(t):
    r'\^='
    return t

# reserved, line, error, comment, ignore
def t_ID(t):
    r'[a-zA-Z_][a-zA-Z_0-9]*'
    t.type = reserved.get(t.value,'ID')
    return t

def t_COMMENT(t):
    r'\#.*'
    pass

def t_newline(t):
    r'\n+'
    t.lexer.lineno += len(t.value)

def t_error(t): t.lexer.skip(1)

t_ignore    = ' \t'

# build lexer
lexer = lex.lex()

# debug code
if __name__ == "__main__":
    lexer.input(input('@'))
    while True:
        tok = lexer.token()
        if not tok: break
        print(":", tok)
