import argparse
import sentencepiece as spm

from tqdm import tqdm

if __name__ == "__main__":
	
	parser = argparse.ArgumentParser(description='learn bpe like a pro')
	parser.add_argument('--input', '-i', type=str, help='input file')
	parser.add_argument('--output', '-o', type=str, help='Output file')
	args = parser.parse_args()
	with open(args.input, 'r') as input, open(args.output, 'w') as output:
		for line in tqdm(input):
			output.write(''.join(line.split()).replace('_', ' '))
	

70a446591ac5e183e52bb174dc17ae0cfd087b81
<mask token>

<mask token>
if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='learn bpe like a pro')
    parser.add_argument('--input', '-i', type=str, help='input file')
    parser.add_argument('--output', '-o', type=str, help='Output file')
    args = parser.parse_args()
    with open(args.input, 'r') as input, open(args.output, 'w') as output:
        for line in tqdm(input):
            output.write(''.join(line.split()).replace('_', ' '))

import argparse
import sentencepiece as spm
from tqdm import tqdm
if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='learn bpe like a pro')
    parser.add_argument('--input', '-i', type=str, help='input file')
    parser.add_argument('--output', '-o', type=str, help='Output file')
    args = parser.parse_args()
    with open(args.input, 'r') as input, open(args.output, 'w') as output:
        for line in tqdm(input):
            output.write(''.join(line.split()).replace('_', ' '))

import argparse
import sentencepiece as spm

from tqdm import tqdm

if __name__ == "__main__":
	
	parser = argparse.ArgumentParser(description='learn bpe like a pro')
	parser.add_argument('--input', '-i', type=str, help='input file')
	parser.add_argument('--output', '-o', type=str, help='Output file')
	args = parser.parse_args()
	with open(args.input, 'r') as input, open(args.output, 'w') as output:
		for line in tqdm(input):
			output.write(''.join(line.split()).replace('_', ' '))
	
