training/utils/preprocess_questions.py [171:199]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        print(h, len(houses[h]))
        for q in houses[h]:
            if len(str(q['answer']).split(' ')) > 1:
                q['answer'] = '_'.join(q['answer'].split(' '))
            questions.append(q)

    print('Total questions: ', len(questions))

    # build vocab if no vocab file provided
    if args.input_vocab == None:
        answerTokenToIdx = buildVocab((str(q['answer']) for q in questions
                                       if q['answer'] != 'NIL'))
        questionTokenToIdx = buildVocab(
            (q['question'] for q in questions if q['answer'] != 'NIL'),
            punctToRemove=['?'],
            addSpecialTok=True)

        vocab = {
            'questionTokenToIdx': questionTokenToIdx,
            'answerTokenToIdx': answerTokenToIdx,
        }
    else:
        vocab = json.load(open(args.input_vocab, 'r'))

    if args.output_vocab != None:
        json.dump(vocab, open(args.output_vocab, 'w'))

    # encode questions
    idx, encoded_questions, question_types, answers, action_labels, action_lengths, pos_queue, envs, boxes = [], [], [], [], [], [], [], [], []
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



training/utils/preprocess_questions_pkl.py [167:195]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        print(h, len(houses[h]))
        for q in houses[h]:
            if len(str(q['answer']).split(' ')) > 1:
                q['answer'] = '_'.join(q['answer'].split(' '))
            questions.append(q)

    print('Total questions: ', len(questions))

    # build vocab if no vocab file provided
    if args.input_vocab == None:
        answerTokenToIdx = buildVocab((str(q['answer']) for q in questions
                                       if q['answer'] != 'NIL'))
        questionTokenToIdx = buildVocab(
            (q['question'] for q in questions if q['answer'] != 'NIL'),
            punctToRemove=['?'],
            addSpecialTok=True)

        vocab = {
            'questionTokenToIdx': questionTokenToIdx,
            'answerTokenToIdx': answerTokenToIdx,
        }
    else:
        vocab = json.load(open(args.input_vocab, 'r'))

    if args.output_vocab != None:
        json.dump(vocab, open(args.output_vocab, 'w'))

    # encode questions
    idx, encoded_questions, question_types, answers, action_labels, action_lengths, pos_queue, envs, boxes = [], [], [], [], [], [], [], [], []
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -