# Copyright (c) 2021 Binbin Zhang(binbzha@qq.com) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import print_function import argparse import copy import logging import os import sys import torch import yaml from torch.utils.data import DataLoader from kws.dataset.dataset import Dataset from kws.model.kws_model import init_model from kws.utils.checkpoint import load_checkpoint def get_args(): parser = argparse.ArgumentParser(description='recognize with your model') parser.add_argument('--config', required=True, help='config file') parser.add_argument('--test_data', required=True, help='test data file') parser.add_argument('--checkpoint', required=True, help='checkpoint model') parser.add_argument('--num_workers', default=0, type=int, help='num of subprocess workers for reading') parser.add_argument('--pin_memory', action='store_true', default=False, help='Use pinned memory buffers used for reading') parser.add_argument('--prefetch', default=100, type=int, help='prefetch number') parser.add_argument('--script_model', required=True, help='output script model') args = parser.parse_args() print(args) return args def main(): args = get_args() logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(levelname)s %(message)s') os.environ['CUDA_VISIBLE_DEVICES'] = str("-1") with open(args.config, 'r') as fin: configs = yaml.load(fin, Loader=yaml.FullLoader) test_conf = copy.deepcopy(configs['dataset_conf']) test_conf['filter_conf']['max_length'] = 102400 test_conf['filter_conf']['min_length'] = 0 test_conf['speed_perturb'] = False test_conf['spec_aug'] = False test_conf['shuffle'] = False test_conf['feature_extraction_conf']['dither'] = 0.0 test_conf['batch_conf']['batch_size'] = 1 test_dataset = Dataset(args.test_data, test_conf) test_data_loader = DataLoader(test_dataset, batch_size=None, pin_memory=args.pin_memory, num_workers=args.num_workers, prefetch_factor=args.prefetch) # Init asr model from configs model_fp32 = init_model(configs['model']) load_checkpoint(model_fp32, args.checkpoint) # model must be set to eval mode for static quantization logic to work model_fp32.eval() # Fuse the activations to preceding layers, where applicable. # This needs to be done manually depending on the model architecture. # Common fusions include `conv + relu` and `conv + batchnorm + relu` print('================ Float 32 ======================') print(model_fp32) print('================ Float 32(fused) ===============') model_fp32.fuse_modules() print(model_fp32) # attach a global qconfig, which contains information about what kind # of observers to attach. Use 'fbgemm' for server inference and # 'qnnpack' for mobile inference. Other quantization configurations such # as selecting symmetric or assymetric quantization and MinMax or L2Norm # calibration techniques can be specified here. model_fp32.qconfig = torch.quantization.get_default_qconfig('qnnpack') # Prepare the model for static quantization. This inserts observers in # the model that will observe activation tensors during calibration. model_fp32_prepared = torch.quantization.prepare(model_fp32) # calibrate the prepared model to determine quantization parameters for # activations in a real world setting, the calibration would be done with # a representative dataset with torch.no_grad(): for batch_idx, batch in enumerate(test_data_loader): keys, feats, target, lengths = batch logits = model_fp32_prepared(feats) if batch_idx % 100 == 0: print('Progress utts {}'.format(batch_idx)) sys.stdout.flush() # Convert the observed model to a quantized model. This does several things: # quantizes the weights, computes and stores the scale and bias value to be # used with each activation tensor, and replaces key operators with # quantized implementations. print('=================== int8 ======================') model_int8 = torch.quantization.convert(model_fp32_prepared) print(model_int8) print('================ int8(script) ==================') script_model = torch.jit.script(model_int8) script_model.save(args.script_model) print(script_model) if __name__ == '__main__': main()