# Copyright (c) Guangsheng Bao. # # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. import os.path import argparse import json import numpy as np def save_lines(lines, file): with open(file, 'w') as fout: fout.write('\n'.join(lines)) def get_auroc(result_file): with open(result_file, 'r') as fin: res = json.load(fin) return res['metrics']['roc_auc'] def get_fpr_tpr(result_file): with open(result_file, 'r') as fin: res = json.load(fin) return res['metrics']['fpr'], res['metrics']['tpr'] def report_main_results(args): datasets = {'xsum': 'XSum', 'squad': 'SQuAD', 'writing': 'WritingPrompts'} source_models = {'gpt2-xl': 'GPT-2', 'opt-2.7b': 'OPT-2.7', 'gpt-neo-2.7B': 'Neo-2.7', 'gpt-j-6B': 'GPT-J', 'gpt-neox-20b': 'NeoX'} methods1 = {'likelihood': 'Likelihood', 'entropy': 'Entropy', 'logrank': 'LogRank', 'lrr': 'LRR', 'npr': 'NPR'} methods2 = {'perturbation_100': 'DetectGPT', 'sampling_discrepancy': 'Fast-DetectGPT'} def _get_method_aurocs(dataset, method, filter=''): cols = [] for model in source_models: result_file = f'{args.result_path}/{dataset}_{model}{filter}.{method}.json' if os.path.exists(result_file): auroc = get_auroc(result_file) else: auroc = 0.0 cols.append(auroc) cols.append(np.mean(cols)) return cols headers = ['Method'] + [source_models[model] for model in source_models] + ['Avg.'] for dataset in datasets: print('----') print(datasets[dataset]) print('----') print(' '.join(headers)) # basic methods for method in methods1: method_name = methods1[method] cols = _get_method_aurocs(dataset, method) cols = [f'{col:.4f}' for col in cols] print(method_name, ' '.join(cols)) # white-box comparison results = {} for method in methods2: method_name = methods2[method] cols = _get_method_aurocs(dataset, method) results[method_name] = cols cols = [f'{col:.4f}' for col in cols] print(method_name, ' '.join(cols)) cols = np.array(results['Fast-DetectGPT']) - np.array(results['DetectGPT']) cols = [f'{col:.4f}' for col in cols] print('(Diff)', ' '.join(cols)) # black-box comparison filters = {'perturbation_100': '.t5-3b_gpt-neo-2.7B', 'sampling_discrepancy': '.gpt-j-6B_gpt-neo-2.7B'} results = {} for method in methods2: method_name = methods2[method] cols = _get_method_aurocs(dataset, method, filters[method]) results[method_name] = cols cols = [f'{col:.4f}' for col in cols] print(method_name, ' '.join(cols)) cols = np.array(results['Fast-DetectGPT']) - np.array(results['DetectGPT']) cols = [f'{col:.4f}' for col in cols] print('(Diff)', ' '.join(cols)) def report_main_ext_results(args): datasets = {'xsum': 'XSum', 'squad': 'SQuAD', 'writing': 'WritingPrompts'} source_models = {'bloom-7b1': 'BLOOM-7.1', 'opt-13b': 'OPT-13', 'llama-13b': 'Llama-13', 'llama2-13b': 'Llama2-13', } methods1 = {'likelihood': 'Likelihood', 'entropy': 'Entropy', 'logrank': 'LogRank', 'lrr': 'LRR', 'npr': 'NPR'} methods2 = {'perturbation_100': 'DetectGPT', 'sampling_discrepancy': 'Fast-DetectGPT'} def _get_method_aurocs(dataset, method, filter=''): cols = [] for model in source_models: result_file = f'{args.result_path}/{dataset}_{model}{filter}.{method}.json' if os.path.exists(result_file): auroc = get_auroc(result_file) else: auroc = 0.0 cols.append(auroc) cols.append(np.mean(cols)) return cols headers = ['Method'] + [source_models[model] for model in source_models] + ['Avg.'] for dataset in datasets: print('----') print(datasets[dataset]) print('----') print(' '.join(headers)) # basic methods for method in methods1: method_name = methods1[method] cols = _get_method_aurocs(dataset, method) cols = [f'{col:.4f}' for col in cols] print(method_name, ' '.join(cols)) # white-box comparison results = {} for method in methods2: method_name = methods2[method] cols = _get_method_aurocs(dataset, method) results[method_name] = cols cols = [f'{col:.4f}' for col in cols] print(method_name, ' '.join(cols)) cols = np.array(results['Fast-DetectGPT']) - np.array(results['DetectGPT']) cols = [f'{col:.4f}' for col in cols] print('(Diff)', ' '.join(cols)) # black-box comparison filters = {'perturbation_100': '.t5-3b_gpt-neo-2.7B', 'sampling_discrepancy': '.gpt-j-6B_gpt-neo-2.7B'} results = {} for method in methods2: method_name = methods2[method] cols = _get_method_aurocs(dataset, method, filters[method]) results[method_name] = cols cols = [f'{col:.4f}' for col in cols] print(method_name, ' '.join(cols)) cols = np.array(results['Fast-DetectGPT']) - np.array(results['DetectGPT']) cols = [f'{col:.4f}' for col in cols] print('(Diff)', ' '.join(cols)) def report_refmodel_results(args): datasets = {'xsum': 'XSum', 'squad': 'SQuAD', 'writing': 'WritingPrompts'} source_models = {'gpt2-xl': 'GPT-2', 'gpt-neo-2.7B': 'Neo-2.7', 'gpt-j-6B': 'GPT-J'} def _get_method_aurocs(method, ref_model=None): cols = [] for dataset in datasets: for model in source_models: filter = '' if ref_model is None or ref_model == model else f'.{ref_model}_{model}' result_file = f'{args.result_path}/{dataset}_{model}{filter}.{method}.json' if os.path.exists(result_file): auroc = get_auroc(result_file) else: auroc = 0.0 cols.append(auroc) cols.append(np.mean(cols)) return cols headers1 = ['----'] + list([datasets[d] for d in datasets]) headers2 = ['Method'] + [source_models[model] for model in source_models] \ + [source_models[model] for model in source_models] \ + [source_models[model] for model in source_models] \ + ['Avg.'] print(' '.join(headers1)) print(' '.join(headers2)) ref_models = [None, 'gpt2-xl', 'gpt-neo-2.7B', 'gpt-j-6B'] for ref_model in ref_models: method = 'sampling_discrepancy' method_name = 'Fast-DetectGPT (*/*)' if ref_model is None else f'Fast-DetectGPT ({source_models[ref_model]}/*)' cols = _get_method_aurocs(method, ref_model) cols = [f'{col:.4f}' for col in cols] print(method_name, ' '.join(cols)) def report_chatgpt_gpt4_results(args): datasets = {'xsum': 'XSum', 'writing': 'Writing', 'pubmed': 'PubMed'} source_models = {'gpt-3.5-turbo': 'ChatGPT', 'gpt-4': 'GPT-4'} score_models = { 't5-11b': 'T5-11B', 'gpt2-xl': 'GPT-2', 'opt-2.7b': 'OPT-2.7', 'gpt-neo-2.7B': 'Neo-2.7', 'gpt-j-6B': 'GPT-J', 'gpt-neox-20b': 'NeoX'} methods1 = {'roberta-base-openai-detector': 'RoBERTa-base', 'roberta-large-openai-detector': 'RoBERTa-large'} methods2 = {'likelihood': 'Likelihood', 'entropy': 'Entropy', 'logrank': 'LogRank'} methods3 = {'lrr': 'LRR', 'npr': 'NPR', 'perturbation_100': 'DetectGPT', 'sampling_discrepancy_analytic': 'Fast'} def _get_method_aurocs(method, filter=''): results = [] for model in source_models: cols = [] for dataset in datasets: result_file = f'{args.result_path}/{dataset}_{model}{filter}.{method}.json' if os.path.exists(result_file): auroc = get_auroc(result_file) else: auroc = 0.0 cols.append(auroc) cols.append(np.mean(cols)) results.extend(cols) return results headers1 = ['--'] + [source_models[model] for model in source_models] headers2 = ['Method'] + [datasets[dataset] for dataset in datasets] + ['Avg.'] \ + [datasets[dataset] for dataset in datasets] + ['Avg.'] print(' '.join(headers1)) print(' '.join(headers2)) # supervised methods for method in methods1: method_name = methods1[method] cols = _get_method_aurocs(method) cols = [f'{col:.4f}' for col in cols] print(method_name, ' '.join(cols)) # zero-shot methods filters2 = {'likelihood': ['.gpt2-xl', '.gpt-neo-2.7B', '.gpt-j-6B', '.gpt-neox-20b'], 'entropy': ['.gpt2-xl', '.gpt-neo-2.7B', '.gpt-j-6B', '.gpt-neox-20b'], 'logrank': ['.gpt2-xl', '.gpt-neo-2.7B', '.gpt-j-6B', '.gpt-neox-20b']} filters3 = {'lrr': ['.t5-11b_gpt2-xl', '.t5-11b_gpt-neo-2.7B', '.t5-11b_gpt-j-6B', '.t5-11b_gpt-neox-20b'], 'npr': ['.t5-11b_gpt2-xl', '.t5-11b_gpt-neo-2.7B', '.t5-11b_gpt-j-6B', '.t5-11b_gpt-neox-20b'], 'perturbation_100': ['.t5-11b_gpt2-xl', '.t5-11b_gpt-neo-2.7B', '.t5-11b_gpt-j-6B', '.t5-11b_gpt-neox-20b'], 'sampling_discrepancy_analytic': ['.gpt-j-6B_gpt2-xl', '.gpt-j-6B_gpt-neo-2.7B', '.gpt-j-6B_gpt-j-6B', '.gpt-neox-20b_gpt-neox-20b']} for method in methods2: for filter in filters2[method]: setting = score_models[filter[1:]] method_name = f'{methods2[method]}({setting})' cols = _get_method_aurocs(method, filter) cols = [f'{col:.4f}' for col in cols] print(method_name, ' '.join(cols)) for method in methods3: for filter in filters3[method]: setting = [score_models[model] for model in filter[1:].split('_')] method_name = f'{methods3[method]}({setting[0]}/{setting[1]})' cols = _get_method_aurocs(method, filter) cols = [f'{col:.4f}' for col in cols] print(method_name, ' '.join(cols)) def report_gpt3_results(args): datasets = {'xsum': 'XSum', 'writing': 'Writing', 'pubmed': 'PubMed'} source_models = {'davinci': 'GPT-3'} score_models = { 't5-11b': 'T5-11B', 'gpt2-xl': 'GPT-2', 'opt-2.7b': 'OPT-2.7', 'gpt-neo-2.7B': 'Neo-2.7', 'gpt-j-6B': 'GPT-J', 'gpt-neox-20b': 'NeoX'} methods1 = {'roberta-base-openai-detector': 'RoBERTa-base', 'roberta-large-openai-detector': 'RoBERTa-large'} methods2 = {'likelihood': 'Likelihood', 'entropy': 'Entropy', 'logrank': 'LogRank'} methods3 = {'lrr': 'LRR', 'npr': 'NPR', 'perturbation_100': 'DetectGPT', 'sampling_discrepancy_analytic': 'Fast'} def _get_method_aurocs(method, filter=''): results = [] for model in source_models: cols = [] for dataset in datasets: result_file = f'{args.result_path}/{dataset}_{model}{filter}.{method}.json' if os.path.exists(result_file): auroc = get_auroc(result_file) else: auroc = 0.0 cols.append(auroc) cols.append(np.mean(cols)) results.extend(cols) return results headers1 = ['--'] + [source_models[model] for model in source_models] headers2 = ['Method'] + [datasets[dataset] for dataset in datasets] + ['Avg.'] \ + [datasets[dataset] for dataset in datasets] + ['Avg.'] print(' '.join(headers1)) print(' '.join(headers2)) # supervised methods for method in methods1: method_name = methods1[method] cols = _get_method_aurocs(method) cols = [f'{col:.4f}' for col in cols] print(method_name, ' '.join(cols)) # zero-shot methods filters2 = {'likelihood': ['.gpt2-xl', '.gpt-neo-2.7B', '.gpt-j-6B', '.gpt-neox-20b'], 'entropy': ['.gpt2-xl', '.gpt-neo-2.7B', '.gpt-j-6B', '.gpt-neox-20b'], 'logrank': ['.gpt2-xl', '.gpt-neo-2.7B', '.gpt-j-6B', '.gpt-neox-20b']} filters3 = {'lrr': ['.t5-11b_gpt2-xl', '.t5-11b_gpt-neo-2.7B', '.t5-11b_gpt-j-6B', '.t5-11b_gpt-neox-20b'], 'npr': ['.t5-11b_gpt2-xl', '.t5-11b_gpt-neo-2.7B', '.t5-11b_gpt-j-6B', '.t5-11b_gpt-neox-20b'], 'perturbation_100': ['.t5-11b_gpt2-xl', '.t5-11b_gpt-neo-2.7B', '.t5-11b_gpt-j-6B', '.t5-11b_gpt-neox-20b'], 'sampling_discrepancy_analytic': ['.gpt-j-6B_gpt2-xl', '.gpt-j-6B_gpt-neo-2.7B', '.gpt-j-6B_gpt-j-6B', '.gpt-neox-20b_gpt-neox-20b']} for method in methods2: for filter in filters2[method]: setting = score_models[filter[1:]] method_name = f'{methods2[method]}({setting})' cols = _get_method_aurocs(method, filter) cols = [f'{col:.4f}' for col in cols] print(method_name, ' '.join(cols)) for method in methods3: for filter in filters3[method]: setting = [score_models[model] for model in filter[1:].split('_')] method_name = f'{methods3[method]}({setting[0]}/{setting[1]})' cols = _get_method_aurocs(method, filter) cols = [f'{col:.4f}' for col in cols] print(method_name, ' '.join(cols)) def report_maxlen_trends(args): datasets = {'xsum': 'XSum', 'writing': 'WritingPrompts'} source_models = {'gpt-3.5-turbo': 'ChatGPT', 'gpt-4': 'GPT-4'} score_models = {'t5-11b': 'T5-11B', 'gpt2-xl': 'GPT-2', 'opt-2.7b': 'OPT-2.7', 'gpt-neo-2.7B': 'Neo-2.7', 'gpt-j-6B': 'GPT-J', 'gpt-neox-20b': 'NeoX'} methods1 = {'roberta-base-openai-detector': 'RoBERTa-base', 'roberta-large-openai-detector': 'RoBERTa-large'} methods2 = {'likelihood': 'Likelihood'} methods3 = {'perturbation_100': 'DetectGPT', 'sampling_discrepancy_analytic': 'Fast-Detect'} maxlens = [30, 60, 90, 120, 150, 180] def _get_method_aurocs(root_path, dataset, source_model, method, filter=''): cols = [] for maxlen in maxlens: result_file = f'{root_path}/exp_maxlen{maxlen}/results/{dataset}_{source_model}{filter}.{method}.json' if os.path.exists(result_file): auroc = get_auroc(result_file) else: auroc = 0.0 cols.append(auroc) return cols filters2 = {'likelihood': '.gpt-neo-2.7B'} filters3 = {'perturbation_100': '.t5-11b_gpt-neo-2.7B', 'sampling_discrepancy_analytic': '.gpt-j-6B_gpt-neo-2.7B'} headers = ['Method'] + [str(maxlen) for maxlen in maxlens] print(' '.join(headers)) # print table per model and dataset results = {} for model in source_models: model_name = source_models[model] for data in datasets: data_name = datasets[data] print('----') print(f'{model_name} / {data_name}') print('----') for method in methods1: method_name = methods1[method] cols = _get_method_aurocs('.', data, model, method) results[f'{model_name}_{data_name}_{method_name}'] = cols cols = [f'{col:.4f}' for col in cols] print(method_name, ' '.join(cols)) for method in methods2: filter = filters2[method] setting = score_models[filter[1:]] method_name = f'{methods2[method]}({setting})' cols = _get_method_aurocs('.', data, model, method, filter) results[f'{model_name}_{data_name}_{method_name}'] = cols cols = [f'{col:.4f}' for col in cols] print(method_name, ' '.join(cols)) for method in methods3: filter = filters3[method] setting = [score_models[model] for model in filter[1:].split('_')] method_name = f'{methods3[method]}({setting[0]}/{setting[1]})' cols = _get_method_aurocs('.', data, model, method, filter) results[f'{model_name}_{data_name}_{method_name}'] = cols cols = [f'{col:.4f}' for col in cols] print(method_name, ' '.join(cols)) import json json_file = './exp_analysis/maxlen_trends.json' with open(json_file, 'w') as fout: json.dump(results, fout) print(f'Write to file {json_file}') def report_auroc_curve(args): datasets = {'xsum': 'XSum', 'writing': 'WritingPrompts'} source_models = {'gpt-3.5-turbo': 'ChatGPT', 'gpt-4': 'GPT-4'} score_models = {'t5-11b': 'T5-11B', 'gpt2-xl': 'GPT-2', 'opt-2.7b': 'OPT-2.7', 'gpt-neo-2.7B': 'Neo-2.7', 'gpt-j-6B': 'GPT-J', 'gpt-neox-20b': 'NeoX'} methods1 = {'roberta-base-openai-detector': 'RoBERTa-base', 'roberta-large-openai-detector': 'RoBERTa-large'} methods2 = {'likelihood': 'Likelihood'} methods3 = {'perturbation_100': 'DetectGPT', 'sampling_discrepancy_analytic': 'Fast-Detect'} def _get_method_fpr_tpr(root_path, dataset, source_model, method, filter=''): maxlen = 180 result_file = f'{root_path}/exp_maxlen{maxlen}/results/{dataset}_{source_model}{filter}.{method}.json' if os.path.exists(result_file): fpr, tpr = get_fpr_tpr(result_file) else: fpr, tpr = [], [] assert len(fpr) == len(tpr) return list(zip(fpr, tpr)) filters2 = {'likelihood': '.gpt-neo-2.7B'} filters3 = {'perturbation_100': '.t5-11b_gpt-neo-2.7B', 'sampling_discrepancy_analytic': '.gpt-j-6B_gpt-neo-2.7B'} # print table per model and dataset results = {} for model in source_models: model_name = source_models[model] for data in datasets: data_name = datasets[data] print('----') print(f'{model_name} / {data_name}') print('----') for method in methods1: method_name = methods1[method] cols = _get_method_fpr_tpr('.', data, model, method) results[f'{model_name}_{data_name}_{method_name}'] = cols cols = [f'({col[0]:.3f},{col[1]:.3f})' for col in cols] print(method_name, ' '.join(cols)) for method in methods2: filter = filters2[method] setting = score_models[filter[1:]] method_name = f'{methods2[method]}({setting})' cols = _get_method_fpr_tpr('.', data, model, method, filter) results[f'{model_name}_{data_name}_{method_name}'] = cols cols = [f'({col[0]:.3f},{col[1]:.3f})' for col in cols] print(method_name, ' '.join(cols)) for method in methods3: filter = filters3[method] setting = [score_models[model] for model in filter[1:].split('_')] method_name = f'{methods3[method]}({setting[0]}/{setting[1]})' cols = _get_method_fpr_tpr('.', data, model, method, filter) results[f'{model_name}_{data_name}_{method_name}'] = cols cols = [f'({col[0]:.3f},{col[1]:.3f})' for col in cols] print(method_name, ' '.join(cols)) import json json_file = './exp_analysis/auroc_curve.json' with open(json_file, 'w') as fout: json.dump(results, fout) print(f'Write to file {json_file}') if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--result_path', type=str, default="./exp_main/results/") parser.add_argument('--report_name', type=str, default="main_results") args = parser.parse_args() if args.report_name == 'main_results': report_main_results(args) elif args.report_name == 'main_ext_results': report_main_ext_results(args) elif args.report_name == 'chatgpt_gpt4_results': report_chatgpt_gpt4_results(args) elif args.report_name == 'gpt3_results': report_gpt3_results(args) elif args.report_name == 'maxlen_trends': report_maxlen_trends(args) elif args.report_name == 'auroc_curve': report_auroc_curve(args) elif args.report_name == 'refmodel_results': report_refmodel_results(args)