res_reg_lmnt_awikner.start_deepthought2_run

  1import numpy as np
  2import os
  3import subprocess
  4import re
  5import time
  6
  7def start_deepthought2_run(system = 'KS', traintype = 'normal', noisetype = 'gaussian', \
  8        noise_realizations = 1, res_size = 500, trainlen = 3000, testlen = 0, rho = 0.5, sigma = 1.0,\
  9        leakage = 1.0, tau = 0.25, win_type = 'full', bias_type = 'old',\
 10        noise_values_array = np.logspace(-3, 0, num = 19, base = 10)[5:11], \
 11        alpha_values = np.append(0., np.logspace(-7, -3, 9)), num_res = 3,\
 12        num_trains = 4, num_tests = 4, metric = 'mss_var', machine = 'deepthought2', \
 13        returnall = False, savepred = False, squarenodes = False, savetime = False, max_valid_time = 500, \
 14        debug = False, num_nodes = 4, cpus_per_node = None, runtime = '2:00:00', \
 15        account = 'physics-hi',debug_part = False, just_process = False, parallel = True,\
 16        res_start = 0, train_start = 0, test_start = 0, \
 17        import_res = False, import_train = False, import_test = False,\
 18        import_noise = False, reg_train_times = None, discard_time = 500, prior = 'zero',\
 19        save_eigenvals = False, pmap = False):
 20
 21    if isinstance(reg_train_times, np.ndarray) or isinstance(reg_train_times, list):
 22        reg_train_times_str = '%d' % reg_train_times[0]
 23        for reg_train_time in reg_train_times[1:]:
 24            reg_train_times_str += ',%d' % reg_train_time
 25    else:
 26        reg_train_times_str = 'None'
 27
 28    noise_values_str = '%e' % noise_values_array[0]
 29    for noise in noise_values_array[1:]:
 30        noise_values_str += ',%e' % noise
 31
 32    reg_values_str = '%e' % alpha_values[0]
 33    for reg in alpha_values[1:]:
 34        reg_values_str += ',%e' % reg
 35
 36    if isinstance(cpus_per_node, int):
 37        cpus_str = '--cpus-per-node=%d' % cpus_per_node
 38    else:
 39        cpus_str = ''
 40
 41    if debug_part:
 42        debug_part_str = '-p debug'
 43    else:
 44        debug_part_str = ''
 45
 46    if savepred:
 47        savepred_str = 'True'
 48    else:
 49        savepred_str = 'False'
 50
 51    if returnall:
 52        returnall_str = 'True'
 53    else:
 54        returnall_str = 'False'
 55    if save_eigenvals:
 56        save_eigenvals_str = 'True'
 57    else:
 58        save_eigenvals_str = 'False'
 59
 60    if debug:
 61        debug_str = 'True'
 62    else:
 63        debug_str ='False'
 64
 65    if savetime:
 66        savetime_str = 'True'
 67    else:
 68        savetime_str = 'False'
 69
 70    if parallel:
 71        parallel_str = 'True'
 72        parallel_str_bash = 'true'
 73    else:
 74        parallel_str = 'False'
 75        parallel_str_bash = 'false'
 76
 77    if squarenodes:
 78        squarenodes_str = 'True'
 79    else:
 80        squarenodes_str = 'False'
 81
 82    if pmap:
 83        pmap_str = 'True'
 84    else:
 85        pmap_str = 'False'
 86
 87    if import_res:
 88        import_res_str = 'True'
 89    else:
 90        import_res_str = 'False'
 91
 92    if import_train:
 93        import_train_str = 'True'
 94    else:
 95        import_train_str = 'False'
 96
 97    if import_test:
 98        import_test_str = 'True'
 99    else:
100        import_test_str = 'False'
101
102    if import_noise:
103        import_noise_str = 'True'
104    else:
105        import_noise_str = 'False'
106
107
108
109    testname = '%s_%s_%s_%d_%dnodes_%dtrain_rho%0.1f_sigma%0.1e_leakage%0.1f_tau%0.3f' % \
110            (system, traintype, noisetype, noise_realizations, res_size, trainlen, rho, sigma, leakage,  tau)
111    options_str = '--savepred=%s --system=%s --noisetype=%s --traintype=%s -r %d --rho=%f --sigma=%f --leakage=%f --win_type=%s --bias_type=%s --tau=%f -N %d -T %d --testtime=%d --res=%d --tests=%d --trains=%d --debug=%s --squarenodes=%s --metric=%s --returnall=%s --savetime=%s --noisevals=%s --regvals=%s --maxvt=%d --machine=%s --parallel=%s --resstart=%d --trainstart=%d --teststart=%d --importres=%s --importtrain=%s --importtest=%s --importnoise=%s --regtraintimes=%s --discardlen=%d --prior=%s --saveeigenvals=%s --pmap=%s' % (savepred_str, system, noisetype, traintype, noise_realizations,  rho,sigma, leakage, win_type, bias_type, tau, res_size, trainlen, testlen, num_res, num_tests, num_trains, debug_str, squarenodes_str, metric, returnall_str, savetime_str, noise_values_str, reg_values_str, max_valid_time, machine, parallel_str, res_start, train_start, test_start, import_res_str, import_train_str, import_test_str, import_noise_str, reg_train_times_str, discard_time, prior,save_eigenvals_str, pmap_str)
112    input_str = 'python slurm-launch.py --ifray %s --exp-name %s --command "python -u reservoir_train_test.py %s" --num-nodes %d %s --load-env "conda activate res39" -t %s -A %s %s' % (parallel_str_bash, testname, options_str, num_nodes, cpus_str, runtime, account, debug_part_str)
113    print(input_str)
114    run_out = subprocess.check_output(input_str, shell=True)
115    time.sleep(1)
116
117    log_file = re.search('log_files/(.*).log', str(run_out))
118    time_str = log_file.group(1)[-11:]
119
120    job_id = str(run_out)[-11:-3]
121    if just_process:
122        os.system('scancel %s' % job_id)
123    template = open('data_scripts/process_test_data.py', 'r')
124    lines = template.readlines()
125    template.close()
126
127    script_name = 'data_scripts/process_test_data_%s.py' % job_id
128    log_name = '%s_%s_process_data' % (testname, time_str)
129    script = open(script_name, 'w')
130    for line in lines:
131        if '#SBATCH -t ' in line and debug_part:
132            debug_part = False
133            script.write('#SBATCH -p debug\n')
134            script.write('#SBATCH -t 15:00\n')
135        elif just_process and '{{JOB_ID}}' in line:
136            pass
137        else:
138            script.write(line.replace('{{JOB_ID}}',job_id).replace('{{JOB_NAME}}', log_name).replace('{{ACCOUNT}}', account))
139    script.close()
140
141    os.system('sbatch %s %s' % (script_name, options_str))
142
143    time.sleep(1)