0

Ray tune上でGPUを用いてハイパーパラメータ最適化を試みましたが、以下のようなエラーに頭を抱えています。

 RuntimeError: No CUDA GPUs are available
(main pid=4099) *** SIGSEGV received at time=1664685800 on cpu 0 ***
(main pid=4099) PC: @     0x7f7999651050  (unknown)  (unknown)
2022-10-02 04:43:20,455 WARNING worker.py:1829 -- A worker died or was killed while executing a task by an unexpected system error. To troubleshoot the problem, check the logs for the dead worker. RayTask ID: ffffffffffffffff7e397495e9840bc1819f011601000000 Worker ID: e9371df84e6c8ca09a2cf2da974ba9e78e9e125beb9488b22dc5a74f Node ID: a898df022b143e3de733f832dfee96aef8385bc6402e8a94da61e9ea Worker IP address: 172.28.0.2 Worker port: 41737 Worker PID: 4099 Worker exit type: SYSTEM_ERROR Worker exit detail: Worker unexpectedly exits with a connection error code 2. End of file. There are some potential root causes. (1) The process is killed by SIGKILL by OOM killer due to high memory usage. (2) ray stop --force is called. (3) The worker is crashed unexpectedly due to SIGSEGV or other unexpected errors.
2022-10-02 04:43:20,456 ERROR trial_runner.py:980 -- Trial main_b7e58_00000: Error processing event.
ray.tune.error._TuneNoNextExecutorEventError: Traceback (most recent call last):
  File "/usr/local/lib/python3.7/dist-packages/ray/tune/execution/ray_trial_executor.py", line 989, in get_next_executor_event
    future_result = ray.get(ready_future)
  File "/usr/local/lib/python3.7/dist-packages/ray/_private/client_mode_hook.py", line 105, in wrapper
    return func(*args, **kwargs)
  File "/usr/local/lib/python3.7/dist-packages/ray/_private/worker.py", line 2277, in get
    raise value
ray.exceptions.RayActorError: The actor died unexpectedly before finishing this task.
    class_name: wrap_function.<locals>.ImplicitFunc
    actor_id: 7e397495e9840bc1819f011601000000
    pid: 4099
    namespace: 8c989dd0-b724-425a-96f7-f4bb2992fe5a
    ip: 172.28.0.2
The actor is dead because its worker process has died. Worker exit type: SYSTEM_ERROR Worker exit detail: Worker unexpectedly exits with a connection error code 2. End of file. There are some potential root causes. (1) The process is killed by SIGKILL by OOM killer due to high memory usage. (2) ray stop --force is called. (3) The worker is crashed unexpectedly due to SIGSEGV or other unexpected errors.

tune.run()ではgpu_per_trialの指定を行なっていて、以下のようにコードを書いています。

def run_search():

  for i in range(len(subj_list)):
    output_dir = '/content/drive/MyDrive/_Results___es_patience20__train_counts1_batch32_lr5e-06_w-decay0.00025'
    subj_dir = output_dir + '/' + subj_list[i]

    config = {
    'lr_init':tune.quniform(1e-8,1e-3,5e-9),
    'weight_decay':tune.qloguniform(1e-4,1e-2,5e-5)
  }

    scheduler = ASHAScheduler(
      metric = 'clip_corr',
      mode = 'max',
      max_t = 5000,
      grace_period = 1 , #学習がうまくいかなくても、1 epochは回す
      reduction_factor = 2
      )
  
    reporter = CLIReporter(
      metric_columns = ['train_loss','train_clip_corr','val_loss','val_clip_corr']
  )

    result = tune.run(main,
                      config = config,
                      num_samples = 1,
                      resources_per_trial = {'cpu':8,'gpu':1},
                      verbose = 3,
                      scheduler = scheduler,
                      local_dir = subj_dir,
                      keep_checkpoints_num = 1, #val_clip_corrの最大化が目標
                      checkpoint_score_attr = 'val_clip_corr',
                      progress_reporter = reporter
                      )
    
    ### extract the best trial run from the search ###
    best_trial = result.get_best_trial(
        'val_clip_corr','max','last'
    )
    
    print('Best trial config :{}'.format(best_trial.config))
    print('Best trial final val_loss : {}'.format(best_trial.last_result['val_loss']))
    print('Best trial final val_clip_corr : {}'.format(best_trial.last_result['val_clip_corr']))

if __name__ == '__main__':
  run_search()

そして以下に表示します、main()ではmodel.cuda(gpu_id)として、gpuの指定を行いました。main()に登場します、transfer_model()という関数では、同じモデルでの以前の学習で得られたパラメータをロードしています。
transfer_model()のセルをmain()の下に添付いたします。ご確認ください。

なぜ、GPUが使用できないか、アイデアがあれば教えていただきたく思います。よろしくお願いします。

    def main(config,
         gpu_id = 0,
         num_epochs = 5000,
         pretrained_type = 'IO',
         pretrained_model = '',
         train_counts =1,
         freeze_layer = ['cnn'],
         overfitting = False,
         early_stopping = False
         ):
  

  seeder(seed)

  output_dir = '/content/drive/MyDrive/_Results___es_patience20__train_counts1_batch32_lr5e-06_w-decay0.00025' 
  print(output_dir)

  for i in range(len(subj_list)):

    pretrained_model = output_dir 
    subj_dir = output_dir + '/' + subj_list[i]

    # get data loader
    train_loader = load_data(scaling = False,
                           downscale_median = True,
                           augmentation = True,
                           train_loader = True)
  
    val_loader = load_data(scaling = False,
                         downscale_median = True,
                         augmentation = True,
                         val_loader = True)
  
    test_loader = load_data(scaling = False,
                         downscale_median = True,
                         augmentation = True,
                         test_loader = True)

    
    

    if pretrained_type == "FT":
      sub_pretrained_model = pretrained_model

    if pretrained_type == "IO":
      sub_pretrained_model = pretrained_model + '/' + subj_list[i]

    
    model = transfer_model(train_counts = 1,
                           pre_model_path = sub_pretrained_model,
                           verbose = True,
                           gpu_id = 0)
    
    #device = os.environ['CUDA_VISIBLE_DEVICES'] = '0'
    model.cuda(gpu_id) 

    for param in model.parameters():
      param.requires_grad = True

    if 'cnn' in freeze_layer:
      for param in model.cnn.parameters():
        param.requires_grad = False

    if 'tdm' in freeze_layer:
      for param in model.tdm.parameters():
        param.requires_grad = False
    
    if 'u_cnn_5' in freeze_layer:
      for idx, param in enumerate(model.cnn.parameters()):
        if idx < 34:
          param.requires_grad = False

    if 'rnn' in freeze_layer:
      for param in model.rnn.parameters():
        param.requires_grad = False

    
    criterion = nn.MSELoss()
    optimizer = optim.Adam(filter(lambda p:p.requires_grad, model.parameters()),
                           lr = config['lr_init'],
                           weight_decay = config['weight_decay'])

    
    if not overfitting:
      es = EarlyStopping(patience=20)


    for epoch in range(num_epochs):

      if epoch == 0:
        pass

      else:
        train_loss, train_clip_corr = train1(train_loader,
                                             model,
                                             criterion,
                                             optimizer,
                                             gpu_id = 0
                                             )
        
        val_loss, val_clip_corr = validate1(val_loader,
                                            model,
                                            gpu_id,
                                            criterion,
                                            corr_w=1.0,
                                            loss_type='MSE&Cosine',
                                            score_metric="spearmanr",
                                            gpu_id = 0)
    
    print('train_loss : {}'.format(train_loss))
    print('train_clip_corr:{}'.format(train_clip_corr))
    print('val_loss :{}'.format(val_loss))
    print('val_clip_corr :{}'.format(val_clip_corr))

    with tune.checkpoint_dir(epoch) as checkpoint_dir:
            path = os.path.join(checkpoint_dir, 'checkpoint')
            torch.save((model.state_dict(), optimizer.state_dict()), path)

    tune.report(
            train_loss = train_loss,
            train_clip_corr = np.mean(train_clip_corr),
            val_loss = val_loss, 
            val_clip_corr = np.mean(val_clip_corr)
        )


def transfer_model(train_counts,
                   pre_model_path,
                   verbose=False,
                   gpu_id = 0):
    """"""

    if train_counts != 0:
        model = CRNN_VGG_BN_3FC_MaxPool(verbose=verbose, 
                                        gpu_id = 1, 
                                        train_counts=train_counts-1)
        model = add_tdm_layer(model, train_counts)

        if train_counts != 1:
            model = add_t_out(model, train_counts-1)
            model.cnn = model.cnn[:-1]
            
    if torch.cuda.is_available():
        model.load_state_dict(torch.load(pre_model_path + '/best_weight.pkl'), 
                                              strict = False)
    else:
      model.load_state_dict(torch.load(pre_model_path + '/best_weight.pkl',
                                       map_location = 'cpu')
                                        ,strict = False)
             
            
        
      
    if train_counts == 0:
        model = CRNN_VGG_BN_3FC_MaxPool(verbose=verbose,
                        gpu_id = 1,
                        train_counts=train_counts)
        
        model.load_state_dict(torch.load(pre_model_path + '/best_weight.pkl'), 
                                              strict = False)
    if verbose:
        print(model)
    
    model.cuda(gpu_id)
    
    return model

0

この質問に回答するには、ログインする必要があります。

のタグが付いた他の質問を参照する。