begin checks for gpu resources
@@ -9,10 +9,45 @@ def gpu_queue(options):
status = (False, None)
- # !! TODO wait until resources are available
+ # device specified, remove all other devices
+ if 'device' in options['gpu_options']:
+ dev = '/dev/nvidia' + options['gpu_options']['device'] + ':rwm'
+ if 'devices' in options:
+ devices = option['devices']:
+ for device in devices:
+ if any(str.isdigit(char) for char in device):
+ if dev is not device:
+ option['devices'].remove(device)
+
print("gpu queue", str(options))
print("gpu queue", str(GpuUsage()))
+ # TODO overriding until this is working
+ # wait = True
+ wait = False
+
+ while wait:
+ usage = GpuUsage()
+ check = 0
+ # TODO check if GPUs are available for the set restrictions
+ # no restrictions set
+ if len(options['gpu_options']) == 1:
+ check = 3
+ else:
+ # check if dev is available
+ if 'device' in options['gpu_options']:
+ if not usage[options['gpu_options']['device']]['processes']:
+ check += 1
+ else:
+ check += 1
+
+ # TODO mem_mb needed available
+
+ # TODO dedicated
+
+ # TODO check if gpus are available
+ if check == 3:
+ wait = False
try:
d_client = docker.from_env()
options = json.loads(options)
@@ -114,7 +149,7 @@ def file_queue(path, template_path="/vent/"):
options_dict = json.loads(config.get(section, 'settings'))
in_base = directory == '/files'
# process base by default
- process_file = True if in_base else False
+ process_file = in_base
# check if this tool shouldn't process the base by default
if 'process_base' in options_dict:
if options_dict['process_base'] == 'no':
