Last active
December 30, 2024 16:48
-
-
Save segeljakt/d854cf6cc8f11a8e268d69542167a6f4 to your computer and use it in GitHub Desktop.
Additional info
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| srun: Setting --cpu-bind=threads as a default of --threads-per-core use | |
| srun: Following options are mutually exclusive with --hint: --ntasks-per-core, --threads-per-core, -B and --cpu-bind, but more than one set by environment variables. Ignoring SLURM_HINT. | |
| srun: defined options | |
| srun: -------------------- -------------------- | |
| srun: (null) : nid005143 | |
| srun: cpus-per-task : 1 | |
| srun: gpus-per-task : 1 | |
| srun: jobid : 8934799 | |
| srun: job-name : AISPECS | |
| srun: mem : 64G | |
| srun: nodes : 1 | |
| srun: ntasks : 1 | |
| srun: threads-per-core : 1 | |
| srun: verbose : 1 | |
| srun: -------------------- -------------------- | |
| srun: end of defined options | |
| srun: jobid 8934799: nodes(1):`nid005143', cpu counts: 56(x1) | |
| srun: Implicitly setting --exact, because -c/--cpus-per-task given. | |
| srun: CpuBindType=threads | |
| srun: launching StepId=8934799.0 on host nid005143, 1 tasks: 0 | |
| srun: route/default: init: route default plugin loaded | |
| srun: topology/none: init: topology NONE plugin loaded | |
| srun: Node nid005143, 1 tasks started | |
| /usr/bin/bash: warning: setlocale: LC_ALL: cannot change locale (en_US.UTF-8) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] WON'T CONVERT forward /opt/aispecs-venv/lib/python3.12/site-packages/torchvision/models/resnet.py line 284 | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] due to: | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] Traceback (most recent call last): | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/convert_frame.py", line 1064, in __call__ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] result = self._inner_convert( | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/convert_frame.py", line 526, in __call__ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return _compile( | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/convert_frame.py", line 952, in _compile | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] raise InternalTorchDynamoError( | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/convert_frame.py", line 924, in _compile | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] guarded_code = compile_inner(code, one_graph, hooks, transform) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/convert_frame.py", line 666, in compile_inner | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return _compile_inner(code, one_graph, hooks, transform) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_utils_internal.py", line 87, in wrapper_function | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return function(*args, **kwargs) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/convert_frame.py", line 699, in _compile_inner | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] out_code = transform_code_object(code, transform) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/bytecode_transformation.py", line 1322, in transform_code_object | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] transformations(instructions, code_options) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/convert_frame.py", line 219, in _fn | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return fn(*args, **kwargs) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/convert_frame.py", line 634, in transform | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] tracer.run() | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 2796, in run | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] super().run() | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 983, in run | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] while self.step(): | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 895, in step | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] self.dispatch_table[inst.opcode](self, inst) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 582, in wrapper | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return inner_fn(self, inst) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 2279, in CALL | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] self._call(inst) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 2273, in _call | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] self.call_function(fn, args, kwargs) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 830, in call_function | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] self.push(fn.call_function(self, args, kwargs)) # type: ignore[arg-type] | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/functions.py", line 385, in call_function | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return super().call_function(tx, args, kwargs) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/functions.py", line 324, in call_function | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return super().call_function(tx, args, kwargs) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/functions.py", line 111, in call_function | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return tx.inline_user_function_return(self, [*self.self_args(), *args], kwargs) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 836, in inline_user_function_return | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return InliningInstructionTranslator.inline_call(self, fn, args, kwargs) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 3011, in inline_call | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return cls.inline_call_(parent, func, args, kwargs) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 3139, in inline_call_ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] tracer.run() | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 983, in run | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] while self.step(): | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 895, in step | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] self.dispatch_table[inst.opcode](self, inst) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 582, in wrapper | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return inner_fn(self, inst) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 2279, in CALL | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] self._call(inst) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 2273, in _call | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] self.call_function(fn, args, kwargs) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 830, in call_function | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] self.push(fn.call_function(self, args, kwargs)) # type: ignore[arg-type] | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/lazy.py", line 156, in realize_and_forward | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return getattr(self.realize(), name)(*args, **kwargs) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/nn_module.py", line 899, in call_function | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return variables.UserFunctionVariable(fn, source=source).call_function( | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/functions.py", line 324, in call_function | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return super().call_function(tx, args, kwargs) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/functions.py", line 111, in call_function | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return tx.inline_user_function_return(self, [*self.self_args(), *args], kwargs) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 836, in inline_user_function_return | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return InliningInstructionTranslator.inline_call(self, fn, args, kwargs) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 3011, in inline_call | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return cls.inline_call_(parent, func, args, kwargs) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 3139, in inline_call_ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] tracer.run() | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 983, in run | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] while self.step(): | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 895, in step | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] self.dispatch_table[inst.opcode](self, inst) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 1744, in LOAD_ATTR | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] self._load_attr(inst) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 1734, in _load_attr | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] result = BuiltinVariable(getattr).call_function( | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/builtin.py", line 967, in call_function | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return handler(tx, args, kwargs) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/builtin.py", line 848, in builtin_dispatch | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] rv = fn(tx, args, kwargs) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/builtin.py", line 766, in call_self_handler | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] result = self_handler(tx, *args, **kwargs) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/builtin.py", line 1727, in call_getattr | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return obj.var_getattr(tx, name) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/nn_module.py", line 1120, in var_getattr | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return super().var_getattr(tx, name) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/user_defined.py", line 1036, in var_getattr | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] if isinstance( | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/base.py", line 110, in __instancecheck__ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] instance = instance.realize() | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/lazy.py", line 63, in realize | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] self._cache.realize() | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/lazy.py", line 29, in realize | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] self.vt = VariableBuilder(tx, self.source)(self.value) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/builder.py", line 377, in __call__ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] vt = self._wrap(value) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/builder.py", line 543, in _wrap | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return type_dispatch(self, value) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/builder.py", line 1593, in wrap_tensor | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] tensor_variable = wrap_fx_proxy( | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/builder.py", line 2037, in wrap_fx_proxy | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return wrap_fx_proxy_cls(target_cls=TensorVariable, **kwargs) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/builder.py", line 2149, in wrap_fx_proxy_cls | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] example_value = wrap_to_fake_tensor_and_record( | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/builder.py", line 2709, in wrap_to_fake_tensor_and_record | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] fake_e = wrap_fake_exception( | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/utils.py", line 1574, in wrap_fake_exception | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return fn() | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/builder.py", line 2710, in <lambda> | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] lambda: tx.fake_mode.from_tensor( | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_subclasses/fake_tensor.py", line 2238, in from_tensor | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return self.fake_tensor_converter.from_real_tensor( | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_subclasses/fake_tensor.py", line 375, in from_real_tensor | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] out = self.meta_converter( | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_subclasses/meta_utils.py", line 1660, in __call__ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] r = self.meta_tensor( | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_subclasses/meta_utils.py", line 1450, in meta_tensor | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] r = callback( | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_subclasses/fake_tensor.py", line 366, in mk_fake_tensor | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return FakeTensor( | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_subclasses/fake_tensor.py", line 694, in __new__ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] init_gpu_context() | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_subclasses/fake_tensor.py", line 484, in init_gpu_context | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] else torch.zeros(1, device="cuda") | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] torch._dynamo.exc.InternalTorchDynamoError: RuntimeError: HIP error: invalid device function | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] Compile with `TORCH_USE_HIP_DSA` to enable device-side assertions. | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] from user code: | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torchvision/models/resnet.py", line 285, in forward | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return self._forward_impl(x) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torchvision/models/resnet.py", line 268, in _forward_impl | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] x = self.conv1(x) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/nn/modules/conv.py", line 554, in forward | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return self._conv_forward(input, self.weight, self.bias) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] Set TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] Traceback (most recent call last): | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/convert_frame.py", line 1064, in __call__ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] result = self._inner_convert( | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/convert_frame.py", line 526, in __call__ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return _compile( | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/convert_frame.py", line 952, in _compile | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] raise InternalTorchDynamoError( | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/convert_frame.py", line 924, in _compile | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] guarded_code = compile_inner(code, one_graph, hooks, transform) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/convert_frame.py", line 666, in compile_inner | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return _compile_inner(code, one_graph, hooks, transform) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_utils_internal.py", line 87, in wrapper_function | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return function(*args, **kwargs) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/convert_frame.py", line 699, in _compile_inner | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] out_code = transform_code_object(code, transform) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/bytecode_transformation.py", line 1322, in transform_code_object | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] transformations(instructions, code_options) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/convert_frame.py", line 219, in _fn | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return fn(*args, **kwargs) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/convert_frame.py", line 634, in transform | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] tracer.run() | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 2796, in run | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] super().run() | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 983, in run | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] while self.step(): | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 895, in step | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] self.dispatch_table[inst.opcode](self, inst) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 582, in wrapper | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return inner_fn(self, inst) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 2279, in CALL | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] self._call(inst) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 2273, in _call | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] self.call_function(fn, args, kwargs) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 830, in call_function | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] self.push(fn.call_function(self, args, kwargs)) # type: ignore[arg-type] | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/functions.py", line 385, in call_function | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return super().call_function(tx, args, kwargs) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/functions.py", line 324, in call_function | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return super().call_function(tx, args, kwargs) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/functions.py", line 111, in call_function | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return tx.inline_user_function_return(self, [*self.self_args(), *args], kwargs) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 836, in inline_user_function_return | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return InliningInstructionTranslator.inline_call(self, fn, args, kwargs) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 3011, in inline_call | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return cls.inline_call_(parent, func, args, kwargs) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 3139, in inline_call_ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] tracer.run() | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 983, in run | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] while self.step(): | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 895, in step | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] self.dispatch_table[inst.opcode](self, inst) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 582, in wrapper | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return inner_fn(self, inst) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 2279, in CALL | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] self._call(inst) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 2273, in _call | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] self.call_function(fn, args, kwargs) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 830, in call_function | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] self.push(fn.call_function(self, args, kwargs)) # type: ignore[arg-type] | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/lazy.py", line 156, in realize_and_forward | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return getattr(self.realize(), name)(*args, **kwargs) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/nn_module.py", line 899, in call_function | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return variables.UserFunctionVariable(fn, source=source).call_function( | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/functions.py", line 324, in call_function | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return super().call_function(tx, args, kwargs) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/functions.py", line 111, in call_function | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return tx.inline_user_function_return(self, [*self.self_args(), *args], kwargs) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 836, in inline_user_function_return | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return InliningInstructionTranslator.inline_call(self, fn, args, kwargs) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 3011, in inline_call | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return cls.inline_call_(parent, func, args, kwargs) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 3139, in inline_call_ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] tracer.run() | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 983, in run | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] while self.step(): | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 895, in step | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] self.dispatch_table[inst.opcode](self, inst) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 1744, in LOAD_ATTR | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] self._load_attr(inst) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 1734, in _load_attr | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] result = BuiltinVariable(getattr).call_function( | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/builtin.py", line 967, in call_function | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return handler(tx, args, kwargs) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/builtin.py", line 848, in builtin_dispatch | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] rv = fn(tx, args, kwargs) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/builtin.py", line 766, in call_self_handler | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] result = self_handler(tx, *args, **kwargs) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/builtin.py", line 1727, in call_getattr | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return obj.var_getattr(tx, name) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/nn_module.py", line 1120, in var_getattr | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return super().var_getattr(tx, name) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/user_defined.py", line 1036, in var_getattr | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] if isinstance( | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/base.py", line 110, in __instancecheck__ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] instance = instance.realize() | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/lazy.py", line 63, in realize | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] self._cache.realize() | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/lazy.py", line 29, in realize | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] self.vt = VariableBuilder(tx, self.source)(self.value) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/builder.py", line 377, in __call__ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] vt = self._wrap(value) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/builder.py", line 543, in _wrap | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return type_dispatch(self, value) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/builder.py", line 1593, in wrap_tensor | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] tensor_variable = wrap_fx_proxy( | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/builder.py", line 2037, in wrap_fx_proxy | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return wrap_fx_proxy_cls(target_cls=TensorVariable, **kwargs) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/builder.py", line 2149, in wrap_fx_proxy_cls | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] example_value = wrap_to_fake_tensor_and_record( | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/builder.py", line 2709, in wrap_to_fake_tensor_and_record | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] fake_e = wrap_fake_exception( | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/utils.py", line 1574, in wrap_fake_exception | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return fn() | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/builder.py", line 2710, in <lambda> | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] lambda: tx.fake_mode.from_tensor( | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_subclasses/fake_tensor.py", line 2238, in from_tensor | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return self.fake_tensor_converter.from_real_tensor( | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_subclasses/fake_tensor.py", line 375, in from_real_tensor | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] out = self.meta_converter( | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_subclasses/meta_utils.py", line 1660, in __call__ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] r = self.meta_tensor( | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_subclasses/meta_utils.py", line 1450, in meta_tensor | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] r = callback( | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_subclasses/fake_tensor.py", line 366, in mk_fake_tensor | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return FakeTensor( | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_subclasses/fake_tensor.py", line 694, in __new__ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] init_gpu_context() | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_subclasses/fake_tensor.py", line 484, in init_gpu_context | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] else torch.zeros(1, device="cuda") | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] torch._dynamo.exc.InternalTorchDynamoError: RuntimeError: HIP error: invalid device function | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] Compile with `TORCH_USE_HIP_DSA` to enable device-side assertions. | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] from user code: | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torchvision/models/resnet.py", line 285, in forward | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return self._forward_impl(x) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torchvision/models/resnet.py", line 268, in _forward_impl | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] x = self.conv1(x) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/nn/modules/conv.py", line 554, in forward | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] return self._conv_forward(input, self.weight, self.bias) | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] Set TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information | |
| W1230 18:34:46.433000 89448 torch/_dynamo/convert_frame.py:1125] | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] WON'T CONVERT _forward_impl /opt/aispecs-venv/lib/python3.12/site-packages/torchvision/models/resnet.py line 266 | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] due to: | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] Traceback (most recent call last): | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/convert_frame.py", line 1064, in __call__ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] result = self._inner_convert( | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/convert_frame.py", line 526, in __call__ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] return _compile( | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/convert_frame.py", line 952, in _compile | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] raise InternalTorchDynamoError( | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/convert_frame.py", line 924, in _compile | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] guarded_code = compile_inner(code, one_graph, hooks, transform) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/convert_frame.py", line 666, in compile_inner | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] return _compile_inner(code, one_graph, hooks, transform) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_utils_internal.py", line 87, in wrapper_function | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] return function(*args, **kwargs) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/convert_frame.py", line 699, in _compile_inner | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] out_code = transform_code_object(code, transform) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/bytecode_transformation.py", line 1322, in transform_code_object | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] transformations(instructions, code_options) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/convert_frame.py", line 219, in _fn | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] return fn(*args, **kwargs) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/convert_frame.py", line 634, in transform | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] tracer.run() | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 2796, in run | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] super().run() | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 983, in run | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] while self.step) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 895, in step | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] self.dispatch_table[inst.opcode](self, inst) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 582, in wrapper | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] return inner_fn(self, inst) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 2279, in CALL | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] self._call(inst) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 2273, in _call | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] self.call_function(fn, args, kwargs) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 830, in call_function | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] self.push(fn.call_function(self, args, kwargs)) # type: ignore[arg-type] | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/lazy.py", line 156, in realize_and_forward | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] return getattr(self.realize(), name)(*args, **kwargs) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/nn_module.py", line 899, in call_function | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] return variables.UserFunctionVariable(fn, source=source).call_function( | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/functions.py", line 324, in call_function | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] return super().call_function(tx, args, kwargs) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/functions.py", line 111, in call_function | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] return tx.inline_user_function_return(self, [*self.self_args(), *args], kwargs) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 836, in inline_user_function_return | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] return InliningInstructionTranslator.inline_call(self, fn, args, kwargs) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 3011, in inline_call | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] return cls.inline_call_(parent, func, args, kwargs) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 3139, in inline_call_ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] tracer.run() | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 983, in run | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] while self.step(): | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 895, in step | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] self.dispatch_table[inst.opcode](self, inst) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 1744, in LOAD_ATTR | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] self._load_attr(inst) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 1734, in _load_attr | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] result = BuiltinVariable(getattr).call_function( | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/builtin.py", line 967, in call_function | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] return handler(tx, args, kwargs) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/builtin.py", line 848, in builtin_dispatch | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] rv = fn(tx, args, kwargs) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/builtin.py", line 766, in call_self_handler | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] result = self_handler(tx, *args, **kwargs) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/builtin.py", line 1727, in call_getattr | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] return obj.var_getattr(tx, name) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/nn_module.py", line 1120, in var_getattr | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] return super().var_getattr(tx, name) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/user_defined.py", line 1036, in var_getattr | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] if isinstance( | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/base.py", line 110, in __instancecheck__ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] instance = instance.realize() | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/lazy.py", line 63, in realize | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] self._cache.realize() | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/lazy.py", line 29, in realize | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] self.vt = VariableBuilder(tx, self.source)(self.value) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/builder.py", line 377, in __call__ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] vt = self._wrap(value) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/builder.py", line 543, in _wrap | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] return type_dispatch(self, value) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/builder.py", line 1593, in wrap_tensor | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] tensor_variable = wrap_fx_proxy( | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/builder.py", line 2037, in wrap_fx_proxy | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] return wrap_fx_proxy_cls(target_cls=TensorVariable, **kwargs) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/builder.py", line 2149, in wrap_fx_proxy_cls | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] example_value = wrap_to_fake_tensor_and_record( | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/builder.py", line 2709, in wrap_to_fake_tensor_and_record | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] fake_e = wrap_fake_exception( | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/utils.py", line 1574, in wrap_fake_exception | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] return fn() | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/builder.py", line 2710, in <lambda> | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] lambda: tx.fake_mode.from_tensor( | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_subclasses/fake_tensor.py", line 2238, in from_tensor | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] return self.fake_tensor_converter.from_real_tensor( | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_subclasses/fake_tensor.py", line 375, in from_real_tensor | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] out = self.meta_converter( | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_subclasses/meta_utils.py", line 1660, in __call__ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] r = self.meta_tensor( | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_subclasses/meta_utils.py", line 1450, in meta_tensor | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] r = callback( | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_subclasses/fake_tensor.py", line 366, in mk_fake_tensor | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] return FakeTensor( | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_subclasses/fake_tensor.py", line 694, in __new__ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] init_gpu_context() | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_subclasses/fake_tensor.py", line 484, in init_gpu_context | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] else torch.zeros(1, device="cuda") | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] torch._dynamo.exc.InternalTorchDynamoError: RuntimeError: HIP error: invalid device function | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] Compile with `TORCH_USE_HIP_DSA` to enable device-side assertions. | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] from user code: | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torchvision/models/resnet.py", line 268, in _forward_impl | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] x = self.conv1(x) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/nn/modules/conv.py", line 554, in forward | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] return self._conv_forward(input, self.weight, self.bias) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] Set TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] Traceback (most recent call last): | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/convert_frame.py", line 1064, in __call__ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] result = self._inner_convert( | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/convert_frame.py", line 526, in __call__ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] return _compile( | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/convert_frame.py", line 952, in _compile | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] raise InternalTorchDynamoError( | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/convert_frame.py", line 924, in _compile | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] guarded_code = compile_inner(code, one_graph, hooks, transform) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/convert_frame.py", line 666, in compile_inner | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] return _compile_inner(code, one_graph, hooks, transform) | |
| :( | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_utils_internal.py", line 87, in wrapper_function | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] return function(*args, **kwargs) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/convert_frame.py", line 699, in _compile_inner | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] out_code = transform_code_object(code, transform) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/bytecode_transformation.py", line 1322, in transform_code_object | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] transformations(instructions, code_options) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/convert_frame.py", line 219, in _fn | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] return fn(*args, **kwargs) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/convert_frame.py", line 634, in transform | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] tracer.run() | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 2796, in run | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] super().run() | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 983, in run | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] while self.step(): | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 895, in step | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] self.dispatch_table[inst.opcode](self, inst) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 582, in wrapper | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] return inner_fn(self, inst) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 2279, in CALL | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] self._call(inst) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 2273, in _call | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] self.call_function(fn, args, kwargs) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 830, in call_function | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] self.push(fn.call_function(self, args, kwargs)) # type: ignore[arg-type] | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/lazy.py", line 156, in realize_and_forward | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] return getattr(self.realize(), name)(*args, **kwargs) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/nn_module.py", line 899, in call_function | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] return variables.UserFunctionVariable(fn, source=source).call_function( | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/functions.py", line 324, in call_function | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] return super().call_function(tx, args, kwargs) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/functions.py", line 111, in call_function | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] return tx.inline_user_function_return(self, [*self.self_args(), *args], kwargs) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 836, in inline_user_function_return | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] return InliningInstructionTranslator.inline_call(self, fn, args, kwargs) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 3011, in inline_call | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] return cls.inline_call_(parent, func, args, kwargs) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 3139, in inline_call_ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] tracer.run() | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 983, in run | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] while self.step(): | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 895, in step | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] self.dispatch_table[inst.opcode](self, inst) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 1744, in LOAD_ATTR | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] self._load_attr(inst) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py", line 1734, in _load_attr | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] result = BuiltinVariable(getattr).call_function( | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/builtin.py", line 967, in call_function | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] return handler(tx, args, kwargs) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/builtin.py", line 848, in builtin_dispatch | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] rv = fn(tx, args, kwargs) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/builtin.py", line 766, in call_self_handler | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] result = self_handler(tx, *args, **kwargs) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/builtin.py", line 1727, in call_getattr | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] return obj.var_getattr(tx, name) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/nn_module.py", line 1120, in var_getattr | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] return super().var_getattr(tx, name) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/user_defined.py", line 1036, in var_getattr | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] if isinstance( | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/base.py", line 110, in __instancecheck__ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] instance = instance.realize() | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/lazy.py", line 63, in realize | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] self._cache.realize() | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/lazy.py", line 29, in realize | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] self.vt = VariableBuilder(tx, self.source)(self.value) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/builder.py", line 377, in __call__ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] vt = self._wrap(value) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/builder.py", line 543, in _wrap | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] return type_dispatch(self, value) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/builder.py", line 1593, in wrap_tensor | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] tensor_variable = wrap_fx_proxy( | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/builder.py", line 2037, in wrap_fx_proxy | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] return wrap_fx_proxy_cls(target_cls=TensorVariable, **kwargs) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/builder.py", line 2149, in wrap_fx_proxy_cls | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] example_value = wrap_to_fake_tensor_and_record( | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/builder.py", line 2709, in wrap_to_fake_tensor_and_record | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] fake_e = wrap_fake_exception( | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/utils.py", line 1574, in wrap_fake_exception | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] return fn() | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/variables/builder.py", line 2710, in <lambda> | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] lambda: tx.fake_mode.from_tensor( | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_subclasses/fake_tensor.py", line 2238, in from_tensor | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] return self.fake_tensor_converter.from_real_tensor( | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_subclasses/fake_tensor.py", line 375, in from_real_tensor | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] out = self.meta_converter( | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_subclasses/meta_utils.py", line 1660, in __call__ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] r = self.meta_tensor( | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_subclasses/meta_utils.py", line 1450, in meta_tensor | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] r = callback( | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_subclasses/fake_tensor.py", line 366, in mk_fake_tensor | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] return FakeTensor( | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_subclasses/fake_tensor.py", line 694, in __new__ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] init_gpu_context() | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_subclasses/fake_tensor.py", line 484, in init_gpu_context | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] else torch.zeros(1, device="cuda") | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] torch._dynamo.exc.InternalTorchDynamoError: RuntimeError: HIP error: invalid device function | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] Compile with `TORCH_USE_HIP_DSA` to enable device-side assertions. | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] from user code: | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torchvision/models/resnet.py", line 268, in _forward_impl | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] x = self.conv1(x) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/nn/modules/conv.py", line 554, in forward | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] return self._conv_forward(input, self.weight, self.bias) | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] Set TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information | |
| W1230 18:34:46.444000 89448 torch/_dynamo/convert_frame.py:1125] | |
| Traceback (most recent call last): | |
| File "<string>", line 24, in <module> | |
| File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl | |
| return self._call_impl(*args, **kwargs) | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl | |
| return forward_call(*args, **kwargs) | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/_dynamo/eval_frame.py", line 465, in _fn | |
| return fn(*args, **kwargs) | |
| ^^^^^^^^^^^^^^^^^^^ | |
| File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl | |
| return self._call_impl(*args, **kwargs) | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl | |
| return forward_call(*args, **kwargs) | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| File "/opt/aispecs-venv/lib/python3.12/site-packages/torchvision/models/resnet.py", line 285, in forward | |
| return self._forward_impl(x) | |
| ^^^^^^^^^^^^^^^^^^^^^ | |
| File "/opt/aispecs-venv/lib/python3.12/site-packages/torchvision/models/resnet.py", line 269, in _forward_impl | |
| x = self.bn1(x) | |
| ^^^^^^^^^^^ | |
| File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl | |
| return self._call_impl(*args, **kwargs) | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl | |
| return forward_call(*args, **kwargs) | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| File "/opt/aispecs-venv/lib/python3.12/site-packages/torch/nn/modules/batchnorm.py", line 173, in forward | |
| self.num_batches_tracked.add_(1) # type: ignore[has-type] | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| RuntimeError: HIP error: invalid device function | |
| Compile with `TORCH_USE_HIP_DSA` to enable device-side assertions. | |
| srun: Received task exit notification for 1 task of StepId=8934799.0 (status=0x0100). | |
| srun: error: nid005143: task 0: Exited with exit code 1 | |
| srun: Terminating StepId=8934799.0 |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| FROM ubuntu:24.04 | |
| ENV AISPECS_WHEEL_DIR="/opt/aispecs/wheelhouse" | |
| ENV AISPECS_INSTALL_VENV="/opt/aispecs-venv" | |
| # ==== PREPARATION ==== # | |
| ENV PATH="$PATH:/usr/local/cuda/bin" | |
| RUN sed -i 's|http://|http://se.|g' /etc/apt/sources.list | |
| RUN apt update | |
| RUN apt install -y build-essential git python3-full python3-pybind11 python3-pip ninja-build cmake python3-pip-whl wget rsync | |
| # ==== Install ROCm ==== # | |
| ENV AISPECS_ROCM_DEB="https://repo.radeon.com/amdgpu-install/6.2.1/ubuntu/noble/amdgpu-install_6.2.60201-1_all.deb" | |
| ENV AISPECS_ROCM_BASE="/opt/rocm" | |
| ENV ROCM_PATH="/opt/rocm-6.2.1" | |
| ENV PATH="$PATH:$ROCM_PATH/bin" | |
| RUN mkdir $AISPECS_ROCM_BASE | |
| RUN cd $AISPECS_ROCM_BASE; wget $AISPECS_ROCM_DEB | |
| RUN cd $AISPECS_ROCM_BASE; dpkg -i `basename $AISPECS_ROCM_DEB` | |
| RUN apt update | |
| RUN apt install -y amdgpu-dkms rocm | |
| # Get wheels | |
| COPY output/amd/wheelhouse $AISPECS_WHEEL_DIR | |
| # Install wheels + optional packages that are not autoinstalled. Flash attention takes a lot of | |
| # resources to build and is crashing due to out of memory unless the parallelism is significantly | |
| # limited. Furthermore, the pre-built package does not install correctly on ARM, so building from | |
| # source is at this time the only option. | |
| RUN python3 -m venv ${AISPECS_INSTALL_VENV} | |
| ENV AISPECS_INSTALL_VENV_PIP3="${AISPECS_INSTALL_VENV}/bin/pip3" | |
| ENV AISPECS_INSTALL_VENV_PY="${AISPECS_INSTALL_VENV}/bin/python3" | |
| RUN ${AISPECS_INSTALL_VENV_PIP3} install "$AISPECS_WHEEL_DIR"/*.whl | |
| RUN ${AISPECS_INSTALL_VENV_PIP3} install accelerate>=0.26.0 | |
| RUN MAX_JOBS=8 ${AISPECS_INSTALL_VENV_PIP3} install flash-attn --no-build-isolation | |
| # Install the amdsmi Python package | |
| RUN cd /opt/rocm-6.2.1/share/amd_smi; ${AISPECS_INSTALL_VENV_PIP3} install . | |
| RUN apt install -y libsndfile1 | |
| # Get scripts | |
| COPY images/eval/common/scripts /opt/scripts | |
| # Get version | |
| COPY output/version.tmp /opt/aispecs-version | |
| COPY output/dirty.tmp /opt/aispecs-version-dirt | |
| # Default run commands prints versions of installed packages | |
| CMD echo "AISPECS eval container version is $(cat /opt/aispecs-version)"; \ | |
| cat /opt/aispecs-version-dirt |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| FROM ubuntu:24.04 | |
| ENV GPU_VERSION="gfx90a" | |
| RUN echo "Building for AMD GPU version $GPU_VERSION" | |
| ENV AISPECS_WHEEL_DIR="/opt/aispecs/wheelhouse" | |
| ENV AISPECS_SETUP_VENV="/opt/aispecs-build-venv" | |
| ENV AISPECS_INSTALL_VENV="/opt/aispecs-venv" | |
| # ==== PREPARATION ==== # | |
| RUN sed -i 's|http://|http://se.|g' /etc/apt/sources.list | |
| RUN apt update | |
| RUN apt install -y build-essential git python3-full python3-pybind11 python3-pip ninja-build cmake python3-pip-whl wget rsync | |
| RUN mkdir -p $AISPECS_WHEEL_DIR | |
| # ==== Install ROCm ==== # | |
| ENV AISPECS_ROCM_DEB="https://repo.radeon.com/amdgpu-install/6.2.1/ubuntu/noble/amdgpu-install_6.2.60201-1_all.deb" | |
| ENV AISPECS_ROCM_BASE="/opt/rocm" | |
| ENV ROCM_PATH="/opt/rocm-6.2.1" | |
| ENV PATH="$PATH:$ROCM_PATH/bin" | |
| RUN mkdir $AISPECS_ROCM_BASE | |
| RUN cd $AISPECS_ROCM_BASE; wget $AISPECS_ROCM_DEB | |
| RUN cd $AISPECS_ROCM_BASE; dpkg -i `basename $AISPECS_ROCM_DEB` | |
| RUN apt update | |
| RUN apt install -y amdgpu-dkms rocm | |
| # ==== Build and install LLVM ==== # | |
| ENV AISPECS_LLVM_COMMIT="10dc3a8e916d73291269e5e2b82dd22681489aa1" | |
| ENV AISPECS_LLVM_BASE="/opt/llvm" | |
| ENV AISPECS_LLVM_BUILD_DIR=${AISPECS_LLVM_BASE}/build | |
| RUN mkdir $AISPECS_LLVM_BASE | |
| RUN mkdir $AISPECS_LLVM_BASE/build | |
| RUN mkdir $AISPECS_LLVM_BASE/install | |
| RUN cd $AISPECS_LLVM_BASE; git clone https://github.com/llvm/llvm-project.git; | |
| RUN cd $AISPECS_LLVM_BASE/llvm-project; git checkout $AISPECS_LLVM_COMMIT; | |
| RUN cd $AISPECS_LLVM_BUILD_DIR; cmake -G Ninja $AISPECS_LLVM_BASE/llvm-project/llvm -DLLVM_ENABLE_PROJECTS="mlir;clang;lld;llvm" -DLLVM_TARGETS_TO_BUILD="host;NVPTX;AMDGPU" -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_ENABLE_RTTI=ON -DCMAKE_INSTALL_PREFIX=$AISPECS_LLVM_BASE/install -DLLVM_BUILD_UTILS=ON -DLLVM_ENABLE_ZSTD=OFF -DLLVM_INSTALL_UTILS=ON -DLLVM_TARGET_ARCH=X86 -DMLIR_ENABLE_BINDINGS_PYTHON=ON | |
| RUN cd $AISPECS_LLVM_BUILD_DIR; cmake --build . | |
| RUN cd $AISPECS_LLVM_BUILD_DIR; cmake --install . | |
| # ==== Prepare for build of Python wheels ==== # | |
| RUN python3 -m venv ${AISPECS_SETUP_VENV} | |
| ENV AISPECS_SETUP_VENV_PIP3="${AISPECS_SETUP_VENV}/bin/pip3" | |
| RUN ${AISPECS_SETUP_VENV_PIP3} install numpy pyyaml typing_extensions packaging setuptools psutil ninja cmake wheel | |
| # ==== Create Triton wheel ==== # | |
| ENV AISPECS_TRITON_COMMIT="origin/release/3.0.x" | |
| ENV AISPECS_TRITON_BASE="/opt/triton" | |
| RUN mkdir $AISPECS_TRITON_BASE | |
| RUN mkdir $AISPECS_TRITON_BASE/install | |
| RUN mkdir $AISPECS_TRITON_BASE/wheelhouse | |
| RUN cd $AISPECS_TRITON_BASE; git clone https://github.com/openai/triton.git | |
| RUN cd $AISPECS_TRITON_BASE/triton; git checkout $AISPECS_TRITON_COMMIT | |
| RUN cd $AISPECS_TRITON_BASE/triton; LLVM_INCLUDE_DIRS=$AISPECS_LLVM_BUILD_DIR/include LLVM_LIBRARY_DIR=$AISPECS_LLVM_BUILD_DIR/lib TRITON_OFFLINE_BUILD=1 ${AISPECS_SETUP_VENV_PIP3} --verbose wheel --use-pep517 --no-deps -w "$AISPECS_WHEEL_DIR" --no-build-isolation -e python | |
| RUN ${AISPECS_SETUP_VENV_PIP3} install "$AISPECS_WHEEL_DIR"/*.whl | |
| # ==== Create PyTorch wheel ==== # | |
| ENV AISPECS_TORCH_BASE="/opt/torch" | |
| ENV AISPECS_TORCH_COMMIT="a8d6afb" | |
| RUN mkdir $AISPECS_TORCH_BASE | |
| RUN cd $AISPECS_TORCH_BASE; git clone --recursive https://github.com/pytorch/pytorch | |
| RUN cd $AISPECS_TORCH_BASE/pytorch; git checkout $AISPECS_TORCH_COMMIT | |
| RUN cd $AISPECS_TORCH_BASE/pytorch; git submodule sync --recursive | |
| RUN cd $AISPECS_TORCH_BASE/pytorch; git submodule update --init --recursive | |
| RUN cd $AISPECS_TORCH_BASE/pytorch; $AISPECS_SETUP_VENV/bin/python3 tools/amd_build/build_amd.py | |
| RUN cd $AISPECS_TORCH_BASE/pytorch; PYTORCH_ROCM_ARCH=$GPU_VERSION USE_KINETO=0 USE_ROCTRACER=0 ${AISPECS_SETUP_VENV_PIP3} --verbose wheel --use-pep517 --no-deps -w "$AISPECS_WHEEL_DIR" --no-build-isolation -e . | |
| RUN ${AISPECS_SETUP_VENV_PIP3} install "$AISPECS_WHEEL_DIR"/*.whl | |
| # ==== Create TorchVision wheel ==== # | |
| ENV AISPECS_TORCHVISION_COMMIT="48b1edf" | |
| ENV AISPECS_TORCHVISION_BASE="/opt/torchvision" | |
| RUN mkdir $AISPECS_TORCHVISION_BASE | |
| RUN cd $AISPECS_TORCHVISION_BASE; git clone --recursive https://github.com/pytorch/vision | |
| RUN cd $AISPECS_TORCHVISION_BASE/vision; git checkout $AISPECS_TORCHVISION_COMMIT; | |
| RUN cd $AISPECS_TORCHVISION_BASE/vision; FORCE_CUDA=1 ${AISPECS_SETUP_VENV_PIP3} --verbose wheel --use-pep517 --no-deps -w "$AISPECS_WHEEL_DIR" --no-build-isolation -e . | |
| RUN ${AISPECS_SETUP_VENV_PIP3} install "$AISPECS_WHEEL_DIR"/*.whl | |
| # ==== Create vLLM non-wheel-distributed dependencies' wheels ==== # | |
| ENV AISPECS_FLASH_GIT="https://github.com/ROCm/flash-attention.git" | |
| ENV AISPECS_FLASH_COMMIT="main" | |
| ENV AISPECS_FLASH_BASE="/opt/vllm-flash-attn" | |
| RUN mkdir $AISPECS_FLASH_BASE | |
| RUN cd $AISPECS_FLASH_BASE; git clone $AISPECS_FLASH_GIT; cd flash-attention; git checkout $AISPECS_FLASH_COMMIT | |
| RUN cd $AISPECS_FLASH_BASE/flash-attention; \ | |
| export AISPECS_TMP_TORCH_VERSION=`${AISPECS_SETUP_VENV_PIP3} show torch | grep Version | cut -d" " -f2`; \ | |
| sed -i 's|PYTORCH_VERSION = .*|PYTORCH_VERSION = "'"${AISPECS_TMP_TORCH_VERSION}"'"|g' setup.py | |
| RUN cd $AISPECS_FLASH_BASE/flash-attention; git diff > "$AISPECS_WHEEL_DIR/vllm-flash-attn.diff" # Record | |
| RUN cd $AISPECS_FLASH_BASE/flash-attention; GPU_ARCHS=$GPU_VERSION ${AISPECS_SETUP_VENV_PIP3} --verbose wheel --use-pep517 --no-deps -w "$AISPECS_WHEEL_DIR" --no-build-isolation . | |
| ENV AISPECS_XFORMERS_VERSION="926f410b79a23754fbdcca3ddc687dc00701abe7" | |
| RUN HIP_ARCHITECTURES=$GPU_VERSION PYTORCH_ROCM_ARCH=$GPU_VERSION ${AISPECS_SETUP_VENV_PIP3} --verbose wheel --use-pep517 --no-deps -w "$AISPECS_WHEEL_DIR" --no-build-isolation git+https://github.com/facebookresearch/xformers.git@${AISPECS_XFORMERS_VERSION}#egg=xformers | |
| RUN ${AISPECS_SETUP_VENV_PIP3} install "$AISPECS_WHEEL_DIR"/*.whl | |
| # ==== Create vLLM wheel ==== # | |
| ENV AISPECS_VLLM_BASE="/opt/vllm" | |
| ENV AISPECS_VLLM_GIT="https://github.com/vllm-project/vllm" | |
| ENV AISPECS_VLLM_COMMIT="cdca899" | |
| RUN mkdir $AISPECS_VLLM_BASE; | |
| RUN cd $AISPECS_VLLM_BASE; git clone $AISPECS_VLLM_GIT; cd vllm; git checkout $AISPECS_VLLM_COMMIT | |
| # Collect version information and update vLLM requirements | |
| RUN export AISPECS_TMP_TORCH_VERSION=`${AISPECS_SETUP_VENV_PIP3} show torch | grep Version | cut -d" " -f2`; \ | |
| export AISPECS_TMP_TORCHVIS_VERSION=`${AISPECS_SETUP_VENV_PIP3} show torchvision | grep Version | cut -d" " -f2`; \ | |
| export AISPECS_TMP_FLASH_VERSION=`${AISPECS_SETUP_VENV_PIP3} show vllm-flash-attn | grep Version | cut -d" " -f2`; \ | |
| sed -i 's|torch\s*==.*|torch=='${AISPECS_TMP_TORCH_VERSION}'|g' $AISPECS_VLLM_BASE/vllm/requirements-*.txt; \ | |
| sed -i 's|torchvision\s*==.*|torchvision=='${AISPECS_TMP_TORCHVIS_VERSION}'|g' $AISPECS_VLLM_BASE/vllm/requirements-*.txt; \ | |
| sed -i 's|xformers\s*==.*|xformers|g' $AISPECS_VLLM_BASE/vllm/requirements-*.txt; \ | |
| sed -i 's|vllm-flash-attn\s*==.*|vllm-flash-attn=='${AISPECS_TMP_FLASH_VERSION}'|g' $AISPECS_VLLM_BASE/vllm/requirements-*.txt | |
| RUN cd $AISPECS_VLLM_BASE/vllm; git diff > "$AISPECS_WHEEL_DIR/vllm.diff" # Record | |
| RUN ${AISPECS_SETUP_VENV_PIP3} install setuptools_scm | |
| ENV PATH="$AISPECS_SETUP_VENV/bin:$PATH" | |
| # Create wheel | |
| RUN cd $AISPECS_VLLM_BASE/vllm; PYTORCH_ROCM_ARCH=$GPU_VERSION ${AISPECS_SETUP_VENV_PIP3} --verbose wheel --use-pep517 --no-deps -w "$AISPECS_WHEEL_DIR" --no-build-isolation -e . | |
| RUN ${AISPECS_SETUP_VENV_PIP3} install "$AISPECS_WHEEL_DIR"/*.whl | |
| # ==== Testing in fresh Python venv ==== # | |
| RUN ls "$AISPECS_WHEEL_DIR" | |
| # ==== Prepare for build of Python wheels ==== # | |
| RUN python3 -m venv ${AISPECS_INSTALL_VENV} | |
| ENV AISPECS_INSTALL_VENV_PIP3="${AISPECS_INSTALL_VENV}/bin/pip3" | |
| ENV AISPECS_INSTALL_VENV_PY="${AISPECS_INSTALL_VENV}/bin/python3" | |
| # Install wheels + optional packages that are not autoinstalled. Flash attention takes a lot of | |
| # resources to build and is crashing due to out of memory unless the parallelism is significantly | |
| # limited. Furthermore, the pre-built package does not install correctly on ARM, so building from | |
| # source is at this time the only option. | |
| RUN ${AISPECS_INSTALL_VENV_PIP3} install "$AISPECS_WHEEL_DIR"/*.whl | |
| RUN ${AISPECS_INSTALL_VENV_PIP3} install accelerate>=0.26.0 | |
| RUN MAX_JOBS=8 ${AISPECS_INSTALL_VENV_PIP3} install flash-attn --no-build-isolation | |
| RUN apt install -y libsndfile1 | |
| CMD echo "This is the build image. Wheel artifacts should be copied out from $AISPECS_WHEEL_DIR" |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment