Created
May 12, 2020 15:25
-
-
Save bilal2vec/ee63a04cd86c5fd45c41dc0b7ce109eb to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| bilal@tf-lm-finetuning:~/lm-finetuning$ python3 train_tfrecords.py --tpu algpt2pod --seq_len 1024 --batch_size 256 --train_len 1000000 --warmup_steps 10000 --model_type gpt2 --config_path gpt2 --epochs 10 --train_path gs://algpt2/train/0.tfrecord --val_path gs://algpt2/train/1.tfrecord | |
| wandb: Tracking run with wandb version 0.8.35 | |
| wandb: Run data is saved locally in wandb/run-20200512_151802-2j4oycre | |
| wandb: Syncing run noble-sunset-1222 | |
| wandb: ⭐️ View project at https://app.wandb.ai/bkkaggle/lm-finetuning | |
| wandb: 🚀 View run at https://app.wandb.ai/bkkaggle/lm-finetuning/runs/2j4oycre | |
| wandb: Run `wandb off` to turn off syncing. | |
| INFO:absl:Entering into master device scope: /job:worker/replica:0/task:0/device:CPU:0 | |
| 2020-05-12 15:18:03.832972: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 AVX512F FMA | |
| 2020-05-12 15:18:03.838986: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2000170000 Hz | |
| 2020-05-12 15:18:03.839674: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x683c750 initialized for platform Host (this does not guarantee that XLA will be used). Devices: | |
| 2020-05-12 15:18:03.839718: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version | |
| 2020-05-12 15:18:03.847103: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:300] Initialize GrpcChannelCache for job worker -> {0 -> 10.44.85.71:8470, 1 -> 10.44.85.76:8470, 2 -> 10.44.85.67:8470, 3 -> 10.44.85.81:8470, 4 -> 10.44.85.70:8470, 5 -> 10.44.85.78:8470, 6 -> 10.44.85.74:8470, 7 -> 10.44.85.75:8470, 8 -> 10.44.85.69:8470, 9 -> 10.44.85.79:8470, 10 -> 10.44.85.72:8470, 11 -> 10.44.85.66:8470, 12 -> 10.44.85.68:8470, 13 -> 10.44.85.77:8470, 14 -> 10.44.85.80:8470, 15 -> 10.44.85.73:8470} | |
| 2020-05-12 15:18:03.847162: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:300] Initialize GrpcChannelCache for job localhost -> {0 -> localhost:44588} | |
| 2020-05-12 15:18:03.869060: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:300] Initialize GrpcChannelCache for job worker -> {0 -> 10.44.85.71:8470, 1 -> 10.44.85.76:8470, 2 -> 10.44.85.67:8470, 3 -> 10.44.85.81:8470, 4 -> 10.44.85.70:8470, 5 -> 10.44.85.78:8470, 6 -> 10.44.85.74:8470, 7 -> 10.44.85.75:8470, 8 -> 10.44.85.69:8470, 9 -> 10.44.85.79:8470, 10 -> 10.44.85.72:8470, 11 -> 10.44.85.66:8470, 12 -> 10.44.85.68:8470, 13 -> 10.44.85.77:8470, 14 -> 10.44.85.80:8470, 15 -> 10.44.85.73:8470} | |
| 2020-05-12 15:18:03.869112: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:300] Initialize GrpcChannelCache for job localhost -> {0 -> localhost:44588} | |
| 2020-05-12 15:18:03.870985: I tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc:390] Started server with target: grpc://localhost:44588 | |
| 2020-05-12 15:18:14.817106: W tensorflow/python/util/util.cc:319] Sets are not currently considered sequences, but this may change in the future, so consider avoiding using them. | |
| Model: "tfgp_t2lm_head_model" | |
| _________________________________________________________________ | |
| Layer (type) Output Shape Param # | |
| ================================================================= | |
| transformer (TFGPT2MainLayer multiple 124439808 | |
| ================================================================= | |
| Total params: 124,439,808 | |
| Trainable params: 124,439,808 | |
| Non-trainable params: 0 | |
| _________________________________________________________________ | |
| Epoch 1/10 | |
| 1/Unknown - 147s 147s/stepsaving model at end of epoch 0 | |
| 1/Unknown - 148s 148s/stepTraceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/keras/engine/training_v2.py", line 128, in run_one_epoch | |
| batch_outs = execution_function(iterator) | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/keras/engine/training_v2_utils.py", line 98, in execution_function | |
| distributed_function(input_fn)) | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/util/nest.py", line 568, in map_structure | |
| structure[0], [func(*x) for x in entries], | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/util/nest.py", line 568, in <listcomp> | |
| structure[0], [func(*x) for x in entries], | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/keras/engine/training_v2_utils.py", line 130, in _non_none_constant_value | |
| constant_value = tensor_util.constant_value(v) | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/framework/tensor_util.py", line 822, in constant_value | |
| return tensor.numpy() | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/framework/ops.py", line 942, in numpy | |
| maybe_arr = self._numpy() # pylint: disable=protected-access | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/framework/ops.py", line 910, in _numpy | |
| six.raise_from(core._status_to_exception(e.code, e.message), None) | |
| File "<string>", line 3, in raise_from | |
| tensorflow.python.framework.errors_impl.OutOfRangeError: 18 root error(s) found. | |
| (0) Out of range: End of sequence | |
| [[{{node IteratorGetNext_86}}]] | |
| Additional GRPC error information: | |
| {"created":"@1589296956.730072126","description":"Error received from peer ipv4:10.44.85.72:8470","file":"third_party/grpc/src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"End of sequence\n\t [[{{node IteratorGetNext_86}}]]","grpc_status":11} | |
| (1) Out of range: End of sequence | |
| [[{{node IteratorGetNext_115}}]] | |
| Additional GRPC error information: | |
| {"created":"@1589296956.732829572","description":"Error received from peer ipv4:10.44.85.80:8470","file":"third_party/grpc/src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"End of sequence\n\t [[{{node IteratorGetNext_115}}]]","grpc_status":11} | |
| (2) Out of range: End of sequence | |
| [[{{node IteratorGetNext_66}}]] | |
| Additional GRPC error information: | |
| {"created":"@1589296956.733323877","description":"Error received from peer ipv4:10.44.85.69:8470","file":"third_party/grpc/src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"End of sequence\n\t [[{{node IteratorGetNext_66}}]]","grpc_status":11} | |
| (3) Out of range: End of sequence | |
| [[{{node IteratorGetNext_71}}]] | |
| Additional GRPC error information: | |
| {"created":"@1589296956.733364564","description":"Error received from peer ipv4:10.44.85.69:8470","file":"third_party/grpc/src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"End of sequence\n\t [[{{node IteratorGetNext_71}}]]","grpc_status":11} | |
| (4) Out of range: End of sequence | |
| [[{{node IteratorGetNext_79}}]] | |
| Additional GRPC error information: | |
| {"created":"@1589296956.737013143","description":"Error received from peer ipv4:10.44.85.79:8470","file":"third_party/grpc/src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"End of sequence\n\t [[{{node IteratorGetNext_79}}]]","grpc_status":11} | |
| (5) Out of range: End of sequence | |
| [[{{node IteratorGetNext_125}}]] | |
| Additional GRPC error information: | |
| {"created":"@1589296956.738007403","description":"Error received from peer ipv4:10.44.85.73:8470","file":"third_party/grpc/src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"End of sequence\n\t [[{{node IteratorGetNext_125}}]]","grpc_status":11} | |
| (6) Out of range: End of sequence | |
| [[{{node IteratorGetNext_20}}]] | |
| Additional GRPC error information: | |
| {"created":"@1589296956.738280380","description":"Error received from peer ipv4:10.44.85.67:8470","file":"third_party/grpc/src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"End of sequence\n\t [[{{node IteratorGetNext_20}}]]","grpc_status":11} | |
| (7) Out of range: End of sequence | |
| [[{{node IteratorGetNext_28}}]] | |
| Additional GRPC error information: | |
| {"created":"@1589296956.740311762","description":"Error received from peer ipv4:10.44.85.81:8470","file":"third_party/grpc/src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"End of sequence\n\t [[{{node IteratorGetNext_28}}]]","grpc_status":11} | |
| (8) Out of range: End of sequence | |
| [[{{node IteratorGetNext_90}}]] | |
| Additional GRPC error information: | |
| {"created":"@1589296956.742798823","descrip ... [truncated] | |
| During handling of the above exception, another exception occurred: | |
| Traceback (most recent call last): | |
| File "train_tfrecords.py", line 218, in <module> | |
| main() | |
| File "train_tfrecords.py", line 214, in main | |
| wandb_callback, checkpoint_callback, lr_callback], initial_epoch=initial_epoch) | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/keras/engine/training.py", line 819, in fit | |
| use_multiprocessing=use_multiprocessing) | |
| File "/home/bilal/.local/lib/python3.7/site-packages/wandb/keras/__init__.py", line 112, in new_v2 | |
| return old_v2(*args, **kwargs) | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/keras/engine/training_v2.py", line 342, in fit | |
| total_epochs=epochs) | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/keras/engine/training_v2.py", line 152, in run_one_epoch | |
| total_epochs * steps_per_epoch)) | |
| TypeError: unsupported operand type(s) for *: 'int' and 'NoneType' | |
| wandb: Waiting for W&B process to finish, PID 14596 | |
| wandb: Program failed with code 1. Press ctrl-c to abort syncing. | |
| wandb: Run summary: | |
| wandb: _timestamp 1589296957.0038688 | |
| wandb: _step 0 | |
| wandb: _runtime 274.9859387874603 | |
| wandb: epoch 0 | |
| wandb: Syncing files in wandb/run-20200512_151802-2j4oycre: | |
| wandb: checkpoint-epoch-1/config.json | |
| wandb: checkpoint-epoch-1/tf_model.h5 | |
| wandb: code/train_tfrecords.py | |
| wandb: final_epoch/config.json | |
| wandb: final_epoch/tf_model.h5 | |
| wandb: plus 8 W&B file(s) and 1 media file(s) | |
| wandb: | |
| wandb: Synced noble-sunset-1222: https://app.wandb.ai/bkkaggle/lm-finetuning/runs/2j4oycre | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f12d30e9d90> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/iterator_ops.py", line 532, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| Exception ignored in: <function _RandomSeedGeneratorDeleter.__del__ at 0x7f12d2cd18c8> | |
| Traceback (most recent call last): | |
| File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/data/ops/dataset_ops.py", line 3462, in __del__ | |
| AttributeError: 'NoneType' object has no attribute 'device' | |
| ^C |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import os | |
| import re | |
| import math | |
| import glob | |
| import time | |
| import argparse | |
| import pickle | |
| import shutil | |
| import numpy as np | |
| from tqdm import tqdm | |
| import tensorflow as tf | |
| from tensorflow import keras | |
| from tensorflow.keras import backend as K | |
| from tensorflow.python.ops import math_ops | |
| from transformers import * | |
| import wandb | |
| from wandb.keras import WandbCallback | |
| from optimizers_tf import * | |
| from detokenizer import wikitext_detokenizer | |
| MODEL_CLASSES = { | |
| 'gpt2': TFGPT2LMHeadModel, | |
| 'algpt2': TFALGPT2LMHeadModel | |
| } | |
| def get_dataset(args): | |
| feature_description = { | |
| 'inputs': tf.io.FixedLenFeature((args.seq_len - 1), tf.int64), | |
| 'labels': tf.io.FixedLenFeature((args.seq_len - 1), tf.int64), | |
| } | |
| def _parse_function(example_proto): | |
| x = tf.io.parse_single_example(example_proto, feature_description) | |
| return (x['inputs'], x['labels']) | |
| train_dataset = tf.data.TFRecordDataset(args.train_path) | |
| train_dataset = train_dataset.map(_parse_function).shuffle( | |
| 1024).batch(args.batch_size, drop_remainder=True).repeat(args.epochs) | |
| val_dataset = tf.data.TFRecordDataset(args.val_path) | |
| val_dataset = val_dataset.map(_parse_function).shuffle( | |
| 1024).batch(args.batch_size, drop_remainder=True).repeat(args.epochs) | |
| return train_dataset, val_dataset | |
| class Checkpoint(tf.keras.callbacks.Callback): | |
| def __init__(self, dir, args, n_batch=0): | |
| super(Checkpoint, self).__init__() | |
| self.dir = dir | |
| self.args = args | |
| self.n_batch = n_batch | |
| def on_batch_end(self, batch, logs=None): | |
| if (self.n_batch + 1) % self.args.log_batches == 0: | |
| wandb.log({'train_batch_loss': logs.get('loss')}, | |
| step=self.n_batch + 1) | |
| if (self.n_batch + 1) % self.args.save_batches == 0: | |
| checkpoint_dir = os.path.join( | |
| self.dir, f'checkpoint-batch-{self.n_batch}') | |
| if not os.path.exists(checkpoint_dir): | |
| os.makedirs(checkpoint_dir) | |
| self.model.save_pretrained(checkpoint_dir) | |
| print(f"saving model at iteration {self.n_batch}") | |
| self.n_batch += 1 | |
| def on_epoch_end(self, epoch, logs=None): | |
| checkpoint_dir = os.path.join( | |
| self.dir, f'checkpoint-epoch-{self.n_batch}') | |
| if not os.path.exists(checkpoint_dir): | |
| os.makedirs(checkpoint_dir) | |
| self.model.save_pretrained(checkpoint_dir) | |
| print(f"saving model at end of epoch {epoch}") | |
| def on_train_end(self, logs=None): | |
| checkpoint_dir = os.path.join(self.dir, 'final_epoch') | |
| if not os.path.exists(checkpoint_dir): | |
| os.makedirs(checkpoint_dir) | |
| self.model.save_pretrained(checkpoint_dir) | |
| def main(): | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument('--tpu', default='grpc://' + | |
| os.environ['COLAB_TPU_ADDR'], required=False) | |
| parser.add_argument('--train_path', nargs='*', | |
| default=['gs://lm-finetuning/wikitext-2/wiki.train.tokens.tfrecord'], required=False) | |
| parser.add_argument('--val_path', nargs='*', | |
| default=['gs://lm-finetuning/wikitext-2/wiki.valid.tokens.tfrecord'], required=False) | |
| parser.add_argument('--train_len', default=100, type=int, required=False) | |
| parser.add_argument('--seq_len', default=256, type=int, required=False) | |
| parser.add_argument('--warmup_steps', default=10000, | |
| type=int, required=False) | |
| parser.add_argument('--config_path', default='./', type=str) | |
| parser.add_argument('--model_type', default='gpt2', type=str) | |
| parser.add_argument('--checkpoint', default=None, type=str) | |
| parser.add_argument('--initial_epoch', default=None, type=int) | |
| parser.add_argument('--optimizer', default='Adafactor', type=str) | |
| parser.add_argument('--lr', default=1e-4, type=float) | |
| parser.add_argument('--momentum', default=0.0, type=float) | |
| parser.add_argument('--relative_update_scale', | |
| default=False, action='store_true') | |
| parser.add_argument('--disable_lr_schedule', | |
| default=False, action='store_true') | |
| parser.add_argument('--batch_size', default=4, type=int) | |
| parser.add_argument('--epochs', default=1, type=int) | |
| parser.add_argument('--save_batches', default=1000, type=int) | |
| parser.add_argument('--log_batches', default=10, type=int) | |
| parser.add_argument('--debug', default=False, action="store_true") | |
| parser.add_argument('--seed', default=42, type=int) | |
| parser.add_argument('--tags', nargs='+') | |
| args = parser.parse_args() | |
| tf.random.set_seed(args.seed) | |
| if args.debug: | |
| import ptvsd | |
| ptvsd.enable_attach(address=('localhost', 5678), | |
| redirect_output=True) | |
| ptvsd.wait_for_attach() | |
| breakpoint() | |
| wandb.login() | |
| wandb.init(project='lm-finetuning', config=args, tags=args.tags) | |
| resolver = tf.distribute.cluster_resolver.TPUClusterResolver(tpu=args.tpu) | |
| tf.config.experimental_connect_to_cluster(resolver) | |
| tf.tpu.experimental.initialize_tpu_system(resolver) | |
| config = AutoConfig.from_pretrained(args.config_path) | |
| model = AutoModelWithLMHead.from_config(config=config) | |
| os.mkdir('./temp') | |
| model.save_pretrained('./temp') | |
| strategy = tf.distribute.experimental.TPUStrategy(resolver) | |
| with strategy.scope(): | |
| model = MODEL_CLASSES[args.model_type] | |
| global_step = 0 | |
| if args.checkpoint: | |
| global_step = int(args.checkpoint.split("-")[-1].split('/')[0]) | |
| print(f'Starting from global step {global_step}') | |
| model = model.from_pretrained(args.checkpoint) | |
| else: | |
| model = model.from_pretrained('./temp', from_pt=True) | |
| model.summary() | |
| shutil.rmtree('./temp') | |
| loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) | |
| if args.optimizer == "SGD": | |
| optimizer = keras.optimizers.SGD(lr=args.lr, momentum=args.momentum) | |
| if args.optimizer == "AdamW": | |
| optimizer = AdamWeightDecay(learning_rate=args.lr) | |
| elif args.optimizer == "Adafactor": | |
| if args.relative_update_scale: | |
| optimizer = AdafactorOptimizer( | |
| beta1=args.momentum, multiply_by_parameter_scale=True) | |
| else: | |
| optimizer = AdafactorOptimizer( | |
| learning_rate=args.lr, beta1=args.momentum) | |
| train_dataset, val_dataset = get_dataset(args) | |
| n_train_steps = (args.train_len // args.batch_size) * args.epochs | |
| wandb_callback = WandbCallback() | |
| checkpoint_callback = Checkpoint(wandb.run.dir, args, global_step) | |
| model.compile(optimizer=optimizer, loss=[ | |
| loss, *[None] * model.config.n_layer]) | |
| initial_epoch = 0 | |
| if args.initial_epoch: | |
| initial_epoch = args.initial_epoch | |
| if args.disable_lr_schedule: | |
| model.fit(train_dataset, validation_data=val_dataset, epochs=args.epochs, callbacks=[ | |
| wandb_callback, checkpoint_callback], initial_epoch=initial_epoch) | |
| else: | |
| lr_callback = WarmUpLinearDecayScheduler( | |
| learning_rate_base=args.lr, total_steps=n_train_steps, warmup_steps=args.warmup_steps, global_step_init=global_step) | |
| model.fit(train_dataset, validation_data=val_dataset, epochs=args.epochs, callbacks=[ | |
| wandb_callback, checkpoint_callback, lr_callback], initial_epoch=initial_epoch) | |
| if __name__ == "__main__": | |
| main() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
To get more tips on the number and recommended size of tfrecord files you can also read "tf.data.Dataset and TFRecords" section of the post:
https://www.kaggle.com/docs/tpu#tpu3
For better data parallelizm you might want to have more than one tfrecord files per TPU.