Generate Fake Hanzi

In this project, Sketch-rnn is used to generate fake Chinese characters which are used in Japan.
Set Colab Runtime Python 3 and use GPU.

Prepare environment

! git clone https://github.com/tensorflow/magenta.git

Cloning into 'magenta'...
remote: Enumerating objects: 12, done.
remote: Counting objects: 100% (12/12), done.
remote: Compressing objects: 100% (12/12), done.
remote: Total 10699 (delta 4), reused 8 (delta 0), pack-reused 10687
Receiving objects: 100% (10699/10699), 30.80 MiB | 27.57 MiB/s, done.
Resolving deltas: 100% (7682/7682), done.

! git clone https://github.com/hardmaru/sketch-rnn-datasets.git

Cloning into 'sketch-rnn-datasets'...
remote: Enumerating objects: 204, done.
remote: Total 204 (delta 0), reused 0 (delta 0), pack-reused 204
Receiving objects: 100% (204/204), 47.21 MiB | 45.01 MiB/s, done.
Resolving deltas: 100% (97/97), done.

!pip install -qU svgwrite

     |████████████████████████████████| 71kB 4.6MB/s 

Using lower version of numpy to avoid strange error.
Need to restart runtime and continue from code below.

import numpy as np
print(np.__version__)
if np.__version__=='1.16.3':
  !pip install numpy==1.16.2

1.16.3
Collecting numpy==1.16.2
  Downloading https://files.pythonhosted.org/packages/35/d5/4f8410ac303e690144f0a0603c4b8fd3b986feb2749c435f7cdbb288f17e/numpy-1.16.2-cp36-cp36m-manylinux1_x86_64.whl (17.3MB)
     |████████████████████████████████| 17.3MB 4.7MB/s 
ERROR: datascience 0.10.6 has requirement folium==0.2.1, but you'll have folium 0.8.3 which is incompatible.
ERROR: albumentations 0.1.12 has requirement imgaug<0.2.7,>=0.2.5, but you'll have imgaug 0.2.8 which is incompatible.
Installing collected packages: numpy
  Found existing installation: numpy 1.16.3
    Uninstalling numpy-1.16.3:
      Successfully uninstalled numpy-1.16.3
Successfully installed numpy-1.16.2

Load dataset

cd /content

/content

import numpy as np
import tensorflow as tf
import random
import os
from IPython.display import SVG, display
import svgwrite
from six.moves import xrange

Use preprocessed kanji dataset.

filename = "sketch-rnn-datasets/kanji/kanji.rdp25.npz"
load_data = np.load(filename, encoding='latin1')
train_set = load_data['train']
valid_set = load_data['valid']
test_set = load_data['test']

print(len(train_set))  
print(len(valid_set))
print(len(test_set))

10358
600
500

def get_bounds(data, factor):
  min_x = 0
  max_x = 0
  min_y = 0
  max_y = 0
    
  abs_x = 0
  abs_y = 0
  for i in xrange(len(data)):
    x = float(data[i,0])/factor
    y = float(data[i,1])/factor
    abs_x += x
    abs_y += y
    min_x = min(min_x, abs_x)
    min_y = min(min_y, abs_y)
    max_x = max(max_x, abs_x)
    max_y = max(max_y, abs_y)
    
  return (min_x, max_x, min_y, max_y)

def draw_strokes(data, factor=0.2, svg_filename = '/tmp/sketch_rnn/svg/sample.svg'):
  tf.gfile.MakeDirs(os.path.dirname(svg_filename))
  min_x, max_x, min_y, max_y = get_bounds(data, factor)
  #print(min_x, max_x, min_y, max_y)
  dims = (50 + max_x - min_x, 50 + max_y - min_y)
  #print(dims)
  dwg = svgwrite.Drawing(svg_filename, size=dims)
  dwg.add(dwg.rect(insert=(0, 0), size=dims,fill='white'))
  lift_pen = 1
  abs_x = 25 - min_x 
  abs_y = 25 - min_y
  p = "M%s,%s " % (abs_x, abs_y)
  command = "m"
  for i in xrange(len(data)):
    if (lift_pen == 1):
      command = "m"
    elif (command != "l"):
      command = "l"
    else:
      command = ""
    x = float(data[i,0])/factor
    y = float(data[i,1])/factor
    lift_pen = data[i, 2]
    p += command+str(x)+","+str(y)+" "
  the_color = "black"
  stroke_width = 1
  dwg.add(dwg.path(p).stroke(the_color,stroke_width).fill("none"))
  dwg.save()
  display(SVG(dwg.tostring()))

def make_grid_svg(s_list, grid_space_y=10.0, grid_space_x=15.0):
  def get_start_and_end(x):
    x = np.array(x)
    x = x[:, 0:2]
    x_start = x[0]
    x_end = x.sum(axis=0)
    x = x.cumsum(axis=0)
    x_max = x.max(axis=0)
    x_min = x.min(axis=0)
    center_loc = (x_max+x_min)*0.5
    return x_start-center_loc, x_end
  x_pos = 0.0
  y_pos = 0.0
  result = [[x_pos, y_pos, 1]]
  for sample in s_list:
    s = sample[0]
    grid_loc = sample[1]
    grid_y = grid_loc[0]*grid_space_y+grid_space_y*0.5
    grid_x = grid_loc[1]*grid_space_x+grid_space_x*0.5
    start_loc, delta_pos = get_start_and_end(s)

    loc_x = start_loc[0]
    loc_y = start_loc[1]
    new_x_pos = grid_x+loc_x
    new_y_pos = grid_y+loc_y
    result.append([new_x_pos-x_pos, new_y_pos-y_pos, 0])

    result += s.tolist()
    result[-1][2] = 1
    x_pos = new_x_pos+delta_pos[0]
    y_pos = new_y_pos+delta_pos[1]
  return np.array(result)

# draw a random example
rdn = random.choice(train_set)
print(rdn[1])
draw_strokes(rdn,factor=1)

[5.51220703 3.61450195 0.        ]

Train

cp -r /content/magenta/magenta/models/sketch_rnn/ /content/

First try: Train Sketch-rnn model using lstm cells as default.

# ! python /content/sketch_rnn/sketch_rnn_train.py --log_root="/content/sketch_rnn/ckpt/" --data_dir="/content/sketch-rnn-datasets/kanji/" --hparams="data_set=[kanji.rdp25.npz]"

WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.
For more information, please see:
  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md
  * https://github.com/tensorflow/addons
If you depend on functionality not listed there, please file an issue.

INFO:tensorflow:sketch-rnn
INFO:tensorflow:Hyperparams:
INFO:tensorflow:data_set = ['kanji.rdp25.npz']
INFO:tensorflow:num_steps = 10000000
INFO:tensorflow:save_every = 500
INFO:tensorflow:max_seq_len = 250
INFO:tensorflow:dec_rnn_size = 512
INFO:tensorflow:dec_model = lstm
INFO:tensorflow:enc_rnn_size = 256
INFO:tensorflow:enc_model = lstm
INFO:tensorflow:z_size = 128
INFO:tensorflow:kl_weight = 0.5
INFO:tensorflow:kl_weight_start = 0.01
INFO:tensorflow:kl_tolerance = 0.2
INFO:tensorflow:batch_size = 100
INFO:tensorflow:grad_clip = 1.0
INFO:tensorflow:num_mixture = 20
INFO:tensorflow:learning_rate = 0.001
INFO:tensorflow:decay_rate = 0.9999
INFO:tensorflow:kl_decay_rate = 0.99995
INFO:tensorflow:min_learning_rate = 1e-05
INFO:tensorflow:use_recurrent_dropout = True
INFO:tensorflow:recurrent_dropout_prob = 0.9
INFO:tensorflow:use_input_dropout = False
INFO:tensorflow:input_dropout_prob = 0.9
INFO:tensorflow:use_output_dropout = False
INFO:tensorflow:output_dropout_prob = 0.9
INFO:tensorflow:random_scale_factor = 0.15
INFO:tensorflow:augment_stroke_prob = 0.1
INFO:tensorflow:conditional = True
INFO:tensorflow:is_training = True
INFO:tensorflow:Loading data files.
INFO:tensorflow:Loaded 10358/600/500 from kanji.rdp25.npz
INFO:tensorflow:Dataset combined: 11458 (10358/600/500), avg len 63
INFO:tensorflow:model_params.max_seq_len 133.
total images <= max_seq_len is 10358
total images <= max_seq_len is 600
total images <= max_seq_len is 500
INFO:tensorflow:normalizing_scale_factor 14.4871.
INFO:tensorflow:Model using gpu.
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.
Instructions for updating:
Colocations handled automatically by placer.
INFO:tensorflow:Input dropout mode = False.
INFO:tensorflow:Output dropout mode = False.
INFO:tensorflow:Recurrent dropout mode = True.
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/magenta/models/sketch_rnn/model.py:100: bidirectional_dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.
Instructions for updating:
Please use `keras.layers.Bidirectional(keras.layers.RNN(cell))`, which is equivalent to this API
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/rnn.py:443: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.
Instructions for updating:
Please use `keras.layers.RNN(cell)`, which is equivalent to this API
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/rnn.py:626: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.cast instead.
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/magenta/models/sketch_rnn/rnn.py:113: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.
Instructions for updating:
Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/magenta/models/sketch_rnn/model.py:266: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Deprecated in favor of operator or tf.math.divide.
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/magenta/models/sketch_rnn/model.py:285: calling reduce_sum_v1 (from tensorflow.python.ops.math_ops) with keep_dims is deprecated and will be removed in a future version.
Instructions for updating:
keep_dims is deprecated, use keepdims instead
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/magenta/models/sketch_rnn/model.py:295: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.
Instructions for updating:

Future major versions of TensorFlow will allow gradients to flow
into the labels input on backprop by default.

See `tf.nn.softmax_cross_entropy_with_logits_v2`.

INFO:tensorflow:Model using gpu.
INFO:tensorflow:Input dropout mode = 0.
INFO:tensorflow:Output dropout mode = 0.
INFO:tensorflow:Recurrent dropout mode = 0.
2019-05-01 22:37:10.523632: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2300000000 Hz
2019-05-01 22:37:10.523932: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x1c6c7e0 executing computations on platform Host. Devices:
2019-05-01 22:37:10.523966: I tensorflow/compiler/xla/service/service.cc:158]   StreamExecutor device (0): <undefined>, <undefined>
2019-05-01 22:37:10.732842: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:998] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-05-01 22:37:10.733379: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x972fc80 executing computations on platform CUDA. Devices:
2019-05-01 22:37:10.733438: I tensorflow/compiler/xla/service/service.cc:158]   StreamExecutor device (0): Tesla T4, Compute Capability 7.5
2019-05-01 22:37:10.733768: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1433] Found device 0 with properties: 
name: Tesla T4 major: 7 minor: 5 memoryClockRate(GHz): 1.59
pciBusID: 0000:00:04.0
totalMemory: 14.73GiB freeMemory: 14.60GiB
2019-05-01 22:37:10.733791: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1512] Adding visible gpu devices: 0
2019-05-01 22:37:12.087844: I tensorflow/core/common_runtime/gpu/gpu_device.cc:984] Device interconnect StreamExecutor with strength 1 edge matrix:
2019-05-01 22:37:12.087907: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990]      0 
2019-05-01 22:37:12.087920: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1003] 0:   N 
2019-05-01 22:37:12.088251: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 14115 MB memory) -> physical GPU (device: 0, name: Tesla T4, pci bus id: 0000:00:04.0, compute capability: 7.5)
INFO:tensorflow:vector_rnn/ENC_RNN/fw/LSTMCell/W_xh:0 (5, 1024) 5120
INFO:tensorflow:vector_rnn/ENC_RNN/fw/LSTMCell/W_hh:0 (256, 1024) 262144
INFO:tensorflow:vector_rnn/ENC_RNN/fw/LSTMCell/bias:0 (1024,) 1024
INFO:tensorflow:vector_rnn/ENC_RNN/bw/LSTMCell/W_xh:0 (5, 1024) 5120
INFO:tensorflow:vector_rnn/ENC_RNN/bw/LSTMCell/W_hh:0 (256, 1024) 262144
INFO:tensorflow:vector_rnn/ENC_RNN/bw/LSTMCell/bias:0 (1024,) 1024
INFO:tensorflow:vector_rnn/ENC_RNN_mu/super_linear_w:0 (512, 128) 65536
INFO:tensorflow:vector_rnn/ENC_RNN_mu/super_linear_b:0 (128,) 128
INFO:tensorflow:vector_rnn/ENC_RNN_sigma/super_linear_w:0 (512, 128) 65536
INFO:tensorflow:vector_rnn/ENC_RNN_sigma/super_linear_b:0 (128,) 128
INFO:tensorflow:vector_rnn/linear/super_linear_w:0 (128, 1024) 131072
INFO:tensorflow:vector_rnn/linear/super_linear_b:0 (1024,) 1024
INFO:tensorflow:vector_rnn/RNN/output_w:0 (512, 123) 62976
INFO:tensorflow:vector_rnn/RNN/output_b:0 (123,) 123
INFO:tensorflow:vector_rnn/RNN/LSTMCell/W_xh:0 (133, 2048) 272384
INFO:tensorflow:vector_rnn/RNN/LSTMCell/W_hh:0 (512, 2048) 1048576
INFO:tensorflow:vector_rnn/RNN/LSTMCell/bias:0 (2048,) 2048
INFO:tensorflow:Total trainable variables 2186107.
2019-05-01 22:37:12.500300: I tensorflow/stream_executor/dso_loader.cc:152] successfully opened CUDA library libcublas.so.10.0 locally
INFO:tensorflow:step: 20, lr: 0.000998, klw: 0.0105, cost: 1.3191, recon: 1.3161, kl: 0.2859, train_time_taken: 8.6223
INFO:tensorflow:step: 40, lr: 0.000996, klw: 0.0110, cost: 1.1023, recon: 1.0909, kl: 1.0447, train_time_taken: 7.2151
INFO:tensorflow:step: 60, lr: 0.000994, klw: 0.0115, cost: 0.9291, recon: 0.9170, kl: 1.0575, train_time_taken: 7.1885
INFO:tensorflow:step: 80, lr: 0.000992, klw: 0.0120, cost: 0.8406, recon: 0.8253, kl: 1.2798, train_time_taken: 7.3627
INFO:tensorflow:step: 100, lr: 0.000990, klw: 0.0124, cost: 0.7909, recon: 0.7744, kl: 1.3224, train_time_taken: 7.5159
INFO:tensorflow:step: 120, lr: 0.000988, klw: 0.0129, cost: 0.7184, recon: 0.7003, kl: 1.3943, train_time_taken: 8.8711
INFO:tensorflow:step: 140, lr: 0.000986, klw: 0.0134, cost: 0.6372, recon: 0.6188, kl: 1.3723, train_time_taken: 7.6897
INFO:tensorflow:step: 160, lr: 0.000984, klw: 0.0139, cost: 0.5516, recon: 0.5304, kl: 1.5248, train_time_taken: 7.0227
INFO:tensorflow:step: 180, lr: 0.000982, klw: 0.0144, cost: 0.4920, recon: 0.4709, kl: 1.4684, train_time_taken: 7.3044
INFO:tensorflow:step: 200, lr: 0.000980, klw: 0.0149, cost: 0.3800, recon: 0.3594, kl: 1.3861, train_time_taken: 7.2839
INFO:tensorflow:step: 220, lr: 0.000978, klw: 0.0154, cost: 0.3336, recon: 0.3121, kl: 1.3998, train_time_taken: 7.3121
INFO:tensorflow:step: 240, lr: 0.000977, klw: 0.0158, cost: 0.3204, recon: 0.3010, kl: 1.2302, train_time_taken: 7.3196
INFO:tensorflow:step: 260, lr: 0.000975, klw: 0.0163, cost: 0.2701, recon: 0.2470, kl: 1.4128, train_time_taken: 7.2318
INFO:tensorflow:step: 280, lr: 0.000973, klw: 0.0168, cost: 0.1893, recon: 0.1653, kl: 1.4304, train_time_taken: 7.2949
INFO:tensorflow:step: 300, lr: 0.000971, klw: 0.0173, cost: 0.1530, recon: 0.1285, kl: 1.4159, train_time_taken: 7.2337
INFO:tensorflow:step: 320, lr: 0.000969, klw: 0.0178, cost: 0.1288, recon: 0.1029, kl: 1.4606, train_time_taken: 7.3294
INFO:tensorflow:step: 340, lr: 0.000967, klw: 0.0183, cost: 0.1463, recon: 0.1203, kl: 1.4271, train_time_taken: 8.5566
INFO:tensorflow:step: 360, lr: 0.000965, klw: 0.0187, cost: 0.0794, recon: 0.0536, kl: 1.3786, train_time_taken: 7.4164
INFO:tensorflow:step: 380, lr: 0.000963, klw: 0.0192, cost: 0.0436, recon: 0.0182, kl: 1.3222, train_time_taken: 7.1288
INFO:tensorflow:step: 400, lr: 0.000961, klw: 0.0197, cost: 0.0335, recon: 0.0056, kl: 1.4138, train_time_taken: 7.4980
INFO:tensorflow:step: 420, lr: 0.000959, klw: 0.0202, cost: 0.0187, recon: -0.0099, kl: 1.4133, train_time_taken: 7.7684
INFO:tensorflow:step: 440, lr: 0.000957, klw: 0.0207, cost: -0.0123, recon: -0.0415, kl: 1.4130, train_time_taken: 7.3933
INFO:tensorflow:step: 460, lr: 0.000955, klw: 0.0211, cost: -0.0362, recon: -0.0665, kl: 1.4370, train_time_taken: 7.3369
INFO:tensorflow:step: 480, lr: 0.000954, klw: 0.0216, cost: -0.0308, recon: -0.0599, kl: 1.3481, train_time_taken: 7.2465
INFO:tensorflow:step: 500, lr: 0.000952, klw: 0.0221, cost: -0.0622, recon: -0.0916, kl: 1.3289, train_time_taken: 7.2462
INFO:tensorflow:best_valid_cost: -0.2362, valid_cost: -0.2362, valid_recon: -0.2496, valid_kl: 1.3422, valid_time_taken: 0.7019
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 500.
INFO:tensorflow:time_taken_save 0.3585.
INFO:tensorflow:eval_cost: -0.2295, eval_recon: -0.2428, eval_kl: 1.3378, eval_time_taken: 0.5153
INFO:tensorflow:step: 520, lr: 0.000950, klw: 0.0226, cost: -0.0703, recon: -0.1003, kl: 1.3274, train_time_taken: 7.4235
INFO:tensorflow:step: 540, lr: 0.000948, klw: 0.0231, cost: -0.0749, recon: -0.1036, kl: 1.2448, train_time_taken: 7.9895
INFO:tensorflow:step: 560, lr: 0.000946, klw: 0.0235, cost: -0.0901, recon: -0.1229, kl: 1.3939, train_time_taken: 8.0771
INFO:tensorflow:step: 580, lr: 0.000944, klw: 0.0240, cost: -0.1046, recon: -0.1357, kl: 1.2964, train_time_taken: 7.4063
INFO:tensorflow:step: 600, lr: 0.000942, klw: 0.0245, cost: -0.1486, recon: -0.1817, kl: 1.3515, train_time_taken: 7.2541
INFO:tensorflow:step: 620, lr: 0.000940, klw: 0.0250, cost: -0.1429, recon: -0.1755, kl: 1.3039, train_time_taken: 7.2065
INFO:tensorflow:step: 640, lr: 0.000939, klw: 0.0254, cost: -0.1666, recon: -0.2013, kl: 1.3637, train_time_taken: 7.2524
INFO:tensorflow:step: 660, lr: 0.000937, klw: 0.0259, cost: -0.1875, recon: -0.2233, kl: 1.3807, train_time_taken: 7.3187
INFO:tensorflow:step: 680, lr: 0.000935, klw: 0.0264, cost: -0.2075, recon: -0.2417, kl: 1.2945, train_time_taken: 7.2507
INFO:tensorflow:step: 700, lr: 0.000933, klw: 0.0269, cost: -0.2134, recon: -0.2482, kl: 1.2977, train_time_taken: 7.2796
INFO:tensorflow:step: 720, lr: 0.000931, klw: 0.0273, cost: -0.2235, recon: -0.2578, kl: 1.2572, train_time_taken: 7.3098
INFO:tensorflow:step: 740, lr: 0.000929, klw: 0.0278, cost: -0.2390, recon: -0.2748, kl: 1.2872, train_time_taken: 7.2677
INFO:tensorflow:step: 760, lr: 0.000928, klw: 0.0283, cost: -0.2175, recon: -0.2518, kl: 1.2152, train_time_taken: 8.0750
INFO:tensorflow:step: 780, lr: 0.000926, klw: 0.0287, cost: -0.2551, recon: -0.2911, kl: 1.2516, train_time_taken: 7.8425
INFO:tensorflow:step: 800, lr: 0.000924, klw: 0.0292, cost: -0.2761, recon: -0.3117, kl: 1.2212, train_time_taken: 7.1962
INFO:tensorflow:step: 820, lr: 0.000922, klw: 0.0297, cost: -0.2932, recon: -0.3278, kl: 1.1628, train_time_taken: 7.2046
INFO:tensorflow:step: 840, lr: 0.000920, klw: 0.0302, cost: -0.2980, recon: -0.3353, kl: 1.2397, train_time_taken: 7.3200
INFO:tensorflow:step: 860, lr: 0.000918, klw: 0.0306, cost: -0.2897, recon: -0.3263, kl: 1.1948, train_time_taken: 7.1510
INFO:tensorflow:step: 880, lr: 0.000917, klw: 0.0311, cost: -0.3229, recon: -0.3606, kl: 1.2125, train_time_taken: 7.2816
INFO:tensorflow:step: 900, lr: 0.000915, klw: 0.0316, cost: -0.3164, recon: -0.3562, kl: 1.2616, train_time_taken: 7.2749
INFO:tensorflow:step: 920, lr: 0.000913, klw: 0.0320, cost: -0.3441, recon: -0.3827, kl: 1.2056, train_time_taken: 7.1965
INFO:tensorflow:step: 940, lr: 0.000911, klw: 0.0325, cost: -0.3512, recon: -0.3900, kl: 1.1942, train_time_taken: 8.0568
INFO:tensorflow:step: 960, lr: 0.000909, klw: 0.0330, cost: -0.3616, recon: -0.4017, kl: 1.2172, train_time_taken: 8.0653
INFO:tensorflow:step: 980, lr: 0.000908, klw: 0.0334, cost: -0.3416, recon: -0.3806, kl: 1.1668, train_time_taken: 8.5533
INFO:tensorflow:step: 1000, lr: 0.000906, klw: 0.0339, cost: -0.3721, recon: -0.4130, kl: 1.2079, train_time_taken: 7.5223
INFO:tensorflow:best_valid_cost: -0.5690, valid_cost: -0.5690, valid_recon: -0.5811, valid_kl: 1.2050, valid_time_taken: 0.6139
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 1000.
INFO:tensorflow:time_taken_save 0.3267.
INFO:tensorflow:eval_cost: -0.5606, eval_recon: -0.5726, eval_kl: 1.1953, eval_time_taken: 0.5171
INFO:tensorflow:step: 1020, lr: 0.000904, klw: 0.0344, cost: -0.3684, recon: -0.4101, kl: 1.2154, train_time_taken: 7.1824
INFO:tensorflow:step: 1040, lr: 0.000902, klw: 0.0348, cost: -0.3824, recon: -0.4226, kl: 1.1552, train_time_taken: 7.1475
INFO:tensorflow:step: 1060, lr: 0.000900, klw: 0.0353, cost: -0.3795, recon: -0.4204, kl: 1.1601, train_time_taken: 7.2026
INFO:tensorflow:step: 1080, lr: 0.000899, klw: 0.0358, cost: -0.3914, recon: -0.4327, kl: 1.1573, train_time_taken: 7.2625
INFO:tensorflow:step: 1100, lr: 0.000897, klw: 0.0362, cost: -0.4239, recon: -0.4655, kl: 1.1484, train_time_taken: 7.2551
INFO:tensorflow:step: 1120, lr: 0.000895, klw: 0.0367, cost: -0.3934, recon: -0.4344, kl: 1.1176, train_time_taken: 7.2839
INFO:tensorflow:step: 1140, lr: 0.000893, klw: 0.0371, cost: -0.4087, recon: -0.4525, kl: 1.1785, train_time_taken: 7.3744
INFO:tensorflow:step: 1160, lr: 0.000892, klw: 0.0376, cost: -0.4248, recon: -0.4679, kl: 1.1449, train_time_taken: 7.2207
INFO:tensorflow:step: 1180, lr: 0.000890, klw: 0.0381, cost: -0.4391, recon: -0.4818, kl: 1.1215, train_time_taken: 7.5310
INFO:tensorflow:step: 1200, lr: 0.000888, klw: 0.0385, cost: -0.4817, recon: -0.5240, kl: 1.0994, train_time_taken: 8.3819
INFO:tensorflow:step: 1220, lr: 0.000886, klw: 0.0390, cost: -0.4209, recon: -0.4655, kl: 1.1443, train_time_taken: 8.0255
INFO:tensorflow:step: 1240, lr: 0.000885, klw: 0.0395, cost: -0.4295, recon: -0.4717, kl: 1.0699, train_time_taken: 7.0900
INFO:tensorflow:step: 1260, lr: 0.000883, klw: 0.0399, cost: -0.4827, recon: -0.5298, kl: 1.1813, train_time_taken: 7.3008
INFO:tensorflow:step: 1280, lr: 0.000881, klw: 0.0404, cost: -0.4985, recon: -0.5424, kl: 1.0863, train_time_taken: 7.3151
INFO:tensorflow:step: 1300, lr: 0.000879, klw: 0.0408, cost: -0.4717, recon: -0.5154, kl: 1.0719, train_time_taken: 7.1923
INFO:tensorflow:step: 1320, lr: 0.000878, klw: 0.0413, cost: -0.4735, recon: -0.5178, kl: 1.0745, train_time_taken: 7.3184
INFO:tensorflow:step: 1340, lr: 0.000876, klw: 0.0418, cost: -0.4976, recon: -0.5426, kl: 1.0787, train_time_taken: 7.3246
INFO:tensorflow:step: 1360, lr: 0.000874, klw: 0.0422, cost: -0.5035, recon: -0.5482, kl: 1.0592, train_time_taken: 7.2877
INFO:tensorflow:step: 1380, lr: 0.000872, klw: 0.0427, cost: -0.5228, recon: -0.5701, kl: 1.1079, train_time_taken: 7.3791
INFO:tensorflow:step: 1400, lr: 0.000871, klw: 0.0431, cost: -0.5568, recon: -0.6054, kl: 1.1259, train_time_taken: 7.9520
INFO:tensorflow:step: 1420, lr: 0.000869, klw: 0.0436, cost: -0.5357, recon: -0.5825, kl: 1.0727, train_time_taken: 8.2137
INFO:tensorflow:step: 1440, lr: 0.000867, klw: 0.0440, cost: -0.5738, recon: -0.6208, kl: 1.0673, train_time_taken: 7.2249
INFO:tensorflow:step: 1460, lr: 0.000866, klw: 0.0445, cost: -0.5583, recon: -0.6044, kl: 1.0361, train_time_taken: 7.2636
INFO:tensorflow:step: 1480, lr: 0.000864, klw: 0.0450, cost: -0.5835, recon: -0.6298, kl: 1.0289, train_time_taken: 7.2441
INFO:tensorflow:step: 1500, lr: 0.000862, klw: 0.0454, cost: -0.5797, recon: -0.6275, kl: 1.0519, train_time_taken: 7.3660
INFO:tensorflow:best_valid_cost: -0.7692, valid_cost: -0.7692, valid_recon: -0.7798, valid_kl: 1.0540, valid_time_taken: 0.6138
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 1500.
INFO:tensorflow:time_taken_save 0.3646.
INFO:tensorflow:eval_cost: -0.7524, eval_recon: -0.7628, eval_kl: 1.0444, eval_time_taken: 0.5143
INFO:tensorflow:step: 1520, lr: 0.000860, klw: 0.0459, cost: -0.5332, recon: -0.5799, kl: 1.0192, train_time_taken: 7.2291
INFO:tensorflow:step: 1540, lr: 0.000859, klw: 0.0463, cost: -0.5350, recon: -0.5821, kl: 1.0174, train_time_taken: 7.2533
INFO:tensorflow:step: 1560, lr: 0.000857, klw: 0.0468, cost: -0.5940, recon: -0.6423, kl: 1.0314, train_time_taken: 7.3756
INFO:tensorflow:step: 1580, lr: 0.000855, klw: 0.0472, cost: -0.6080, recon: -0.6575, kl: 1.0488, train_time_taken: 7.3327
INFO:tensorflow:step: 1600, lr: 0.000854, klw: 0.0477, cost: -0.5755, recon: -0.6233, kl: 1.0016, train_time_taken: 7.2932
INFO:tensorflow:step: 1620, lr: 0.000852, klw: 0.0481, cost: -0.5689, recon: -0.6173, kl: 1.0052, train_time_taken: 8.3413
INFO:tensorflow:step: 1640, lr: 0.000850, klw: 0.0486, cost: -0.5898, recon: -0.6387, kl: 1.0073, train_time_taken: 7.6304
INFO:tensorflow:step: 1660, lr: 0.000849, klw: 0.0490, cost: -0.5660, recon: -0.6152, kl: 1.0032, train_time_taken: 7.2339
INFO:tensorflow:step: 1680, lr: 0.000847, klw: 0.0495, cost: -0.6750, recon: -0.7253, kl: 1.0162, train_time_taken: 7.3440
INFO:tensorflow:step: 1700, lr: 0.000845, klw: 0.0499, cost: -0.5995, recon: -0.6467, kl: 0.9455, train_time_taken: 7.2593
INFO:tensorflow:step: 1720, lr: 0.000844, klw: 0.0504, cost: -0.6171, recon: -0.6670, kl: 0.9912, train_time_taken: 7.3803
INFO:tensorflow:step: 1740, lr: 0.000842, klw: 0.0508, cost: -0.6497, recon: -0.7008, kl: 1.0049, train_time_taken: 7.3227
INFO:tensorflow:step: 1760, lr: 0.000840, klw: 0.0513, cost: -0.6192, recon: -0.6705, kl: 1.0005, train_time_taken: 8.0602
INFO:tensorflow:step: 1780, lr: 0.000839, klw: 0.0517, cost: -0.6565, recon: -0.7070, kl: 0.9750, train_time_taken: 8.3331
INFO:tensorflow:step: 1800, lr: 0.000837, klw: 0.0522, cost: -0.6446, recon: -0.6943, kl: 0.9510, train_time_taken: 7.3881
INFO:tensorflow:step: 1820, lr: 0.000835, klw: 0.0526, cost: -0.6425, recon: -0.6934, kl: 0.9659, train_time_taken: 7.7394
INFO:tensorflow:step: 1840, lr: 0.000834, klw: 0.0531, cost: -0.6297, recon: -0.6792, kl: 0.9337, train_time_taken: 8.3415
INFO:tensorflow:step: 1860, lr: 0.000832, klw: 0.0535, cost: -0.6720, recon: -0.7227, kl: 0.9484, train_time_taken: 7.4161
INFO:tensorflow:step: 1880, lr: 0.000830, klw: 0.0540, cost: -0.6401, recon: -0.6906, kl: 0.9347, train_time_taken: 7.3754
INFO:tensorflow:step: 1900, lr: 0.000829, klw: 0.0544, cost: -0.6570, recon: -0.7079, kl: 0.9346, train_time_taken: 7.2945
INFO:tensorflow:step: 1920, lr: 0.000827, klw: 0.0549, cost: -0.6664, recon: -0.7173, kl: 0.9271, train_time_taken: 7.3517
INFO:tensorflow:step: 1940, lr: 0.000825, klw: 0.0553, cost: -0.6842, recon: -0.7351, kl: 0.9206, train_time_taken: 7.3124
INFO:tensorflow:step: 1960, lr: 0.000824, klw: 0.0557, cost: -0.6918, recon: -0.7440, kl: 0.9350, train_time_taken: 7.3677
INFO:tensorflow:step: 1980, lr: 0.000822, klw: 0.0562, cost: -0.6266, recon: -0.6775, kl: 0.9067, train_time_taken: 7.2828
INFO:tensorflow:step: 2000, lr: 0.000821, klw: 0.0566, cost: -0.6824, recon: -0.7367, kl: 0.9595, train_time_taken: 7.4510
INFO:tensorflow:best_valid_cost: -0.9203, valid_cost: -0.9203, valid_recon: -0.9300, valid_kl: 0.9675, valid_time_taken: 0.6144
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 2000.
INFO:tensorflow:time_taken_save 0.3646.
INFO:tensorflow:eval_cost: -0.9046, eval_recon: -0.9142, eval_kl: 0.9571, eval_time_taken: 0.5192
INFO:tensorflow:step: 2020, lr: 0.000819, klw: 0.0571, cost: -0.6867, recon: -0.7383, kl: 0.9043, train_time_taken: 7.6763
INFO:tensorflow:step: 2040, lr: 0.000817, klw: 0.0575, cost: -0.7328, recon: -0.7860, kl: 0.9237, train_time_taken: 8.6493
INFO:tensorflow:step: 2060, lr: 0.000816, klw: 0.0580, cost: -0.6965, recon: -0.7488, kl: 0.9025, train_time_taken: 7.6559
INFO:tensorflow:step: 2080, lr: 0.000814, klw: 0.0584, cost: -0.6991, recon: -0.7528, kl: 0.9185, train_time_taken: 7.2103
INFO:tensorflow:step: 2100, lr: 0.000812, klw: 0.0588, cost: -0.7053, recon: -0.7585, kl: 0.9045, train_time_taken: 7.3142
INFO:tensorflow:step: 2120, lr: 0.000811, klw: 0.0593, cost: -0.6842, recon: -0.7369, kl: 0.8896, train_time_taken: 7.3014
INFO:tensorflow:step: 2140, lr: 0.000809, klw: 0.0597, cost: -0.7594, recon: -0.8127, kl: 0.8926, train_time_taken: 7.3272
INFO:tensorflow:step: 2160, lr: 0.000808, klw: 0.0602, cost: -0.6830, recon: -0.7365, kl: 0.8902, train_time_taken: 7.3676
INFO:tensorflow:step: 2180, lr: 0.000806, klw: 0.0606, cost: -0.7459, recon: -0.8002, kl: 0.8966, train_time_taken: 7.2042
INFO:tensorflow:step: 2200, lr: 0.000804, klw: 0.0610, cost: -0.7794, recon: -0.8349, kl: 0.9090, train_time_taken: 7.3856
INFO:tensorflow:step: 2220, lr: 0.000803, klw: 0.0615, cost: -0.7605, recon: -0.8140, kl: 0.8711, train_time_taken: 7.2313
INFO:tensorflow:step: 2240, lr: 0.000801, klw: 0.0619, cost: -0.6437, recon: -0.6959, kl: 0.8433, train_time_taken: 7.4211
INFO:tensorflow:step: 2260, lr: 0.000800, klw: 0.0624, cost: -0.7529, recon: -0.8073, kl: 0.8722, train_time_taken: 8.5320
INFO:tensorflow:step: 2280, lr: 0.000798, klw: 0.0628, cost: -0.7422, recon: -0.7972, kl: 0.8749, train_time_taken: 7.3646
INFO:tensorflow:step: 2300, lr: 0.000797, klw: 0.0632, cost: -0.7393, recon: -0.7925, kl: 0.8415, train_time_taken: 7.3675
INFO:tensorflow:step: 2320, lr: 0.000795, klw: 0.0637, cost: -0.7474, recon: -0.8019, kl: 0.8561, train_time_taken: 7.2676
INFO:tensorflow:step: 2340, lr: 0.000793, klw: 0.0641, cost: -0.7218, recon: -0.7754, kl: 0.8361, train_time_taken: 7.1975
INFO:tensorflow:step: 2360, lr: 0.000792, klw: 0.0645, cost: -0.6918, recon: -0.7459, kl: 0.8376, train_time_taken: 7.4149
INFO:tensorflow:step: 2380, lr: 0.000790, klw: 0.0650, cost: -0.8111, recon: -0.8678, kl: 0.8735, train_time_taken: 7.2528
INFO:tensorflow:step: 2400, lr: 0.000789, klw: 0.0654, cost: -0.7738, recon: -0.8293, kl: 0.8488, train_time_taken: 7.1398
INFO:tensorflow:step: 2420, lr: 0.000787, klw: 0.0658, cost: -0.7652, recon: -0.8193, kl: 0.8225, train_time_taken: 7.4203
INFO:tensorflow:step: 2440, lr: 0.000786, klw: 0.0663, cost: -0.7640, recon: -0.8188, kl: 0.8272, train_time_taken: 7.3736
INFO:tensorflow:step: 2460, lr: 0.000784, klw: 0.0667, cost: -0.7318, recon: -0.7855, kl: 0.8036, train_time_taken: 7.6523
INFO:tensorflow:step: 2480, lr: 0.000783, klw: 0.0671, cost: -0.7626, recon: -0.8177, kl: 0.8214, train_time_taken: 8.4838
INFO:tensorflow:step: 2500, lr: 0.000781, klw: 0.0676, cost: -0.7946, recon: -0.8505, kl: 0.8268, train_time_taken: 7.3747
INFO:tensorflow:best_valid_cost: -1.0089, valid_cost: -1.0089, valid_recon: -1.0173, valid_kl: 0.8435, valid_time_taken: 0.6357
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 2500.
INFO:tensorflow:time_taken_save 0.3722.
INFO:tensorflow:eval_cost: -0.9892, eval_recon: -0.9976, eval_kl: 0.8344, eval_time_taken: 0.5135
INFO:tensorflow:step: 2520, lr: 0.000779, klw: 0.0680, cost: -0.7688, recon: -0.8233, kl: 0.8021, train_time_taken: 7.4201
INFO:tensorflow:step: 2540, lr: 0.000778, klw: 0.0684, cost: -0.7886, recon: -0.8446, kl: 0.8187, train_time_taken: 7.3707
INFO:tensorflow:step: 2560, lr: 0.000776, klw: 0.0689, cost: -0.7787, recon: -0.8347, kl: 0.8138, train_time_taken: 7.3878
INFO:tensorflow:step: 2580, lr: 0.000775, klw: 0.0693, cost: -0.8331, recon: -0.8901, kl: 0.8232, train_time_taken: 8.1964
INFO:tensorflow:step: 2600, lr: 0.000773, klw: 0.0697, cost: -0.8312, recon: -0.8894, kl: 0.8344, train_time_taken: 8.2636
INFO:tensorflow:step: 2620, lr: 0.000772, klw: 0.0702, cost: -0.8042, recon: -0.8593, kl: 0.7862, train_time_taken: 7.3198
INFO:tensorflow:step: 2640, lr: 0.000770, klw: 0.0706, cost: -0.8797, recon: -0.9375, kl: 0.8191, train_time_taken: 7.2750
INFO:tensorflow:step: 2660, lr: 0.000769, klw: 0.0710, cost: -0.8737, recon: -0.9324, kl: 0.8255, train_time_taken: 7.4157
INFO:tensorflow:step: 2680, lr: 0.000767, klw: 0.0715, cost: -0.7686, recon: -0.8234, kl: 0.7667, train_time_taken: 8.3326
INFO:tensorflow:step: 2700, lr: 0.000766, klw: 0.0719, cost: -0.8280, recon: -0.8869, kl: 0.8199, train_time_taken: 7.6371
INFO:tensorflow:step: 2720, lr: 0.000764, klw: 0.0723, cost: -0.8150, recon: -0.8711, kl: 0.7751, train_time_taken: 7.3972
INFO:tensorflow:step: 2740, lr: 0.000763, klw: 0.0727, cost: -0.8407, recon: -0.8972, kl: 0.7776, train_time_taken: 7.3795
INFO:tensorflow:step: 2760, lr: 0.000761, klw: 0.0732, cost: -0.8666, recon: -0.9240, kl: 0.7842, train_time_taken: 7.3613
INFO:tensorflow:step: 2780, lr: 0.000760, klw: 0.0736, cost: -0.8413, recon: -0.8994, kl: 0.7892, train_time_taken: 7.3905
INFO:tensorflow:step: 2800, lr: 0.000758, klw: 0.0740, cost: -0.8807, recon: -0.9375, kl: 0.7675, train_time_taken: 7.2915
INFO:tensorflow:step: 2820, lr: 0.000757, klw: 0.0744, cost: -0.9019, recon: -0.9590, kl: 0.7670, train_time_taken: 7.3814
INFO:tensorflow:step: 2840, lr: 0.000755, klw: 0.0749, cost: -0.8212, recon: -0.8773, kl: 0.7493, train_time_taken: 8.0952
INFO:tensorflow:step: 2860, lr: 0.000754, klw: 0.0753, cost: -0.8548, recon: -0.9120, kl: 0.7603, train_time_taken: 7.4026
INFO:tensorflow:step: 2880, lr: 0.000752, klw: 0.0757, cost: -0.8002, recon: -0.8572, kl: 0.7523, train_time_taken: 7.6567
INFO:tensorflow:step: 2900, lr: 0.000751, klw: 0.0761, cost: -0.8678, recon: -0.9266, kl: 0.7726, train_time_taken: 8.3955
INFO:tensorflow:step: 2920, lr: 0.000749, klw: 0.0766, cost: -0.8395, recon: -0.8955, kl: 0.7315, train_time_taken: 7.3326
INFO:tensorflow:step: 2940, lr: 0.000748, klw: 0.0770, cost: -0.7792, recon: -0.8349, kl: 0.7239, train_time_taken: 7.4503
INFO:tensorflow:step: 2960, lr: 0.000746, klw: 0.0774, cost: -0.8376, recon: -0.8948, kl: 0.7388, train_time_taken: 7.3802
INFO:tensorflow:step: 2980, lr: 0.000745, klw: 0.0778, cost: -0.8726, recon: -0.9294, kl: 0.7300, train_time_taken: 7.4435
INFO:tensorflow:step: 3000, lr: 0.000743, klw: 0.0783, cost: -0.8243, recon: -0.8814, kl: 0.7296, train_time_taken: 7.3589
INFO:tensorflow:best_valid_cost: -1.0895, valid_cost: -1.0895, valid_recon: -1.0971, valid_kl: 0.7612, valid_time_taken: 0.6211
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 3000.
INFO:tensorflow:time_taken_save 0.3893.
INFO:tensorflow:eval_cost: -1.0700, eval_recon: -1.0775, eval_kl: 0.7519, eval_time_taken: 0.5077
INFO:tensorflow:step: 3020, lr: 0.000742, klw: 0.0787, cost: -0.8432, recon: -0.9003, kl: 0.7253, train_time_taken: 7.4185
INFO:tensorflow:step: 3040, lr: 0.000740, klw: 0.0791, cost: -0.9052, recon: -0.9648, kl: 0.7541, train_time_taken: 7.4497
INFO:tensorflow:step: 3060, lr: 0.000739, klw: 0.0795, cost: -0.9153, recon: -0.9745, kl: 0.7445, train_time_taken: 7.2914
INFO:tensorflow:step: 3080, lr: 0.000738, klw: 0.0799, cost: -0.9083, recon: -0.9672, kl: 0.7364, train_time_taken: 7.3581
INFO:tensorflow:step: 3100, lr: 0.000736, klw: 0.0804, cost: -0.8468, recon: -0.9051, kl: 0.7253, train_time_taken: 8.4140
INFO:tensorflow:step: 3120, lr: 0.000735, klw: 0.0808, cost: -0.8525, recon: -0.9110, kl: 0.7241, train_time_taken: 7.8176
INFO:tensorflow:step: 3140, lr: 0.000733, klw: 0.0812, cost: -0.8847, recon: -0.9434, kl: 0.7227, train_time_taken: 7.3041
INFO:tensorflow:step: 3160, lr: 0.000732, klw: 0.0816, cost: -0.9074, recon: -0.9666, kl: 0.7256, train_time_taken: 7.4787
INFO:tensorflow:step: 3180, lr: 0.000730, klw: 0.0820, cost: -0.8589, recon: -0.9155, kl: 0.6901, train_time_taken: 7.5465
INFO:tensorflow:step: 3200, lr: 0.000729, klw: 0.0825, cost: -0.9414, recon: -1.0009, kl: 0.7224, train_time_taken: 7.3599
INFO:tensorflow:step: 3220, lr: 0.000727, klw: 0.0829, cost: -0.9158, recon: -0.9753, kl: 0.7187, train_time_taken: 7.5643
INFO:tensorflow:step: 3240, lr: 0.000726, klw: 0.0833, cost: -0.8894, recon: -0.9479, kl: 0.7028, train_time_taken: 7.3698
INFO:tensorflow:step: 3260, lr: 0.000725, klw: 0.0837, cost: -0.9201, recon: -0.9791, kl: 0.7046, train_time_taken: 7.4145
INFO:tensorflow:step: 3280, lr: 0.000723, klw: 0.0841, cost: -0.9235, recon: -0.9812, kl: 0.6858, train_time_taken: 7.3529
INFO:tensorflow:step: 3300, lr: 0.000722, klw: 0.0845, cost: -0.8908, recon: -0.9496, kl: 0.6961, train_time_taken: 7.6967
INFO:tensorflow:step: 3320, lr: 0.000720, klw: 0.0849, cost: -0.9816, recon: -1.0441, kl: 0.7360, train_time_taken: 8.4029
INFO:tensorflow:step: 3340, lr: 0.000719, klw: 0.0854, cost: -0.9033, recon: -0.9623, kl: 0.6913, train_time_taken: 7.4428
INFO:tensorflow:step: 3360, lr: 0.000717, klw: 0.0858, cost: -0.8645, recon: -0.9215, kl: 0.6640, train_time_taken: 7.4145
INFO:tensorflow:step: 3380, lr: 0.000716, klw: 0.0862, cost: -0.9437, recon: -1.0040, kl: 0.6986, train_time_taken: 7.6340
INFO:tensorflow:step: 3400, lr: 0.000715, klw: 0.0866, cost: -0.9673, recon: -1.0279, kl: 0.6991, train_time_taken: 8.6053
INFO:tensorflow:step: 3420, lr: 0.000713, klw: 0.0870, cost: -0.9094, recon: -0.9688, kl: 0.6820, train_time_taken: 7.9053
INFO:tensorflow:step: 3440, lr: 0.000712, klw: 0.0874, cost: -0.9028, recon: -0.9621, kl: 0.6789, train_time_taken: 7.4149
INFO:tensorflow:step: 3460, lr: 0.000710, klw: 0.0878, cost: -0.9308, recon: -0.9908, kl: 0.6836, train_time_taken: 7.3785
INFO:tensorflow:step: 3480, lr: 0.000709, klw: 0.0883, cost: -0.9508, recon: -1.0119, kl: 0.6928, train_time_taken: 7.3682
INFO:tensorflow:step: 3500, lr: 0.000708, klw: 0.0887, cost: -0.9211, recon: -0.9817, kl: 0.6834, train_time_taken: 7.3646
INFO:tensorflow:best_valid_cost: -1.1565, valid_cost: -1.1565, valid_recon: -1.1635, valid_kl: 0.7013, valid_time_taken: 0.6373
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 3500.
INFO:tensorflow:time_taken_save 0.3898.
INFO:tensorflow:eval_cost: -1.1348, eval_recon: -1.1417, eval_kl: 0.6912, eval_time_taken: 0.5149
INFO:tensorflow:step: 3520, lr: 0.000706, klw: 0.0891, cost: -0.9307, recon: -0.9914, kl: 0.6816, train_time_taken: 8.5186
INFO:tensorflow:step: 3540, lr: 0.000705, klw: 0.0895, cost: -0.9565, recon: -1.0171, kl: 0.6774, train_time_taken: 7.6756
INFO:tensorflow:step: 3560, lr: 0.000703, klw: 0.0899, cost: -0.9152, recon: -0.9771, kl: 0.6882, train_time_taken: 7.4259
INFO:tensorflow:step: 3580, lr: 0.000702, klw: 0.0903, cost: -0.9371, recon: -0.9995, kl: 0.6909, train_time_taken: 7.3420
INFO:tensorflow:step: 3600, lr: 0.000701, klw: 0.0907, cost: -0.9808, recon: -1.0411, kl: 0.6653, train_time_taken: 7.4102
INFO:tensorflow:step: 3620, lr: 0.000699, klw: 0.0911, cost: -0.9580, recon: -1.0194, kl: 0.6742, train_time_taken: 7.6096
INFO:tensorflow:step: 3640, lr: 0.000698, klw: 0.0915, cost: -0.9387, recon: -1.0007, kl: 0.6775, train_time_taken: 8.0278
INFO:tensorflow:step: 3660, lr: 0.000697, klw: 0.0919, cost: -0.9157, recon: -0.9755, kl: 0.6502, train_time_taken: 7.4238
INFO:tensorflow:step: 3680, lr: 0.000695, klw: 0.0924, cost: -0.9036, recon: -0.9647, kl: 0.6617, train_time_taken: 7.4705
INFO:tensorflow:step: 3700, lr: 0.000694, klw: 0.0928, cost: -0.9893, recon: -1.0514, kl: 0.6705, train_time_taken: 7.3837
INFO:tensorflow:step: 3720, lr: 0.000692, klw: 0.0932, cost: -0.8960, recon: -0.9559, kl: 0.6426, train_time_taken: 7.8334
INFO:tensorflow:step: 3740, lr: 0.000691, klw: 0.0936, cost: -0.9352, recon: -0.9959, kl: 0.6487, train_time_taken: 8.2326
INFO:tensorflow:step: 3760, lr: 0.000690, klw: 0.0940, cost: -0.9479, recon: -1.0088, kl: 0.6485, train_time_taken: 7.3154
INFO:tensorflow:step: 3780, lr: 0.000688, klw: 0.0944, cost: -1.0072, recon: -1.0704, kl: 0.6692, train_time_taken: 7.2466
INFO:tensorflow:step: 3800, lr: 0.000687, klw: 0.0948, cost: -0.9719, recon: -1.0319, kl: 0.6338, train_time_taken: 7.4232
INFO:tensorflow:step: 3820, lr: 0.000686, klw: 0.0952, cost: -0.9905, recon: -1.0550, kl: 0.6775, train_time_taken: 7.3918
INFO:tensorflow:step: 3840, lr: 0.000684, klw: 0.0956, cost: -0.9332, recon: -0.9944, kl: 0.6404, train_time_taken: 7.4053
INFO:tensorflow:step: 3860, lr: 0.000683, klw: 0.0960, cost: -0.9809, recon: -1.0439, kl: 0.6564, train_time_taken: 7.3856
INFO:tensorflow:step: 3880, lr: 0.000682, klw: 0.0964, cost: -0.9247, recon: -0.9856, kl: 0.6311, train_time_taken: 7.3458
INFO:tensorflow:step: 3900, lr: 0.000680, klw: 0.0968, cost: -0.9674, recon: -1.0298, kl: 0.6443, train_time_taken: 7.4334
INFO:tensorflow:step: 3920, lr: 0.000679, klw: 0.0972, cost: -0.9473, recon: -1.0072, kl: 0.6166, train_time_taken: 7.5043
INFO:tensorflow:step: 3940, lr: 0.000678, klw: 0.0976, cost: -1.0115, recon: -1.0755, kl: 0.6558, train_time_taken: 8.1960
INFO:tensorflow:step: 3960, lr: 0.000676, klw: 0.0980, cost: -0.9420, recon: -1.0042, kl: 0.6349, train_time_taken: 8.0267
INFO:tensorflow:step: 3980, lr: 0.000675, klw: 0.0984, cost: -0.9919, recon: -1.0569, kl: 0.6607, train_time_taken: 7.3754
INFO:tensorflow:step: 4000, lr: 0.000674, klw: 0.0988, cost: -0.9864, recon: -1.0489, kl: 0.6320, train_time_taken: 7.4570
INFO:tensorflow:best_valid_cost: -1.2026, valid_cost: -1.2026, valid_recon: -1.2091, valid_kl: 0.6468, valid_time_taken: 0.6423
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 4000.
INFO:tensorflow:time_taken_save 0.5492.
INFO:tensorflow:eval_cost: -1.1804, eval_recon: -1.1868, eval_kl: 0.6368, eval_time_taken: 0.5319
INFO:tensorflow:step: 4020, lr: 0.000672, klw: 0.0992, cost: -0.9461, recon: -1.0085, kl: 0.6286, train_time_taken: 7.4367
INFO:tensorflow:step: 4040, lr: 0.000671, klw: 0.0996, cost: -0.9861, recon: -1.0495, kl: 0.6368, train_time_taken: 7.3395
INFO:tensorflow:step: 4060, lr: 0.000670, klw: 0.1000, cost: -0.9206, recon: -0.9821, kl: 0.6150, train_time_taken: 7.4457
INFO:tensorflow:step: 4080, lr: 0.000668, klw: 0.1004, cost: -0.9119, recon: -0.9731, kl: 0.6096, train_time_taken: 7.4329
INFO:tensorflow:step: 4100, lr: 0.000667, klw: 0.1008, cost: -1.0706, recon: -1.1349, kl: 0.6384, train_time_taken: 7.3642
INFO:tensorflow:step: 4120, lr: 0.000666, klw: 0.1012, cost: -1.0133, recon: -1.0767, kl: 0.6258, train_time_taken: 7.2229
INFO:tensorflow:step: 4140, lr: 0.000664, klw: 0.1016, cost: -1.0454, recon: -1.1105, kl: 0.6404, train_time_taken: 7.6007
INFO:tensorflow:step: 4160, lr: 0.000663, klw: 0.1020, cost: -0.9385, recon: -1.0023, kl: 0.6247, train_time_taken: 8.4360
INFO:tensorflow:step: 4180, lr: 0.000662, klw: 0.1024, cost: -0.9511, recon: -1.0130, kl: 0.6046, train_time_taken: 7.3244
INFO:tensorflow:step: 4200, lr: 0.000660, klw: 0.1028, cost: -0.9803, recon: -1.0449, kl: 0.6278, train_time_taken: 7.9604
INFO:tensorflow:step: 4220, lr: 0.000659, klw: 0.1032, cost: -0.9747, recon: -1.0393, kl: 0.6259, train_time_taken: 8.5735
INFO:tensorflow:step: 4240, lr: 0.000658, klw: 0.1036, cost: -1.0008, recon: -1.0644, kl: 0.6146, train_time_taken: 7.3638
INFO:tensorflow:step: 4260, lr: 0.000657, klw: 0.1040, cost: -1.0040, recon: -1.0675, kl: 0.6105, train_time_taken: 7.2626
INFO:tensorflow:step: 4280, lr: 0.000655, klw: 0.1044, cost: -1.0093, recon: -1.0737, kl: 0.6164, train_time_taken: 7.4093
INFO:tensorflow:step: 4300, lr: 0.000654, klw: 0.1048, cost: -1.0449, recon: -1.1093, kl: 0.6145, train_time_taken: 7.3506
INFO:tensorflow:step: 4320, lr: 0.000653, klw: 0.1052, cost: -0.9779, recon: -1.0398, kl: 0.5883, train_time_taken: 7.3231
INFO:tensorflow:step: 4340, lr: 0.000651, klw: 0.1056, cost: -1.0397, recon: -1.1036, kl: 0.6047, train_time_taken: 7.4747
INFO:tensorflow:step: 4360, lr: 0.000650, klw: 0.1060, cost: -0.9775, recon: -1.0421, kl: 0.6096, train_time_taken: 8.2334
INFO:tensorflow:step: 4380, lr: 0.000649, klw: 0.1064, cost: -1.0050, recon: -1.0692, kl: 0.6038, train_time_taken: 7.9008
INFO:tensorflow:step: 4400, lr: 0.000648, klw: 0.1068, cost: -0.9906, recon: -1.0548, kl: 0.6018, train_time_taken: 7.3675
INFO:tensorflow:step: 4420, lr: 0.000646, klw: 0.1072, cost: -1.0260, recon: -1.0912, kl: 0.6078, train_time_taken: 7.5038
INFO:tensorflow:step: 4440, lr: 0.000645, klw: 0.1076, cost: -1.0145, recon: -1.0795, kl: 0.6045, train_time_taken: 7.9028
INFO:tensorflow:step: 4460, lr: 0.000644, klw: 0.1079, cost: -1.0435, recon: -1.1093, kl: 0.6103, train_time_taken: 7.3804
INFO:tensorflow:step: 4480, lr: 0.000643, klw: 0.1083, cost: -1.0043, recon: -1.0678, kl: 0.5868, train_time_taken: 7.3975
INFO:tensorflow:step: 4500, lr: 0.000641, klw: 0.1087, cost: -1.0766, recon: -1.1435, kl: 0.6159, train_time_taken: 7.4503
INFO:tensorflow:best_valid_cost: -1.2525, valid_cost: -1.2525, valid_recon: -1.2586, valid_kl: 0.6101, valid_time_taken: 0.6347
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 4500.
INFO:tensorflow:time_taken_save 0.4027.
INFO:tensorflow:eval_cost: -1.2274, eval_recon: -1.2334, eval_kl: 0.5996, eval_time_taken: 0.5298
INFO:tensorflow:step: 4520, lr: 0.000640, klw: 0.1091, cost: -1.0277, recon: -1.0944, kl: 0.6117, train_time_taken: 7.4478
INFO:tensorflow:step: 4540, lr: 0.000639, klw: 0.1095, cost: -1.0235, recon: -1.0870, kl: 0.5798, train_time_taken: 7.2583
INFO:tensorflow:step: 4560, lr: 0.000637, klw: 0.1099, cost: -1.0168, recon: -1.0827, kl: 0.6000, train_time_taken: 7.7237
INFO:tensorflow:step: 4580, lr: 0.000636, klw: 0.1103, cost: -1.0591, recon: -1.1260, kl: 0.6066, train_time_taken: 8.3066
INFO:tensorflow:step: 4600, lr: 0.000635, klw: 0.1107, cost: -1.0452, recon: -1.1106, kl: 0.5912, train_time_taken: 7.3717
INFO:tensorflow:step: 4620, lr: 0.000634, klw: 0.1111, cost: -1.0326, recon: -1.0976, kl: 0.5855, train_time_taken: 7.2779
INFO:tensorflow:step: 4640, lr: 0.000632, klw: 0.1115, cost: -1.0458, recon: -1.1106, kl: 0.5814, train_time_taken: 7.3093
INFO:tensorflow:step: 4660, lr: 0.000631, klw: 0.1118, cost: -1.0707, recon: -1.1376, kl: 0.5984, train_time_taken: 7.3664
INFO:tensorflow:step: 4680, lr: 0.000630, klw: 0.1122, cost: -1.0461, recon: -1.1125, kl: 0.5920, train_time_taken: 7.3875
INFO:tensorflow:step: 4700, lr: 0.000629, klw: 0.1126, cost: -1.0324, recon: -1.1001, kl: 0.6017, train_time_taken: 7.4391
INFO:tensorflow:step: 4720, lr: 0.000628, klw: 0.1130, cost: -1.0612, recon: -1.1285, kl: 0.5952, train_time_taken: 7.2617
INFO:tensorflow:step: 4740, lr: 0.000626, klw: 0.1134, cost: -1.0820, recon: -1.1488, kl: 0.5897, train_time_taken: 7.2975
INFO:tensorflow:step: 4760, lr: 0.000625, klw: 0.1138, cost: -1.0954, recon: -1.1619, kl: 0.5840, train_time_taken: 7.3532
INFO:tensorflow:step: 4780, lr: 0.000624, klw: 0.1142, cost: -1.0672, recon: -1.1333, kl: 0.5786, train_time_taken: 7.8073
INFO:tensorflow:step: 4800, lr: 0.000623, klw: 0.1146, cost: -0.9912, recon: -1.0566, kl: 0.5714, train_time_taken: 8.1208
INFO:tensorflow:step: 4820, lr: 0.000621, klw: 0.1149, cost: -1.0915, recon: -1.1585, kl: 0.5829, train_time_taken: 7.4119
INFO:tensorflow:step: 4840, lr: 0.000620, klw: 0.1153, cost: -1.0736, recon: -1.1408, kl: 0.5820, train_time_taken: 7.2905
INFO:tensorflow:step: 4860, lr: 0.000619, klw: 0.1157, cost: -1.0151, recon: -1.0818, kl: 0.5765, train_time_taken: 7.3004
INFO:tensorflow:step: 4880, lr: 0.000618, klw: 0.1161, cost: -1.0680, recon: -1.1341, kl: 0.5693, train_time_taken: 7.4028
INFO:tensorflow:step: 4900, lr: 0.000616, klw: 0.1165, cost: -1.0706, recon: -1.1379, kl: 0.5776, train_time_taken: 7.2727
INFO:tensorflow:step: 4920, lr: 0.000615, klw: 0.1169, cost: -1.0647, recon: -1.1335, kl: 0.5893, train_time_taken: 7.4246
INFO:tensorflow:step: 4940, lr: 0.000614, klw: 0.1172, cost: -1.0677, recon: -1.1335, kl: 0.5616, train_time_taken: 7.5746
INFO:tensorflow:step: 4960, lr: 0.000613, klw: 0.1176, cost: -1.0247, recon: -1.0906, kl: 0.5603, train_time_taken: 7.3806
INFO:tensorflow:step: 4980, lr: 0.000612, klw: 0.1180, cost: -1.0231, recon: -1.0889, kl: 0.5572, train_time_taken: 7.3185
INFO:tensorflow:step: 5000, lr: 0.000610, klw: 0.1184, cost: -1.0783, recon: -1.1465, kl: 0.5763, train_time_taken: 8.3066
INFO:tensorflow:best_valid_cost: -1.2815, valid_cost: -1.2815, valid_recon: -1.2873, valid_kl: 0.5825, valid_time_taken: 0.7171
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 5000.
INFO:tensorflow:time_taken_save 0.4616.
INFO:tensorflow:eval_cost: -1.2547, eval_recon: -1.2604, eval_kl: 0.5727, eval_time_taken: 0.6008
INFO:tensorflow:step: 5020, lr: 0.000609, klw: 0.1188, cost: -1.0530, recon: -1.1202, kl: 0.5652, train_time_taken: 8.3996
INFO:tensorflow:step: 5040, lr: 0.000608, klw: 0.1192, cost: -1.0339, recon: -1.1013, kl: 0.5662, train_time_taken: 8.1535
INFO:tensorflow:step: 5060, lr: 0.000607, klw: 0.1195, cost: -1.0690, recon: -1.1367, kl: 0.5661, train_time_taken: 7.4400
INFO:tensorflow:step: 5080, lr: 0.000606, klw: 0.1199, cost: -1.1322, recon: -1.2018, kl: 0.5804, train_time_taken: 7.3538
INFO:tensorflow:step: 5100, lr: 0.000604, klw: 0.1203, cost: -1.0468, recon: -1.1140, kl: 0.5586, train_time_taken: 7.3647
INFO:tensorflow:step: 5120, lr: 0.000603, klw: 0.1207, cost: -1.0511, recon: -1.1177, kl: 0.5518, train_time_taken: 7.3577
INFO:tensorflow:step: 5140, lr: 0.000602, klw: 0.1211, cost: -1.1117, recon: -1.1797, kl: 0.5623, train_time_taken: 7.2570
INFO:tensorflow:step: 5160, lr: 0.000601, klw: 0.1214, cost: -1.0360, recon: -1.1019, kl: 0.5426, train_time_taken: 7.3048
INFO:tensorflow:step: 5180, lr: 0.000600, klw: 0.1218, cost: -1.0341, recon: -1.1022, kl: 0.5592, train_time_taken: 7.3907
INFO:tensorflow:step: 5200, lr: 0.000599, klw: 0.1222, cost: -1.0445, recon: -1.1127, kl: 0.5583, train_time_taken: 7.9133
INFO:tensorflow:step: 5220, lr: 0.000597, klw: 0.1226, cost: -1.0729, recon: -1.1420, kl: 0.5634, train_time_taken: 8.4308
INFO:tensorflow:step: 5240, lr: 0.000596, klw: 0.1229, cost: -1.0952, recon: -1.1663, kl: 0.5779, train_time_taken: 8.1064
INFO:tensorflow:step: 5260, lr: 0.000595, klw: 0.1233, cost: -1.0617, recon: -1.1287, kl: 0.5439, train_time_taken: 7.3409
INFO:tensorflow:step: 5280, lr: 0.000594, klw: 0.1237, cost: -1.1318, recon: -1.2028, kl: 0.5742, train_time_taken: 7.2108
INFO:tensorflow:step: 5300, lr: 0.000593, klw: 0.1241, cost: -1.0134, recon: -1.0822, kl: 0.5540, train_time_taken: 7.1878
INFO:tensorflow:step: 5320, lr: 0.000592, klw: 0.1244, cost: -1.0790, recon: -1.1479, kl: 0.5537, train_time_taken: 7.4219
INFO:tensorflow:step: 5340, lr: 0.000590, klw: 0.1248, cost: -1.0836, recon: -1.1539, kl: 0.5634, train_time_taken: 7.3055
INFO:tensorflow:step: 5360, lr: 0.000589, klw: 0.1252, cost: -1.0558, recon: -1.1242, kl: 0.5460, train_time_taken: 7.2263
INFO:tensorflow:step: 5380, lr: 0.000588, klw: 0.1256, cost: -1.0778, recon: -1.1464, kl: 0.5464, train_time_taken: 7.4301
INFO:tensorflow:step: 5400, lr: 0.000587, klw: 0.1259, cost: -1.1672, recon: -1.2389, kl: 0.5695, train_time_taken: 7.3857
INFO:tensorflow:step: 5420, lr: 0.000586, klw: 0.1263, cost: -1.1547, recon: -1.2242, kl: 0.5499, train_time_taken: 8.3004
INFO:tensorflow:step: 5440, lr: 0.000585, klw: 0.1267, cost: -1.0719, recon: -1.1411, kl: 0.5458, train_time_taken: 7.8258
INFO:tensorflow:step: 5460, lr: 0.000583, klw: 0.1271, cost: -1.1504, recon: -1.2198, kl: 0.5457, train_time_taken: 7.2420
INFO:tensorflow:step: 5480, lr: 0.000582, klw: 0.1274, cost: -1.1015, recon: -1.1721, kl: 0.5545, train_time_taken: 7.3537
INFO:tensorflow:step: 5500, lr: 0.000581, klw: 0.1278, cost: -1.0917, recon: -1.1616, kl: 0.5470, train_time_taken: 7.3175
INFO:tensorflow:best_valid_cost: -1.3201, valid_cost: -1.3201, valid_recon: -1.3257, valid_kl: 0.5515, valid_time_taken: 0.6294
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 5500.
INFO:tensorflow:time_taken_save 0.4094.
INFO:tensorflow:eval_cost: -1.2945, eval_recon: -1.2999, eval_kl: 0.5412, eval_time_taken: 0.5194
INFO:tensorflow:step: 5520, lr: 0.000580, klw: 0.1282, cost: -1.0686, recon: -1.1372, kl: 0.5356, train_time_taken: 7.2693
INFO:tensorflow:step: 5540, lr: 0.000579, klw: 0.1286, cost: -1.0446, recon: -1.1149, kl: 0.5466, train_time_taken: 7.4311
INFO:tensorflow:step: 5560, lr: 0.000578, klw: 0.1289, cost: -1.0997, recon: -1.1710, kl: 0.5527, train_time_taken: 7.3812
INFO:tensorflow:step: 5580, lr: 0.000577, klw: 0.1293, cost: -1.1156, recon: -1.1857, kl: 0.5423, train_time_taken: 7.3742
INFO:tensorflow:step: 5600, lr: 0.000575, klw: 0.1297, cost: -1.1095, recon: -1.1797, kl: 0.5410, train_time_taken: 7.2864
INFO:tensorflow:step: 5620, lr: 0.000574, klw: 0.1300, cost: -1.1713, recon: -1.2429, kl: 0.5511, train_time_taken: 7.6637
INFO:tensorflow:step: 5640, lr: 0.000573, klw: 0.1304, cost: -1.1555, recon: -1.2270, kl: 0.5485, train_time_taken: 8.5161
INFO:tensorflow:step: 5660, lr: 0.000572, klw: 0.1308, cost: -1.1053, recon: -1.1766, kl: 0.5452, train_time_taken: 7.4460
INFO:tensorflow:step: 5680, lr: 0.000571, klw: 0.1311, cost: -1.1960, recon: -1.2708, kl: 0.5704, train_time_taken: 7.3856
INFO:tensorflow:step: 5700, lr: 0.000570, klw: 0.1315, cost: -1.1713, recon: -1.2438, kl: 0.5514, train_time_taken: 7.4029
INFO:tensorflow:step: 5720, lr: 0.000569, klw: 0.1319, cost: -1.1478, recon: -1.2197, kl: 0.5450, train_time_taken: 7.3320
INFO:tensorflow:step: 5740, lr: 0.000568, klw: 0.1323, cost: -1.1112, recon: -1.1820, kl: 0.5358, train_time_taken: 7.3740
INFO:tensorflow:step: 5760, lr: 0.000567, klw: 0.1326, cost: -1.1797, recon: -1.2536, kl: 0.5574, train_time_taken: 7.3215
INFO:tensorflow:step: 5780, lr: 0.000565, klw: 0.1330, cost: -1.0947, recon: -1.1649, kl: 0.5285, train_time_taken: 7.4161
INFO:tensorflow:step: 5800, lr: 0.000564, klw: 0.1334, cost: -1.0958, recon: -1.1683, kl: 0.5431, train_time_taken: 7.3463
INFO:tensorflow:step: 5820, lr: 0.000563, klw: 0.1337, cost: -1.1219, recon: -1.1932, kl: 0.5332, train_time_taken: 7.3259
INFO:tensorflow:step: 5840, lr: 0.000562, klw: 0.1341, cost: -1.1862, recon: -1.2616, kl: 0.5620, train_time_taken: 8.1111
INFO:tensorflow:step: 5860, lr: 0.000561, klw: 0.1345, cost: -1.1132, recon: -1.1855, kl: 0.5382, train_time_taken: 8.2733
INFO:tensorflow:step: 5880, lr: 0.000560, klw: 0.1348, cost: -1.1234, recon: -1.1968, kl: 0.5445, train_time_taken: 7.3649
INFO:tensorflow:step: 5900, lr: 0.000559, klw: 0.1352, cost: -1.1424, recon: -1.2135, kl: 0.5261, train_time_taken: 7.2469
INFO:tensorflow:step: 5920, lr: 0.000558, klw: 0.1355, cost: -1.1466, recon: -1.2204, kl: 0.5443, train_time_taken: 7.4406
INFO:tensorflow:step: 5940, lr: 0.000557, klw: 0.1359, cost: -1.1457, recon: -1.2191, kl: 0.5404, train_time_taken: 7.2937
INFO:tensorflow:step: 5960, lr: 0.000555, klw: 0.1363, cost: -1.1246, recon: -1.1975, kl: 0.5345, train_time_taken: 7.3523
INFO:tensorflow:step: 5980, lr: 0.000554, klw: 0.1366, cost: -1.1401, recon: -1.2118, kl: 0.5247, train_time_taken: 7.3157
INFO:tensorflow:step: 6000, lr: 0.000553, klw: 0.1370, cost: -1.0114, recon: -1.0846, kl: 0.5339, train_time_taken: 7.2694
INFO:tensorflow:best_valid_cost: -1.3595, valid_cost: -1.3595, valid_recon: -1.3652, valid_kl: 0.5625, valid_time_taken: 0.6173
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 6000.
INFO:tensorflow:time_taken_save 0.4320.
INFO:tensorflow:eval_cost: -1.3323, eval_recon: -1.3378, eval_kl: 0.5552, eval_time_taken: 0.5261
INFO:tensorflow:step: 6020, lr: 0.000552, klw: 0.1374, cost: -1.1344, recon: -1.2079, kl: 0.5352, train_time_taken: 7.4239
INFO:tensorflow:step: 6040, lr: 0.000551, klw: 0.1377, cost: -1.1847, recon: -1.2577, kl: 0.5298, train_time_taken: 8.1627
INFO:tensorflow:step: 6060, lr: 0.000550, klw: 0.1381, cost: -1.1106, recon: -1.1831, kl: 0.5247, train_time_taken: 8.5352
INFO:tensorflow:step: 6080, lr: 0.000549, klw: 0.1385, cost: -1.0749, recon: -1.1476, kl: 0.5247, train_time_taken: 7.4779
INFO:tensorflow:step: 6100, lr: 0.000548, klw: 0.1388, cost: -1.1716, recon: -1.2469, kl: 0.5425, train_time_taken: 7.3730
INFO:tensorflow:step: 6120, lr: 0.000547, klw: 0.1392, cost: -1.0538, recon: -1.1241, kl: 0.5053, train_time_taken: 7.2960
INFO:tensorflow:step: 6140, lr: 0.000546, klw: 0.1395, cost: -1.1567, recon: -1.2310, kl: 0.5326, train_time_taken: 7.3494
INFO:tensorflow:step: 6160, lr: 0.000545, klw: 0.1399, cost: -1.0734, recon: -1.1455, kl: 0.5155, train_time_taken: 7.3494
INFO:tensorflow:step: 6180, lr: 0.000544, klw: 0.1403, cost: -1.0843, recon: -1.1581, kl: 0.5264, train_time_taken: 7.4205
INFO:tensorflow:step: 6200, lr: 0.000543, klw: 0.1406, cost: -1.1573, recon: -1.2307, kl: 0.5218, train_time_taken: 7.3201
INFO:tensorflow:step: 6220, lr: 0.000541, klw: 0.1410, cost: -1.1433, recon: -1.2171, kl: 0.5235, train_time_taken: 7.4654
INFO:tensorflow:step: 6240, lr: 0.000540, klw: 0.1413, cost: -1.0804, recon: -1.1521, kl: 0.5077, train_time_taken: 7.4067
INFO:tensorflow:step: 6260, lr: 0.000539, klw: 0.1417, cost: -1.1498, recon: -1.2228, kl: 0.5150, train_time_taken: 7.8629
INFO:tensorflow:step: 6280, lr: 0.000538, klw: 0.1420, cost: -1.0798, recon: -1.1524, kl: 0.5112, train_time_taken: 8.2074
INFO:tensorflow:step: 6300, lr: 0.000537, klw: 0.1424, cost: -1.1327, recon: -1.2070, kl: 0.5211, train_time_taken: 7.4691
INFO:tensorflow:step: 6320, lr: 0.000536, klw: 0.1428, cost: -1.2052, recon: -1.2815, kl: 0.5339, train_time_taken: 7.3568
INFO:tensorflow:step: 6340, lr: 0.000535, klw: 0.1431, cost: -1.1648, recon: -1.2395, kl: 0.5213, train_time_taken: 7.3380
INFO:tensorflow:step: 6360, lr: 0.000534, klw: 0.1435, cost: -1.1674, recon: -1.2417, kl: 0.5181, train_time_taken: 7.3547
INFO:tensorflow:step: 6380, lr: 0.000533, klw: 0.1438, cost: -1.1594, recon: -1.2359, kl: 0.5320, train_time_taken: 7.3551
INFO:tensorflow:step: 6400, lr: 0.000532, klw: 0.1442, cost: -1.2056, recon: -1.2828, kl: 0.5348, train_time_taken: 7.4736
INFO:tensorflow:step: 6420, lr: 0.000531, klw: 0.1445, cost: -1.1172, recon: -1.1894, kl: 0.4995, train_time_taken: 7.2722
INFO:tensorflow:step: 6440, lr: 0.000530, klw: 0.1449, cost: -1.2172, recon: -1.2947, kl: 0.5348, train_time_taken: 7.3438
INFO:tensorflow:step: 6460, lr: 0.000529, klw: 0.1453, cost: -1.1235, recon: -1.1979, kl: 0.5120, train_time_taken: 7.2314
INFO:tensorflow:step: 6480, lr: 0.000528, klw: 0.1456, cost: -1.1310, recon: -1.2048, kl: 0.5069, train_time_taken: 8.0803
INFO:tensorflow:step: 6500, lr: 0.000527, klw: 0.1460, cost: -1.1906, recon: -1.2668, kl: 0.5220, train_time_taken: 8.0498
INFO:tensorflow:best_valid_cost: -1.3798, valid_cost: -1.3798, valid_recon: -1.3849, valid_kl: 0.5140, valid_time_taken: 0.6562
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 6500.
INFO:tensorflow:time_taken_save 0.4254.
INFO:tensorflow:eval_cost: -1.3501, eval_recon: -1.3551, eval_kl: 0.5050, eval_time_taken: 0.5264
INFO:tensorflow:step: 6520, lr: 0.000526, klw: 0.1463, cost: -1.1294, recon: -1.2048, kl: 0.5152, train_time_taken: 7.1727
INFO:tensorflow:step: 6540, lr: 0.000525, klw: 0.1467, cost: -1.1309, recon: -1.2044, kl: 0.5014, train_time_taken: 7.2472
INFO:tensorflow:step: 6560, lr: 0.000524, klw: 0.1470, cost: -1.0429, recon: -1.1173, kl: 0.5056, train_time_taken: 7.2635
INFO:tensorflow:step: 6580, lr: 0.000523, klw: 0.1474, cost: -1.1477, recon: -1.2252, kl: 0.5256, train_time_taken: 7.3412
INFO:tensorflow:step: 6600, lr: 0.000522, klw: 0.1477, cost: -1.1091, recon: -1.1850, kl: 0.5137, train_time_taken: 7.2662
INFO:tensorflow:step: 6620, lr: 0.000521, klw: 0.1481, cost: -1.1604, recon: -1.2353, kl: 0.5060, train_time_taken: 7.2682
INFO:tensorflow:step: 6640, lr: 0.000520, klw: 0.1484, cost: -1.1569, recon: -1.2327, kl: 0.5107, train_time_taken: 7.1885
INFO:tensorflow:step: 6660, lr: 0.000519, klw: 0.1488, cost: -1.1149, recon: -1.1898, kl: 0.5030, train_time_taken: 7.9463
INFO:tensorflow:step: 6680, lr: 0.000518, klw: 0.1491, cost: -1.2388, recon: -1.3174, kl: 0.5267, train_time_taken: 8.4598
INFO:tensorflow:step: 6700, lr: 0.000517, klw: 0.1495, cost: -1.1394, recon: -1.2154, kl: 0.5084, train_time_taken: 8.6702
INFO:tensorflow:step: 6720, lr: 0.000516, klw: 0.1498, cost: -1.1285, recon: -1.2050, kl: 0.5102, train_time_taken: 7.2929
INFO:tensorflow:step: 6740, lr: 0.000515, klw: 0.1502, cost: -1.1157, recon: -1.1911, kl: 0.5023, train_time_taken: 7.4073
INFO:tensorflow:step: 6760, lr: 0.000514, klw: 0.1505, cost: -1.1291, recon: -1.2023, kl: 0.4866, train_time_taken: 7.2455
INFO:tensorflow:step: 6780, lr: 0.000513, klw: 0.1509, cost: -1.1918, recon: -1.2698, kl: 0.5169, train_time_taken: 7.2530
INFO:tensorflow:step: 6800, lr: 0.000512, klw: 0.1512, cost: -1.1242, recon: -1.1973, kl: 0.4831, train_time_taken: 7.2723
INFO:tensorflow:step: 6820, lr: 0.000511, klw: 0.1516, cost: -1.0796, recon: -1.1551, kl: 0.4978, train_time_taken: 7.3499
INFO:tensorflow:step: 6840, lr: 0.000510, klw: 0.1519, cost: -1.1862, recon: -1.2640, kl: 0.5119, train_time_taken: 7.7466
INFO:tensorflow:step: 6860, lr: 0.000509, klw: 0.1523, cost: -1.2245, recon: -1.3036, kl: 0.5196, train_time_taken: 7.6971
INFO:tensorflow:step: 6880, lr: 0.000508, klw: 0.1526, cost: -1.1409, recon: -1.2164, kl: 0.4948, train_time_taken: 7.3155
INFO:tensorflow:step: 6900, lr: 0.000507, klw: 0.1530, cost: -1.2044, recon: -1.2829, kl: 0.5132, train_time_taken: 7.8905
INFO:tensorflow:step: 6920, lr: 0.000506, klw: 0.1533, cost: -1.1011, recon: -1.1769, kl: 0.4949, train_time_taken: 8.0844
INFO:tensorflow:step: 6940, lr: 0.000505, klw: 0.1537, cost: -1.1816, recon: -1.2586, kl: 0.5012, train_time_taken: 7.2335
INFO:tensorflow:step: 6960, lr: 0.000504, klw: 0.1540, cost: -1.2048, recon: -1.2832, kl: 0.5091, train_time_taken: 7.3608
INFO:tensorflow:step: 6980, lr: 0.000503, klw: 0.1544, cost: -1.2346, recon: -1.3139, kl: 0.5137, train_time_taken: 7.1425
INFO:tensorflow:step: 7000, lr: 0.000502, klw: 0.1547, cost: -1.1952, recon: -1.2722, kl: 0.4980, train_time_taken: 7.3159
INFO:tensorflow:best_valid_cost: -1.4070, valid_cost: -1.4070, valid_recon: -1.4121, valid_kl: 0.5099, valid_time_taken: 0.6306
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 7000.
INFO:tensorflow:time_taken_save 0.4322.
INFO:tensorflow:eval_cost: -1.3784, eval_recon: -1.3834, eval_kl: 0.5011, eval_time_taken: 0.5313
INFO:tensorflow:step: 7020, lr: 0.000501, klw: 0.1551, cost: -1.3010, recon: -1.3815, kl: 0.5189, train_time_taken: 7.4296
INFO:tensorflow:step: 7040, lr: 0.000500, klw: 0.1554, cost: -1.1620, recon: -1.2401, kl: 0.5025, train_time_taken: 7.2019
INFO:tensorflow:step: 7060, lr: 0.000499, klw: 0.1557, cost: -1.2595, recon: -1.3387, kl: 0.5087, train_time_taken: 7.3571
INFO:tensorflow:step: 7080, lr: 0.000498, klw: 0.1561, cost: -1.2023, recon: -1.2807, kl: 0.5023, train_time_taken: 7.2317
INFO:tensorflow:step: 7100, lr: 0.000497, klw: 0.1564, cost: -1.1438, recon: -1.2200, kl: 0.4869, train_time_taken: 7.1927
INFO:tensorflow:step: 7120, lr: 0.000496, klw: 0.1568, cost: -1.1862, recon: -1.2633, kl: 0.4919, train_time_taken: 8.4153
INFO:tensorflow:step: 7140, lr: 0.000495, klw: 0.1571, cost: -1.2232, recon: -1.3036, kl: 0.5114, train_time_taken: 7.6562
INFO:tensorflow:step: 7160, lr: 0.000494, klw: 0.1575, cost: -1.1985, recon: -1.2783, kl: 0.5067, train_time_taken: 7.3316
INFO:tensorflow:step: 7180, lr: 0.000493, klw: 0.1578, cost: -1.2397, recon: -1.3195, kl: 0.5057, train_time_taken: 7.2234
INFO:tensorflow:step: 7200, lr: 0.000492, klw: 0.1581, cost: -1.1579, recon: -1.2366, kl: 0.4978, train_time_taken: 7.2495
INFO:tensorflow:step: 7220, lr: 0.000491, klw: 0.1585, cost: -1.2165, recon: -1.2974, kl: 0.5107, train_time_taken: 7.2915
INFO:tensorflow:step: 7240, lr: 0.000490, klw: 0.1588, cost: -1.1831, recon: -1.2626, kl: 0.5008, train_time_taken: 7.3238
INFO:tensorflow:step: 7260, lr: 0.000489, klw: 0.1592, cost: -1.1528, recon: -1.2308, kl: 0.4901, train_time_taken: 7.2029
INFO:tensorflow:step: 7280, lr: 0.000488, klw: 0.1595, cost: -1.2110, recon: -1.2903, kl: 0.4972, train_time_taken: 7.3856
INFO:tensorflow:step: 7300, lr: 0.000487, klw: 0.1598, cost: -1.2267, recon: -1.3062, kl: 0.4976, train_time_taken: 7.2006
INFO:tensorflow:step: 7320, lr: 0.000486, klw: 0.1602, cost: -1.1836, recon: -1.2639, kl: 0.5014, train_time_taken: 7.5249
INFO:tensorflow:step: 7340, lr: 0.000485, klw: 0.1605, cost: -1.2122, recon: -1.2910, kl: 0.4905, train_time_taken: 8.4008
INFO:tensorflow:step: 7360, lr: 0.000484, klw: 0.1609, cost: -1.1360, recon: -1.2143, kl: 0.4868, train_time_taken: 7.4166
INFO:tensorflow:step: 7380, lr: 0.000483, klw: 0.1612, cost: -1.1892, recon: -1.2669, kl: 0.4819, train_time_taken: 7.4651
INFO:tensorflow:step: 7400, lr: 0.000482, klw: 0.1615, cost: -1.1949, recon: -1.2759, kl: 0.5013, train_time_taken: 7.3196
INFO:tensorflow:step: 7420, lr: 0.000481, klw: 0.1619, cost: -1.1639, recon: -1.2433, kl: 0.4904, train_time_taken: 7.2559
INFO:tensorflow:step: 7440, lr: 0.000480, klw: 0.1622, cost: -1.1725, recon: -1.2533, kl: 0.4983, train_time_taken: 7.3603
INFO:tensorflow:step: 7460, lr: 0.000479, klw: 0.1626, cost: -1.2163, recon: -1.2958, kl: 0.4892, train_time_taken: 7.2459
INFO:tensorflow:step: 7480, lr: 0.000479, klw: 0.1629, cost: -1.2506, recon: -1.3323, kl: 0.5018, train_time_taken: 7.6817
INFO:tensorflow:step: 7500, lr: 0.000478, klw: 0.1632, cost: -1.1955, recon: -1.2744, kl: 0.4833, train_time_taken: 8.6461
INFO:tensorflow:best_valid_cost: -1.4320, valid_cost: -1.4320, valid_recon: -1.4369, valid_kl: 0.4913, valid_time_taken: 0.6223
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 7500.
INFO:tensorflow:time_taken_save 0.4482.
INFO:tensorflow:eval_cost: -1.3974, eval_recon: -1.4022, eval_kl: 0.4832, eval_time_taken: 0.5060
INFO:tensorflow:step: 7520, lr: 0.000477, klw: 0.1636, cost: -1.2346, recon: -1.3166, kl: 0.5008, train_time_taken: 7.2552
INFO:tensorflow:step: 7540, lr: 0.000476, klw: 0.1639, cost: -1.1741, recon: -1.2534, kl: 0.4837, train_time_taken: 8.0512
INFO:tensorflow:step: 7560, lr: 0.000475, klw: 0.1642, cost: -1.2276, recon: -1.3078, kl: 0.4887, train_time_taken: 7.9371
INFO:tensorflow:step: 7580, lr: 0.000474, klw: 0.1646, cost: -1.1223, recon: -1.2024, kl: 0.4866, train_time_taken: 7.1473
INFO:tensorflow:step: 7600, lr: 0.000473, klw: 0.1649, cost: -1.1678, recon: -1.2477, kl: 0.4845, train_time_taken: 7.2136
INFO:tensorflow:step: 7620, lr: 0.000472, klw: 0.1652, cost: -1.1711, recon: -1.2534, kl: 0.4977, train_time_taken: 7.1565
INFO:tensorflow:step: 7640, lr: 0.000471, klw: 0.1656, cost: -1.2287, recon: -1.3094, kl: 0.4874, train_time_taken: 7.3926
INFO:tensorflow:step: 7660, lr: 0.000470, klw: 0.1659, cost: -1.1394, recon: -1.2167, kl: 0.4656, train_time_taken: 7.9960
INFO:tensorflow:step: 7680, lr: 0.000469, klw: 0.1662, cost: -1.1967, recon: -1.2774, kl: 0.4854, train_time_taken: 7.3017
INFO:tensorflow:step: 7700, lr: 0.000468, klw: 0.1666, cost: -1.2038, recon: -1.2857, kl: 0.4915, train_time_taken: 7.3297
INFO:tensorflow:step: 7720, lr: 0.000467, klw: 0.1669, cost: -1.2354, recon: -1.3178, kl: 0.4937, train_time_taken: 7.3713
INFO:tensorflow:step: 7740, lr: 0.000467, klw: 0.1672, cost: -1.2168, recon: -1.2989, kl: 0.4910, train_time_taken: 7.3089
INFO:tensorflow:step: 7760, lr: 0.000466, klw: 0.1676, cost: -1.1863, recon: -1.2669, kl: 0.4811, train_time_taken: 8.5173
INFO:tensorflow:step: 7780, lr: 0.000465, klw: 0.1679, cost: -1.1493, recon: -1.2294, kl: 0.4775, train_time_taken: 7.5976
INFO:tensorflow:step: 7800, lr: 0.000464, klw: 0.1682, cost: -1.1732, recon: -1.2544, kl: 0.4824, train_time_taken: 7.4006
INFO:tensorflow:step: 7820, lr: 0.000463, klw: 0.1686, cost: -1.1742, recon: -1.2550, kl: 0.4796, train_time_taken: 7.3495
INFO:tensorflow:step: 7840, lr: 0.000462, klw: 0.1689, cost: -1.2582, recon: -1.3425, kl: 0.4994, train_time_taken: 7.2571
INFO:tensorflow:step: 7860, lr: 0.000461, klw: 0.1692, cost: -1.3558, recon: -1.4388, kl: 0.4900, train_time_taken: 7.3204
INFO:tensorflow:step: 7880, lr: 0.000460, klw: 0.1696, cost: -1.2548, recon: -1.3368, kl: 0.4837, train_time_taken: 7.2725
INFO:tensorflow:step: 7900, lr: 0.000459, klw: 0.1699, cost: -1.2515, recon: -1.3357, kl: 0.4959, train_time_taken: 7.1530
INFO:tensorflow:step: 7920, lr: 0.000458, klw: 0.1702, cost: -1.2480, recon: -1.3325, kl: 0.4964, train_time_taken: 7.2432
INFO:tensorflow:step: 7940, lr: 0.000457, klw: 0.1706, cost: -1.2432, recon: -1.3260, kl: 0.4856, train_time_taken: 7.1982
INFO:tensorflow:step: 7960, lr: 0.000457, klw: 0.1709, cost: -1.2281, recon: -1.3100, kl: 0.4791, train_time_taken: 7.3194
INFO:tensorflow:step: 7980, lr: 0.000456, klw: 0.1712, cost: -1.2263, recon: -1.3106, kl: 0.4928, train_time_taken: 8.4191
INFO:tensorflow:step: 8000, lr: 0.000455, klw: 0.1715, cost: -1.2581, recon: -1.3401, kl: 0.4780, train_time_taken: 7.4486
INFO:tensorflow:best_valid_cost: -1.4494, valid_cost: -1.4494, valid_recon: -1.4543, valid_kl: 0.4848, valid_time_taken: 0.6396
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 8000.
INFO:tensorflow:time_taken_save 0.4466.
INFO:tensorflow:eval_cost: -1.4182, eval_recon: -1.4230, eval_kl: 0.4754, eval_time_taken: 0.5229
INFO:tensorflow:step: 8020, lr: 0.000454, klw: 0.1719, cost: -1.1971, recon: -1.2811, kl: 0.4888, train_time_taken: 7.3699
INFO:tensorflow:step: 8040, lr: 0.000453, klw: 0.1722, cost: -1.1603, recon: -1.2435, kl: 0.4831, train_time_taken: 7.2493
INFO:tensorflow:step: 8060, lr: 0.000452, klw: 0.1725, cost: -1.2924, recon: -1.3781, kl: 0.4967, train_time_taken: 7.2316
INFO:tensorflow:step: 8080, lr: 0.000451, klw: 0.1729, cost: -1.2357, recon: -1.3191, kl: 0.4824, train_time_taken: 7.2600
INFO:tensorflow:step: 8100, lr: 0.000450, klw: 0.1732, cost: -1.1975, recon: -1.2793, kl: 0.4721, train_time_taken: 7.2318
INFO:tensorflow:step: 8120, lr: 0.000450, klw: 0.1735, cost: -1.1335, recon: -1.2148, kl: 0.4684, train_time_taken: 7.2064
INFO:tensorflow:step: 8140, lr: 0.000449, klw: 0.1738, cost: -1.2136, recon: -1.2966, kl: 0.4771, train_time_taken: 7.3772
INFO:tensorflow:step: 8160, lr: 0.000448, klw: 0.1742, cost: -1.1416, recon: -1.2227, kl: 0.4656, train_time_taken: 7.2355
INFO:tensorflow:step: 8180, lr: 0.000447, klw: 0.1745, cost: -1.2218, recon: -1.3083, kl: 0.4958, train_time_taken: 7.8602
INFO:tensorflow:step: 8200, lr: 0.000446, klw: 0.1748, cost: -1.2028, recon: -1.2853, kl: 0.4724, train_time_taken: 8.1669
INFO:tensorflow:step: 8220, lr: 0.000445, klw: 0.1751, cost: -1.2338, recon: -1.3181, kl: 0.4810, train_time_taken: 7.1513
INFO:tensorflow:step: 8240, lr: 0.000444, klw: 0.1755, cost: -1.2110, recon: -1.2934, kl: 0.4696, train_time_taken: 7.2074
INFO:tensorflow:step: 8260, lr: 0.000443, klw: 0.1758, cost: -1.2298, recon: -1.3143, kl: 0.4808, train_time_taken: 7.1559
INFO:tensorflow:step: 8280, lr: 0.000443, klw: 0.1761, cost: -1.2535, recon: -1.3379, kl: 0.4795, train_time_taken: 7.2232
INFO:tensorflow:step: 8300, lr: 0.000442, klw: 0.1764, cost: -1.2489, recon: -1.3353, kl: 0.4894, train_time_taken: 7.3029
INFO:tensorflow:step: 8320, lr: 0.000441, klw: 0.1768, cost: -1.2906, recon: -1.3755, kl: 0.4808, train_time_taken: 8.8062
INFO:tensorflow:step: 8340, lr: 0.000440, klw: 0.1771, cost: -1.1757, recon: -1.2570, kl: 0.4588, train_time_taken: 7.2603
INFO:tensorflow:step: 8360, lr: 0.000439, klw: 0.1774, cost: -1.2096, recon: -1.2958, kl: 0.4858, train_time_taken: 7.3165
INFO:tensorflow:step: 8380, lr: 0.000438, klw: 0.1777, cost: -1.2738, recon: -1.3572, kl: 0.4693, train_time_taken: 7.2987
INFO:tensorflow:step: 8400, lr: 0.000437, klw: 0.1781, cost: -1.1970, recon: -1.2811, kl: 0.4722, train_time_taken: 8.2240
INFO:tensorflow:step: 8420, lr: 0.000437, klw: 0.1784, cost: -1.2401, recon: -1.3248, kl: 0.4746, train_time_taken: 7.8029
INFO:tensorflow:step: 8440, lr: 0.000436, klw: 0.1787, cost: -1.2208, recon: -1.3056, kl: 0.4745, train_time_taken: 7.1842
INFO:tensorflow:step: 8460, lr: 0.000435, klw: 0.1790, cost: -1.1562, recon: -1.2395, kl: 0.4654, train_time_taken: 7.6549
INFO:tensorflow:step: 8480, lr: 0.000434, klw: 0.1793, cost: -1.2321, recon: -1.3174, kl: 0.4759, train_time_taken: 7.6906
INFO:tensorflow:step: 8500, lr: 0.000433, klw: 0.1797, cost: -1.1448, recon: -1.2274, kl: 0.4597, train_time_taken: 7.3527
INFO:tensorflow:best_valid_cost: -1.4586, valid_cost: -1.4586, valid_recon: -1.4633, valid_kl: 0.4739, valid_time_taken: 0.6251
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 8500.
INFO:tensorflow:time_taken_save 0.4539.
INFO:tensorflow:eval_cost: -1.4241, eval_recon: -1.4288, eval_kl: 0.4646, eval_time_taken: 0.5235
INFO:tensorflow:step: 8520, lr: 0.000432, klw: 0.1800, cost: -1.2404, recon: -1.3238, kl: 0.4638, train_time_taken: 7.2028
INFO:tensorflow:step: 8540, lr: 0.000431, klw: 0.1803, cost: -1.1937, recon: -1.2791, kl: 0.4740, train_time_taken: 7.2140
INFO:tensorflow:step: 8560, lr: 0.000431, klw: 0.1806, cost: -1.3131, recon: -1.4011, kl: 0.4874, train_time_taken: 7.2532
INFO:tensorflow:step: 8580, lr: 0.000430, klw: 0.1809, cost: -1.2191, recon: -1.3028, kl: 0.4621, train_time_taken: 7.3083
INFO:tensorflow:step: 8600, lr: 0.000429, klw: 0.1813, cost: -1.3378, recon: -1.4253, kl: 0.4830, train_time_taken: 7.3855
INFO:tensorflow:step: 8620, lr: 0.000428, klw: 0.1816, cost: -1.3084, recon: -1.3954, kl: 0.4794, train_time_taken: 8.5251
INFO:tensorflow:step: 8640, lr: 0.000427, klw: 0.1819, cost: -1.2490, recon: -1.3356, kl: 0.4758, train_time_taken: 7.3041
INFO:tensorflow:step: 8660, lr: 0.000426, klw: 0.1822, cost: -1.2736, recon: -1.3632, kl: 0.4917, train_time_taken: 7.3769
INFO:tensorflow:step: 8680, lr: 0.000426, klw: 0.1825, cost: -1.2270, recon: -1.3148, kl: 0.4812, train_time_taken: 7.3379
INFO:tensorflow:step: 8700, lr: 0.000425, klw: 0.1828, cost: -1.1533, recon: -1.2359, kl: 0.4519, train_time_taken: 7.4143
INFO:tensorflow:step: 8720, lr: 0.000424, klw: 0.1832, cost: -1.2224, recon: -1.3073, kl: 0.4637, train_time_taken: 7.2661
INFO:tensorflow:step: 8740, lr: 0.000423, klw: 0.1835, cost: -1.2469, recon: -1.3378, kl: 0.4957, train_time_taken: 7.3098
INFO:tensorflow:step: 8760, lr: 0.000422, klw: 0.1838, cost: -1.2560, recon: -1.3439, kl: 0.4782, train_time_taken: 7.3220
INFO:tensorflow:step: 8780, lr: 0.000421, klw: 0.1841, cost: -1.2582, recon: -1.3478, kl: 0.4866, train_time_taken: 7.2846
INFO:tensorflow:step: 8800, lr: 0.000421, klw: 0.1844, cost: -1.3290, recon: -1.4166, kl: 0.4750, train_time_taken: 7.3260
INFO:tensorflow:step: 8820, lr: 0.000420, klw: 0.1847, cost: -1.1702, recon: -1.2561, kl: 0.4653, train_time_taken: 7.7581
INFO:tensorflow:step: 8840, lr: 0.000419, klw: 0.1851, cost: -1.2123, recon: -1.2987, kl: 0.4672, train_time_taken: 8.2674
INFO:tensorflow:step: 8860, lr: 0.000418, klw: 0.1854, cost: -1.2347, recon: -1.3226, kl: 0.4739, train_time_taken: 7.3353
INFO:tensorflow:step: 8880, lr: 0.000417, klw: 0.1857, cost: -1.2023, recon: -1.2881, kl: 0.4623, train_time_taken: 7.1607
INFO:tensorflow:step: 8900, lr: 0.000417, klw: 0.1860, cost: -1.2504, recon: -1.3362, kl: 0.4616, train_time_taken: 7.4360
INFO:tensorflow:step: 8920, lr: 0.000416, klw: 0.1863, cost: -1.2644, recon: -1.3516, kl: 0.4680, train_time_taken: 7.2423
INFO:tensorflow:step: 8940, lr: 0.000415, klw: 0.1866, cost: -1.3186, recon: -1.4073, kl: 0.4756, train_time_taken: 7.3364
INFO:tensorflow:step: 8960, lr: 0.000414, klw: 0.1869, cost: -1.3207, recon: -1.4117, kl: 0.4864, train_time_taken: 7.2971
INFO:tensorflow:step: 8980, lr: 0.000413, klw: 0.1873, cost: -1.2689, recon: -1.3574, kl: 0.4727, train_time_taken: 7.2134
INFO:tensorflow:step: 9000, lr: 0.000412, klw: 0.1876, cost: -1.2871, recon: -1.3742, kl: 0.4643, train_time_taken: 7.3785
INFO:tensorflow:best_valid_cost: -1.4873, valid_cost: -1.4873, valid_recon: -1.4920, valid_kl: 0.4723, valid_time_taken: 0.6246
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 9000.
INFO:tensorflow:time_taken_save 0.4616.
INFO:tensorflow:eval_cost: -1.4534, eval_recon: -1.4581, eval_kl: 0.4632, eval_time_taken: 0.5242
INFO:tensorflow:step: 9020, lr: 0.000412, klw: 0.1879, cost: -1.1962, recon: -1.2847, kl: 0.4711, train_time_taken: 7.3644
INFO:tensorflow:step: 9040, lr: 0.000411, klw: 0.1882, cost: -1.2733, recon: -1.3620, kl: 0.4713, train_time_taken: 8.2298
INFO:tensorflow:step: 9060, lr: 0.000410, klw: 0.1885, cost: -1.2191, recon: -1.3069, kl: 0.4656, train_time_taken: 7.8658
INFO:tensorflow:step: 9080, lr: 0.000409, klw: 0.1888, cost: -1.3127, recon: -1.4014, kl: 0.4699, train_time_taken: 7.3060
INFO:tensorflow:step: 9100, lr: 0.000408, klw: 0.1891, cost: -1.2640, recon: -1.3526, kl: 0.4680, train_time_taken: 7.3357
INFO:tensorflow:step: 9120, lr: 0.000408, klw: 0.1894, cost: -1.2613, recon: -1.3482, kl: 0.4590, train_time_taken: 7.4144
INFO:tensorflow:step: 9140, lr: 0.000407, klw: 0.1897, cost: -1.1135, recon: -1.1973, kl: 0.4420, train_time_taken: 8.6810
INFO:tensorflow:step: 9160, lr: 0.000406, klw: 0.1901, cost: -1.2937, recon: -1.3824, kl: 0.4667, train_time_taken: 7.3384
INFO:tensorflow:step: 9180, lr: 0.000405, klw: 0.1904, cost: -1.3210, recon: -1.4102, kl: 0.4686, train_time_taken: 7.3149
INFO:tensorflow:step: 9200, lr: 0.000405, klw: 0.1907, cost: -1.2514, recon: -1.3395, kl: 0.4621, train_time_taken: 7.3673
INFO:tensorflow:step: 9220, lr: 0.000404, klw: 0.1910, cost: -1.3053, recon: -1.3971, kl: 0.4804, train_time_taken: 7.5133
INFO:tensorflow:step: 9240, lr: 0.000403, klw: 0.1913, cost: -1.3174, recon: -1.4073, kl: 0.4698, train_time_taken: 7.5492
INFO:tensorflow:step: 9260, lr: 0.000402, klw: 0.1916, cost: -1.2314, recon: -1.3190, kl: 0.4576, train_time_taken: 8.5214
INFO:tensorflow:step: 9280, lr: 0.000401, klw: 0.1919, cost: -1.3360, recon: -1.4264, kl: 0.4710, train_time_taken: 8.0427
INFO:tensorflow:step: 9300, lr: 0.000401, klw: 0.1922, cost: -1.3647, recon: -1.4578, kl: 0.4842, train_time_taken: 7.2908
INFO:tensorflow:step: 9320, lr: 0.000400, klw: 0.1925, cost: -1.2749, recon: -1.3626, kl: 0.4560, train_time_taken: 7.2484
INFO:tensorflow:step: 9340, lr: 0.000399, klw: 0.1928, cost: -1.2863, recon: -1.3759, kl: 0.4647, train_time_taken: 7.3777
INFO:tensorflow:step: 9360, lr: 0.000398, klw: 0.1931, cost: -1.3203, recon: -1.4093, kl: 0.4607, train_time_taken: 7.1864
INFO:tensorflow:step: 9380, lr: 0.000397, klw: 0.1934, cost: -1.2376, recon: -1.3256, kl: 0.4549, train_time_taken: 7.2516
INFO:tensorflow:step: 9400, lr: 0.000397, klw: 0.1938, cost: -1.3110, recon: -1.4030, kl: 0.4750, train_time_taken: 7.3063
INFO:tensorflow:step: 9420, lr: 0.000396, klw: 0.1941, cost: -1.2537, recon: -1.3436, kl: 0.4628, train_time_taken: 7.4356
INFO:tensorflow:step: 9440, lr: 0.000395, klw: 0.1944, cost: -1.2843, recon: -1.3738, kl: 0.4607, train_time_taken: 7.3686
INFO:tensorflow:step: 9460, lr: 0.000394, klw: 0.1947, cost: -1.1814, recon: -1.2672, kl: 0.4410, train_time_taken: 7.9602
INFO:tensorflow:step: 9480, lr: 0.000394, klw: 0.1950, cost: -1.3038, recon: -1.3928, kl: 0.4564, train_time_taken: 8.0648
INFO:tensorflow:step: 9500, lr: 0.000393, klw: 0.1953, cost: -1.2396, recon: -1.3298, kl: 0.4621, train_time_taken: 7.4636
INFO:tensorflow:best_valid_cost: -1.4986, valid_cost: -1.4986, valid_recon: -1.5032, valid_kl: 0.4624, valid_time_taken: 0.6230
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 9500.
INFO:tensorflow:time_taken_save 0.4734.
INFO:tensorflow:eval_cost: -1.4673, eval_recon: -1.4719, eval_kl: 0.4536, eval_time_taken: 0.5153
INFO:tensorflow:step: 9520, lr: 0.000392, klw: 0.1956, cost: -1.2795, recon: -1.3685, kl: 0.4548, train_time_taken: 7.1505
INFO:tensorflow:step: 9540, lr: 0.000391, klw: 0.1959, cost: -1.1876, recon: -1.2757, kl: 0.4501, train_time_taken: 7.3710
INFO:tensorflow:step: 9560, lr: 0.000391, klw: 0.1962, cost: -1.1949, recon: -1.2824, kl: 0.4457, train_time_taken: 7.3222
INFO:tensorflow:step: 9580, lr: 0.000390, klw: 0.1965, cost: -1.2207, recon: -1.3087, kl: 0.4476, train_time_taken: 7.2670
INFO:tensorflow:step: 9600, lr: 0.000389, klw: 0.1968, cost: -1.1672, recon: -1.2563, kl: 0.4527, train_time_taken: 7.2117
INFO:tensorflow:step: 9620, lr: 0.000388, klw: 0.1971, cost: -1.2481, recon: -1.3371, kl: 0.4516, train_time_taken: 7.2103
INFO:tensorflow:step: 9640, lr: 0.000388, klw: 0.1974, cost: -1.2140, recon: -1.3026, kl: 0.4485, train_time_taken: 7.2365
INFO:tensorflow:step: 9660, lr: 0.000387, klw: 0.1977, cost: -1.3078, recon: -1.3995, kl: 0.4638, train_time_taken: 7.3621
INFO:tensorflow:step: 9680, lr: 0.000386, klw: 0.1980, cost: -1.2600, recon: -1.3491, kl: 0.4499, train_time_taken: 8.3531
INFO:tensorflow:step: 9700, lr: 0.000385, klw: 0.1983, cost: -1.2453, recon: -1.3343, kl: 0.4488, train_time_taken: 7.6969
INFO:tensorflow:step: 9720, lr: 0.000385, klw: 0.1986, cost: -1.2516, recon: -1.3418, kl: 0.4544, train_time_taken: 7.3101
INFO:tensorflow:step: 9740, lr: 0.000384, klw: 0.1989, cost: -1.2554, recon: -1.3468, kl: 0.4597, train_time_taken: 7.3110
INFO:tensorflow:step: 9760, lr: 0.000383, klw: 0.1992, cost: -1.2703, recon: -1.3605, kl: 0.4527, train_time_taken: 7.2532
INFO:tensorflow:step: 9780, lr: 0.000382, klw: 0.1995, cost: -1.3903, recon: -1.4829, kl: 0.4641, train_time_taken: 7.4137
INFO:tensorflow:step: 9800, lr: 0.000382, klw: 0.1998, cost: -1.2850, recon: -1.3768, kl: 0.4599, train_time_taken: 7.3117
INFO:tensorflow:step: 9820, lr: 0.000381, klw: 0.2001, cost: -1.3057, recon: -1.3967, kl: 0.4548, train_time_taken: 7.3944
INFO:tensorflow:step: 9840, lr: 0.000380, klw: 0.2004, cost: -1.2312, recon: -1.3219, kl: 0.4524, train_time_taken: 7.2032
INFO:tensorflow:step: 9860, lr: 0.000379, klw: 0.2007, cost: -1.3524, recon: -1.4437, kl: 0.4548, train_time_taken: 7.2361
INFO:tensorflow:step: 9880, lr: 0.000379, klw: 0.2010, cost: -1.1890, recon: -1.2766, kl: 0.4357, train_time_taken: 7.5308
INFO:tensorflow:step: 9900, lr: 0.000378, klw: 0.2013, cost: -1.2480, recon: -1.3384, kl: 0.4492, train_time_taken: 8.5311
INFO:tensorflow:step: 9920, lr: 0.000377, klw: 0.2016, cost: -1.3606, recon: -1.4553, kl: 0.4698, train_time_taken: 7.4324
INFO:tensorflow:step: 9940, lr: 0.000376, klw: 0.2019, cost: -1.2708, recon: -1.3617, kl: 0.4499, train_time_taken: 7.3468
INFO:tensorflow:step: 9960, lr: 0.000376, klw: 0.2022, cost: -1.3940, recon: -1.4888, kl: 0.4686, train_time_taken: 8.6397
INFO:tensorflow:step: 9980, lr: 0.000375, klw: 0.2025, cost: -1.2892, recon: -1.3800, kl: 0.4484, train_time_taken: 7.5937
INFO:tensorflow:step: 10000, lr: 0.000374, klw: 0.2028, cost: -1.2752, recon: -1.3650, kl: 0.4425, train_time_taken: 7.3100
INFO:tensorflow:best_valid_cost: -1.5015, valid_cost: -1.5015, valid_recon: -1.5061, valid_kl: 0.4558, valid_time_taken: 0.6184
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 10000.
INFO:tensorflow:time_taken_save 0.4739.
INFO:tensorflow:eval_cost: -1.4743, eval_recon: -1.4787, eval_kl: 0.4467, eval_time_taken: 0.5242
INFO:tensorflow:step: 10020, lr: 0.000373, klw: 0.2031, cost: -1.2940, recon: -1.3849, kl: 0.4474, train_time_taken: 7.3664
INFO:tensorflow:step: 10040, lr: 0.000373, klw: 0.2034, cost: -1.4103, recon: -1.5064, kl: 0.4724, train_time_taken: 7.2787
INFO:tensorflow:step: 10060, lr: 0.000372, klw: 0.2037, cost: -1.3357, recon: -1.4304, kl: 0.4653, train_time_taken: 7.2492
INFO:tensorflow:step: 10080, lr: 0.000371, klw: 0.2040, cost: -1.2672, recon: -1.3593, kl: 0.4515, train_time_taken: 8.1064
INFO:tensorflow:step: 10100, lr: 0.000371, klw: 0.2043, cost: -1.2987, recon: -1.3920, kl: 0.4564, train_time_taken: 8.3830
INFO:tensorflow:step: 10120, lr: 0.000370, klw: 0.2046, cost: -1.2959, recon: -1.3877, kl: 0.4486, train_time_taken: 7.6872
INFO:tensorflow:step: 10140, lr: 0.000369, klw: 0.2049, cost: -1.2284, recon: -1.3197, kl: 0.4458, train_time_taken: 7.2289
INFO:tensorflow:step: 10160, lr: 0.000368, klw: 0.2052, cost: -1.1958, recon: -1.2854, kl: 0.4368, train_time_taken: 7.2742
INFO:tensorflow:step: 10180, lr: 0.000368, klw: 0.2055, cost: -1.2002, recon: -1.2904, kl: 0.4391, train_time_taken: 7.3927
INFO:tensorflow:step: 10200, lr: 0.000367, klw: 0.2058, cost: -1.2689, recon: -1.3603, kl: 0.4440, train_time_taken: 7.2961
INFO:tensorflow:step: 10220, lr: 0.000366, klw: 0.2061, cost: -1.1973, recon: -1.2892, kl: 0.4462, train_time_taken: 7.2081
INFO:tensorflow:step: 10240, lr: 0.000366, klw: 0.2063, cost: -1.1832, recon: -1.2743, kl: 0.4416, train_time_taken: 7.1642
INFO:tensorflow:step: 10260, lr: 0.000365, klw: 0.2066, cost: -1.3709, recon: -1.4678, kl: 0.4692, train_time_taken: 7.1444
INFO:tensorflow:step: 10280, lr: 0.000364, klw: 0.2069, cost: -1.2871, recon: -1.3809, kl: 0.4537, train_time_taken: 7.3382
INFO:tensorflow:step: 10300, lr: 0.000363, klw: 0.2072, cost: -1.1729, recon: -1.2642, kl: 0.4406, train_time_taken: 7.2611
INFO:tensorflow:step: 10320, lr: 0.000363, klw: 0.2075, cost: -1.3064, recon: -1.3990, kl: 0.4461, train_time_taken: 8.5325
INFO:tensorflow:step: 10340, lr: 0.000362, klw: 0.2078, cost: -1.2524, recon: -1.3443, kl: 0.4425, train_time_taken: 7.5416
INFO:tensorflow:step: 10360, lr: 0.000361, klw: 0.2081, cost: -1.3691, recon: -1.4655, kl: 0.4636, train_time_taken: 7.3281
INFO:tensorflow:step: 10380, lr: 0.000361, klw: 0.2084, cost: -1.2562, recon: -1.3475, kl: 0.4382, train_time_taken: 7.2345
INFO:tensorflow:step: 10400, lr: 0.000360, klw: 0.2087, cost: -1.2396, recon: -1.3332, kl: 0.4487, train_time_taken: 7.3071
INFO:tensorflow:step: 10420, lr: 0.000359, klw: 0.2090, cost: -1.3230, recon: -1.4167, kl: 0.4481, train_time_taken: 7.2076
INFO:tensorflow:step: 10440, lr: 0.000359, klw: 0.2093, cost: -1.3762, recon: -1.4714, kl: 0.4549, train_time_taken: 7.3229
INFO:tensorflow:step: 10460, lr: 0.000358, klw: 0.2096, cost: -1.3497, recon: -1.4458, kl: 0.4586, train_time_taken: 7.1518
INFO:tensorflow:step: 10480, lr: 0.000357, klw: 0.2099, cost: -1.2745, recon: -1.3662, kl: 0.4367, train_time_taken: 7.3050
INFO:tensorflow:step: 10500, lr: 0.000356, klw: 0.2101, cost: -1.3262, recon: -1.4222, kl: 0.4566, train_time_taken: 7.2754
INFO:tensorflow:best_valid_cost: -1.5223, valid_cost: -1.5223, valid_recon: -1.5268, valid_kl: 0.4494, valid_time_taken: 0.6328
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 10500.
INFO:tensorflow:time_taken_save 0.4860.
INFO:tensorflow:eval_cost: -1.4863, eval_recon: -1.4907, eval_kl: 0.4409, eval_time_taken: 0.5243
INFO:tensorflow:step: 10520, lr: 0.000356, klw: 0.2104, cost: -1.2433, recon: -1.3349, kl: 0.4354, train_time_taken: 7.6975
INFO:tensorflow:step: 10540, lr: 0.000355, klw: 0.2107, cost: -1.2416, recon: -1.3358, kl: 0.4469, train_time_taken: 8.3728
INFO:tensorflow:step: 10560, lr: 0.000354, klw: 0.2110, cost: -1.2218, recon: -1.3144, kl: 0.4389, train_time_taken: 7.3306
INFO:tensorflow:step: 10580, lr: 0.000354, klw: 0.2113, cost: -1.2584, recon: -1.3515, kl: 0.4406, train_time_taken: 7.2014
INFO:tensorflow:step: 10600, lr: 0.000353, klw: 0.2116, cost: -1.3603, recon: -1.4565, kl: 0.4547, train_time_taken: 7.3095
INFO:tensorflow:step: 10620, lr: 0.000352, klw: 0.2119, cost: -1.3521, recon: -1.4483, kl: 0.4540, train_time_taken: 7.3127
INFO:tensorflow:step: 10640, lr: 0.000352, klw: 0.2122, cost: -1.3972, recon: -1.4942, kl: 0.4573, train_time_taken: 7.3684
INFO:tensorflow:step: 10660, lr: 0.000351, klw: 0.2125, cost: -1.3305, recon: -1.4266, kl: 0.4524, train_time_taken: 7.1684
INFO:tensorflow:step: 10680, lr: 0.000350, klw: 0.2127, cost: -1.3615, recon: -1.4565, kl: 0.4464, train_time_taken: 7.3024
INFO:tensorflow:step: 10700, lr: 0.000350, klw: 0.2130, cost: -1.2775, recon: -1.3717, kl: 0.4424, train_time_taken: 7.2607
INFO:tensorflow:step: 10720, lr: 0.000349, klw: 0.2133, cost: -1.2864, recon: -1.3821, kl: 0.4485, train_time_taken: 7.3806
INFO:tensorflow:step: 10740, lr: 0.000348, klw: 0.2136, cost: -1.2066, recon: -1.2985, kl: 0.4302, train_time_taken: 7.8650
INFO:tensorflow:step: 10760, lr: 0.000348, klw: 0.2139, cost: -1.2959, recon: -1.3898, kl: 0.4389, train_time_taken: 8.0881
INFO:tensorflow:step: 10780, lr: 0.000347, klw: 0.2142, cost: -1.3105, recon: -1.4057, kl: 0.4446, train_time_taken: 8.5584
INFO:tensorflow:step: 10800, lr: 0.000346, klw: 0.2145, cost: -1.2684, recon: -1.3611, kl: 0.4321, train_time_taken: 7.7259
INFO:tensorflow:step: 10820, lr: 0.000346, klw: 0.2147, cost: -1.2685, recon: -1.3626, kl: 0.4384, train_time_taken: 7.2124
INFO:tensorflow:step: 10840, lr: 0.000345, klw: 0.2150, cost: -1.2899, recon: -1.3832, kl: 0.4336, train_time_taken: 7.2270
INFO:tensorflow:step: 10860, lr: 0.000344, klw: 0.2153, cost: -1.3039, recon: -1.4009, kl: 0.4507, train_time_taken: 7.4120
INFO:tensorflow:step: 10880, lr: 0.000344, klw: 0.2156, cost: -1.3423, recon: -1.4384, kl: 0.4459, train_time_taken: 7.4627
INFO:tensorflow:step: 10900, lr: 0.000343, klw: 0.2159, cost: -1.2487, recon: -1.3456, kl: 0.4487, train_time_taken: 8.0404
INFO:tensorflow:step: 10920, lr: 0.000342, klw: 0.2162, cost: -1.2597, recon: -1.3547, kl: 0.4396, train_time_taken: 7.4088
INFO:tensorflow:step: 10940, lr: 0.000342, klw: 0.2164, cost: -1.3613, recon: -1.4582, kl: 0.4475, train_time_taken: 7.5285
INFO:tensorflow:step: 10960, lr: 0.000341, klw: 0.2167, cost: -1.2978, recon: -1.3939, kl: 0.4434, train_time_taken: 8.5238
INFO:tensorflow:step: 10980, lr: 0.000340, klw: 0.2170, cost: -1.2707, recon: -1.3681, kl: 0.4487, train_time_taken: 7.5070
INFO:tensorflow:step: 11000, lr: 0.000340, klw: 0.2173, cost: -1.2085, recon: -1.3038, kl: 0.4385, train_time_taken: 7.3712
INFO:tensorflow:best_valid_cost: -1.5309, valid_cost: -1.5309, valid_recon: -1.5354, valid_kl: 0.4456, valid_time_taken: 0.6316
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 11000.
INFO:tensorflow:time_taken_save 0.5032.
INFO:tensorflow:eval_cost: -1.4988, eval_recon: -1.5032, eval_kl: 0.4368, eval_time_taken: 0.5212
INFO:tensorflow:step: 11020, lr: 0.000339, klw: 0.2176, cost: -1.3458, recon: -1.4439, kl: 0.4510, train_time_taken: 7.4446
INFO:tensorflow:step: 11040, lr: 0.000338, klw: 0.2179, cost: -1.3074, recon: -1.4027, kl: 0.4375, train_time_taken: 7.3870
INFO:tensorflow:step: 11060, lr: 0.000338, klw: 0.2181, cost: -1.2496, recon: -1.3443, kl: 0.4341, train_time_taken: 7.3658
INFO:tensorflow:step: 11080, lr: 0.000337, klw: 0.2184, cost: -1.2912, recon: -1.3855, kl: 0.4317, train_time_taken: 7.3353
INFO:tensorflow:step: 11100, lr: 0.000336, klw: 0.2187, cost: -1.2406, recon: -1.3363, kl: 0.4376, train_time_taken: 7.3414
INFO:tensorflow:step: 11120, lr: 0.000336, klw: 0.2190, cost: -1.3514, recon: -1.4491, kl: 0.4464, train_time_taken: 7.3963
INFO:tensorflow:step: 11140, lr: 0.000335, klw: 0.2193, cost: -1.3427, recon: -1.4403, kl: 0.4450, train_time_taken: 7.4371
INFO:tensorflow:step: 11160, lr: 0.000334, klw: 0.2196, cost: -1.3016, recon: -1.3992, kl: 0.4443, train_time_taken: 8.2610
INFO:tensorflow:step: 11180, lr: 0.000334, klw: 0.2198, cost: -1.3158, recon: -1.4113, kl: 0.4343, train_time_taken: 8.0239
INFO:tensorflow:step: 11200, lr: 0.000333, klw: 0.2201, cost: -1.2789, recon: -1.3727, kl: 0.4264, train_time_taken: 7.2363
INFO:tensorflow:step: 11220, lr: 0.000332, klw: 0.2204, cost: -1.3391, recon: -1.4377, kl: 0.4471, train_time_taken: 7.2422
INFO:tensorflow:step: 11240, lr: 0.000332, klw: 0.2207, cost: -1.3505, recon: -1.4485, kl: 0.4444, train_time_taken: 7.5132
INFO:tensorflow:step: 11260, lr: 0.000331, klw: 0.2209, cost: -1.3376, recon: -1.4342, kl: 0.4369, train_time_taken: 7.2540
INFO:tensorflow:step: 11280, lr: 0.000330, klw: 0.2212, cost: -1.2937, recon: -1.3917, kl: 0.4430, train_time_taken: 7.3080
INFO:tensorflow:step: 11300, lr: 0.000330, klw: 0.2215, cost: -1.3167, recon: -1.4137, kl: 0.4383, train_time_taken: 7.3851
INFO:tensorflow:step: 11320, lr: 0.000329, klw: 0.2218, cost: -1.3019, recon: -1.3975, kl: 0.4311, train_time_taken: 7.3806
INFO:tensorflow:step: 11340, lr: 0.000329, klw: 0.2221, cost: -1.2472, recon: -1.3455, kl: 0.4427, train_time_taken: 7.2730
INFO:tensorflow:step: 11360, lr: 0.000328, klw: 0.2223, cost: -1.3626, recon: -1.4621, kl: 0.4475, train_time_taken: 7.4549
INFO:tensorflow:step: 11380, lr: 0.000327, klw: 0.2226, cost: -1.3322, recon: -1.4293, kl: 0.4361, train_time_taken: 8.3747
INFO:tensorflow:step: 11400, lr: 0.000327, klw: 0.2229, cost: -1.3492, recon: -1.4477, kl: 0.4418, train_time_taken: 7.8665
INFO:tensorflow:step: 11420, lr: 0.000326, klw: 0.2232, cost: -1.1685, recon: -1.2621, kl: 0.4196, train_time_taken: 7.3969
INFO:tensorflow:step: 11440, lr: 0.000325, klw: 0.2235, cost: -1.3213, recon: -1.4189, kl: 0.4371, train_time_taken: 7.4716
INFO:tensorflow:step: 11460, lr: 0.000325, klw: 0.2237, cost: -1.3540, recon: -1.4532, kl: 0.4432, train_time_taken: 7.4328
INFO:tensorflow:step: 11480, lr: 0.000324, klw: 0.2240, cost: -1.4146, recon: -1.5166, kl: 0.4552, train_time_taken: 7.4756
INFO:tensorflow:step: 11500, lr: 0.000323, klw: 0.2243, cost: -1.3014, recon: -1.3978, kl: 0.4301, train_time_taken: 7.4263
INFO:tensorflow:best_valid_cost: -1.5347, valid_cost: -1.5347, valid_recon: -1.5391, valid_kl: 0.4398, valid_time_taken: 0.6402
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 11500.
INFO:tensorflow:time_taken_save 0.5135.
INFO:tensorflow:eval_cost: -1.5012, eval_recon: -1.5055, eval_kl: 0.4309, eval_time_taken: 0.5307
INFO:tensorflow:step: 11520, lr: 0.000323, klw: 0.2246, cost: -1.3561, recon: -1.4555, kl: 0.4428, train_time_taken: 7.4549
INFO:tensorflow:step: 11540, lr: 0.000322, klw: 0.2248, cost: -1.2253, recon: -1.3212, kl: 0.4266, train_time_taken: 7.4900
INFO:tensorflow:step: 11560, lr: 0.000322, klw: 0.2251, cost: -1.2917, recon: -1.3880, kl: 0.4279, train_time_taken: 7.4917
INFO:tensorflow:step: 11580, lr: 0.000321, klw: 0.2254, cost: -1.2590, recon: -1.3552, kl: 0.4267, train_time_taken: 7.9572
INFO:tensorflow:step: 11600, lr: 0.000320, klw: 0.2257, cost: -1.2538, recon: -1.3499, kl: 0.4260, train_time_taken: 8.7527
INFO:tensorflow:step: 11620, lr: 0.000320, klw: 0.2259, cost: -1.2610, recon: -1.3569, kl: 0.4247, train_time_taken: 7.5207
INFO:tensorflow:step: 11640, lr: 0.000319, klw: 0.2262, cost: -1.2810, recon: -1.3789, kl: 0.4326, train_time_taken: 7.5048
INFO:tensorflow:step: 11660, lr: 0.000318, klw: 0.2265, cost: -1.3178, recon: -1.4166, kl: 0.4363, train_time_taken: 7.3493
INFO:tensorflow:step: 11680, lr: 0.000318, klw: 0.2267, cost: -1.2582, recon: -1.3546, kl: 0.4251, train_time_taken: 7.8898
INFO:tensorflow:step: 11700, lr: 0.000317, klw: 0.2270, cost: -1.2063, recon: -1.3016, kl: 0.4200, train_time_taken: 7.8907
INFO:tensorflow:step: 11720, lr: 0.000317, klw: 0.2273, cost: -1.3680, recon: -1.4687, kl: 0.4430, train_time_taken: 7.5456
INFO:tensorflow:step: 11740, lr: 0.000316, klw: 0.2276, cost: -1.2689, recon: -1.3665, kl: 0.4289, train_time_taken: 7.4301
INFO:tensorflow:step: 11760, lr: 0.000315, klw: 0.2278, cost: -1.2854, recon: -1.3828, kl: 0.4274, train_time_taken: 7.5472
INFO:tensorflow:step: 11780, lr: 0.000315, klw: 0.2281, cost: -1.2814, recon: -1.3783, kl: 0.4244, train_time_taken: 7.4558
INFO:tensorflow:step: 11800, lr: 0.000314, klw: 0.2284, cost: -1.3167, recon: -1.4169, kl: 0.4384, train_time_taken: 8.6484
INFO:tensorflow:step: 11820, lr: 0.000314, klw: 0.2287, cost: -1.2893, recon: -1.3860, kl: 0.4228, train_time_taken: 7.6114
INFO:tensorflow:step: 11840, lr: 0.000313, klw: 0.2289, cost: -1.2874, recon: -1.3852, kl: 0.4273, train_time_taken: 7.5597
INFO:tensorflow:step: 11860, lr: 0.000312, klw: 0.2292, cost: -1.2722, recon: -1.3696, kl: 0.4250, train_time_taken: 7.5620
INFO:tensorflow:step: 11880, lr: 0.000312, klw: 0.2295, cost: -1.2884, recon: -1.3868, kl: 0.4290, train_time_taken: 7.5029
INFO:tensorflow:step: 11900, lr: 0.000311, klw: 0.2297, cost: -1.3444, recon: -1.4440, kl: 0.4336, train_time_taken: 7.4153
INFO:tensorflow:step: 11920, lr: 0.000311, klw: 0.2300, cost: -1.3254, recon: -1.4239, kl: 0.4286, train_time_taken: 7.5544
INFO:tensorflow:step: 11940, lr: 0.000310, klw: 0.2303, cost: -1.2126, recon: -1.3091, kl: 0.4193, train_time_taken: 7.4804
INFO:tensorflow:step: 11960, lr: 0.000309, klw: 0.2305, cost: -1.3872, recon: -1.4906, kl: 0.4482, train_time_taken: 7.5182
INFO:tensorflow:step: 11980, lr: 0.000309, klw: 0.2308, cost: -1.3042, recon: -1.4031, kl: 0.4284, train_time_taken: 7.5285
INFO:tensorflow:step: 12000, lr: 0.000308, klw: 0.2311, cost: -1.3468, recon: -1.4479, kl: 0.4373, train_time_taken: 8.0855
INFO:tensorflow:best_valid_cost: -1.5471, valid_cost: -1.5471, valid_recon: -1.5514, valid_kl: 0.4344, valid_time_taken: 0.7073
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 12000.
INFO:tensorflow:time_taken_save 0.5972.
INFO:tensorflow:eval_cost: -1.5129, eval_recon: -1.5172, eval_kl: 0.4262, eval_time_taken: 0.5862
INFO:tensorflow:step: 12020, lr: 0.000308, klw: 0.2314, cost: -1.2178, recon: -1.3148, kl: 0.4191, train_time_taken: 8.0120
INFO:tensorflow:step: 12040, lr: 0.000307, klw: 0.2316, cost: -1.3898, recon: -1.4899, kl: 0.4324, train_time_taken: 7.3778
INFO:tensorflow:step: 12060, lr: 0.000306, klw: 0.2319, cost: -1.3055, recon: -1.4026, kl: 0.4187, train_time_taken: 7.6305
INFO:tensorflow:step: 12080, lr: 0.000306, klw: 0.2322, cost: -1.3625, recon: -1.4636, kl: 0.4355, train_time_taken: 7.4895
INFO:tensorflow:step: 12100, lr: 0.000305, klw: 0.2324, cost: -1.2978, recon: -1.3948, kl: 0.4173, train_time_taken: 7.5834
INFO:tensorflow:step: 12120, lr: 0.000305, klw: 0.2327, cost: -1.3816, recon: -1.4826, kl: 0.4343, train_time_taken: 7.5908
INFO:tensorflow:step: 12140, lr: 0.000304, klw: 0.2330, cost: -1.3178, recon: -1.4172, kl: 0.4267, train_time_taken: 7.4002
INFO:tensorflow:step: 12160, lr: 0.000303, klw: 0.2332, cost: -1.4135, recon: -1.5162, kl: 0.4404, train_time_taken: 7.5569
INFO:tensorflow:step: 12180, lr: 0.000303, klw: 0.2335, cost: -1.2980, recon: -1.3963, kl: 0.4214, train_time_taken: 7.5707
INFO:tensorflow:step: 12200, lr: 0.000302, klw: 0.2338, cost: -1.4296, recon: -1.5307, kl: 0.4327, train_time_taken: 7.7079
INFO:tensorflow:step: 12220, lr: 0.000302, klw: 0.2340, cost: -1.3961, recon: -1.4978, kl: 0.4348, train_time_taken: 8.5966
INFO:tensorflow:step: 12240, lr: 0.000301, klw: 0.2343, cost: -1.2456, recon: -1.3450, kl: 0.4244, train_time_taken: 7.4871
INFO:tensorflow:step: 12260, lr: 0.000301, klw: 0.2346, cost: -1.3282, recon: -1.4293, kl: 0.4312, train_time_taken: 7.5014
INFO:tensorflow:step: 12280, lr: 0.000300, klw: 0.2348, cost: -1.3379, recon: -1.4388, kl: 0.4297, train_time_taken: 7.6076
INFO:tensorflow:step: 12300, lr: 0.000299, klw: 0.2351, cost: -1.3756, recon: -1.4774, kl: 0.4330, train_time_taken: 7.4829
INFO:tensorflow:step: 12320, lr: 0.000299, klw: 0.2354, cost: -1.2934, recon: -1.3939, kl: 0.4268, train_time_taken: 7.5761
INFO:tensorflow:step: 12340, lr: 0.000298, klw: 0.2356, cost: -1.3128, recon: -1.4127, kl: 0.4240, train_time_taken: 7.4947
INFO:tensorflow:step: 12360, lr: 0.000298, klw: 0.2359, cost: -1.3068, recon: -1.4059, kl: 0.4200, train_time_taken: 7.5711
INFO:tensorflow:step: 12380, lr: 0.000297, klw: 0.2361, cost: -1.2392, recon: -1.3378, kl: 0.4177, train_time_taken: 7.4380
INFO:tensorflow:step: 12400, lr: 0.000296, klw: 0.2364, cost: -1.3615, recon: -1.4636, kl: 0.4320, train_time_taken: 8.7601
INFO:tensorflow:step: 12420, lr: 0.000296, klw: 0.2367, cost: -1.2438, recon: -1.3436, kl: 0.4216, train_time_taken: 8.6663
INFO:tensorflow:step: 12440, lr: 0.000295, klw: 0.2369, cost: -1.2194, recon: -1.3177, kl: 0.4147, train_time_taken: 7.8665
INFO:tensorflow:step: 12460, lr: 0.000295, klw: 0.2372, cost: -1.3605, recon: -1.4613, kl: 0.4249, train_time_taken: 7.5410
INFO:tensorflow:step: 12480, lr: 0.000294, klw: 0.2375, cost: -1.3043, recon: -1.4033, kl: 0.4168, train_time_taken: 8.2298
INFO:tensorflow:step: 12500, lr: 0.000294, klw: 0.2377, cost: -1.3385, recon: -1.4395, kl: 0.4249, train_time_taken: 7.6078
INFO:tensorflow:best_valid_cost: -1.5662, valid_cost: -1.5662, valid_recon: -1.5705, valid_kl: 0.4271, valid_time_taken: 0.6486
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 12500.
INFO:tensorflow:time_taken_save 0.5203.
INFO:tensorflow:eval_cost: -1.5312, eval_recon: -1.5354, eval_kl: 0.4187, eval_time_taken: 0.5295
INFO:tensorflow:step: 12520, lr: 0.000293, klw: 0.2380, cost: -1.3570, recon: -1.4587, kl: 0.4274, train_time_taken: 7.5221
INFO:tensorflow:step: 12540, lr: 0.000292, klw: 0.2383, cost: -1.3933, recon: -1.4961, kl: 0.4314, train_time_taken: 7.5196
INFO:tensorflow:step: 12560, lr: 0.000292, klw: 0.2385, cost: -1.3528, recon: -1.4575, kl: 0.4391, train_time_taken: 7.6105
INFO:tensorflow:step: 12580, lr: 0.000291, klw: 0.2388, cost: -1.3812, recon: -1.4825, kl: 0.4245, train_time_taken: 7.4280
INFO:tensorflow:step: 12600, lr: 0.000291, klw: 0.2390, cost: -1.3261, recon: -1.4263, kl: 0.4192, train_time_taken: 7.4836
INFO:tensorflow:step: 12620, lr: 0.000290, klw: 0.2393, cost: -1.3594, recon: -1.4611, kl: 0.4250, train_time_taken: 8.1788
INFO:tensorflow:step: 12640, lr: 0.000290, klw: 0.2396, cost: -1.3380, recon: -1.4399, kl: 0.4251, train_time_taken: 8.0378
INFO:tensorflow:step: 12660, lr: 0.000289, klw: 0.2398, cost: -1.3901, recon: -1.4954, kl: 0.4393, train_time_taken: 7.5119
INFO:tensorflow:step: 12680, lr: 0.000289, klw: 0.2401, cost: -1.2613, recon: -1.3637, kl: 0.4263, train_time_taken: 7.5923
INFO:tensorflow:step: 12700, lr: 0.000288, klw: 0.2403, cost: -1.2423, recon: -1.3406, kl: 0.4090, train_time_taken: 7.5557
INFO:tensorflow:step: 12720, lr: 0.000287, klw: 0.2406, cost: -1.3466, recon: -1.4502, kl: 0.4306, train_time_taken: 7.5626
INFO:tensorflow:step: 12740, lr: 0.000287, klw: 0.2409, cost: -1.3739, recon: -1.4767, kl: 0.4267, train_time_taken: 7.5972
INFO:tensorflow:step: 12760, lr: 0.000286, klw: 0.2411, cost: -1.3413, recon: -1.4434, kl: 0.4238, train_time_taken: 7.5518
INFO:tensorflow:step: 12780, lr: 0.000286, klw: 0.2414, cost: -1.2799, recon: -1.3797, kl: 0.4135, train_time_taken: 7.4523
INFO:tensorflow:step: 12800, lr: 0.000285, klw: 0.2416, cost: -1.3324, recon: -1.4326, kl: 0.4146, train_time_taken: 7.5698
INFO:tensorflow:step: 12820, lr: 0.000285, klw: 0.2419, cost: -1.2558, recon: -1.3561, kl: 0.4146, train_time_taken: 7.6416
INFO:tensorflow:step: 12840, lr: 0.000284, klw: 0.2421, cost: -1.3086, recon: -1.4104, kl: 0.4204, train_time_taken: 8.5640
INFO:tensorflow:step: 12860, lr: 0.000284, klw: 0.2424, cost: -1.3837, recon: -1.4894, kl: 0.4362, train_time_taken: 7.5704
INFO:tensorflow:step: 12880, lr: 0.000283, klw: 0.2427, cost: -1.2245, recon: -1.3227, kl: 0.4045, train_time_taken: 7.3819
INFO:tensorflow:step: 12900, lr: 0.000283, klw: 0.2429, cost: -1.3189, recon: -1.4199, kl: 0.4158, train_time_taken: 7.4084
INFO:tensorflow:step: 12920, lr: 0.000282, klw: 0.2432, cost: -1.3246, recon: -1.4290, kl: 0.4294, train_time_taken: 7.5136
INFO:tensorflow:step: 12940, lr: 0.000281, klw: 0.2434, cost: -1.3027, recon: -1.4063, kl: 0.4253, train_time_taken: 7.5685
INFO:tensorflow:step: 12960, lr: 0.000281, klw: 0.2437, cost: -1.3439, recon: -1.4458, kl: 0.4181, train_time_taken: 7.6604
INFO:tensorflow:step: 12980, lr: 0.000280, klw: 0.2439, cost: -1.3724, recon: -1.4760, kl: 0.4247, train_time_taken: 7.5287
INFO:tensorflow:step: 13000, lr: 0.000280, klw: 0.2442, cost: -1.3531, recon: -1.4564, kl: 0.4233, train_time_taken: 7.5128
INFO:tensorflow:best_valid_cost: -1.5738, valid_cost: -1.5738, valid_recon: -1.5780, valid_kl: 0.4263, valid_time_taken: 0.6490
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 13000.
INFO:tensorflow:time_taken_save 0.5290.
INFO:tensorflow:eval_cost: -1.5442, eval_recon: -1.5483, eval_kl: 0.4178, eval_time_taken: 0.5482
INFO:tensorflow:step: 13020, lr: 0.000279, klw: 0.2445, cost: -1.3134, recon: -1.4154, kl: 0.4173, train_time_taken: 7.4272
INFO:tensorflow:step: 13040, lr: 0.000279, klw: 0.2447, cost: -1.3154, recon: -1.4174, kl: 0.4169, train_time_taken: 8.2619
INFO:tensorflow:step: 13060, lr: 0.000278, klw: 0.2450, cost: -1.2911, recon: -1.3924, kl: 0.4136, train_time_taken: 7.9997
INFO:tensorflow:step: 13080, lr: 0.000278, klw: 0.2452, cost: -1.4067, recon: -1.5154, kl: 0.4432, train_time_taken: 7.5079
INFO:tensorflow:step: 13100, lr: 0.000277, klw: 0.2455, cost: -1.3574, recon: -1.4607, kl: 0.4210, train_time_taken: 7.5692
INFO:tensorflow:step: 13120, lr: 0.000277, klw: 0.2457, cost: -1.3197, recon: -1.4217, kl: 0.4152, train_time_taken: 7.4702
INFO:tensorflow:step: 13140, lr: 0.000276, klw: 0.2460, cost: -1.4022, recon: -1.5071, kl: 0.4265, train_time_taken: 7.4821
INFO:tensorflow:step: 13160, lr: 0.000276, klw: 0.2462, cost: -1.3732, recon: -1.4775, kl: 0.4234, train_time_taken: 7.4803
INFO:tensorflow:step: 13180, lr: 0.000275, klw: 0.2465, cost: -1.3240, recon: -1.4281, kl: 0.4224, train_time_taken: 7.5959
INFO:tensorflow:step: 13200, lr: 0.000274, klw: 0.2467, cost: -1.4054, recon: -1.5092, kl: 0.4206, train_time_taken: 8.8486
INFO:tensorflow:step: 13220, lr: 0.000274, klw: 0.2470, cost: -1.3282, recon: -1.4301, kl: 0.4126, train_time_taken: 7.7208
INFO:tensorflow:step: 13240, lr: 0.000273, klw: 0.2473, cost: -1.4120, recon: -1.5188, kl: 0.4322, train_time_taken: 8.0679
INFO:tensorflow:step: 13260, lr: 0.000273, klw: 0.2475, cost: -1.3225, recon: -1.4255, kl: 0.4162, train_time_taken: 8.7364
INFO:tensorflow:step: 13280, lr: 0.000272, klw: 0.2478, cost: -1.2006, recon: -1.3023, kl: 0.4106, train_time_taken: 7.4441
INFO:tensorflow:step: 13300, lr: 0.000272, klw: 0.2480, cost: -1.4313, recon: -1.5368, kl: 0.4252, train_time_taken: 7.5501
INFO:tensorflow:step: 13320, lr: 0.000271, klw: 0.2483, cost: -1.3304, recon: -1.4340, kl: 0.4176, train_time_taken: 7.4627
INFO:tensorflow:step: 13340, lr: 0.000271, klw: 0.2485, cost: -1.2667, recon: -1.3685, kl: 0.4098, train_time_taken: 7.5533
INFO:tensorflow:step: 13360, lr: 0.000270, klw: 0.2488, cost: -1.2991, recon: -1.4026, kl: 0.4161, train_time_taken: 7.6120
INFO:tensorflow:step: 13380, lr: 0.000270, klw: 0.2490, cost: -1.2900, recon: -1.3931, kl: 0.4139, train_time_taken: 7.5723
INFO:tensorflow:step: 13400, lr: 0.000269, klw: 0.2493, cost: -1.2857, recon: -1.3862, kl: 0.4033, train_time_taken: 7.3620
INFO:tensorflow:step: 13420, lr: 0.000269, klw: 0.2495, cost: -1.3471, recon: -1.4513, kl: 0.4178, train_time_taken: 7.4581
INFO:tensorflow:step: 13440, lr: 0.000268, klw: 0.2498, cost: -1.2749, recon: -1.3786, kl: 0.4154, train_time_taken: 7.4946
INFO:tensorflow:step: 13460, lr: 0.000268, klw: 0.2500, cost: -1.3463, recon: -1.4529, kl: 0.4262, train_time_taken: 8.5485
INFO:tensorflow:step: 13480, lr: 0.000267, klw: 0.2503, cost: -1.3965, recon: -1.5032, kl: 0.4261, train_time_taken: 7.7602
INFO:tensorflow:step: 13500, lr: 0.000267, klw: 0.2505, cost: -1.3632, recon: -1.4694, kl: 0.4239, train_time_taken: 7.4126
INFO:tensorflow:best_valid_cost: -1.5797, valid_cost: -1.5797, valid_recon: -1.5840, valid_kl: 0.4240, valid_time_taken: 0.6267
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 13500.
INFO:tensorflow:time_taken_save 0.7056.
INFO:tensorflow:eval_cost: -1.5452, eval_recon: -1.5493, eval_kl: 0.4157, eval_time_taken: 0.5284
INFO:tensorflow:step: 13520, lr: 0.000266, klw: 0.2508, cost: -1.3930, recon: -1.4976, kl: 0.4170, train_time_taken: 7.3791
INFO:tensorflow:step: 13540, lr: 0.000266, klw: 0.2510, cost: -1.3221, recon: -1.4272, kl: 0.4189, train_time_taken: 7.4496
INFO:tensorflow:step: 13560, lr: 0.000265, klw: 0.2513, cost: -1.3432, recon: -1.4479, kl: 0.4168, train_time_taken: 7.4378
INFO:tensorflow:step: 13580, lr: 0.000265, klw: 0.2515, cost: -1.3642, recon: -1.4687, kl: 0.4154, train_time_taken: 7.5701
INFO:tensorflow:step: 13600, lr: 0.000264, klw: 0.2518, cost: -1.3023, recon: -1.4047, kl: 0.4071, train_time_taken: 7.5671
INFO:tensorflow:step: 13620, lr: 0.000264, klw: 0.2520, cost: -1.3706, recon: -1.4754, kl: 0.4160, train_time_taken: 7.5424
INFO:tensorflow:step: 13640, lr: 0.000263, klw: 0.2523, cost: -1.3339, recon: -1.4370, kl: 0.4087, train_time_taken: 7.4347
INFO:tensorflow:step: 13660, lr: 0.000263, klw: 0.2525, cost: -1.3561, recon: -1.4611, kl: 0.4159, train_time_taken: 8.3306
INFO:tensorflow:step: 13680, lr: 0.000262, klw: 0.2528, cost: -1.3519, recon: -1.4551, kl: 0.4085, train_time_taken: 7.9966
INFO:tensorflow:step: 13700, lr: 0.000262, klw: 0.2530, cost: -1.3056, recon: -1.4080, kl: 0.4048, train_time_taken: 7.5426
INFO:tensorflow:step: 13720, lr: 0.000261, klw: 0.2532, cost: -1.3202, recon: -1.4257, kl: 0.4168, train_time_taken: 7.4290
INFO:tensorflow:step: 13740, lr: 0.000261, klw: 0.2535, cost: -1.3679, recon: -1.4746, kl: 0.4207, train_time_taken: 7.4804
INFO:tensorflow:step: 13760, lr: 0.000260, klw: 0.2537, cost: -1.3120, recon: -1.4160, kl: 0.4099, train_time_taken: 7.4640
INFO:tensorflow:step: 13780, lr: 0.000260, klw: 0.2540, cost: -1.3245, recon: -1.4278, kl: 0.4068, train_time_taken: 7.4077
INFO:tensorflow:step: 13800, lr: 0.000259, klw: 0.2542, cost: -1.4080, recon: -1.5143, kl: 0.4182, train_time_taken: 7.5486
INFO:tensorflow:step: 13820, lr: 0.000259, klw: 0.2545, cost: -1.3439, recon: -1.4483, kl: 0.4103, train_time_taken: 7.3706
INFO:tensorflow:step: 13840, lr: 0.000258, klw: 0.2547, cost: -1.3839, recon: -1.4901, kl: 0.4168, train_time_taken: 7.5649
INFO:tensorflow:step: 13860, lr: 0.000258, klw: 0.2550, cost: -1.4554, recon: -1.5636, kl: 0.4244, train_time_taken: 7.6359
INFO:tensorflow:step: 13880, lr: 0.000257, klw: 0.2552, cost: -1.3639, recon: -1.4705, kl: 0.4176, train_time_taken: 8.5581
INFO:tensorflow:step: 13900, lr: 0.000257, klw: 0.2555, cost: -1.4137, recon: -1.5206, kl: 0.4183, train_time_taken: 7.4939
INFO:tensorflow:step: 13920, lr: 0.000256, klw: 0.2557, cost: -1.3829, recon: -1.4905, kl: 0.4209, train_time_taken: 7.4817
INFO:tensorflow:step: 13940, lr: 0.000256, klw: 0.2559, cost: -1.4779, recon: -1.5871, kl: 0.4268, train_time_taken: 7.4730
INFO:tensorflow:step: 13960, lr: 0.000255, klw: 0.2562, cost: -1.3069, recon: -1.4116, kl: 0.4083, train_time_taken: 7.5586
INFO:tensorflow:step: 13980, lr: 0.000255, klw: 0.2564, cost: -1.3294, recon: -1.4369, kl: 0.4195, train_time_taken: 7.5991
INFO:tensorflow:step: 14000, lr: 0.000254, klw: 0.2567, cost: -1.2862, recon: -1.3923, kl: 0.4133, train_time_taken: 8.5503
INFO:tensorflow:best_valid_cost: -1.5892, valid_cost: -1.5892, valid_recon: -1.5934, valid_kl: 0.4181, valid_time_taken: 0.7263
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 14000.
INFO:tensorflow:time_taken_save 0.6265.
INFO:tensorflow:eval_cost: -1.5552, eval_recon: -1.5593, eval_kl: 0.4100, eval_time_taken: 0.5916
INFO:tensorflow:step: 14020, lr: 0.000254, klw: 0.2569, cost: -1.3297, recon: -1.4360, kl: 0.4138, train_time_taken: 7.6434
INFO:tensorflow:step: 14040, lr: 0.000253, klw: 0.2572, cost: -1.3690, recon: -1.4757, kl: 0.4150, train_time_taken: 7.5894
INFO:tensorflow:step: 14060, lr: 0.000253, klw: 0.2574, cost: -1.2859, recon: -1.3914, kl: 0.4095, train_time_taken: 7.9200
INFO:tensorflow:step: 14080, lr: 0.000252, klw: 0.2576, cost: -1.4821, recon: -1.5940, kl: 0.4344, train_time_taken: 8.4423
INFO:tensorflow:step: 14100, lr: 0.000252, klw: 0.2579, cost: -1.3078, recon: -1.4132, kl: 0.4084, train_time_taken: 7.7808
INFO:tensorflow:step: 14120, lr: 0.000251, klw: 0.2581, cost: -1.3980, recon: -1.5069, kl: 0.4217, train_time_taken: 7.4493
INFO:tensorflow:step: 14140, lr: 0.000251, klw: 0.2584, cost: -1.3552, recon: -1.4637, kl: 0.4198, train_time_taken: 7.4826
INFO:tensorflow:step: 14160, lr: 0.000250, klw: 0.2586, cost: -1.4273, recon: -1.5372, kl: 0.4251, train_time_taken: 7.4573
INFO:tensorflow:step: 14180, lr: 0.000250, klw: 0.2589, cost: -1.3498, recon: -1.4566, kl: 0.4127, train_time_taken: 7.4187
INFO:tensorflow:step: 14200, lr: 0.000249, klw: 0.2591, cost: -1.3588, recon: -1.4648, kl: 0.4092, train_time_taken: 7.4163
INFO:tensorflow:step: 14220, lr: 0.000249, klw: 0.2593, cost: -1.3631, recon: -1.4697, kl: 0.4111, train_time_taken: 7.5255
INFO:tensorflow:step: 14240, lr: 0.000248, klw: 0.2596, cost: -1.3891, recon: -1.4992, kl: 0.4241, train_time_taken: 7.4607
INFO:tensorflow:step: 14260, lr: 0.000248, klw: 0.2598, cost: -1.3377, recon: -1.4451, kl: 0.4136, train_time_taken: 7.3860
INFO:tensorflow:step: 14280, lr: 0.000247, klw: 0.2601, cost: -1.3883, recon: -1.4972, kl: 0.4186, train_time_taken: 7.8768
INFO:tensorflow:step: 14300, lr: 0.000247, klw: 0.2603, cost: -1.3309, recon: -1.4375, kl: 0.4093, train_time_taken: 8.3293
INFO:tensorflow:step: 14320, lr: 0.000246, klw: 0.2605, cost: -1.3040, recon: -1.4115, kl: 0.4125, train_time_taken: 7.6107
INFO:tensorflow:step: 14340, lr: 0.000246, klw: 0.2608, cost: -1.3754, recon: -1.4839, kl: 0.4163, train_time_taken: 7.5173
INFO:tensorflow:step: 14360, lr: 0.000245, klw: 0.2610, cost: -1.4232, recon: -1.5305, kl: 0.4111, train_time_taken: 7.5812
INFO:tensorflow:step: 14380, lr: 0.000245, klw: 0.2613, cost: -1.3691, recon: -1.4766, kl: 0.4112, train_time_taken: 7.4490
INFO:tensorflow:step: 14400, lr: 0.000245, klw: 0.2615, cost: -1.4274, recon: -1.5356, kl: 0.4136, train_time_taken: 7.6789
INFO:tensorflow:step: 14420, lr: 0.000244, klw: 0.2617, cost: -1.3502, recon: -1.4582, kl: 0.4125, train_time_taken: 7.3705
INFO:tensorflow:step: 14440, lr: 0.000244, klw: 0.2620, cost: -1.3219, recon: -1.4299, kl: 0.4123, train_time_taken: 7.5063
INFO:tensorflow:step: 14460, lr: 0.000243, klw: 0.2622, cost: -1.3177, recon: -1.4251, kl: 0.4098, train_time_taken: 7.4157
INFO:tensorflow:step: 14480, lr: 0.000243, klw: 0.2624, cost: -1.3670, recon: -1.4757, kl: 0.4141, train_time_taken: 7.4636
INFO:tensorflow:step: 14500, lr: 0.000242, klw: 0.2627, cost: -1.2179, recon: -1.3233, kl: 0.4013, train_time_taken: 8.3320
INFO:tensorflow:best_valid_cost: -1.5963, valid_cost: -1.5963, valid_recon: -1.6004, valid_kl: 0.4137, valid_time_taken: 0.7288
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 14500.
INFO:tensorflow:time_taken_save 0.6074.
INFO:tensorflow:eval_cost: -1.5588, eval_recon: -1.5629, eval_kl: 0.4054, eval_time_taken: 0.6039
INFO:tensorflow:step: 14520, lr: 0.000242, klw: 0.2629, cost: -1.2619, recon: -1.3662, kl: 0.3966, train_time_taken: 7.5863
INFO:tensorflow:step: 14540, lr: 0.000241, klw: 0.2632, cost: -1.4713, recon: -1.5826, kl: 0.4230, train_time_taken: 7.5183
INFO:tensorflow:step: 14560, lr: 0.000241, klw: 0.2634, cost: -1.2849, recon: -1.3926, kl: 0.4091, train_time_taken: 7.4938
INFO:tensorflow:step: 14580, lr: 0.000240, klw: 0.2636, cost: -1.3287, recon: -1.4370, kl: 0.4107, train_time_taken: 7.4925
INFO:tensorflow:step: 14600, lr: 0.000240, klw: 0.2639, cost: -1.4049, recon: -1.5146, kl: 0.4158, train_time_taken: 7.4837
INFO:tensorflow:step: 14620, lr: 0.000239, klw: 0.2641, cost: -1.3465, recon: -1.4532, kl: 0.4039, train_time_taken: 7.4000
INFO:tensorflow:step: 14640, lr: 0.000239, klw: 0.2643, cost: -1.4508, recon: -1.5617, kl: 0.4193, train_time_taken: 7.5230
INFO:tensorflow:step: 14660, lr: 0.000239, klw: 0.2646, cost: -1.3767, recon: -1.4853, kl: 0.4106, train_time_taken: 7.4551
INFO:tensorflow:step: 14680, lr: 0.000238, klw: 0.2648, cost: -1.4101, recon: -1.5194, kl: 0.4128, train_time_taken: 7.5026
INFO:tensorflow:step: 14700, lr: 0.000238, klw: 0.2650, cost: -1.3486, recon: -1.4584, kl: 0.4142, train_time_taken: 8.0341
INFO:tensorflow:step: 14720, lr: 0.000237, klw: 0.2653, cost: -1.3273, recon: -1.4372, kl: 0.4142, train_time_taken: 8.1431
INFO:tensorflow:step: 14740, lr: 0.000237, klw: 0.2655, cost: -1.3360, recon: -1.4450, kl: 0.4105, train_time_taken: 7.5623
INFO:tensorflow:step: 14760, lr: 0.000236, klw: 0.2658, cost: -1.3103, recon: -1.4159, kl: 0.3971, train_time_taken: 7.4699
INFO:tensorflow:step: 14780, lr: 0.000236, klw: 0.2660, cost: -1.3193, recon: -1.4266, kl: 0.4034, train_time_taken: 7.4673
INFO:tensorflow:step: 14800, lr: 0.000235, klw: 0.2662, cost: -1.3448, recon: -1.4527, kl: 0.4055, train_time_taken: 8.5938
INFO:tensorflow:step: 14820, lr: 0.000235, klw: 0.2665, cost: -1.3110, recon: -1.4183, kl: 0.4026, train_time_taken: 8.0298
INFO:tensorflow:step: 14840, lr: 0.000234, klw: 0.2667, cost: -1.3439, recon: -1.4530, kl: 0.4092, train_time_taken: 8.1428
INFO:tensorflow:step: 14860, lr: 0.000234, klw: 0.2669, cost: -1.3803, recon: -1.4903, kl: 0.4120, train_time_taken: 7.5880
INFO:tensorflow:step: 14880, lr: 0.000234, klw: 0.2672, cost: -1.3102, recon: -1.4197, kl: 0.4096, train_time_taken: 7.7539
INFO:tensorflow:step: 14900, lr: 0.000233, klw: 0.2674, cost: -1.3457, recon: -1.4541, kl: 0.4053, train_time_taken: 7.7114
INFO:tensorflow:step: 14920, lr: 0.000233, klw: 0.2676, cost: -1.3484, recon: -1.4590, kl: 0.4134, train_time_taken: 8.4373
INFO:tensorflow:step: 14940, lr: 0.000232, klw: 0.2678, cost: -1.4321, recon: -1.5452, kl: 0.4226, train_time_taken: 7.5908
INFO:tensorflow:step: 14960, lr: 0.000232, klw: 0.2681, cost: -1.4257, recon: -1.5366, kl: 0.4137, train_time_taken: 7.3626
INFO:tensorflow:step: 14980, lr: 0.000231, klw: 0.2683, cost: -1.3466, recon: -1.4548, kl: 0.4033, train_time_taken: 7.4615
INFO:tensorflow:step: 15000, lr: 0.000231, klw: 0.2685, cost: -1.3359, recon: -1.4437, kl: 0.4014, train_time_taken: 7.3715
INFO:tensorflow:best_valid_cost: -1.6024, valid_cost: -1.6024, valid_recon: -1.6065, valid_kl: 0.4091, valid_time_taken: 0.6417
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 15000.
INFO:tensorflow:time_taken_save 0.5696.
INFO:tensorflow:eval_cost: -1.5687, eval_recon: -1.5727, eval_kl: 0.4010, eval_time_taken: 0.5207
INFO:tensorflow:step: 15020, lr: 0.000230, klw: 0.2688, cost: -1.2695, recon: -1.3770, kl: 0.3999, train_time_taken: 7.5136
INFO:tensorflow:step: 15040, lr: 0.000230, klw: 0.2690, cost: -1.4252, recon: -1.5349, kl: 0.4080, train_time_taken: 7.3954
INFO:tensorflow:step: 15060, lr: 0.000230, klw: 0.2692, cost: -1.4314, recon: -1.5439, kl: 0.4180, train_time_taken: 7.5356
INFO:tensorflow:step: 15080, lr: 0.000229, klw: 0.2695, cost: -1.3667, recon: -1.4752, kl: 0.4026, train_time_taken: 7.3837
INFO:tensorflow:step: 15100, lr: 0.000229, klw: 0.2697, cost: -1.3467, recon: -1.4565, kl: 0.4071, train_time_taken: 7.3962
INFO:tensorflow:step: 15120, lr: 0.000228, klw: 0.2699, cost: -1.3387, recon: -1.4474, kl: 0.4028, train_time_taken: 8.5579
INFO:tensorflow:step: 15140, lr: 0.000228, klw: 0.2702, cost: -1.4574, recon: -1.5692, kl: 0.4139, train_time_taken: 7.8797
INFO:tensorflow:step: 15160, lr: 0.000227, klw: 0.2704, cost: -1.3836, recon: -1.4955, kl: 0.4138, train_time_taken: 7.5086
INFO:tensorflow:step: 15180, lr: 0.000227, klw: 0.2706, cost: -1.4012, recon: -1.5125, kl: 0.4114, train_time_taken: 7.4627
INFO:tensorflow:step: 15200, lr: 0.000227, klw: 0.2708, cost: -1.2982, recon: -1.4064, kl: 0.3995, train_time_taken: 7.6184
INFO:tensorflow:step: 15220, lr: 0.000226, klw: 0.2711, cost: -1.2988, recon: -1.4078, kl: 0.4021, train_time_taken: 7.4553
INFO:tensorflow:step: 15240, lr: 0.000226, klw: 0.2713, cost: -1.2797, recon: -1.3876, kl: 0.3978, train_time_taken: 7.5606
INFO:tensorflow:step: 15260, lr: 0.000225, klw: 0.2715, cost: -1.4355, recon: -1.5476, kl: 0.4128, train_time_taken: 7.4923
INFO:tensorflow:step: 15280, lr: 0.000225, klw: 0.2718, cost: -1.4217, recon: -1.5345, kl: 0.4152, train_time_taken: 7.4429
INFO:tensorflow:step: 15300, lr: 0.000224, klw: 0.2720, cost: -1.3597, recon: -1.4703, kl: 0.4069, train_time_taken: 7.3904
INFO:tensorflow:step: 15320, lr: 0.000224, klw: 0.2722, cost: -1.3721, recon: -1.4834, kl: 0.4086, train_time_taken: 7.9821
INFO:tensorflow:step: 15340, lr: 0.000223, klw: 0.2724, cost: -1.3814, recon: -1.4923, kl: 0.4071, train_time_taken: 8.3239
INFO:tensorflow:step: 15360, lr: 0.000223, klw: 0.2727, cost: -1.3958, recon: -1.5100, kl: 0.4187, train_time_taken: 7.4818
INFO:tensorflow:step: 15380, lr: 0.000223, klw: 0.2729, cost: -1.4025, recon: -1.5150, kl: 0.4123, train_time_taken: 7.5270
INFO:tensorflow:step: 15400, lr: 0.000222, klw: 0.2731, cost: -1.3873, recon: -1.4990, kl: 0.4089, train_time_taken: 7.4048
INFO:tensorflow:step: 15420, lr: 0.000222, klw: 0.2734, cost: -1.4173, recon: -1.5307, kl: 0.4148, train_time_taken: 7.5344
INFO:tensorflow:step: 15440, lr: 0.000221, klw: 0.2736, cost: -1.4773, recon: -1.5935, kl: 0.4247, train_time_taken: 7.6155
INFO:tensorflow:step: 15460, lr: 0.000221, klw: 0.2738, cost: -1.4170, recon: -1.5302, kl: 0.4132, train_time_taken: 7.5544
INFO:tensorflow:step: 15480, lr: 0.000221, klw: 0.2740, cost: -1.3499, recon: -1.4600, kl: 0.4019, train_time_taken: 7.4685
INFO:tensorflow:step: 15500, lr: 0.000220, klw: 0.2743, cost: -1.2445, recon: -1.3537, kl: 0.3981, train_time_taken: 7.4914
INFO:tensorflow:best_valid_cost: -1.6089, valid_cost: -1.6089, valid_recon: -1.6129, valid_kl: 0.4072, valid_time_taken: 0.6344
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 15500.
INFO:tensorflow:time_taken_save 0.5653.
INFO:tensorflow:eval_cost: -1.5737, eval_recon: -1.5777, eval_kl: 0.3995, eval_time_taken: 0.5430
INFO:tensorflow:step: 15520, lr: 0.000220, klw: 0.2745, cost: -1.3924, recon: -1.5055, kl: 0.4121, train_time_taken: 7.5101
INFO:tensorflow:step: 15540, lr: 0.000219, klw: 0.2747, cost: -1.4373, recon: -1.5509, kl: 0.4133, train_time_taken: 8.5720
INFO:tensorflow:step: 15560, lr: 0.000219, klw: 0.2749, cost: -1.3340, recon: -1.4442, kl: 0.4008, train_time_taken: 7.6674
INFO:tensorflow:step: 15580, lr: 0.000218, klw: 0.2752, cost: -1.2977, recon: -1.4073, kl: 0.3981, train_time_taken: 7.4301
INFO:tensorflow:step: 15600, lr: 0.000218, klw: 0.2754, cost: -1.3869, recon: -1.4997, kl: 0.4097, train_time_taken: 8.4568
INFO:tensorflow:step: 15620, lr: 0.000218, klw: 0.2756, cost: -1.3517, recon: -1.4598, kl: 0.3923, train_time_taken: 8.2367
INFO:tensorflow:step: 15640, lr: 0.000217, klw: 0.2758, cost: -1.3839, recon: -1.4963, kl: 0.4076, train_time_taken: 7.8606
INFO:tensorflow:step: 15660, lr: 0.000217, klw: 0.2761, cost: -1.3420, recon: -1.4526, kl: 0.4007, train_time_taken: 7.5229
INFO:tensorflow:step: 15680, lr: 0.000216, klw: 0.2763, cost: -1.4258, recon: -1.5383, kl: 0.4074, train_time_taken: 7.4981
INFO:tensorflow:step: 15700, lr: 0.000216, klw: 0.2765, cost: -1.2440, recon: -1.3515, kl: 0.3887, train_time_taken: 7.3773
INFO:tensorflow:step: 15720, lr: 0.000216, klw: 0.2767, cost: -1.2731, recon: -1.3812, kl: 0.3907, train_time_taken: 7.4260
INFO:tensorflow:step: 15740, lr: 0.000215, klw: 0.2770, cost: -1.3524, recon: -1.4649, kl: 0.4063, train_time_taken: 8.2800
INFO:tensorflow:step: 15760, lr: 0.000215, klw: 0.2772, cost: -1.3405, recon: -1.4503, kl: 0.3959, train_time_taken: 7.8204
INFO:tensorflow:step: 15780, lr: 0.000214, klw: 0.2774, cost: -1.3632, recon: -1.4761, kl: 0.4071, train_time_taken: 7.3927
INFO:tensorflow:step: 15800, lr: 0.000214, klw: 0.2776, cost: -1.3698, recon: -1.4826, kl: 0.4063, train_time_taken: 7.4286
INFO:tensorflow:step: 15820, lr: 0.000213, klw: 0.2778, cost: -1.4467, recon: -1.5603, kl: 0.4090, train_time_taken: 7.5234
INFO:tensorflow:step: 15840, lr: 0.000213, klw: 0.2781, cost: -1.2603, recon: -1.3692, kl: 0.3914, train_time_taken: 7.5007
INFO:tensorflow:step: 15860, lr: 0.000213, klw: 0.2783, cost: -1.3359, recon: -1.4472, kl: 0.4000, train_time_taken: 7.5128
INFO:tensorflow:step: 15880, lr: 0.000212, klw: 0.2785, cost: -1.3386, recon: -1.4516, kl: 0.4060, train_time_taken: 7.4831
INFO:tensorflow:step: 15900, lr: 0.000212, klw: 0.2787, cost: -1.3620, recon: -1.4736, kl: 0.4003, train_time_taken: 7.5139
INFO:tensorflow:step: 15920, lr: 0.000211, klw: 0.2790, cost: -1.3286, recon: -1.4413, kl: 0.4040, train_time_taken: 7.5010
INFO:tensorflow:step: 15940, lr: 0.000211, klw: 0.2792, cost: -1.3897, recon: -1.5027, kl: 0.4046, train_time_taken: 7.8121
INFO:tensorflow:step: 15960, lr: 0.000211, klw: 0.2794, cost: -1.3296, recon: -1.4403, kl: 0.3964, train_time_taken: 8.6120
INFO:tensorflow:step: 15980, lr: 0.000210, klw: 0.2796, cost: -1.2954, recon: -1.4071, kl: 0.3996, train_time_taken: 7.4483
INFO:tensorflow:step: 16000, lr: 0.000210, klw: 0.2798, cost: -1.4302, recon: -1.5444, kl: 0.4082, train_time_taken: 7.4899
INFO:tensorflow:best_valid_cost: -1.6200, valid_cost: -1.6200, valid_recon: -1.6241, valid_kl: 0.4026, valid_time_taken: 0.6403
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 16000.
INFO:tensorflow:time_taken_save 0.5671.
INFO:tensorflow:eval_cost: -1.5805, eval_recon: -1.5844, eval_kl: 0.3950, eval_time_taken: 0.5377
INFO:tensorflow:step: 16020, lr: 0.000209, klw: 0.2801, cost: -1.3393, recon: -1.4517, kl: 0.4014, train_time_taken: 7.2989
INFO:tensorflow:step: 16040, lr: 0.000209, klw: 0.2803, cost: -1.4482, recon: -1.5601, kl: 0.3992, train_time_taken: 7.4805
INFO:tensorflow:step: 16060, lr: 0.000209, klw: 0.2805, cost: -1.3476, recon: -1.4614, kl: 0.4057, train_time_taken: 7.3812
INFO:tensorflow:step: 16080, lr: 0.000208, klw: 0.2807, cost: -1.3905, recon: -1.5042, kl: 0.4050, train_time_taken: 7.4267
INFO:tensorflow:step: 16100, lr: 0.000208, klw: 0.2809, cost: -1.3497, recon: -1.4611, kl: 0.3966, train_time_taken: 7.5103
INFO:tensorflow:step: 16120, lr: 0.000207, klw: 0.2812, cost: -1.3872, recon: -1.5018, kl: 0.4075, train_time_taken: 7.4394
INFO:tensorflow:step: 16140, lr: 0.000207, klw: 0.2814, cost: -1.3528, recon: -1.4642, kl: 0.3961, train_time_taken: 7.5849
INFO:tensorflow:step: 16160, lr: 0.000207, klw: 0.2816, cost: -1.3720, recon: -1.4865, kl: 0.4067, train_time_taken: 8.4211
INFO:tensorflow:step: 16180, lr: 0.000206, klw: 0.2818, cost: -1.3928, recon: -1.5055, kl: 0.4000, train_time_taken: 7.8425
INFO:tensorflow:step: 16200, lr: 0.000206, klw: 0.2820, cost: -1.4157, recon: -1.5295, kl: 0.4038, train_time_taken: 7.4403
INFO:tensorflow:step: 16220, lr: 0.000206, klw: 0.2822, cost: -1.2960, recon: -1.4069, kl: 0.3931, train_time_taken: 7.6246
INFO:tensorflow:step: 16240, lr: 0.000205, klw: 0.2825, cost: -1.4244, recon: -1.5377, kl: 0.4010, train_time_taken: 7.5059
INFO:tensorflow:step: 16260, lr: 0.000205, klw: 0.2827, cost: -1.4839, recon: -1.5998, kl: 0.4100, train_time_taken: 7.4924
INFO:tensorflow:step: 16280, lr: 0.000204, klw: 0.2829, cost: -1.5081, recon: -1.6247, kl: 0.4120, train_time_taken: 7.4801
INFO:tensorflow:step: 16300, lr: 0.000204, klw: 0.2831, cost: -1.4615, recon: -1.5765, kl: 0.4063, train_time_taken: 7.4904
INFO:tensorflow:step: 16320, lr: 0.000204, klw: 0.2833, cost: -1.3924, recon: -1.5085, kl: 0.4097, train_time_taken: 7.4421
INFO:tensorflow:step: 16340, lr: 0.000203, klw: 0.2835, cost: -1.4219, recon: -1.5378, kl: 0.4090, train_time_taken: 7.4197
INFO:tensorflow:step: 16360, lr: 0.000203, klw: 0.2838, cost: -1.2360, recon: -1.3458, kl: 0.3873, train_time_taken: 7.5298
INFO:tensorflow:step: 16380, lr: 0.000202, klw: 0.2840, cost: -1.4426, recon: -1.5583, kl: 0.4075, train_time_taken: 8.5332
INFO:tensorflow:step: 16400, lr: 0.000202, klw: 0.2842, cost: -1.3208, recon: -1.4358, kl: 0.4046, train_time_taken: 7.9885
INFO:tensorflow:step: 16420, lr: 0.000202, klw: 0.2844, cost: -1.3763, recon: -1.4917, kl: 0.4059, train_time_taken: 8.7965
INFO:tensorflow:step: 16440, lr: 0.000201, klw: 0.2846, cost: -1.3881, recon: -1.5036, kl: 0.4057, train_time_taken: 7.4962
INFO:tensorflow:step: 16460, lr: 0.000201, klw: 0.2848, cost: -1.3876, recon: -1.5015, kl: 0.3997, train_time_taken: 7.5176
INFO:tensorflow:step: 16480, lr: 0.000200, klw: 0.2851, cost: -1.3466, recon: -1.4605, kl: 0.3993, train_time_taken: 7.4462
INFO:tensorflow:step: 16500, lr: 0.000200, klw: 0.2853, cost: -1.4794, recon: -1.5949, kl: 0.4049, train_time_taken: 7.5176
INFO:tensorflow:best_valid_cost: -1.6207, valid_cost: -1.6207, valid_recon: -1.6247, valid_kl: 0.3980, valid_time_taken: 0.6342
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 16500.
INFO:tensorflow:time_taken_save 0.5683.
INFO:tensorflow:eval_cost: -1.5808, eval_recon: -1.5847, eval_kl: 0.3908, eval_time_taken: 0.5388
INFO:tensorflow:step: 16520, lr: 0.000200, klw: 0.2855, cost: -1.4186, recon: -1.5332, kl: 0.4012, train_time_taken: 7.3874
INFO:tensorflow:step: 16540, lr: 0.000199, klw: 0.2857, cost: -1.4079, recon: -1.5222, kl: 0.3999, train_time_taken: 7.4183
INFO:tensorflow:step: 16560, lr: 0.000199, klw: 0.2859, cost: -1.4029, recon: -1.5202, kl: 0.4105, train_time_taken: 7.4245
INFO:tensorflow:step: 16580, lr: 0.000199, klw: 0.2861, cost: -1.3174, recon: -1.4294, kl: 0.3913, train_time_taken: 8.6335
INFO:tensorflow:step: 16600, lr: 0.000198, klw: 0.2863, cost: -1.4252, recon: -1.5412, kl: 0.4050, train_time_taken: 7.5183
INFO:tensorflow:step: 16620, lr: 0.000198, klw: 0.2866, cost: -1.4159, recon: -1.5312, kl: 0.4022, train_time_taken: 7.4897
INFO:tensorflow:step: 16640, lr: 0.000197, klw: 0.2868, cost: -1.4382, recon: -1.5543, kl: 0.4046, train_time_taken: 7.5375
INFO:tensorflow:step: 16660, lr: 0.000197, klw: 0.2870, cost: -1.3654, recon: -1.4799, kl: 0.3990, train_time_taken: 7.5252
INFO:tensorflow:step: 16680, lr: 0.000197, klw: 0.2872, cost: -1.4497, recon: -1.5637, kl: 0.3968, train_time_taken: 7.4289
INFO:tensorflow:step: 16700, lr: 0.000196, klw: 0.2874, cost: -1.4414, recon: -1.5583, kl: 0.4065, train_time_taken: 7.4745
INFO:tensorflow:step: 16720, lr: 0.000196, klw: 0.2876, cost: -1.3908, recon: -1.5066, kl: 0.4025, train_time_taken: 7.4589
INFO:tensorflow:step: 16740, lr: 0.000196, klw: 0.2878, cost: -1.4087, recon: -1.5219, kl: 0.3932, train_time_taken: 7.4760
INFO:tensorflow:step: 16760, lr: 0.000195, klw: 0.2880, cost: -1.5229, recon: -1.6410, kl: 0.4102, train_time_taken: 7.4661
INFO:tensorflow:step: 16780, lr: 0.000195, klw: 0.2883, cost: -1.3413, recon: -1.4560, kl: 0.3978, train_time_taken: 7.9637
INFO:tensorflow:step: 16800, lr: 0.000194, klw: 0.2885, cost: -1.3387, recon: -1.4536, kl: 0.3985, train_time_taken: 8.4066
INFO:tensorflow:step: 16820, lr: 0.000194, klw: 0.2887, cost: -1.3944, recon: -1.5098, kl: 0.3999, train_time_taken: 7.3887
INFO:tensorflow:step: 16840, lr: 0.000194, klw: 0.2889, cost: -1.4569, recon: -1.5728, kl: 0.4014, train_time_taken: 7.4488
INFO:tensorflow:step: 16860, lr: 0.000193, klw: 0.2891, cost: -1.2901, recon: -1.4023, kl: 0.3879, train_time_taken: 7.4618
INFO:tensorflow:step: 16880, lr: 0.000193, klw: 0.2893, cost: -1.4208, recon: -1.5384, kl: 0.4064, train_time_taken: 7.3865
INFO:tensorflow:step: 16900, lr: 0.000193, klw: 0.2895, cost: -1.3116, recon: -1.4261, kl: 0.3956, train_time_taken: 7.3662
INFO:tensorflow:step: 16920, lr: 0.000192, klw: 0.2897, cost: -1.3453, recon: -1.4590, kl: 0.3924, train_time_taken: 7.4086
INFO:tensorflow:step: 16940, lr: 0.000192, klw: 0.2899, cost: -1.3852, recon: -1.5020, kl: 0.4029, train_time_taken: 7.4480
INFO:tensorflow:step: 16960, lr: 0.000192, klw: 0.2902, cost: -1.2483, recon: -1.3597, kl: 0.3839, train_time_taken: 7.4749
INFO:tensorflow:step: 16980, lr: 0.000191, klw: 0.2904, cost: -1.3501, recon: -1.4646, kl: 0.3946, train_time_taken: 7.4605
INFO:tensorflow:step: 17000, lr: 0.000191, klw: 0.2906, cost: -1.3289, recon: -1.4416, kl: 0.3879, train_time_taken: 8.4420
INFO:tensorflow:best_valid_cost: -1.6259, valid_cost: -1.6259, valid_recon: -1.6299, valid_kl: 0.3966, valid_time_taken: 0.7056
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 17000.
INFO:tensorflow:time_taken_save 0.6523.
INFO:tensorflow:eval_cost: -1.5916, eval_recon: -1.5955, eval_kl: 0.3889, eval_time_taken: 0.5852
INFO:tensorflow:step: 17020, lr: 0.000190, klw: 0.2908, cost: -1.3452, recon: -1.4588, kl: 0.3907, train_time_taken: 7.5426
INFO:tensorflow:step: 17040, lr: 0.000190, klw: 0.2910, cost: -1.4123, recon: -1.5276, kl: 0.3964, train_time_taken: 7.3345
INFO:tensorflow:step: 17060, lr: 0.000190, klw: 0.2912, cost: -1.3288, recon: -1.4418, kl: 0.3880, train_time_taken: 7.5031
INFO:tensorflow:step: 17080, lr: 0.000189, klw: 0.2914, cost: -1.3725, recon: -1.4872, kl: 0.3935, train_time_taken: 7.3657
INFO:tensorflow:step: 17100, lr: 0.000189, klw: 0.2916, cost: -1.4204, recon: -1.5356, kl: 0.3952, train_time_taken: 7.4617
INFO:tensorflow:step: 17120, lr: 0.000189, klw: 0.2918, cost: -1.3667, recon: -1.4793, kl: 0.3858, train_time_taken: 7.4183
INFO:tensorflow:step: 17140, lr: 0.000188, klw: 0.2920, cost: -1.4360, recon: -1.5514, kl: 0.3950, train_time_taken: 7.4890
INFO:tensorflow:step: 17160, lr: 0.000188, klw: 0.2922, cost: -1.4522, recon: -1.5689, kl: 0.3996, train_time_taken: 7.4901
INFO:tensorflow:step: 17180, lr: 0.000188, klw: 0.2924, cost: -1.3838, recon: -1.4990, kl: 0.3938, train_time_taken: 7.4443
INFO:tensorflow:step: 17200, lr: 0.000187, klw: 0.2927, cost: -1.3842, recon: -1.4985, kl: 0.3905, train_time_taken: 8.0906
INFO:tensorflow:step: 17220, lr: 0.000187, klw: 0.2929, cost: -1.4362, recon: -1.5534, kl: 0.4003, train_time_taken: 8.7132
INFO:tensorflow:step: 17240, lr: 0.000187, klw: 0.2931, cost: -1.3492, recon: -1.4634, kl: 0.3898, train_time_taken: 7.6380
INFO:tensorflow:step: 17260, lr: 0.000186, klw: 0.2933, cost: -1.3966, recon: -1.5139, kl: 0.4000, train_time_taken: 7.4543
INFO:tensorflow:step: 17280, lr: 0.000186, klw: 0.2935, cost: -1.3227, recon: -1.4364, kl: 0.3875, train_time_taken: 7.4487
INFO:tensorflow:step: 17300, lr: 0.000185, klw: 0.2937, cost: -1.3821, recon: -1.4986, kl: 0.3969, train_time_taken: 7.4399
INFO:tensorflow:step: 17320, lr: 0.000185, klw: 0.2939, cost: -1.3948, recon: -1.5094, kl: 0.3898, train_time_taken: 7.4465
INFO:tensorflow:step: 17340, lr: 0.000185, klw: 0.2941, cost: -1.4231, recon: -1.5396, kl: 0.3963, train_time_taken: 7.3555
INFO:tensorflow:step: 17360, lr: 0.000184, klw: 0.2943, cost: -1.4335, recon: -1.5505, kl: 0.3974, train_time_taken: 7.2772
INFO:tensorflow:step: 17380, lr: 0.000184, klw: 0.2945, cost: -1.3699, recon: -1.4850, kl: 0.3909, train_time_taken: 7.4809
INFO:tensorflow:step: 17400, lr: 0.000184, klw: 0.2947, cost: -1.3216, recon: -1.4347, kl: 0.3837, train_time_taken: 7.5154
INFO:tensorflow:step: 17420, lr: 0.000183, klw: 0.2949, cost: -1.3504, recon: -1.4660, kl: 0.3921, train_time_taken: 8.5254
INFO:tensorflow:step: 17440, lr: 0.000183, klw: 0.2951, cost: -1.3459, recon: -1.4629, kl: 0.3966, train_time_taken: 7.7689
INFO:tensorflow:step: 17460, lr: 0.000183, klw: 0.2953, cost: -1.3954, recon: -1.5118, kl: 0.3944, train_time_taken: 7.4560
INFO:tensorflow:step: 17480, lr: 0.000182, klw: 0.2955, cost: -1.4616, recon: -1.5800, kl: 0.4006, train_time_taken: 7.4511
INFO:tensorflow:step: 17500, lr: 0.000182, klw: 0.2957, cost: -1.3721, recon: -1.4901, kl: 0.3990, train_time_taken: 7.5170
INFO:tensorflow:best_valid_cost: -1.6259, valid_cost: -1.6255, valid_recon: -1.6294, valid_kl: 0.3951, valid_time_taken: 0.6264
INFO:tensorflow:step: 17520, lr: 0.000182, klw: 0.2959, cost: -1.3054, recon: -1.4216, kl: 0.3926, train_time_taken: 7.4155
INFO:tensorflow:step: 17540, lr: 0.000181, klw: 0.2962, cost: -1.5058, recon: -1.6239, kl: 0.3990, train_time_taken: 7.5234
INFO:tensorflow:step: 17560, lr: 0.000181, klw: 0.2964, cost: -1.4365, recon: -1.5547, kl: 0.3990, train_time_taken: 7.4201
INFO:tensorflow:step: 17580, lr: 0.000181, klw: 0.2966, cost: -1.4175, recon: -1.5348, kl: 0.3954, train_time_taken: 7.4489
INFO:tensorflow:step: 17600, lr: 0.000180, klw: 0.2968, cost: -1.2765, recon: -1.3898, kl: 0.3817, train_time_taken: 7.3686
INFO:tensorflow:step: 17620, lr: 0.000180, klw: 0.2970, cost: -1.4241, recon: -1.5419, kl: 0.3965, train_time_taken: 7.8112
INFO:tensorflow:step: 17640, lr: 0.000180, klw: 0.2972, cost: -1.4442, recon: -1.5617, kl: 0.3956, train_time_taken: 8.3859
INFO:tensorflow:step: 17660, lr: 0.000179, klw: 0.2974, cost: -1.4165, recon: -1.5343, kl: 0.3963, train_time_taken: 7.4500
INFO:tensorflow:step: 17680, lr: 0.000179, klw: 0.2976, cost: -1.3960, recon: -1.5135, kl: 0.3947, train_time_taken: 7.3318
INFO:tensorflow:step: 17700, lr: 0.000179, klw: 0.2978, cost: -1.2756, recon: -1.3901, kl: 0.3844, train_time_taken: 7.5875
INFO:tensorflow:step: 17720, lr: 0.000178, klw: 0.2980, cost: -1.4138, recon: -1.5307, kl: 0.3922, train_time_taken: 7.5881
INFO:tensorflow:step: 17740, lr: 0.000178, klw: 0.2982, cost: -1.4330, recon: -1.5510, kl: 0.3955, train_time_taken: 7.4303
INFO:tensorflow:step: 17760, lr: 0.000178, klw: 0.2984, cost: -1.3127, recon: -1.4282, kl: 0.3873, train_time_taken: 7.4027
INFO:tensorflow:step: 17780, lr: 0.000177, klw: 0.2986, cost: -1.4301, recon: -1.5468, kl: 0.3909, train_time_taken: 7.4633
INFO:tensorflow:step: 17800, lr: 0.000177, klw: 0.2988, cost: -1.3844, recon: -1.5021, kl: 0.3939, train_time_taken: 7.4726
INFO:tensorflow:step: 17820, lr: 0.000177, klw: 0.2990, cost: -1.3879, recon: -1.5068, kl: 0.3976, train_time_taken: 7.5001
INFO:tensorflow:step: 17840, lr: 0.000176, klw: 0.2992, cost: -1.4449, recon: -1.5632, kl: 0.3954, train_time_taken: 8.3374
INFO:tensorflow:step: 17860, lr: 0.000176, klw: 0.2994, cost: -1.3958, recon: -1.5145, kl: 0.3966, train_time_taken: 7.9265
INFO:tensorflow:step: 17880, lr: 0.000176, klw: 0.2996, cost: -1.4696, recon: -1.5892, kl: 0.3992, train_time_taken: 7.5195
INFO:tensorflow:step: 17900, lr: 0.000175, klw: 0.2998, cost: -1.4488, recon: -1.5698, kl: 0.4036, train_time_taken: 7.4476
INFO:tensorflow:step: 17920, lr: 0.000175, klw: 0.3000, cost: -1.3556, recon: -1.4717, kl: 0.3871, train_time_taken: 7.4000
INFO:tensorflow:step: 17940, lr: 0.000175, klw: 0.3002, cost: -1.4265, recon: -1.5475, kl: 0.4030, train_time_taken: 7.3558
INFO:tensorflow:step: 17960, lr: 0.000174, klw: 0.3004, cost: -1.4636, recon: -1.5830, kl: 0.3977, train_time_taken: 7.3879
INFO:tensorflow:step: 17980, lr: 0.000174, klw: 0.3006, cost: -1.3825, recon: -1.4990, kl: 0.3876, train_time_taken: 7.4674
INFO:tensorflow:step: 18000, lr: 0.000174, klw: 0.3008, cost: -1.4274, recon: -1.5451, kl: 0.3912, train_time_taken: 7.3809
INFO:tensorflow:best_valid_cost: -1.6337, valid_cost: -1.6337, valid_recon: -1.6376, valid_kl: 0.3912, valid_time_taken: 0.6382
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 18000.
INFO:tensorflow:time_taken_save 0.5806.
INFO:tensorflow:eval_cost: -1.5968, eval_recon: -1.6006, eval_kl: 0.3836, eval_time_taken: 0.5256
INFO:tensorflow:step: 18020, lr: 0.000173, klw: 0.3010, cost: -1.3078, recon: -1.4239, kl: 0.3855, train_time_taken: 8.5513
INFO:tensorflow:step: 18040, lr: 0.000173, klw: 0.3012, cost: -1.4883, recon: -1.6088, kl: 0.3998, train_time_taken: 8.7068
INFO:tensorflow:step: 18060, lr: 0.000173, klw: 0.3014, cost: -1.4747, recon: -1.5959, kl: 0.4023, train_time_taken: 8.0427
INFO:tensorflow:step: 18080, lr: 0.000172, klw: 0.3016, cost: -1.5147, recon: -1.6367, kl: 0.4044, train_time_taken: 7.5113
INFO:tensorflow:step: 18100, lr: 0.000172, klw: 0.3018, cost: -1.5563, recon: -1.6802, kl: 0.4107, train_time_taken: 7.4615
INFO:tensorflow:step: 18120, lr: 0.000172, klw: 0.3020, cost: -1.3024, recon: -1.4177, kl: 0.3820, train_time_taken: 7.4229
INFO:tensorflow:step: 18140, lr: 0.000171, klw: 0.3022, cost: -1.3772, recon: -1.4956, kl: 0.3919, train_time_taken: 7.2883
INFO:tensorflow:step: 18160, lr: 0.000171, klw: 0.3024, cost: -1.3205, recon: -1.4360, kl: 0.3821, train_time_taken: 7.4162
INFO:tensorflow:step: 18180, lr: 0.000171, klw: 0.3026, cost: -1.3422, recon: -1.4588, kl: 0.3854, train_time_taken: 7.3714
INFO:tensorflow:step: 18200, lr: 0.000170, klw: 0.3028, cost: -1.4207, recon: -1.5418, kl: 0.3997, train_time_taken: 7.4711
INFO:tensorflow:step: 18220, lr: 0.000170, klw: 0.3030, cost: -1.5528, recon: -1.6777, kl: 0.4121, train_time_taken: 7.4000
INFO:tensorflow:step: 18240, lr: 0.000170, klw: 0.3032, cost: -1.3288, recon: -1.4451, kl: 0.3836, train_time_taken: 7.4927
INFO:tensorflow:step: 18260, lr: 0.000169, klw: 0.3034, cost: -1.3602, recon: -1.4779, kl: 0.3879, train_time_taken: 8.4544
INFO:tensorflow:step: 18280, lr: 0.000169, klw: 0.3036, cost: -1.3781, recon: -1.4968, kl: 0.3908, train_time_taken: 7.7018
INFO:tensorflow:step: 18300, lr: 0.000169, klw: 0.3038, cost: -1.4035, recon: -1.5206, kl: 0.3856, train_time_taken: 7.2875
INFO:tensorflow:step: 18320, lr: 0.000168, klw: 0.3039, cost: -1.3236, recon: -1.4417, kl: 0.3885, train_time_taken: 7.3587
INFO:tensorflow:step: 18340, lr: 0.000168, klw: 0.3041, cost: -1.3334, recon: -1.4504, kl: 0.3849, train_time_taken: 7.4878
INFO:tensorflow:step: 18360, lr: 0.000168, klw: 0.3043, cost: -1.3533, recon: -1.4728, kl: 0.3925, train_time_taken: 7.3358
INFO:tensorflow:step: 18380, lr: 0.000168, klw: 0.3045, cost: -1.3243, recon: -1.4421, kl: 0.3869, train_time_taken: 7.4029
INFO:tensorflow:step: 18400, lr: 0.000167, klw: 0.3047, cost: -1.3679, recon: -1.4843, kl: 0.3818, train_time_taken: 7.5831
INFO:tensorflow:step: 18420, lr: 0.000167, klw: 0.3049, cost: -1.2453, recon: -1.3594, kl: 0.3739, train_time_taken: 7.4617
INFO:tensorflow:step: 18440, lr: 0.000167, klw: 0.3051, cost: -1.3641, recon: -1.4824, kl: 0.3877, train_time_taken: 7.4823
INFO:tensorflow:step: 18460, lr: 0.000166, klw: 0.3053, cost: -1.2958, recon: -1.4105, kl: 0.3757, train_time_taken: 7.8009
INFO:tensorflow:step: 18480, lr: 0.000166, klw: 0.3055, cost: -1.3709, recon: -1.4886, kl: 0.3854, train_time_taken: 8.4365
INFO:tensorflow:step: 18500, lr: 0.000166, klw: 0.3057, cost: -1.3848, recon: -1.5015, kl: 0.3817, train_time_taken: 7.3896
INFO:tensorflow:best_valid_cost: -1.6366, valid_cost: -1.6366, valid_recon: -1.6405, valid_kl: 0.3882, valid_time_taken: 0.6251
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 18500.
INFO:tensorflow:time_taken_save 0.5923.
INFO:tensorflow:eval_cost: -1.6040, eval_recon: -1.6078, eval_kl: 0.3806, eval_time_taken: 0.5196
INFO:tensorflow:step: 18520, lr: 0.000165, klw: 0.3059, cost: -1.4217, recon: -1.5400, kl: 0.3868, train_time_taken: 7.3497
INFO:tensorflow:step: 18540, lr: 0.000165, klw: 0.3061, cost: -1.4128, recon: -1.5332, kl: 0.3935, train_time_taken: 7.4474
INFO:tensorflow:step: 18560, lr: 0.000165, klw: 0.3063, cost: -1.3975, recon: -1.5157, kl: 0.3861, train_time_taken: 7.4303
INFO:tensorflow:step: 18580, lr: 0.000164, klw: 0.3065, cost: -1.3795, recon: -1.4985, kl: 0.3884, train_time_taken: 7.4582
INFO:tensorflow:step: 18600, lr: 0.000164, klw: 0.3067, cost: -1.3407, recon: -1.4591, kl: 0.3861, train_time_taken: 7.5132
INFO:tensorflow:step: 18620, lr: 0.000164, klw: 0.3069, cost: -1.4244, recon: -1.5452, kl: 0.3936, train_time_taken: 7.3178
INFO:tensorflow:step: 18640, lr: 0.000163, klw: 0.3071, cost: -1.3337, recon: -1.4520, kl: 0.3853, train_time_taken: 7.5327
INFO:tensorflow:step: 18660, lr: 0.000163, klw: 0.3073, cost: -1.3315, recon: -1.4504, kl: 0.3868, train_time_taken: 7.4923
INFO:tensorflow:step: 18680, lr: 0.000163, klw: 0.3074, cost: -1.3439, recon: -1.4617, kl: 0.3832, train_time_taken: 8.5502
INFO:tensorflow:step: 18700, lr: 0.000163, klw: 0.3076, cost: -1.3973, recon: -1.5157, kl: 0.3848, train_time_taken: 7.7964
INFO:tensorflow:step: 18720, lr: 0.000162, klw: 0.3078, cost: -1.3306, recon: -1.4476, kl: 0.3800, train_time_taken: 7.4955
INFO:tensorflow:step: 18740, lr: 0.000162, klw: 0.3080, cost: -1.3803, recon: -1.4981, kl: 0.3825, train_time_taken: 7.4521
INFO:tensorflow:step: 18760, lr: 0.000162, klw: 0.3082, cost: -1.3960, recon: -1.5137, kl: 0.3817, train_time_taken: 7.4494
INFO:tensorflow:step: 18780, lr: 0.000161, klw: 0.3084, cost: -1.3752, recon: -1.4940, kl: 0.3850, train_time_taken: 7.5483
INFO:tensorflow:step: 18800, lr: 0.000161, klw: 0.3086, cost: -1.3478, recon: -1.4688, kl: 0.3919, train_time_taken: 7.5826
INFO:tensorflow:step: 18820, lr: 0.000161, klw: 0.3088, cost: -1.3836, recon: -1.5036, kl: 0.3887, train_time_taken: 8.2692
INFO:tensorflow:step: 18840, lr: 0.000160, klw: 0.3090, cost: -1.3811, recon: -1.4994, kl: 0.3827, train_time_taken: 8.6657
INFO:tensorflow:step: 18860, lr: 0.000160, klw: 0.3092, cost: -1.3520, recon: -1.4718, kl: 0.3875, train_time_taken: 7.5904
INFO:tensorflow:step: 18880, lr: 0.000160, klw: 0.3094, cost: -1.4457, recon: -1.5669, kl: 0.3918, train_time_taken: 8.2011
INFO:tensorflow:step: 18900, lr: 0.000160, klw: 0.3096, cost: -1.3749, recon: -1.4931, kl: 0.3817, train_time_taken: 8.0594
INFO:tensorflow:step: 18920, lr: 0.000159, klw: 0.3097, cost: -1.5049, recon: -1.6273, kl: 0.3952, train_time_taken: 7.4059
INFO:tensorflow:step: 18940, lr: 0.000159, klw: 0.3099, cost: -1.3875, recon: -1.5092, kl: 0.3927, train_time_taken: 7.4133
INFO:tensorflow:step: 18960, lr: 0.000159, klw: 0.3101, cost: -1.4372, recon: -1.5579, kl: 0.3892, train_time_taken: 7.5298
INFO:tensorflow:step: 18980, lr: 0.000158, klw: 0.3103, cost: -1.3918, recon: -1.5113, kl: 0.3853, train_time_taken: 7.5261
INFO:tensorflow:step: 19000, lr: 0.000158, klw: 0.3105, cost: -1.4183, recon: -1.5389, kl: 0.3884, train_time_taken: 7.4780
INFO:tensorflow:best_valid_cost: -1.6397, valid_cost: -1.6397, valid_recon: -1.6436, valid_kl: 0.3864, valid_time_taken: 0.6399
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 19000.
INFO:tensorflow:time_taken_save 0.5947.
INFO:tensorflow:eval_cost: -1.6027, eval_recon: -1.6065, eval_kl: 0.3789, eval_time_taken: 0.5251
INFO:tensorflow:step: 19020, lr: 0.000158, klw: 0.3107, cost: -1.3737, recon: -1.4934, kl: 0.3853, train_time_taken: 7.3748
INFO:tensorflow:step: 19040, lr: 0.000157, klw: 0.3109, cost: -1.4001, recon: -1.5208, kl: 0.3883, train_time_taken: 7.4011
INFO:tensorflow:step: 19060, lr: 0.000157, klw: 0.3111, cost: -1.2922, recon: -1.4084, kl: 0.3736, train_time_taken: 7.4885
INFO:tensorflow:step: 19080, lr: 0.000157, klw: 0.3113, cost: -1.4045, recon: -1.5260, kl: 0.3903, train_time_taken: 7.7414
INFO:tensorflow:step: 19100, lr: 0.000157, klw: 0.3114, cost: -1.3833, recon: -1.5014, kl: 0.3793, train_time_taken: 8.5460
INFO:tensorflow:step: 19120, lr: 0.000156, klw: 0.3116, cost: -1.3882, recon: -1.5102, kl: 0.3915, train_time_taken: 7.2959
INFO:tensorflow:step: 19140, lr: 0.000156, klw: 0.3118, cost: -1.3934, recon: -1.5123, kl: 0.3813, train_time_taken: 7.4617
INFO:tensorflow:step: 19160, lr: 0.000156, klw: 0.3120, cost: -1.4045, recon: -1.5261, kl: 0.3900, train_time_taken: 7.4355
INFO:tensorflow:step: 19180, lr: 0.000155, klw: 0.3122, cost: -1.4006, recon: -1.5238, kl: 0.3946, train_time_taken: 7.5044
INFO:tensorflow:step: 19200, lr: 0.000155, klw: 0.3124, cost: -1.4948, recon: -1.6191, kl: 0.3979, train_time_taken: 7.2827
INFO:tensorflow:step: 19220, lr: 0.000155, klw: 0.3126, cost: -1.3878, recon: -1.5072, kl: 0.3822, train_time_taken: 7.4491
INFO:tensorflow:step: 19240, lr: 0.000155, klw: 0.3128, cost: -1.4360, recon: -1.5585, kl: 0.3916, train_time_taken: 7.4708
INFO:tensorflow:step: 19260, lr: 0.000154, klw: 0.3129, cost: -1.3670, recon: -1.4857, kl: 0.3794, train_time_taken: 7.4238
INFO:tensorflow:step: 19280, lr: 0.000154, klw: 0.3131, cost: -1.4763, recon: -1.6010, kl: 0.3982, train_time_taken: 7.5343
INFO:tensorflow:step: 19300, lr: 0.000154, klw: 0.3133, cost: -1.2859, recon: -1.4041, kl: 0.3772, train_time_taken: 8.1259
INFO:tensorflow:step: 19320, lr: 0.000153, klw: 0.3135, cost: -1.4426, recon: -1.5663, kl: 0.3947, train_time_taken: 8.0752
INFO:tensorflow:step: 19340, lr: 0.000153, klw: 0.3137, cost: -1.3663, recon: -1.4877, kl: 0.3871, train_time_taken: 7.4666
INFO:tensorflow:step: 19360, lr: 0.000153, klw: 0.3139, cost: -1.4168, recon: -1.5396, kl: 0.3913, train_time_taken: 7.6435
INFO:tensorflow:step: 19380, lr: 0.000153, klw: 0.3141, cost: -1.4017, recon: -1.5243, kl: 0.3904, train_time_taken: 7.4709
INFO:tensorflow:step: 19400, lr: 0.000152, klw: 0.3143, cost: -1.3724, recon: -1.4938, kl: 0.3863, train_time_taken: 7.3938
INFO:tensorflow:step: 19420, lr: 0.000152, klw: 0.3144, cost: -1.3772, recon: -1.4997, kl: 0.3899, train_time_taken: 7.3588
INFO:tensorflow:step: 19440, lr: 0.000152, klw: 0.3146, cost: -1.3118, recon: -1.4291, kl: 0.3728, train_time_taken: 7.3938
INFO:tensorflow:step: 19460, lr: 0.000151, klw: 0.3148, cost: -1.4400, recon: -1.5628, kl: 0.3904, train_time_taken: 7.5618
INFO:tensorflow:step: 19480, lr: 0.000151, klw: 0.3150, cost: -1.4267, recon: -1.5488, kl: 0.3877, train_time_taken: 7.5552
INFO:tensorflow:step: 19500, lr: 0.000151, klw: 0.3152, cost: -1.3797, recon: -1.5001, kl: 0.3821, train_time_taken: 7.4880
INFO:tensorflow:best_valid_cost: -1.6485, valid_cost: -1.6485, valid_recon: -1.6524, valid_kl: 0.3875, valid_time_taken: 0.6891
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 19500.
INFO:tensorflow:time_taken_save 0.6780.
INFO:tensorflow:eval_cost: -1.6105, eval_recon: -1.6143, eval_kl: 0.3799, eval_time_taken: 0.5799
INFO:tensorflow:step: 19520, lr: 0.000151, klw: 0.3154, cost: -1.4317, recon: -1.5542, kl: 0.3882, train_time_taken: 8.5055
INFO:tensorflow:step: 19540, lr: 0.000150, klw: 0.3155, cost: -1.4142, recon: -1.5364, kl: 0.3874, train_time_taken: 7.4765
INFO:tensorflow:step: 19560, lr: 0.000150, klw: 0.3157, cost: -1.3795, recon: -1.5017, kl: 0.3869, train_time_taken: 7.3905
INFO:tensorflow:step: 19580, lr: 0.000150, klw: 0.3159, cost: -1.2982, recon: -1.4171, kl: 0.3764, train_time_taken: 7.3124
INFO:tensorflow:step: 19600, lr: 0.000149, klw: 0.3161, cost: -1.4032, recon: -1.5248, kl: 0.3846, train_time_taken: 7.7495
INFO:tensorflow:step: 19620, lr: 0.000149, klw: 0.3163, cost: -1.3860, recon: -1.5079, kl: 0.3855, train_time_taken: 7.6588
INFO:tensorflow:step: 19640, lr: 0.000149, klw: 0.3165, cost: -1.4574, recon: -1.5816, kl: 0.3925, train_time_taken: 8.3676
INFO:tensorflow:step: 19660, lr: 0.000149, klw: 0.3167, cost: -1.4434, recon: -1.5666, kl: 0.3890, train_time_taken: 8.0153
INFO:tensorflow:step: 19680, lr: 0.000148, klw: 0.3168, cost: -1.3944, recon: -1.5161, kl: 0.3842, train_time_taken: 7.3075
INFO:tensorflow:step: 19700, lr: 0.000148, klw: 0.3170, cost: -1.3492, recon: -1.4690, kl: 0.3779, train_time_taken: 7.3541
INFO:tensorflow:step: 19720, lr: 0.000148, klw: 0.3172, cost: -1.4690, recon: -1.5936, kl: 0.3928, train_time_taken: 8.5253
INFO:tensorflow:step: 19740, lr: 0.000147, klw: 0.3174, cost: -1.4869, recon: -1.6122, kl: 0.3947, train_time_taken: 7.8362
INFO:tensorflow:step: 19760, lr: 0.000147, klw: 0.3176, cost: -1.3748, recon: -1.4964, kl: 0.3828, train_time_taken: 7.4191
INFO:tensorflow:step: 19780, lr: 0.000147, klw: 0.3177, cost: -1.4203, recon: -1.5437, kl: 0.3882, train_time_taken: 7.4555
INFO:tensorflow:step: 19800, lr: 0.000147, klw: 0.3179, cost: -1.4083, recon: -1.5304, kl: 0.3841, train_time_taken: 7.4743
INFO:tensorflow:step: 19820, lr: 0.000146, klw: 0.3181, cost: -1.4604, recon: -1.5842, kl: 0.3892, train_time_taken: 7.5095
INFO:tensorflow:step: 19840, lr: 0.000146, klw: 0.3183, cost: -1.4973, recon: -1.6212, kl: 0.3892, train_time_taken: 7.3943
INFO:tensorflow:step: 19860, lr: 0.000146, klw: 0.3185, cost: -1.3822, recon: -1.5032, kl: 0.3799, train_time_taken: 7.3522
INFO:tensorflow:step: 19880, lr: 0.000146, klw: 0.3187, cost: -1.4012, recon: -1.5243, kl: 0.3862, train_time_taken: 7.3944
INFO:tensorflow:step: 19900, lr: 0.000145, klw: 0.3188, cost: -1.4691, recon: -1.5920, kl: 0.3854, train_time_taken: 7.3640
INFO:tensorflow:step: 19920, lr: 0.000145, klw: 0.3190, cost: -1.3289, recon: -1.4514, kl: 0.3840, train_time_taken: 7.5326
INFO:tensorflow:step: 19940, lr: 0.000145, klw: 0.3192, cost: -1.4777, recon: -1.6023, kl: 0.3902, train_time_taken: 8.5740
INFO:tensorflow:step: 19960, lr: 0.000145, klw: 0.3194, cost: -1.4924, recon: -1.6178, kl: 0.3925, train_time_taken: 7.4951
INFO:tensorflow:step: 19980, lr: 0.000144, klw: 0.3196, cost: -1.4725, recon: -1.5952, kl: 0.3841, train_time_taken: 7.4335
INFO:tensorflow:step: 20000, lr: 0.000144, klw: 0.3197, cost: -1.3385, recon: -1.4599, kl: 0.3796, train_time_taken: 7.4358
INFO:tensorflow:best_valid_cost: -1.6496, valid_cost: -1.6496, valid_recon: -1.6534, valid_kl: 0.3826, valid_time_taken: 0.6369
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 20000.
INFO:tensorflow:time_taken_save 0.6317.
INFO:tensorflow:eval_cost: -1.6121, eval_recon: -1.6158, eval_kl: 0.3752, eval_time_taken: 0.5334
INFO:tensorflow:step: 20020, lr: 0.000144, klw: 0.3199, cost: -1.3726, recon: -1.4935, kl: 0.3780, train_time_taken: 7.4540
INFO:tensorflow:step: 20040, lr: 0.000143, klw: 0.3201, cost: -1.3138, recon: -1.4327, kl: 0.3714, train_time_taken: 7.3618
INFO:tensorflow:step: 20060, lr: 0.000143, klw: 0.3203, cost: -1.3868, recon: -1.5074, kl: 0.3765, train_time_taken: 7.4123
INFO:tensorflow:step: 20080, lr: 0.000143, klw: 0.3205, cost: -1.3870, recon: -1.5099, kl: 0.3835, train_time_taken: 7.5259
INFO:tensorflow:step: 20100, lr: 0.000143, klw: 0.3206, cost: -1.3529, recon: -1.4745, kl: 0.3791, train_time_taken: 7.4583
INFO:tensorflow:step: 20120, lr: 0.000142, klw: 0.3208, cost: -1.3851, recon: -1.5056, kl: 0.3756, train_time_taken: 7.4988
INFO:tensorflow:step: 20140, lr: 0.000142, klw: 0.3210, cost: -1.3319, recon: -1.4540, kl: 0.3803, train_time_taken: 8.2677
INFO:tensorflow:step: 20160, lr: 0.000142, klw: 0.3212, cost: -1.3694, recon: -1.4910, kl: 0.3786, train_time_taken: 7.8858
INFO:tensorflow:step: 20180, lr: 0.000142, klw: 0.3214, cost: -1.3888, recon: -1.5097, kl: 0.3764, train_time_taken: 7.5264
INFO:tensorflow:step: 20200, lr: 0.000141, klw: 0.3215, cost: -1.4595, recon: -1.5854, kl: 0.3917, train_time_taken: 7.3074
INFO:tensorflow:step: 20220, lr: 0.000141, klw: 0.3217, cost: -1.4000, recon: -1.5216, kl: 0.3781, train_time_taken: 7.3666
INFO:tensorflow:step: 20240, lr: 0.000141, klw: 0.3219, cost: -1.2882, recon: -1.4074, kl: 0.3704, train_time_taken: 7.4251
INFO:tensorflow:step: 20260, lr: 0.000141, klw: 0.3221, cost: -1.3915, recon: -1.5148, kl: 0.3828, train_time_taken: 7.3882
INFO:tensorflow:step: 20280, lr: 0.000140, klw: 0.3222, cost: -1.4423, recon: -1.5677, kl: 0.3893, train_time_taken: 7.2982
INFO:tensorflow:step: 20300, lr: 0.000140, klw: 0.3224, cost: -1.4511, recon: -1.5759, kl: 0.3870, train_time_taken: 7.3667
INFO:tensorflow:step: 20320, lr: 0.000140, klw: 0.3226, cost: -1.3983, recon: -1.5253, kl: 0.3937, train_time_taken: 7.3360
INFO:tensorflow:step: 20340, lr: 0.000139, klw: 0.3228, cost: -1.4056, recon: -1.5280, kl: 0.3793, train_time_taken: 7.6646
INFO:tensorflow:step: 20360, lr: 0.000139, klw: 0.3230, cost: -1.4517, recon: -1.5759, kl: 0.3843, train_time_taken: 8.5281
INFO:tensorflow:step: 20380, lr: 0.000139, klw: 0.3231, cost: -1.3260, recon: -1.4476, kl: 0.3761, train_time_taken: 7.5286
INFO:tensorflow:step: 20400, lr: 0.000139, klw: 0.3233, cost: -1.2927, recon: -1.4130, kl: 0.3720, train_time_taken: 8.1334
INFO:tensorflow:step: 20420, lr: 0.000138, klw: 0.3235, cost: -1.3912, recon: -1.5159, kl: 0.3852, train_time_taken: 7.6066
INFO:tensorflow:step: 20440, lr: 0.000138, klw: 0.3237, cost: -1.3662, recon: -1.4904, kl: 0.3837, train_time_taken: 7.6150
INFO:tensorflow:step: 20460, lr: 0.000138, klw: 0.3238, cost: -1.3543, recon: -1.4764, kl: 0.3771, train_time_taken: 8.7829
INFO:tensorflow:step: 20480, lr: 0.000138, klw: 0.3240, cost: -1.4706, recon: -1.5965, kl: 0.3887, train_time_taken: 7.4275
INFO:tensorflow:step: 20500, lr: 0.000137, klw: 0.3242, cost: -1.3904, recon: -1.5136, kl: 0.3801, train_time_taken: 7.4065
INFO:tensorflow:best_valid_cost: -1.6498, valid_cost: -1.6498, valid_recon: -1.6536, valid_kl: 0.3824, valid_time_taken: 0.6250
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 20500.
INFO:tensorflow:time_taken_save 0.6158.
INFO:tensorflow:eval_cost: -1.6139, eval_recon: -1.6177, eval_kl: 0.3746, eval_time_taken: 0.5283
INFO:tensorflow:step: 20520, lr: 0.000137, klw: 0.3244, cost: -1.4187, recon: -1.5426, kl: 0.3821, train_time_taken: 7.3660
INFO:tensorflow:step: 20540, lr: 0.000137, klw: 0.3245, cost: -1.4273, recon: -1.5491, kl: 0.3752, train_time_taken: 7.3053
INFO:tensorflow:step: 20560, lr: 0.000137, klw: 0.3247, cost: -1.4305, recon: -1.5547, kl: 0.3826, train_time_taken: 8.6867
INFO:tensorflow:step: 20580, lr: 0.000136, klw: 0.3249, cost: -1.4077, recon: -1.5334, kl: 0.3868, train_time_taken: 7.6389
INFO:tensorflow:step: 20600, lr: 0.000136, klw: 0.3251, cost: -1.3525, recon: -1.4759, kl: 0.3796, train_time_taken: 7.4172
INFO:tensorflow:step: 20620, lr: 0.000136, klw: 0.3252, cost: -1.3331, recon: -1.4538, kl: 0.3712, train_time_taken: 7.4641
INFO:tensorflow:step: 20640, lr: 0.000136, klw: 0.3254, cost: -1.4056, recon: -1.5280, kl: 0.3763, train_time_taken: 7.4131
INFO:tensorflow:step: 20660, lr: 0.000135, klw: 0.3256, cost: -1.5446, recon: -1.6736, kl: 0.3963, train_time_taken: 7.3484
INFO:tensorflow:step: 20680, lr: 0.000135, klw: 0.3258, cost: -1.4368, recon: -1.5622, kl: 0.3850, train_time_taken: 7.3957
INFO:tensorflow:step: 20700, lr: 0.000135, klw: 0.3259, cost: -1.4507, recon: -1.5764, kl: 0.3858, train_time_taken: 7.3843
INFO:tensorflow:step: 20720, lr: 0.000135, klw: 0.3261, cost: -1.4509, recon: -1.5766, kl: 0.3856, train_time_taken: 7.3797
INFO:tensorflow:step: 20740, lr: 0.000134, klw: 0.3263, cost: -1.4030, recon: -1.5268, kl: 0.3794, train_time_taken: 7.2943
INFO:tensorflow:step: 20760, lr: 0.000134, klw: 0.3265, cost: -1.3411, recon: -1.4640, kl: 0.3765, train_time_taken: 7.8529
INFO:tensorflow:step: 20780, lr: 0.000134, klw: 0.3266, cost: -1.3779, recon: -1.5024, kl: 0.3811, train_time_taken: 8.3644
INFO:tensorflow:step: 20800, lr: 0.000134, klw: 0.3268, cost: -1.3408, recon: -1.4639, kl: 0.3767, train_time_taken: 7.4091
INFO:tensorflow:step: 20820, lr: 0.000133, klw: 0.3270, cost: -1.3981, recon: -1.5253, kl: 0.3891, train_time_taken: 7.3032
INFO:tensorflow:step: 20840, lr: 0.000133, klw: 0.3272, cost: -1.4382, recon: -1.5644, kl: 0.3857, train_time_taken: 7.3863
INFO:tensorflow:step: 20860, lr: 0.000133, klw: 0.3273, cost: -1.4256, recon: -1.5509, kl: 0.3826, train_time_taken: 7.3060
INFO:tensorflow:step: 20880, lr: 0.000133, klw: 0.3275, cost: -1.3697, recon: -1.4929, kl: 0.3761, train_time_taken: 7.2849
INFO:tensorflow:step: 20900, lr: 0.000132, klw: 0.3277, cost: -1.3327, recon: -1.4535, kl: 0.3689, train_time_taken: 7.4107
INFO:tensorflow:step: 20920, lr: 0.000132, klw: 0.3278, cost: -1.4800, recon: -1.6075, kl: 0.3888, train_time_taken: 7.5214
INFO:tensorflow:step: 20940, lr: 0.000132, klw: 0.3280, cost: -1.3558, recon: -1.4779, kl: 0.3722, train_time_taken: 7.4918
INFO:tensorflow:step: 20960, lr: 0.000132, klw: 0.3282, cost: -1.3895, recon: -1.5154, kl: 0.3837, train_time_taken: 7.3725
INFO:tensorflow:step: 20980, lr: 0.000131, klw: 0.3284, cost: -1.4293, recon: -1.5570, kl: 0.3891, train_time_taken: 8.0902
INFO:tensorflow:step: 21000, lr: 0.000131, klw: 0.3285, cost: -1.3899, recon: -1.5127, kl: 0.3738, train_time_taken: 8.1776
INFO:tensorflow:best_valid_cost: -1.6564, valid_cost: -1.6564, valid_recon: -1.6602, valid_kl: 0.3797, valid_time_taken: 0.6231
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 21000.
INFO:tensorflow:time_taken_save 0.6309.
INFO:tensorflow:eval_cost: -1.6201, eval_recon: -1.6238, eval_kl: 0.3722, eval_time_taken: 0.5340
INFO:tensorflow:step: 21020, lr: 0.000131, klw: 0.3287, cost: -1.4088, recon: -1.5319, kl: 0.3747, train_time_taken: 7.4871
INFO:tensorflow:step: 21040, lr: 0.000131, klw: 0.3289, cost: -1.4294, recon: -1.5540, kl: 0.3790, train_time_taken: 7.3313
INFO:tensorflow:step: 21060, lr: 0.000130, klw: 0.3290, cost: -1.4155, recon: -1.5379, kl: 0.3720, train_time_taken: 7.4730
INFO:tensorflow:step: 21080, lr: 0.000130, klw: 0.3292, cost: -1.3598, recon: -1.4833, kl: 0.3752, train_time_taken: 7.3291
INFO:tensorflow:step: 21100, lr: 0.000130, klw: 0.3294, cost: -1.4134, recon: -1.5386, kl: 0.3799, train_time_taken: 7.4562
INFO:tensorflow:step: 21120, lr: 0.000130, klw: 0.3296, cost: -1.3975, recon: -1.5217, kl: 0.3767, train_time_taken: 7.5184
INFO:tensorflow:step: 21140, lr: 0.000130, klw: 0.3297, cost: -1.4425, recon: -1.5682, kl: 0.3812, train_time_taken: 7.3333
INFO:tensorflow:step: 21160, lr: 0.000129, klw: 0.3299, cost: -1.3837, recon: -1.5065, kl: 0.3720, train_time_taken: 7.3986
INFO:tensorflow:step: 21180, lr: 0.000129, klw: 0.3301, cost: -1.5412, recon: -1.6685, kl: 0.3858, train_time_taken: 7.6243
INFO:tensorflow:step: 21200, lr: 0.000129, klw: 0.3302, cost: -1.2924, recon: -1.4128, kl: 0.3646, train_time_taken: 8.6654
INFO:tensorflow:step: 21220, lr: 0.000129, klw: 0.3304, cost: -1.3822, recon: -1.5045, kl: 0.3701, train_time_taken: 7.4072
INFO:tensorflow:step: 21240, lr: 0.000128, klw: 0.3306, cost: -1.4486, recon: -1.5715, kl: 0.3719, train_time_taken: 7.5207
INFO:tensorflow:step: 21260, lr: 0.000128, klw: 0.3307, cost: -1.5400, recon: -1.6686, kl: 0.3887, train_time_taken: 8.4600
INFO:tensorflow:step: 21280, lr: 0.000128, klw: 0.3309, cost: -1.5348, recon: -1.6648, kl: 0.3930, train_time_taken: 7.9529
INFO:tensorflow:step: 21300, lr: 0.000128, klw: 0.3311, cost: -1.3776, recon: -1.5016, kl: 0.3747, train_time_taken: 7.4792
INFO:tensorflow:step: 21320, lr: 0.000127, klw: 0.3313, cost: -1.3439, recon: -1.4696, kl: 0.3796, train_time_taken: 7.4626
INFO:tensorflow:step: 21340, lr: 0.000127, klw: 0.3314, cost: -1.4336, recon: -1.5580, kl: 0.3753, train_time_taken: 7.4648
INFO:tensorflow:step: 21360, lr: 0.000127, klw: 0.3316, cost: -1.4415, recon: -1.5672, kl: 0.3791, train_time_taken: 7.3687
INFO:tensorflow:step: 21380, lr: 0.000127, klw: 0.3318, cost: -1.4437, recon: -1.5695, kl: 0.3794, train_time_taken: 7.3079
INFO:tensorflow:step: 21400, lr: 0.000126, klw: 0.3319, cost: -1.4109, recon: -1.5352, kl: 0.3744, train_time_taken: 8.2994
INFO:tensorflow:step: 21420, lr: 0.000126, klw: 0.3321, cost: -1.4743, recon: -1.6041, kl: 0.3908, train_time_taken: 7.8294
INFO:tensorflow:step: 21440, lr: 0.000126, klw: 0.3323, cost: -1.3957, recon: -1.5215, kl: 0.3786, train_time_taken: 7.3856
INFO:tensorflow:step: 21460, lr: 0.000126, klw: 0.3324, cost: -1.4434, recon: -1.5701, kl: 0.3812, train_time_taken: 7.3125
INFO:tensorflow:step: 21480, lr: 0.000126, klw: 0.3326, cost: -1.4035, recon: -1.5298, kl: 0.3797, train_time_taken: 7.5244
INFO:tensorflow:step: 21500, lr: 0.000125, klw: 0.3328, cost: -1.4061, recon: -1.5315, kl: 0.3770, train_time_taken: 7.4777
INFO:tensorflow:best_valid_cost: -1.6572, valid_cost: -1.6572, valid_recon: -1.6609, valid_kl: 0.3790, valid_time_taken: 0.6252
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 21500.
INFO:tensorflow:time_taken_save 0.6309.
INFO:tensorflow:eval_cost: -1.6194, eval_recon: -1.6231, eval_kl: 0.3716, eval_time_taken: 0.5391
INFO:tensorflow:step: 21520, lr: 0.000125, klw: 0.3329, cost: -1.4460, recon: -1.5702, kl: 0.3732, train_time_taken: 7.4170
INFO:tensorflow:step: 21540, lr: 0.000125, klw: 0.3331, cost: -1.3651, recon: -1.4899, kl: 0.3749, train_time_taken: 7.4319
INFO:tensorflow:step: 21560, lr: 0.000125, klw: 0.3333, cost: -1.4259, recon: -1.5517, kl: 0.3776, train_time_taken: 7.5118
INFO:tensorflow:step: 21580, lr: 0.000124, klw: 0.3334, cost: -1.4394, recon: -1.5658, kl: 0.3790, train_time_taken: 7.3966
INFO:tensorflow:step: 21600, lr: 0.000124, klw: 0.3336, cost: -1.4706, recon: -1.5973, kl: 0.3797, train_time_taken: 7.7540
INFO:tensorflow:step: 21620, lr: 0.000124, klw: 0.3338, cost: -1.3962, recon: -1.5206, kl: 0.3727, train_time_taken: 8.2732
INFO:tensorflow:step: 21640, lr: 0.000124, klw: 0.3339, cost: -1.4747, recon: -1.6049, kl: 0.3900, train_time_taken: 7.4233
INFO:tensorflow:step: 21660, lr: 0.000123, klw: 0.3341, cost: -1.4246, recon: -1.5516, kl: 0.3802, train_time_taken: 7.3245
INFO:tensorflow:step: 21680, lr: 0.000123, klw: 0.3343, cost: -1.3870, recon: -1.5121, kl: 0.3743, train_time_taken: 7.5317
INFO:tensorflow:step: 21700, lr: 0.000123, klw: 0.3344, cost: -1.4451, recon: -1.5695, kl: 0.3721, train_time_taken: 7.3551
INFO:tensorflow:step: 21720, lr: 0.000123, klw: 0.3346, cost: -1.3485, recon: -1.4735, kl: 0.3736, train_time_taken: 7.2981
INFO:tensorflow:step: 21740, lr: 0.000123, klw: 0.3348, cost: -1.4091, recon: -1.5355, kl: 0.3776, train_time_taken: 7.3051
INFO:tensorflow:step: 21760, lr: 0.000122, klw: 0.3349, cost: -1.4664, recon: -1.5974, kl: 0.3911, train_time_taken: 7.2854
INFO:tensorflow:step: 21780, lr: 0.000122, klw: 0.3351, cost: -1.4822, recon: -1.6122, kl: 0.3881, train_time_taken: 7.3823
INFO:tensorflow:step: 21800, lr: 0.000122, klw: 0.3353, cost: -1.3850, recon: -1.5105, kl: 0.3743, train_time_taken: 7.3820
INFO:tensorflow:step: 21820, lr: 0.000122, klw: 0.3354, cost: -1.4228, recon: -1.5497, kl: 0.3782, train_time_taken: 8.1174
INFO:tensorflow:step: 21840, lr: 0.000121, klw: 0.3356, cost: -1.3658, recon: -1.4909, kl: 0.3728, train_time_taken: 7.9711
INFO:tensorflow:step: 21860, lr: 0.000121, klw: 0.3358, cost: -1.4795, recon: -1.6056, kl: 0.3755, train_time_taken: 7.3966
INFO:tensorflow:step: 21880, lr: 0.000121, klw: 0.3359, cost: -1.4517, recon: -1.5809, kl: 0.3847, train_time_taken: 7.2795
INFO:tensorflow:step: 21900, lr: 0.000121, klw: 0.3361, cost: -1.4842, recon: -1.6135, kl: 0.3846, train_time_taken: 7.3527
INFO:tensorflow:step: 21920, lr: 0.000121, klw: 0.3362, cost: -1.3734, recon: -1.4993, kl: 0.3745, train_time_taken: 7.3762
INFO:tensorflow:step: 21940, lr: 0.000120, klw: 0.3364, cost: -1.4413, recon: -1.5703, kl: 0.3834, train_time_taken: 7.4090
INFO:tensorflow:step: 21960, lr: 0.000120, klw: 0.3366, cost: -1.3826, recon: -1.5094, kl: 0.3768, train_time_taken: 7.3918
INFO:tensorflow:step: 21980, lr: 0.000120, klw: 0.3367, cost: -1.4385, recon: -1.5652, kl: 0.3761, train_time_taken: 7.3065
INFO:tensorflow:step: 22000, lr: 0.000120, klw: 0.3369, cost: -1.4298, recon: -1.5578, kl: 0.3799, train_time_taken: 8.0569
INFO:tensorflow:best_valid_cost: -1.6629, valid_cost: -1.6629, valid_recon: -1.6667, valid_kl: 0.3766, valid_time_taken: 0.7147
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 22000.
INFO:tensorflow:time_taken_save 0.6361.
INFO:tensorflow:eval_cost: -1.6256, eval_recon: -1.6293, eval_kl: 0.3693, eval_time_taken: 0.5219
INFO:tensorflow:step: 22020, lr: 0.000119, klw: 0.3371, cost: -1.4465, recon: -1.5734, kl: 0.3763, train_time_taken: 7.4690
INFO:tensorflow:step: 22040, lr: 0.000119, klw: 0.3372, cost: -1.3919, recon: -1.5190, kl: 0.3770, train_time_taken: 8.4884
INFO:tensorflow:step: 22060, lr: 0.000119, klw: 0.3374, cost: -1.4997, recon: -1.6298, kl: 0.3855, train_time_taken: 7.6818
INFO:tensorflow:step: 22080, lr: 0.000119, klw: 0.3375, cost: -1.3442, recon: -1.4675, kl: 0.3655, train_time_taken: 8.7836
INFO:tensorflow:step: 22100, lr: 0.000119, klw: 0.3377, cost: -1.3488, recon: -1.4735, kl: 0.3695, train_time_taken: 7.4319
INFO:tensorflow:step: 22120, lr: 0.000118, klw: 0.3379, cost: -1.3931, recon: -1.5186, kl: 0.3714, train_time_taken: 7.4714
INFO:tensorflow:step: 22140, lr: 0.000118, klw: 0.3380, cost: -1.3352, recon: -1.4592, kl: 0.3670, train_time_taken: 7.4310
INFO:tensorflow:step: 22160, lr: 0.000118, klw: 0.3382, cost: -1.4524, recon: -1.5816, kl: 0.3821, train_time_taken: 7.4518
INFO:tensorflow:step: 22180, lr: 0.000118, klw: 0.3384, cost: -1.3883, recon: -1.5171, kl: 0.3807, train_time_taken: 7.3365
INFO:tensorflow:step: 22200, lr: 0.000118, klw: 0.3385, cost: -1.3694, recon: -1.4970, kl: 0.3769, train_time_taken: 7.2921
INFO:tensorflow:step: 22220, lr: 0.000117, klw: 0.3387, cost: -1.4259, recon: -1.5538, kl: 0.3775, train_time_taken: 7.3437
INFO:tensorflow:step: 22240, lr: 0.000117, klw: 0.3388, cost: -1.3342, recon: -1.4595, kl: 0.3700, train_time_taken: 8.1770
INFO:tensorflow:step: 22260, lr: 0.000117, klw: 0.3390, cost: -1.4919, recon: -1.6206, kl: 0.3798, train_time_taken: 7.8857
INFO:tensorflow:step: 22280, lr: 0.000117, klw: 0.3392, cost: -1.4820, recon: -1.6089, kl: 0.3743, train_time_taken: 7.3852
INFO:tensorflow:step: 22300, lr: 0.000116, klw: 0.3393, cost: -1.3268, recon: -1.4516, kl: 0.3677, train_time_taken: 7.3469
INFO:tensorflow:step: 22320, lr: 0.000116, klw: 0.3395, cost: -1.4537, recon: -1.5828, kl: 0.3805, train_time_taken: 7.4201
INFO:tensorflow:step: 22340, lr: 0.000116, klw: 0.3396, cost: -1.5159, recon: -1.6494, kl: 0.3929, train_time_taken: 7.5099
INFO:tensorflow:step: 22360, lr: 0.000116, klw: 0.3398, cost: -1.3545, recon: -1.4815, kl: 0.3737, train_time_taken: 7.4176
INFO:tensorflow:step: 22380, lr: 0.000116, klw: 0.3400, cost: -1.4429, recon: -1.5700, kl: 0.3738, train_time_taken: 7.2874
INFO:tensorflow:step: 22400, lr: 0.000115, klw: 0.3401, cost: -1.4643, recon: -1.5926, kl: 0.3773, train_time_taken: 7.2630
INFO:tensorflow:step: 22420, lr: 0.000115, klw: 0.3403, cost: -1.4467, recon: -1.5782, kl: 0.3865, train_time_taken: 7.3378
INFO:tensorflow:step: 22440, lr: 0.000115, klw: 0.3404, cost: -1.3713, recon: -1.4974, kl: 0.3702, train_time_taken: 7.3619
INFO:tensorflow:step: 22460, lr: 0.000115, klw: 0.3406, cost: -1.3824, recon: -1.5081, kl: 0.3692, train_time_taken: 8.4944
INFO:tensorflow:step: 22480, lr: 0.000115, klw: 0.3408, cost: -1.3712, recon: -1.5000, kl: 0.3782, train_time_taken: 7.6500
INFO:tensorflow:step: 22500, lr: 0.000114, klw: 0.3409, cost: -1.4672, recon: -1.5973, kl: 0.3816, train_time_taken: 7.4240
INFO:tensorflow:best_valid_cost: -1.6631, valid_cost: -1.6631, valid_recon: -1.6668, valid_kl: 0.3749, valid_time_taken: 0.6222
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 22500.
INFO:tensorflow:time_taken_save 0.6498.
INFO:tensorflow:eval_cost: -1.6256, eval_recon: -1.6293, eval_kl: 0.3675, eval_time_taken: 0.5145
INFO:tensorflow:step: 22520, lr: 0.000114, klw: 0.3411, cost: -1.4131, recon: -1.5421, kl: 0.3783, train_time_taken: 7.4824
INFO:tensorflow:step: 22540, lr: 0.000114, klw: 0.3412, cost: -1.4489, recon: -1.5760, kl: 0.3725, train_time_taken: 7.3885
INFO:tensorflow:step: 22560, lr: 0.000114, klw: 0.3414, cost: -1.4660, recon: -1.5933, kl: 0.3728, train_time_taken: 7.4041
INFO:tensorflow:step: 22580, lr: 0.000114, klw: 0.3416, cost: -1.3916, recon: -1.5178, kl: 0.3696, train_time_taken: 7.2622
INFO:tensorflow:step: 22600, lr: 0.000113, klw: 0.3417, cost: -1.3741, recon: -1.4990, kl: 0.3654, train_time_taken: 7.4045
INFO:tensorflow:step: 22620, lr: 0.000113, klw: 0.3419, cost: -1.3731, recon: -1.4988, kl: 0.3675, train_time_taken: 7.5025
INFO:tensorflow:step: 22640, lr: 0.000113, klw: 0.3420, cost: -1.4386, recon: -1.5674, kl: 0.3767, train_time_taken: 7.3097
INFO:tensorflow:step: 22660, lr: 0.000113, klw: 0.3422, cost: -1.4434, recon: -1.5725, kl: 0.3772, train_time_taken: 8.0065
INFO:tensorflow:step: 22680, lr: 0.000112, klw: 0.3424, cost: -1.4204, recon: -1.5481, kl: 0.3729, train_time_taken: 7.9948
INFO:tensorflow:step: 22700, lr: 0.000112, klw: 0.3425, cost: -1.4560, recon: -1.5888, kl: 0.3877, train_time_taken: 7.3616
INFO:tensorflow:step: 22720, lr: 0.000112, klw: 0.3427, cost: -1.3871, recon: -1.5157, kl: 0.3753, train_time_taken: 7.3523
INFO:tensorflow:step: 22740, lr: 0.000112, klw: 0.3428, cost: -1.3679, recon: -1.4966, kl: 0.3754, train_time_taken: 7.3236
INFO:tensorflow:step: 22760, lr: 0.000112, klw: 0.3430, cost: -1.5205, recon: -1.6523, kl: 0.3841, train_time_taken: 7.3277
INFO:tensorflow:step: 22780, lr: 0.000111, klw: 0.3431, cost: -1.3901, recon: -1.5169, kl: 0.3694, train_time_taken: 7.3908
INFO:tensorflow:step: 22800, lr: 0.000111, klw: 0.3433, cost: -1.3679, recon: -1.4954, kl: 0.3714, train_time_taken: 8.1117
INFO:tensorflow:step: 22820, lr: 0.000111, klw: 0.3434, cost: -1.5319, recon: -1.6637, kl: 0.3838, train_time_taken: 7.5286
INFO:tensorflow:step: 22840, lr: 0.000111, klw: 0.3436, cost: -1.4560, recon: -1.5868, kl: 0.3806, train_time_taken: 7.4842
INFO:tensorflow:step: 22860, lr: 0.000111, klw: 0.3438, cost: -1.4457, recon: -1.5743, kl: 0.3742, train_time_taken: 7.4243
INFO:tensorflow:step: 22880, lr: 0.000110, klw: 0.3439, cost: -1.4534, recon: -1.5837, kl: 0.3787, train_time_taken: 8.5580
INFO:tensorflow:step: 22900, lr: 0.000110, klw: 0.3441, cost: -1.4716, recon: -1.6034, kl: 0.3831, train_time_taken: 8.5340
INFO:tensorflow:step: 22920, lr: 0.000110, klw: 0.3442, cost: -1.3477, recon: -1.4744, kl: 0.3682, train_time_taken: 7.3196
INFO:tensorflow:step: 22940, lr: 0.000110, klw: 0.3444, cost: -1.4498, recon: -1.5796, kl: 0.3768, train_time_taken: 7.3323
INFO:tensorflow:step: 22960, lr: 0.000110, klw: 0.3445, cost: -1.3863, recon: -1.5145, kl: 0.3720, train_time_taken: 7.4115
INFO:tensorflow:step: 22980, lr: 0.000109, klw: 0.3447, cost: -1.4296, recon: -1.5577, kl: 0.3716, train_time_taken: 7.2934
INFO:tensorflow:step: 23000, lr: 0.000109, klw: 0.3449, cost: -1.3493, recon: -1.4766, kl: 0.3691, train_time_taken: 7.4157
INFO:tensorflow:best_valid_cost: -1.6660, valid_cost: -1.6660, valid_recon: -1.6697, valid_kl: 0.3747, valid_time_taken: 0.6334
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 23000.
INFO:tensorflow:time_taken_save 0.8255.
INFO:tensorflow:eval_cost: -1.6277, eval_recon: -1.6313, eval_kl: 0.3670, eval_time_taken: 0.5230
INFO:tensorflow:step: 23020, lr: 0.000109, klw: 0.3450, cost: -1.5529, recon: -1.6856, kl: 0.3847, train_time_taken: 7.4466
INFO:tensorflow:step: 23040, lr: 0.000109, klw: 0.3452, cost: -1.4552, recon: -1.5842, kl: 0.3738, train_time_taken: 7.2505
INFO:tensorflow:step: 23060, lr: 0.000109, klw: 0.3453, cost: -1.4736, recon: -1.6040, kl: 0.3776, train_time_taken: 7.4999
INFO:tensorflow:step: 23080, lr: 0.000108, klw: 0.3455, cost: -1.4322, recon: -1.5609, kl: 0.3727, train_time_taken: 8.0556
INFO:tensorflow:step: 23100, lr: 0.000108, klw: 0.3456, cost: -1.3577, recon: -1.4866, kl: 0.3730, train_time_taken: 8.0690
INFO:tensorflow:step: 23120, lr: 0.000108, klw: 0.3458, cost: -1.4419, recon: -1.5724, kl: 0.3772, train_time_taken: 7.3832
INFO:tensorflow:step: 23140, lr: 0.000108, klw: 0.3459, cost: -1.4705, recon: -1.6050, kl: 0.3886, train_time_taken: 7.2737
INFO:tensorflow:step: 23160, lr: 0.000108, klw: 0.3461, cost: -1.4447, recon: -1.5726, kl: 0.3696, train_time_taken: 7.3077
INFO:tensorflow:step: 23180, lr: 0.000107, klw: 0.3462, cost: -1.4796, recon: -1.6125, kl: 0.3838, train_time_taken: 7.4995
INFO:tensorflow:step: 23200, lr: 0.000107, klw: 0.3464, cost: -1.3554, recon: -1.4818, kl: 0.3651, train_time_taken: 7.4105
INFO:tensorflow:step: 23220, lr: 0.000107, klw: 0.3465, cost: -1.4761, recon: -1.6065, kl: 0.3762, train_time_taken: 7.2663
INFO:tensorflow:step: 23240, lr: 0.000107, klw: 0.3467, cost: -1.4472, recon: -1.5768, kl: 0.3738, train_time_taken: 7.5081
INFO:tensorflow:step: 23260, lr: 0.000107, klw: 0.3469, cost: -1.4029, recon: -1.5343, kl: 0.3788, train_time_taken: 7.2770
INFO:tensorflow:step: 23280, lr: 0.000107, klw: 0.3470, cost: -1.4704, recon: -1.6034, kl: 0.3830, train_time_taken: 7.2834
INFO:tensorflow:step: 23300, lr: 0.000106, klw: 0.3472, cost: -1.2689, recon: -1.3964, kl: 0.3671, train_time_taken: 8.3235
INFO:tensorflow:step: 23320, lr: 0.000106, klw: 0.3473, cost: -1.3668, recon: -1.4948, kl: 0.3684, train_time_taken: 7.8592
INFO:tensorflow:step: 23340, lr: 0.000106, klw: 0.3475, cost: -1.4234, recon: -1.5538, kl: 0.3753, train_time_taken: 7.3335
INFO:tensorflow:step: 23360, lr: 0.000106, klw: 0.3476, cost: -1.4994, recon: -1.6332, kl: 0.3849, train_time_taken: 7.3851
INFO:tensorflow:step: 23380, lr: 0.000106, klw: 0.3478, cost: -1.4131, recon: -1.5426, kl: 0.3723, train_time_taken: 7.3263
INFO:tensorflow:step: 23400, lr: 0.000105, klw: 0.3479, cost: -1.4008, recon: -1.5299, kl: 0.3710, train_time_taken: 7.4881
INFO:tensorflow:step: 23420, lr: 0.000105, klw: 0.3481, cost: -1.4118, recon: -1.5439, kl: 0.3794, train_time_taken: 7.4606
INFO:tensorflow:step: 23440, lr: 0.000105, klw: 0.3482, cost: -1.5247, recon: -1.6563, kl: 0.3779, train_time_taken: 7.3646
INFO:tensorflow:step: 23460, lr: 0.000105, klw: 0.3484, cost: -1.3502, recon: -1.4778, kl: 0.3661, train_time_taken: 7.3957
INFO:tensorflow:step: 23480, lr: 0.000105, klw: 0.3485, cost: -1.4464, recon: -1.5756, kl: 0.3708, train_time_taken: 7.3911
INFO:tensorflow:step: 23500, lr: 0.000104, klw: 0.3487, cost: -1.4233, recon: -1.5525, kl: 0.3707, train_time_taken: 7.6707
INFO:tensorflow:best_valid_cost: -1.6716, valid_cost: -1.6716, valid_recon: -1.6754, valid_kl: 0.3727, valid_time_taken: 0.7122
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 23500.
INFO:tensorflow:time_taken_save 0.7314.
INFO:tensorflow:eval_cost: -1.6323, eval_recon: -1.6359, eval_kl: 0.3653, eval_time_taken: 0.5911
INFO:tensorflow:step: 23520, lr: 0.000104, klw: 0.3488, cost: -1.4025, recon: -1.5302, kl: 0.3663, train_time_taken: 8.2935
INFO:tensorflow:step: 23540, lr: 0.000104, klw: 0.3490, cost: -1.5285, recon: -1.6617, kl: 0.3818, train_time_taken: 7.3993
INFO:tensorflow:step: 23560, lr: 0.000104, klw: 0.3491, cost: -1.4345, recon: -1.5646, kl: 0.3726, train_time_taken: 7.2848
INFO:tensorflow:step: 23580, lr: 0.000104, klw: 0.3493, cost: -1.4947, recon: -1.6281, kl: 0.3821, train_time_taken: 7.4098
INFO:tensorflow:step: 23600, lr: 0.000103, klw: 0.3494, cost: -1.3566, recon: -1.4834, kl: 0.3630, train_time_taken: 8.1168
INFO:tensorflow:step: 23620, lr: 0.000103, klw: 0.3496, cost: -1.4154, recon: -1.5475, kl: 0.3781, train_time_taken: 7.2532
INFO:tensorflow:step: 23640, lr: 0.000103, klw: 0.3497, cost: -1.3287, recon: -1.4553, kl: 0.3619, train_time_taken: 7.4158
INFO:tensorflow:step: 23660, lr: 0.000103, klw: 0.3499, cost: -1.3855, recon: -1.5164, kl: 0.3743, train_time_taken: 7.4396
INFO:tensorflow:step: 23680, lr: 0.000103, klw: 0.3500, cost: -1.4376, recon: -1.5667, kl: 0.3687, train_time_taken: 7.2092
INFO:tensorflow:step: 23700, lr: 0.000103, klw: 0.3502, cost: -1.2729, recon: -1.3975, kl: 0.3558, train_time_taken: 8.1804
INFO:tensorflow:step: 23720, lr: 0.000102, klw: 0.3503, cost: -1.4384, recon: -1.5711, kl: 0.3790, train_time_taken: 8.6919
INFO:tensorflow:step: 23740, lr: 0.000102, klw: 0.3505, cost: -1.3823, recon: -1.5103, kl: 0.3652, train_time_taken: 7.6246
INFO:tensorflow:step: 23760, lr: 0.000102, klw: 0.3506, cost: -1.4296, recon: -1.5600, kl: 0.3720, train_time_taken: 7.3607
INFO:tensorflow:step: 23780, lr: 0.000102, klw: 0.3508, cost: -1.3392, recon: -1.4673, kl: 0.3650, train_time_taken: 7.3430
INFO:tensorflow:step: 23800, lr: 0.000102, klw: 0.3509, cost: -1.4746, recon: -1.6066, kl: 0.3760, train_time_taken: 7.2363
INFO:tensorflow:step: 23820, lr: 0.000101, klw: 0.3511, cost: -1.4415, recon: -1.5718, kl: 0.3713, train_time_taken: 7.2232
INFO:tensorflow:step: 23840, lr: 0.000101, klw: 0.3512, cost: -1.4505, recon: -1.5799, kl: 0.3683, train_time_taken: 7.3486
INFO:tensorflow:step: 23860, lr: 0.000101, klw: 0.3514, cost: -1.3944, recon: -1.5260, kl: 0.3746, train_time_taken: 7.3003
INFO:tensorflow:step: 23880, lr: 0.000101, klw: 0.3515, cost: -1.4854, recon: -1.6169, kl: 0.3742, train_time_taken: 7.3702
INFO:tensorflow:step: 23900, lr: 0.000101, klw: 0.3517, cost: -1.3839, recon: -1.5141, kl: 0.3701, train_time_taken: 7.4446
INFO:tensorflow:step: 23920, lr: 0.000101, klw: 0.3518, cost: -1.3834, recon: -1.5123, kl: 0.3665, train_time_taken: 7.5366
INFO:tensorflow:step: 23940, lr: 0.000100, klw: 0.3520, cost: -1.4843, recon: -1.6181, kl: 0.3802, train_time_taken: 8.5822
INFO:tensorflow:step: 23960, lr: 0.000100, klw: 0.3521, cost: -1.3912, recon: -1.5180, kl: 0.3603, train_time_taken: 7.5191
INFO:tensorflow:step: 23980, lr: 0.000100, klw: 0.3523, cost: -1.4532, recon: -1.5853, kl: 0.3751, train_time_taken: 7.4979
INFO:tensorflow:step: 24000, lr: 0.000100, klw: 0.3524, cost: -1.5059, recon: -1.6389, kl: 0.3775, train_time_taken: 7.3803
INFO:tensorflow:best_valid_cost: -1.6716, valid_cost: -1.6706, valid_recon: -1.6743, valid_kl: 0.3708, valid_time_taken: 0.6248
INFO:tensorflow:step: 24020, lr: 0.000100, klw: 0.3526, cost: -1.3039, recon: -1.4314, kl: 0.3617, train_time_taken: 7.4611
INFO:tensorflow:step: 24040, lr: 0.000099, klw: 0.3527, cost: -1.4894, recon: -1.6221, kl: 0.3764, train_time_taken: 7.5501
INFO:tensorflow:step: 24060, lr: 0.000099, klw: 0.3529, cost: -1.3900, recon: -1.5179, kl: 0.3624, train_time_taken: 7.3230
INFO:tensorflow:step: 24080, lr: 0.000099, klw: 0.3530, cost: -1.4113, recon: -1.5401, kl: 0.3648, train_time_taken: 7.5602
INFO:tensorflow:step: 24100, lr: 0.000099, klw: 0.3532, cost: -1.5668, recon: -1.7032, kl: 0.3860, train_time_taken: 7.4158
INFO:tensorflow:step: 24120, lr: 0.000099, klw: 0.3533, cost: -1.4680, recon: -1.5987, kl: 0.3699, train_time_taken: 7.4650
INFO:tensorflow:step: 24140, lr: 0.000099, klw: 0.3534, cost: -1.4632, recon: -1.5947, kl: 0.3722, train_time_taken: 8.2151
INFO:tensorflow:step: 24160, lr: 0.000098, klw: 0.3536, cost: -1.4020, recon: -1.5324, kl: 0.3688, train_time_taken: 8.0919
INFO:tensorflow:step: 24180, lr: 0.000098, klw: 0.3537, cost: -1.4155, recon: -1.5458, kl: 0.3685, train_time_taken: 7.5330
INFO:tensorflow:step: 24200, lr: 0.000098, klw: 0.3539, cost: -1.4249, recon: -1.5541, kl: 0.3651, train_time_taken: 7.4553
INFO:tensorflow:step: 24220, lr: 0.000098, klw: 0.3540, cost: -1.4304, recon: -1.5603, kl: 0.3669, train_time_taken: 7.4218
INFO:tensorflow:step: 24240, lr: 0.000098, klw: 0.3542, cost: -1.4791, recon: -1.6128, kl: 0.3775, train_time_taken: 7.3145
INFO:tensorflow:step: 24260, lr: 0.000097, klw: 0.3543, cost: -1.4388, recon: -1.5703, kl: 0.3713, train_time_taken: 7.4384
INFO:tensorflow:step: 24280, lr: 0.000097, klw: 0.3545, cost: -1.4299, recon: -1.5612, kl: 0.3703, train_time_taken: 7.3714
INFO:tensorflow:step: 24300, lr: 0.000097, klw: 0.3546, cost: -1.3303, recon: -1.4582, kl: 0.3607, train_time_taken: 7.3995
INFO:tensorflow:step: 24320, lr: 0.000097, klw: 0.3548, cost: -1.4661, recon: -1.5986, kl: 0.3734, train_time_taken: 7.3096
INFO:tensorflow:step: 24340, lr: 0.000097, klw: 0.3549, cost: -1.5055, recon: -1.6364, kl: 0.3690, train_time_taken: 7.3349
INFO:tensorflow:step: 24360, lr: 0.000097, klw: 0.3551, cost: -1.3148, recon: -1.4448, kl: 0.3661, train_time_taken: 8.3765
INFO:tensorflow:step: 24380, lr: 0.000096, klw: 0.3552, cost: -1.3956, recon: -1.5280, kl: 0.3725, train_time_taken: 7.8210
INFO:tensorflow:step: 24400, lr: 0.000096, klw: 0.3553, cost: -1.3519, recon: -1.4806, kl: 0.3622, train_time_taken: 8.6133
INFO:tensorflow:step: 24420, lr: 0.000096, klw: 0.3555, cost: -1.3708, recon: -1.4986, kl: 0.3596, train_time_taken: 7.3824
INFO:tensorflow:step: 24440, lr: 0.000096, klw: 0.3556, cost: -1.3470, recon: -1.4754, kl: 0.3611, train_time_taken: 7.2835
INFO:tensorflow:step: 24460, lr: 0.000096, klw: 0.3558, cost: -1.4974, recon: -1.6298, kl: 0.3722, train_time_taken: 7.3940
INFO:tensorflow:step: 24480, lr: 0.000096, klw: 0.3559, cost: -1.4083, recon: -1.5383, kl: 0.3651, train_time_taken: 7.4292
INFO:tensorflow:step: 24500, lr: 0.000095, klw: 0.3561, cost: -1.4613, recon: -1.5922, kl: 0.3677, train_time_taken: 7.4800
INFO:tensorflow:best_valid_cost: -1.6716, valid_cost: -1.6697, valid_recon: -1.6733, valid_kl: 0.3684, valid_time_taken: 0.6291
INFO:tensorflow:step: 24520, lr: 0.000095, klw: 0.3562, cost: -1.3799, recon: -1.5104, kl: 0.3663, train_time_taken: 8.7947
INFO:tensorflow:step: 24540, lr: 0.000095, klw: 0.3564, cost: -1.3651, recon: -1.4935, kl: 0.3603, train_time_taken: 7.5932
INFO:tensorflow:step: 24560, lr: 0.000095, klw: 0.3565, cost: -1.3838, recon: -1.5137, kl: 0.3642, train_time_taken: 8.2139
INFO:tensorflow:step: 24580, lr: 0.000095, klw: 0.3566, cost: -1.5204, recon: -1.6558, kl: 0.3798, train_time_taken: 7.9786
INFO:tensorflow:step: 24600, lr: 0.000095, klw: 0.3568, cost: -1.3633, recon: -1.4928, kl: 0.3631, train_time_taken: 7.5163
INFO:tensorflow:step: 24620, lr: 0.000094, klw: 0.3569, cost: -1.3639, recon: -1.4940, kl: 0.3646, train_time_taken: 7.3079
INFO:tensorflow:step: 24640, lr: 0.000094, klw: 0.3571, cost: -1.3479, recon: -1.4779, kl: 0.3641, train_time_taken: 7.5251
INFO:tensorflow:step: 24660, lr: 0.000094, klw: 0.3572, cost: -1.3003, recon: -1.4292, kl: 0.3609, train_time_taken: 7.4058
INFO:tensorflow:step: 24680, lr: 0.000094, klw: 0.3574, cost: -1.4100, recon: -1.5405, kl: 0.3650, train_time_taken: 7.3940
INFO:tensorflow:step: 24700, lr: 0.000094, klw: 0.3575, cost: -1.4934, recon: -1.6298, kl: 0.3817, train_time_taken: 7.3025
INFO:tensorflow:step: 24720, lr: 0.000094, klw: 0.3576, cost: -1.4157, recon: -1.5457, kl: 0.3637, train_time_taken: 7.4216
INFO:tensorflow:step: 24740, lr: 0.000093, klw: 0.3578, cost: -1.5321, recon: -1.6676, kl: 0.3788, train_time_taken: 7.5075
INFO:tensorflow:step: 24760, lr: 0.000093, klw: 0.3579, cost: -1.4732, recon: -1.6072, kl: 0.3742, train_time_taken: 7.4567
INFO:tensorflow:step: 24780, lr: 0.000093, klw: 0.3581, cost: -1.3318, recon: -1.4610, kl: 0.3609, train_time_taken: 8.5715
INFO:tensorflow:step: 24800, lr: 0.000093, klw: 0.3582, cost: -1.3724, recon: -1.5007, kl: 0.3583, train_time_taken: 7.5415
INFO:tensorflow:step: 24820, lr: 0.000093, klw: 0.3583, cost: -1.5508, recon: -1.6855, kl: 0.3760, train_time_taken: 7.3501
INFO:tensorflow:step: 24840, lr: 0.000093, klw: 0.3585, cost: -1.4139, recon: -1.5468, kl: 0.3708, train_time_taken: 7.3045
INFO:tensorflow:step: 24860, lr: 0.000092, klw: 0.3586, cost: -1.5066, recon: -1.6419, kl: 0.3771, train_time_taken: 7.3954
INFO:tensorflow:step: 24880, lr: 0.000092, klw: 0.3588, cost: -1.2639, recon: -1.3895, kl: 0.3501, train_time_taken: 7.4281
INFO:tensorflow:step: 24900, lr: 0.000092, klw: 0.3589, cost: -1.4866, recon: -1.6219, kl: 0.3771, train_time_taken: 7.2406
INFO:tensorflow:step: 24920, lr: 0.000092, klw: 0.3591, cost: -1.5279, recon: -1.6615, kl: 0.3720, train_time_taken: 7.2437
INFO:tensorflow:step: 24940, lr: 0.000092, klw: 0.3592, cost: -1.4369, recon: -1.5678, kl: 0.3644, train_time_taken: 7.3647
INFO:tensorflow:step: 24960, lr: 0.000092, klw: 0.3593, cost: -1.4653, recon: -1.6016, kl: 0.3792, train_time_taken: 7.3567
INFO:tensorflow:step: 24980, lr: 0.000091, klw: 0.3595, cost: -1.5108, recon: -1.6446, kl: 0.3724, train_time_taken: 7.6377
INFO:tensorflow:step: 25000, lr: 0.000091, klw: 0.3596, cost: -1.3119, recon: -1.4417, kl: 0.3609, train_time_taken: 8.4354
INFO:tensorflow:best_valid_cost: -1.6772, valid_cost: -1.6772, valid_recon: -1.6808, valid_kl: 0.3673, valid_time_taken: 0.6429
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 25000.
INFO:tensorflow:time_taken_save 0.6568.
INFO:tensorflow:eval_cost: -1.6389, eval_recon: -1.6425, eval_kl: 0.3596, eval_time_taken: 0.5303
INFO:tensorflow:step: 25020, lr: 0.000091, klw: 0.3598, cost: -1.4835, recon: -1.6149, kl: 0.3654, train_time_taken: 7.2637
INFO:tensorflow:step: 25040, lr: 0.000091, klw: 0.3599, cost: -1.4916, recon: -1.6271, kl: 0.3764, train_time_taken: 7.3057
INFO:tensorflow:step: 25060, lr: 0.000091, klw: 0.3600, cost: -1.3999, recon: -1.5308, kl: 0.3637, train_time_taken: 7.4255
INFO:tensorflow:step: 25080, lr: 0.000091, klw: 0.3602, cost: -1.4016, recon: -1.5351, kl: 0.3704, train_time_taken: 7.3230
INFO:tensorflow:step: 25100, lr: 0.000090, klw: 0.3603, cost: -1.4008, recon: -1.5290, kl: 0.3557, train_time_taken: 7.4300
INFO:tensorflow:step: 25120, lr: 0.000090, klw: 0.3605, cost: -1.4274, recon: -1.5586, kl: 0.3639, train_time_taken: 7.3263
INFO:tensorflow:step: 25140, lr: 0.000090, klw: 0.3606, cost: -1.4115, recon: -1.5447, kl: 0.3693, train_time_taken: 7.4525
INFO:tensorflow:step: 25160, lr: 0.000090, klw: 0.3607, cost: -1.4258, recon: -1.5579, kl: 0.3663, train_time_taken: 7.1656
INFO:tensorflow:step: 25180, lr: 0.000090, klw: 0.3609, cost: -1.4406, recon: -1.5709, kl: 0.3611, train_time_taken: 7.3931
INFO:tensorflow:step: 25200, lr: 0.000090, klw: 0.3610, cost: -1.4842, recon: -1.6186, kl: 0.3722, train_time_taken: 8.4463
INFO:tensorflow:step: 25220, lr: 0.000089, klw: 0.3612, cost: -1.4341, recon: -1.5678, kl: 0.3703, train_time_taken: 7.6647
INFO:tensorflow:step: 25240, lr: 0.000089, klw: 0.3613, cost: -1.2917, recon: -1.4214, kl: 0.3589, train_time_taken: 7.2730
INFO:tensorflow:step: 25260, lr: 0.000089, klw: 0.3614, cost: -1.4286, recon: -1.5642, kl: 0.3751, train_time_taken: 7.2706
INFO:tensorflow:step: 25280, lr: 0.000089, klw: 0.3616, cost: -1.4256, recon: -1.5621, kl: 0.3775, train_time_taken: 7.4509
INFO:tensorflow:step: 25300, lr: 0.000089, klw: 0.3617, cost: -1.5257, recon: -1.6605, kl: 0.3728, train_time_taken: 7.3250
INFO:tensorflow:step: 25320, lr: 0.000089, klw: 0.3618, cost: -1.4526, recon: -1.5851, kl: 0.3661, train_time_taken: 7.2519
INFO:tensorflow:step: 25340, lr: 0.000089, klw: 0.3620, cost: -1.5104, recon: -1.6426, kl: 0.3651, train_time_taken: 8.7258
INFO:tensorflow:step: 25360, lr: 0.000088, klw: 0.3621, cost: -1.3937, recon: -1.5264, kl: 0.3664, train_time_taken: 7.5014
INFO:tensorflow:step: 25380, lr: 0.000088, klw: 0.3623, cost: -1.4508, recon: -1.5826, kl: 0.3640, train_time_taken: 7.3457
INFO:tensorflow:step: 25400, lr: 0.000088, klw: 0.3624, cost: -1.4085, recon: -1.5423, kl: 0.3693, train_time_taken: 7.7673
INFO:tensorflow:step: 25420, lr: 0.000088, klw: 0.3625, cost: -1.5254, recon: -1.6617, kl: 0.3760, train_time_taken: 8.5149
INFO:tensorflow:step: 25440, lr: 0.000088, klw: 0.3627, cost: -1.4098, recon: -1.5417, kl: 0.3634, train_time_taken: 7.3674
INFO:tensorflow:step: 25460, lr: 0.000088, klw: 0.3628, cost: -1.4325, recon: -1.5652, kl: 0.3660, train_time_taken: 7.3148
INFO:tensorflow:step: 25480, lr: 0.000087, klw: 0.3629, cost: -1.4191, recon: -1.5531, kl: 0.3691, train_time_taken: 7.3482
INFO:tensorflow:step: 25500, lr: 0.000087, klw: 0.3631, cost: -1.4534, recon: -1.5875, kl: 0.3694, train_time_taken: 7.3831
INFO:tensorflow:best_valid_cost: -1.6772, valid_cost: -1.6766, valid_recon: -1.6802, valid_kl: 0.3660, valid_time_taken: 0.6337
INFO:tensorflow:step: 25520, lr: 0.000087, klw: 0.3632, cost: -1.4525, recon: -1.5850, kl: 0.3649, train_time_taken: 7.3271
INFO:tensorflow:step: 25540, lr: 0.000087, klw: 0.3634, cost: -1.4349, recon: -1.5676, kl: 0.3651, train_time_taken: 7.2480
INFO:tensorflow:step: 25560, lr: 0.000087, klw: 0.3635, cost: -1.4433, recon: -1.5763, kl: 0.3658, train_time_taken: 7.4294
INFO:tensorflow:step: 25580, lr: 0.000087, klw: 0.3636, cost: -1.2605, recon: -1.3890, kl: 0.3535, train_time_taken: 7.4109
INFO:tensorflow:step: 25600, lr: 0.000087, klw: 0.3638, cost: -1.4748, recon: -1.6081, kl: 0.3665, train_time_taken: 7.3945
INFO:tensorflow:step: 25620, lr: 0.000086, klw: 0.3639, cost: -1.4298, recon: -1.5636, kl: 0.3677, train_time_taken: 7.9907
INFO:tensorflow:step: 25640, lr: 0.000086, klw: 0.3640, cost: -1.4423, recon: -1.5781, kl: 0.3730, train_time_taken: 8.0587
INFO:tensorflow:step: 25660, lr: 0.000086, klw: 0.3642, cost: -1.4716, recon: -1.6036, kl: 0.3625, train_time_taken: 7.2933
INFO:tensorflow:step: 25680, lr: 0.000086, klw: 0.3643, cost: -1.4336, recon: -1.5653, kl: 0.3615, train_time_taken: 7.2825
INFO:tensorflow:step: 25700, lr: 0.000086, klw: 0.3644, cost: -1.4768, recon: -1.6124, kl: 0.3721, train_time_taken: 7.2836
INFO:tensorflow:step: 25720, lr: 0.000086, klw: 0.3646, cost: -1.4709, recon: -1.6038, kl: 0.3643, train_time_taken: 7.2753
INFO:tensorflow:step: 25740, lr: 0.000085, klw: 0.3647, cost: -1.4428, recon: -1.5770, kl: 0.3680, train_time_taken: 7.4146
INFO:tensorflow:step: 25760, lr: 0.000085, klw: 0.3649, cost: -1.4260, recon: -1.5601, kl: 0.3675, train_time_taken: 7.3060
INFO:tensorflow:step: 25780, lr: 0.000085, klw: 0.3650, cost: -1.5059, recon: -1.6445, kl: 0.3797, train_time_taken: 7.2414
INFO:tensorflow:step: 25800, lr: 0.000085, klw: 0.3651, cost: -1.5247, recon: -1.6591, kl: 0.3679, train_time_taken: 7.4100
INFO:tensorflow:step: 25820, lr: 0.000085, klw: 0.3653, cost: -1.4437, recon: -1.5796, kl: 0.3721, train_time_taken: 7.3594
INFO:tensorflow:step: 25840, lr: 0.000085, klw: 0.3654, cost: -1.3424, recon: -1.4718, kl: 0.3541, train_time_taken: 8.2772
INFO:tensorflow:step: 25860, lr: 0.000085, klw: 0.3655, cost: -1.4668, recon: -1.6026, kl: 0.3716, train_time_taken: 7.7787
INFO:tensorflow:step: 25880, lr: 0.000084, klw: 0.3657, cost: -1.3821, recon: -1.5137, kl: 0.3599, train_time_taken: 7.3296
INFO:tensorflow:step: 25900, lr: 0.000084, klw: 0.3658, cost: -1.3995, recon: -1.5320, kl: 0.3622, train_time_taken: 7.2789
INFO:tensorflow:step: 25920, lr: 0.000084, klw: 0.3659, cost: -1.4799, recon: -1.6163, kl: 0.3727, train_time_taken: 7.4010
INFO:tensorflow:step: 25940, lr: 0.000084, klw: 0.3661, cost: -1.4381, recon: -1.5708, kl: 0.3625, train_time_taken: 7.3110
INFO:tensorflow:step: 25960, lr: 0.000084, klw: 0.3662, cost: -1.4016, recon: -1.5324, kl: 0.3573, train_time_taken: 7.3548
INFO:tensorflow:step: 25980, lr: 0.000084, klw: 0.3663, cost: -1.4773, recon: -1.6133, kl: 0.3712, train_time_taken: 7.2706
INFO:tensorflow:step: 26000, lr: 0.000084, klw: 0.3665, cost: -1.4125, recon: -1.5453, kl: 0.3624, train_time_taken: 7.3737
INFO:tensorflow:best_valid_cost: -1.6772, valid_cost: -1.6762, valid_recon: -1.6798, valid_kl: 0.3654, valid_time_taken: 0.6221
INFO:tensorflow:step: 26020, lr: 0.000083, klw: 0.3666, cost: -1.3365, recon: -1.4657, kl: 0.3525, train_time_taken: 8.1200
INFO:tensorflow:step: 26040, lr: 0.000083, klw: 0.3667, cost: -1.4733, recon: -1.6078, kl: 0.3667, train_time_taken: 7.5553
INFO:tensorflow:step: 26060, lr: 0.000083, klw: 0.3669, cost: -1.4472, recon: -1.5822, kl: 0.3680, train_time_taken: 8.5469
INFO:tensorflow:step: 26080, lr: 0.000083, klw: 0.3670, cost: -1.4553, recon: -1.5901, kl: 0.3671, train_time_taken: 7.3172
INFO:tensorflow:step: 26100, lr: 0.000083, klw: 0.3671, cost: -1.3648, recon: -1.4962, kl: 0.3580, train_time_taken: 7.2434
INFO:tensorflow:step: 26120, lr: 0.000083, klw: 0.3673, cost: -1.4058, recon: -1.5382, kl: 0.3605, train_time_taken: 7.3052
INFO:tensorflow:step: 26140, lr: 0.000082, klw: 0.3674, cost: -1.3918, recon: -1.5235, kl: 0.3585, train_time_taken: 7.3362
INFO:tensorflow:step: 26160, lr: 0.000082, klw: 0.3675, cost: -1.4034, recon: -1.5365, kl: 0.3619, train_time_taken: 8.7124
INFO:tensorflow:step: 26180, lr: 0.000082, klw: 0.3677, cost: -1.3482, recon: -1.4804, kl: 0.3597, train_time_taken: 7.7671
INFO:tensorflow:step: 26200, lr: 0.000082, klw: 0.3678, cost: -1.4643, recon: -1.5999, kl: 0.3686, train_time_taken: 7.2356
INFO:tensorflow:step: 26220, lr: 0.000082, klw: 0.3679, cost: -1.3894, recon: -1.5220, kl: 0.3604, train_time_taken: 7.3212
INFO:tensorflow:step: 26240, lr: 0.000082, klw: 0.3681, cost: -1.4950, recon: -1.6309, kl: 0.3691, train_time_taken: 7.3593
INFO:tensorflow:step: 26260, lr: 0.000082, klw: 0.3682, cost: -1.4182, recon: -1.5503, kl: 0.3588, train_time_taken: 8.2378
INFO:tensorflow:step: 26280, lr: 0.000081, klw: 0.3683, cost: -1.5076, recon: -1.6415, kl: 0.3635, train_time_taken: 7.9952
INFO:tensorflow:step: 26300, lr: 0.000081, klw: 0.3685, cost: -1.4486, recon: -1.5834, kl: 0.3659, train_time_taken: 7.4554
INFO:tensorflow:step: 26320, lr: 0.000081, klw: 0.3686, cost: -1.4160, recon: -1.5494, kl: 0.3618, train_time_taken: 7.3368
INFO:tensorflow:step: 26340, lr: 0.000081, klw: 0.3687, cost: -1.4600, recon: -1.5939, kl: 0.3633, train_time_taken: 7.4577
INFO:tensorflow:step: 26360, lr: 0.000081, klw: 0.3688, cost: -1.3943, recon: -1.5273, kl: 0.3604, train_time_taken: 7.4053
INFO:tensorflow:step: 26380, lr: 0.000081, klw: 0.3690, cost: -1.3928, recon: -1.5241, kl: 0.3558, train_time_taken: 7.2427
INFO:tensorflow:step: 26400, lr: 0.000081, klw: 0.3691, cost: -1.4660, recon: -1.6007, kl: 0.3647, train_time_taken: 7.4015
INFO:tensorflow:step: 26420, lr: 0.000080, klw: 0.3692, cost: -1.3980, recon: -1.5290, kl: 0.3549, train_time_taken: 7.3574
INFO:tensorflow:step: 26440, lr: 0.000080, klw: 0.3694, cost: -1.4389, recon: -1.5723, kl: 0.3612, train_time_taken: 7.3145
INFO:tensorflow:step: 26460, lr: 0.000080, klw: 0.3695, cost: -1.3824, recon: -1.5121, kl: 0.3511, train_time_taken: 7.2467
INFO:tensorflow:step: 26480, lr: 0.000080, klw: 0.3696, cost: -1.4695, recon: -1.6061, kl: 0.3696, train_time_taken: 8.5549
INFO:tensorflow:step: 26500, lr: 0.000080, klw: 0.3698, cost: -1.3599, recon: -1.4940, kl: 0.3627, train_time_taken: 7.6938
INFO:tensorflow:best_valid_cost: -1.6844, valid_cost: -1.6844, valid_recon: -1.6880, valid_kl: 0.3650, valid_time_taken: 0.6367
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 26500.
INFO:tensorflow:time_taken_save 0.6791.
INFO:tensorflow:eval_cost: -1.6453, eval_recon: -1.6489, eval_kl: 0.3576, eval_time_taken: 0.5146
INFO:tensorflow:step: 26520, lr: 0.000080, klw: 0.3699, cost: -1.3770, recon: -1.5105, kl: 0.3608, train_time_taken: 7.3032
INFO:tensorflow:step: 26540, lr: 0.000080, klw: 0.3700, cost: -1.4893, recon: -1.6245, kl: 0.3655, train_time_taken: 7.4282
INFO:tensorflow:step: 26560, lr: 0.000080, klw: 0.3702, cost: -1.4346, recon: -1.5698, kl: 0.3651, train_time_taken: 7.4349
INFO:tensorflow:step: 26580, lr: 0.000079, klw: 0.3703, cost: -1.3685, recon: -1.5001, kl: 0.3552, train_time_taken: 7.2468
INFO:tensorflow:step: 26600, lr: 0.000079, klw: 0.3704, cost: -1.4697, recon: -1.6055, kl: 0.3666, train_time_taken: 7.2737
INFO:tensorflow:step: 26620, lr: 0.000079, klw: 0.3705, cost: -1.3377, recon: -1.4694, kl: 0.3554, train_time_taken: 7.3585
INFO:tensorflow:step: 26640, lr: 0.000079, klw: 0.3707, cost: -1.4712, recon: -1.6071, kl: 0.3667, train_time_taken: 7.3643
INFO:tensorflow:step: 26660, lr: 0.000079, klw: 0.3708, cost: -1.5166, recon: -1.6532, kl: 0.3685, train_time_taken: 7.3529
INFO:tensorflow:step: 26680, lr: 0.000079, klw: 0.3709, cost: -1.4739, recon: -1.6085, kl: 0.3630, train_time_taken: 7.8300
INFO:tensorflow:step: 26700, lr: 0.000079, klw: 0.3711, cost: -1.4596, recon: -1.5961, kl: 0.3680, train_time_taken: 8.2647
INFO:tensorflow:step: 26720, lr: 0.000078, klw: 0.3712, cost: -1.3369, recon: -1.4694, kl: 0.3570, train_time_taken: 7.3176
INFO:tensorflow:step: 26740, lr: 0.000078, klw: 0.3713, cost: -1.5047, recon: -1.6386, kl: 0.3605, train_time_taken: 7.3447
INFO:tensorflow:step: 26760, lr: 0.000078, klw: 0.3714, cost: -1.3839, recon: -1.5151, kl: 0.3532, train_time_taken: 7.3368
INFO:tensorflow:step: 26780, lr: 0.000078, klw: 0.3716, cost: -1.4698, recon: -1.6044, kl: 0.3622, train_time_taken: 7.3764
INFO:tensorflow:step: 26800, lr: 0.000078, klw: 0.3717, cost: -1.3551, recon: -1.4865, kl: 0.3537, train_time_taken: 7.4104
INFO:tensorflow:step: 26820, lr: 0.000078, klw: 0.3718, cost: -1.4377, recon: -1.5709, kl: 0.3585, train_time_taken: 8.1725
INFO:tensorflow:step: 26840, lr: 0.000078, klw: 0.3720, cost: -1.4366, recon: -1.5720, kl: 0.3638, train_time_taken: 7.3635
INFO:tensorflow:step: 26860, lr: 0.000077, klw: 0.3721, cost: -1.3901, recon: -1.5251, kl: 0.3628, train_time_taken: 7.3176
INFO:tensorflow:step: 26880, lr: 0.000077, klw: 0.3722, cost: -1.5070, recon: -1.6450, kl: 0.3708, train_time_taken: 7.4673
INFO:tensorflow:step: 26900, lr: 0.000077, klw: 0.3723, cost: -1.4874, recon: -1.6208, kl: 0.3581, train_time_taken: 8.4009
INFO:tensorflow:step: 26920, lr: 0.000077, klw: 0.3725, cost: -1.4459, recon: -1.5801, kl: 0.3601, train_time_taken: 7.8606
INFO:tensorflow:step: 26940, lr: 0.000077, klw: 0.3726, cost: -1.4753, recon: -1.6113, kl: 0.3650, train_time_taken: 7.3217
INFO:tensorflow:step: 26960, lr: 0.000077, klw: 0.3727, cost: -1.5010, recon: -1.6383, kl: 0.3684, train_time_taken: 7.3035
INFO:tensorflow:step: 26980, lr: 0.000077, klw: 0.3728, cost: -1.5350, recon: -1.6735, kl: 0.3713, train_time_taken: 8.7282
INFO:tensorflow:step: 27000, lr: 0.000077, klw: 0.3730, cost: -1.2951, recon: -1.4264, kl: 0.3521, train_time_taken: 7.5487
INFO:tensorflow:best_valid_cost: -1.6844, valid_cost: -1.6837, valid_recon: -1.6873, valid_kl: 0.3630, valid_time_taken: 0.6271
INFO:tensorflow:step: 27020, lr: 0.000076, klw: 0.3731, cost: -1.3923, recon: -1.5258, kl: 0.3578, train_time_taken: 7.3060
INFO:tensorflow:step: 27040, lr: 0.000076, klw: 0.3732, cost: -1.4440, recon: -1.5792, kl: 0.3622, train_time_taken: 7.2780
INFO:tensorflow:step: 27060, lr: 0.000076, klw: 0.3734, cost: -1.4334, recon: -1.5684, kl: 0.3616, train_time_taken: 7.4269
INFO:tensorflow:step: 27080, lr: 0.000076, klw: 0.3735, cost: -1.4342, recon: -1.5685, kl: 0.3596, train_time_taken: 7.2924
INFO:tensorflow:step: 27100, lr: 0.000076, klw: 0.3736, cost: -1.5030, recon: -1.6403, kl: 0.3674, train_time_taken: 7.8018
INFO:tensorflow:step: 27120, lr: 0.000076, klw: 0.3737, cost: -1.4816, recon: -1.6195, kl: 0.3690, train_time_taken: 8.3470
INFO:tensorflow:step: 27140, lr: 0.000076, klw: 0.3739, cost: -1.4260, recon: -1.5617, kl: 0.3629, train_time_taken: 7.4536
INFO:tensorflow:step: 27160, lr: 0.000075, klw: 0.3740, cost: -1.3989, recon: -1.5307, kl: 0.3527, train_time_taken: 7.4936
INFO:tensorflow:step: 27180, lr: 0.000075, klw: 0.3741, cost: -1.3602, recon: -1.4938, kl: 0.3572, train_time_taken: 7.5213
INFO:tensorflow:step: 27200, lr: 0.000075, klw: 0.3742, cost: -1.4066, recon: -1.5419, kl: 0.3616, train_time_taken: 7.6848
INFO:tensorflow:step: 27220, lr: 0.000075, klw: 0.3744, cost: -1.3416, recon: -1.4757, kl: 0.3582, train_time_taken: 7.4672
INFO:tensorflow:step: 27240, lr: 0.000075, klw: 0.3745, cost: -1.3919, recon: -1.5254, kl: 0.3565, train_time_taken: 7.4142
INFO:tensorflow:step: 27260, lr: 0.000075, klw: 0.3746, cost: -1.4239, recon: -1.5591, kl: 0.3610, train_time_taken: 7.6291
INFO:tensorflow:step: 27280, lr: 0.000075, klw: 0.3747, cost: -1.4508, recon: -1.5865, kl: 0.3621, train_time_taken: 7.6132
INFO:tensorflow:step: 27300, lr: 0.000075, klw: 0.3749, cost: -1.3925, recon: -1.5269, kl: 0.3586, train_time_taken: 7.5186
INFO:tensorflow:step: 27320, lr: 0.000074, klw: 0.3750, cost: -1.5408, recon: -1.6817, kl: 0.3757, train_time_taken: 8.5463
INFO:tensorflow:step: 27340, lr: 0.000074, klw: 0.3751, cost: -1.4200, recon: -1.5566, kl: 0.3641, train_time_taken: 7.8078
INFO:tensorflow:step: 27360, lr: 0.000074, klw: 0.3752, cost: -1.4927, recon: -1.6315, kl: 0.3699, train_time_taken: 7.4669
INFO:tensorflow:step: 27380, lr: 0.000074, klw: 0.3754, cost: -1.3844, recon: -1.5185, kl: 0.3573, train_time_taken: 7.5186
INFO:tensorflow:step: 27400, lr: 0.000074, klw: 0.3755, cost: -1.3703, recon: -1.5079, kl: 0.3665, train_time_taken: 7.5717
INFO:tensorflow:step: 27420, lr: 0.000074, klw: 0.3756, cost: -1.4442, recon: -1.5804, kl: 0.3627, train_time_taken: 7.6351
INFO:tensorflow:step: 27440, lr: 0.000074, klw: 0.3757, cost: -1.4267, recon: -1.5629, kl: 0.3625, train_time_taken: 7.6378
INFO:tensorflow:step: 27460, lr: 0.000074, klw: 0.3759, cost: -1.4627, recon: -1.5989, kl: 0.3623, train_time_taken: 7.4553
INFO:tensorflow:step: 27480, lr: 0.000073, klw: 0.3760, cost: -1.5048, recon: -1.6452, kl: 0.3736, train_time_taken: 7.3848
INFO:tensorflow:step: 27500, lr: 0.000073, klw: 0.3761, cost: -1.4273, recon: -1.5628, kl: 0.3603, train_time_taken: 7.2518
INFO:tensorflow:best_valid_cost: -1.6844, valid_cost: -1.6781, valid_recon: -1.6817, valid_kl: 0.3623, valid_time_taken: 0.6268
INFO:tensorflow:step: 27520, lr: 0.000073, klw: 0.3762, cost: -1.3382, recon: -1.4687, kl: 0.3466, train_time_taken: 7.8452
INFO:tensorflow:step: 27540, lr: 0.000073, klw: 0.3764, cost: -1.4848, recon: -1.6213, kl: 0.3627, train_time_taken: 8.2438
INFO:tensorflow:step: 27560, lr: 0.000073, klw: 0.3765, cost: -1.4156, recon: -1.5517, kl: 0.3617, train_time_taken: 7.3575
INFO:tensorflow:step: 27580, lr: 0.000073, klw: 0.3766, cost: -1.4752, recon: -1.6115, kl: 0.3618, train_time_taken: 7.3710
INFO:tensorflow:step: 27600, lr: 0.000073, klw: 0.3767, cost: -1.4256, recon: -1.5626, kl: 0.3636, train_time_taken: 7.4074
INFO:tensorflow:step: 27620, lr: 0.000073, klw: 0.3769, cost: -1.4231, recon: -1.5607, kl: 0.3650, train_time_taken: 8.2838
INFO:tensorflow:step: 27640, lr: 0.000072, klw: 0.3770, cost: -1.2670, recon: -1.3958, kl: 0.3417, train_time_taken: 7.4221
INFO:tensorflow:step: 27660, lr: 0.000072, klw: 0.3771, cost: -1.4593, recon: -1.5963, kl: 0.3633, train_time_taken: 7.4287
INFO:tensorflow:step: 27680, lr: 0.000072, klw: 0.3772, cost: -1.3970, recon: -1.5329, kl: 0.3604, train_time_taken: 7.4349
INFO:tensorflow:step: 27700, lr: 0.000072, klw: 0.3773, cost: -1.2688, recon: -1.4003, kl: 0.3486, train_time_taken: 7.4688
INFO:tensorflow:step: 27720, lr: 0.000072, klw: 0.3775, cost: -1.4167, recon: -1.5520, kl: 0.3584, train_time_taken: 7.6109
INFO:tensorflow:step: 27740, lr: 0.000072, klw: 0.3776, cost: -1.4137, recon: -1.5483, kl: 0.3563, train_time_taken: 8.4402
INFO:tensorflow:step: 27760, lr: 0.000072, klw: 0.3777, cost: -1.4307, recon: -1.5657, kl: 0.3574, train_time_taken: 7.9837
INFO:tensorflow:step: 27780, lr: 0.000072, klw: 0.3778, cost: -1.4967, recon: -1.6349, kl: 0.3658, train_time_taken: 8.4152
INFO:tensorflow:step: 27800, lr: 0.000071, klw: 0.3780, cost: -1.4476, recon: -1.5879, kl: 0.3712, train_time_taken: 8.2149
INFO:tensorflow:step: 27820, lr: 0.000071, klw: 0.3781, cost: -1.4674, recon: -1.6034, kl: 0.3598, train_time_taken: 7.5749
INFO:tensorflow:step: 27840, lr: 0.000071, klw: 0.3782, cost: -1.5555, recon: -1.6975, kl: 0.3754, train_time_taken: 7.4549
INFO:tensorflow:step: 27860, lr: 0.000071, klw: 0.3783, cost: -1.4468, recon: -1.5839, kl: 0.3624, train_time_taken: 7.6216
INFO:tensorflow:step: 27880, lr: 0.000071, klw: 0.3784, cost: -1.3841, recon: -1.5171, kl: 0.3513, train_time_taken: 7.4227
INFO:tensorflow:step: 27900, lr: 0.000071, klw: 0.3786, cost: -1.3462, recon: -1.4814, kl: 0.3571, train_time_taken: 7.5465
INFO:tensorflow:step: 27920, lr: 0.000071, klw: 0.3787, cost: -1.4375, recon: -1.5729, kl: 0.3577, train_time_taken: 7.5700
INFO:tensorflow:step: 27940, lr: 0.000071, klw: 0.3788, cost: -1.3092, recon: -1.4409, kl: 0.3477, train_time_taken: 8.1869
INFO:tensorflow:step: 27960, lr: 0.000070, klw: 0.3789, cost: -1.4552, recon: -1.5930, kl: 0.3638, train_time_taken: 8.1919
INFO:tensorflow:step: 27980, lr: 0.000070, klw: 0.3791, cost: -1.4503, recon: -1.5887, kl: 0.3653, train_time_taken: 7.4275
INFO:tensorflow:step: 28000, lr: 0.000070, klw: 0.3792, cost: -1.5620, recon: -1.7037, kl: 0.3736, train_time_taken: 7.6228
INFO:tensorflow:best_valid_cost: -1.6866, valid_cost: -1.6866, valid_recon: -1.6902, valid_kl: 0.3595, valid_time_taken: 0.6392
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 28000.
INFO:tensorflow:time_taken_save 0.7085.
INFO:tensorflow:eval_cost: -1.6476, eval_recon: -1.6511, eval_kl: 0.3518, eval_time_taken: 0.5258
INFO:tensorflow:step: 28020, lr: 0.000070, klw: 0.3793, cost: -1.4848, recon: -1.6232, kl: 0.3647, train_time_taken: 7.4598
INFO:tensorflow:step: 28040, lr: 0.000070, klw: 0.3794, cost: -1.3604, recon: -1.4943, kl: 0.3530, train_time_taken: 7.4879
INFO:tensorflow:step: 28060, lr: 0.000070, klw: 0.3795, cost: -1.4897, recon: -1.6314, kl: 0.3734, train_time_taken: 7.5855
INFO:tensorflow:step: 28080, lr: 0.000070, klw: 0.3797, cost: -1.4394, recon: -1.5786, kl: 0.3666, train_time_taken: 7.4950
INFO:tensorflow:step: 28100, lr: 0.000070, klw: 0.3798, cost: -1.4240, recon: -1.5599, kl: 0.3577, train_time_taken: 7.4639
INFO:tensorflow:step: 28120, lr: 0.000069, klw: 0.3799, cost: -1.4684, recon: -1.6033, kl: 0.3550, train_time_taken: 7.4950
INFO:tensorflow:step: 28140, lr: 0.000069, klw: 0.3800, cost: -1.4009, recon: -1.5358, kl: 0.3550, train_time_taken: 7.8252
INFO:tensorflow:step: 28160, lr: 0.000069, klw: 0.3801, cost: -1.3404, recon: -1.4744, kl: 0.3525, train_time_taken: 8.4426
INFO:tensorflow:step: 28180, lr: 0.000069, klw: 0.3803, cost: -1.3997, recon: -1.5353, kl: 0.3565, train_time_taken: 7.6159
INFO:tensorflow:step: 28200, lr: 0.000069, klw: 0.3804, cost: -1.4153, recon: -1.5556, kl: 0.3688, train_time_taken: 7.6368
INFO:tensorflow:step: 28220, lr: 0.000069, klw: 0.3805, cost: -1.3662, recon: -1.5009, kl: 0.3541, train_time_taken: 7.4609
INFO:tensorflow:step: 28240, lr: 0.000069, klw: 0.3806, cost: -1.4371, recon: -1.5761, kl: 0.3650, train_time_taken: 7.4463
INFO:tensorflow:step: 28260, lr: 0.000069, klw: 0.3807, cost: -1.4953, recon: -1.6345, kl: 0.3657, train_time_taken: 7.3481
INFO:tensorflow:step: 28280, lr: 0.000069, klw: 0.3809, cost: -1.5136, recon: -1.6550, kl: 0.3712, train_time_taken: 7.2146
INFO:tensorflow:step: 28300, lr: 0.000068, klw: 0.3810, cost: -1.5188, recon: -1.6573, kl: 0.3637, train_time_taken: 7.3744
INFO:tensorflow:step: 28320, lr: 0.000068, klw: 0.3811, cost: -1.3985, recon: -1.5336, kl: 0.3544, train_time_taken: 7.2604
INFO:tensorflow:step: 28340, lr: 0.000068, klw: 0.3812, cost: -1.3777, recon: -1.5129, kl: 0.3549, train_time_taken: 7.3412
INFO:tensorflow:step: 28360, lr: 0.000068, klw: 0.3813, cost: -1.4542, recon: -1.5919, kl: 0.3613, train_time_taken: 7.8561
INFO:tensorflow:step: 28380, lr: 0.000068, klw: 0.3814, cost: -1.5011, recon: -1.6405, kl: 0.3653, train_time_taken: 8.0869
INFO:tensorflow:step: 28400, lr: 0.000068, klw: 0.3816, cost: -1.2740, recon: -1.4039, kl: 0.3404, train_time_taken: 7.7680
INFO:tensorflow:step: 28420, lr: 0.000068, klw: 0.3817, cost: -1.4547, recon: -1.5919, kl: 0.3595, train_time_taken: 7.7377
INFO:tensorflow:step: 28440, lr: 0.000068, klw: 0.3818, cost: -1.5341, recon: -1.6787, kl: 0.3787, train_time_taken: 7.4093
INFO:tensorflow:step: 28460, lr: 0.000067, klw: 0.3819, cost: -1.5309, recon: -1.6725, kl: 0.3707, train_time_taken: 7.3787
INFO:tensorflow:step: 28480, lr: 0.000067, klw: 0.3820, cost: -1.4470, recon: -1.5869, kl: 0.3662, train_time_taken: 7.3984
INFO:tensorflow:step: 28500, lr: 0.000067, klw: 0.3822, cost: -1.4823, recon: -1.6187, kl: 0.3569, train_time_taken: 7.2930
INFO:tensorflow:best_valid_cost: -1.6888, valid_cost: -1.6888, valid_recon: -1.6923, valid_kl: 0.3583, valid_time_taken: 0.6244
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 28500.
INFO:tensorflow:time_taken_save 0.6916.
INFO:tensorflow:eval_cost: -1.6497, eval_recon: -1.6532, eval_kl: 0.3506, eval_time_taken: 0.5239
INFO:tensorflow:step: 28520, lr: 0.000067, klw: 0.3823, cost: -1.3669, recon: -1.5016, kl: 0.3524, train_time_taken: 7.3030
INFO:tensorflow:step: 28540, lr: 0.000067, klw: 0.3824, cost: -1.4371, recon: -1.5737, kl: 0.3573, train_time_taken: 7.4124
INFO:tensorflow:step: 28560, lr: 0.000067, klw: 0.3825, cost: -1.5129, recon: -1.6539, kl: 0.3687, train_time_taken: 7.7327
INFO:tensorflow:step: 28580, lr: 0.000067, klw: 0.3826, cost: -1.4987, recon: -1.6376, kl: 0.3631, train_time_taken: 8.7060
INFO:tensorflow:step: 28600, lr: 0.000067, klw: 0.3827, cost: -1.4409, recon: -1.5788, kl: 0.3602, train_time_taken: 8.4592
INFO:tensorflow:step: 28620, lr: 0.000067, klw: 0.3829, cost: -1.3681, recon: -1.5032, kl: 0.3528, train_time_taken: 7.4067
INFO:tensorflow:step: 28640, lr: 0.000066, klw: 0.3830, cost: -1.4872, recon: -1.6265, kl: 0.3636, train_time_taken: 7.3861
INFO:tensorflow:step: 28660, lr: 0.000066, klw: 0.3831, cost: -1.4809, recon: -1.6232, kl: 0.3712, train_time_taken: 7.2834
INFO:tensorflow:step: 28680, lr: 0.000066, klw: 0.3832, cost: -1.4816, recon: -1.6227, kl: 0.3682, train_time_taken: 7.4048
INFO:tensorflow:step: 28700, lr: 0.000066, klw: 0.3833, cost: -1.4824, recon: -1.6200, kl: 0.3592, train_time_taken: 7.3443
INFO:tensorflow:step: 28720, lr: 0.000066, klw: 0.3834, cost: -1.4048, recon: -1.5393, kl: 0.3507, train_time_taken: 7.4755
INFO:tensorflow:step: 28740, lr: 0.000066, klw: 0.3836, cost: -1.4360, recon: -1.5731, kl: 0.3575, train_time_taken: 7.3762
INFO:tensorflow:step: 28760, lr: 0.000066, klw: 0.3837, cost: -1.4199, recon: -1.5560, kl: 0.3548, train_time_taken: 7.3141
INFO:tensorflow:step: 28780, lr: 0.000066, klw: 0.3838, cost: -1.4151, recon: -1.5540, kl: 0.3619, train_time_taken: 8.2396
INFO:tensorflow:step: 28800, lr: 0.000066, klw: 0.3839, cost: -1.4907, recon: -1.6286, kl: 0.3592, train_time_taken: 7.8924
INFO:tensorflow:step: 28820, lr: 0.000065, klw: 0.3840, cost: -1.5786, recon: -1.7198, kl: 0.3678, train_time_taken: 7.3509
INFO:tensorflow:step: 28840, lr: 0.000065, klw: 0.3841, cost: -1.4658, recon: -1.6053, kl: 0.3630, train_time_taken: 7.3624
INFO:tensorflow:step: 28860, lr: 0.000065, klw: 0.3843, cost: -1.3859, recon: -1.5210, kl: 0.3515, train_time_taken: 7.3644
INFO:tensorflow:step: 28880, lr: 0.000065, klw: 0.3844, cost: -1.4842, recon: -1.6215, kl: 0.3572, train_time_taken: 7.5530
INFO:tensorflow:step: 28900, lr: 0.000065, klw: 0.3845, cost: -1.4857, recon: -1.6238, kl: 0.3593, train_time_taken: 7.3555
INFO:tensorflow:step: 28920, lr: 0.000065, klw: 0.3846, cost: -1.4205, recon: -1.5569, kl: 0.3546, train_time_taken: 7.4900
INFO:tensorflow:step: 28940, lr: 0.000065, klw: 0.3847, cost: -1.4979, recon: -1.6382, kl: 0.3647, train_time_taken: 7.5316
INFO:tensorflow:step: 28960, lr: 0.000065, klw: 0.3848, cost: -1.4532, recon: -1.5923, kl: 0.3614, train_time_taken: 7.4854
INFO:tensorflow:step: 28980, lr: 0.000065, klw: 0.3849, cost: -1.4496, recon: -1.5865, kl: 0.3557, train_time_taken: 7.4305
INFO:tensorflow:step: 29000, lr: 0.000064, klw: 0.3851, cost: -1.4503, recon: -1.5872, kl: 0.3556, train_time_taken: 8.4884
INFO:tensorflow:best_valid_cost: -1.6926, valid_cost: -1.6926, valid_recon: -1.6962, valid_kl: 0.3591, valid_time_taken: 0.7268
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 29000.
INFO:tensorflow:time_taken_save 0.7794.
INFO:tensorflow:eval_cost: -1.6528, eval_recon: -1.6563, eval_kl: 0.3514, eval_time_taken: 0.5266
INFO:tensorflow:step: 29020, lr: 0.000064, klw: 0.3852, cost: -1.4527, recon: -1.5920, kl: 0.3615, train_time_taken: 7.5727
INFO:tensorflow:step: 29040, lr: 0.000064, klw: 0.3853, cost: -1.5028, recon: -1.6450, kl: 0.3693, train_time_taken: 7.3857
INFO:tensorflow:step: 29060, lr: 0.000064, klw: 0.3854, cost: -1.4228, recon: -1.5584, kl: 0.3520, train_time_taken: 7.4466
INFO:tensorflow:step: 29080, lr: 0.000064, klw: 0.3855, cost: -1.4441, recon: -1.5796, kl: 0.3513, train_time_taken: 7.4451
INFO:tensorflow:step: 29100, lr: 0.000064, klw: 0.3856, cost: -1.3490, recon: -1.4826, kl: 0.3464, train_time_taken: 7.4669
INFO:tensorflow:step: 29120, lr: 0.000064, klw: 0.3858, cost: -1.3826, recon: -1.5166, kl: 0.3474, train_time_taken: 7.4547
INFO:tensorflow:step: 29140, lr: 0.000064, klw: 0.3859, cost: -1.4180, recon: -1.5575, kl: 0.3616, train_time_taken: 7.3836
INFO:tensorflow:step: 29160, lr: 0.000064, klw: 0.3860, cost: -1.4746, recon: -1.6142, kl: 0.3619, train_time_taken: 7.4151
INFO:tensorflow:step: 29180, lr: 0.000063, klw: 0.3861, cost: -1.3698, recon: -1.5041, kl: 0.3478, train_time_taken: 7.4847
INFO:tensorflow:step: 29200, lr: 0.000063, klw: 0.3862, cost: -1.4924, recon: -1.6297, kl: 0.3557, train_time_taken: 8.0688
INFO:tensorflow:step: 29220, lr: 0.000063, klw: 0.3863, cost: -1.4468, recon: -1.5853, kl: 0.3584, train_time_taken: 7.9648
INFO:tensorflow:step: 29240, lr: 0.000063, klw: 0.3864, cost: -1.5210, recon: -1.6637, kl: 0.3693, train_time_taken: 7.3993
INFO:tensorflow:step: 29260, lr: 0.000063, klw: 0.3865, cost: -1.3648, recon: -1.5029, kl: 0.3572, train_time_taken: 7.4567
INFO:tensorflow:step: 29280, lr: 0.000063, klw: 0.3867, cost: -1.5435, recon: -1.6824, kl: 0.3594, train_time_taken: 7.4107
INFO:tensorflow:step: 29300, lr: 0.000063, klw: 0.3868, cost: -1.4661, recon: -1.6062, kl: 0.3624, train_time_taken: 7.3383
INFO:tensorflow:step: 29320, lr: 0.000063, klw: 0.3869, cost: -1.5146, recon: -1.6534, kl: 0.3589, train_time_taken: 7.2498
INFO:tensorflow:step: 29340, lr: 0.000063, klw: 0.3870, cost: -1.5291, recon: -1.6709, kl: 0.3663, train_time_taken: 7.4306
INFO:tensorflow:step: 29360, lr: 0.000063, klw: 0.3871, cost: -1.3840, recon: -1.5216, kl: 0.3557, train_time_taken: 7.3910
INFO:tensorflow:step: 29380, lr: 0.000062, klw: 0.3872, cost: -1.4798, recon: -1.6203, kl: 0.3628, train_time_taken: 7.4832
INFO:tensorflow:step: 29400, lr: 0.000062, klw: 0.3873, cost: -1.4369, recon: -1.5752, kl: 0.3572, train_time_taken: 8.2027
INFO:tensorflow:step: 29420, lr: 0.000062, klw: 0.3875, cost: -1.5002, recon: -1.6410, kl: 0.3634, train_time_taken: 8.7213
INFO:tensorflow:step: 29440, lr: 0.000062, klw: 0.3876, cost: -1.4995, recon: -1.6408, kl: 0.3644, train_time_taken: 7.5489
INFO:tensorflow:step: 29460, lr: 0.000062, klw: 0.3877, cost: -1.4063, recon: -1.5433, kl: 0.3534, train_time_taken: 7.4352
INFO:tensorflow:step: 29480, lr: 0.000062, klw: 0.3878, cost: -1.5425, recon: -1.6845, kl: 0.3663, train_time_taken: 7.3599
INFO:tensorflow:step: 29500, lr: 0.000062, klw: 0.3879, cost: -1.4245, recon: -1.5642, kl: 0.3602, train_time_taken: 7.2898
INFO:tensorflow:best_valid_cost: -1.6934, valid_cost: -1.6934, valid_recon: -1.6970, valid_kl: 0.3588, valid_time_taken: 0.6239
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 29500.
INFO:tensorflow:time_taken_save 0.7124.
INFO:tensorflow:eval_cost: -1.6549, eval_recon: -1.6584, eval_kl: 0.3511, eval_time_taken: 0.5209
INFO:tensorflow:step: 29520, lr: 0.000062, klw: 0.3880, cost: -1.4323, recon: -1.5713, kl: 0.3580, train_time_taken: 7.3012
INFO:tensorflow:step: 29540, lr: 0.000062, klw: 0.3881, cost: -1.4503, recon: -1.5916, kl: 0.3641, train_time_taken: 7.4134
INFO:tensorflow:step: 29560, lr: 0.000061, klw: 0.3882, cost: -1.4428, recon: -1.5826, kl: 0.3601, train_time_taken: 7.3633
INFO:tensorflow:step: 29580, lr: 0.000061, klw: 0.3884, cost: -1.3055, recon: -1.4408, kl: 0.3483, train_time_taken: 7.2733
INFO:tensorflow:step: 29600, lr: 0.000061, klw: 0.3885, cost: -1.5090, recon: -1.6502, kl: 0.3636, train_time_taken: 7.2671
INFO:tensorflow:step: 29620, lr: 0.000061, klw: 0.3886, cost: -1.4758, recon: -1.6146, kl: 0.3573, train_time_taken: 7.9866
INFO:tensorflow:step: 29640, lr: 0.000061, klw: 0.3887, cost: -1.4860, recon: -1.6266, kl: 0.3617, train_time_taken: 8.2020
INFO:tensorflow:step: 29660, lr: 0.000061, klw: 0.3888, cost: -1.5160, recon: -1.6563, kl: 0.3608, train_time_taken: 7.3430
INFO:tensorflow:step: 29680, lr: 0.000061, klw: 0.3889, cost: -1.3488, recon: -1.4832, kl: 0.3455, train_time_taken: 7.3456
INFO:tensorflow:step: 29700, lr: 0.000061, klw: 0.3890, cost: -1.3926, recon: -1.5299, kl: 0.3528, train_time_taken: 7.4479
INFO:tensorflow:step: 29720, lr: 0.000061, klw: 0.3891, cost: -1.4355, recon: -1.5737, kl: 0.3551, train_time_taken: 7.2926
INFO:tensorflow:step: 29740, lr: 0.000061, klw: 0.3892, cost: -1.4393, recon: -1.5762, kl: 0.3519, train_time_taken: 7.4200
INFO:tensorflow:step: 29760, lr: 0.000060, klw: 0.3894, cost: -1.4749, recon: -1.6172, kl: 0.3657, train_time_taken: 7.4192
INFO:tensorflow:step: 29780, lr: 0.000060, klw: 0.3895, cost: -1.3399, recon: -1.4751, kl: 0.3472, train_time_taken: 7.3829
INFO:tensorflow:step: 29800, lr: 0.000060, klw: 0.3896, cost: -1.3706, recon: -1.5062, kl: 0.3479, train_time_taken: 7.3054
INFO:tensorflow:step: 29820, lr: 0.000060, klw: 0.3897, cost: -1.3932, recon: -1.5334, kl: 0.3599, train_time_taken: 7.6001
INFO:tensorflow:step: 29840, lr: 0.000060, klw: 0.3898, cost: -1.4477, recon: -1.5850, kl: 0.3522, train_time_taken: 8.5925
INFO:tensorflow:step: 29860, lr: 0.000060, klw: 0.3899, cost: -1.4562, recon: -1.5959, kl: 0.3584, train_time_taken: 7.8465
INFO:tensorflow:step: 29880, lr: 0.000060, klw: 0.3900, cost: -1.4637, recon: -1.6047, kl: 0.3616, train_time_taken: 7.5358
INFO:tensorflow:step: 29900, lr: 0.000060, klw: 0.3901, cost: -1.4072, recon: -1.5466, kl: 0.3573, train_time_taken: 7.5152
INFO:tensorflow:step: 29920, lr: 0.000060, klw: 0.3902, cost: -1.4843, recon: -1.6246, kl: 0.3594, train_time_taken: 7.5995
INFO:tensorflow:step: 29940, lr: 0.000060, klw: 0.3903, cost: -1.4474, recon: -1.5884, kl: 0.3611, train_time_taken: 7.5696
INFO:tensorflow:step: 29960, lr: 0.000059, klw: 0.3905, cost: -1.4611, recon: -1.6016, kl: 0.3600, train_time_taken: 7.6521
INFO:tensorflow:step: 29980, lr: 0.000059, klw: 0.3906, cost: -1.3375, recon: -1.4721, kl: 0.3447, train_time_taken: 7.4283
INFO:tensorflow:step: 30000, lr: 0.000059, klw: 0.3907, cost: -1.3905, recon: -1.5266, kl: 0.3485, train_time_taken: 8.0251
INFO:tensorflow:best_valid_cost: -1.6939, valid_cost: -1.6939, valid_recon: -1.6975, valid_kl: 0.3570, valid_time_taken: 0.7201
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 30000.
INFO:tensorflow:time_taken_save 0.8414.
INFO:tensorflow:eval_cost: -1.6526, eval_recon: -1.6561, eval_kl: 0.3494, eval_time_taken: 0.5352
INFO:tensorflow:step: 30020, lr: 0.000059, klw: 0.3908, cost: -1.4464, recon: -1.5871, kl: 0.3600, train_time_taken: 7.5190
INFO:tensorflow:step: 30040, lr: 0.000059, klw: 0.3909, cost: -1.5352, recon: -1.6789, kl: 0.3676, train_time_taken: 8.4700
INFO:tensorflow:step: 30060, lr: 0.000059, klw: 0.3910, cost: -1.4610, recon: -1.6007, kl: 0.3573, train_time_taken: 8.1520
INFO:tensorflow:step: 30080, lr: 0.000059, klw: 0.3911, cost: -1.4497, recon: -1.5888, kl: 0.3558, train_time_taken: 7.4357
INFO:tensorflow:step: 30100, lr: 0.000059, klw: 0.3912, cost: -1.3398, recon: -1.4734, kl: 0.3417, train_time_taken: 7.6033
INFO:tensorflow:step: 30120, lr: 0.000059, klw: 0.3913, cost: -1.4880, recon: -1.6282, kl: 0.3582, train_time_taken: 7.5589
INFO:tensorflow:step: 30140, lr: 0.000059, klw: 0.3914, cost: -1.3410, recon: -1.4774, kl: 0.3485, train_time_taken: 7.7962
INFO:tensorflow:step: 30160, lr: 0.000058, klw: 0.3915, cost: -1.4176, recon: -1.5578, kl: 0.3579, train_time_taken: 7.5654
INFO:tensorflow:step: 30180, lr: 0.000058, klw: 0.3916, cost: -1.3544, recon: -1.4938, kl: 0.3560, train_time_taken: 7.6184
INFO:tensorflow:step: 30200, lr: 0.000058, klw: 0.3918, cost: -1.4116, recon: -1.5495, kl: 0.3519, train_time_taken: 8.0328
INFO:tensorflow:step: 30220, lr: 0.000058, klw: 0.3919, cost: -1.5129, recon: -1.6545, kl: 0.3614, train_time_taken: 8.6142
INFO:tensorflow:step: 30240, lr: 0.000058, klw: 0.3920, cost: -1.4149, recon: -1.5541, kl: 0.3550, train_time_taken: 8.2127
INFO:tensorflow:step: 30260, lr: 0.000058, klw: 0.3921, cost: -1.4529, recon: -1.5916, kl: 0.3538, train_time_taken: 8.2949
INFO:tensorflow:step: 30280, lr: 0.000058, klw: 0.3922, cost: -1.4276, recon: -1.5635, kl: 0.3465, train_time_taken: 7.4588
INFO:tensorflow:step: 30300, lr: 0.000058, klw: 0.3923, cost: -1.4465, recon: -1.5871, kl: 0.3584, train_time_taken: 7.5934
INFO:tensorflow:step: 30320, lr: 0.000058, klw: 0.3924, cost: -1.4580, recon: -1.5985, kl: 0.3582, train_time_taken: 7.4974
INFO:tensorflow:step: 30340, lr: 0.000058, klw: 0.3925, cost: -1.4296, recon: -1.5687, kl: 0.3543, train_time_taken: 7.5334
INFO:tensorflow:step: 30360, lr: 0.000058, klw: 0.3926, cost: -1.2905, recon: -1.4269, kl: 0.3474, train_time_taken: 7.5379
INFO:tensorflow:step: 30380, lr: 0.000057, klw: 0.3927, cost: -1.3181, recon: -1.4550, kl: 0.3485, train_time_taken: 7.5233
INFO:tensorflow:step: 30400, lr: 0.000057, klw: 0.3928, cost: -1.3870, recon: -1.5261, kl: 0.3540, train_time_taken: 7.6433
INFO:tensorflow:step: 30420, lr: 0.000057, klw: 0.3929, cost: -1.4240, recon: -1.5615, kl: 0.3499, train_time_taken: 7.6603
INFO:tensorflow:step: 30440, lr: 0.000057, klw: 0.3930, cost: -1.4476, recon: -1.5862, kl: 0.3526, train_time_taken: 7.5647
INFO:tensorflow:step: 30460, lr: 0.000057, klw: 0.3932, cost: -1.2813, recon: -1.4159, kl: 0.3424, train_time_taken: 8.6767
INFO:tensorflow:step: 30480, lr: 0.000057, klw: 0.3933, cost: -1.3966, recon: -1.5363, kl: 0.3552, train_time_taken: 7.6049
INFO:tensorflow:step: 30500, lr: 0.000057, klw: 0.3934, cost: -1.4153, recon: -1.5515, kl: 0.3465, train_time_taken: 7.5135
INFO:tensorflow:best_valid_cost: -1.6939, valid_cost: -1.6916, valid_recon: -1.6952, valid_kl: 0.3548, valid_time_taken: 0.6425
INFO:tensorflow:step: 30520, lr: 0.000057, klw: 0.3935, cost: -1.4051, recon: -1.5443, kl: 0.3538, train_time_taken: 7.5325
INFO:tensorflow:step: 30540, lr: 0.000057, klw: 0.3936, cost: -1.4209, recon: -1.5618, kl: 0.3580, train_time_taken: 7.6765
INFO:tensorflow:step: 30560, lr: 0.000057, klw: 0.3937, cost: -1.5046, recon: -1.6431, kl: 0.3518, train_time_taken: 7.5663
INFO:tensorflow:step: 30580, lr: 0.000057, klw: 0.3938, cost: -1.5484, recon: -1.6927, kl: 0.3665, train_time_taken: 7.4687
INFO:tensorflow:step: 30600, lr: 0.000056, klw: 0.3939, cost: -1.4319, recon: -1.5717, kl: 0.3550, train_time_taken: 7.4983
INFO:tensorflow:step: 30620, lr: 0.000056, klw: 0.3940, cost: -1.4726, recon: -1.6126, kl: 0.3553, train_time_taken: 7.5963
INFO:tensorflow:step: 30640, lr: 0.000056, klw: 0.3941, cost: -1.3814, recon: -1.5213, kl: 0.3549, train_time_taken: 7.5044
INFO:tensorflow:step: 30660, lr: 0.000056, klw: 0.3942, cost: -1.4410, recon: -1.5820, kl: 0.3576, train_time_taken: 8.2234
INFO:tensorflow:step: 30680, lr: 0.000056, klw: 0.3943, cost: -1.3706, recon: -1.5067, kl: 0.3451, train_time_taken: 8.1513
INFO:tensorflow:step: 30700, lr: 0.000056, klw: 0.3944, cost: -1.4275, recon: -1.5682, kl: 0.3566, train_time_taken: 7.5086
INFO:tensorflow:step: 30720, lr: 0.000056, klw: 0.3945, cost: -1.5048, recon: -1.6471, kl: 0.3606, train_time_taken: 7.6723
INFO:tensorflow:step: 30740, lr: 0.000056, klw: 0.3946, cost: -1.4134, recon: -1.5532, kl: 0.3542, train_time_taken: 7.6110
INFO:tensorflow:step: 30760, lr: 0.000056, klw: 0.3947, cost: -1.4949, recon: -1.6363, kl: 0.3584, train_time_taken: 7.6593
INFO:tensorflow:step: 30780, lr: 0.000056, klw: 0.3949, cost: -1.3473, recon: -1.4830, kl: 0.3437, train_time_taken: 8.1191
INFO:tensorflow:step: 30800, lr: 0.000055, klw: 0.3950, cost: -1.4638, recon: -1.6055, kl: 0.3589, train_time_taken: 7.9347
INFO:tensorflow:step: 30820, lr: 0.000055, klw: 0.3951, cost: -1.5096, recon: -1.6538, kl: 0.3650, train_time_taken: 7.6323
INFO:tensorflow:step: 30840, lr: 0.000055, klw: 0.3952, cost: -1.4185, recon: -1.5569, kl: 0.3502, train_time_taken: 7.7784
INFO:tensorflow:step: 30860, lr: 0.000055, klw: 0.3953, cost: -1.5006, recon: -1.6408, kl: 0.3545, train_time_taken: 7.9644
INFO:tensorflow:step: 30880, lr: 0.000055, klw: 0.3954, cost: -1.4014, recon: -1.5419, kl: 0.3555, train_time_taken: 8.6183
INFO:tensorflow:step: 30900, lr: 0.000055, klw: 0.3955, cost: -1.3861, recon: -1.5239, kl: 0.3486, train_time_taken: 7.5963
INFO:tensorflow:step: 30920, lr: 0.000055, klw: 0.3956, cost: -1.5231, recon: -1.6632, kl: 0.3540, train_time_taken: 7.5285
INFO:tensorflow:step: 30940, lr: 0.000055, klw: 0.3957, cost: -1.5425, recon: -1.6848, kl: 0.3597, train_time_taken: 7.6309
INFO:tensorflow:step: 30960, lr: 0.000055, klw: 0.3958, cost: -1.4811, recon: -1.6239, kl: 0.3607, train_time_taken: 7.6182
INFO:tensorflow:step: 30980, lr: 0.000055, klw: 0.3959, cost: -1.3665, recon: -1.5051, kl: 0.3502, train_time_taken: 7.6363
INFO:tensorflow:step: 31000, lr: 0.000055, klw: 0.3960, cost: -1.4944, recon: -1.6379, kl: 0.3623, train_time_taken: 8.1875
INFO:tensorflow:best_valid_cost: -1.6964, valid_cost: -1.6964, valid_recon: -1.6999, valid_kl: 0.3543, valid_time_taken: 0.7478
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 31000.
INFO:tensorflow:time_taken_save 0.8598.
INFO:tensorflow:eval_cost: -1.6575, eval_recon: -1.6610, eval_kl: 0.3466, eval_time_taken: 0.5990
INFO:tensorflow:step: 31020, lr: 0.000055, klw: 0.3961, cost: -1.3432, recon: -1.4795, kl: 0.3440, train_time_taken: 8.0361
INFO:tensorflow:step: 31040, lr: 0.000054, klw: 0.3962, cost: -1.3808, recon: -1.5170, kl: 0.3438, train_time_taken: 7.4614
INFO:tensorflow:step: 31060, lr: 0.000054, klw: 0.3963, cost: -1.4165, recon: -1.5560, kl: 0.3520, train_time_taken: 7.9625
INFO:tensorflow:step: 31080, lr: 0.000054, klw: 0.3964, cost: -1.5063, recon: -1.6472, kl: 0.3554, train_time_taken: 8.4534
INFO:tensorflow:step: 31100, lr: 0.000054, klw: 0.3965, cost: -1.4065, recon: -1.5457, kl: 0.3510, train_time_taken: 7.6277
INFO:tensorflow:step: 31120, lr: 0.000054, klw: 0.3966, cost: -1.3489, recon: -1.4842, kl: 0.3412, train_time_taken: 7.5489
INFO:tensorflow:step: 31140, lr: 0.000054, klw: 0.3967, cost: -1.5368, recon: -1.6786, kl: 0.3575, train_time_taken: 7.4908
INFO:tensorflow:step: 31160, lr: 0.000054, klw: 0.3968, cost: -1.4863, recon: -1.6292, kl: 0.3601, train_time_taken: 7.5511
INFO:tensorflow:step: 31180, lr: 0.000054, klw: 0.3969, cost: -1.3856, recon: -1.5246, kl: 0.3501, train_time_taken: 7.5598
INFO:tensorflow:step: 31200, lr: 0.000054, klw: 0.3970, cost: -1.4892, recon: -1.6297, kl: 0.3537, train_time_taken: 7.7545
INFO:tensorflow:step: 31220, lr: 0.000054, klw: 0.3971, cost: -1.4468, recon: -1.5893, kl: 0.3589, train_time_taken: 7.6298
INFO:tensorflow:step: 31240, lr: 0.000054, klw: 0.3972, cost: -1.4663, recon: -1.6081, kl: 0.3568, train_time_taken: 7.5828
INFO:tensorflow:step: 31260, lr: 0.000053, klw: 0.3973, cost: -1.4496, recon: -1.5902, kl: 0.3540, train_time_taken: 7.5766
INFO:tensorflow:step: 31280, lr: 0.000053, klw: 0.3974, cost: -1.3611, recon: -1.4994, kl: 0.3478, train_time_taken: 8.4655
INFO:tensorflow:step: 31300, lr: 0.000053, klw: 0.3976, cost: -1.5423, recon: -1.6877, kl: 0.3659, train_time_taken: 7.9406
INFO:tensorflow:step: 31320, lr: 0.000053, klw: 0.3977, cost: -1.4953, recon: -1.6392, kl: 0.3620, train_time_taken: 7.5412
INFO:tensorflow:step: 31340, lr: 0.000053, klw: 0.3978, cost: -1.4048, recon: -1.5459, kl: 0.3548, train_time_taken: 7.6341
INFO:tensorflow:step: 31360, lr: 0.000053, klw: 0.3979, cost: -1.4276, recon: -1.5702, kl: 0.3584, train_time_taken: 7.6218
INFO:tensorflow:step: 31380, lr: 0.000053, klw: 0.3980, cost: -1.4047, recon: -1.5471, kl: 0.3577, train_time_taken: 7.6463
INFO:tensorflow:step: 31400, lr: 0.000053, klw: 0.3981, cost: -1.3681, recon: -1.5055, kl: 0.3451, train_time_taken: 7.5347
INFO:tensorflow:step: 31420, lr: 0.000053, klw: 0.3982, cost: -1.5102, recon: -1.6531, kl: 0.3589, train_time_taken: 7.6161
INFO:tensorflow:step: 31440, lr: 0.000053, klw: 0.3983, cost: -1.5081, recon: -1.6517, kl: 0.3606, train_time_taken: 7.5358
INFO:tensorflow:step: 31460, lr: 0.000053, klw: 0.3984, cost: -1.4464, recon: -1.5922, kl: 0.3660, train_time_taken: 7.5421
INFO:tensorflow:step: 31480, lr: 0.000053, klw: 0.3985, cost: -1.3657, recon: -1.5060, kl: 0.3522, train_time_taken: 7.9978
INFO:tensorflow:step: 31500, lr: 0.000052, klw: 0.3986, cost: -1.4610, recon: -1.6045, kl: 0.3599, train_time_taken: 8.3463
INFO:tensorflow:best_valid_cost: -1.6975, valid_cost: -1.6975, valid_recon: -1.7010, valid_kl: 0.3553, valid_time_taken: 0.6377
INFO:tensorflow:saving model /content/sketch_rnn/ckpt/vector.
INFO:tensorflow:global_step 31500.
INFO:tensorflow:time_taken_save 0.7747.
INFO:tensorflow:eval_cost: -1.6573, eval_recon: -1.6608, eval_kl: 0.3477, eval_time_taken: 0.5306
INFO:tensorflow:step: 31520, lr: 0.000052, klw: 0.3987, cost: -1.4871, recon: -1.6286, kl: 0.3549, train_time_taken: 7.6964
INFO:tensorflow:step: 31540, lr: 0.000052, klw: 0.3988, cost: -1.3636, recon: -1.5012, kl: 0.3451, train_time_taken: 7.4753
INFO:tensorflow:step: 31560, lr: 0.000052, klw: 0.3989, cost: -1.4228, recon: -1.5649, kl: 0.3564, train_time_taken: 8.3573
INFO:tensorflow:step: 31580, lr: 0.000052, klw: 0.3990, cost: -1.4720, recon: -1.6143, kl: 0.3566, train_time_taken: 7.6481
INFO:tensorflow:step: 31600, lr: 0.000052, klw: 0.3991, cost: -1.5488, recon: -1.6934, kl: 0.3624, train_time_taken: 7.4273
INFO:tensorflow:step: 31620, lr: 0.000052, klw: 0.3992, cost: -1.3333, recon: -1.4696, kl: 0.3415, train_time_taken: 7.6244
INFO:tensorflow:step: 31640, lr: 0.000052, klw: 0.3993, cost: -1.3936, recon: -1.5341, kl: 0.3520, train_time_taken: 7.7340
Traceback (most recent call last):
  File "/content/sketch_rnn/sketch_rnn_train.py", line 478, in <module>
    console_entry_point()
  File "/content/sketch_rnn/sketch_rnn_train.py", line 474, in console_entry_point
    tf.app.run(main)
  File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/platform/app.py", line 125, in run
    _sys.exit(main(argv))
  File "/content/sketch_rnn/sketch_rnn_train.py", line 470, in main
    trainer(model_params)
  File "/content/sketch_rnn/sketch_rnn_train.py", line 462, in trainer
    train(sess, model, eval_model, train_set, valid_set, test_set)
  File "/content/sketch_rnn/sketch_rnn_train.py", line 301, in train
    ], feed)
  File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py", line 929, in run
    run_metadata_ptr)
  File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py", line 1152, in _run
    feed_dict_tensor, options, run_metadata)
  File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py", line 1328, in _do_run
    run_metadata)
  File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py", line 1334, in _do_call
    return fn(*args)
  File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py", line 1319, in _run_fn
    options, feed_dict, fetch_list, target_list, run_metadata)
  File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py", line 1407, in _call_tf_sessionrun
    run_metadata)
KeyboardInterrupt

Second try: Train Sketch-rnn model using LSTM with Layer Normalization (layer_norm for both enc_model and dec_model) as suggested for small dataset.
Also, use random scale factor and augment stroke prob in data augmentation.
Set z_sise = 128, use recurrent dropout.

! python /content/sketch_rnn/sketch_rnn_train.py --log_root="/content/sketch_rnn/ckpt_layer_norm/" --data_dir="/content/sketch-rnn-datasets/kanji/" --hparams="data_set=[kanji.rdp25.npz],save_every=2000,dec_model=layer_norm,enc_model=layer_norm"

WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.
For more information, please see:
  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md
  * https://github.com/tensorflow/addons
If you depend on functionality not listed there, please file an issue.

INFO:tensorflow:sketch-rnn
INFO:tensorflow:Hyperparams:
INFO:tensorflow:data_set = ['kanji.rdp25.npz']
INFO:tensorflow:num_steps = 10000000
INFO:tensorflow:save_every = 2000
INFO:tensorflow:max_seq_len = 250
INFO:tensorflow:dec_rnn_size = 512
INFO:tensorflow:dec_model = layer_norm
INFO:tensorflow:enc_rnn_size = 256
INFO:tensorflow:enc_model = layer_norm
INFO:tensorflow:z_size = 128
INFO:tensorflow:kl_weight = 0.5
INFO:tensorflow:kl_weight_start = 0.01
INFO:tensorflow:kl_tolerance = 0.2
INFO:tensorflow:batch_size = 100
INFO:tensorflow:grad_clip = 1.0
INFO:tensorflow:num_mixture = 20
INFO:tensorflow:learning_rate = 0.001
INFO:tensorflow:decay_rate = 0.9999
INFO:tensorflow:kl_decay_rate = 0.99995
INFO:tensorflow:min_learning_rate = 1e-05
INFO:tensorflow:use_recurrent_dropout = True
INFO:tensorflow:recurrent_dropout_prob = 0.9
INFO:tensorflow:use_input_dropout = False
INFO:tensorflow:input_dropout_prob = 0.9
INFO:tensorflow:use_output_dropout = False
INFO:tensorflow:output_dropout_prob = 0.9
INFO:tensorflow:random_scale_factor = 0.15
INFO:tensorflow:augment_stroke_prob = 0.1
INFO:tensorflow:conditional = True
INFO:tensorflow:is_training = True
INFO:tensorflow:Loading data files.
INFO:tensorflow:Loaded 10358/600/500 from kanji.rdp25.npz
INFO:tensorflow:Dataset combined: 11458 (10358/600/500), avg len 63
INFO:tensorflow:model_params.max_seq_len 133.
total images <= max_seq_len is 10358
total images <= max_seq_len is 600
total images <= max_seq_len is 500
INFO:tensorflow:normalizing_scale_factor 14.4871.
INFO:tensorflow:Model using gpu.
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.
Instructions for updating:
Colocations handled automatically by placer.
INFO:tensorflow:Input dropout mode = False.
INFO:tensorflow:Output dropout mode = False.
INFO:tensorflow:Recurrent dropout mode = True.
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/magenta/models/sketch_rnn/model.py:100: bidirectional_dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.
Instructions for updating:
Please use `keras.layers.Bidirectional(keras.layers.RNN(cell))`, which is equivalent to this API
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/rnn.py:443: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.
Instructions for updating:
Please use `keras.layers.RNN(cell)`, which is equivalent to this API
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/rnn.py:626: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.cast instead.
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/magenta/models/sketch_rnn/rnn.py:301: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.
Instructions for updating:
Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/magenta/models/sketch_rnn/model.py:266: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Deprecated in favor of operator or tf.math.divide.
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/magenta/models/sketch_rnn/model.py:285: calling reduce_sum_v1 (from tensorflow.python.ops.math_ops) with keep_dims is deprecated and will be removed in a future version.
Instructions for updating:
keep_dims is deprecated, use keepdims instead
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/magenta/models/sketch_rnn/model.py:295: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.
Instructions for updating:

Future major versions of TensorFlow will allow gradients to flow
into the labels input on backprop by default.

See `tf.nn.softmax_cross_entropy_with_logits_v2`.

INFO:tensorflow:Model using gpu.
INFO:tensorflow:Input dropout mode = 0.
INFO:tensorflow:Output dropout mode = 0.
INFO:tensorflow:Recurrent dropout mode = 0.
2019-05-02 22:43:31.284769: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2200000000 Hz
2019-05-02 22:43:31.285084: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x159ec00 executing computations on platform Host. Devices:
2019-05-02 22:43:31.285129: I tensorflow/compiler/xla/service/service.cc:158]   StreamExecutor device (0): <undefined>, <undefined>
2019-05-02 22:43:31.527201: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:998] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-05-02 22:43:31.527761: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x9033de0 executing computations on platform CUDA. Devices:
2019-05-02 22:43:31.527792: I tensorflow/compiler/xla/service/service.cc:158]   StreamExecutor device (0): Tesla T4, Compute Capability 7.5
2019-05-02 22:43:31.528189: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1433] Found device 0 with properties: 
name: Tesla T4 major: 7 minor: 5 memoryClockRate(GHz): 1.59
pciBusID: 0000:00:04.0
totalMemory: 14.73GiB freeMemory: 14.60GiB
2019-05-02 22:43:31.528213: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1512] Adding visible gpu devices: 0
2019-05-02 22:43:32.864571: I tensorflow/core/common_runtime/gpu/gpu_device.cc:984] Device interconnect StreamExecutor with strength 1 edge matrix:
2019-05-02 22:43:32.864632: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990]      0 
2019-05-02 22:43:32.864645: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1003] 0:   N 
2019-05-02 22:43:32.864955: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 14115 MB memory) -> physical GPU (device: 0, name: Tesla T4, pci bus id: 0000:00:04.0, compute capability: 7.5)
INFO:tensorflow:vector_rnn/ENC_RNN/fw/LayerNormLSTMCell/W_xh:0 (5, 1024) 5120
INFO:tensorflow:vector_rnn/ENC_RNN/fw/LayerNormLSTMCell/W_hh:0 (256, 1024) 262144
INFO:tensorflow:vector_rnn/ENC_RNN/fw/LayerNormLSTMCell/ln_all/ln_gamma:0 (1024,) 1024
INFO:tensorflow:vector_rnn/ENC_RNN/fw/LayerNormLSTMCell/ln_all/ln_beta:0 (1024,) 1024
INFO:tensorflow:vector_rnn/ENC_RNN/fw/LayerNormLSTMCell/ln_c/ln_gamma:0 (256,) 256
INFO:tensorflow:vector_rnn/ENC_RNN/fw/LayerNormLSTMCell/ln_c/ln_beta:0 (256,) 256
INFO:tensorflow:vector_rnn/ENC_RNN/bw/LayerNormLSTMCell/W_xh:0 (5, 1024) 5120
INFO:tensorflow:vector_rnn/ENC_RNN/bw/LayerNormLSTMCell/W_hh:0 (256, 1024) 262144
INFO:tensorflow:vector_rnn/ENC_RNN/bw/LayerNormLSTMCell/ln_all/ln_gamma:0 (1024,) 1024
INFO:tensorflow:vector_rnn/ENC_RNN/bw/LayerNormLSTMCell/ln_all/ln_beta:0 (1024,) 1024
INFO:tensorflow:vector_rnn/ENC_RNN/bw/LayerNormLSTMCell/ln_c/ln_gamma:0 (256,) 256
INFO:tensorflow:vector_rnn/ENC_RNN/bw/LayerNormLSTMCell/ln_c/ln_beta:0 (256,) 256
INFO:tensorflow:vector_rnn/ENC_RNN_mu/super_linear_w:0 (512, 128) 65536
INFO:tensorflow:vector_rnn/ENC_RNN_mu/super_linear_b:0 (128,) 128
INFO:tensorflow:vector_rnn/ENC_RNN_sigma/super_linear_w:0 (512, 128) 65536
INFO:tensorflow:vector_rnn/ENC_RNN_sigma/super_linear_b:0 (128,) 128
INFO:tensorflow:vector_rnn/linear/super_linear_w:0 (128, 1024) 131072
INFO:tensorflow:vector_rnn/linear/super_linear_b:0 (1024,) 1024
INFO:tensorflow:vector_rnn/RNN/output_w:0 (512, 123) 62976
INFO:tensorflow:vector_rnn/RNN/output_b:0 (123,) 123
INFO:tensorflow:vector_rnn/RNN/LayerNormLSTMCell/W_xh:0 (133, 2048) 272384
INFO:tensorflow:vector_rnn/RNN/LayerNormLSTMCell/W_hh:0 (512, 2048) 1048576
INFO:tensorflow:vector_rnn/RNN/LayerNormLSTMCell/ln_all/ln_gamma:0 (2048,) 2048
INFO:tensorflow:vector_rnn/RNN/LayerNormLSTMCell/ln_all/ln_beta:0 (2048,) 2048
INFO:tensorflow:vector_rnn/RNN/LayerNormLSTMCell/ln_c/ln_gamma:0 (512,) 512
INFO:tensorflow:vector_rnn/RNN/LayerNormLSTMCell/ln_c/ln_beta:0 (512,) 512
INFO:tensorflow:Total trainable variables 2192251.
2019-05-02 22:43:33.415158: I tensorflow/stream_executor/dso_loader.cc:152] successfully opened CUDA library libcublas.so.10.0 locally
INFO:tensorflow:step: 20, lr: 0.000998, klw: 0.0105, cost: 1.4046, recon: 1.4013, kl: 0.3231, train_time_taken: 17.6171
INFO:tensorflow:step: 40, lr: 0.000996, klw: 0.0110, cost: 1.0554, recon: 1.0492, kl: 0.5647, train_time_taken: 15.9336
INFO:tensorflow:step: 60, lr: 0.000994, klw: 0.0115, cost: 0.9355, recon: 0.9264, kl: 0.7953, train_time_taken: 15.4139
INFO:tensorflow:step: 80, lr: 0.000992, klw: 0.0120, cost: 0.9139, recon: 0.9027, kl: 0.9318, train_time_taken: 15.1181
INFO:tensorflow:step: 100, lr: 0.000990, klw: 0.0124, cost: 1.2165, recon: 1.2051, kl: 0.9161, train_time_taken: 15.2678
INFO:tensorflow:step: 120, lr: 0.000988, klw: 0.0129, cost: 0.8681, recon: 0.8590, kl: 0.7036, train_time_taken: 15.1011
INFO:tensorflow:step: 140, lr: 0.000986, klw: 0.0134, cost: 0.7899, recon: 0.7765, kl: 0.9962, train_time_taken: 16.8030
INFO:tensorflow:step: 160, lr: 0.000984, klw: 0.0139, cost: 0.6128, recon: 0.5945, kl: 1.3133, train_time_taken: 15.9702
INFO:tensorflow:step: 180, lr: 0.000982, klw: 0.0144, cost: 0.5558, recon: 0.5322, kl: 1.6398, train_time_taken: 15.2527
INFO:tensorflow:step: 200, lr: 0.000980, klw: 0.0149, cost: 0.3976, recon: 0.3714, kl: 1.7583, train_time_taken: 15.3530
INFO:tensorflow:step: 220, lr: 0.000978, klw: 0.0154, cost: 0.3570, recon: 0.3303, kl: 1.7402, train_time_taken: 15.1046
INFO:tensorflow:step: 240, lr: 0.000977, klw: 0.0158, cost: 0.2893, recon: 0.2611, kl: 1.7788, train_time_taken: 16.2959
INFO:tensorflow:step: 260, lr: 0.000975, klw: 0.0163, cost: 0.2228, recon: 0.1942, kl: 1.7514, train_time_taken: 14.9040
INFO:tensorflow:step: 280, lr: 0.000973, klw: 0.0168, cost: 0.1731, recon: 0.1432, kl: 1.7764, train_time_taken: 15.1372
INFO:tensorflow:step: 300, lr: 0.000971, klw: 0.0173, cost: 0.1162, recon: 0.0849, kl: 1.8093, train_time_taken: 15.0004
INFO:tensorflow:step: 320, lr: 0.000969, klw: 0.0178, cost: 0.0911, recon: 0.0589, kl: 1.8100, train_time_taken: 15.0538
INFO:tensorflow:step: 340, lr: 0.000967, klw: 0.0183, cost: 0.0245, recon: -0.0090, kl: 1.8318, train_time_taken: 15.8057
INFO:tensorflow:step: 360, lr: 0.000965, klw: 0.0187, cost: -0.0070, recon: -0.0419, kl: 1.8628, train_time_taken: 15.1196
INFO:tensorflow:step: 380, lr: 0.000963, klw: 0.0192, cost: -0.0065, recon: -0.0432, kl: 1.9068, train_time_taken: 15.4855
INFO:tensorflow:step: 400, lr: 0.000961, klw: 0.0197, cost: -0.0343, recon: -0.0710, kl: 1.8649, train_time_taken: 14.5534
INFO:tensorflow:step: 420, lr: 0.000959, klw: 0.0202, cost: -0.0782, recon: -0.1157, kl: 1.8603, train_time_taken: 15.0564
INFO:tensorflow:step: 440, lr: 0.000957, klw: 0.0207, cost: -0.0718, recon: -0.1105, kl: 1.8724, train_time_taken: 15.0789
INFO:tensorflow:step: 460, lr: 0.000955, klw: 0.0211, cost: -0.1279, recon: -0.1677, kl: 1.8827, train_time_taken: 15.7082
INFO:tensorflow:step: 480, lr: 0.000954, klw: 0.0216, cost: -0.1692, recon: -0.2090, kl: 1.8413, train_time_taken: 14.6951
INFO:tensorflow:step: 500, lr: 0.000952, klw: 0.0221, cost: -0.1535, recon: -0.1956, kl: 1.9020, train_time_taken: 14.8963
INFO:tensorflow:step: 520, lr: 0.000950, klw: 0.0226, cost: -0.2239, recon: -0.2679, kl: 1.9488, train_time_taken: 15.1816
INFO:tensorflow:step: 540, lr: 0.000948, klw: 0.0231, cost: -0.1838, recon: -0.2266, kl: 1.8547, train_time_taken: 15.1099
INFO:tensorflow:step: 560, lr: 0.000946, klw: 0.0235, cost: -0.2210, recon: -0.2642, kl: 1.8364, train_time_taken: 16.7787
INFO:tensorflow:step: 580, lr: 0.000944, klw: 0.0240, cost: -0.2674, recon: -0.3141, kl: 1.9479, train_time_taken: 14.7203
INFO:tensorflow:step: 600, lr: 0.000942, klw: 0.0245, cost: -0.2454, recon: -0.2914, kl: 1.8786, train_time_taken: 14.6288
INFO:tensorflow:step: 620, lr: 0.000940, klw: 0.0250, cost: -0.2607, recon: -0.3079, kl: 1.8903, train_time_taken: 14.9070
INFO:tensorflow:step: 640, lr: 0.000939, klw: 0.0254, cost: -0.3167, recon: -0.3626, kl: 1.8041, train_time_taken: 14.8509
INFO:tensorflow:step: 660, lr: 0.000937, klw: 0.0259, cost: -0.3400, recon: -0.3891, kl: 1.8958, train_time_taken: 15.8519
INFO:tensorflow:step: 680, lr: 0.000935, klw: 0.0264, cost: -0.3251, recon: -0.3734, kl: 1.8320, train_time_taken: 15.0270
INFO:tensorflow:step: 700, lr: 0.000933, klw: 0.0269, cost: -0.3881, recon: -0.4385, kl: 1.8788, train_time_taken: 14.9498
INFO:tensorflow:step: 720, lr: 0.000931, klw: 0.0273, cost: -0.3628, recon: -0.4130, kl: 1.8368, train_time_taken: 15.2158
INFO:tensorflow:step: 740, lr: 0.000929, klw: 0.0278, cost: -0.3674, recon: -0.4190, kl: 1.8548, train_time_taken: 14.9825
INFO:tensorflow:step: 760, lr: 0.000928, klw: 0.0283, cost: -0.3790, recon: -0.4305, kl: 1.8226, train_time_taken: 15.3180
INFO:tensorflow:step: 780, lr: 0.000926, klw: 0.0287, cost: -0.3914, recon: -0.4436, kl: 1.8177, train_time_taken: 16.7342
INFO:tensorflow:step: 800, lr: 0.000924, klw: 0.0292, cost: -0.4307, recon: -0.4857, kl: 1.8831, train_time_taken: 14.9019
INFO:tensorflow:step: 820, lr: 0.000922, klw: 0.0297, cost: -0.4410, recon: -0.4939, kl: 1.7809, train_time_taken: 14.9627
INFO:tensorflow:step: 840, lr: 0.000920, klw: 0.0302, cost: -0.4208, recon: -0.4720, kl: 1.6980, train_time_taken: 14.9604
INFO:tensorflow:step: 860, lr: 0.000918, klw: 0.0306, cost: -0.3823, recon: -0.4359, kl: 1.7498, train_time_taken: 14.9374
INFO:tensorflow:step: 880, lr: 0.000917, klw: 0.0311, cost: -0.4782, recon: -0.5345, kl: 1.8111, train_time_taken: 16.3284
INFO:tensorflow:step: 900, lr: 0.000915, klw: 0.0316, cost: -0.4805, recon: -0.5386, kl: 1.8415, train_time_taken: 14.8411
INFO:tensorflow:step: 920, lr: 0.000913, klw: 0.0320, cost: -0.4924, recon: -0.5476, kl: 1.7231, train_time_taken: 14.8338
INFO:tensorflow:step: 940, lr: 0.000911, klw: 0.0325, cost: -0.4445, recon: -0.5018, kl: 1.7656, train_time_taken: 14.8003
INFO:tensorflow:step: 960, lr: 0.000909, klw: 0.0330, cost: -0.4869, recon: -0.5439, kl: 1.7294, train_time_taken: 15.7112
INFO:tensorflow:step: 980, lr: 0.000908, klw: 0.0334, cost: -0.4613, recon: -0.5174, kl: 1.6765, train_time_taken: 16.6289
INFO:tensorflow:step: 1000, lr: 0.000906, klw: 0.0339, cost: -0.5102, recon: -0.5693, kl: 1.7433, train_time_taken: 15.0021
INFO:tensorflow:step: 1020, lr: 0.000904, klw: 0.0344, cost: -0.5074, recon: -0.5670, kl: 1.7346, train_time_taken: 14.6809
INFO:tensorflow:step: 1040, lr: 0.000902, klw: 0.0348, cost: -0.5065, recon: -0.5653, kl: 1.6884, train_time_taken: 14.6423
INFO:tensorflow:step: 1060, lr: 0.000900, klw: 0.0353, cost: -0.5135, recon: -0.5739, kl: 1.7113, train_time_taken: 14.8366
INFO:tensorflow:step: 1080, lr: 0.000899, klw: 0.0358, cost: -0.5600, recon: -0.6229, kl: 1.7594, train_time_taken: 16.0198
INFO:tensorflow:step: 1100, lr: 0.000897, klw: 0.0362, cost: -0.5831, recon: -0.6461, kl: 1.7383, train_time_taken: 15.1474
INFO:tensorflow:step: 1120, lr: 0.000895, klw: 0.0367, cost: -0.5325, recon: -0.5928, kl: 1.6431, train_time_taken: 14.9056
INFO:tensorflow:step: 1140, lr: 0.000893, klw: 0.0371, cost: -0.5485, recon: -0.6097, kl: 1.6477, train_time_taken: 14.9514
INFO:tensorflow:step: 1160, lr: 0.000892, klw: 0.0376, cost: -0.6065, recon: -0.6708, kl: 1.7096, train_time_taken: 15.0427
INFO:tensorflow:step: 1180, lr: 0.000890, klw: 0.0381, cost: -0.5671, recon: -0.6297, kl: 1.6421, train_time_taken: 15.3184
INFO:tensorflow:step: 1200, lr: 0.000888, klw: 0.0385, cost: -0.5911, recon: -0.6541, kl: 1.6335, train_time_taken: 16.0535
INFO:tensorflow:step: 1220, lr: 0.000886, klw: 0.0390, cost: -0.6904, recon: -0.7553, kl: 1.6638, train_time_taken: 14.7239
INFO:tensorflow:step: 1240, lr: 0.000885, klw: 0.0395, cost: -0.6804, recon: -0.7451, kl: 1.6389, train_time_taken: 14.8303
INFO:tensorflow:step: 1260, lr: 0.000883, klw: 0.0399, cost: -0.6286, recon: -0.6964, kl: 1.6989, train_time_taken: 15.0468
INFO:tensorflow:step: 1280, lr: 0.000881, klw: 0.0404, cost: -0.6924, recon: -0.7598, kl: 1.6676, train_time_taken: 14.9018
INFO:tensorflow:step: 1300, lr: 0.000879, klw: 0.0408, cost: -0.6539, recon: -0.7227, kl: 1.6849, train_time_taken: 16.2326
INFO:tensorflow:step: 1320, lr: 0.000878, klw: 0.0413, cost: -0.6486, recon: -0.7163, kl: 1.6403, train_time_taken: 14.8763
INFO:tensorflow:step: 1340, lr: 0.000876, klw: 0.0418, cost: -0.7556, recon: -0.8244, kl: 1.6488, train_time_taken: 14.8361
INFO:tensorflow:step: 1360, lr: 0.000874, klw: 0.0422, cost: -0.6358, recon: -0.7039, kl: 1.6151, train_time_taken: 14.9185
INFO:tensorflow:step: 1380, lr: 0.000872, klw: 0.0427, cost: -0.6550, recon: -0.7231, kl: 1.5976, train_time_taken: 16.1764
INFO:tensorflow:step: 1400, lr: 0.000871, klw: 0.0431, cost: -0.6968, recon: -0.7639, kl: 1.5555, train_time_taken: 16.1401
INFO:tensorflow:step: 1420, lr: 0.000869, klw: 0.0436, cost: -0.7351, recon: -0.8064, kl: 1.6345, train_time_taken: 14.7690
INFO:tensorflow:step: 1440, lr: 0.000867, klw: 0.0440, cost: -0.7150, recon: -0.7876, kl: 1.6483, train_time_taken: 14.9747
INFO:tensorflow:step: 1460, lr: 0.000866, klw: 0.0445, cost: -0.7316, recon: -0.8035, kl: 1.6155, train_time_taken: 15.0557
INFO:tensorflow:step: 1480, lr: 0.000864, klw: 0.0450, cost: -0.7072, recon: -0.7781, kl: 1.5770, train_time_taken: 14.7567
INFO:tensorflow:step: 1500, lr: 0.000862, klw: 0.0454, cost: -0.7122, recon: -0.7847, kl: 1.5961, train_time_taken: 15.6970
INFO:tensorflow:step: 1520, lr: 0.000860, klw: 0.0459, cost: -0.6951, recon: -0.7662, kl: 1.5491, train_time_taken: 15.6665
INFO:tensorflow:step: 1540, lr: 0.000859, klw: 0.0463, cost: -0.6942, recon: -0.7661, kl: 1.5530, train_time_taken: 14.8642
INFO:tensorflow:step: 1560, lr: 0.000857, klw: 0.0468, cost: -0.7252, recon: -0.7993, kl: 1.5839, train_time_taken: 14.5646
INFO:tensorflow:step: 1580, lr: 0.000855, klw: 0.0472, cost: -0.7033, recon: -0.7767, kl: 1.5546, train_time_taken: 15.4385
INFO:tensorflow:step: 1600, lr: 0.000854, klw: 0.0477, cost: -0.7883, recon: -0.8613, kl: 1.5309, train_time_taken: 14.9096
INFO:tensorflow:step: 1620, lr: 0.000852, klw: 0.0481, cost: -0.7707, recon: -0.8433, kl: 1.5085, train_time_taken: 16.3302
INFO:tensorflow:step: 1640, lr: 0.000850, klw: 0.0486, cost: -0.7607, recon: -0.8366, kl: 1.5631, train_time_taken: 14.6985
INFO:tensorflow:step: 1660, lr: 0.000849, klw: 0.0490, cost: -0.8194, recon: -0.8959, kl: 1.5610, train_time_taken: 15.0007
INFO:tensorflow:step: 1680, lr: 0.000847, klw: 0.0495, cost: -0.7657, recon: -0.8404, kl: 1.5102, train_time_taken: 14.8600
INFO:tensorflow:step: 1700, lr: 0.000845, klw: 0.0499, cost: -0.7579, recon: -0.8336, kl: 1.5175, train_time_taken: 14.7284
INFO:tensorflow:step: 1720, lr: 0.000844, klw: 0.0504, cost: -0.7634, recon: -0.8405, kl: 1.5307, train_time_taken: 16.1615
INFO:tensorflow:step: 1740, lr: 0.000842, klw: 0.0508, cost: -0.8541, recon: -0.9316, kl: 1.5252, train_time_taken: 14.9664
INFO:tensorflow:step: 1760, lr: 0.000840, klw: 0.0513, cost: -0.7882, recon: -0.8660, kl: 1.5179, train_time_taken: 14.7544
INFO:tensorflow:step: 1780, lr: 0.000839, klw: 0.0517, cost: -0.8548, recon: -0.9320, kl: 1.4937, train_time_taken: 16.5383
INFO:tensorflow:step: 1800, lr: 0.000837, klw: 0.0522, cost: -0.7933, recon: -0.8710, kl: 1.4894, train_time_taken: 14.7018
INFO:tensorflow:step: 1820, lr: 0.000835, klw: 0.0526, cost: -0.8157, recon: -0.8939, kl: 1.4869, train_time_taken: 16.0399
INFO:tensorflow:step: 1840, lr: 0.000834, klw: 0.0531, cost: -0.8660, recon: -0.9452, kl: 1.4924, train_time_taken: 15.1404
INFO:tensorflow:step: 1860, lr: 0.000832, klw: 0.0535, cost: -0.7760, recon: -0.8534, kl: 1.4469, train_time_taken: 15.0160
INFO:tensorflow:step: 1880, lr: 0.000830, klw: 0.0540, cost: -0.8044, recon: -0.8792, kl: 1.3864, train_time_taken: 14.8450
INFO:tensorflow:step: 1900, lr: 0.000829, klw: 0.0544, cost: -0.8352, recon: -0.9132, kl: 1.4334, train_time_taken: 14.9659
INFO:tensorflow:step: 1920, lr: 0.000827, klw: 0.0549, cost: -0.7674, recon: -0.8450, kl: 1.4143, train_time_taken: 15.4301
INFO:tensorflow:step: 1940, lr: 0.000825, klw: 0.0553, cost: -0.7933, recon: -0.8712, kl: 1.4092, train_time_taken: 15.3380
INFO:tensorflow:step: 1960, lr: 0.000824, klw: 0.0557, cost: -0.8766, recon: -0.9580, kl: 1.4612, train_time_taken: 14.4416
INFO:tensorflow:step: 1980, lr: 0.000822, klw: 0.0562, cost: -0.9204, recon: -1.0025, kl: 1.4612, train_time_taken: 15.2660
INFO:tensorflow:step: 2000, lr: 0.000821, klw: 0.0566, cost: -0.7862, recon: -0.8656, kl: 1.4025, train_time_taken: 14.9001
INFO:tensorflow:best_valid_cost: -1.0968, valid_cost: -1.0968, valid_recon: -1.1112, valid_kl: 1.4380, valid_time_taken: 0.9547
INFO:tensorflow:saving model /content/sketch_rnn/ckpt_layer_norm/vector.
INFO:tensorflow:global_step 2000.
INFO:tensorflow:time_taken_save 0.5137.
INFO:tensorflow:eval_cost: -1.0795, eval_recon: -1.0938, eval_kl: 1.4311, eval_time_taken: 0.7246
INFO:tensorflow:step: 2020, lr: 0.000819, klw: 0.0571, cost: -0.8228, recon: -0.9005, kl: 1.3611, train_time_taken: 14.6554
INFO:tensorflow:step: 2040, lr: 0.000817, klw: 0.0575, cost: -0.8733, recon: -0.9522, kl: 1.3713, train_time_taken: 16.1433
INFO:tensorflow:step: 2060, lr: 0.000816, klw: 0.0580, cost: -0.8651, recon: -0.9447, kl: 1.3724, train_time_taken: 14.8922
INFO:tensorflow:step: 2080, lr: 0.000814, klw: 0.0584, cost: -0.8592, recon: -0.9386, kl: 1.3582, train_time_taken: 15.0410
INFO:tensorflow:step: 2100, lr: 0.000812, klw: 0.0588, cost: -0.8502, recon: -0.9325, kl: 1.3984, train_time_taken: 14.6848
INFO:tensorflow:step: 2120, lr: 0.000811, klw: 0.0593, cost: -0.7935, recon: -0.8720, kl: 1.3229, train_time_taken: 14.9746
INFO:tensorflow:step: 2140, lr: 0.000809, klw: 0.0597, cost: -0.8987, recon: -0.9824, kl: 1.4009, train_time_taken: 16.2138
INFO:tensorflow:step: 2160, lr: 0.000808, klw: 0.0602, cost: -0.9182, recon: -1.0015, kl: 1.3854, train_time_taken: 14.7423
INFO:tensorflow:step: 2180, lr: 0.000806, klw: 0.0606, cost: -0.8806, recon: -0.9626, kl: 1.3539, train_time_taken: 15.7374
INFO:tensorflow:step: 2200, lr: 0.000804, klw: 0.0610, cost: -0.9337, recon: -1.0151, kl: 1.3328, train_time_taken: 15.5706
INFO:tensorflow:step: 2220, lr: 0.000803, klw: 0.0615, cost: -0.8478, recon: -0.9266, kl: 1.2832, train_time_taken: 14.8208
INFO:tensorflow:step: 2240, lr: 0.000801, klw: 0.0619, cost: -0.8357, recon: -0.9159, kl: 1.2958, train_time_taken: 16.0319
INFO:tensorflow:step: 2260, lr: 0.000800, klw: 0.0624, cost: -0.9675, recon: -1.0517, kl: 1.3498, train_time_taken: 15.0804
INFO:tensorflow:step: 2280, lr: 0.000798, klw: 0.0628, cost: -0.8675, recon: -0.9475, kl: 1.2729, train_time_taken: 14.8397
INFO:tensorflow:step: 2300, lr: 0.000797, klw: 0.0632, cost: -0.8763, recon: -0.9578, kl: 1.2891, train_time_taken: 14.8150
INFO:tensorflow:step: 2320, lr: 0.000795, klw: 0.0637, cost: -0.8913, recon: -0.9734, kl: 1.2888, train_time_taken: 14.8443
INFO:tensorflow:step: 2340, lr: 0.000793, klw: 0.0641, cost: -0.8784, recon: -0.9600, kl: 1.2724, train_time_taken: 14.9253
INFO:tensorflow:step: 2360, lr: 0.000792, klw: 0.0645, cost: -0.8996, recon: -0.9800, kl: 1.2451, train_time_taken: 15.6768
INFO:tensorflow:step: 2380, lr: 0.000790, klw: 0.0650, cost: -0.9326, recon: -1.0155, kl: 1.2749, train_time_taken: 15.6019
INFO:tensorflow:step: 2400, lr: 0.000789, klw: 0.0654, cost: -0.9782, recon: -1.0646, kl: 1.3215, train_time_taken: 14.9224
INFO:tensorflow:step: 2420, lr: 0.000787, klw: 0.0658, cost: -0.9132, recon: -0.9939, kl: 1.2249, train_time_taken: 14.7062
INFO:tensorflow:step: 2440, lr: 0.000786, klw: 0.0663, cost: -0.8733, recon: -0.9544, kl: 1.2237, train_time_taken: 14.6682
INFO:tensorflow:step: 2460, lr: 0.000784, klw: 0.0667, cost: -0.9018, recon: -0.9850, kl: 1.2470, train_time_taken: 16.3243
INFO:tensorflow:step: 2480, lr: 0.000783, klw: 0.0671, cost: -0.9781, recon: -1.0647, kl: 1.2891, train_time_taken: 14.8171
INFO:tensorflow:step: 2500, lr: 0.000781, klw: 0.0676, cost: -0.8786, recon: -0.9614, kl: 1.2260, train_time_taken: 14.5667
INFO:tensorflow:step: 2520, lr: 0.000779, klw: 0.0680, cost: -0.9006, recon: -0.9840, kl: 1.2271, train_time_taken: 15.0070
INFO:tensorflow:step: 2540, lr: 0.000778, klw: 0.0684, cost: -0.9321, recon: -1.0146, kl: 1.2050, train_time_taken: 14.7281
INFO:tensorflow:step: 2560, lr: 0.000776, klw: 0.0689, cost: -0.9276, recon: -1.0107, kl: 1.2055, train_time_taken: 15.8430
INFO:tensorflow:step: 2580, lr: 0.000775, klw: 0.0693, cost: -0.9556, recon: -1.0405, kl: 1.2251, train_time_taken: 14.9549
INFO:tensorflow:step: 2600, lr: 0.000773, klw: 0.0697, cost: -0.8929, recon: -0.9740, kl: 1.1636, train_time_taken: 16.2214
INFO:tensorflow:step: 2620, lr: 0.000772, klw: 0.0702, cost: -1.0030, recon: -1.0874, kl: 1.2023, train_time_taken: 14.7460
INFO:tensorflow:step: 2640, lr: 0.000770, klw: 0.0706, cost: -1.0052, recon: -1.0892, kl: 1.1896, train_time_taken: 14.8718
INFO:tensorflow:step: 2660, lr: 0.000769, klw: 0.0710, cost: -1.0106, recon: -1.0954, kl: 1.1942, train_time_taken: 15.6537
INFO:tensorflow:step: 2680, lr: 0.000767, klw: 0.0715, cost: -0.9830, recon: -1.0695, kl: 1.2100, train_time_taken: 15.2342
INFO:tensorflow:step: 2700, lr: 0.000766, klw: 0.0719, cost: -0.9220, recon: -1.0047, kl: 1.1503, train_time_taken: 14.6001
INFO:tensorflow:step: 2720, lr: 0.000764, klw: 0.0723, cost: -1.0509, recon: -1.1385, kl: 1.2110, train_time_taken: 14.5680
INFO:tensorflow:step: 2740, lr: 0.000763, klw: 0.0727, cost: -0.9703, recon: -1.0558, kl: 1.1753, train_time_taken: 14.7536
INFO:tensorflow:step: 2760, lr: 0.000761, klw: 0.0732, cost: -1.0087, recon: -1.0924, kl: 1.1435, train_time_taken: 14.4480
INFO:tensorflow:step: 2780, lr: 0.000760, klw: 0.0736, cost: -0.9783, recon: -1.0626, kl: 1.1458, train_time_taken: 16.0120
INFO:tensorflow:step: 2800, lr: 0.000758, klw: 0.0740, cost: -0.9784, recon: -1.0628, kl: 1.1398, train_time_taken: 15.4700
INFO:tensorflow:step: 2820, lr: 0.000757, klw: 0.0744, cost: -1.0219, recon: -1.1068, kl: 1.1402, train_time_taken: 14.9992
INFO:tensorflow:step: 2840, lr: 0.000755, klw: 0.0749, cost: -0.8871, recon: -0.9661, kl: 1.0551, train_time_taken: 14.9298
INFO:tensorflow:step: 2860, lr: 0.000754, klw: 0.0753, cost: -0.9497, recon: -1.0327, kl: 1.1034, train_time_taken: 14.8501
INFO:tensorflow:step: 2880, lr: 0.000752, klw: 0.0757, cost: -0.9413, recon: -1.0240, kl: 1.0927, train_time_taken: 16.0360
INFO:tensorflow:step: 2900, lr: 0.000751, klw: 0.0761, cost: -1.0035, recon: -1.0874, kl: 1.1018, train_time_taken: 15.0162
INFO:tensorflow:step: 2920, lr: 0.000749, klw: 0.0766, cost: -0.9823, recon: -1.0665, kl: 1.0991, train_time_taken: 14.7171
INFO:tensorflow:step: 2940, lr: 0.000748, klw: 0.0770, cost: -0.9791, recon: -1.0623, kl: 1.0805, train_time_taken: 14.4523
INFO:tensorflow:step: 2960, lr: 0.000746, klw: 0.0774, cost: -1.0162, recon: -1.1002, kl: 1.0862, train_time_taken: 14.8905
INFO:tensorflow:step: 2980, lr: 0.000745, klw: 0.0778, cost: -1.0342, recon: -1.1188, kl: 1.0874, train_time_taken: 15.4544
INFO:tensorflow:step: 3000, lr: 0.000743, klw: 0.0783, cost: -0.9020, recon: -0.9836, kl: 1.0423, train_time_taken: 16.3504
INFO:tensorflow:step: 3020, lr: 0.000742, klw: 0.0787, cost: -0.9559, recon: -1.0418, kl: 1.0911, train_time_taken: 15.4961
INFO:tensorflow:step: 3040, lr: 0.000740, klw: 0.0791, cost: -1.0592, recon: -1.1459, kl: 1.0966, train_time_taken: 14.7605
INFO:tensorflow:step: 3060, lr: 0.000739, klw: 0.0795, cost: -1.0397, recon: -1.1256, kl: 1.0801, train_time_taken: 14.7320
INFO:tensorflow:step: 3080, lr: 0.000738, klw: 0.0799, cost: -1.0118, recon: -1.0958, kl: 1.0512, train_time_taken: 14.8132
INFO:tensorflow:step: 3100, lr: 0.000736, klw: 0.0804, cost: -1.0638, recon: -1.1496, kl: 1.0683, train_time_taken: 16.0043
INFO:tensorflow:step: 3120, lr: 0.000735, klw: 0.0808, cost: -0.9563, recon: -1.0408, kl: 1.0455, train_time_taken: 14.9234
INFO:tensorflow:step: 3140, lr: 0.000733, klw: 0.0812, cost: -0.9609, recon: -1.0434, kl: 1.0153, train_time_taken: 14.6212
INFO:tensorflow:step: 3160, lr: 0.000732, klw: 0.0816, cost: -1.0573, recon: -1.1460, kl: 1.0865, train_time_taken: 15.0109
INFO:tensorflow:step: 3180, lr: 0.000730, klw: 0.0820, cost: -1.0204, recon: -1.1035, kl: 1.0132, train_time_taken: 14.9073
INFO:tensorflow:step: 3200, lr: 0.000729, klw: 0.0825, cost: -1.0354, recon: -1.1213, kl: 1.0418, train_time_taken: 16.5910
INFO:tensorflow:step: 3220, lr: 0.000727, klw: 0.0829, cost: -0.9700, recon: -1.0539, kl: 1.0134, train_time_taken: 14.6866
INFO:tensorflow:step: 3240, lr: 0.000726, klw: 0.0833, cost: -1.0195, recon: -1.1022, kl: 0.9936, train_time_taken: 15.1430
INFO:tensorflow:step: 3260, lr: 0.000725, klw: 0.0837, cost: -0.9981, recon: -1.0827, kl: 1.0103, train_time_taken: 14.8233
INFO:tensorflow:step: 3280, lr: 0.000723, klw: 0.0841, cost: -0.9985, recon: -1.0783, kl: 0.9490, train_time_taken: 14.6877
INFO:tensorflow:step: 3300, lr: 0.000722, klw: 0.0845, cost: -1.0955, recon: -1.1837, kl: 1.0426, train_time_taken: 15.6132
INFO:tensorflow:step: 3320, lr: 0.000720, klw: 0.0849, cost: -1.1052, recon: -1.1918, kl: 1.0196, train_time_taken: 15.2526
INFO:tensorflow:step: 3340, lr: 0.000719, klw: 0.0854, cost: -1.0172, recon: -1.1002, kl: 0.9722, train_time_taken: 14.8566
INFO:tensorflow:step: 3360, lr: 0.000717, klw: 0.0858, cost: -1.0197, recon: -1.1053, kl: 0.9984, train_time_taken: 14.8347
INFO:tensorflow:step: 3380, lr: 0.000716, klw: 0.0862, cost: -1.0138, recon: -1.0970, kl: 0.9649, train_time_taken: 14.7737
INFO:tensorflow:step: 3400, lr: 0.000715, klw: 0.0866, cost: -1.0418, recon: -1.1270, kl: 0.9842, train_time_taken: 14.8974
INFO:tensorflow:step: 3420, lr: 0.000713, klw: 0.0870, cost: -1.0515, recon: -1.1359, kl: 0.9709, train_time_taken: 16.5343
INFO:tensorflow:step: 3440, lr: 0.000712, klw: 0.0874, cost: -1.0313, recon: -1.1139, kl: 0.9450, train_time_taken: 14.7438
INFO:tensorflow:step: 3460, lr: 0.000710, klw: 0.0878, cost: -1.0610, recon: -1.1449, kl: 0.9548, train_time_taken: 14.7726
INFO:tensorflow:step: 3480, lr: 0.000709, klw: 0.0883, cost: -1.1289, recon: -1.2155, kl: 0.9816, train_time_taken: 15.0472
INFO:tensorflow:step: 3500, lr: 0.000708, klw: 0.0887, cost: -1.1288, recon: -1.2161, kl: 0.9844, train_time_taken: 14.7256
INFO:tensorflow:step: 3520, lr: 0.000706, klw: 0.0891, cost: -1.1396, recon: -1.2270, kl: 0.9810, train_time_taken: 16.1653
INFO:tensorflow:step: 3540, lr: 0.000705, klw: 0.0895, cost: -1.0969, recon: -1.1815, kl: 0.9453, train_time_taken: 14.9267
INFO:tensorflow:step: 3560, lr: 0.000703, klw: 0.0899, cost: -1.0738, recon: -1.1572, kl: 0.9278, train_time_taken: 14.6710
INFO:tensorflow:step: 3580, lr: 0.000702, klw: 0.0903, cost: -1.0704, recon: -1.1518, kl: 0.9008, train_time_taken: 14.9284
INFO:tensorflow:step: 3600, lr: 0.000701, klw: 0.0907, cost: -0.9902, recon: -1.0692, kl: 0.8709, train_time_taken: 15.3855
INFO:tensorflow:step: 3620, lr: 0.000699, klw: 0.0911, cost: -1.1211, recon: -1.2071, kl: 0.9434, train_time_taken: 15.4767
INFO:tensorflow:step: 3640, lr: 0.000698, klw: 0.0915, cost: -1.1341, recon: -1.2181, kl: 0.9180, train_time_taken: 15.2477
INFO:tensorflow:step: 3660, lr: 0.000697, klw: 0.0919, cost: -1.1030, recon: -1.1865, kl: 0.9079, train_time_taken: 14.9272
INFO:tensorflow:step: 3680, lr: 0.000695, klw: 0.0924, cost: -1.1065, recon: -1.1924, kl: 0.9297, train_time_taken: 15.0778
INFO:tensorflow:step: 3700, lr: 0.000694, klw: 0.0928, cost: -1.0832, recon: -1.1647, kl: 0.8788, train_time_taken: 14.9077
INFO:tensorflow:step: 3720, lr: 0.000692, klw: 0.0932, cost: -1.0432, recon: -1.1259, kl: 0.8879, train_time_taken: 15.0667
INFO:tensorflow:step: 3740, lr: 0.000691, klw: 0.0936, cost: -1.1456, recon: -1.2313, kl: 0.9161, train_time_taken: 16.1730
INFO:tensorflow:step: 3760, lr: 0.000690, klw: 0.0940, cost: -1.0942, recon: -1.1765, kl: 0.8754, train_time_taken: 14.7735
INFO:tensorflow:step: 3780, lr: 0.000688, klw: 0.0944, cost: -1.0092, recon: -1.0910, kl: 0.8666, train_time_taken: 14.9740
INFO:tensorflow:step: 3800, lr: 0.000687, klw: 0.0948, cost: -1.0891, recon: -1.1712, kl: 0.8659, train_time_taken: 14.9376
INFO:tensorflow:step: 3820, lr: 0.000686, klw: 0.0952, cost: -1.0835, recon: -1.1669, kl: 0.8756, train_time_taken: 15.5390
INFO:tensorflow:step: 3840, lr: 0.000684, klw: 0.0956, cost: -1.1492, recon: -1.2350, kl: 0.8973, train_time_taken: 16.4457
INFO:tensorflow:step: 3860, lr: 0.000683, klw: 0.0960, cost: -1.0722, recon: -1.1545, kl: 0.8568, train_time_taken: 15.1090
INFO:tensorflow:step: 3880, lr: 0.000682, klw: 0.0964, cost: -1.0739, recon: -1.1558, kl: 0.8503, train_time_taken: 14.6978
INFO:tensorflow:step: 3900, lr: 0.000680, klw: 0.0968, cost: -1.1958, recon: -1.2815, kl: 0.8843, train_time_taken: 14.5360
INFO:tensorflow:step: 3920, lr: 0.000679, klw: 0.0972, cost: -1.0305, recon: -1.1106, kl: 0.8245, train_time_taken: 14.8637
INFO:tensorflow:step: 3940, lr: 0.000678, klw: 0.0976, cost: -1.1509, recon: -1.2349, kl: 0.8604, train_time_taken: 15.8957
INFO:tensorflow:step: 3960, lr: 0.000676, klw: 0.0980, cost: -1.0963, recon: -1.1774, kl: 0.8276, train_time_taken: 15.1853
INFO:tensorflow:step: 3980, lr: 0.000675, klw: 0.0984, cost: -1.0821, recon: -1.1628, kl: 0.8203, train_time_taken: 14.8594
INFO:tensorflow:step: 4000, lr: 0.000674, klw: 0.0988, cost: -1.1663, recon: -1.2507, kl: 0.8543, train_time_taken: 15.5175
INFO:tensorflow:best_valid_cost: -1.3463, valid_cost: -1.3463, valid_recon: -1.3549, valid_kl: 0.8582, valid_time_taken: 0.8739
INFO:tensorflow:saving model /content/sketch_rnn/ckpt_layer_norm/vector.
INFO:tensorflow:global_step 4000.
INFO:tensorflow:time_taken_save 0.4801.
INFO:tensorflow:eval_cost: -1.3219, eval_recon: -1.3303, eval_kl: 0.8471, eval_time_taken: 0.7258
INFO:tensorflow:step: 4020, lr: 0.000672, klw: 0.0992, cost: -1.0938, recon: -1.1759, kl: 0.8267, train_time_taken: 14.8966
INFO:tensorflow:step: 4040, lr: 0.000671, klw: 0.0996, cost: -1.0891, recon: -1.1693, kl: 0.8050, train_time_taken: 15.4963
INFO:tensorflow:step: 4060, lr: 0.000670, klw: 0.1000, cost: -1.0775, recon: -1.1573, kl: 0.7985, train_time_taken: 15.6113
INFO:tensorflow:step: 4080, lr: 0.000668, klw: 0.1004, cost: -1.0946, recon: -1.1758, kl: 0.8088, train_time_taken: 14.5256
INFO:tensorflow:step: 4100, lr: 0.000667, klw: 0.1008, cost: -1.1780, recon: -1.2632, kl: 0.8458, train_time_taken: 14.6598
INFO:tensorflow:step: 4120, lr: 0.000666, klw: 0.1012, cost: -1.1378, recon: -1.2210, kl: 0.8219, train_time_taken: 14.9897
INFO:tensorflow:step: 4140, lr: 0.000664, klw: 0.1016, cost: -1.1592, recon: -1.2402, kl: 0.7971, train_time_taken: 14.9193
INFO:tensorflow:step: 4160, lr: 0.000663, klw: 0.1020, cost: -1.0441, recon: -1.1231, kl: 0.7743, train_time_taken: 16.0860
INFO:tensorflow:step: 4180, lr: 0.000662, klw: 0.1024, cost: -1.1360, recon: -1.2169, kl: 0.7899, train_time_taken: 14.7420
INFO:tensorflow:step: 4200, lr: 0.000660, klw: 0.1028, cost: -1.1304, recon: -1.2139, kl: 0.8119, train_time_taken: 14.7919
INFO:tensorflow:step: 4220, lr: 0.000659, klw: 0.1032, cost: -1.1551, recon: -1.2394, kl: 0.8173, train_time_taken: 15.0089
INFO:tensorflow:step: 4240, lr: 0.000658, klw: 0.1036, cost: -1.0767, recon: -1.1569, kl: 0.7741, train_time_taken: 16.3560
INFO:tensorflow:step: 4260, lr: 0.000657, klw: 0.1040, cost: -1.2266, recon: -1.3095, kl: 0.7975, train_time_taken: 16.0885
INFO:tensorflow:step: 4280, lr: 0.000655, klw: 0.1044, cost: -1.1186, recon: -1.1982, kl: 0.7624, train_time_taken: 15.0405
INFO:tensorflow:step: 4300, lr: 0.000654, klw: 0.1048, cost: -1.0950, recon: -1.1724, kl: 0.7390, train_time_taken: 14.7887
INFO:tensorflow:step: 4320, lr: 0.000653, klw: 0.1052, cost: -1.1026, recon: -1.1817, kl: 0.7523, train_time_taken: 14.6389
INFO:tensorflow:step: 4340, lr: 0.000651, klw: 0.1056, cost: -1.1666, recon: -1.2491, kl: 0.7810, train_time_taken: 14.9685
INFO:tensorflow:step: 4360, lr: 0.000650, klw: 0.1060, cost: -1.0939, recon: -1.1741, kl: 0.7563, train_time_taken: 15.8388
INFO:tensorflow:step: 4380, lr: 0.000649, klw: 0.1064, cost: -1.1264, recon: -1.2063, kl: 0.7511, train_time_taken: 15.1653
INFO:tensorflow:step: 4400, lr: 0.000648, klw: 0.1068, cost: -1.0694, recon: -1.1472, kl: 0.7291, train_time_taken: 15.3503
INFO:tensorflow:step: 4420, lr: 0.000646, klw: 0.1072, cost: -1.0967, recon: -1.1735, kl: 0.7168, train_time_taken: 14.6284
INFO:tensorflow:step: 4440, lr: 0.000645, klw: 0.1076, cost: -1.1771, recon: -1.2588, kl: 0.7593, train_time_taken: 14.8533
INFO:tensorflow:step: 4460, lr: 0.000644, klw: 0.1079, cost: -1.1572, recon: -1.2380, kl: 0.7483, train_time_taken: 15.0741
INFO:tensorflow:step: 4480, lr: 0.000643, klw: 0.1083, cost: -1.0544, recon: -1.1306, kl: 0.7036, train_time_taken: 15.5791
INFO:tensorflow:step: 4500, lr: 0.000641, klw: 0.1087, cost: -1.0838, recon: -1.1636, kl: 0.7337, train_time_taken: 14.5534
INFO:tensorflow:step: 4520, lr: 0.000640, klw: 0.1091, cost: -1.1389, recon: -1.2167, kl: 0.7133, train_time_taken: 14.5743
INFO:tensorflow:step: 4540, lr: 0.000639, klw: 0.1095, cost: -1.1908, recon: -1.2699, kl: 0.7218, train_time_taken: 14.8733
INFO:tensorflow:step: 4560, lr: 0.000637, klw: 0.1099, cost: -1.1201, recon: -1.1996, kl: 0.7231, train_time_taken: 14.7112
INFO:tensorflow:step: 4580, lr: 0.000636, klw: 0.1103, cost: -1.1680, recon: -1.2481, kl: 0.7259, train_time_taken: 15.9605
INFO:tensorflow:step: 4600, lr: 0.000635, klw: 0.1107, cost: -1.1392, recon: -1.2157, kl: 0.6917, train_time_taken: 14.7270
INFO:tensorflow:step: 4620, lr: 0.000634, klw: 0.1111, cost: -1.2054, recon: -1.2864, kl: 0.7290, train_time_taken: 14.9115
INFO:tensorflow:step: 4640, lr: 0.000632, klw: 0.1115, cost: -1.1346, recon: -1.2146, kl: 0.7176, train_time_taken: 16.0267
INFO:tensorflow:step: 4660, lr: 0.000631, klw: 0.1118, cost: -1.2409, recon: -1.3214, kl: 0.7199, train_time_taken: 15.3651
INFO:tensorflow:step: 4680, lr: 0.000630, klw: 0.1122, cost: -1.1071, recon: -1.1820, kl: 0.6668, train_time_taken: 15.6880
INFO:tensorflow:step: 4700, lr: 0.000629, klw: 0.1126, cost: -1.0987, recon: -1.1752, kl: 0.6792, train_time_taken: 14.6335
INFO:tensorflow:step: 4720, lr: 0.000628, klw: 0.1130, cost: -1.1402, recon: -1.2178, kl: 0.6870, train_time_taken: 14.6971
INFO:tensorflow:step: 4740, lr: 0.000626, klw: 0.1134, cost: -1.1172, recon: -1.1930, kl: 0.6680, train_time_taken: 14.6378
INFO:tensorflow:step: 4760, lr: 0.000625, klw: 0.1138, cost: -1.1499, recon: -1.2267, kl: 0.6747, train_time_taken: 14.5897
INFO:tensorflow:step: 4780, lr: 0.000624, klw: 0.1142, cost: -1.2284, recon: -1.3097, kl: 0.7122, train_time_taken: 14.9544
INFO:tensorflow:step: 4800, lr: 0.000623, klw: 0.1146, cost: -1.1602, recon: -1.2376, kl: 0.6754, train_time_taken: 16.3737
INFO:tensorflow:step: 4820, lr: 0.000621, klw: 0.1149, cost: -1.1741, recon: -1.2520, kl: 0.6773, train_time_taken: 14.7267
INFO:tensorflow:step: 4840, lr: 0.000620, klw: 0.1153, cost: -1.1813, recon: -1.2601, kl: 0.6833, train_time_taken: 14.9816
INFO:tensorflow:step: 4860, lr: 0.000619, klw: 0.1157, cost: -1.1459, recon: -1.2218, kl: 0.6562, train_time_taken: 14.7730
INFO:tensorflow:step: 4880, lr: 0.000618, klw: 0.1161, cost: -1.1197, recon: -1.1962, kl: 0.6586, train_time_taken: 14.7429
INFO:tensorflow:step: 4900, lr: 0.000616, klw: 0.1165, cost: -1.1482, recon: -1.2245, kl: 0.6553, train_time_taken: 16.0540
INFO:tensorflow:step: 4920, lr: 0.000615, klw: 0.1169, cost: -1.1250, recon: -1.2002, kl: 0.6439, train_time_taken: 14.5812
INFO:tensorflow:step: 4940, lr: 0.000614, klw: 0.1172, cost: -1.2410, recon: -1.3206, kl: 0.6783, train_time_taken: 14.9628
INFO:tensorflow:step: 4960, lr: 0.000613, klw: 0.1176, cost: -1.1496, recon: -1.2256, kl: 0.6463, train_time_taken: 14.7253
INFO:tensorflow:step: 4980, lr: 0.000612, klw: 0.1180, cost: -1.1314, recon: -1.2104, kl: 0.6698, train_time_taken: 14.5949
INFO:tensorflow:step: 5000, lr: 0.000610, klw: 0.1184, cost: -1.2130, recon: -1.2902, kl: 0.6528, train_time_taken: 15.6052
INFO:tensorflow:step: 5020, lr: 0.000609, klw: 0.1188, cost: -1.1689, recon: -1.2451, kl: 0.6410, train_time_taken: 14.9787
INFO:tensorflow:step: 5040, lr: 0.000608, klw: 0.1192, cost: -1.1014, recon: -1.1765, kl: 0.6299, train_time_taken: 14.6376
INFO:tensorflow:step: 5060, lr: 0.000607, klw: 0.1195, cost: -1.1835, recon: -1.2616, kl: 0.6531, train_time_taken: 16.0789
INFO:tensorflow:step: 5080, lr: 0.000606, klw: 0.1199, cost: -1.2910, recon: -1.3707, kl: 0.6648, train_time_taken: 14.6769
INFO:tensorflow:step: 5100, lr: 0.000604, klw: 0.1203, cost: -1.1772, recon: -1.2544, kl: 0.6420, train_time_taken: 14.6761
INFO:tensorflow:step: 5120, lr: 0.000603, klw: 0.1207, cost: -1.1717, recon: -1.2459, kl: 0.6151, train_time_taken: 15.6372
INFO:tensorflow:step: 5140, lr: 0.000602, klw: 0.1211, cost: -1.1997, recon: -1.2789, kl: 0.6543, train_time_taken: 14.8469
INFO:tensorflow:step: 5160, lr: 0.000601, klw: 0.1214, cost: -1.2915, recon: -1.3727, kl: 0.6688, train_time_taken: 14.5137
INFO:tensorflow:step: 5180, lr: 0.000600, klw: 0.1218, cost: -1.1843, recon: -1.2598, kl: 0.6199, train_time_taken: 14.9727
INFO:tensorflow:step: 5200, lr: 0.000599, klw: 0.1222, cost: -1.1502, recon: -1.2257, kl: 0.6178, train_time_taken: 14.4800
INFO:tensorflow:step: 5220, lr: 0.000597, klw: 0.1226, cost: -1.1933, recon: -1.2691, kl: 0.6179, train_time_taken: 16.5412
INFO:tensorflow:step: 5240, lr: 0.000596, klw: 0.1229, cost: -1.2031, recon: -1.2790, kl: 0.6173, train_time_taken: 14.8850
INFO:tensorflow:step: 5260, lr: 0.000595, klw: 0.1233, cost: -1.2292, recon: -1.3050, kl: 0.6141, train_time_taken: 14.7595
INFO:tensorflow:step: 5280, lr: 0.000594, klw: 0.1237, cost: -1.0668, recon: -1.1407, kl: 0.5973, train_time_taken: 14.7492
INFO:tensorflow:step: 5300, lr: 0.000593, klw: 0.1241, cost: -1.3229, recon: -1.4021, kl: 0.6386, train_time_taken: 14.7495
INFO:tensorflow:step: 5320, lr: 0.000592, klw: 0.1244, cost: -1.2775, recon: -1.3549, kl: 0.6219, train_time_taken: 15.6939
INFO:tensorflow:step: 5340, lr: 0.000590, klw: 0.1248, cost: -1.2303, recon: -1.3086, kl: 0.6274, train_time_taken: 15.1356
INFO:tensorflow:step: 5360, lr: 0.000589, klw: 0.1252, cost: -1.2528, recon: -1.3303, kl: 0.6188, train_time_taken: 14.4273
INFO:tensorflow:step: 5380, lr: 0.000588, klw: 0.1256, cost: -1.3109, recon: -1.3901, kl: 0.6305, train_time_taken: 15.0632
INFO:tensorflow:step: 5400, lr: 0.000587, klw: 0.1259, cost: -1.1377, recon: -1.2105, kl: 0.5784, train_time_taken: 14.8087
INFO:tensorflow:step: 5420, lr: 0.000586, klw: 0.1263, cost: -1.2023, recon: -1.2763, kl: 0.5861, train_time_taken: 14.9945
INFO:tensorflow:step: 5440, lr: 0.000585, klw: 0.1267, cost: -1.2433, recon: -1.3209, kl: 0.6126, train_time_taken: 16.0758
INFO:tensorflow:step: 5460, lr: 0.000583, klw: 0.1271, cost: -1.2197, recon: -1.2963, kl: 0.6029, train_time_taken: 15.5119
INFO:tensorflow:step: 5480, lr: 0.000582, klw: 0.1274, cost: -1.1776, recon: -1.2515, kl: 0.5797, train_time_taken: 15.7214
INFO:tensorflow:step: 5500, lr: 0.000581, klw: 0.1278, cost: -1.1291, recon: -1.2018, kl: 0.5692, train_time_taken: 14.5581
INFO:tensorflow:step: 5520, lr: 0.000580, klw: 0.1282, cost: -1.2359, recon: -1.3120, kl: 0.5941, train_time_taken: 14.5967
INFO:tensorflow:step: 5540, lr: 0.000579, klw: 0.1286, cost: -1.1476, recon: -1.2212, kl: 0.5723, train_time_taken: 15.8888
INFO:tensorflow:step: 5560, lr: 0.000578, klw: 0.1289, cost: -1.2249, recon: -1.3016, kl: 0.5951, train_time_taken: 14.8596
INFO:tensorflow:step: 5580, lr: 0.000577, klw: 0.1293, cost: -1.2348, recon: -1.3117, kl: 0.5945, train_time_taken: 14.8429
INFO:tensorflow:step: 5600, lr: 0.000575, klw: 0.1297, cost: -1.2546, recon: -1.3301, kl: 0.5825, train_time_taken: 14.7594
INFO:tensorflow:step: 5620, lr: 0.000574, klw: 0.1300, cost: -1.2591, recon: -1.3342, kl: 0.5774, train_time_taken: 15.5594
INFO:tensorflow:step: 5640, lr: 0.000573, klw: 0.1304, cost: -1.1895, recon: -1.2635, kl: 0.5675, train_time_taken: 15.7598
INFO:tensorflow:step: 5660, lr: 0.000572, klw: 0.1308, cost: -1.2592, recon: -1.3360, kl: 0.5874, train_time_taken: 14.9060
INFO:tensorflow:step: 5680, lr: 0.000571, klw: 0.1311, cost: -1.2262, recon: -1.3025, kl: 0.5816, train_time_taken: 14.7711
INFO:tensorflow:step: 5700, lr: 0.000570, klw: 0.1315, cost: -1.2644, recon: -1.3417, kl: 0.5880, train_time_taken: 14.5513
INFO:tensorflow:step: 5720, lr: 0.000569, klw: 0.1319, cost: -1.2558, recon: -1.3325, kl: 0.5818, train_time_taken: 14.7860
INFO:tensorflow:step: 5740, lr: 0.000568, klw: 0.1323, cost: -1.2768, recon: -1.3533, kl: 0.5788, train_time_taken: 14.9302
INFO:tensorflow:step: 5760, lr: 0.000567, klw: 0.1326, cost: -1.3009, recon: -1.3764, kl: 0.5691, train_time_taken: 15.7290
INFO:tensorflow:step: 5780, lr: 0.000565, klw: 0.1330, cost: -1.2861, recon: -1.3631, kl: 0.5786, train_time_taken: 14.5692
INFO:tensorflow:step: 5800, lr: 0.000564, klw: 0.1334, cost: -1.1700, recon: -1.2445, kl: 0.5589, train_time_taken: 14.6906
INFO:tensorflow:step: 5820, lr: 0.000563, klw: 0.1337, cost: -1.2114, recon: -1.2857, kl: 0.5556, train_time_taken: 14.6493
INFO:tensorflow:step: 5840, lr: 0.000562, klw: 0.1341, cost: -1.2576, recon: -1.3316, kl: 0.5514, train_time_taken: 14.6829
INFO:tensorflow:step: 5860, lr: 0.000561, klw: 0.1345, cost: -1.2226, recon: -1.2941, kl: 0.5317, train_time_taken: 16.3657
INFO:tensorflow:step: 5880, lr: 0.000560, klw: 0.1348, cost: -1.1881, recon: -1.2608, kl: 0.5392, train_time_taken: 16.3913
INFO:tensorflow:step: 5900, lr: 0.000559, klw: 0.1352, cost: -1.2210, recon: -1.2956, kl: 0.5513, train_time_taken: 14.7005
INFO:tensorflow:step: 5920, lr: 0.000558, klw: 0.1355, cost: -1.2229, recon: -1.2970, kl: 0.5464, train_time_taken: 14.8722
INFO:tensorflow:step: 5940, lr: 0.000557, klw: 0.1359, cost: -1.3233, recon: -1.4005, kl: 0.5679, train_time_taken: 14.9521
INFO:tensorflow:step: 5960, lr: 0.000555, klw: 0.1363, cost: -1.2551, recon: -1.3295, kl: 0.5466, train_time_taken: 15.9913
INFO:tensorflow:step: 5980, lr: 0.000554, klw: 0.1366, cost: -1.2980, recon: -1.3747, kl: 0.5618, train_time_taken: 14.7884
INFO:tensorflow:step: 6000, lr: 0.000553, klw: 0.1370, cost: -1.2708, recon: -1.3461, kl: 0.5493, train_time_taken: 14.8443
INFO:tensorflow:best_valid_cost: -1.4688, valid_cost: -1.4688, valid_recon: -1.4743, valid_kl: 0.5510, valid_time_taken: 0.8864
INFO:tensorflow:saving model /content/sketch_rnn/ckpt_layer_norm/vector.
INFO:tensorflow:global_step 6000.
INFO:tensorflow:time_taken_save 0.5105.
INFO:tensorflow:eval_cost: -1.4388, eval_recon: -1.4442, eval_kl: 0.5421, eval_time_taken: 0.7349
INFO:tensorflow:step: 6020, lr: 0.000552, klw: 0.1374, cost: -1.2215, recon: -1.2946, kl: 0.5320, train_time_taken: 15.5558
INFO:tensorflow:step: 6040, lr: 0.000551, klw: 0.1377, cost: -1.1734, recon: -1.2469, kl: 0.5336, train_time_taken: 14.7908
INFO:tensorflow:step: 6060, lr: 0.000550, klw: 0.1381, cost: -1.2940, recon: -1.3683, kl: 0.5383, train_time_taken: 15.7547
INFO:tensorflow:step: 6080, lr: 0.000549, klw: 0.1385, cost: -1.3084, recon: -1.3851, kl: 0.5540, train_time_taken: 15.2245
INFO:tensorflow:step: 6100, lr: 0.000548, klw: 0.1388, cost: -1.2298, recon: -1.3036, kl: 0.5321, train_time_taken: 14.6376
INFO:tensorflow:step: 6120, lr: 0.000547, klw: 0.1392, cost: -1.2965, recon: -1.3715, kl: 0.5387, train_time_taken: 14.5908
INFO:tensorflow:step: 6140, lr: 0.000546, klw: 0.1395, cost: -1.2563, recon: -1.3302, kl: 0.5296, train_time_taken: 14.5941
INFO:tensorflow:step: 6160, lr: 0.000545, klw: 0.1399, cost: -1.2578, recon: -1.3314, kl: 0.5263, train_time_taken: 14.5950
INFO:tensorflow:step: 6180, lr: 0.000544, klw: 0.1403, cost: -1.2413, recon: -1.3162, kl: 0.5343, train_time_taken: 16.2525
INFO:tensorflow:step: 6200, lr: 0.000543, klw: 0.1406, cost: -1.2603, recon: -1.3337, kl: 0.5223, train_time_taken: 14.7317
INFO:tensorflow:step: 6220, lr: 0.000541, klw: 0.1410, cost: -1.2469, recon: -1.3208, kl: 0.5243, train_time_taken: 14.4615
INFO:tensorflow:step: 6240, lr: 0.000540, klw: 0.1413, cost: -1.2882, recon: -1.3616, kl: 0.5195, train_time_taken: 14.7620
INFO:tensorflow:step: 6260, lr: 0.000539, klw: 0.1417, cost: -1.3274, recon: -1.4066, kl: 0.5591, train_time_taken: 14.8141
INFO:tensorflow:step: 6280, lr: 0.000538, klw: 0.1420, cost: -1.2665, recon: -1.3398, kl: 0.5160, train_time_taken: 16.0892
INFO:tensorflow:step: 6300, lr: 0.000537, klw: 0.1424, cost: -1.2590, recon: -1.3321, kl: 0.5137, train_time_taken: 15.6311
INFO:tensorflow:step: 6320, lr: 0.000536, klw: 0.1428, cost: -1.2884, recon: -1.3631, kl: 0.5232, train_time_taken: 15.1279
INFO:tensorflow:step: 6340, lr: 0.000535, klw: 0.1431, cost: -1.2994, recon: -1.3743, kl: 0.5233, train_time_taken: 15.0774
INFO:tensorflow:step: 6360, lr: 0.000534, klw: 0.1435, cost: -1.1380, recon: -1.2090, kl: 0.4949, train_time_taken: 14.7364
INFO:tensorflow:step: 6380, lr: 0.000533, klw: 0.1438, cost: -1.3489, recon: -1.4246, kl: 0.5269, train_time_taken: 15.5995
INFO:tensorflow:step: 6400, lr: 0.000532, klw: 0.1442, cost: -1.2597, recon: -1.3335, kl: 0.5117, train_time_taken: 15.5726
INFO:tensorflow:step: 6420, lr: 0.000531, klw: 0.1445, cost: -1.3196, recon: -1.3961, kl: 0.5292, train_time_taken: 15.4723
INFO:tensorflow:step: 6440, lr: 0.000530, klw: 0.1449, cost: -1.1899, recon: -1.2623, kl: 0.4998, train_time_taken: 14.8370
INFO:tensorflow:step: 6460, lr: 0.000529, klw: 0.1453, cost: -1.3435, recon: -1.4208, kl: 0.5320, train_time_taken: 14.8964
INFO:tensorflow:step: 6480, lr: 0.000528, klw: 0.1456, cost: -1.2584, recon: -1.3323, kl: 0.5077, train_time_taken: 14.9576
INFO:tensorflow:step: 6500, lr: 0.000527, klw: 0.1460, cost: -1.3034, recon: -1.3770, kl: 0.5046, train_time_taken: 16.1286
INFO:tensorflow:step: 6520, lr: 0.000526, klw: 0.1463, cost: -1.1899, recon: -1.2611, kl: 0.4862, train_time_taken: 14.5752
INFO:tensorflow:step: 6540, lr: 0.000525, klw: 0.1467, cost: -1.2851, recon: -1.3578, kl: 0.4960, train_time_taken: 14.8356
INFO:tensorflow:step: 6560, lr: 0.000524, klw: 0.1470, cost: -1.2509, recon: -1.3264, kl: 0.5132, train_time_taken: 14.6700
INFO:tensorflow:step: 6580, lr: 0.000523, klw: 0.1474, cost: -1.2965, recon: -1.3707, kl: 0.5039, train_time_taken: 14.7523
INFO:tensorflow:step: 6600, lr: 0.000522, klw: 0.1477, cost: -1.2856, recon: -1.3577, kl: 0.4874, train_time_taken: 16.1859
INFO:tensorflow:step: 6620, lr: 0.000521, klw: 0.1481, cost: -1.2322, recon: -1.3061, kl: 0.4992, train_time_taken: 14.6983
INFO:tensorflow:step: 6640, lr: 0.000520, klw: 0.1484, cost: -1.1782, recon: -1.2496, kl: 0.4810, train_time_taken: 14.7319
INFO:tensorflow:step: 6660, lr: 0.000519, klw: 0.1488, cost: -1.3383, recon: -1.4155, kl: 0.5194, train_time_taken: 14.8621
INFO:tensorflow:step: 6680, lr: 0.000518, klw: 0.1491, cost: -1.2721, recon: -1.3439, kl: 0.4820, train_time_taken: 14.8106
INFO:tensorflow:step: 6700, lr: 0.000517, klw: 0.1495, cost: -1.3049, recon: -1.3807, kl: 0.5068, train_time_taken: 16.3629
INFO:tensorflow:step: 6720, lr: 0.000516, klw: 0.1498, cost: -1.2734, recon: -1.3487, kl: 0.5028, train_time_taken: 15.2156
INFO:tensorflow:step: 6740, lr: 0.000515, klw: 0.1502, cost: -1.2471, recon: -1.3199, kl: 0.4852, train_time_taken: 15.0575
INFO:tensorflow:step: 6760, lr: 0.000514, klw: 0.1505, cost: -1.2886, recon: -1.3622, kl: 0.4888, train_time_taken: 14.6764
INFO:tensorflow:step: 6780, lr: 0.000513, klw: 0.1509, cost: -1.3567, recon: -1.4331, kl: 0.5070, train_time_taken: 14.6222
INFO:tensorflow:step: 6800, lr: 0.000512, klw: 0.1512, cost: -1.2268, recon: -1.2991, kl: 0.4780, train_time_taken: 15.0467
INFO:tensorflow:step: 6820, lr: 0.000511, klw: 0.1516, cost: -1.2625, recon: -1.3351, kl: 0.4787, train_time_taken: 16.9793
INFO:tensorflow:step: 6840, lr: 0.000510, klw: 0.1519, cost: -1.2321, recon: -1.3037, kl: 0.4709, train_time_taken: 14.9045
INFO:tensorflow:step: 6860, lr: 0.000509, klw: 0.1523, cost: -1.2412, recon: -1.3120, kl: 0.4649, train_time_taken: 14.4258
INFO:tensorflow:step: 6880, lr: 0.000508, klw: 0.1526, cost: -1.3412, recon: -1.4162, kl: 0.4914, train_time_taken: 14.5257
INFO:tensorflow:step: 6900, lr: 0.000507, klw: 0.1530, cost: -1.3074, recon: -1.3821, kl: 0.4879, train_time_taken: 14.5600
INFO:tensorflow:step: 6920, lr: 0.000506, klw: 0.1533, cost: -1.3205, recon: -1.3935, kl: 0.4761, train_time_taken: 15.9532
INFO:tensorflow:step: 6940, lr: 0.000505, klw: 0.1537, cost: -1.3084, recon: -1.3823, kl: 0.4814, train_time_taken: 14.6525
INFO:tensorflow:step: 6960, lr: 0.000504, klw: 0.1540, cost: -1.2121, recon: -1.2837, kl: 0.4649, train_time_taken: 14.7817
INFO:tensorflow:step: 6980, lr: 0.000503, klw: 0.1544, cost: -1.2863, recon: -1.3586, kl: 0.4685, train_time_taken: 15.0295
INFO:tensorflow:step: 7000, lr: 0.000502, klw: 0.1547, cost: -1.3398, recon: -1.4156, kl: 0.4898, train_time_taken: 14.7084
INFO:tensorflow:step: 7020, lr: 0.000501, klw: 0.1551, cost: -1.3754, recon: -1.4507, kl: 0.4859, train_time_taken: 15.2751
INFO:tensorflow:step: 7040, lr: 0.000500, klw: 0.1554, cost: -1.3130, recon: -1.3876, kl: 0.4799, train_time_taken: 14.9740
INFO:tensorflow:step: 7060, lr: 0.000499, klw: 0.1557, cost: -1.2644, recon: -1.3379, kl: 0.4720, train_time_taken: 15.0329
INFO:tensorflow:step: 7080, lr: 0.000498, klw: 0.1561, cost: -1.2095, recon: -1.2803, kl: 0.4541, train_time_taken: 14.9276
INFO:tensorflow:step: 7100, lr: 0.000497, klw: 0.1564, cost: -1.1867, recon: -1.2555, kl: 0.4396, train_time_taken: 15.2337
INFO:tensorflow:step: 7120, lr: 0.000496, klw: 0.1568, cost: -1.2963, recon: -1.3709, kl: 0.4757, train_time_taken: 16.2198
INFO:tensorflow:step: 7140, lr: 0.000495, klw: 0.1571, cost: -1.3487, recon: -1.4243, kl: 0.4812, train_time_taken: 15.8899
INFO:tensorflow:step: 7160, lr: 0.000494, klw: 0.1575, cost: -1.2187, recon: -1.2898, kl: 0.4513, train_time_taken: 14.3987
INFO:tensorflow:step: 7180, lr: 0.000493, klw: 0.1578, cost: -1.3155, recon: -1.3889, kl: 0.4655, train_time_taken: 14.8106
INFO:tensorflow:step: 7200, lr: 0.000492, klw: 0.1581, cost: -1.2994, recon: -1.3725, kl: 0.4624, train_time_taken: 15.0814
INFO:tensorflow:step: 7220, lr: 0.000491, klw: 0.1585, cost: -1.3675, recon: -1.4422, kl: 0.4717, train_time_taken: 15.4532
INFO:tensorflow:step: 7240, lr: 0.000490, klw: 0.1588, cost: -1.2163, recon: -1.2873, kl: 0.4469, train_time_taken: 16.2018
INFO:tensorflow:step: 7260, lr: 0.000489, klw: 0.1592, cost: -1.1812, recon: -1.2534, kl: 0.4533, train_time_taken: 14.9330
INFO:tensorflow:step: 7280, lr: 0.000488, klw: 0.1595, cost: -1.2987, recon: -1.3712, kl: 0.4545, train_time_taken: 14.5236
INFO:tensorflow:step: 7300, lr: 0.000487, klw: 0.1598, cost: -1.2805, recon: -1.3520, kl: 0.4471, train_time_taken: 15.0562
INFO:tensorflow:step: 7320, lr: 0.000486, klw: 0.1602, cost: -1.3455, recon: -1.4201, kl: 0.4660, train_time_taken: 15.1439
INFO:tensorflow:step: 7340, lr: 0.000485, klw: 0.1605, cost: -1.2880, recon: -1.3603, kl: 0.4502, train_time_taken: 16.0227
INFO:tensorflow:step: 7360, lr: 0.000484, klw: 0.1609, cost: -1.3117, recon: -1.3863, kl: 0.4642, train_time_taken: 14.7345
INFO:tensorflow:step: 7380, lr: 0.000483, klw: 0.1612, cost: -1.2972, recon: -1.3715, kl: 0.4609, train_time_taken: 14.8289
INFO:tensorflow:step: 7400, lr: 0.000482, klw: 0.1615, cost: -1.3138, recon: -1.3895, kl: 0.4687, train_time_taken: 14.8452
INFO:tensorflow:step: 7420, lr: 0.000481, klw: 0.1619, cost: -1.3599, recon: -1.4329, kl: 0.4509, train_time_taken: 14.7612
INFO:tensorflow:step: 7440, lr: 0.000480, klw: 0.1622, cost: -1.3292, recon: -1.4036, kl: 0.4587, train_time_taken: 15.3777
INFO:tensorflow:step: 7460, lr: 0.000479, klw: 0.1626, cost: -1.3371, recon: -1.4114, kl: 0.4572, train_time_taken: 15.6809
INFO:tensorflow:step: 7480, lr: 0.000479, klw: 0.1629, cost: -1.3679, recon: -1.4435, kl: 0.4642, train_time_taken: 14.6863
INFO:tensorflow:step: 7500, lr: 0.000478, klw: 0.1632, cost: -1.2499, recon: -1.3225, kl: 0.4448, train_time_taken: 14.6982
INFO:tensorflow:step: 7520, lr: 0.000477, klw: 0.1636, cost: -1.2799, recon: -1.3545, kl: 0.4565, train_time_taken: 16.3262
INFO:tensorflow:step: 7540, lr: 0.000476, klw: 0.1639, cost: -1.2622, recon: -1.3351, kl: 0.4450, train_time_taken: 14.7315
INFO:tensorflow:step: 7560, lr: 0.000475, klw: 0.1642, cost: -1.3258, recon: -1.4004, kl: 0.4545, train_time_taken: 15.9974
INFO:tensorflow:step: 7580, lr: 0.000474, klw: 0.1646, cost: -1.2358, recon: -1.3072, kl: 0.4340, train_time_taken: 14.3653
INFO:tensorflow:step: 7600, lr: 0.000473, klw: 0.1649, cost: -1.3050, recon: -1.3776, kl: 0.4404, train_time_taken: 14.7028
INFO:tensorflow:step: 7620, lr: 0.000472, klw: 0.1652, cost: -1.2956, recon: -1.3684, kl: 0.4406, train_time_taken: 14.9895
INFO:tensorflow:step: 7640, lr: 0.000471, klw: 0.1656, cost: -1.2309, recon: -1.3024, kl: 0.4320, train_time_taken: 15.6895
INFO:tensorflow:step: 7660, lr: 0.000470, klw: 0.1659, cost: -1.2675, recon: -1.3386, kl: 0.4286, train_time_taken: 15.7091
INFO:tensorflow:step: 7680, lr: 0.000469, klw: 0.1662, cost: -1.3265, recon: -1.4015, kl: 0.4514, train_time_taken: 14.5900
INFO:tensorflow:step: 7700, lr: 0.000468, klw: 0.1666, cost: -1.4049, recon: -1.4796, kl: 0.4485, train_time_taken: 14.8723
INFO:tensorflow:step: 7720, lr: 0.000467, klw: 0.1669, cost: -1.3324, recon: -1.4081, kl: 0.4537, train_time_taken: 14.6025
INFO:tensorflow:step: 7740, lr: 0.000467, klw: 0.1672, cost: -1.2991, recon: -1.3736, kl: 0.4454, train_time_taken: 14.5792
INFO:tensorflow:step: 7760, lr: 0.000466, klw: 0.1676, cost: -1.3404, recon: -1.4141, kl: 0.4397, train_time_taken: 15.0033
INFO:tensorflow:step: 7780, lr: 0.000465, klw: 0.1679, cost: -1.2439, recon: -1.3169, kl: 0.4347, train_time_taken: 15.7018
INFO:tensorflow:step: 7800, lr: 0.000464, klw: 0.1682, cost: -1.3048, recon: -1.3779, kl: 0.4349, train_time_taken: 14.7773
INFO:tensorflow:step: 7820, lr: 0.000463, klw: 0.1686, cost: -1.3467, recon: -1.4206, kl: 0.4382, train_time_taken: 14.5404
INFO:tensorflow:step: 7840, lr: 0.000462, klw: 0.1689, cost: -1.3611, recon: -1.4375, kl: 0.4523, train_time_taken: 15.0880
INFO:tensorflow:step: 7860, lr: 0.000461, klw: 0.1692, cost: -1.4804, recon: -1.5564, kl: 0.4488, train_time_taken: 14.8775
INFO:tensorflow:step: 7880, lr: 0.000460, klw: 0.1696, cost: -1.3290, recon: -1.4022, kl: 0.4317, train_time_taken: 15.9456
INFO:tensorflow:step: 7900, lr: 0.000459, klw: 0.1699, cost: -1.3447, recon: -1.4209, kl: 0.4485, train_time_taken: 14.7888
INFO:tensorflow:step: 7920, lr: 0.000458, klw: 0.1702, cost: -1.3550, recon: -1.4291, kl: 0.4353, train_time_taken: 14.6456
INFO:tensorflow:step: 7940, lr: 0.000457, klw: 0.1706, cost: -1.2598, recon: -1.3315, kl: 0.4206, train_time_taken: 16.2995
INFO:tensorflow:step: 7960, lr: 0.000457, klw: 0.1709, cost: -1.4202, recon: -1.4965, kl: 0.4464, train_time_taken: 14.8434
INFO:tensorflow:step: 7980, lr: 0.000456, klw: 0.1712, cost: -1.3590, recon: -1.4338, kl: 0.4365, train_time_taken: 15.7440
INFO:tensorflow:step: 8000, lr: 0.000455, klw: 0.1715, cost: -1.3125, recon: -1.3861, kl: 0.4288, train_time_taken: 14.7813
INFO:tensorflow:best_valid_cost: -1.5550, valid_cost: -1.5550, valid_recon: -1.5594, valid_kl: 0.4386, valid_time_taken: 0.8694
INFO:tensorflow:saving model /content/sketch_rnn/ckpt_layer_norm/vector.
INFO:tensorflow:global_step 8000.
INFO:tensorflow:time_taken_save 0.4994.
INFO:tensorflow:eval_cost: -1.5215, eval_recon: -1.5258, eval_kl: 0.4308, eval_time_taken: 0.7304
INFO:tensorflow:step: 8020, lr: 0.000454, klw: 0.1719, cost: -1.4856, recon: -1.5632, kl: 0.4510, train_time_taken: 14.8651
INFO:tensorflow:step: 8040, lr: 0.000453, klw: 0.1722, cost: -1.3929, recon: -1.4677, kl: 0.4348, train_time_taken: 15.3862
INFO:tensorflow:step: 8060, lr: 0.000452, klw: 0.1725, cost: -1.3516, recon: -1.4250, kl: 0.4255, train_time_taken: 14.9869
INFO:tensorflow:step: 8080, lr: 0.000451, klw: 0.1729, cost: -1.3611, recon: -1.4385, kl: 0.4475, train_time_taken: 15.7312
INFO:tensorflow:step: 8100, lr: 0.000450, klw: 0.1732, cost: -1.3706, recon: -1.4457, kl: 0.4333, train_time_taken: 15.0557
INFO:tensorflow:step: 8120, lr: 0.000450, klw: 0.1735, cost: -1.2328, recon: -1.3053, kl: 0.4175, train_time_taken: 14.8282
INFO:tensorflow:step: 8140, lr: 0.000449, klw: 0.1738, cost: -1.3839, recon: -1.4583, kl: 0.4283, train_time_taken: 14.5415
INFO:tensorflow:step: 8160, lr: 0.000448, klw: 0.1742, cost: -1.4412, recon: -1.5183, kl: 0.4427, train_time_taken: 14.6824
INFO:tensorflow:step: 8180, lr: 0.000447, klw: 0.1745, cost: -1.3161, recon: -1.3874, kl: 0.4085, train_time_taken: 14.7118
INFO:tensorflow:step: 8200, lr: 0.000446, klw: 0.1748, cost: -1.3395, recon: -1.4138, kl: 0.4250, train_time_taken: 16.1450
INFO:tensorflow:step: 8220, lr: 0.000445, klw: 0.1751, cost: -1.3651, recon: -1.4409, kl: 0.4331, train_time_taken: 14.8494
INFO:tensorflow:step: 8240, lr: 0.000444, klw: 0.1755, cost: -1.3148, recon: -1.3866, kl: 0.4094, train_time_taken: 14.8874
INFO:tensorflow:step: 8260, lr: 0.000443, klw: 0.1758, cost: -1.2728, recon: -1.3466, kl: 0.4199, train_time_taken: 14.6223
INFO:tensorflow:step: 8280, lr: 0.000443, klw: 0.1761, cost: -1.3578, recon: -1.4330, kl: 0.4271, train_time_taken: 14.5713
INFO:tensorflow:step: 8300, lr: 0.000442, klw: 0.1764, cost: -1.3671, recon: -1.4419, kl: 0.4240, train_time_taken: 16.0781
INFO:tensorflow:step: 8320, lr: 0.000441, klw: 0.1768, cost: -1.4135, recon: -1.4891, kl: 0.4274, train_time_taken: 14.5639
INFO:tensorflow:step: 8340, lr: 0.000440, klw: 0.1771, cost: -1.3624, recon: -1.4384, kl: 0.4291, train_time_taken: 16.2054
INFO:tensorflow:step: 8360, lr: 0.000439, klw: 0.1774, cost: -1.2697, recon: -1.3421, kl: 0.4083, train_time_taken: 14.9351
INFO:tensorflow:step: 8380, lr: 0.000438, klw: 0.1777, cost: -1.4882, recon: -1.5649, kl: 0.4311, train_time_taken: 14.6701
INFO:tensorflow:step: 8400, lr: 0.000437, klw: 0.1781, cost: -1.3110, recon: -1.3856, kl: 0.4191, train_time_taken: 15.7595
INFO:tensorflow:step: 8420, lr: 0.000437, klw: 0.1784, cost: -1.3553, recon: -1.4304, kl: 0.4214, train_time_taken: 14.8236
INFO:tensorflow:step: 8440, lr: 0.000436, klw: 0.1787, cost: -1.2653, recon: -1.3373, kl: 0.4027, train_time_taken: 15.3262
INFO:tensorflow:step: 8460, lr: 0.000435, klw: 0.1790, cost: -1.3430, recon: -1.4178, kl: 0.4176, train_time_taken: 14.7269
INFO:tensorflow:step: 8480, lr: 0.000434, klw: 0.1793, cost: -1.3117, recon: -1.3853, kl: 0.4103, train_time_taken: 14.6026
INFO:tensorflow:step: 8500, lr: 0.000433, klw: 0.1797, cost: -1.3457, recon: -1.4224, kl: 0.4270, train_time_taken: 14.9303
INFO:tensorflow:step: 8520, lr: 0.000432, klw: 0.1800, cost: -1.4438, recon: -1.5193, kl: 0.4197, train_time_taken: 15.8842
INFO:tensorflow:step: 8540, lr: 0.000431, klw: 0.1803, cost: -1.3295, recon: -1.4042, kl: 0.4142, train_time_taken: 14.7331
INFO:tensorflow:step: 8560, lr: 0.000431, klw: 0.1806, cost: -1.3893, recon: -1.4642, kl: 0.4144, train_time_taken: 15.0578
INFO:tensorflow:step: 8580, lr: 0.000430, klw: 0.1809, cost: -1.3421, recon: -1.4174, kl: 0.4164, train_time_taken: 14.9082
INFO:tensorflow:step: 8600, lr: 0.000429, klw: 0.1813, cost: -1.4061, recon: -1.4833, kl: 0.4260, train_time_taken: 14.7508
INFO:tensorflow:step: 8620, lr: 0.000428, klw: 0.1816, cost: -1.3126, recon: -1.3873, kl: 0.4118, train_time_taken: 15.9398
INFO:tensorflow:step: 8640, lr: 0.000427, klw: 0.1819, cost: -1.3621, recon: -1.4380, kl: 0.4171, train_time_taken: 14.7940
INFO:tensorflow:step: 8660, lr: 0.000426, klw: 0.1822, cost: -1.3438, recon: -1.4179, kl: 0.4070, train_time_taken: 14.6684
INFO:tensorflow:step: 8680, lr: 0.000426, klw: 0.1825, cost: -1.3229, recon: -1.3973, kl: 0.4079, train_time_taken: 14.5523
INFO:tensorflow:step: 8700, lr: 0.000425, klw: 0.1828, cost: -1.2947, recon: -1.3684, kl: 0.4030, train_time_taken: 14.4484
INFO:tensorflow:step: 8720, lr: 0.000424, klw: 0.1832, cost: -1.3839, recon: -1.4606, kl: 0.4187, train_time_taken: 15.5290
INFO:tensorflow:step: 8740, lr: 0.000423, klw: 0.1835, cost: -1.4005, recon: -1.4765, kl: 0.4141, train_time_taken: 15.2017
INFO:tensorflow:step: 8760, lr: 0.000422, klw: 0.1838, cost: -1.3583, recon: -1.4348, kl: 0.4165, train_time_taken: 16.1096
INFO:tensorflow:step: 8780, lr: 0.000421, klw: 0.1841, cost: -1.3286, recon: -1.4035, kl: 0.4067, train_time_taken: 14.9135
INFO:tensorflow:step: 8800, lr: 0.000421, klw: 0.1844, cost: -1.3334, recon: -1.4089, kl: 0.4093, train_time_taken: 14.5662
INFO:tensorflow:step: 8820, lr: 0.000420, klw: 0.1847, cost: -1.3451, recon: -1.4199, kl: 0.4050, train_time_taken: 15.0416
INFO:tensorflow:step: 8840, lr: 0.000419, klw: 0.1851, cost: -1.3409, recon: -1.4166, kl: 0.4091, train_time_taken: 16.3246
INFO:tensorflow:step: 8860, lr: 0.000418, klw: 0.1854, cost: -1.2745, recon: -1.3474, kl: 0.3932, train_time_taken: 14.5370
INFO:tensorflow:step: 8880, lr: 0.000417, klw: 0.1857, cost: -1.3491, recon: -1.4242, kl: 0.4045, train_time_taken: 14.5648
INFO:tensorflow:step: 8900, lr: 0.000417, klw: 0.1860, cost: -1.2679, recon: -1.3431, kl: 0.4040, train_time_taken: 14.5962
INFO:tensorflow:step: 8920, lr: 0.000416, klw: 0.1863, cost: -1.3700, recon: -1.4440, kl: 0.3972, train_time_taken: 14.7539
INFO:tensorflow:step: 8940, lr: 0.000415, klw: 0.1866, cost: -1.3616, recon: -1.4368, kl: 0.4026, train_time_taken: 16.0713
INFO:tensorflow:step: 8960, lr: 0.000414, klw: 0.1869, cost: -1.3741, recon: -1.4495, kl: 0.4038, train_time_taken: 14.5159
INFO:tensorflow:step: 8980, lr: 0.000413, klw: 0.1873, cost: -1.3994, recon: -1.4761, kl: 0.4094, train_time_taken: 14.5120
INFO:tensorflow:step: 9000, lr: 0.000412, klw: 0.1876, cost: -1.4272, recon: -1.5037, kl: 0.4078, train_time_taken: 14.7321
INFO:tensorflow:step: 9020, lr: 0.000412, klw: 0.1879, cost: -1.3367, recon: -1.4112, kl: 0.3963, train_time_taken: 14.6389
INFO:tensorflow:step: 9040, lr: 0.000411, klw: 0.1882, cost: -1.3480, recon: -1.4224, kl: 0.3952, train_time_taken: 15.3457
INFO:tensorflow:step: 9060, lr: 0.000410, klw: 0.1885, cost: -1.4298, recon: -1.5047, kl: 0.3971, train_time_taken: 15.4518
INFO:tensorflow:step: 9080, lr: 0.000409, klw: 0.1888, cost: -1.4727, recon: -1.5523, kl: 0.4213, train_time_taken: 14.5636
INFO:tensorflow:step: 9100, lr: 0.000408, klw: 0.1891, cost: -1.3813, recon: -1.4569, kl: 0.3995, train_time_taken: 14.6337
INFO:tensorflow:step: 9120, lr: 0.000408, klw: 0.1894, cost: -1.4033, recon: -1.4788, kl: 0.3988, train_time_taken: 14.8271
INFO:tensorflow:step: 9140, lr: 0.000407, klw: 0.1897, cost: -1.2801, recon: -1.3547, kl: 0.3935, train_time_taken: 14.6744
INFO:tensorflow:step: 9160, lr: 0.000406, klw: 0.1901, cost: -1.3665, recon: -1.4424, kl: 0.3988, train_time_taken: 16.9352
INFO:tensorflow:step: 9180, lr: 0.000405, klw: 0.1904, cost: -1.2688, recon: -1.3415, kl: 0.3817, train_time_taken: 15.1656
INFO:tensorflow:step: 9200, lr: 0.000405, klw: 0.1907, cost: -1.4175, recon: -1.4948, kl: 0.4053, train_time_taken: 14.6784
INFO:tensorflow:step: 9220, lr: 0.000404, klw: 0.1910, cost: -1.3591, recon: -1.4336, kl: 0.3899, train_time_taken: 14.6980
INFO:tensorflow:step: 9240, lr: 0.000403, klw: 0.1913, cost: -1.4540, recon: -1.5328, kl: 0.4121, train_time_taken: 14.8158
INFO:tensorflow:step: 9260, lr: 0.000402, klw: 0.1916, cost: -1.2324, recon: -1.3056, kl: 0.3817, train_time_taken: 16.2807
INFO:tensorflow:step: 9280, lr: 0.000401, klw: 0.1919, cost: -1.4193, recon: -1.4954, kl: 0.3965, train_time_taken: 14.6037
INFO:tensorflow:step: 9300, lr: 0.000401, klw: 0.1922, cost: -1.4796, recon: -1.5590, kl: 0.4126, train_time_taken: 14.8334
INFO:tensorflow:step: 9320, lr: 0.000400, klw: 0.1925, cost: -1.3475, recon: -1.4223, kl: 0.3882, train_time_taken: 14.6716
INFO:tensorflow:step: 9340, lr: 0.000399, klw: 0.1928, cost: -1.3402, recon: -1.4152, kl: 0.3892, train_time_taken: 14.7516
INFO:tensorflow:step: 9360, lr: 0.000398, klw: 0.1931, cost: -1.4354, recon: -1.5131, kl: 0.4020, train_time_taken: 15.3639
INFO:tensorflow:step: 9380, lr: 0.000397, klw: 0.1934, cost: -1.3460, recon: -1.4213, kl: 0.3892, train_time_taken: 15.5285
INFO:tensorflow:step: 9400, lr: 0.000397, klw: 0.1938, cost: -1.3967, recon: -1.4739, kl: 0.3984, train_time_taken: 14.9694
INFO:tensorflow:step: 9420, lr: 0.000396, klw: 0.1941, cost: -1.3578, recon: -1.4342, kl: 0.3936, train_time_taken: 14.8362
INFO:tensorflow:step: 9440, lr: 0.000395, klw: 0.1944, cost: -1.4290, recon: -1.5073, kl: 0.4026, train_time_taken: 14.7863
INFO:tensorflow:step: 9460, lr: 0.000394, klw: 0.1947, cost: -1.3873, recon: -1.4629, kl: 0.3884, train_time_taken: 15.0018
INFO:tensorflow:step: 9480, lr: 0.000394, klw: 0.1950, cost: -1.3855, recon: -1.4618, kl: 0.3916, train_time_taken: 15.9920
INFO:tensorflow:step: 9500, lr: 0.000393, klw: 0.1953, cost: -1.3872, recon: -1.4643, kl: 0.3948, train_time_taken: 14.6595
INFO:tensorflow:step: 9520, lr: 0.000392, klw: 0.1956, cost: -1.2935, recon: -1.3691, kl: 0.3868, train_time_taken: 14.6915
INFO:tensorflow:step: 9540, lr: 0.000391, klw: 0.1959, cost: -1.3689, recon: -1.4452, kl: 0.3897, train_time_taken: 15.0580
INFO:tensorflow:step: 9560, lr: 0.000391, klw: 0.1962, cost: -1.4570, recon: -1.5358, kl: 0.4016, train_time_taken: 14.8528
INFO:tensorflow:step: 9580, lr: 0.000390, klw: 0.1965, cost: -1.4042, recon: -1.4804, kl: 0.3878, train_time_taken: 16.2496
INFO:tensorflow:step: 9600, lr: 0.000389, klw: 0.1968, cost: -1.3106, recon: -1.3842, kl: 0.3741, train_time_taken: 14.5808
INFO:tensorflow:step: 9620, lr: 0.000388, klw: 0.1971, cost: -1.3950, recon: -1.4736, kl: 0.3984, train_time_taken: 14.7529
INFO:tensorflow:step: 9640, lr: 0.000388, klw: 0.1974, cost: -1.4526, recon: -1.5312, kl: 0.3982, train_time_taken: 14.7013
INFO:tensorflow:step: 9660, lr: 0.000387, klw: 0.1977, cost: -1.4311, recon: -1.5089, kl: 0.3935, train_time_taken: 15.5789
INFO:tensorflow:step: 9680, lr: 0.000386, klw: 0.1980, cost: -1.3999, recon: -1.4769, kl: 0.3888, train_time_taken: 15.4599
INFO:tensorflow:step: 9700, lr: 0.000385, klw: 0.1983, cost: -1.3858, recon: -1.4629, kl: 0.3888, train_time_taken: 15.3887
INFO:tensorflow:step: 9720, lr: 0.000385, klw: 0.1986, cost: -1.4479, recon: -1.5268, kl: 0.3971, train_time_taken: 14.6932
INFO:tensorflow:step: 9740, lr: 0.000384, klw: 0.1989, cost: -1.3381, recon: -1.4150, kl: 0.3865, train_time_taken: 14.5892
INFO:tensorflow:step: 9760, lr: 0.000383, klw: 0.1992, cost: -1.3524, recon: -1.4293, kl: 0.3859, train_time_taken: 14.4694
INFO:tensorflow:step: 9780, lr: 0.000382, klw: 0.1995, cost: -1.3755, recon: -1.4524, kl: 0.3857, train_time_taken: 14.6333
INFO:tensorflow:step: 9800, lr: 0.000382, klw: 0.1998, cost: -1.4297, recon: -1.5070, kl: 0.3869, train_time_taken: 16.0362
INFO:tensorflow:step: 9820, lr: 0.000381, klw: 0.2001, cost: -1.4277, recon: -1.5045, kl: 0.3837, train_time_taken: 14.7825
INFO:tensorflow:step: 9840, lr: 0.000380, klw: 0.2004, cost: -1.3124, recon: -1.3887, kl: 0.3807, train_time_taken: 14.8711
INFO:tensorflow:step: 9860, lr: 0.000379, klw: 0.2007, cost: -1.3821, recon: -1.4584, kl: 0.3803, train_time_taken: 14.6231
INFO:tensorflow:step: 9880, lr: 0.000379, klw: 0.2010, cost: -1.3813, recon: -1.4585, kl: 0.3841, train_time_taken: 14.9101
INFO:tensorflow:step: 9900, lr: 0.000378, klw: 0.2013, cost: -1.3497, recon: -1.4268, kl: 0.3831, train_time_taken: 15.7579
INFO:tensorflow:step: 9920, lr: 0.000377, klw: 0.2016, cost: -1.3537, recon: -1.4302, kl: 0.3795, train_time_taken: 14.9320
INFO:tensorflow:step: 9940, lr: 0.000376, klw: 0.2019, cost: -1.3232, recon: -1.3995, kl: 0.3783, train_time_taken: 14.6302
INFO:tensorflow:step: 9960, lr: 0.000376, klw: 0.2022, cost: -1.4404, recon: -1.5182, kl: 0.3847, train_time_taken: 14.7674
INFO:tensorflow:step: 9980, lr: 0.000375, klw: 0.2025, cost: -1.4210, recon: -1.4997, kl: 0.3890, train_time_taken: 14.5514
INFO:tensorflow:step: 10000, lr: 0.000374, klw: 0.2028, cost: -1.4057, recon: -1.4847, kl: 0.3897, train_time_taken: 16.7110
INFO:tensorflow:best_valid_cost: -1.6120, valid_cost: -1.6120, valid_recon: -1.6159, valid_kl: 0.3899, valid_time_taken: 1.0076
INFO:tensorflow:saving model /content/sketch_rnn/ckpt_layer_norm/vector.
INFO:tensorflow:global_step 10000.
INFO:tensorflow:time_taken_save 0.5816.
INFO:tensorflow:eval_cost: -1.5772, eval_recon: -1.5810, eval_kl: 0.3833, eval_time_taken: 0.8617
INFO:tensorflow:step: 10020, lr: 0.000373, klw: 0.2031, cost: -1.3419, recon: -1.4189, kl: 0.3791, train_time_taken: 15.2572
INFO:tensorflow:step: 10040, lr: 0.000373, klw: 0.2034, cost: -1.4442, recon: -1.5218, kl: 0.3813, train_time_taken: 14.5767
INFO:tensorflow:step: 10060, lr: 0.000372, klw: 0.2037, cost: -1.4106, recon: -1.4882, kl: 0.3807, train_time_taken: 15.6101
INFO:tensorflow:step: 10080, lr: 0.000371, klw: 0.2040, cost: -1.3554, recon: -1.4318, kl: 0.3744, train_time_taken: 14.7088
INFO:tensorflow:step: 10100, lr: 0.000371, klw: 0.2043, cost: -1.3463, recon: -1.4244, kl: 0.3824, train_time_taken: 15.0675
INFO:tensorflow:step: 10120, lr: 0.000370, klw: 0.2046, cost: -1.3936, recon: -1.4707, kl: 0.3769, train_time_taken: 15.6290
INFO:tensorflow:step: 10140, lr: 0.000369, klw: 0.2049, cost: -1.3740, recon: -1.4506, kl: 0.3740, train_time_taken: 14.5537
INFO:tensorflow:step: 10160, lr: 0.000368, klw: 0.2052, cost: -1.3424, recon: -1.4191, kl: 0.3742, train_time_taken: 14.4961
INFO:tensorflow:step: 10180, lr: 0.000368, klw: 0.2055, cost: -1.3314, recon: -1.4082, kl: 0.3737, train_time_taken: 14.8715
INFO:tensorflow:step: 10200, lr: 0.000367, klw: 0.2058, cost: -1.4936, recon: -1.5735, kl: 0.3886, train_time_taken: 14.4723
INFO:tensorflow:step: 10220, lr: 0.000366, klw: 0.2061, cost: -1.2667, recon: -1.3408, kl: 0.3595, train_time_taken: 16.2196
INFO:tensorflow:step: 10240, lr: 0.000366, klw: 0.2063, cost: -1.4274, recon: -1.5059, kl: 0.3803, train_time_taken: 14.8117
INFO:tensorflow:step: 10260, lr: 0.000365, klw: 0.2066, cost: -1.4615, recon: -1.5389, kl: 0.3746, train_time_taken: 14.7267
INFO:tensorflow:step: 10280, lr: 0.000364, klw: 0.2069, cost: -1.4993, recon: -1.5782, kl: 0.3813, train_time_taken: 14.5475
INFO:tensorflow:step: 10300, lr: 0.000363, klw: 0.2072, cost: -1.3917, recon: -1.4699, kl: 0.3774, train_time_taken: 14.8167
INFO:tensorflow:step: 10320, lr: 0.000363, klw: 0.2075, cost: -1.3956, recon: -1.4728, kl: 0.3721, train_time_taken: 15.2925
INFO:tensorflow:step: 10340, lr: 0.000362, klw: 0.2078, cost: -1.4138, recon: -1.4924, kl: 0.3783, train_time_taken: 15.5014
INFO:tensorflow:step: 10360, lr: 0.000361, klw: 0.2081, cost: -1.4436, recon: -1.5227, kl: 0.3798, train_time_taken: 14.8918
INFO:tensorflow:step: 10380, lr: 0.000361, klw: 0.2084, cost: -1.5027, recon: -1.5848, kl: 0.3942, train_time_taken: 14.8347
INFO:tensorflow:step: 10400, lr: 0.000360, klw: 0.2087, cost: -1.4032, recon: -1.4815, kl: 0.3754, train_time_taken: 15.8741
INFO:tensorflow:step: 10420, lr: 0.000359, klw: 0.2090, cost: -1.4427, recon: -1.5220, kl: 0.3795, train_time_taken: 15.0206
INFO:tensorflow:step: 10440, lr: 0.000359, klw: 0.2093, cost: -1.3720, recon: -1.4496, kl: 0.3710, train_time_taken: 15.9185
INFO:tensorflow:step: 10460, lr: 0.000358, klw: 0.2096, cost: -1.4644, recon: -1.5441, kl: 0.3802, train_time_taken: 15.6274
INFO:tensorflow:step: 10480, lr: 0.000357, klw: 0.2099, cost: -1.3314, recon: -1.4071, kl: 0.3610, train_time_taken: 14.7214
INFO:tensorflow:step: 10500, lr: 0.000356, klw: 0.2101, cost: -1.2751, recon: -1.3515, kl: 0.3638, train_time_taken: 14.6445
INFO:tensorflow:step: 10520, lr: 0.000356, klw: 0.2104, cost: -1.4141, recon: -1.4921, kl: 0.3704, train_time_taken: 14.5850
INFO:tensorflow:step: 10540, lr: 0.000355, klw: 0.2107, cost: -1.4665, recon: -1.5473, kl: 0.3838, train_time_taken: 15.9367
INFO:tensorflow:step: 10560, lr: 0.000354, klw: 0.2110, cost: -1.3824, recon: -1.4601, kl: 0.3685, train_time_taken: 14.6364
INFO:tensorflow:step: 10580, lr: 0.000354, klw: 0.2113, cost: -1.4344, recon: -1.5136, kl: 0.3748, train_time_taken: 14.8630
INFO:tensorflow:step: 10600, lr: 0.000353, klw: 0.2116, cost: -1.4270, recon: -1.5056, kl: 0.3717, train_time_taken: 14.9612
INFO:tensorflow:step: 10620, lr: 0.000352, klw: 0.2119, cost: -1.4334, recon: -1.5126, kl: 0.3737, train_time_taken: 14.6723
INFO:tensorflow:step: 10640, lr: 0.000352, klw: 0.2122, cost: -1.3920, recon: -1.4707, kl: 0.3711, train_time_taken: 15.5535
INFO:tensorflow:step: 10660, lr: 0.000351, klw: 0.2125, cost: -1.3834, recon: -1.4611, kl: 0.3659, train_time_taken: 15.3735
INFO:tensorflow:step: 10680, lr: 0.000350, klw: 0.2127, cost: -1.4576, recon: -1.5372, kl: 0.3740, train_time_taken: 14.7229
INFO:tensorflow:step: 10700, lr: 0.000350, klw: 0.2130, cost: -1.4230, recon: -1.5023, kl: 0.3719, train_time_taken: 14.6498
INFO:tensorflow:step: 10720, lr: 0.000349, klw: 0.2133, cost: -1.4646, recon: -1.5442, kl: 0.3730, train_time_taken: 14.8372
INFO:tensorflow:step: 10740, lr: 0.000348, klw: 0.2136, cost: -1.4341, recon: -1.5124, kl: 0.3669, train_time_taken: 14.6579
INFO:tensorflow:step: 10760, lr: 0.000348, klw: 0.2139, cost: -1.3421, recon: -1.4207, kl: 0.3678, train_time_taken: 15.9768
INFO:tensorflow:step: 10780, lr: 0.000347, klw: 0.2142, cost: -1.4572, recon: -1.5397, kl: 0.3853, train_time_taken: 14.7480
INFO:tensorflow:step: 10800, lr: 0.000346, klw: 0.2145, cost: -1.3737, recon: -1.4520, kl: 0.3652, train_time_taken: 14.7978
INFO:tensorflow:step: 10820, lr: 0.000346, klw: 0.2147, cost: -1.4117, recon: -1.4913, kl: 0.3707, train_time_taken: 16.1485
INFO:tensorflow:step: 10840, lr: 0.000345, klw: 0.2150, cost: -1.3993, recon: -1.4773, kl: 0.3630, train_time_taken: 14.7157
INFO:tensorflow:step: 10860, lr: 0.000344, klw: 0.2153, cost: -1.4548, recon: -1.5338, kl: 0.3670, train_time_taken: 16.3680
INFO:tensorflow:step: 10880, lr: 0.000344, klw: 0.2156, cost: -1.3864, recon: -1.4641, kl: 0.3606, train_time_taken: 14.8202
INFO:tensorflow:step: 10900, lr: 0.000343, klw: 0.2159, cost: -1.4178, recon: -1.4971, kl: 0.3673, train_time_taken: 14.7745
INFO:tensorflow:step: 10920, lr: 0.000342, klw: 0.2162, cost: -1.4873, recon: -1.5681, kl: 0.3736, train_time_taken: 14.6030
INFO:tensorflow:step: 10940, lr: 0.000342, klw: 0.2164, cost: -1.4813, recon: -1.5621, kl: 0.3733, train_time_taken: 14.6401
INFO:tensorflow:step: 10960, lr: 0.000341, klw: 0.2167, cost: -1.3184, recon: -1.3973, kl: 0.3638, train_time_taken: 15.4959
INFO:tensorflow:step: 10980, lr: 0.000340, klw: 0.2170, cost: -1.4272, recon: -1.5077, kl: 0.3710, train_time_taken: 15.3282
INFO:tensorflow:step: 11000, lr: 0.000340, klw: 0.2173, cost: -1.3706, recon: -1.4506, kl: 0.3683, train_time_taken: 14.6650
INFO:tensorflow:step: 11020, lr: 0.000339, klw: 0.2176, cost: -1.3857, recon: -1.4664, kl: 0.3710, train_time_taken: 14.6419
INFO:tensorflow:step: 11040, lr: 0.000338, klw: 0.2179, cost: -1.3418, recon: -1.4205, kl: 0.3614, train_time_taken: 14.7735
INFO:tensorflow:step: 11060, lr: 0.000338, klw: 0.2181, cost: -1.3673, recon: -1.4464, kl: 0.3628, train_time_taken: 14.7711
INFO:tensorflow:step: 11080, lr: 0.000337, klw: 0.2184, cost: -1.4514, recon: -1.5325, kl: 0.3710, train_time_taken: 15.9855
INFO:tensorflow:step: 11100, lr: 0.000336, klw: 0.2187, cost: -1.4086, recon: -1.4880, kl: 0.3628, train_time_taken: 14.6284
INFO:tensorflow:step: 11120, lr: 0.000336, klw: 0.2190, cost: -1.3496, recon: -1.4275, kl: 0.3561, train_time_taken: 14.7937
INFO:tensorflow:step: 11140, lr: 0.000335, klw: 0.2193, cost: -1.4798, recon: -1.5602, kl: 0.3665, train_time_taken: 14.7963
INFO:tensorflow:step: 11160, lr: 0.000334, klw: 0.2196, cost: -1.5714, recon: -1.6535, kl: 0.3736, train_time_taken: 14.7640
INFO:tensorflow:step: 11180, lr: 0.000334, klw: 0.2198, cost: -1.4446, recon: -1.5249, kl: 0.3654, train_time_taken: 16.0894
INFO:tensorflow:step: 11200, lr: 0.000333, klw: 0.2201, cost: -1.3921, recon: -1.4720, kl: 0.3630, train_time_taken: 14.7296
INFO:tensorflow:step: 11220, lr: 0.000332, klw: 0.2204, cost: -1.4268, recon: -1.5078, kl: 0.3672, train_time_taken: 15.4435
INFO:tensorflow:step: 11240, lr: 0.000332, klw: 0.2207, cost: -1.4092, recon: -1.4892, kl: 0.3628, train_time_taken: 15.5521
INFO:tensorflow:step: 11260, lr: 0.000331, klw: 0.2209, cost: -1.3688, recon: -1.4477, kl: 0.3572, train_time_taken: 14.7379
INFO:tensorflow:step: 11280, lr: 0.000330, klw: 0.2212, cost: -1.4577, recon: -1.5404, kl: 0.3736, train_time_taken: 16.6857
INFO:tensorflow:step: 11300, lr: 0.000330, klw: 0.2215, cost: -1.4688, recon: -1.5490, kl: 0.3617, train_time_taken: 15.3380
INFO:tensorflow:step: 11320, lr: 0.000329, klw: 0.2218, cost: -1.4618, recon: -1.5436, kl: 0.3690, train_time_taken: 14.7363
INFO:tensorflow:step: 11340, lr: 0.000329, klw: 0.2221, cost: -1.5134, recon: -1.5961, kl: 0.3724, train_time_taken: 14.6381
INFO:tensorflow:step: 11360, lr: 0.000328, klw: 0.2223, cost: -1.5252, recon: -1.6075, kl: 0.3704, train_time_taken: 14.4193
INFO:tensorflow:step: 11380, lr: 0.000327, klw: 0.2226, cost: -1.4616, recon: -1.5437, kl: 0.3687, train_time_taken: 14.9114
INFO:tensorflow:step: 11400, lr: 0.000327, klw: 0.2229, cost: -1.5632, recon: -1.6468, kl: 0.3753, train_time_taken: 15.8895
INFO:tensorflow:step: 11420, lr: 0.000326, klw: 0.2232, cost: -1.4906, recon: -1.5716, kl: 0.3627, train_time_taken: 14.7723
INFO:tensorflow:step: 11440, lr: 0.000325, klw: 0.2235, cost: -1.2958, recon: -1.3737, kl: 0.3486, train_time_taken: 14.8253
INFO:tensorflow:step: 11460, lr: 0.000325, klw: 0.2237, cost: -1.4735, recon: -1.5557, kl: 0.3676, train_time_taken: 14.8700
INFO:tensorflow:step: 11480, lr: 0.000324, klw: 0.2240, cost: -1.4413, recon: -1.5213, kl: 0.3570, train_time_taken: 14.9412
INFO:tensorflow:step: 11500, lr: 0.000323, klw: 0.2243, cost: -1.3697, recon: -1.4488, kl: 0.3526, train_time_taken: 15.9356
INFO:tensorflow:step: 11520, lr: 0.000323, klw: 0.2246, cost: -1.4626, recon: -1.5447, kl: 0.3657, train_time_taken: 14.5050
INFO:tensorflow:step: 11540, lr: 0.000322, klw: 0.2248, cost: -1.4749, recon: -1.5557, kl: 0.3596, train_time_taken: 14.6008
INFO:tensorflow:step: 11560, lr: 0.000322, klw: 0.2251, cost: -1.3696, recon: -1.4499, kl: 0.3567, train_time_taken: 14.9421
INFO:tensorflow:step: 11580, lr: 0.000321, klw: 0.2254, cost: -1.5350, recon: -1.6201, kl: 0.3775, train_time_taken: 14.8641
INFO:tensorflow:step: 11600, lr: 0.000320, klw: 0.2257, cost: -1.5478, recon: -1.6322, kl: 0.3740, train_time_taken: 15.7278
INFO:tensorflow:step: 11620, lr: 0.000320, klw: 0.2259, cost: -1.3606, recon: -1.4406, kl: 0.3544, train_time_taken: 15.0947
INFO:tensorflow:step: 11640, lr: 0.000319, klw: 0.2262, cost: -1.4935, recon: -1.5747, kl: 0.3591, train_time_taken: 16.2732
INFO:tensorflow:step: 11660, lr: 0.000318, klw: 0.2265, cost: -1.5039, recon: -1.5879, kl: 0.3708, train_time_taken: 14.5533
INFO:tensorflow:step: 11680, lr: 0.000318, klw: 0.2267, cost: -1.3876, recon: -1.4681, kl: 0.3552, train_time_taken: 15.5540
INFO:tensorflow:step: 11700, lr: 0.000317, klw: 0.2270, cost: -1.5839, recon: -1.6686, kl: 0.3733, train_time_taken: 15.0372
INFO:tensorflow:step: 11720, lr: 0.000317, klw: 0.2273, cost: -1.5026, recon: -1.5860, kl: 0.3669, train_time_taken: 15.6542
INFO:tensorflow:step: 11740, lr: 0.000316, klw: 0.2276, cost: -1.4379, recon: -1.5186, kl: 0.3546, train_time_taken: 14.7119
INFO:tensorflow:step: 11760, lr: 0.000315, klw: 0.2278, cost: -1.5381, recon: -1.6208, kl: 0.3627, train_time_taken: 14.9766
INFO:tensorflow:step: 11780, lr: 0.000315, klw: 0.2281, cost: -1.4173, recon: -1.4995, kl: 0.3605, train_time_taken: 14.8879
INFO:tensorflow:step: 11800, lr: 0.000314, klw: 0.2284, cost: -1.4756, recon: -1.5601, kl: 0.3700, train_time_taken: 14.6431
INFO:tensorflow:step: 11820, lr: 0.000314, klw: 0.2287, cost: -1.4068, recon: -1.4879, kl: 0.3549, train_time_taken: 16.0627
INFO:tensorflow:step: 11840, lr: 0.000313, klw: 0.2289, cost: -1.4753, recon: -1.5575, kl: 0.3589, train_time_taken: 14.6287
INFO:tensorflow:step: 11860, lr: 0.000312, klw: 0.2292, cost: -1.4558, recon: -1.5372, kl: 0.3551, train_time_taken: 14.8735
INFO:tensorflow:step: 11880, lr: 0.000312, klw: 0.2295, cost: -1.4728, recon: -1.5542, kl: 0.3548, train_time_taken: 14.8281
INFO:tensorflow:step: 11900, lr: 0.000311, klw: 0.2297, cost: -1.3466, recon: -1.4265, kl: 0.3478, train_time_taken: 14.5756
INFO:tensorflow:step: 11920, lr: 0.000311, klw: 0.2300, cost: -1.4142, recon: -1.4952, kl: 0.3519, train_time_taken: 15.8217
INFO:tensorflow:step: 11940, lr: 0.000310, klw: 0.2303, cost: -1.4383, recon: -1.5198, kl: 0.3542, train_time_taken: 15.0453
INFO:tensorflow:step: 11960, lr: 0.000309, klw: 0.2305, cost: -1.4135, recon: -1.4960, kl: 0.3581, train_time_taken: 14.9856
INFO:tensorflow:step: 11980, lr: 0.000309, klw: 0.2308, cost: -1.2850, recon: -1.3654, kl: 0.3483, train_time_taken: 14.5081
INFO:tensorflow:step: 12000, lr: 0.000308, klw: 0.2311, cost: -1.4705, recon: -1.5527, kl: 0.3557, train_time_taken: 14.4154
INFO:tensorflow:best_valid_cost: -1.6567, valid_cost: -1.6567, valid_recon: -1.6602, valid_kl: 0.3550, valid_time_taken: 0.8918
INFO:tensorflow:saving model /content/sketch_rnn/ckpt_layer_norm/vector.
INFO:tensorflow:global_step 12000.
INFO:tensorflow:time_taken_save 0.5335.
INFO:tensorflow:eval_cost: -1.6236, eval_recon: -1.6271, eval_kl: 0.3490, eval_time_taken: 0.7338
INFO:tensorflow:step: 12020, lr: 0.000308, klw: 0.2314, cost: -1.4757, recon: -1.5576, kl: 0.3541, train_time_taken: 15.3267
INFO:tensorflow:step: 12040, lr: 0.000307, klw: 0.2316, cost: -1.4714, recon: -1.5550, kl: 0.3611, train_time_taken: 16.6326
INFO:tensorflow:step: 12060, lr: 0.000306, klw: 0.2319, cost: -1.4245, recon: -1.5069, kl: 0.3551, train_time_taken: 15.0351
INFO:tensorflow:step: 12080, lr: 0.000306, klw: 0.2322, cost: -1.4461, recon: -1.5283, kl: 0.3541, train_time_taken: 15.4968
INFO:tensorflow:step: 12100, lr: 0.000305, klw: 0.2324, cost: -1.4884, recon: -1.5709, kl: 0.3550, train_time_taken: 14.6286
INFO:tensorflow:step: 12120, lr: 0.000305, klw: 0.2327, cost: -1.4371, recon: -1.5205, kl: 0.3587, train_time_taken: 14.5652
INFO:tensorflow:step: 12140, lr: 0.000304, klw: 0.2330, cost: -1.4649, recon: -1.5479, kl: 0.3566, train_time_taken: 16.2399
INFO:tensorflow:step: 12160, lr: 0.000303, klw: 0.2332, cost: -1.4788, recon: -1.5622, kl: 0.3578, train_time_taken: 14.7076
INFO:tensorflow:step: 12180, lr: 0.000303, klw: 0.2335, cost: -1.4797, recon: -1.5635, kl: 0.3587, train_time_taken: 14.9540
INFO:tensorflow:step: 12200, lr: 0.000302, klw: 0.2338, cost: -1.4798, recon: -1.5622, kl: 0.3524, train_time_taken: 14.6695
INFO:tensorflow:step: 12220, lr: 0.000302, klw: 0.2340, cost: -1.5062, recon: -1.5901, kl: 0.3584, train_time_taken: 14.8207
INFO:tensorflow:step: 12240, lr: 0.000301, klw: 0.2343, cost: -1.4338, recon: -1.5169, kl: 0.3549, train_time_taken: 16.3610
INFO:tensorflow:step: 12260, lr: 0.000301, klw: 0.2346, cost: -1.4864, recon: -1.5705, kl: 0.3586, train_time_taken: 14.6549
INFO:tensorflow:step: 12280, lr: 0.000300, klw: 0.2348, cost: -1.4526, recon: -1.5364, kl: 0.3566, train_time_taken: 14.7563
INFO:tensorflow:step: 12300, lr: 0.000299, klw: 0.2351, cost: -1.5186, recon: -1.6017, kl: 0.3533, train_time_taken: 14.8705
INFO:tensorflow:step: 12320, lr: 0.000299, klw: 0.2354, cost: -1.4713, recon: -1.5558, kl: 0.3590, train_time_taken: 14.8797
INFO:tensorflow:step: 12340, lr: 0.000298, klw: 0.2356, cost: -1.4126, recon: -1.4955, kl: 0.3518, train_time_taken: 15.3426
INFO:tensorflow:step: 12360, lr: 0.000298, klw: 0.2359, cost: -1.5308, recon: -1.6163, kl: 0.3623, train_time_taken: 15.3603
INFO:tensorflow:step: 12380, lr: 0.000297, klw: 0.2361, cost: -1.4837, recon: -1.5686, kl: 0.3596, train_time_taken: 14.7427
INFO:tensorflow:step: 12400, lr: 0.000296, klw: 0.2364, cost: -1.4752, recon: -1.5595, kl: 0.3563, train_time_taken: 14.6269
INFO:tensorflow:step: 12420, lr: 0.000296, klw: 0.2367, cost: -1.4127, recon: -1.4945, kl: 0.3453, train_time_taken: 14.5711
INFO:tensorflow:step: 12440, lr: 0.000295, klw: 0.2369, cost: -1.5452, recon: -1.6302, kl: 0.3588, train_time_taken: 14.7281
INFO:tensorflow:step: 12460, lr: 0.000295, klw: 0.2372, cost: -1.5126, recon: -1.5959, kl: 0.3508, train_time_taken: 16.5361
INFO:tensorflow:step: 12480, lr: 0.000294, klw: 0.2375, cost: -1.4676, recon: -1.5517, kl: 0.3544, train_time_taken: 15.5602
INFO:tensorflow:step: 12500, lr: 0.000294, klw: 0.2377, cost: -1.4291, recon: -1.5127, kl: 0.3515, train_time_taken: 14.6929
INFO:tensorflow:step: 12520, lr: 0.000293, klw: 0.2380, cost: -1.4349, recon: -1.5186, kl: 0.3517, train_time_taken: 14.7933
INFO:tensorflow:step: 12540, lr: 0.000292, klw: 0.2383, cost: -1.4960, recon: -1.5791, kl: 0.3486, train_time_taken: 14.5739
INFO:tensorflow:step: 12560, lr: 0.000292, klw: 0.2385, cost: -1.4480, recon: -1.5318, kl: 0.3511, train_time_taken: 15.8909
INFO:tensorflow:step: 12580, lr: 0.000291, klw: 0.2388, cost: -1.4669, recon: -1.5499, kl: 0.3480, train_time_taken: 14.4805
INFO:tensorflow:step: 12600, lr: 0.000291, klw: 0.2390, cost: -1.4214, recon: -1.5042, kl: 0.3461, train_time_taken: 14.5971
INFO:tensorflow:step: 12620, lr: 0.000290, klw: 0.2393, cost: -1.4852, recon: -1.5695, kl: 0.3521, train_time_taken: 14.8482
INFO:tensorflow:step: 12640, lr: 0.000290, klw: 0.2396, cost: -1.3711, recon: -1.4532, kl: 0.3425, train_time_taken: 14.5695
INFO:tensorflow:step: 12660, lr: 0.000289, klw: 0.2398, cost: -1.4226, recon: -1.5051, kl: 0.3443, train_time_taken: 14.7765
INFO:tensorflow:step: 12680, lr: 0.000289, klw: 0.2401, cost: -1.4816, recon: -1.5658, kl: 0.3507, train_time_taken: 15.2885
INFO:tensorflow:step: 12700, lr: 0.000288, klw: 0.2403, cost: -1.4586, recon: -1.5424, kl: 0.3484, train_time_taken: 14.8013
INFO:tensorflow:step: 12720, lr: 0.000287, klw: 0.2406, cost: -1.4240, recon: -1.5062, kl: 0.3419, train_time_taken: 14.7029
INFO:tensorflow:step: 12740, lr: 0.000287, klw: 0.2409, cost: -1.4922, recon: -1.5781, kl: 0.3565, train_time_taken: 15.0691
INFO:tensorflow:step: 12760, lr: 0.000286, klw: 0.2411, cost: -1.4651, recon: -1.5497, kl: 0.3509, train_time_taken: 14.5802
INFO:tensorflow:step: 12780, lr: 0.000286, klw: 0.2414, cost: -1.3925, recon: -1.4760, kl: 0.3459, train_time_taken: 15.9470
INFO:tensorflow:step: 12800, lr: 0.000285, klw: 0.2416, cost: -1.4355, recon: -1.5198, kl: 0.3488, train_time_taken: 14.5264
INFO:tensorflow:step: 12820, lr: 0.000285, klw: 0.2419, cost: -1.3920, recon: -1.4745, kl: 0.3411, train_time_taken: 14.6707
INFO:tensorflow:step: 12840, lr: 0.000284, klw: 0.2421, cost: -1.4338, recon: -1.5165, kl: 0.3416, train_time_taken: 14.6982
INFO:tensorflow:step: 12860, lr: 0.000284, klw: 0.2424, cost: -1.4912, recon: -1.5747, kl: 0.3444, train_time_taken: 14.9840
INFO:tensorflow:step: 12880, lr: 0.000283, klw: 0.2427, cost: -1.4218, recon: -1.5036, kl: 0.3374, train_time_taken: 17.1541
INFO:tensorflow:step: 12900, lr: 0.000283, klw: 0.2429, cost: -1.5065, recon: -1.5919, kl: 0.3518, train_time_taken: 15.3200
INFO:tensorflow:step: 12920, lr: 0.000282, klw: 0.2432, cost: -1.3314, recon: -1.4148, kl: 0.3429, train_time_taken: 14.6949
INFO:tensorflow:step: 12940, lr: 0.000281, klw: 0.2434, cost: -1.5840, recon: -1.6712, kl: 0.3583, train_time_taken: 14.5941
INFO:tensorflow:step: 12960, lr: 0.000281, klw: 0.2437, cost: -1.3805, recon: -1.4638, kl: 0.3418, train_time_taken: 14.4447
INFO:tensorflow:step: 12980, lr: 0.000280, klw: 0.2439, cost: -1.4418, recon: -1.5257, kl: 0.3439, train_time_taken: 15.0791
INFO:tensorflow:step: 13000, lr: 0.000280, klw: 0.2442, cost: -1.3626, recon: -1.4463, kl: 0.3427, train_time_taken: 15.7122
INFO:tensorflow:step: 13020, lr: 0.000279, klw: 0.2445, cost: -1.4842, recon: -1.5694, kl: 0.3485, train_time_taken: 14.7124
INFO:tensorflow:step: 13040, lr: 0.000279, klw: 0.2447, cost: -1.4635, recon: -1.5492, kl: 0.3503, train_time_taken: 14.6090
INFO:tensorflow:step: 13060, lr: 0.000278, klw: 0.2450, cost: -1.4263, recon: -1.5105, kl: 0.3440, train_time_taken: 14.7935
INFO:tensorflow:step: 13080, lr: 0.000278, klw: 0.2452, cost: -1.5605, recon: -1.6463, kl: 0.3499, train_time_taken: 14.7969
INFO:tensorflow:step: 13100, lr: 0.000277, klw: 0.2455, cost: -1.5430, recon: -1.6292, kl: 0.3511, train_time_taken: 15.9065
INFO:tensorflow:step: 13120, lr: 0.000277, klw: 0.2457, cost: -1.5647, recon: -1.6513, kl: 0.3522, train_time_taken: 14.6788
INFO:tensorflow:step: 13140, lr: 0.000276, klw: 0.2460, cost: -1.5319, recon: -1.6180, kl: 0.3501, train_time_taken: 14.5264
INFO:tensorflow:step: 13160, lr: 0.000276, klw: 0.2462, cost: -1.4194, recon: -1.5036, kl: 0.3418, train_time_taken: 14.4168
INFO:tensorflow:step: 13180, lr: 0.000275, klw: 0.2465, cost: -1.4049, recon: -1.4894, kl: 0.3427, train_time_taken: 14.7338
INFO:tensorflow:step: 13200, lr: 0.000274, klw: 0.2467, cost: -1.5238, recon: -1.6103, kl: 0.3503, train_time_taken: 15.4413
INFO:tensorflow:step: 13220, lr: 0.000274, klw: 0.2470, cost: -1.5314, recon: -1.6170, kl: 0.3467, train_time_taken: 15.3395
INFO:tensorflow:step: 13240, lr: 0.000273, klw: 0.2473, cost: -1.4947, recon: -1.5804, kl: 0.3469, train_time_taken: 14.7009
INFO:tensorflow:step: 13260, lr: 0.000273, klw: 0.2475, cost: -1.5439, recon: -1.6292, kl: 0.3448, train_time_taken: 14.5102
INFO:tensorflow:step: 13280, lr: 0.000272, klw: 0.2478, cost: -1.4764, recon: -1.5619, kl: 0.3449, train_time_taken: 15.4559
INFO:tensorflow:step: 13300, lr: 0.000272, klw: 0.2480, cost: -1.5352, recon: -1.6222, kl: 0.3507, train_time_taken: 16.3367
INFO:tensorflow:step: 13320, lr: 0.000271, klw: 0.2483, cost: -1.4769, recon: -1.5624, kl: 0.3445, train_time_taken: 15.8971
INFO:tensorflow:step: 13340, lr: 0.000271, klw: 0.2485, cost: -1.4403, recon: -1.5243, kl: 0.3380, train_time_taken: 14.5712
INFO:tensorflow:step: 13360, lr: 0.000270, klw: 0.2488, cost: -1.5697, recon: -1.6568, kl: 0.3502, train_time_taken: 14.7615
INFO:tensorflow:step: 13380, lr: 0.000270, klw: 0.2490, cost: -1.3293, recon: -1.4130, kl: 0.3360, train_time_taken: 14.7041
INFO:tensorflow:step: 13400, lr: 0.000269, klw: 0.2493, cost: -1.5790, recon: -1.6661, kl: 0.3495, train_time_taken: 14.8026
INFO:tensorflow:step: 13420, lr: 0.000269, klw: 0.2495, cost: -1.4366, recon: -1.5210, kl: 0.3384, train_time_taken: 15.9022
INFO:tensorflow:step: 13440, lr: 0.000268, klw: 0.2498, cost: -1.4888, recon: -1.5751, kl: 0.3453, train_time_taken: 14.6942
INFO:tensorflow:step: 13460, lr: 0.000268, klw: 0.2500, cost: -1.4879, recon: -1.5743, kl: 0.3458, train_time_taken: 14.7291
INFO:tensorflow:step: 13480, lr: 0.000267, klw: 0.2503, cost: -1.4638, recon: -1.5506, kl: 0.3470, train_time_taken: 14.8540
INFO:tensorflow:step: 13500, lr: 0.000267, klw: 0.2505, cost: -1.4307, recon: -1.5159, kl: 0.3403, train_time_taken: 14.7843
INFO:tensorflow:step: 13520, lr: 0.000266, klw: 0.2508, cost: -1.6069, recon: -1.6955, kl: 0.3533, train_time_taken: 15.5524
INFO:tensorflow:step: 13540, lr: 0.000266, klw: 0.2510, cost: -1.5568, recon: -1.6434, kl: 0.3449, train_time_taken: 15.0965
INFO:tensorflow:step: 13560, lr: 0.000265, klw: 0.2513, cost: -1.4980, recon: -1.5847, kl: 0.3453, train_time_taken: 14.6659
INFO:tensorflow:step: 13580, lr: 0.000265, klw: 0.2515, cost: -1.4449, recon: -1.5301, kl: 0.3388, train_time_taken: 14.9138
INFO:tensorflow:step: 13600, lr: 0.000264, klw: 0.2518, cost: -1.5090, recon: -1.5947, kl: 0.3404, train_time_taken: 14.6214
INFO:tensorflow:step: 13620, lr: 0.000264, klw: 0.2520, cost: -1.4805, recon: -1.5656, kl: 0.3380, train_time_taken: 14.7473
INFO:tensorflow:step: 13640, lr: 0.000263, klw: 0.2523, cost: -1.5538, recon: -1.6409, kl: 0.3451, train_time_taken: 16.2475
INFO:tensorflow:step: 13660, lr: 0.000263, klw: 0.2525, cost: -1.4156, recon: -1.5008, kl: 0.3375, train_time_taken: 14.6469
INFO:tensorflow:step: 13680, lr: 0.000262, klw: 0.2528, cost: -1.5186, recon: -1.6052, kl: 0.3427, train_time_taken: 14.7468
INFO:tensorflow:step: 13700, lr: 0.000262, klw: 0.2530, cost: -1.5118, recon: -1.5985, kl: 0.3427, train_time_taken: 16.5888
INFO:tensorflow:step: 13720, lr: 0.000261, klw: 0.2532, cost: -1.5006, recon: -1.5869, kl: 0.3408, train_time_taken: 14.8597
INFO:tensorflow:step: 13740, lr: 0.000261, klw: 0.2535, cost: -1.5240, recon: -1.6092, kl: 0.3362, train_time_taken: 16.1751
INFO:tensorflow:step: 13760, lr: 0.000260, klw: 0.2537, cost: -1.5177, recon: -1.6049, kl: 0.3439, train_time_taken: 14.8766
INFO:tensorflow:step: 13780, lr: 0.000260, klw: 0.2540, cost: -1.4649, recon: -1.5529, kl: 0.3463, train_time_taken: 14.6154
INFO:tensorflow:step: 13800, lr: 0.000259, klw: 0.2542, cost: -1.5365, recon: -1.6253, kl: 0.3496, train_time_taken: 14.7062
INFO:tensorflow:step: 13820, lr: 0.000259, klw: 0.2545, cost: -1.4304, recon: -1.5149, kl: 0.3323, train_time_taken: 14.7793
INFO:tensorflow:step: 13840, lr: 0.000258, klw: 0.2547, cost: -1.4786, recon: -1.5655, kl: 0.3412, train_time_taken: 16.2152
INFO:tensorflow:step: 13860, lr: 0.000258, klw: 0.2550, cost: -1.6016, recon: -1.6900, kl: 0.3467, train_time_taken: 15.0815
INFO:tensorflow:step: 13880, lr: 0.000257, klw: 0.2552, cost: -1.4959, recon: -1.5834, kl: 0.3430, train_time_taken: 14.7333
INFO:tensorflow:step: 13900, lr: 0.000257, klw: 0.2555, cost: -1.4923, recon: -1.5780, kl: 0.3357, train_time_taken: 14.8133
INFO:tensorflow:step: 13920, lr: 0.000256, klw: 0.2557, cost: -1.4588, recon: -1.5452, kl: 0.3379, train_time_taken: 14.7430
INFO:tensorflow:step: 13940, lr: 0.000256, klw: 0.2559, cost: -1.4903, recon: -1.5763, kl: 0.3363, train_time_taken: 14.7890
INFO:tensorflow:step: 13960, lr: 0.000255, klw: 0.2562, cost: -1.5562, recon: -1.6441, kl: 0.3433, train_time_taken: 16.0514
INFO:tensorflow:step: 13980, lr: 0.000255, klw: 0.2564, cost: -1.5039, recon: -1.5912, kl: 0.3405, train_time_taken: 14.9444
INFO:tensorflow:step: 14000, lr: 0.000254, klw: 0.2567, cost: -1.4648, recon: -1.5517, kl: 0.3385, train_time_taken: 14.9175
INFO:tensorflow:best_valid_cost: -1.6825, valid_cost: -1.6825, valid_recon: -1.6859, valid_kl: 0.3395, valid_time_taken: 0.8951
INFO:tensorflow:saving model /content/sketch_rnn/ckpt_layer_norm/vector.
INFO:tensorflow:global_step 14000.
INFO:tensorflow:time_taken_save 0.5383.
INFO:tensorflow:eval_cost: -1.6448, eval_recon: -1.6482, eval_kl: 0.3340, eval_time_taken: 0.7646
INFO:tensorflow:step: 14020, lr: 0.000254, klw: 0.2569, cost: -1.4452, recon: -1.5307, kl: 0.3327, train_time_taken: 14.7010
INFO:tensorflow:step: 14040, lr: 0.000253, klw: 0.2572, cost: -1.4494, recon: -1.5347, kl: 0.3315, train_time_taken: 14.4119
INFO:tensorflow:step: 14060, lr: 0.000253, klw: 0.2574, cost: -1.4880, recon: -1.5754, kl: 0.3397, train_time_taken: 15.7497
INFO:tensorflow:step: 14080, lr: 0.000252, klw: 0.2576, cost: -1.5587, recon: -1.6466, kl: 0.3412, train_time_taken: 15.0190
INFO:tensorflow:step: 14100, lr: 0.000252, klw: 0.2579, cost: -1.3932, recon: -1.4794, kl: 0.3340, train_time_taken: 15.8415
INFO:tensorflow:step: 14120, lr: 0.000251, klw: 0.2581, cost: -1.5150, recon: -1.6041, kl: 0.3452, train_time_taken: 15.3776
INFO:tensorflow:step: 14140, lr: 0.000251, klw: 0.2584, cost: -1.4541, recon: -1.5411, kl: 0.3367, train_time_taken: 14.8515
INFO:tensorflow:step: 14160, lr: 0.000250, klw: 0.2586, cost: -1.3637, recon: -1.4495, kl: 0.3320, train_time_taken: 16.0964
INFO:tensorflow:step: 14180, lr: 0.000250, klw: 0.2589, cost: -1.5447, recon: -1.6329, kl: 0.3409, train_time_taken: 14.6207
INFO:tensorflow:step: 14200, lr: 0.000249, klw: 0.2591, cost: -1.4447, recon: -1.5308, kl: 0.3325, train_time_taken: 14.7637
INFO:tensorflow:step: 14220, lr: 0.000249, klw: 0.2593, cost: -1.5767, recon: -1.6660, kl: 0.3446, train_time_taken: 14.7675
INFO:tensorflow:step: 14240, lr: 0.000248, klw: 0.2596, cost: -1.4689, recon: -1.5580, kl: 0.3434, train_time_taken: 14.9870
INFO:tensorflow:step: 14260, lr: 0.000248, klw: 0.2598, cost: -1.4604, recon: -1.5470, kl: 0.3334, train_time_taken: 15.4584
INFO:tensorflow:step: 14280, lr: 0.000247, klw: 0.2601, cost: -1.5355, recon: -1.6238, kl: 0.3396, train_time_taken: 15.5343
INFO:tensorflow:step: 14300, lr: 0.000247, klw: 0.2603, cost: -1.4612, recon: -1.5490, kl: 0.3372, train_time_taken: 14.5413
INFO:tensorflow:step: 14320, lr: 0.000246, klw: 0.2605, cost: -1.5515, recon: -1.6403, kl: 0.3409, train_time_taken: 14.7625
INFO:tensorflow:step: 14340, lr: 0.000246, klw: 0.2608, cost: -1.4894, recon: -1.5765, kl: 0.3342, train_time_taken: 14.7398
INFO:tensorflow:step: 14360, lr: 0.000245, klw: 0.2610, cost: -1.5734, recon: -1.6619, kl: 0.3389, train_time_taken: 14.6986
INFO:tensorflow:step: 14380, lr: 0.000245, klw: 0.2613, cost: -1.4035, recon: -1.4901, kl: 0.3314, train_time_taken: 16.1286
INFO:tensorflow:step: 14400, lr: 0.000245, klw: 0.2615, cost: -1.4666, recon: -1.5539, kl: 0.3336, train_time_taken: 14.7099
INFO:tensorflow:step: 14420, lr: 0.000244, klw: 0.2617, cost: -1.5305, recon: -1.6204, kl: 0.3433, train_time_taken: 14.8536
INFO:tensorflow:step: 14440, lr: 0.000244, klw: 0.2620, cost: -1.4626, recon: -1.5502, kl: 0.3343, train_time_taken: 14.6788
INFO:tensorflow:step: 14460, lr: 0.000243, klw: 0.2622, cost: -1.5388, recon: -1.6279, kl: 0.3399, train_time_taken: 14.7702
INFO:tensorflow:step: 14480, lr: 0.000243, klw: 0.2624, cost: -1.4793, recon: -1.5682, kl: 0.3386, train_time_taken: 16.1763
INFO:tensorflow:step: 14500, lr: 0.000242, klw: 0.2627, cost: -1.5068, recon: -1.5946, kl: 0.3341, train_time_taken: 15.0017
INFO:tensorflow:step: 14520, lr: 0.000242, klw: 0.2629, cost: -1.5147, recon: -1.6033, kl: 0.3369, train_time_taken: 16.6599
INFO:tensorflow:step: 14540, lr: 0.000241, klw: 0.2632, cost: -1.4852, recon: -1.5737, kl: 0.3365, train_time_taken: 14.8557
INFO:tensorflow:step: 14560, lr: 0.000241, klw: 0.2634, cost: -1.5573, recon: -1.6465, kl: 0.3387, train_time_taken: 14.9987
INFO:tensorflow:step: 14580, lr: 0.000240, klw: 0.2636, cost: -1.5180, recon: -1.6078, kl: 0.3407, train_time_taken: 15.4870
INFO:tensorflow:step: 14600, lr: 0.000240, klw: 0.2639, cost: -1.4883, recon: -1.5761, kl: 0.3328, train_time_taken: 15.4234
INFO:tensorflow:step: 14620, lr: 0.000239, klw: 0.2641, cost: -1.4570, recon: -1.5461, kl: 0.3377, train_time_taken: 14.4602
INFO:tensorflow:step: 14640, lr: 0.000239, klw: 0.2643, cost: -1.4806, recon: -1.5666, kl: 0.3254, train_time_taken: 14.7144
INFO:tensorflow:step: 14660, lr: 0.000239, klw: 0.2646, cost: -1.4936, recon: -1.5825, kl: 0.3357, train_time_taken: 14.7173
INFO:tensorflow:step: 14680, lr: 0.000238, klw: 0.2648, cost: -1.5491, recon: -1.6391, kl: 0.3398, train_time_taken: 14.7341
INFO:tensorflow:step: 14700, lr: 0.000238, klw: 0.2650, cost: -1.4324, recon: -1.5210, kl: 0.3344, train_time_taken: 15.7986
INFO:tensorflow:step: 14720, lr: 0.000237, klw: 0.2653, cost: -1.4341, recon: -1.5227, kl: 0.3337, train_time_taken: 14.6301
INFO:tensorflow:step: 14740, lr: 0.000237, klw: 0.2655, cost: -1.3884, recon: -1.4752, kl: 0.3267, train_time_taken: 14.5350
INFO:tensorflow:step: 14760, lr: 0.000236, klw: 0.2658, cost: -1.4527, recon: -1.5417, kl: 0.3349, train_time_taken: 15.1109
INFO:tensorflow:step: 14780, lr: 0.000236, klw: 0.2660, cost: -1.4586, recon: -1.5488, kl: 0.3392, train_time_taken: 14.6921
INFO:tensorflow:step: 14800, lr: 0.000235, klw: 0.2662, cost: -1.4856, recon: -1.5738, kl: 0.3313, train_time_taken: 16.0768
INFO:tensorflow:step: 14820, lr: 0.000235, klw: 0.2665, cost: -1.5552, recon: -1.6449, kl: 0.3366, train_time_taken: 14.7790
INFO:tensorflow:step: 14840, lr: 0.000234, klw: 0.2667, cost: -1.6034, recon: -1.6947, kl: 0.3424, train_time_taken: 14.7583
INFO:tensorflow:step: 14860, lr: 0.000234, klw: 0.2669, cost: -1.4703, recon: -1.5596, kl: 0.3348, train_time_taken: 14.4780
INFO:tensorflow:step: 14880, lr: 0.000234, klw: 0.2672, cost: -1.5644, recon: -1.6557, kl: 0.3416, train_time_taken: 14.5832
INFO:tensorflow:step: 14900, lr: 0.000233, klw: 0.2674, cost: -1.4229, recon: -1.5100, kl: 0.3257, train_time_taken: 14.8115
INFO:tensorflow:step: 14920, lr: 0.000233, klw: 0.2676, cost: -1.4319, recon: -1.5199, kl: 0.3288, train_time_taken: 16.5503
INFO:tensorflow:step: 14940, lr: 0.000232, klw: 0.2678, cost: -1.5417, recon: -1.6307, kl: 0.3325, train_time_taken: 15.5636
INFO:tensorflow:step: 14960, lr: 0.000232, klw: 0.2681, cost: -1.4515, recon: -1.5395, kl: 0.3280, train_time_taken: 14.7836
INFO:tensorflow:step: 14980, lr: 0.000231, klw: 0.2683, cost: -1.5340, recon: -1.6252, kl: 0.3399, train_time_taken: 14.6933
INFO:tensorflow:step: 15000, lr: 0.000231, klw: 0.2685, cost: -1.5062, recon: -1.5953, kl: 0.3320, train_time_taken: 14.7069
INFO:tensorflow:step: 15020, lr: 0.000230, klw: 0.2688, cost: -1.4447, recon: -1.5334, kl: 0.3302, train_time_taken: 16.2774
INFO:tensorflow:step: 15040, lr: 0.000230, klw: 0.2690, cost: -1.6227, recon: -1.7129, kl: 0.3353, train_time_taken: 14.7887
INFO:tensorflow:step: 15060, lr: 0.000230, klw: 0.2692, cost: -1.4792, recon: -1.5701, kl: 0.3375, train_time_taken: 14.8523
INFO:tensorflow:step: 15080, lr: 0.000229, klw: 0.2695, cost: -1.4434, recon: -1.5325, kl: 0.3309, train_time_taken: 15.0063
INFO:tensorflow:step: 15100, lr: 0.000229, klw: 0.2697, cost: -1.5118, recon: -1.6023, kl: 0.3356, train_time_taken: 14.7709
INFO:tensorflow:step: 15120, lr: 0.000228, klw: 0.2699, cost: -1.5903, recon: -1.6810, kl: 0.3361, train_time_taken: 16.2374
INFO:tensorflow:step: 15140, lr: 0.000228, klw: 0.2702, cost: -1.5175, recon: -1.6080, kl: 0.3350, train_time_taken: 14.7494
INFO:tensorflow:step: 15160, lr: 0.000227, klw: 0.2704, cost: -1.5001, recon: -1.5907, kl: 0.3350, train_time_taken: 14.6547
INFO:tensorflow:step: 15180, lr: 0.000227, klw: 0.2706, cost: -1.4771, recon: -1.5662, kl: 0.3293, train_time_taken: 14.7182
INFO:tensorflow:step: 15200, lr: 0.000227, klw: 0.2708, cost: -1.4731, recon: -1.5623, kl: 0.3291, train_time_taken: 14.6989
INFO:tensorflow:step: 15220, lr: 0.000226, klw: 0.2711, cost: -1.4327, recon: -1.5207, kl: 0.3246, train_time_taken: 15.5303
INFO:tensorflow:step: 15240, lr: 0.000226, klw: 0.2713, cost: -1.5131, recon: -1.6051, kl: 0.3390, train_time_taken: 15.8436
INFO:tensorflow:step: 15260, lr: 0.000225, klw: 0.2715, cost: -1.4753, recon: -1.5652, kl: 0.3311, train_time_taken: 14.7973
INFO:tensorflow:step: 15280, lr: 0.000225, klw: 0.2718, cost: -1.4813, recon: -1.5722, kl: 0.3345, train_time_taken: 14.9488
INFO:tensorflow:step: 15300, lr: 0.000224, klw: 0.2720, cost: -1.5516, recon: -1.6428, kl: 0.3353, train_time_taken: 15.1132
INFO:tensorflow:step: 15320, lr: 0.000224, klw: 0.2722, cost: -1.5629, recon: -1.6543, kl: 0.3357, train_time_taken: 15.8649
INFO:tensorflow:step: 15340, lr: 0.000223, klw: 0.2724, cost: -1.4158, recon: -1.5061, kl: 0.3312, train_time_taken: 16.8858
INFO:tensorflow:step: 15360, lr: 0.000223, klw: 0.2727, cost: -1.5285, recon: -1.6196, kl: 0.3341, train_time_taken: 14.6923
INFO:tensorflow:step: 15380, lr: 0.000223, klw: 0.2729, cost: -1.4662, recon: -1.5570, kl: 0.3330, train_time_taken: 14.5791
INFO:tensorflow:step: 15400, lr: 0.000222, klw: 0.2731, cost: -1.4911, recon: -1.5809, kl: 0.3288, train_time_taken: 14.7061
INFO:tensorflow:step: 15420, lr: 0.000222, klw: 0.2734, cost: -1.5356, recon: -1.6267, kl: 0.3331, train_time_taken: 14.7322
INFO:tensorflow:step: 15440, lr: 0.000221, klw: 0.2736, cost: -1.4523, recon: -1.5420, kl: 0.3280, train_time_taken: 16.2804
INFO:tensorflow:step: 15460, lr: 0.000221, klw: 0.2738, cost: -1.4637, recon: -1.5530, kl: 0.3262, train_time_taken: 14.7591
INFO:tensorflow:step: 15480, lr: 0.000221, klw: 0.2740, cost: -1.5160, recon: -1.6073, kl: 0.3331, train_time_taken: 14.7247
INFO:tensorflow:step: 15500, lr: 0.000220, klw: 0.2743, cost: -1.4720, recon: -1.5600, kl: 0.3206, train_time_taken: 14.6426
INFO:tensorflow:step: 15520, lr: 0.000220, klw: 0.2745, cost: -1.5106, recon: -1.6007, kl: 0.3283, train_time_taken: 14.8338
INFO:tensorflow:step: 15540, lr: 0.000219, klw: 0.2747, cost: -1.5130, recon: -1.6041, kl: 0.3316, train_time_taken: 15.5307
INFO:tensorflow:step: 15560, lr: 0.000219, klw: 0.2749, cost: -1.5016, recon: -1.5927, kl: 0.3315, train_time_taken: 15.4700
INFO:tensorflow:step: 15580, lr: 0.000218, klw: 0.2752, cost: -1.5349, recon: -1.6268, kl: 0.3341, train_time_taken: 14.5319
INFO:tensorflow:step: 15600, lr: 0.000218, klw: 0.2754, cost: -1.4005, recon: -1.4903, kl: 0.3260, train_time_taken: 14.9183
INFO:tensorflow:step: 15620, lr: 0.000218, klw: 0.2756, cost: -1.5935, recon: -1.6853, kl: 0.3330, train_time_taken: 14.6742
INFO:tensorflow:step: 15640, lr: 0.000217, klw: 0.2758, cost: -1.5039, recon: -1.5949, kl: 0.3298, train_time_taken: 14.8040
INFO:tensorflow:step: 15660, lr: 0.000217, klw: 0.2761, cost: -1.5178, recon: -1.6091, kl: 0.3308, train_time_taken: 16.1370
INFO:tensorflow:step: 15680, lr: 0.000216, klw: 0.2763, cost: -1.4736, recon: -1.5641, kl: 0.3275, train_time_taken: 14.6174
INFO:tensorflow:step: 15700, lr: 0.000216, klw: 0.2765, cost: -1.5309, recon: -1.6234, kl: 0.3343, train_time_taken: 14.4889
INFO:tensorflow:step: 15720, lr: 0.000216, klw: 0.2767, cost: -1.5756, recon: -1.6664, kl: 0.3281, train_time_taken: 15.7363
INFO:tensorflow:step: 15740, lr: 0.000215, klw: 0.2770, cost: -1.4376, recon: -1.5292, kl: 0.3306, train_time_taken: 14.9854
INFO:tensorflow:step: 15760, lr: 0.000215, klw: 0.2772, cost: -1.5368, recon: -1.6286, kl: 0.3314, train_time_taken: 16.8270
INFO:tensorflow:step: 15780, lr: 0.000214, klw: 0.2774, cost: -1.4455, recon: -1.5360, kl: 0.3265, train_time_taken: 14.6729
INFO:tensorflow:step: 15800, lr: 0.000214, klw: 0.2776, cost: -1.5713, recon: -1.6657, kl: 0.3404, train_time_taken: 14.9042
INFO:tensorflow:step: 15820, lr: 0.000213, klw: 0.2778, cost: -1.5186, recon: -1.6117, kl: 0.3350, train_time_taken: 14.6833
INFO:tensorflow:step: 15840, lr: 0.000213, klw: 0.2781, cost: -1.4372, recon: -1.5287, kl: 0.3291, train_time_taken: 15.0450
INFO:tensorflow:step: 15860, lr: 0.000213, klw: 0.2783, cost: -1.5128, recon: -1.6046, kl: 0.3299, train_time_taken: 15.5932
INFO:tensorflow:step: 15880, lr: 0.000212, klw: 0.2785, cost: -1.4140, recon: -1.5050, kl: 0.3266, train_time_taken: 15.3736
INFO:tensorflow:step: 15900, lr: 0.000212, klw: 0.2787, cost: -1.5021, recon: -1.5956, kl: 0.3354, train_time_taken: 14.7104
INFO:tensorflow:step: 15920, lr: 0.000211, klw: 0.2790, cost: -1.5597, recon: -1.6517, kl: 0.3299, train_time_taken: 14.8563
INFO:tensorflow:step: 15940, lr: 0.000211, klw: 0.2792, cost: -1.4415, recon: -1.5344, kl: 0.3329, train_time_taken: 14.6934
INFO:tensorflow:step: 15960, lr: 0.000211, klw: 0.2794, cost: -1.3837, recon: -1.4742, kl: 0.3237, train_time_taken: 15.0193
INFO:tensorflow:step: 15980, lr: 0.000210, klw: 0.2796, cost: -1.4884, recon: -1.5785, kl: 0.3225, train_time_taken: 16.0547
INFO:tensorflow:step: 16000, lr: 0.000210, klw: 0.2798, cost: -1.4010, recon: -1.4909, kl: 0.3212, train_time_taken: 14.9111
INFO:tensorflow:best_valid_cost: -1.7078, valid_cost: -1.7078, valid_recon: -1.7111, valid_kl: 0.3287, valid_time_taken: 0.8947
INFO:tensorflow:saving model /content/sketch_rnn/ckpt_layer_norm/vector.
INFO:tensorflow:global_step 16000.
INFO:tensorflow:time_taken_save 0.5484.
INFO:tensorflow:eval_cost: -1.6748, eval_recon: -1.6780, eval_kl: 0.3232, eval_time_taken: 0.7559
INFO:tensorflow:step: 16020, lr: 0.000209, klw: 0.2801, cost: -1.5712, recon: -1.6641, kl: 0.3314, train_time_taken: 15.2387
INFO:tensorflow:step: 16040, lr: 0.000209, klw: 0.2803, cost: -1.4607, recon: -1.5512, kl: 0.3231, train_time_taken: 14.7154
INFO:tensorflow:step: 16060, lr: 0.000209, klw: 0.2805, cost: -1.5146, recon: -1.6080, kl: 0.3330, train_time_taken: 14.6833
INFO:tensorflow:step: 16080, lr: 0.000208, klw: 0.2807, cost: -1.5866, recon: -1.6798, kl: 0.3321, train_time_taken: 16.1342
INFO:tensorflow:step: 16100, lr: 0.000208, klw: 0.2809, cost: -1.4958, recon: -1.5891, kl: 0.3319, train_time_taken: 14.9036
INFO:tensorflow:step: 16120, lr: 0.000207, klw: 0.2812, cost: -1.4451, recon: -1.5357, kl: 0.3223, train_time_taken: 15.4616
INFO:tensorflow:step: 16140, lr: 0.000207, klw: 0.2814, cost: -1.5075, recon: -1.6010, kl: 0.3322, train_time_taken: 15.0243
INFO:tensorflow:step: 16160, lr: 0.000207, klw: 0.2816, cost: -1.5958, recon: -1.6891, kl: 0.3311, train_time_taken: 16.2509
INFO:tensorflow:step: 16180, lr: 0.000206, klw: 0.2818, cost: -1.5440, recon: -1.6370, kl: 0.3300, train_time_taken: 16.2873
INFO:tensorflow:step: 16200, lr: 0.000206, klw: 0.2820, cost: -1.4900, recon: -1.5828, kl: 0.3288, train_time_taken: 14.9751
INFO:tensorflow:step: 16220, lr: 0.000206, klw: 0.2822, cost: -1.4821, recon: -1.5739, kl: 0.3251, train_time_taken: 14.8749
INFO:tensorflow:step: 16240, lr: 0.000205, klw: 0.2825, cost: -1.5488, recon: -1.6415, kl: 0.3282, train_time_taken: 14.7788
INFO:tensorflow:step: 16260, lr: 0.000205, klw: 0.2827, cost: -1.5140, recon: -1.6075, kl: 0.3309, train_time_taken: 15.1101
INFO:tensorflow:step: 16280, lr: 0.000204, klw: 0.2829, cost: -1.5956, recon: -1.6904, kl: 0.3350, train_time_taken: 15.6481
INFO:tensorflow:step: 16300, lr: 0.000204, klw: 0.2831, cost: -1.5473, recon: -1.6407, kl: 0.3299, train_time_taken: 14.9711
INFO:tensorflow:step: 16320, lr: 0.000204, klw: 0.2833, cost: -1.5106, recon: -1.6024, kl: 0.3241, train_time_taken: 14.8720
INFO:tensorflow:step: 16340, lr: 0.000203, klw: 0.2835, cost: -1.5326, recon: -1.6268, kl: 0.3324, train_time_taken: 14.6974
INFO:tensorflow:step: 16360, lr: 0.000203, klw: 0.2838, cost: -1.4261, recon: -1.5169, kl: 0.3203, train_time_taken: 14.8308
INFO:tensorflow:step: 16380, lr: 0.000202, klw: 0.2840, cost: -1.5260, recon: -1.6194, kl: 0.3289, train_time_taken: 15.2641
INFO:tensorflow:step: 16400, lr: 0.000202, klw: 0.2842, cost: -1.5614, recon: -1.6547, kl: 0.3284, train_time_taken: 15.9314
INFO:tensorflow:step: 16420, lr: 0.000202, klw: 0.2844, cost: -1.4272, recon: -1.5183, kl: 0.3206, train_time_taken: 14.7033
INFO:tensorflow:step: 16440, lr: 0.000201, klw: 0.2846, cost: -1.4936, recon: -1.5849, kl: 0.3207, train_time_taken: 14.7780
INFO:tensorflow:step: 16460, lr: 0.000201, klw: 0.2848, cost: -1.6541, recon: -1.7498, kl: 0.3361, train_time_taken: 15.1175
INFO:tensorflow:step: 16480, lr: 0.000200, klw: 0.2851, cost: -1.5102, recon: -1.6030, kl: 0.3255, train_time_taken: 14.6707
INFO:tensorflow:step: 16500, lr: 0.000200, klw: 0.2853, cost: -1.5514, recon: -1.6449, kl: 0.3279, train_time_taken: 16.0077
INFO:tensorflow:step: 16520, lr: 0.000200, klw: 0.2855, cost: -1.4485, recon: -1.5400, kl: 0.3205, train_time_taken: 15.4991
INFO:tensorflow:step: 16540, lr: 0.000199, klw: 0.2857, cost: -1.4672, recon: -1.5586, kl: 0.3199, train_time_taken: 14.7428
INFO:tensorflow:step: 16560, lr: 0.000199, klw: 0.2859, cost: -1.5890, recon: -1.6843, kl: 0.3332, train_time_taken: 15.9673
INFO:tensorflow:step: 16580, lr: 0.000199, klw: 0.2861, cost: -1.5766, recon: -1.6709, kl: 0.3294, train_time_taken: 15.5709
INFO:tensorflow:step: 16600, lr: 0.000198, klw: 0.2863, cost: -1.5107, recon: -1.6033, kl: 0.3235, train_time_taken: 15.8335
INFO:tensorflow:step: 16620, lr: 0.000198, klw: 0.2866, cost: -1.4439, recon: -1.5357, kl: 0.3204, train_time_taken: 15.0087
INFO:tensorflow:step: 16640, lr: 0.000197, klw: 0.2868, cost: -1.6385, recon: -1.7336, kl: 0.3315, train_time_taken: 14.9436
INFO:tensorflow:step: 16660, lr: 0.000197, klw: 0.2870, cost: -1.4622, recon: -1.5552, kl: 0.3241, train_time_taken: 14.9666
INFO:tensorflow:step: 16680, lr: 0.000197, klw: 0.2872, cost: -1.5163, recon: -1.6105, kl: 0.3282, train_time_taken: 14.8331
INFO:tensorflow:step: 16700, lr: 0.000196, klw: 0.2874, cost: -1.4541, recon: -1.5477, kl: 0.3256, train_time_taken: 15.2969
INFO:tensorflow:step: 16720, lr: 0.000196, klw: 0.2876, cost: -1.6559, recon: -1.7504, kl: 0.3284, train_time_taken: 15.6660
INFO:tensorflow:step: 16740, lr: 0.000196, klw: 0.2878, cost: -1.5492, recon: -1.6423, kl: 0.3235, train_time_taken: 15.0263
INFO:tensorflow:step: 16760, lr: 0.000195, klw: 0.2880, cost: -1.6272, recon: -1.7213, kl: 0.3267, train_time_taken: 14.9004
INFO:tensorflow:step: 16780, lr: 0.000195, klw: 0.2883, cost: -1.5078, recon: -1.6011, kl: 0.3236, train_time_taken: 14.9248
INFO:tensorflow:step: 16800, lr: 0.000194, klw: 0.2885, cost: -1.4592, recon: -1.5509, kl: 0.3180, train_time_taken: 14.8170
INFO:tensorflow:step: 16820, lr: 0.000194, klw: 0.2887, cost: -1.5408, recon: -1.6341, kl: 0.3233, train_time_taken: 16.2412
INFO:tensorflow:step: 16840, lr: 0.000194, klw: 0.2889, cost: -1.5543, recon: -1.6473, kl: 0.3219, train_time_taken: 14.8570
INFO:tensorflow:step: 16860, lr: 0.000193, klw: 0.2891, cost: -1.4696, recon: -1.5639, kl: 0.3259, train_time_taken: 14.9144
INFO:tensorflow:step: 16880, lr: 0.000193, klw: 0.2893, cost: -1.3907, recon: -1.4815, kl: 0.3139, train_time_taken: 14.4351
INFO:tensorflow:step: 16900, lr: 0.000193, klw: 0.2895, cost: -1.4737, recon: -1.5655, kl: 0.3172, train_time_taken: 14.6232
INFO:tensorflow:step: 16920, lr: 0.000192, klw: 0.2897, cost: -1.4373, recon: -1.5295, kl: 0.3184, train_time_taken: 16.0433
INFO:tensorflow:step: 16940, lr: 0.000192, klw: 0.2899, cost: -1.5721, recon: -1.6666, kl: 0.3259, train_time_taken: 14.7884
INFO:tensorflow:step: 16960, lr: 0.000192, klw: 0.2902, cost: -1.5464, recon: -1.6406, kl: 0.3247, train_time_taken: 14.9780
INFO:tensorflow:step: 16980, lr: 0.000191, klw: 0.2904, cost: -1.5077, recon: -1.6015, kl: 0.3230, train_time_taken: 16.0433
INFO:tensorflow:step: 17000, lr: 0.000191, klw: 0.2906, cost: -1.5056, recon: -1.6001, kl: 0.3251, train_time_taken: 14.5526
INFO:tensorflow:step: 17020, lr: 0.000190, klw: 0.2908, cost: -1.5529, recon: -1.6488, kl: 0.3299, train_time_taken: 15.4880
INFO:tensorflow:step: 17040, lr: 0.000190, klw: 0.2910, cost: -1.4402, recon: -1.5326, kl: 0.3175, train_time_taken: 15.4479
INFO:tensorflow:step: 17060, lr: 0.000190, klw: 0.2912, cost: -1.5142, recon: -1.6074, kl: 0.3200, train_time_taken: 14.8245
INFO:tensorflow:step: 17080, lr: 0.000189, klw: 0.2914, cost: -1.5283, recon: -1.6219, kl: 0.3212, train_time_taken: 14.8961
INFO:tensorflow:step: 17100, lr: 0.000189, klw: 0.2916, cost: -1.5191, recon: -1.6139, kl: 0.3251, train_time_taken: 14.7626
INFO:tensorflow:step: 17120, lr: 0.000189, klw: 0.2918, cost: -1.6028, recon: -1.6973, kl: 0.3240, train_time_taken: 14.9461
INFO:tensorflow:step: 17140, lr: 0.000188, klw: 0.2920, cost: -1.4750, recon: -1.5680, kl: 0.3183, train_time_taken: 16.2551
INFO:tensorflow:step: 17160, lr: 0.000188, klw: 0.2922, cost: -1.4542, recon: -1.5471, kl: 0.3177, train_time_taken: 14.9381
INFO:tensorflow:step: 17180, lr: 0.000188, klw: 0.2924, cost: -1.5277, recon: -1.6210, kl: 0.3191, train_time_taken: 14.7055
INFO:tensorflow:step: 17200, lr: 0.000187, klw: 0.2927, cost: -1.5756, recon: -1.6702, kl: 0.3231, train_time_taken: 14.8441
INFO:tensorflow:step: 17220, lr: 0.000187, klw: 0.2929, cost: -1.4894, recon: -1.5831, kl: 0.3202, train_time_taken: 14.8521
INFO:tensorflow:step: 17240, lr: 0.000187, klw: 0.2931, cost: -1.5520, recon: -1.6455, kl: 0.3191, train_time_taken: 16.1384
INFO:tensorflow:step: 17260, lr: 0.000186, klw: 0.2933, cost: -1.5013, recon: -1.5967, kl: 0.3253, train_time_taken: 14.5599
INFO:tensorflow:step: 17280, lr: 0.000186, klw: 0.2935, cost: -1.5603, recon: -1.6546, kl: 0.3214, train_time_taken: 15.0963
INFO:tensorflow:step: 17300, lr: 0.000185, klw: 0.2937, cost: -1.4476, recon: -1.5407, kl: 0.3170, train_time_taken: 14.7734
INFO:tensorflow:step: 17320, lr: 0.000185, klw: 0.2939, cost: -1.4846, recon: -1.5797, kl: 0.3237, train_time_taken: 15.0252
INFO:tensorflow:step: 17340, lr: 0.000185, klw: 0.2941, cost: -1.5491, recon: -1.6457, kl: 0.3286, train_time_taken: 16.4491
INFO:tensorflow:step: 17360, lr: 0.000184, klw: 0.2943, cost: -1.5457, recon: -1.6411, kl: 0.3239, train_time_taken: 15.3862
INFO:tensorflow:step: 17380, lr: 0.000184, klw: 0.2945, cost: -1.4921, recon: -1.5872, kl: 0.3227, train_time_taken: 16.2456
INFO:tensorflow:step: 17400, lr: 0.000184, klw: 0.2947, cost: -1.4384, recon: -1.5337, kl: 0.3232, train_time_taken: 15.0616
INFO:tensorflow:step: 17420, lr: 0.000183, klw: 0.2949, cost: -1.4576, recon: -1.5520, kl: 0.3204, train_time_taken: 14.8758
INFO:tensorflow:step: 17440, lr: 0.000183, klw: 0.2951, cost: -1.6927, recon: -1.7908, kl: 0.3324, train_time_taken: 15.2184
INFO:tensorflow:step: 17460, lr: 0.000183, klw: 0.2953, cost: -1.4232, recon: -1.5177, kl: 0.3201, train_time_taken: 15.7113
INFO:tensorflow:step: 17480, lr: 0.000182, klw: 0.2955, cost: -1.5065, recon: -1.6002, kl: 0.3170, train_time_taken: 14.9635
INFO:tensorflow:step: 17500, lr: 0.000182, klw: 0.2957, cost: -1.5442, recon: -1.6398, kl: 0.3232, train_time_taken: 14.8913
INFO:tensorflow:step: 17520, lr: 0.000182, klw: 0.2959, cost: -1.6059, recon: -1.7008, kl: 0.3206, train_time_taken: 14.9053
INFO:tensorflow:step: 17540, lr: 0.000181, klw: 0.2962, cost: -1.4690, recon: -1.5631, kl: 0.3177, train_time_taken: 14.9602
INFO:tensorflow:step: 17560, lr: 0.000181, klw: 0.2964, cost: -1.6099, recon: -1.7075, kl: 0.3292, train_time_taken: 16.8370
INFO:tensorflow:step: 17580, lr: 0.000181, klw: 0.2966, cost: -1.4638, recon: -1.5590, kl: 0.3210, train_time_taken: 14.8429
INFO:tensorflow:step: 17600, lr: 0.000180, klw: 0.2968, cost: -1.5186, recon: -1.6135, kl: 0.3198, train_time_taken: 14.8111
INFO:tensorflow:step: 17620, lr: 0.000180, klw: 0.2970, cost: -1.5630, recon: -1.6590, kl: 0.3233, train_time_taken: 14.8741
INFO:tensorflow:step: 17640, lr: 0.000180, klw: 0.2972, cost: -1.6465, recon: -1.7426, kl: 0.3235, train_time_taken: 15.2084
INFO:tensorflow:step: 17660, lr: 0.000179, klw: 0.2974, cost: -1.3873, recon: -1.4806, kl: 0.3139, train_time_taken: 16.2972
INFO:tensorflow:step: 17680, lr: 0.000179, klw: 0.2976, cost: -1.6363, recon: -1.7340, kl: 0.3283, train_time_taken: 15.3029
INFO:tensorflow:step: 17700, lr: 0.000179, klw: 0.2978, cost: -1.5321, recon: -1.6268, kl: 0.3179, train_time_taken: 15.0400
INFO:tensorflow:step: 17720, lr: 0.000178, klw: 0.2980, cost: -1.5294, recon: -1.6253, kl: 0.3216, train_time_taken: 15.5702
INFO:tensorflow:step: 17740, lr: 0.000178, klw: 0.2982, cost: -1.5571, recon: -1.6536, kl: 0.3235, train_time_taken: 15.1176
INFO:tensorflow:step: 17760, lr: 0.000178, klw: 0.2984, cost: -1.5497, recon: -1.6453, kl: 0.3203, train_time_taken: 15.9419
INFO:tensorflow:step: 17780, lr: 0.000177, klw: 0.2986, cost: -1.4267, recon: -1.5217, kl: 0.3183, train_time_taken: 16.1673
INFO:tensorflow:step: 17800, lr: 0.000177, klw: 0.2988, cost: -1.4287, recon: -1.5243, kl: 0.3200, train_time_taken: 15.8069
INFO:tensorflow:step: 17820, lr: 0.000177, klw: 0.2990, cost: -1.5172, recon: -1.6139, kl: 0.3234, train_time_taken: 15.0983
INFO:tensorflow:step: 17840, lr: 0.000176, klw: 0.2992, cost: -1.5833, recon: -1.6806, kl: 0.3255, train_time_taken: 15.2161
INFO:tensorflow:step: 17860, lr: 0.000176, klw: 0.2994, cost: -1.5754, recon: -1.6719, kl: 0.3224, train_time_taken: 15.7256
INFO:tensorflow:step: 17880, lr: 0.000176, klw: 0.2996, cost: -1.5546, recon: -1.6515, kl: 0.3232, train_time_taken: 15.7134
INFO:tensorflow:step: 17900, lr: 0.000175, klw: 0.2998, cost: -1.5592, recon: -1.6553, kl: 0.3206, train_time_taken: 15.0722
INFO:tensorflow:step: 17920, lr: 0.000175, klw: 0.3000, cost: -1.4982, recon: -1.5937, kl: 0.3182, train_time_taken: 14.9417
INFO:tensorflow:step: 17940, lr: 0.000175, klw: 0.3002, cost: -1.6005, recon: -1.6977, kl: 0.3238, train_time_taken: 14.8349
INFO:tensorflow:step: 17960, lr: 0.000174, klw: 0.3004, cost: -1.6487, recon: -1.7461, kl: 0.3243, train_time_taken: 15.1710
INFO:tensorflow:step: 17980, lr: 0.000174, klw: 0.3006, cost: -1.5924, recon: -1.6894, kl: 0.3226, train_time_taken: 15.9514
INFO:tensorflow:step: 18000, lr: 0.000174, klw: 0.3008, cost: -1.4199, recon: -1.5148, kl: 0.3155, train_time_taken: 14.9292
INFO:tensorflow:best_valid_cost: -1.7273, valid_cost: -1.7273, valid_recon: -1.7305, valid_kl: 0.3178, valid_time_taken: 0.9119
INFO:tensorflow:saving model /content/sketch_rnn/ckpt_layer_norm/vector.
INFO:tensorflow:global_step 18000.
INFO:tensorflow:time_taken_save 0.5768.
INFO:tensorflow:eval_cost: -1.6911, eval_recon: -1.6942, eval_kl: 0.3125, eval_time_taken: 0.7711
INFO:tensorflow:step: 18020, lr: 0.000173, klw: 0.3010, cost: -1.6297, recon: -1.7273, kl: 0.3244, train_time_taken: 15.2863
INFO:tensorflow:step: 18040, lr: 0.000173, klw: 0.3012, cost: -1.5720, recon: -1.6683, kl: 0.3197, train_time_taken: 14.9550
INFO:tensorflow:step: 18060, lr: 0.000173, klw: 0.3014, cost: -1.5616, recon: -1.6587, kl: 0.3222, train_time_taken: 15.0790
INFO:tensorflow:step: 18080, lr: 0.000172, klw: 0.3016, cost: -1.4386, recon: -1.5320, kl: 0.3096, train_time_taken: 16.1707
INFO:tensorflow:step: 18100, lr: 0.000172, klw: 0.3018, cost: -1.3840, recon: -1.4782, kl: 0.3121, train_time_taken: 15.2161
INFO:tensorflow:step: 18120, lr: 0.000172, klw: 0.3020, cost: -1.6326, recon: -1.7300, kl: 0.3227, train_time_taken: 15.6560
INFO:tensorflow:step: 18140, lr: 0.000171, klw: 0.3022, cost: -1.5014, recon: -1.5967, kl: 0.3154, train_time_taken: 15.1717
Traceback (most recent call last):
  File "/content/sketch_rnn/sketch_rnn_train.py", line 478, in <module>
    console_entry_point()
  File "/content/sketch_rnn/sketch_rnn_train.py", line 474, in console_entry_point
    tf.app.run(main)
  File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/platform/app.py", line 125, in run
    _sys.exit(main(argv))
  File "/content/sketch_rnn/sketch_rnn_train.py", line 470, in main
    trainer(model_params)
  File "/content/sketch_rnn/sketch_rnn_train.py", line 462, in trainer
    train(sess, model, eval_model, train_set, valid_set, test_set)
  File "/content/sketch_rnn/sketch_rnn_train.py", line 301, in train
    ], feed)
  File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py", line 929, in run
    run_metadata_ptr)
  File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py", line 1152, in _run
    feed_dict_tensor, options, run_metadata)
  File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py", line 1328, in _do_run
    run_metadata)
  File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py", line 1334, in _do_call
    return fn(*args)
  File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py", line 1319, in _run_fn
    options, feed_dict, fetch_list, target_list, run_metadata)
  File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py", line 1407, in _call_tf_sessionrun
    run_metadata)
KeyboardInterrupt

Load checkpoint

# !pip install -q magenta

# import our command line tools
from magenta.models.sketch_rnn.sketch_rnn_train import *
from magenta.models.sketch_rnn.model import *
from magenta.models.sketch_rnn.utils import *
from magenta.models.sketch_rnn.rnn import *

WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.
For more information, please see:
  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md
  * https://github.com/tensorflow/addons
If you depend on functionality not listed there, please file an issue.

Load model checkpoints saved in google drive.

Download checkpoint from https://drive.google.com/drive/folders/1E3ZpsUOMhi8HFJqr7JHMONAwZktYCx-A?usp=sharing
Then copy to /content/sketch_rnn/ckpt_layer_norm/

model_dir = '/content/sketch_rnn/ckpt_layer_norm/'

def load_model_compatible(model_dir):
  """Loads model for inference mode, used in jupyter notebook."""
  # modified https://github.com/tensorflow/magenta/blob/master/magenta/models/sketch_rnn/sketch_rnn_train.py
  # to work with depreciated tf.HParams functionality
  model_params = sketch_rnn_model.get_default_hparams()
  with tf.gfile.Open(os.path.join(model_dir, 'model_config.json'), 'r') as f:
    data = json.load(f)
  fix_list = ['conditional', 'is_training', 'use_input_dropout', 'use_output_dropout', 'use_recurrent_dropout']
  for fix in fix_list:
    data[fix] = (data[fix] == 1)
  model_params.parse_json(json.dumps(data))

  model_params.batch_size = 1  # only sample one at a time
  eval_model_params = sketch_rnn_model.copy_hparams(model_params)
  eval_model_params.use_input_dropout = 0
  eval_model_params.use_recurrent_dropout = 0
  eval_model_params.use_output_dropout = 0
  eval_model_params.is_training = 0
  sample_model_params = sketch_rnn_model.copy_hparams(eval_model_params)
  sample_model_params.max_seq_len = 1  # sample one point at a time
  return [model_params, eval_model_params, sample_model_params]

[hps_model, eval_hps_model, sample_hps_model] = load_model_compatible(model_dir)

# construct the sketch-rnn model here:
reset_graph()
model = Model(hps_model)
eval_model = Model(eval_hps_model, reuse=True)
sample_model = Model(sample_hps_model, reuse=True)

INFO:tensorflow:Model using gpu.
INFO:tensorflow:Input dropout mode = False.
INFO:tensorflow:Output dropout mode = False.
INFO:tensorflow:Recurrent dropout mode = True.
INFO:tensorflow:Model using gpu.
INFO:tensorflow:Input dropout mode = 0.
INFO:tensorflow:Output dropout mode = 0.
INFO:tensorflow:Recurrent dropout mode = 0.
INFO:tensorflow:Model using gpu.
INFO:tensorflow:Input dropout mode = 0.
INFO:tensorflow:Output dropout mode = 0.
INFO:tensorflow:Recurrent dropout mode = 0.

sess = tf.InteractiveSession()
sess.run(tf.global_variables_initializer())

load_checkpoint(sess, model_dir)

INFO:tensorflow:Loading model /content/sketch_rnn/ckpt_layer_norm/vector-18000.
INFO:tensorflow:Restoring parameters from /content/sketch_rnn/ckpt_layer_norm/vector-18000

Sample

def encode(input_strokes):
  strokes = to_big_strokes(input_strokes).tolist()
  strokes.insert(0, [0, 0, 1, 0, 0])
  seq_len = [len(input_strokes)]
  #print(strokes)
  print(seq_len)
  #draw_strokes(to_normal_strokes(np.array(strokes)))
  return sess.run(eval_model.batch_z, feed_dict={eval_model.input_data: [strokes], eval_model.sequence_lengths: seq_len})[0]

def decode(z_input=None, draw_mode=True, temperature=0.1, factor=0.2):
  z = None
  if z_input is not None:
    z = [z_input]
  sample_strokes, m = sample(sess, sample_model, seq_len=eval_model.hps.max_seq_len, temperature=temperature, z=z)
  strokes = to_normal_strokes(sample_strokes)
  if draw_mode:
    draw_strokes(strokes, factor)
  return strokes

Interpolate

theunique = random.choice(test_set)
draw_strokes(theunique,factor=1)

print(theunique.shape)

(75, 3)

Interpolate spherically between z_0 and z_1

z_0 = np.random.randn(eval_model.hps.z_size)
_ = decode(z_0, draw_mode=True, temperature=0.1, factor=0.1)

z_1 = np.random.randn(eval_model.hps.z_size)
_ = decode(z_1, draw_mode=True, temperature=0.1, factor=0.1)

z_list = [] # interpolate spherically between z_0 and z_1
N = 8
for t in np.linspace(0, 1, N):
  z_list.append(slerp(z_0, z_1, t))
# for every latent vector in z_list, sample a vector image
reconstructions = []
for i in range(N):
  reconstructions.append([decode(z_list[i], draw_mode=False, temperature=0.1), [0, i]])

stroke_grid = make_grid_svg(reconstructions)
draw_strokes(stroke_grid, factor=0.1)

Chaos writer

Generate a paragraph of fake characters and insert punctuations to look more "real".

# randomly generate N examples using IID gaussian latent vectors
from ipywidgets import IntProgress
from random import randint
    
N = randint(70,90)

f = IntProgress(min=0, max=N) # instantiate the bar
display(f) # display the bar

reconstructions = []
j = 0
k = 2
zoom_value = 0.1

# generate punctuation position
pos_punc = np.random.permutation(np.arange(3,N-3))[:int(N/10)]
for i in pos_punc:
  if i+1 in pos_punc:
    index, = np.where(pos_punc == i+1)
    pos_punc=np.delete(pos_punc,index)
  elif i+2 in pos_punc:
    index, = np.where(pos_punc == i+2)
    pos_punc=np.delete(pos_punc,index)
           
pos_stop = pos_punc[:int(len(pos_punc)/2)]
pos_comma= pos_punc[int(len(pos_punc)/2):]

stop = np.array([[0,0,0],[1.5,-3.5,0],[3.5,-1.5,0],[3.5,1.5,0],[1.5,3.5,0],[-1.5,3.5,0],[-3.5,1.5,0],[-3.5,-1.5,0],[-1.5,-3.5,1]])*zoom_value
comma = np.array([[0,0,0],[2.5,4,0],[-5,5,1]])*zoom_value
space = np.array([[0,0,0]])

reconstructions.append([space, [0, 0]])
reconstructions.append([space, [0, 1]])

def update_jk(j,k):
  if k<15:
    k=k+1
  else:
    k=0
    j=j+1 
  return j,k

# generate examples
for i in range(N):    
  keep_generate = True
  if i in pos_stop:
    reconstructions.append([stop, [j, k]])
    j,k = update_jk(j,k)  
  elif i in pos_comma:
    reconstructions.append([comma, [j, k]])
    j,k = update_jk(j,k)      
  while keep_generate:
    z = np.random.randn(eval_model.hps.z_size)
    character = decode(z, temperature=0.1, draw_mode=False)
    min_x, max_x, min_y, max_y = get_bounds(character, factor=zoom_value)  
    dims = (50 + max_x - min_x, 50 + max_y - min_y)
    if (dims[0]<120 and dims[1]<120) and (dims[0]>70 and dims[1]>70) and (character.shape[0]>20 and character.shape[0]<90):   
    #print((j,k),dims,'accept')
      reconstructions.append([character, [j, k]])
      j,k = update_jk(j,k)
      keep_generate = False
    
    #print(dims)
  f.value = i # signal to increment the progress bar
reconstructions.append([stop, [j, k]])

stroke_grid = make_grid_svg(reconstructions, grid_space_y=10.0, grid_space_x=7.0)
draw_strokes(stroke_grid,factor=zoom_value,svg_filename='/content/sketch_rnn/FakeHanzi.svg')

As shown above, these are nonsense, but the structure of some characters accord with real ones.
Some samples:

draw_strokes(reconstructions[51][0],factor=zoom_value)
print('guess it means: metal shop')

guess it means: metal shop

draw_strokes(reconstructions[52][0],factor=zoom_value)
print('guess it means: ill white son')

guess it means: ill white son

There are also some characters coincidently reconstruct real ones, which are rare:

draw_strokes(reconstructions[68][0],factor=zoom_value)
print('kind; sort')

kind; sort

draw_strokes(reconstructions[73][0],factor=zoom_value)
print('oil')

oil