我正在尝试使用 Tensorflow 的数据集 API 从多个 csv 文件中读取数据。
以下版本的代码工作正常:
record_defaults = [[""], [0.0], [0.0], [0.0], [0.0], [0.0], [0.]]
def decode_csv(line):
col1, col2, col3, col4, col5, col6, col7 = tf.decode_csv(line, record_defaults)
features = tf.stack([col2, col3, col4, col5, col6])
labels = tf.stack([col7])
return features, labels
filenames = tf.placeholder(tf.string, shape=[None])
dataset5 = tf.data.Dataset.from_tensor_slices(filenames)
dataset5 = dataset5.flat_map(lambda filename: tf.data.TextLineDataset(filename).skip(1).map(decode_csv))
dataset5 = dataset5.shuffle(buffer_size=1000)
dataset5 = dataset5.batch(7)
iterator5 = dataset5.make_initializable_iterator()
但我想让它更具动态性，因为#columns(#features)可能会在不同的项目中发生变化。但是当我如下更改代码时，它就不起作用了。在这个问题上花费大量时间也无济于事..
record_defaults = [[""], [0.0], [0.0], [0.0], [0.0], [0.0], [0.]]
def decode_csv(line):
csv_columns = tf.decode_csv(line, record_defaults)
labels = csv_columns[-1] # last column is the label
del csv_columns[-1] # delete the last column
del csv_columns[0] # delete the first column bcz not a feature
features = csv_columns
return features, labels
filenames = tf.placeholder(tf.string, shape=[None])
dataset5 = tf.data.Dataset.from_tensor_slices(filenames)
dataset5 = dataset5.flat_map(lambda filename: tf.data.TextLineDataset(filename).skip(1).map(decode_csv))
dataset5 = dataset5.shuffle(buffer_size=1000)
dataset5 = dataset5.batch(7)
iterator5 = dataset5.make_initializable_iterator()
当我运行上面的第二个版本时出现以下错误..也许更有经验的人在这里立即看到问题..?
---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
<ipython-input-21-92ea8cc44da0> in <module>()
18 filenames = tf.placeholder(tf.string, shape=[None])
19 dataset5 = tf.data.Dataset.from_tensor_slices(filenames)
---> 20 dataset5 = dataset5.flat_map(lambda filename: tf.data.TextLineDataset(filename).skip(1).map(decode_csv))
21 dataset5 = dataset5.shuffle(buffer_size=1000)
22 dataset5 = dataset5.batch(7)
~/.local/lib/python3.5/site-packages/tensorflow/python/data/ops/dataset_ops.py in flat_map(self, map_func)
799 Dataset: A `Dataset`.
800 """
--> 801 return FlatMapDataset(self, map_func)
802
803 def interleave(self, map_func, cycle_length, block_length=1):
~/.local/lib/python3.5/site-packages/tensorflow/python/data/ops/dataset_ops.py in __init__(self, input_dataset, map_func)
1676
1677 self._map_func = tf_map_func
-> 1678 self._map_func.add_to_graph(ops.get_default_graph())
1679
1680 def _as_variant_tensor(self):
~/.local/lib/python3.5/site-packages/tensorflow/python/framework/function.py in add_to_graph(self, g)
484 def add_to_graph(self, g):
485 """Adds this function into the graph g."""
--> 486 self._create_definition_if_needed()
487
488 # Adds this function into 'g'.
~/.local/lib/python3.5/site-packages/tensorflow/python/framework/function.py in _create_definition_if_needed(self)
319 """Creates the function definition if it's not created yet."""
320 with context.graph_mode():
--> 321 self._create_definition_if_needed_impl()
322
323 def _create_definition_if_needed_impl(self):
~/.local/lib/python3.5/site-packages/tensorflow/python/framework/function.py in _create_definition_if_needed_impl(self)
336 # Call func and gather the output tensors.
337 with vs.variable_scope("", custom_getter=temp_graph.getvar):
--> 338 outputs = self._func(*inputs)
339
340 # There is no way of distinguishing between a function not returning
~/.local/lib/python3.5/site-packages/tensorflow/python/data/ops/dataset_ops.py in tf_map_func(*args)
1664 dataset = map_func(*nested_args)
1665 else:
-> 1666 dataset = map_func(nested_args)
1667
1668 if not isinstance(dataset, Dataset):
<ipython-input-21-92ea8cc44da0> in <lambda>(filename)
18 filenames = tf.placeholder(tf.string, shape=[None])
19 dataset5 = tf.data.Dataset.from_tensor_slices(filenames)
---> 20 dataset5 = dataset5.flat_map(lambda filename: tf.data.TextLineDataset(filename).skip(1).map(decode_csv))
21 dataset5 = dataset5.shuffle(buffer_size=1000)
22 dataset5 = dataset5.batch(7)
~/.local/lib/python3.5/site-packages/tensorflow/python/data/ops/dataset_ops.py in map(self, map_func, num_parallel_calls)
784 """
785 if num_parallel_calls is None:
--> 786 return MapDataset(self, map_func)
787 else:
788 return ParallelMapDataset(self, map_func, num_parallel_calls)
~/.local/lib/python3.5/site-packages/tensorflow/python/data/ops/dataset_ops.py in __init__(self, input_dataset, map_func)
1587
1588 self._map_func = tf_map_func
-> 1589 self._map_func.add_to_graph(ops.get_default_graph())
1590
1591 def _as_variant_tensor(self):
~/.local/lib/python3.5/site-packages/tensorflow/python/framework/function.py in add_to_graph(self, g)
484 def add_to_graph(self, g):
485 """Adds this function into the graph g."""
--> 486 self._create_definition_if_needed()
487
488 # Adds this function into 'g'.
~/.local/lib/python3.5/site-packages/tensorflow/python/framework/function.py in _create_definition_if_needed(self)
319 """Creates the function definition if it's not created yet."""
320 with context.graph_mode():
--> 321 self._create_definition_if_needed_impl()
322
323 def _create_definition_if_needed_impl(self):
~/.local/lib/python3.5/site-packages/tensorflow/python/framework/function.py in _create_definition_if_needed_impl(self)
336 # Call func and gather the output tensors.
337 with vs.variable_scope("", custom_getter=temp_graph.getvar):
--> 338 outputs = self._func(*inputs)
339
340 # There is no way of distinguishing between a function not returning
~/.local/lib/python3.5/site-packages/tensorflow/python/data/ops/dataset_ops.py in tf_map_func(*args)
1575 self._output_classes = sparse.get_classes(ret)
1576 self._output_shapes = nest.pack_sequence_as(
-> 1577 ret, [t.get_shape() for t in nest.flatten(ret)])
1578 self._output_types = nest.pack_sequence_as(
1579 ret, [t.dtype for t in nest.flatten(ret)])
~/.local/lib/python3.5/site-packages/tensorflow/python/data/ops/dataset_ops.py in <listcomp>(.0)
1575 self._output_classes = sparse.get_classes(ret)
1576 self._output_shapes = nest.pack_sequence_as(
-> 1577 ret, [t.get_shape() for t in nest.flatten(ret)])
1578 self._output_types = nest.pack_sequence_as(
1579 ret, [t.dtype for t in nest.flatten(ret)])
AttributeError: 'list' object has no attribute 'get_shape'
附录:
以下也适用。
feature_names = ['f0','f1','f2','f3','f4','f5']
record_defaults = [[""], [0.0], [0.0], [0.0], [0.0], [0.0], [0.]]
def decode_csv(line):
parsed_line = tf.decode_csv(line, record_defaults) # => tensor
label = parsed_line[-1]
del parsed_line[-1]
features = parsed_line
d = dict(zip(feature_names,features)),label
return d
filenames = tf.placeholder(tf.string, shape=[None])
dataset5 = tf.data.Dataset.from_tensor_slices(filenames)
dataset5 = dataset5.flat_map(lambda filename: tf.data.TextLineDataset(filename).skip(1).map(decode_csv))
dataset5 = dataset5.shuffle(buffer_size=1000)
dataset5 = dataset5.batch(7)
iterator5 = dataset5.make_initializable_iterator()
但现在 decode_csv 函数正在返回一个 (feature_name,feature_value) 对的字典。为什么有人喜欢从这个函数返回一个字典？向量化计算(如前向传播等)不会变得非常困难吗？
请您参考如下方法:
已解决。以下是工作版本。我不是为了节省一些空间而复制整个东西。 在excel文件中，第一列不是特征，只是训练样例ID。最后一列只是标签。使用 tf.stack(...) 函数堆叠特征解决了这个问题。
feature_names = ['f1','f2','f3','f4','f5']
record_defaults = [[""], [0.0], [0.0], [0.0], [0.0], [0.0], [0.0]]
def decode_csv(line):
parsed_line = tf.decode_csv(line, record_defaults)
label = parsed_line[-1]
del parsed_line[-1]
del parsed_line[0]
features = tf.stack(parsed_line) # ADDED LINE
d = features, label
return d
filenames = tf.placeholder(tf.string, shape=[None])
dataset5 = tf.data.Dataset.from_tensor_slices(filenames)
dataset5 = dataset5.flat_map(lambda filename: tf.data.TextLineDataset(filename).skip(1).map(decode_csv))
dataset5 = dataset5.shuffle(buffer_size=1000)
dataset5 = dataset5.batch(7)
iterator5 = dataset5.make_initializable_iterator()
