Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions tensorflow_datasets/datasets/stl10/stl10_dataset_builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,11 +61,11 @@ def gen_filenames(filenames):
# Define the splits
splits = [
tfds.core.SplitGenerator(
name=tfds.Split.TRAIN,
name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute]
gen_kwargs={"filepaths": gen_filenames(train_files)},
),
tfds.core.SplitGenerator(
name=tfds.Split.TEST,
name=tfds.Split.TEST, # pyrefly: ignore[missing-attribute]
gen_kwargs={"filepaths": gen_filenames(test_files)},
),
tfds.core.SplitGenerator(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -61,23 +61,23 @@ def _info(self) -> tfds.core.DatasetInfo:

def _split_generators(self, dl_manager: tfds.download.DownloadManager):
"""Returns SplitGenerators."""
if self.builder_config.name == '2016':
if self.builder_config.name == '2016': # pyrefly: ignore[missing-attribute]
file_postfix = 'spring2016'
elif self.builder_config.name == '2018':
file_postfix = 'winter2018'

val_file = 'cloze_test_val__' + file_postfix + '.csv'
val_file = 'cloze_test_val__' + file_postfix + '.csv' # pyrefly: ignore[unbound-name]
test_file = 'cloze_test_test__' + file_postfix + '.csv'

return [
tfds.core.SplitGenerator(
name=tfds.Split.VALIDATION,
name=tfds.Split.VALIDATION, # pyrefly: ignore[missing-attribute]
gen_kwargs={
'filepath': os.path.join(dl_manager.manual_dir, val_file)
},
),
tfds.core.SplitGenerator(
name=tfds.Split.TEST,
name=tfds.Split.TEST, # pyrefly: ignore[missing-attribute]
gen_kwargs={
'filepath': os.path.join(dl_manager.manual_dir, test_file)
},
Expand Down
22 changes: 11 additions & 11 deletions tensorflow_datasets/datasets/sun397/sun397_dataset_builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,7 @@ def __init__(self, tfds_split_files=None, **kwargs):
"va": "sun397_tfds_va.txt",
}
for split, filename in tfds_split_files.items():
tfds_split_files[split] = tfds.core.tfds_path(
tfds_split_files[split] = tfds.core.tfds_path( # pyrefly: ignore[bad-assignment]
os.path.join("datasets", "sun397", filename)
)
self._tfds_split_files = tfds_split_files
Expand Down Expand Up @@ -222,25 +222,25 @@ def _split_generators(self, dl_manager):
path=paths["images"],
extract_method=tfds.download.ExtractMethod.TAR_GZ_STREAM,
)
if self.builder_config.name == "tfds":
if self.builder_config.name == "tfds": # pyrefly: ignore[missing-attribute]
subset_images = self._get_tfds_subsets_images()
return [
tfds.core.SplitGenerator(
name=tfds.Split.TRAIN,
name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute]
gen_kwargs=dict(
archive=dl_manager.iter_archive(images),
subset_images=subset_images["tr"],
),
),
tfds.core.SplitGenerator(
name=tfds.Split.TEST,
name=tfds.Split.TEST, # pyrefly: ignore[missing-attribute]
gen_kwargs=dict(
archive=dl_manager.iter_archive(images),
subset_images=subset_images["te"],
),
),
tfds.core.SplitGenerator(
name=tfds.Split.VALIDATION,
name=tfds.Split.VALIDATION, # pyrefly: ignore[missing-attribute]
gen_kwargs=dict(
archive=dl_manager.iter_archive(images),
subset_images=subset_images["va"],
Expand All @@ -251,14 +251,14 @@ def _split_generators(self, dl_manager):
subset_images = self._get_partition_subsets_images(paths["partitions"])
return [
tfds.core.SplitGenerator(
name=tfds.Split.TRAIN,
name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute]
gen_kwargs=dict(
archive=dl_manager.iter_archive(images),
subset_images=subset_images["tr"],
),
),
tfds.core.SplitGenerator(
name=tfds.Split.TEST,
name=tfds.Split.TEST, # pyrefly: ignore[missing-attribute]
gen_kwargs=dict(
archive=dl_manager.iter_archive(images),
subset_images=subset_images["te"],
Expand All @@ -284,8 +284,8 @@ def _generate_examples(self, archive, subset_images):
fobj,
sess,
filename,
quality=self.builder_config.quality,
target_pixels=self.builder_config.target_pixels,
quality=self.builder_config.quality, # pyrefly: ignore[missing-attribute]
target_pixels=self.builder_config.target_pixels, # pyrefly: ignore[missing-attribute]
)
record = {
"file_name": filename,
Expand All @@ -307,8 +307,8 @@ def _get_partition_subsets_images(self, partitions_dir):
all_images.update(split_images)
# Load the images in the training/test split of this partition.
filenames = {
"tr": "Training_%02d.txt" % self.builder_config.partition,
"te": "Testing_%02d.txt" % self.builder_config.partition,
"tr": "Training_%02d.txt" % self.builder_config.partition, # pyrefly: ignore[missing-attribute]
"te": "Testing_%02d.txt" % self.builder_config.partition, # pyrefly: ignore[missing-attribute]
}
splits_sets = {}
for split, filename in filenames.items():
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ def _info(self):
"label": tfds.features.ClassLabel(num_classes=10),
}
if self.version > "3.0.0":
features_dict["id"] = tfds.features.Text()
features_dict["id"] = tfds.features.Text() # pyrefly: ignore[bad-assignment]
return self.dataset_info_from_configs(
description=(
"The Street View House Numbers (SVHN) Dataset is an image digit "
Expand All @@ -61,14 +61,14 @@ def _split_generators(self, dl_manager):

return [
tfds.core.SplitGenerator(
name=tfds.Split.TRAIN,
name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute]
gen_kwargs=dict(
split_prefix="train_",
filepath=output_files["train"],
),
),
tfds.core.SplitGenerator(
name=tfds.Split.TEST,
name=tfds.Split.TEST, # pyrefly: ignore[missing-attribute]
gen_kwargs=dict(
split_prefix="test_",
filepath=output_files["test"],
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ def _info(self) -> tfds.core.DatasetInfo:

def _split_generators(self, dl_manager: tfds.download.DownloadManager):
"""Returns SplitGenerators."""
lang = _LANGS[self.builder_config.language]
lang = _LANGS[self.builder_config.language] # pyrefly: ignore[missing-attribute]

tatoeba_source_path = os.path.join(
_DATA_URLS, "tatoeba.{lang}-eng.{lang}".format(lang=lang)
Expand Down Expand Up @@ -138,6 +138,6 @@ def _generate_examples(self, source_file, target_file):
yield i, {
"source_sentence": source_sentences[i],
"target_sentence": target_sentences[i],
"source_language": self.builder_config.language,
"source_language": self.builder_config.language, # pyrefly: ignore[missing-attribute]
"target_language": "en",
}
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ def __init__(self, *, language_pair=(None, None), **kwargs):
first will be used as source and second as target in supervised mode.
**kwargs: keyword arguments forwarded to super.
"""
name = "%s_to_%s" % (language_pair[0].replace("_", ""), language_pair[1])
name = "%s_to_%s" % (language_pair[0].replace("_", ""), language_pair[1]) # pyrefly: ignore[missing-attribute]

description = "Translation dataset from %s to %s in plain text." % (
language_pair[0],
Expand Down Expand Up @@ -99,21 +99,21 @@ class Builder(tfds.core.GeneratorBasedBuilder):
def _info(self):
return self.dataset_info_from_configs(
features=tfds.features.Translation(
languages=self.builder_config.language_pair
languages=self.builder_config.language_pair # pyrefly: ignore[missing-attribute]
),
homepage="https://github.com/neulab/word-embeddings-for-nmt",
supervised_keys=self.builder_config.language_pair,
supervised_keys=self.builder_config.language_pair, # pyrefly: ignore[missing-attribute]
)

def _split_generators(self, dl_manager):
dl_dir = dl_manager.download_and_extract(_DATA_URL)
source, target = self.builder_config.language_pair
source, target = self.builder_config.language_pair # pyrefly: ignore[missing-attribute]

data_dir = os.path.join(dl_dir, "datasets", "%s_to_%s" % (source, target))

return [
tfds.core.SplitGenerator(
name=tfds.Split.TRAIN,
name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute]
gen_kwargs={
"source_file": os.path.join(
data_dir, "{}.train".format(source.replace("_", "-"))
Expand All @@ -124,7 +124,7 @@ def _split_generators(self, dl_manager):
},
),
tfds.core.SplitGenerator(
name=tfds.Split.VALIDATION,
name=tfds.Split.VALIDATION, # pyrefly: ignore[missing-attribute]
gen_kwargs={
"source_file": os.path.join(
data_dir, "{}.dev".format(source.split("_")[0])
Expand All @@ -133,7 +133,7 @@ def _split_generators(self, dl_manager):
},
),
tfds.core.SplitGenerator(
name=tfds.Split.TEST,
name=tfds.Split.TEST, # pyrefly: ignore[missing-attribute]
gen_kwargs={
"source_file": os.path.join(
data_dir, "{}.test".format(source.split("_")[0])
Expand All @@ -159,7 +159,7 @@ def _generate_examples(self, source_file, target_file):
target_file,
)

source, target = self.builder_config.language_pair
source, target = self.builder_config.language_pair # pyrefly: ignore[missing-attribute]
for idx, (l1, l2) in enumerate(zip(source_sentences, target_sentences)):
result = {source: l1, target: l2}
# Make sure that both translations are non-empty.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -108,17 +108,17 @@ def _split_generators(self, dl_manager):

return [
tfds.core.SplitGenerator(
name=tfds.Split.TRAIN,
name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute]
gen_kwargs={
'data_file': os.path.join(dl_dir, 'all_talks_train.tsv')
},
),
tfds.core.SplitGenerator(
name=tfds.Split.VALIDATION,
name=tfds.Split.VALIDATION, # pyrefly: ignore[missing-attribute]
gen_kwargs={'data_file': os.path.join(dl_dir, 'all_talks_dev.tsv')},
),
tfds.core.SplitGenerator(
name=tfds.Split.TEST,
name=tfds.Split.TEST, # pyrefly: ignore[missing-attribute]
gen_kwargs={
'data_file': os.path.join(dl_dir, 'all_talks_test.tsv')
},
Expand Down
26 changes: 13 additions & 13 deletions tensorflow_datasets/datasets/tedlium/tedlium_dataset_builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,9 +63,9 @@ def _make_builder_configs():
url="https://www.openslr.org/7/",
download_url="http://www.openslr.org/resources/7/TEDLIUM_release1.tar.gz",
split_paths=[
(tfds.Split.TRAIN, os.path.join("TEDLIUM_release1", "train")),
(tfds.Split.VALIDATION, os.path.join("TEDLIUM_release1", "dev")),
(tfds.Split.TEST, os.path.join("TEDLIUM_release1", "test")),
(tfds.Split.TRAIN, os.path.join("TEDLIUM_release1", "train")), # pyrefly: ignore[missing-attribute]
(tfds.Split.VALIDATION, os.path.join("TEDLIUM_release1", "dev")), # pyrefly: ignore[missing-attribute]
(tfds.Split.TEST, os.path.join("TEDLIUM_release1", "test")), # pyrefly: ignore[missing-attribute]
],
)

Expand Down Expand Up @@ -99,9 +99,9 @@ def _make_builder_configs():
"http://www.openslr.org/resources/19/TEDLIUM_release2.tar.gz"
),
split_paths=[
(tfds.Split.TRAIN, os.path.join("TEDLIUM_release2", "train")),
(tfds.Split.VALIDATION, os.path.join("TEDLIUM_release2", "dev")),
(tfds.Split.TEST, os.path.join("TEDLIUM_release2", "test")),
(tfds.Split.TRAIN, os.path.join("TEDLIUM_release2", "train")), # pyrefly: ignore[missing-attribute]
(tfds.Split.VALIDATION, os.path.join("TEDLIUM_release2", "dev")), # pyrefly: ignore[missing-attribute]
(tfds.Split.TEST, os.path.join("TEDLIUM_release2", "test")), # pyrefly: ignore[missing-attribute]
],
)

Expand Down Expand Up @@ -152,17 +152,17 @@ def _make_builder_configs():
download_url="http://www.openslr.org/resources/51/TEDLIUM_release-3.tgz",
split_paths=[
(
tfds.Split.VALIDATION,
tfds.Split.VALIDATION, # pyrefly: ignore[missing-attribute]
os.path.join("TEDLIUM_release-3", "legacy", "dev"),
),
(
tfds.Split.TEST,
tfds.Split.TEST, # pyrefly: ignore[missing-attribute]
os.path.join("TEDLIUM_release-3", "legacy", "test"),
),
# The legacy/train directory contains symlinks to "data",
# which are skipped by extraction (see above).
# Work around this by manually dereferencing the links here.
(tfds.Split.TRAIN, os.path.join("TEDLIUM_release-3", "data")),
(tfds.Split.TRAIN, os.path.join("TEDLIUM_release-3", "data")), # pyrefly: ignore[missing-attribute]
],
)

Expand All @@ -186,19 +186,19 @@ def _info(self):
"id": np.str_,
}),
supervised_keys=("speech", "text"),
homepage=self.builder_config.url,
citation=self.builder_config.citation,
homepage=self.builder_config.url, # pyrefly: ignore[missing-attribute]
citation=self.builder_config.citation, # pyrefly: ignore[missing-attribute]
metadata=tfds.core.MetadataDict(
sample_rate=16000,
),
)

def _split_generators(self, dl_manager):
extracted_dir = dl_manager.download_and_extract(
self.builder_config.download_url
self.builder_config.download_url # pyrefly: ignore[missing-attribute]
)
splits = []
for split, path in self.builder_config.split_paths:
for split, path in self.builder_config.split_paths: # pyrefly: ignore[missing-attribute]
kwargs = {"directory": os.path.join(extracted_dir, path)}
splits.append(tfds.core.SplitGenerator(name=split, gen_kwargs=kwargs))
return splits
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ def _split_generators(self, dl_manager):
# There is no predefined train/val/test split for this dataset.
return [
tfds.core.SplitGenerator(
name=tfds.Split.TRAIN,
name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute]
gen_kwargs={"images_dir_path": dl_manager.iter_archive(path)},
),
]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ def _split_generators(self, dl_manager):
extracted_path = dl_manager.download_and_extract(_DATASET_URL)
return [
tfds.core.SplitGenerator(
name=tfds.Split.TRAIN,
name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute]
gen_kwargs={
"image_dir_path": os.path.join(extracted_path, "300W_LP"),
},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,19 +57,19 @@ def _split_generators(self, dl_manager):

return [
tfds.core.SplitGenerator(
name=tfds.Split.TRAIN,
name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute]
# These kwargs will be passed to _generate_examples
gen_kwargs={'split_key': 'train', 'split_text': train_text},
),
tfds.core.SplitGenerator(
name=tfds.Split.VALIDATION,
name=tfds.Split.VALIDATION, # pyrefly: ignore[missing-attribute]
gen_kwargs={
'split_key': 'validation',
'split_text': validation_text,
},
),
tfds.core.SplitGenerator(
name=tfds.Split.TEST,
name=tfds.Split.TEST, # pyrefly: ignore[missing-attribute]
gen_kwargs={'split_key': 'test', 'split_text': test_text},
),
]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ def _split_generators(self, dl_manager):
# There is no predefined train/val/test split for this dataset.
return [
tfds.core.SplitGenerator(
name=tfds.Split.TRAIN, gen_kwargs={"file_path": path}
name=tfds.Split.TRAIN, gen_kwargs={"file_path": path} # pyrefly: ignore[missing-attribute]
),
]

Expand Down
4 changes: 2 additions & 2 deletions tensorflow_datasets/datasets/trec/trec_dataset_builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -102,13 +102,13 @@ def _split_generators(self, dl_manager):
dl_files = dl_manager.download_and_extract(_URLs)
return [
tfds.core.SplitGenerator(
name=tfds.Split.TRAIN,
name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute]
gen_kwargs={
"filepath": dl_files["train"],
},
),
tfds.core.SplitGenerator(
name=tfds.Split.TEST,
name=tfds.Split.TEST, # pyrefly: ignore[missing-attribute]
gen_kwargs={
"filepath": dl_files["test"],
},
Expand Down
Loading
Loading