-
Notifications
You must be signed in to change notification settings - Fork 1
/
configuration.py
186 lines (145 loc) · 7.56 KB
/
configuration.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
import os
import collections
import yaml
import tensorflow as tf
########################################################################################################################
# You need to edit these.
########################################################################################################################
PROJECT_FOLDER = '/data/PycharmProjects/Spider-Monkey-Whinny-Detection'
DATA_FOLDER = '/data' + '/Data'
########################################################################################################################
# Leave as are.
########################################################################################################################
TFRECORDS_FOLDER = DATA_FOLDER + "/tfrecords_jenna"
OUTPUT_FOLDER = PROJECT_FOLDER + '/Results'
# These are the paper results -- train partition a bit smaller, test partition a bit bigger.
PARTITIONS = {"train": ["Luna", "Other", "Catappa", "Osa 1", "Osa 2", "Live-recording", "Piro-new calls"],
"devel": ["Tape", "Shady", "Eleanor"],
"test": ["Corcovado Calls", "new", "Will", "Corcovado Round 2 positives"]}
PRAAT_FILE_LIST = sorted([f for f in os.listdir(DATA_FOLDER + '/praat-files') if not f.startswith('.')])
YAML_CONFIGURATION_FOLDER = PROJECT_FOLDER + "/experiment_configurations"
def get_name_to_metadata(tf_names):
name_to_metadata = dict()
for name in [
"support",
"waveform",
"logmel_spectrogram",
"segment_id",
"version_id",
"whinny_single",
]:
name_to_metadata[name] = dict()
name_to_metadata["support"]["numpy_shape"] = (48000, 1)
name_to_metadata["waveform"]["numpy_shape"] = (75, 640)
name_to_metadata["logmel_spectrogram"]["numpy_shape"] = (300, 128)
name_to_metadata["segment_id"]["numpy_shape"] = (1, )
name_to_metadata["version_id"]["numpy_shape"] = (1, )
name_to_metadata["whinny_single"]["numpy_shape"] = (2, )
name_to_metadata["support"]["tfrecords_type"] = tf.string
name_to_metadata["waveform"]["tfrecords_type"] = tf.string
name_to_metadata["logmel_spectrogram"]["tfrecords_type"] = tf.string
name_to_metadata["segment_id"]["tfrecords_type"] = tf.int64
name_to_metadata["version_id"]["tfrecords_type"] = tf.int64
name_to_metadata["whinny_single"]["tfrecords_type"] = tf.string
name_to_metadata["support"]["variable_type"] = "support"
name_to_metadata["waveform"]["variable_type"] = "x"
name_to_metadata["logmel_spectrogram"]["variable_type"] = "x"
name_to_metadata["segment_id"]["variable_type"] = "id"
name_to_metadata["version_id"]["variable_type"] = "id"
name_to_metadata["whinny_single"]["variable_type"] = "y"
name_to_metadata["support"]["shape"] = (-1, 1)
name_to_metadata["waveform"]["shape"] = (-1, 640)
name_to_metadata["logmel_spectrogram"]["shape"] = (-1, 128)
name_to_metadata["segment_id"]["shape"] = (1,)
name_to_metadata["version_id"]["shape"] = (1,)
name_to_metadata["whinny_single"]["shape"] = (2,)
name_to_metadata["support"]["tf_dtype"] = tf.float32
name_to_metadata["waveform"]["tf_dtype"] = tf.float32
name_to_metadata["logmel_spectrogram"]["tf_dtype"] = tf.float32
name_to_metadata["segment_id"]["tf_dtype"] = tf.int32
name_to_metadata["version_id"]["tf_dtype"] = tf.int32
name_to_metadata["whinny_single"]["tf_dtype"] = tf.float32
name_to_metadata["support"]["padded_shape"] = (None, 1)
name_to_metadata["waveform"]["padded_shape"] = (None, 640)
name_to_metadata["logmel_spectrogram"]["padded_shape"] = (None, 128)
name_to_metadata["segment_id"]["padded_shape"] = (1,)
name_to_metadata["version_id"]["padded_shape"] = (1,)
name_to_metadata["whinny_single"]["padded_shape"] = (2,)
name_to_metadata["whinny_single"]["number_of_outputs"] = 2
name_to_metadata["support"]["placeholder_shape"] = (None, None, 1)
name_to_metadata["waveform"]["placeholder_shape"] = (None, None, 640)
name_to_metadata["logmel_spectrogram"]["placeholder_shape"] = (None, None, 128)
name_to_metadata["segment_id"]["placeholder_shape"] = (None, 1)
name_to_metadata["version_id"]["placeholder_shape"] = (None, 1)
name_to_metadata["whinny_single"]["placeholder_shape"] = (None, 2)
name_to_metadata = {k: name_to_metadata[k] for k in tf_names}
return name_to_metadata
def filter_names(all_path_list,
pos_variations,
neg_variations):
path_dict = collections.defaultdict(list)
for path in all_path_list:
path_split = path[:-10].split("_")
segment_id = int(path_split[-2])
version_id = int(path_split[-1])
name = "_".join(path_split[1:4])
path_dict[name].append(path)
all_path_list_new = list()
for k, v in path_dict.items():
if "pos" in k:
if pos_variations is None:
number_of_variations = len(v)
else:
number_of_variations = pos_variations
elif "neg" in k:
if neg_variations is None:
number_of_variations = len(v)
else:
number_of_variations = neg_variations
else:
raise ValueError
for i, vv in enumerate(v):
if i < number_of_variations:
all_path_list_new.append(v[i])
return all_path_list_new
def get_dataset_info(tfrecords_folder):
partitions = ["train",
"devel",
"test"]
path_list_dict = dict()
partition_size_dict = dict()
for partition in partitions:
partition_eff = partition
all_path_list = os.listdir(tfrecords_folder + "/" + partition_eff)
if partition_eff == "train":
all_path_list = filter_names(all_path_list,
pos_variations=5, # Multiple versions per positive sample exist -- offline random time shift.
neg_variations=None) # Get all negatives.
elif partition_eff in ["devel", "test"]:
all_path_list = filter_names(all_path_list,
pos_variations=1, # Only 1 version per positive sample exists (and should exist).
neg_variations=None) # Get all negatives.
else:
raise ValueError
all_path_list = [tfrecords_folder + "/" + partition_eff + "/" + name for name in all_path_list]
path_list_dict[partition] = all_path_list
partition_size_dict[partition] = len(all_path_list)
return path_list_dict, partition_size_dict
def get_config_dict_from_yaml(file_name):
# Read the parameters from the YAML file.
stream = open(YAML_CONFIGURATION_FOLDER + "/" + file_name + ".yaml", 'r')
CONFIG_DICT = yaml.load(stream)
stream.close()
# Get the list of TFRECORDS file paths per partition.
PATH_LIST_DICT, \
PARTITIONS_SIZE_DICT = get_dataset_info(TFRECORDS_FOLDER)
CONFIG_DICT["tfrecords_folder"] = TFRECORDS_FOLDER
CONFIG_DICT["output_folder"] = OUTPUT_FOLDER
CONFIG_DICT["path_list_dict"] = PATH_LIST_DICT
CONFIG_DICT["train_size"] = PARTITIONS_SIZE_DICT["train"]
CONFIG_DICT["devel_size"] = PARTITIONS_SIZE_DICT["devel"]
CONFIG_DICT["test_size"] = PARTITIONS_SIZE_DICT["test"]
CONFIG_DICT["model_configuration"]["name_to_metadata"] = get_name_to_metadata(CONFIG_DICT["tf_names"])
CONFIG_DICT["results_summary_path"] = OUTPUT_FOLDER + "/" + CONFIG_DICT["method_string"] + "/results_summary"
CONFIG_DICT["target_to_measures"] = {"whinny_single": ["pos_au_pr",]}
return CONFIG_DICT