diff --git a/audinterface/core/process.py b/audinterface/core/process.py index c7e9977..a724873 100644 --- a/audinterface/core/process.py +++ b/audinterface/core/process.py @@ -2,6 +2,7 @@ import inspect import itertools import os +import pathlib import typing import numpy as np @@ -36,6 +37,15 @@ def identity(signal, sampling_rate) -> np.ndarray: return signal +def data_identity(data): + r"""Default processing function for non-signal data. + + In analogy to the identity function for signals, + it returns the data itself + """ + return data + + class Process: r"""Processing interface. @@ -212,6 +222,7 @@ def __init__( self.num_workers = num_workers r"""Number of workers.""" + # leaving process_func unaltered here, currently deferred self.process_func = process_func r"""Processing function.""" @@ -271,10 +282,14 @@ def _process_file( end = utils.to_timedelta(end, self.sampling_rate) ext = audeer.file_extension(file).lower() - + # print(ext, exit is None) # Text files if ext in ["json", "txt"]: + self._processing_mode = "text" # convenience + print("set processing mode") data = utils.read_text(file, root=root) + # should be idempotent, but is currently deferred + # self._handle_processing_func_args(data_type="text") y, file = self._process_data( data, idx=idx, @@ -288,6 +303,7 @@ def _process_file( # Audio/video files else: + self._processing_mode = "signal" # convenience signal, sampling_rate = utils.read_audio( file, start=start, @@ -568,6 +584,22 @@ def process_folder( process_func_args=process_func_args, ) + def _set_processing_mode(self, index): + """Set processsing mode to text depending on input data extension.""" + + self._processing_mode = "signal" + + if audformat.is_segmented_index(index): + extensions = [ + pathlib.Path(x).suffix[1:] for x in index.get_level_values("file") + ] + else: + # never getting filewise indices so far? + pass + + if set(extensions).issubset(set(["json", "txt"])): + self._processing_mode = "text" + def _process_index_wo_segment( self, index: pd.Index, @@ -592,6 +624,9 @@ def _process_index_wo_segment( for idx, (file, start, end) in enumerate(index) ] + # modify procesing mode variable when getting json or text files + self._set_processing_mode(index) + xs = audeer.run_tasks( self._process_file, params, @@ -601,20 +636,38 @@ def _process_index_wo_segment( task_description=f"Process {len(index)} segments", ) - y = list(itertools.chain.from_iterable([x[0] for x in xs])) - files = list(itertools.chain.from_iterable([x[1] for x in xs])) - starts = list(itertools.chain.from_iterable([x[2] for x in xs])) - ends = list(itertools.chain.from_iterable([x[3] for x in xs])) + if self._processing_mode == "text": + y = [x[0] for x in xs] + files = list(itertools.chain.from_iterable([x[1] for x in xs])) + starts = [x[2] for x in xs] + ends = [x[3] for x in xs] + if ( + len(audeer.unique(starts)) == 1 + and audeer.unique(starts)[0] is None + and len(audeer.unique(ends)) == 1 + and audeer.unique(ends)[0] is None + ): + index = audformat.filewise_index(files) + else: + # leave index untouched + # index = audformat.segmented_index(files, starts, ends) + pass - if ( - len(audeer.unique(starts)) == 1 - and audeer.unique(starts)[0] is None - and len(audeer.unique(ends)) == 1 - and audeer.unique(ends)[0] is None - ): - index = audformat.filewise_index(files) else: - index = audformat.segmented_index(files, starts, ends) + y = list(itertools.chain.from_iterable([x[0] for x in xs])) + files = list(itertools.chain.from_iterable([x[1] for x in xs])) + starts = list(itertools.chain.from_iterable([x[2] for x in xs])) + ends = list(itertools.chain.from_iterable([x[3] for x in xs])) + + if ( + len(audeer.unique(starts)) == 1 + and audeer.unique(starts)[0] is None + and len(audeer.unique(ends)) == 1 + and audeer.unique(ends)[0] is None + ): + index = audformat.filewise_index(files) + else: + index = audformat.segmented_index(files, starts, ends) y = pd.Series(y, index) @@ -686,9 +739,7 @@ def process_index( ) y = self._process_index_wo_segment( - segmented_index, - root, - process_func_args=process_func_args, + segmented_index, root, process_func_args=process_func_args ) if cache_path is not None: @@ -1117,6 +1168,25 @@ def _helper(x): return y + def _handle_processing_func_for_text(self): + """Handle data identity based on input modality. + + As text data never have sampling rate the + functional is exchanged + + """ + + f = data_identity + process_func = f + self.process_func = process_func + signature = inspect.signature(process_func) + self._process_func_signature = dict(signature.parameters) + + @staticmethod + def _handle_processing_func_args_for_text(process_func_args): + """If process_func_args has an sr, strip it.""" + ... + def _call_data( self, data: typing.Any, @@ -1146,7 +1216,9 @@ def _call_data( """ process_func_args = process_func_args or self.process_func_args special_args = self._special_args(idx, root, file, process_func_args) + self._handle_processing_func_for_text() y = self.process_func(data, **special_args, **process_func_args) + return y def _special_args( diff --git a/audinterface/utils/__init__.py b/audinterface/utils/__init__.py index 0679a8b..0d9ed08 100644 --- a/audinterface/utils/__init__.py +++ b/audinterface/utils/__init__.py @@ -1,4 +1,5 @@ from audinterface.core.utils import read_audio +from audinterface.core.utils import read_text from audinterface.core.utils import signal_index from audinterface.core.utils import sliding_window from audinterface.core.utils import to_timedelta diff --git a/tests/test_process_text.py b/tests/test_process_text.py index 4238ace..9d0878e 100644 --- a/tests/test_process_text.py +++ b/tests/test_process_text.py @@ -204,8 +204,9 @@ def test_process_index( ): cache_root = os.path.join(tmpdir, "cache") + process_func = None process = audinterface.Process( - process_func=None, + process_func=process_func, num_workers=num_workers, multiprocessing=multiprocessing, verbose=False, @@ -239,9 +240,16 @@ def test_process_index( ) if preserve_index: pd.testing.assert_index_equal(y.index, index) - for (path, _, _), value in y.items(): - assert audinterface.utils.read_text(path) == data - assert value == data + + # only works for preserved index, otherwise too many to unpack + if preserve_index: + for (path, _, _), value in y.items(): + assert audinterface.utils.read_text(path) == data + assert value == data + else: + for path, value in y.items(): + assert audinterface.utils.read_text(path) == data + assert value == data # Segmented index with relative paths index = audformat.segmented_index( @@ -256,9 +264,9 @@ def test_process_index( ) if preserve_index: pd.testing.assert_index_equal(y.index, index) - for (file, _, _), value in y.items(): - assert audinterface.utils.read_text(file, root=root) == data - assert value == data + for (file, _, _), value in y.items(): + assert audinterface.utils.read_text(file, root=root) == data + assert value == data # Filewise index with absolute paths index = audformat.filewise_index(path) @@ -274,7 +282,7 @@ def test_process_index( else: expected_index = audformat.filewise_index(files=list(index)) pd.testing.assert_index_equal(y.index, expected_index) - for (path, _, _), value in y.items(): + for path, value in y.items(): assert audinterface.utils.read_text(path) == data assert value == data @@ -291,7 +299,7 @@ def test_process_index( assert audinterface.utils.read_text(file, root=root) == data assert value == data else: - for (file, _, _), value in y.items(): + for file, value in y.items(): assert audinterface.utils.read_text(file, root=root) == data assert value == data @@ -305,7 +313,7 @@ def test_process_index( os.remove(path) # Fails because second file does not exist - with pytest.raises(RuntimeError): + with pytest.raises(FileNotFoundError): process.process_index( index, preserve_index=preserve_index,