gaitsetpy.preprocessing.preprocessors

  1'''
  2Preprocessor classes for gait data preprocessing.
  3
  4This module contains individual preprocessor classes that inherit from BasePreprocessor
  5and provide specific preprocessing functionality.
  6
  7Maintainer: @aharshit123456
  8'''
  9
 10from typing import Union, Dict, Any
 11import numpy as np
 12import pandas as pd
 13from scipy.signal import butter, filtfilt
 14from ..core.base_classes import BasePreprocessor
 15
 16
 17class ClippingPreprocessor(BasePreprocessor):
 18    """
 19    Preprocessor for clipping values to a specified range.
 20    """
 21    
 22    def __init__(self, min_val: float = -1, max_val: float = 1):
 23        super().__init__(
 24            name="clipping",
 25            description="Clips values in the data to be within a specified range"
 26        )
 27        self.config = {
 28            'min_val': min_val,
 29            'max_val': max_val
 30        }
 31    
 32    def fit(self, data: Union[pd.DataFrame, np.ndarray], **kwargs):
 33        """
 34        Fit the preprocessor (no fitting needed for clipping).
 35        
 36        Args:
 37            data: Input data to fit on
 38            **kwargs: Additional arguments
 39        """
 40        # Update config with any passed arguments
 41        self.config.update({k: v for k, v in kwargs.items() if k in ['min_val', 'max_val']})
 42        self.fitted = True
 43    
 44    def transform(self, data: Union[pd.DataFrame, np.ndarray], **kwargs) -> Union[pd.DataFrame, np.ndarray]:
 45        """
 46        Clip values in the data to be within the specified range.
 47        
 48        Args:
 49            data: Input data to transform
 50            **kwargs: Additional arguments
 51            
 52        Returns:
 53            Clipped data
 54        """
 55        min_val = kwargs.get('min_val', self.config['min_val'])
 56        max_val = kwargs.get('max_val', self.config['max_val'])
 57        
 58        return np.clip(data, min_val, max_val)
 59
 60
 61class NoiseRemovalPreprocessor(BasePreprocessor):
 62    """
 63    Preprocessor for removing noise using moving average filter.
 64    """
 65    
 66    def __init__(self, window_size: int = 5):
 67        super().__init__(
 68            name="noise_removal",
 69            description="Applies a moving average filter to reduce noise"
 70        )
 71        self.config = {
 72            'window_size': window_size
 73        }
 74    
 75    def fit(self, data: Union[pd.DataFrame, np.ndarray], **kwargs):
 76        """
 77        Fit the preprocessor (no fitting needed for noise removal).
 78        
 79        Args:
 80            data: Input data to fit on
 81            **kwargs: Additional arguments
 82        """
 83        self.config.update({k: v for k, v in kwargs.items() if k in ['window_size']})
 84        self.fitted = True
 85    
 86    def transform(self, data: Union[pd.DataFrame, np.ndarray], **kwargs) -> Union[pd.DataFrame, np.ndarray]:
 87        """
 88        Apply a moving average filter to reduce noise.
 89        
 90        Args:
 91            data: Input data to transform
 92            **kwargs: Additional arguments
 93            
 94        Returns:
 95            Noise-reduced data
 96        """
 97        window_size = kwargs.get('window_size', self.config['window_size'])
 98        
 99        if isinstance(data, pd.DataFrame):
100            return data.rolling(window=window_size, center=True).mean().bfill().ffill()
101        elif isinstance(data, pd.Series):
102            return data.rolling(window=window_size, center=True).mean().bfill().ffill()
103        else:
104            # For numpy arrays, use uniform filter
105            from scipy.ndimage import uniform_filter1d
106            return uniform_filter1d(data, size=window_size, mode='nearest')
107
108
109class OutlierRemovalPreprocessor(BasePreprocessor):
110    """
111    Preprocessor for removing outliers using Z-score method.
112    """
113    
114    def __init__(self, threshold: float = 3):
115        super().__init__(
116            name="outlier_removal",
117            description="Removes outliers beyond a given threshold using the Z-score method"
118        )
119        self.config = {
120            'threshold': threshold
121        }
122        self.mean_ = None
123        self.std_ = None
124    
125    def fit(self, data: Union[pd.DataFrame, np.ndarray], **kwargs):
126        """
127        Fit the preprocessor by computing mean and standard deviation.
128        
129        Args:
130            data: Input data to fit on
131            **kwargs: Additional arguments
132        """
133        self.config.update({k: v for k, v in kwargs.items() if k in ['threshold']})
134        
135        if isinstance(data, (pd.DataFrame, pd.Series)):
136            self.mean_ = data.mean()
137            self.std_ = data.std()
138        else:
139            self.mean_ = np.mean(data)
140            self.std_ = np.std(data)
141        
142        self.fitted = True
143    
144    def transform(self, data: Union[pd.DataFrame, np.ndarray], **kwargs) -> Union[pd.DataFrame, np.ndarray]:
145        """
146        Remove outliers beyond the threshold using Z-score method.
147        
148        Args:
149            data: Input data to transform
150            **kwargs: Additional arguments
151            
152        Returns:
153            Data with outliers removed
154        """
155        threshold = kwargs.get('threshold', self.config['threshold'])
156        
157        if isinstance(data, (pd.DataFrame, pd.Series)):
158            z_scores = (data - self.mean_).abs() / self.std_
159            return data[z_scores <= threshold]
160        else:
161            z_scores = np.abs(data - self.mean_) / self.std_
162            return data[z_scores <= threshold]
163
164
165class BaselineRemovalPreprocessor(BasePreprocessor):
166    """
167    Preprocessor for removing baseline by subtracting the mean.
168    """
169    
170    def __init__(self):
171        super().__init__(
172            name="baseline_removal",
173            description="Removes baseline by subtracting the mean"
174        )
175        self.mean_ = None
176    
177    def fit(self, data: Union[pd.DataFrame, np.ndarray], **kwargs):
178        """
179        Fit the preprocessor by computing the mean.
180        
181        Args:
182            data: Input data to fit on
183            **kwargs: Additional arguments
184        """
185        if isinstance(data, (pd.DataFrame, pd.Series)):
186            self.mean_ = data.mean()
187        else:
188            self.mean_ = np.mean(data)
189        
190        self.fitted = True
191    
192    def transform(self, data: Union[pd.DataFrame, np.ndarray], **kwargs) -> Union[pd.DataFrame, np.ndarray]:
193        """
194        Remove baseline by subtracting the mean.
195        
196        Args:
197            data: Input data to transform
198            **kwargs: Additional arguments
199            
200        Returns:
201            Baseline-corrected data
202        """
203        return data - self.mean_
204
205
206class DriftRemovalPreprocessor(BasePreprocessor):
207    """
208    Preprocessor for removing low-frequency drift using high-pass filter.
209    """
210    
211    def __init__(self, cutoff: float = 0.01, fs: int = 100):
212        super().__init__(
213            name="drift_removal",
214            description="Removes low-frequency drift using a high-pass filter"
215        )
216        self.config = {
217            'cutoff': cutoff,
218            'fs': fs
219        }
220    
221    def fit(self, data: Union[pd.DataFrame, np.ndarray], **kwargs):
222        """
223        Fit the preprocessor (no fitting needed for drift removal).
224        
225        Args:
226            data: Input data to fit on
227            **kwargs: Additional arguments
228        """
229        self.config.update({k: v for k, v in kwargs.items() if k in ['cutoff', 'fs']})
230        self.fitted = True
231    
232    def transform(self, data: Union[pd.DataFrame, np.ndarray], **kwargs) -> Union[pd.DataFrame, np.ndarray]:
233        """
234        Remove low-frequency drift using a high-pass filter.
235        
236        Args:
237            data: Input data to transform
238            **kwargs: Additional arguments
239            
240        Returns:
241            Drift-corrected data
242        """
243        cutoff = kwargs.get('cutoff', self.config['cutoff'])
244        fs = kwargs.get('fs', self.config['fs'])
245        
246        b, a = butter(1, cutoff / (fs / 2), btype='highpass')
247        
248        if isinstance(data, (pd.DataFrame, pd.Series)):
249            return pd.Series(filtfilt(b, a, data), index=data.index)
250        else:
251            return filtfilt(b, a, data)
252
253
254class HighFrequencyNoiseRemovalPreprocessor(BasePreprocessor):
255    """
256    Preprocessor for removing high-frequency noise using low-pass filter.
257    """
258    
259    def __init__(self, cutoff: float = 10, fs: int = 100):
260        super().__init__(
261            name="high_frequency_noise_removal",
262            description="Applies a low-pass filter to remove high-frequency noise"
263        )
264        self.config = {
265            'cutoff': cutoff,
266            'fs': fs
267        }
268    
269    def fit(self, data: Union[pd.DataFrame, np.ndarray], **kwargs):
270        """
271        Fit the preprocessor (no fitting needed for filtering).
272        
273        Args:
274            data: Input data to fit on
275            **kwargs: Additional arguments
276        """
277        self.config.update({k: v for k, v in kwargs.items() if k in ['cutoff', 'fs']})
278        self.fitted = True
279    
280    def transform(self, data: Union[pd.DataFrame, np.ndarray], **kwargs) -> Union[pd.DataFrame, np.ndarray]:
281        """
282        Apply a low-pass filter to remove high-frequency noise.
283        
284        Args:
285            data: Input data to transform
286            **kwargs: Additional arguments
287            
288        Returns:
289            Filtered data
290        """
291        cutoff = kwargs.get('cutoff', self.config['cutoff'])
292        fs = kwargs.get('fs', self.config['fs'])
293        
294        b, a = butter(1, cutoff / (fs / 2), btype='lowpass')
295        
296        if isinstance(data, (pd.DataFrame, pd.Series)):
297            return pd.Series(filtfilt(b, a, data), index=data.index)
298        else:
299            return filtfilt(b, a, data)
300
301
302class LowFrequencyNoiseRemovalPreprocessor(BasePreprocessor):
303    """
304    Preprocessor for removing low-frequency noise using high-pass filter.
305    """
306    
307    def __init__(self, cutoff: float = 0.5, fs: int = 100):
308        super().__init__(
309            name="low_frequency_noise_removal",
310            description="Applies a high-pass filter to remove low-frequency noise"
311        )
312        self.config = {
313            'cutoff': cutoff,
314            'fs': fs
315        }
316    
317    def fit(self, data: Union[pd.DataFrame, np.ndarray], **kwargs):
318        """
319        Fit the preprocessor (no fitting needed for filtering).
320        
321        Args:
322            data: Input data to fit on
323            **kwargs: Additional arguments
324        """
325        self.config.update({k: v for k, v in kwargs.items() if k in ['cutoff', 'fs']})
326        self.fitted = True
327    
328    def transform(self, data: Union[pd.DataFrame, np.ndarray], **kwargs) -> Union[pd.DataFrame, np.ndarray]:
329        """
330        Apply a high-pass filter to remove low-frequency noise.
331        
332        Args:
333            data: Input data to transform
334            **kwargs: Additional arguments
335            
336        Returns:
337            Filtered data
338        """
339        cutoff = kwargs.get('cutoff', self.config['cutoff'])
340        fs = kwargs.get('fs', self.config['fs'])
341        
342        b, a = butter(1, cutoff / (fs / 2), btype='highpass')
343        
344        if isinstance(data, (pd.DataFrame, pd.Series)):
345            return pd.Series(filtfilt(b, a, data), index=data.index)
346        else:
347            return filtfilt(b, a, data)
348
349
350class ArtifactRemovalPreprocessor(BasePreprocessor):
351    """
352    Preprocessor for removing artifacts by interpolating missing values.
353    """
354    
355    def __init__(self, method: str = "linear"):
356        super().__init__(
357            name="artifact_removal",
358            description="Removes artifacts by interpolating missing values"
359        )
360        self.config = {
361            'method': method
362        }
363    
364    def fit(self, data: Union[pd.DataFrame, np.ndarray], **kwargs):
365        """
366        Fit the preprocessor (no fitting needed for interpolation).
367        
368        Args:
369            data: Input data to fit on
370            **kwargs: Additional arguments
371        """
372        self.config.update({k: v for k, v in kwargs.items() if k in ['method']})
373        self.fitted = True
374    
375    def transform(self, data: Union[pd.DataFrame, np.ndarray], **kwargs) -> Union[pd.DataFrame, np.ndarray]:
376        """
377        Remove artifacts by interpolating missing values.
378        
379        Args:
380            data: Input data to transform
381            **kwargs: Additional arguments
382            
383        Returns:
384            Artifact-free data
385        """
386        method = kwargs.get('method', self.config['method'])
387        
388        if isinstance(data, (pd.DataFrame, pd.Series)):
389            return data.interpolate(method=method).bfill().ffill()
390        else:
391            # For numpy arrays, use linear interpolation
392            from scipy.interpolate import interp1d
393            x = np.arange(len(data))
394            valid_mask = ~np.isnan(data)
395            if np.any(valid_mask):
396                f = interp1d(x[valid_mask], data[valid_mask], kind='linear', fill_value='extrapolate')
397                return f(x)
398            else:
399                return data
400
401
402class TrendRemovalPreprocessor(BasePreprocessor):
403    """
404    Preprocessor for removing trends using polynomial fitting.
405    """
406    
407    def __init__(self, order: int = 2):
408        super().__init__(
409            name="trend_removal",
410            description="Removes trends using polynomial fitting"
411        )
412        self.config = {
413            'order': order
414        }
415    
416    def fit(self, data: Union[pd.DataFrame, np.ndarray], **kwargs):
417        """
418        Fit the preprocessor (no fitting needed for detrending).
419        
420        Args:
421            data: Input data to fit on
422            **kwargs: Additional arguments
423        """
424        self.config.update({k: v for k, v in kwargs.items() if k in ['order']})
425        self.fitted = True
426    
427    def transform(self, data: Union[pd.DataFrame, np.ndarray], **kwargs) -> Union[pd.DataFrame, np.ndarray]:
428        """
429        Remove trends using polynomial fitting.
430        
431        Args:
432            data: Input data to transform
433            **kwargs: Additional arguments
434            
435        Returns:
436            Detrended data
437        """
438        order = kwargs.get('order', self.config['order'])
439        
440        if isinstance(data, (pd.DataFrame, pd.Series)):
441            x = np.arange(len(data))
442            poly_coeffs = np.polyfit(x, data, order)
443            trend = np.polyval(poly_coeffs, x)
444            return data - trend
445        else:
446            x = np.arange(len(data))
447            poly_coeffs = np.polyfit(x, data, order)
448            trend = np.polyval(poly_coeffs, x)
449            return data - trend
450
451
452class DCOffsetRemovalPreprocessor(BasePreprocessor):
453    """
454    Preprocessor for removing DC offset by subtracting the mean.
455    """
456    
457    def __init__(self):
458        super().__init__(
459            name="dc_offset_removal",
460            description="Removes DC offset by subtracting the mean"
461        )
462        self.mean_ = None
463    
464    def fit(self, data: Union[pd.DataFrame, np.ndarray], **kwargs):
465        """
466        Fit the preprocessor by computing the mean.
467        
468        Args:
469            data: Input data to fit on
470            **kwargs: Additional arguments
471        """
472        if isinstance(data, (pd.DataFrame, pd.Series)):
473            self.mean_ = data.mean()
474        else:
475            self.mean_ = np.mean(data)
476        
477        self.fitted = True
478    
479    def transform(self, data: Union[pd.DataFrame, np.ndarray], **kwargs) -> Union[pd.DataFrame, np.ndarray]:
480        """
481        Remove DC offset by subtracting the mean.
482        
483        Args:
484            data: Input data to transform
485            **kwargs: Additional arguments
486            
487        Returns:
488            DC-corrected data
489        """
490        return data - self.mean_