gaitsetpy.dataset

 18class DaphnetLoader(BaseDatasetLoader):
 19    """
 20    Daphnet dataset loader class.
 21    
 22    This class handles loading and processing of the Daphnet dataset for gait analysis.
 23    """
 24    
 25    def __init__(self, max_workers: int = 8):
 26        """
 27        Initialize Daphnet loader with concurrent download support.
 28        
 29        Args:
 30            max_workers: Maximum number of concurrent download threads (default: 8)
 31        """
 32        super().__init__(
 33            name="daphnet",
 34            description="Daphnet Freezing of Gait Dataset - Contains accelerometer data from subjects with Parkinson's disease",
 35            max_workers=max_workers
 36        )
 37        self.metadata = {
 38            'sensors': ['shank', 'thigh', 'trunk'],
 39            'components': ['h_fd', 'v', 'h_l'],  # horizontal forward, vertical, horizontal lateral
 40            'sampling_frequency': 64,
 41            'annotations': {
 42                0: 'not_valid',
 43                1: 'no_freeze',
 44                2: 'freeze'
 45            }
 46        }
 47    
 48    def load_data(self, data_dir: str, **kwargs) -> Tuple[List[pd.DataFrame], List[str]]:
 49        """
 50        Load Daphnet dataset from the specified directory.
 51        
 52        Args:
 53            data_dir: Directory to store/find the dataset
 54            **kwargs: Additional arguments (unused for Daphnet)
 55            
 56        Returns:
 57            Tuple of (data_list, names_list)
 58        """
 59        # Download and extract if needed
 60        download_dataset("daphnet", data_dir)
 61        extract_dataset("daphnet", data_dir)
 62        
 63        file_path = os.path.join(data_dir, "dataset_fog_release/dataset")
 64        daphnet_data = []
 65        daphnet_names = []
 66        
 67        # Load all subject files
 68        for file in sorted(glob(os.path.join(file_path, "S*.txt"))):
 69            # Extract filename from path
 70            filename = os.path.basename(file)
 71            daphnet_names.append(filename)
 72            
 73            # Load CSV with proper column names
 74            column_names = [
 75                "time", "shank_h_fd", "shank_v", "shank_h_l", 
 76                "thigh_h_fd", "thigh_v", "thigh_h_l", 
 77                "trunk_h_fd", "trunk_v", "trunk_h_l", "annotations"
 78            ]
 79            
 80            df = pd.read_csv(file, sep=" ", names=column_names)
 81            
 82            # Set time as index
 83            df = df.set_index("time")
 84            
 85            # Calculate magnitude for each sensor
 86            df["thigh"] = np.sqrt(df["thigh_h_l"]**2 + df["thigh_v"]**2 + df["thigh_h_fd"]**2)
 87            df["shank"] = np.sqrt(df["shank_h_l"]**2 + df["shank_v"]**2 + df["shank_h_fd"]**2)
 88            df["trunk"] = np.sqrt(df["trunk_h_l"]**2 + df["trunk_v"]**2 + df["trunk_h_fd"]**2)
 89            
 90            # Reorder columns for consistency
 91            df = df[["shank", "shank_h_fd", "shank_v", "shank_h_l", 
 92                    "thigh", "thigh_h_fd", "thigh_v", "thigh_h_l", 
 93                    "trunk", "trunk_h_fd", "trunk_v", "trunk_h_l", "annotations"]]
 94            
 95            daphnet_data.append(df)
 96        
 97        # Store loaded data
 98        self.data = daphnet_data
 99        self.names = daphnet_names
100        
101        return daphnet_data, daphnet_names
102    
103    def create_sliding_windows(self, data: List[pd.DataFrame], names: List[str], 
104                             window_size: int = 192, step_size: int = 32) -> List[Dict]:
105        """
106        Create sliding windows from the Daphnet dataset.
107        
108        Args:
109            data: List of DataFrames containing Daphnet data
110            names: List of names corresponding to the data
111            window_size: Size of the sliding window (default: 192)
112            step_size: Step size for the sliding window (default: 32)
113            
114        Returns:
115            List of dictionaries containing sliding windows for each DataFrame
116        """
117        windows_data = []
118        
119        for idx, df in enumerate(data):
120            # Filter out invalid data (annotations == 0)
121            df_filtered = df[df.annotations > 0]
122            
123            if df_filtered.empty:
124                continue
125                
126            windows = []
127            processed_columns = set()
128            
129            # Process each sensor column
130            for col in df_filtered.columns:
131                if col != "annotations" and col not in processed_columns:
132                    window_data = sliding_window(df_filtered[col], window_size, step_size)
133                    windows.append({"name": col, "data": window_data})
134                    processed_columns.add(col)
135            
136            # Include annotations separately
137            annotations_window = sliding_window(df_filtered["annotations"], window_size, step_size)
138            windows.append({"name": "annotations", "data": annotations_window})
139            
140            windows_data.append({"name": names[idx], "windows": windows})
141        
142        return windows_data
143    
144    def get_supported_formats(self) -> List[str]:
145        """
146        Get list of supported file formats for Daphnet dataset.
147        
148        Returns:
149            List of supported file extensions
150        """
151        return ['.txt']
152    
153    def get_sensor_info(self) -> Dict[str, List[str]]:
154        """
155        Get information about sensors in the dataset.
156        
157        Returns:
158            Dictionary containing sensor information
159        """
160        return {
161            'sensors': self.metadata['sensors'],
162            'components': self.metadata['components'],
163            'sampling_frequency': self.metadata['sampling_frequency']
164        }
165    
166    def get_annotation_info(self) -> Dict[int, str]:
167        """
168        Get information about annotations in the dataset.
169        
170        Returns:
171            Dictionary mapping annotation values to descriptions
172        """
173        return self.metadata['annotations']

 17class MobiFallLoader(BaseDatasetLoader):
 18    """
 19    MobiFall dataset loader class.
 20    
 21    This class handles loading and processing of the MobiFall dataset for gait analysis.
 22    """
 23    
 24    def __init__(self, max_workers: int = 8):
 25        """
 26        Initialize MobiFall loader with concurrent download support.
 27        
 28        Args:
 29            max_workers: Maximum number of concurrent download threads (default: 8)
 30        """
 31        super().__init__(
 32            name="mobifall",
 33            description="MobiFall Dataset - Contains accelerometer and gyroscope data for fall detection",
 34            max_workers=max_workers
 35        )
 36        self.metadata = {
 37            'sensors': ['accelerometer', 'gyroscope'],
 38            'components': ['x', 'y', 'z'],
 39            'sampling_frequency': 100,  # Typical for MobiFall
 40            'activities': ['ADL', 'FALL']  # Activities of Daily Living and Falls
 41        }
 42    
 43    def load_data(self, data_dir: str, **kwargs) -> Tuple[List[pd.DataFrame], List[str]]:
 44        """
 45        Load MobiFall dataset from the specified directory.
 46        
 47        Args:
 48            data_dir: Directory to store/find the dataset
 49            **kwargs: Additional arguments (unused for MobiFall)
 50            
 51        Returns:
 52            Tuple of (data_list, names_list)
 53        """
 54        # TODO: Implement MobiFall data loading
 55        # This is a placeholder implementation
 56        print("MobiFall data loading is not yet implemented")
 57        return [], []
 58    
 59    def create_sliding_windows(self, data: List[pd.DataFrame], names: List[str], 
 60                             window_size: int = 192, step_size: int = 32) -> List[Dict]:
 61        """
 62        Create sliding windows from the MobiFall dataset.
 63        
 64        Args:
 65            data: List of DataFrames containing MobiFall data
 66            names: List of names corresponding to the data
 67            window_size: Size of the sliding window (default: 192)
 68            step_size: Step size for the sliding window (default: 32)
 69            
 70        Returns:
 71            List of dictionaries containing sliding windows for each DataFrame
 72        """
 73        # TODO: Implement MobiFall sliding window creation
 74        # This is a placeholder implementation
 75        print("MobiFall sliding window creation is not yet implemented")
 76        return []
 77    
 78    def get_supported_formats(self) -> List[str]:
 79        """
 80        Get list of supported file formats for MobiFall dataset.
 81        
 82        Returns:
 83            List of supported file extensions
 84        """
 85        return ['.csv', '.txt']
 86    
 87    def get_sensor_info(self) -> Dict[str, List[str]]:
 88        """
 89        Get information about sensors in the dataset.
 90        
 91        Returns:
 92            Dictionary containing sensor information
 93        """
 94        return {
 95            'sensors': self.metadata['sensors'],
 96            'components': self.metadata['components'],
 97            'sampling_frequency': self.metadata['sampling_frequency']
 98        }
 99    
100    def get_activity_info(self) -> List[str]:
101        """
102        Get information about activities in the dataset.
103        
104        Returns:
105            List of activity types
106        """
107        return self.metadata['activities']

 17class ArduousLoader(BaseDatasetLoader):
 18    """
 19    Arduous dataset loader class.
 20    
 21    This class handles loading and processing of the Arduous dataset for gait analysis.
 22    """
 23    
 24    def __init__(self, max_workers: int = 8):
 25        """
 26        Initialize Arduous loader with concurrent download support.
 27        
 28        Args:
 29            max_workers: Maximum number of concurrent download threads (default: 8)
 30        """
 31        super().__init__(
 32            name="arduous",
 33            description="Arduous Dataset - Contains multi-sensor wearable data for daily activity recognition",
 34            max_workers=max_workers
 35        )
 36        self.metadata = {
 37            'sensors': ['accelerometer', 'gyroscope', 'magnetometer'],
 38            'components': ['x', 'y', 'z'],
 39            'sampling_frequency': 50,  # Typical for Arduous
 40            'activities': ['walking', 'running', 'sitting', 'standing', 'lying']
 41        }
 42    
 43    def load_data(self, data_dir: str, **kwargs) -> Tuple[List[pd.DataFrame], List[str]]:
 44        """
 45        Load Arduous dataset from the specified directory.
 46        
 47        Args:
 48            data_dir: Directory to store/find the dataset
 49            **kwargs: Additional arguments (unused for Arduous)
 50            
 51        Returns:
 52            Tuple of (data_list, names_list)
 53        """
 54        # TODO: Implement Arduous data loading
 55        # This is a placeholder implementation
 56        print("Arduous data loading is not yet implemented")
 57        return [], []
 58    
 59    def create_sliding_windows(self, data: List[pd.DataFrame], names: List[str], 
 60                             window_size: int = 192, step_size: int = 32) -> List[Dict]:
 61        """
 62        Create sliding windows from the Arduous dataset.
 63        
 64        Args:
 65            data: List of DataFrames containing Arduous data
 66            names: List of names corresponding to the data
 67            window_size: Size of the sliding window (default: 192)
 68            step_size: Step size for the sliding window (default: 32)
 69            
 70        Returns:
 71            List of dictionaries containing sliding windows for each DataFrame
 72        """
 73        # TODO: Implement Arduous sliding window creation
 74        # This is a placeholder implementation
 75        print("Arduous sliding window creation is not yet implemented")
 76        return []
 77    
 78    def get_supported_formats(self) -> List[str]:
 79        """
 80        Get list of supported file formats for Arduous dataset.
 81        
 82        Returns:
 83            List of supported file extensions
 84        """
 85        return ['.csv', '.txt']
 86    
 87    def get_sensor_info(self) -> Dict[str, List[str]]:
 88        """
 89        Get information about sensors in the dataset.
 90        
 91        Returns:
 92            Dictionary containing sensor information
 93        """
 94        return {
 95            'sensors': self.metadata['sensors'],
 96            'components': self.metadata['components'],
 97            'sampling_frequency': self.metadata['sampling_frequency']
 98        }
 99    
100    def get_activity_info(self) -> List[str]:
101        """
102        Get information about activities in the dataset.
103        
104        Returns:
105            List of activity types
106        """
107        return self.metadata['activities']

 25class PhysioNetLoader(BaseDatasetLoader):
 26    """
 27    PhysioNet VGRF dataset loader class.
 28    
 29    This class handles loading and processing of the PhysioNet Gait in Parkinson's Disease dataset.
 30    The dataset contains vertical ground reaction force (VGRF) data from subjects with Parkinson's 
 31    disease and healthy controls.
 32    
 33    Features concurrent downloading for efficient data retrieval.
 34    """
 35    
 36    def __init__(self, max_workers: int = 8):
 37        """
 38        Initialize PhysioNet loader with concurrent download support.
 39        
 40        Args:
 41            max_workers: Maximum number of concurrent download threads (default: 8)
 42        """
 43        super().__init__(
 44            name="physionet",
 45            description="PhysioNet Gait in Parkinson's Disease Dataset - Contains VGRF data from subjects with Parkinson's disease and healthy controls",
 46            max_workers=max_workers
 47        )
 48        self.metadata = {
 49            'sensors': ['VGRF_L1', 'VGRF_L2', 'VGRF_L3', 'VGRF_L4', 'VGRF_L5', 'VGRF_L6', 'VGRF_L7', 'VGRF_L8',
 50                       'VGRF_R1', 'VGRF_R2', 'VGRF_R3', 'VGRF_R4', 'VGRF_R5', 'VGRF_R6', 'VGRF_R7', 'VGRF_R8'],
 51            'sampling_frequency': 100,  # 100 Hz sampling frequency
 52            'subjects': {
 53                'Co': 'Control subjects',
 54                'Pt': 'Parkinson\'s disease patients'
 55            },
 56            'window_size': 600,  # 6 seconds at 100 Hz
 57            'url': 'https://physionet.org/files/gaitpdb/1.0.0/'
 58        }
 59        self.labels = []
 60        self.subject_types = []
 61    
 62    def _download_physionet_data(self, data_dir: str) -> str:
 63        """
 64        Download PhysioNet dataset if not already present using concurrent downloads.
 65        
 66        This method uses multi-threaded downloading to significantly speed up the
 67        download process for the 100+ files in the PhysioNet dataset.
 68        
 69        Args:
 70            data_dir: Directory to store the dataset
 71            
 72        Returns:
 73            Path to the downloaded/existing dataset directory
 74        """
 75        dataset_path = os.path.join(data_dir, "physionet_gaitpdb")
 76        
 77        if os.path.exists(dataset_path) and len(os.listdir(dataset_path)) > 0:
 78            print(f"PhysioNet dataset already exists at: {dataset_path}")
 79            return dataset_path
 80        
 81        os.makedirs(dataset_path, exist_ok=True)
 82        
 83        # Download the dataset files
 84        base_url = "https://physionet.org/files/gaitpdb/1.0.0/"
 85        
 86        # Get list of files (basic file names based on the reference)
 87        file_patterns = [
 88            # Control subjects - Ga prefix
 89            *[f"GaCo{i:02d}_{j:02d}.txt" for i in range(1, 18) for j in range(1, 3)],
 90            "GaCo22_01.txt", "GaCo22_10.txt",
 91            
 92            # Parkinson's patients - Ga prefix
 93            *[f"GaPt{i:02d}_{j:02d}.txt" for i in range(3, 10) for j in range(1, 3)],
 94            *[f"GaPt{i:02d}_{j:02d}.txt" for i in range(12, 34) for j in range(1, 3)],
 95            *[f"GaPt{i:02d}_10.txt" for i in range(13, 34)],
 96            
 97            # Control subjects - Ju prefix
 98            *[f"JuCo{i:02d}_01.txt" for i in range(1, 27)],
 99            
100            # Parkinson's patients - Ju prefix
101            *[f"JuPt{i:02d}_{j:02d}.txt" for i in range(1, 30) for j in range(1, 8)],
102            
103            # Control subjects - Si prefix
104            *[f"SiCo{i:02d}_01.txt" for i in range(1, 31)],
105            
106            # Parkinson's patients - Si prefix
107            *[f"SiPt{i:02d}_01.txt" for i in range(2, 41)]
108        ]
109        
110        # Prepare download tasks for concurrent execution
111        download_tasks = [
112            {
113                'url': base_url + filename,
114                'dest_path': os.path.join(dataset_path, filename)
115            }
116            for filename in file_patterns
117        ]
118        
119        print(f"Downloading PhysioNet dataset to {dataset_path} using {self.max_workers} threads")
120        
121        # Use concurrent downloading from base class
122        results = self.download_files_concurrent(
123            download_tasks, 
124            show_progress=True, 
125            desc="Downloading PhysioNet files"
126        )
127        
128        # Print summary
129        print(f"\nDownload Summary:")
130        print(f"  Total files: {results['total']}")
131        print(f"  Successfully downloaded: {results['success']}")
132        print(f"  Already existed (skipped): {results['skipped']}")
133        print(f"  Failed: {results['failed']}")
134        
135        if results['failed'] > 0 and len(results['failed_downloads']) > 0:
136            print(f"\nFailed downloads (showing first 10):")
137            for failed in results['failed_downloads'][:10]:
138                print(f"  - {os.path.basename(failed['dest_path'])}: {failed['error']}")
139            if len(results['failed_downloads']) > 10:
140                print(f"  ... and {len(results['failed_downloads']) - 10} more failures")
141        
142        return dataset_path
143    
144    def load_data(self, data_dir: str, **kwargs) -> Tuple[List[pd.DataFrame], List[str]]:
145        """
146        Load PhysioNet VGRF dataset from the specified directory.
147        
148        Args:
149            data_dir: Directory to store/find the dataset
150            **kwargs: Additional arguments (unused for PhysioNet)
151            
152        Returns:
153            Tuple of (data_list, names_list)
154        """
155        # Download dataset if needed
156        dataset_path = self._download_physionet_data(data_dir)
157        
158        physionet_data = []
159        physionet_names = []
160        self.labels = []
161        self.subject_types = []
162        
163        # Load all available files
164        for filepath in sorted(glob(os.path.join(dataset_path, "Ga*.txt"))):
165            filename = os.path.basename(filepath)
166            
167            # Extract subject type from filename
168            if 'Co' in filename:
169                subject_type = 'Control'
170                label = 'Co'
171            elif 'Pt' in filename:
172                subject_type = 'Patient'
173                label = 'Pt'
174            else:
175                continue  # Skip files that don't match expected pattern
176            
177            try:
178                # Read the file - PhysioNet files are tab-delimited with variable columns
179                # Column 0: time, Columns 1-16: VGRF sensors, additional columns may exist
180                df = pd.read_csv(filepath, delimiter='\t', header=None)
181                
182                # Handle variable number of columns
183                n_cols = min(df.shape[1], 19)  # Limit to 19 columns max
184                df = df.iloc[:, :n_cols]
185                
186                # Create column names
187                col_names = ['time']
188                for i in range(1, n_cols):
189                    if i <= 8:
190                        col_names.append(f'VGRF_L{i}')
191                    elif i <= 16:
192                        col_names.append(f'VGRF_R{i-8}')
193                    else:
194                        col_names.append(f'sensor_{i}')
195                
196                df.columns = col_names
197                
198                # Set time as index
199                df = df.set_index('time')
200                
201                # Add subject metadata
202                df['subject_type'] = subject_type
203                df['label'] = label
204                
205                physionet_data.append(df)
206                physionet_names.append(filename)
207                self.labels.append(label)
208                self.subject_types.append(subject_type)
209                
210            except Exception as e:
211                print(f"Error loading {filename}: {e}")
212                continue
213        
214        # Store loaded data
215        self.data = physionet_data
216        self.names = physionet_names
217        
218        print(f"Loaded {len(physionet_data)} PhysioNet files")
219        print(f"Subject distribution: {dict(zip(*np.unique(self.subject_types, return_counts=True)))}")
220        
221        return physionet_data, physionet_names
222    
223    def create_sliding_windows(self, data: List[pd.DataFrame], names: List[str], 
224                             window_size: int = 600, step_size: int = 100) -> List[Dict]:
225        """
226        Create sliding windows from the PhysioNet dataset.
227        
228        Args:
229            data: List of DataFrames containing PhysioNet data
230            names: List of names corresponding to the data
231            window_size: Size of the sliding window (default: 600 for 6 seconds at 100Hz)
232            step_size: Step size for the sliding window (default: 100)
233            
234        Returns:
235            List of dictionaries containing sliding windows for each DataFrame
236        """
237        windows_data = []
238        
239        for idx, df in enumerate(data):
240            # Remove metadata columns for windowing
241            sensor_columns = [col for col in df.columns if col.startswith('VGRF_') or col.startswith('sensor_')]
242            df_sensors = df[sensor_columns]
243            
244            if df_sensors.empty or len(df_sensors) < window_size:
245                continue
246                
247            windows = []
248            
249            # Create windows for each sensor
250            for col in sensor_columns:
251                try:
252                    window_data = sliding_window(df_sensors[col].values, window_size, step_size)
253                    windows.append({"name": col, "data": window_data})
254                except Exception as e:
255                    print(f"Error creating windows for {col} in {names[idx]}: {e}")
256                    continue
257            
258            if windows:
259                windows_data.append({
260                    "name": names[idx],
261                    "windows": windows,
262                    "metadata": {
263                        "subject_type": df['subject_type'].iloc[0] if 'subject_type' in df.columns else 'Unknown',
264                        "label": df['label'].iloc[0] if 'label' in df.columns else 'Unknown',
265                        "window_size": window_size,
266                        "step_size": step_size,
267                        "num_windows": len(windows[0]["data"]) if windows else 0
268                    }
269                })
270        
271        return windows_data
272    
273    def get_supported_formats(self) -> List[str]:
274        """
275        Get list of supported file formats for PhysioNet dataset.
276        
277        Returns:
278            List of supported file extensions
279        """
280        return ['.txt']
281    
282    def get_sensor_info(self) -> Dict[str, List[str]]:
283        """
284        Get information about sensors in the dataset.
285        
286        Returns:
287            Dictionary containing sensor information
288        """
289        return {
290            'sensors': self.metadata['sensors'],
291            'sampling_frequency': self.metadata['sampling_frequency'],
292            'window_size': self.metadata['window_size']
293        }
294    
295    def get_subject_info(self) -> Dict[str, str]:
296        """
297        Get information about subjects in the dataset.
298        
299        Returns:
300            Dictionary containing subject information
301        """
302        return self.metadata['subjects']
303    
304    def get_labels(self) -> List[str]:
305        """
306        Get labels for loaded data.
307        
308        Returns:
309            List of labels corresponding to loaded data
310        """
311        return self.labels
312    
313    def filter_by_subject_type(self, subject_type: str) -> Tuple[List[pd.DataFrame], List[str]]:
314        """
315        Filter loaded data by subject type.
316        
317        Args:
318            subject_type: 'Control' or 'Patient'
319            
320        Returns:
321            Tuple of (filtered_data, filtered_names)
322        """
323        if not self.data:
324            raise ValueError("No data loaded. Call load_data() first.")
325        
326        filtered_data = []
327        filtered_names = []
328        
329        for i, df in enumerate(self.data):
330            if df['subject_type'].iloc[0] == subject_type:
331                filtered_data.append(df)
332                filtered_names.append(self.names[i])
333        
334        return filtered_data, filtered_names