Coverage for nilearn/tests/test_signal.py: 12%

616 statements  

« prev     ^ index     » next       coverage.py v7.9.1, created at 2025-06-20 10:58 +0200

1"""Test the signals module.""" 

2 

3from pathlib import Path 

4 

5import numpy as np 

6import pytest 

7import scipy.signal 

8from numpy import array_equal 

9from numpy.testing import assert_almost_equal, assert_array_equal, assert_equal 

10from pandas import read_csv 

11 

12from nilearn._utils.exceptions import AllVolumesRemovedError 

13from nilearn.conftest import _rng 

14from nilearn.glm.first_level.design_matrix import create_cosine_drift 

15from nilearn.signal import ( 

16 _censor_signals, 

17 _create_cosine_drift_terms, 

18 _detrend, 

19 _handle_scrubbed_volumes, 

20 _mean_of_squares, 

21 butterworth, 

22 clean, 

23 high_variance_confounds, 

24 row_sum_of_squares, 

25 standardize_signal, 

26) 

27 

28EPS = np.finfo(np.float64).eps 

29 

30 

31def generate_signals( 

32 n_features=17, n_confounds=5, length=41, same_variance=True, order="C" 

33): 

34 """Generate test signals. 

35 

36 All returned signals have no trends at all (to machine precision). 

37 

38 Parameters 

39 ---------- 

40 n_features, n_confounds : int, optional 

41 respectively number of features to generate, and number of confounds 

42 to use for generating noise signals. 

43 

44 length : int, optional 

45 number of samples for every signal. 

46 

47 same_variance : bool, optional 

48 if True, every column of "signals" have a unit variance. Otherwise, 

49 a random amplitude is applied. 

50 

51 order : "C" or "F" 

52 gives the contiguousness of the output arrays. 

53 

54 Returns 

55 ------- 

56 signals : numpy.ndarray, shape (length, n_features) 

57 unperturbed signals. 

58 

59 noises : numpy.ndarray, shape (length, n_features) 

60 confound-based noises. Each column is a signal obtained by linear 

61 combination of all confounds signals (below). The coefficients in 

62 the linear combination are also random. 

63 

64 confounds : numpy.ndarray, shape (length, n_confounds) 

65 random signals used as confounds. 

66 """ 

67 rng = _rng() 

68 

69 # Generate random confounds 

70 confounds_shape = (length, n_confounds) 

71 confounds = np.ndarray(confounds_shape, order=order) 

72 confounds[...] = rng.standard_normal(size=confounds_shape) 

73 confounds[...] = scipy.signal.detrend(confounds, axis=0) 

74 

75 # Compute noise based on confounds, with random factors 

76 factors = rng.standard_normal(size=(n_confounds, n_features)) 

77 noises_shape = (length, n_features) 

78 noises = np.ndarray(noises_shape, order=order) 

79 noises[...] = np.dot(confounds, factors) 

80 noises[...] = scipy.signal.detrend(noises, axis=0) 

81 

82 # Generate random signals with random amplitudes 

83 signals_shape = noises_shape 

84 signals = np.ndarray(signals_shape, order=order) 

85 if same_variance: 

86 signals[...] = rng.standard_normal(size=signals_shape) 

87 else: 

88 signals[...] = ( 

89 4.0 * abs(rng.standard_normal(size=signals_shape[1])) + 0.5 

90 ) * rng.standard_normal(size=signals_shape) 

91 

92 signals[...] = scipy.signal.detrend(signals, axis=0) 

93 return signals, noises, confounds 

94 

95 

96def generate_trends(n_features=17, length=41): 

97 """Generate linearly-varying signals, with zero mean. 

98 

99 Parameters 

100 ---------- 

101 n_features, length : int 

102 respectively number of signals and number of samples to generate. 

103 

104 Returns 

105 ------- 

106 trends : numpy.ndarray, shape (length, n_features) 

107 output signals, one per column. 

108 """ 

109 rng = _rng() 

110 trends = scipy.signal.detrend(np.linspace(0, 1.0, length), type="constant") 

111 trends = np.repeat(np.atleast_2d(trends).T, n_features, axis=1) 

112 factors = rng.standard_normal(size=n_features) 

113 return trends * factors 

114 

115 

116def generate_signals_plus_trends(n_features=17, n_samples=41): 

117 """Generate signal with a trend.""" 

118 signals, _, _ = generate_signals(n_features=n_features, length=n_samples) 

119 trends = generate_trends(n_features=n_features, length=n_samples) 

120 return signals + trends 

121 

122 

123@pytest.fixture 

124def data_butterworth_single_timeseries(rng): 

125 """Generate single timeseries for butterworth tests.""" 

126 n_samples = 100 

127 return rng.standard_normal(size=n_samples) 

128 

129 

130@pytest.fixture 

131def data_butterworth_multiple_timeseries( 

132 rng, data_butterworth_single_timeseries 

133): 

134 """Generate mutltiple timeseries for butterworth tests.""" 

135 n_features = 20000 

136 n_samples = 100 

137 data = rng.standard_normal(size=(n_samples, n_features)) 

138 # set first timeseries to previous data 

139 data[:, 0] = data_butterworth_single_timeseries 

140 return data 

141 

142 

143def test_butterworth(data_butterworth_single_timeseries): 

144 """Check butterworth onsingle timeseries.""" 

145 sampling = 100 

146 low_pass = 30 

147 high_pass = 10 

148 

149 # Compare output for different options. 

150 # single timeseries 

151 data = data_butterworth_single_timeseries 

152 data_original = data.copy() 

153 

154 out_single = butterworth( 

155 data, sampling, low_pass=low_pass, high_pass=high_pass, copy=True 

156 ) 

157 

158 assert_almost_equal(data, data_original) 

159 

160 butterworth( 

161 data, sampling, low_pass=low_pass, high_pass=high_pass, copy=False 

162 ) 

163 

164 assert_almost_equal(out_single, data) 

165 assert id(out_single) != id(data) 

166 

167 

168def test_butterworth_multiple_timeseries( 

169 data_butterworth_single_timeseries, data_butterworth_multiple_timeseries 

170): 

171 """Check butterworth on multiple / single timeseries do the same thing.""" 

172 sampling = 100 

173 low_pass = 30 

174 high_pass = 10 

175 

176 data = data_butterworth_multiple_timeseries 

177 data_original = data.copy() 

178 

179 out_single = butterworth( 

180 data_butterworth_single_timeseries, 

181 sampling, 

182 low_pass=low_pass, 

183 high_pass=high_pass, 

184 copy=True, 

185 ) 

186 

187 out1 = butterworth( 

188 data, sampling, low_pass=low_pass, high_pass=high_pass, copy=True 

189 ) 

190 

191 assert_almost_equal(data, data_original) 

192 assert id(out1) != id(data_original) 

193 

194 assert_almost_equal(out1[:, 0], out_single) 

195 

196 butterworth( 

197 data, sampling, low_pass=low_pass, high_pass=high_pass, copy=False 

198 ) 

199 

200 assert_almost_equal(out1, data) 

201 

202 

203def test_butterworth_nyquist_frequency_clipping( 

204 data_butterworth_multiple_timeseries, 

205): 

206 """Test nyquist frequency clipping. 

207 

208 issue #482 

209 """ 

210 sampling = 100 

211 

212 data = data_butterworth_multiple_timeseries 

213 

214 out1 = butterworth(data, sampling, low_pass=50.0, copy=True) 

215 out2 = butterworth( 

216 data, 

217 sampling, 

218 low_pass=80.0, 

219 copy=True, # Greater than nyq frequency 

220 ) 

221 

222 assert_almost_equal(out1, out2) 

223 assert id(out1) != id(out2) 

224 

225 

226def test_butterworth_warnings_critical_frequencies( 

227 data_butterworth_single_timeseries, 

228): 

229 """Check for equal values in critical frequencies.""" 

230 data = data_butterworth_single_timeseries 

231 sampling = 1 

232 low_pass = 2 

233 high_pass = 1 

234 

235 with pytest.warns( 

236 UserWarning, 

237 match=( 

238 "Signals are returned unfiltered because " 

239 "band-pass critical frequencies are equal. " 

240 "Please check that inputs for sampling_rate, " 

241 "low_pass, and high_pass are valid." 

242 ), 

243 ): 

244 out = butterworth( 

245 data, 

246 sampling, 

247 low_pass=low_pass, 

248 high_pass=high_pass, 

249 copy=True, 

250 ) 

251 assert (out == data).all() 

252 

253 

254def test_butterworth_warnings_lpf_too_high(data_butterworth_single_timeseries): 

255 """Check for frequency higher than allowed (>=Nyquist). 

256 

257 The frequency should be modified and the filter should be run. 

258 """ 

259 data = data_butterworth_single_timeseries 

260 

261 sampling = 1 

262 high_pass = 0.01 

263 low_pass = 0.5 

264 with pytest.warns( 

265 UserWarning, 

266 match="The frequency specified for the low pass filter is too high", 

267 ): 

268 out = butterworth( 

269 data, 

270 sampling, 

271 low_pass=low_pass, 

272 high_pass=high_pass, 

273 copy=True, 

274 ) 

275 assert not array_equal(data, out) 

276 

277 

278def test_butterworth_warnings_hpf_too_low(data_butterworth_single_timeseries): 

279 """Check for frequency lower than allowed (<0). 

280 

281 The frequency should be modified and the filter should be run. 

282 """ 

283 data = data_butterworth_single_timeseries 

284 sampling = 1 

285 high_pass = -1 

286 low_pass = 0.4 

287 

288 with pytest.warns( 

289 UserWarning, 

290 match="The frequency specified for the high pass filter is too low", 

291 ): 

292 out = butterworth( 

293 data, 

294 sampling, 

295 low_pass=low_pass, 

296 high_pass=high_pass, 

297 copy=True, 

298 ) 

299 assert not array_equal(data, out) 

300 

301 

302def test_butterworth_errors(data_butterworth_single_timeseries): 

303 """Check for high-pass frequency higher than low-pass frequency.""" 

304 sampling = 1 

305 high_pass = 0.2 

306 low_pass = 0.1 

307 with pytest.raises( 

308 ValueError, 

309 match=( 

310 r"High pass cutoff frequency \([0-9.]+\) is greater than or " 

311 r"equal to low pass filter frequency \([0-9.]+\)\." 

312 ), 

313 ): 

314 butterworth( 

315 data_butterworth_single_timeseries, 

316 sampling, 

317 low_pass=low_pass, 

318 high_pass=high_pass, 

319 copy=True, 

320 ) 

321 

322 

323def test_standardize_error(rng): 

324 """Test raise error for wrong strategy.""" 

325 n_features = 10 

326 n_samples = 17 

327 

328 # Create random signals with offsets and and negative mean 

329 a = rng.random((n_samples, n_features)) 

330 a += np.linspace(0, 2.0, n_features) 

331 

332 with pytest.raises(ValueError, match="no valid standardize strategy"): 

333 standardize_signal(a, standardize="foo") 

334 

335 # test warning for strategy that will be removed 

336 with pytest.warns( 

337 DeprecationWarning, match="default strategy for standardize" 

338 ): 

339 standardize_signal(a, standardize="zscore") 

340 

341 

342def test_standardize(rng): 

343 """Test starndardize_signal with several options.""" 

344 n_features = 10 

345 n_samples = 17 

346 

347 # Create random signals with offsets and and negative mean 

348 a = rng.random((n_samples, n_features)) 

349 a += np.linspace(0, 2.0, n_features) 

350 

351 # ensure PSC rescaled correctly, correlation should be 1 

352 z = standardize_signal(a, standardize="zscore_sample") 

353 psc = standardize_signal(a, standardize="psc") 

354 corr_coef_feature = np.corrcoef(z[:, 0], psc[:, 0])[0, 1] 

355 

356 assert corr_coef_feature.mean() == 1 

357 

358 # transpose array to fit standardize input. 

359 # Without trend removal 

360 b = standardize_signal(a, standardize="zscore_sample") 

361 

362 stds = np.std(b) 

363 assert_almost_equal(stds, np.ones(n_features), decimal=1) 

364 assert_almost_equal(b.sum(axis=0), np.zeros(n_features)) 

365 

366 # With trend removal 

367 a = np.atleast_2d(np.linspace(0, 2.0, n_features)).T 

368 b = standardize_signal(a, detrend=True, standardize=False) 

369 

370 assert_almost_equal(b, np.zeros(b.shape)) 

371 

372 b = standardize_signal(a, detrend=True, standardize="zscore_sample") 

373 

374 assert_almost_equal(b, np.zeros(b.shape)) 

375 

376 length_1_signal = np.atleast_2d(np.linspace(0, 2.0, n_features)) 

377 

378 assert_array_equal( 

379 length_1_signal, 

380 standardize_signal(length_1_signal, standardize="zscore_sample"), 

381 ) 

382 

383 

384def test_detrend(): 

385 """Test custom detrend implementation.""" 

386 point_number = 703 

387 features = 17 

388 signals, _, _ = generate_signals( 

389 n_features=features, length=point_number, same_variance=True 

390 ) 

391 trends = generate_trends(n_features=features, length=point_number) 

392 x = signals + trends + 1 

393 original = x.copy() 

394 

395 # Mean removal only (out-of-place) 

396 detrended = _detrend(x, inplace=False, type="constant") 

397 

398 assert abs(detrended.mean(axis=0)).max() < 15.0 * EPS 

399 

400 # out-of-place detrending. Use scipy as a reference implementation 

401 detrended = _detrend(x, inplace=False) 

402 

403 detrended_scipy = scipy.signal.detrend(x, axis=0) 

404 

405 # "x" must be left untouched 

406 assert_almost_equal(original, x, decimal=14) 

407 assert abs(detrended.mean(axis=0)).max() < 15.0 * EPS 

408 assert_almost_equal(detrended_scipy, detrended, decimal=14) 

409 # for this to work, there must be no trends at all in "signals" 

410 assert_almost_equal(detrended, signals, decimal=14) 

411 

412 # inplace detrending 

413 _detrend(x, inplace=True) 

414 

415 assert abs(x.mean(axis=0)).max() < 15.0 * EPS 

416 # for this to work, there must be no trends at all in "signals" 

417 assert_almost_equal(detrended_scipy, detrended, decimal=14) 

418 assert_almost_equal(x, signals, decimal=14) 

419 

420 length_1_signal = x[0] 

421 length_1_signal = length_1_signal[np.newaxis, :] 

422 assert_array_equal(length_1_signal, _detrend(length_1_signal)) 

423 

424 # Mean removal on integers 

425 detrended = _detrend(x.astype(np.int64), inplace=True, type="constant") 

426 

427 assert abs(detrended.mean(axis=0)).max() < 20.0 * EPS 

428 

429 

430def test_mean_of_squares(): 

431 """Test _mean_of_squares.""" 

432 n_samples = 11 

433 n_features = 501 # Higher than 500 required 

434 signals, _, _ = generate_signals( 

435 n_features=n_features, length=n_samples, same_variance=True 

436 ) 

437 # Reference computation 

438 var1 = np.copy(signals) 

439 var1 **= 2 

440 var1 = var1.mean(axis=0) 

441 

442 var2 = _mean_of_squares(signals) 

443 

444 assert_almost_equal(var1, var2) 

445 

446 

447def test_row_sum_of_squares(): 

448 """Test row_sum_of_squares.""" 

449 n_samples = 11 

450 n_features = 501 # Higher than 500 required 

451 signals, _, _ = generate_signals( 

452 n_features=n_features, length=n_samples, same_variance=True 

453 ) 

454 # Reference computation 

455 var1 = signals**2 

456 var1 = var1.sum(axis=0) 

457 

458 var2 = row_sum_of_squares(signals) 

459 

460 assert_almost_equal(var1, var2) 

461 

462 

463def test_clean_detrending(): 

464 """Check effect of clean with detrending. 

465 

466 This test is inspired from Scipy docstring of detrend function. 

467 

468 - clean should not modify inputs 

469 - check effect when fintie results requested 

470 """ 

471 n_samples = 21 

472 n_features = 501 # Must be higher than 500 

473 signals, _, _ = generate_signals(n_features=n_features, length=n_samples) 

474 trends = generate_trends(n_features=n_features, length=n_samples) 

475 x = signals + trends 

476 x_orig = x.copy() 

477 

478 # if NANs, data out should be False with ensure_finite=True 

479 y = signals + trends 

480 y[20, 150] = np.nan 

481 y[5, 500] = np.nan 

482 y[15, 14] = np.inf 

483 y_orig = y.copy() 

484 

485 y_clean = clean(y, ensure_finite=True) 

486 

487 assert np.any(np.isfinite(y_clean)) 

488 # clean should not modify inputs 

489 # using assert_almost_equal instead of array_equal due to NaNs 

490 assert_almost_equal(y_orig, y, decimal=13) 

491 

492 # This should remove trends 

493 x_detrended = clean( 

494 x, standardize=False, detrend=True, low_pass=None, high_pass=None 

495 ) 

496 

497 assert_almost_equal(x_detrended, signals, decimal=13) 

498 # clean should not modify inputs 

499 assert array_equal(x_orig, x) 

500 

501 # This should do nothing 

502 x_undetrended = clean( 

503 x, standardize=False, detrend=False, low_pass=None, high_pass=None 

504 ) 

505 

506 assert abs(x_undetrended - signals).max() >= 0.06 

507 # clean should not modify inputs 

508 assert array_equal(x_orig, x) 

509 

510 

511def test_clean_t_r(rng): 

512 """Different TRs produce different results after butterworth filtering.""" 

513 n_samples = 34 

514 # n_features Must be higher than 500 

515 n_features = 501 

516 x_orig = generate_signals_plus_trends( 

517 n_features=n_features, n_samples=n_samples 

518 ) 

519 random_tr_list1 = np.round(rng.uniform(size=3) * 10, decimals=2) 

520 random_tr_list2 = np.round(rng.uniform(size=3) * 10, decimals=2) 

521 for tr1, tr2 in zip(random_tr_list1, random_tr_list2): 

522 low_pass_freq_list = tr1 * np.array([1.0 / 100, 1.0 / 110]) 

523 high_pass_freq_list = tr1 * np.array([1.0 / 210, 1.0 / 190]) 

524 for low_cutoff, high_cutoff in zip( 

525 low_pass_freq_list, high_pass_freq_list 

526 ): 

527 det_one_tr = clean( 

528 x_orig, t_r=tr1, low_pass=low_cutoff, high_pass=high_cutoff 

529 ) 

530 det_diff_tr = clean( 

531 x_orig, t_r=tr2, low_pass=low_cutoff, high_pass=high_cutoff 

532 ) 

533 

534 if not np.isclose(tr1, tr2, atol=0.3): 

535 msg = ( 

536 "results do not differ " 

537 f"for different TRs: {tr1} and {tr2} " 

538 f"at cutoffs: low_pass={low_cutoff}, " 

539 f"high_pass={high_cutoff} " 

540 f"n_samples={n_samples}, n_features={n_features}" 

541 ) 

542 assert np.any(np.not_equal(det_one_tr, det_diff_tr)), msg 

543 del det_one_tr, det_diff_tr 

544 

545 

546@pytest.mark.parametrize( 

547 "kwarg_set", 

548 [ 

549 { 

550 "butterworth__padtype": "even", 

551 "butterworth__padlen": 10, 

552 "butterworth__order": 3, 

553 }, 

554 { 

555 "butterworth__padtype": None, 

556 "butterworth__padlen": None, 

557 "butterworth__order": 1, 

558 }, 

559 { 

560 "butterworth__padtype": "constant", 

561 "butterworth__padlen": 20, 

562 "butterworth__order": 10, 

563 }, 

564 ], 

565) 

566def test_clean_kwargs(kwarg_set): 

567 """Providing kwargs to clean should change the filtered results.""" 

568 n_samples = 34 

569 n_features = 501 

570 x_orig = generate_signals_plus_trends( 

571 n_features=n_features, n_samples=n_samples 

572 ) 

573 

574 # Base result 

575 t_r, high_pass, low_pass = 0.8, 0.01, 0.08 

576 base_filtered = clean( 

577 x_orig, t_r=t_r, low_pass=low_pass, high_pass=high_pass 

578 ) 

579 

580 test_filtered = clean( 

581 x_orig, 

582 t_r=t_r, 

583 low_pass=low_pass, 

584 high_pass=high_pass, 

585 **kwarg_set, 

586 ) 

587 

588 # Check that results are **not** the same. 

589 assert np.any(np.not_equal(base_filtered, test_filtered)) 

590 

591 

592def test_clean_frequencies(): 

593 """Check several values for low and high pass.""" 

594 sx1 = np.sin(np.linspace(0, 100, 2000)) 

595 sx2 = np.sin(np.linspace(0, 100, 2000)) 

596 sx = np.vstack((sx1, sx2)).T 

597 

598 t_r = 2.5 

599 standardize = False 

600 

601 cleaned_signal = clean( 

602 sx, standardize=standardize, high_pass=0.002, low_pass=None, t_r=t_r 

603 ) 

604 assert cleaned_signal.max() > 0.1 

605 

606 cleaned_signal = clean( 

607 sx, standardize=standardize, high_pass=0.2, low_pass=None, t_r=t_r 

608 ) 

609 assert cleaned_signal.max() < 0.01 

610 

611 cleaned_signal = clean(sx, standardize=standardize, low_pass=0.01, t_r=t_r) 

612 assert cleaned_signal.max() > 0.9 

613 

614 with pytest.raises( 

615 ValueError, match="High pass .* greater than .* low pass" 

616 ): 

617 clean(sx, low_pass=0.4, high_pass=0.5, t_r=t_r) 

618 

619 

620def test_clean_leaves_input_untouched(): 

621 """Clean should not modify inputs.""" 

622 sx1 = np.sin(np.linspace(0, 100, 2000)) 

623 sx2 = np.sin(np.linspace(0, 100, 2000)) 

624 sx = np.vstack((sx1, sx2)).T 

625 sx_orig = sx.copy() 

626 

627 t_r = 2.5 

628 standardize = False 

629 

630 _ = clean( 

631 sx, standardize=standardize, detrend=False, low_pass=0.2, t_r=t_r 

632 ) 

633 

634 assert array_equal(sx_orig, sx) 

635 

636 

637def test_clean_runs(): 

638 """Check cleaning across runs.""" 

639 n_samples = 21 

640 n_features = 501 # Must be higher than 500 

641 signals, _, confounds = generate_signals( 

642 n_features=n_features, length=n_samples 

643 ) 

644 trends = generate_trends(n_features=n_features, length=n_samples) 

645 x = signals + trends 

646 x_orig = x.copy() 

647 # Create run info 

648 runs = np.ones(n_samples) 

649 runs[: n_samples // 2] = 0 

650 

651 x_detrended = clean( 

652 x, 

653 confounds=confounds, 

654 standardize=False, 

655 detrend=True, 

656 low_pass=None, 

657 high_pass=None, 

658 runs=runs, 

659 ) 

660 

661 # clean should not modify inputs 

662 assert array_equal(x_orig, x) 

663 

664 # check the runs are individually cleaned 

665 x_run1 = clean( 

666 x[0 : n_samples // 2, :], 

667 confounds=confounds[0 : n_samples // 2, :], 

668 standardize=False, 

669 detrend=True, 

670 low_pass=None, 

671 high_pass=None, 

672 ) 

673 assert array_equal(x_run1, x_detrended[0 : n_samples // 2, :]) 

674 

675 

676@pytest.fixture 

677def signals(): 

678 """Return generic signal.""" 

679 return generate_signals(n_features=41, n_confounds=5, length=45)[0] 

680 

681 

682@pytest.fixture 

683def confounds(): 

684 """Return generic condounds.""" 

685 return generate_signals(n_features=41, n_confounds=5, length=45)[2] 

686 

687 

688def test_clean_confounds_errros(signals): 

689 """Test error handling.""" 

690 with pytest.raises( 

691 TypeError, match="confounds keyword has an unhandled type" 

692 ): 

693 clean(signals, confounds=1) 

694 

695 with pytest.raises(TypeError, match="confound has an unhandled type"): 

696 clean(signals, confounds=[None]) 

697 

698 msg = "Confound signal has an incorrect length." 

699 with pytest.raises(ValueError, match=msg): 

700 clean(signals, confounds=np.zeros(2)) 

701 with pytest.raises(ValueError, match=msg): 

702 clean(signals, confounds=np.zeros((2, 2))) 

703 with pytest.raises(ValueError, match=msg): 

704 current_dir = Path(__file__).parent 

705 filename1 = current_dir / "data" / "spm_confounds.txt" 

706 clean(signals[:-1, :], confounds=filename1) 

707 

708 

709def test_clean_errros(signals): 

710 """Test error handling.""" 

711 with pytest.raises( 

712 ValueError, 

713 match="confound array has an incorrect number of dimensions", 

714 ): 

715 clean(signals, confounds=np.zeros((2, 3, 4))) 

716 

717 with pytest.raises( 

718 ValueError, 

719 match="Repetition time .* and low cutoff frequency .*", 

720 ): 

721 clean(signals, filter="cosine", t_r=None, high_pass=0.008) 

722 

723 with pytest.raises( 

724 ValueError, 

725 match="Repetition time .* must be specified for butterworth.", 

726 ): 

727 # using butterworth filter here 

728 clean(signals, t_r=None, low_pass=0.01) 

729 

730 with pytest.raises( 

731 ValueError, match="Filter method not_implemented not implemented." 

732 ): 

733 clean(signals, filter="not_implemented") 

734 

735 with pytest.raises( 

736 ValueError, match="'ensure_finite' must be boolean type True or False" 

737 ): 

738 clean(signals, ensure_finite=None) 

739 

740 # test boolean is not given to signal.clean 

741 with pytest.raises(TypeError, match="high/low pass must be float or None"): 

742 clean(signals, low_pass=False) 

743 

744 with pytest.raises(TypeError, match="high/low pass must be float or None"): 

745 clean(signals, high_pass=False) 

746 

747 

748def test_clean_confounds(): 

749 """Check output of cleaning when counfoun is passed.""" 

750 signals, noises, confounds = generate_signals( 

751 n_features=41, n_confounds=5, length=45 

752 ) 

753 # No signal: output must be zero. 

754 noises1 = noises.copy() 

755 cleaned_signals = clean( 

756 noises, confounds=confounds, detrend=True, standardize=False 

757 ) 

758 

759 assert abs(cleaned_signals).max() < 100.0 * EPS 

760 # clean should not modify inputs 

761 assert array_equal(noises, noises1) 

762 

763 # With signal: output must be orthogonal to confounds 

764 cleaned_signals = clean( 

765 signals + noises, confounds=confounds, detrend=False, standardize=True 

766 ) 

767 

768 assert abs(np.dot(confounds.T, cleaned_signals)).max() < 1000.0 * EPS 

769 

770 # Same output when a constant confound is added 

771 confounds1 = np.hstack((np.ones((45, 1)), confounds)) 

772 cleaned_signals1 = clean( 

773 signals + noises, confounds=confounds1, detrend=False, standardize=True 

774 ) 

775 

776 assert_almost_equal(cleaned_signals1, cleaned_signals) 

777 

778 

779def test_clean_confounds_detrending(): 

780 """Test detrending. 

781 

782 No trend should exist in the output. 

783 """ 

784 signals, noises, confounds = generate_signals( 

785 n_features=41, n_confounds=5, length=45 

786 ) 

787 # Use confounds with a trend. 

788 temp = confounds.T 

789 temp += np.arange(confounds.shape[0]) 

790 

791 cleaned_signals = clean( 

792 signals + noises, confounds=confounds, detrend=False, standardize=False 

793 ) 

794 coeffs = np.polyfit( 

795 np.arange(cleaned_signals.shape[0]), cleaned_signals, 1 

796 ) 

797 

798 assert (abs(coeffs) > 1e-3).any() # trends remain 

799 

800 cleaned_signals = clean( 

801 signals + noises, confounds=confounds, detrend=True, standardize=False 

802 ) 

803 coeffs = np.polyfit( 

804 np.arange(cleaned_signals.shape[0]), cleaned_signals, 1 

805 ) 

806 

807 assert (abs(coeffs) < 1000.0 * EPS).all() # trend removed 

808 

809 

810def test_clean_standardize_trye_false(): 

811 """Check difference between standardize False and True.""" 

812 signals, _, _ = generate_signals(n_features=41, n_confounds=5, length=45) 

813 

814 input_signals = 10 * signals 

815 cleaned_signals = clean(input_signals, detrend=False, standardize=False) 

816 

817 assert_almost_equal(cleaned_signals, input_signals) 

818 

819 cleaned_signals = clean(input_signals, detrend=False, standardize=True) 

820 

821 assert_almost_equal( 

822 cleaned_signals.var(axis=0), np.ones(cleaned_signals.shape[1]) 

823 ) 

824 

825 

826def test_clean_confounds_inputs(): 

827 """Check several types of supported inputs.""" 

828 signals, _, confounds = generate_signals( 

829 n_features=41, n_confounds=3, length=20 

830 ) 

831 # Test with confounds read from a file. 

832 # Smoke test only (result has no meaning). 

833 current_dir = Path(__file__).parent 

834 filename1 = current_dir / "data" / "spm_confounds.txt" 

835 filename2 = current_dir / "data" / "confounds_with_header.csv" 

836 

837 clean(signals, detrend=False, standardize=False, confounds=filename1) 

838 clean(signals, detrend=False, standardize=False, confounds=filename2) 

839 clean(signals, detrend=False, standardize=False, confounds=confounds[:, 1]) 

840 

841 # test with confounds as a pandas DataFrame 

842 confounds_df = read_csv(filename2, sep="\t") 

843 clean( 

844 signals, 

845 detrend=False, 

846 standardize=False, 

847 confounds=confounds_df.values, 

848 ) 

849 clean(signals, detrend=False, standardize=False, confounds=confounds_df) 

850 

851 # test array-like signals 

852 list_signal = signals.tolist() 

853 clean(list_signal) 

854 

855 # Use a list containing two filenames, a 2D array and a 1D array 

856 clean( 

857 signals, 

858 detrend=False, 

859 standardize=False, 

860 confounds=[filename1, confounds[:, 0:2], filename2, confounds[:, 2]], 

861 ) 

862 

863 

864def test_clean_warning(signals): 

865 """Check warnings are thrown.""" 

866 # Check warning message when no confound methods were specified, 

867 # but cutoff frequency provided. 

868 with pytest.warns(UserWarning, match="not perform filtering"): 

869 clean(signals, t_r=2.5, filter=False, low_pass=0.01) 

870 

871 # Test without standardizing that constant parts of confounds are 

872 # accounted for 

873 # passing standardize_confounds=False, detrend=False should raise warning 

874 warning_message = r"must perform detrend and/or standardize confounds" 

875 with pytest.warns(UserWarning, match=warning_message): 

876 assert_almost_equal( 

877 clean( 

878 np.ones((20, 2)), 

879 standardize=False, 

880 confounds=np.ones(20), 

881 standardize_confounds=False, 

882 detrend=False, 

883 ).mean(), 

884 np.zeros((20, 2)), 

885 ) 

886 

887 

888def test_clean_confounds_are_removed(signals, confounds): 

889 """Check that confounders effects are effectively removed. 

890 

891 Check that confounders effects are effectively removed from 

892 the signals when having a detrending and filtering operation together. 

893 This did not happen originally due to a different order in which 

894 these operations were being applied to the data and confounders. 

895 see https://github.com/nilearn/nilearn/issues/2730 

896 """ 

897 signals_clean = clean( 

898 signals, 

899 detrend=True, 

900 high_pass=0.01, 

901 standardize_confounds=True, 

902 standardize=True, 

903 confounds=confounds, 

904 ) 

905 confounds_clean = clean( 

906 confounds, detrend=True, high_pass=0.01, standardize=True 

907 ) 

908 assert abs(np.dot(confounds_clean.T, signals_clean)).max() < 1000.0 * EPS 

909 

910 

911def test_clean_frequencies_using_power_spectrum_density(): 

912 """Check on power spectrum that expected frequencies were removed.""" 

913 # Create signal 

914 sx = np.array( 

915 [ 

916 np.sin(np.linspace(0, 100, 100) * 1.5), 

917 np.sin(np.linspace(0, 100, 100) * 3.0), 

918 np.sin(np.linspace(0, 100, 100) / 8.0), 

919 ] 

920 ).T 

921 

922 # Apply low- and high-pass filter with butterworth (separately) 

923 t_r = 1.0 

924 low_pass = 0.1 

925 high_pass = 0.4 

926 res_low = clean( 

927 sx, 

928 detrend=False, 

929 standardize=False, 

930 filter="butterworth", 

931 low_pass=low_pass, 

932 high_pass=None, 

933 t_r=t_r, 

934 ) 

935 res_high = clean( 

936 sx, 

937 detrend=False, 

938 standardize=False, 

939 filter="butterworth", 

940 low_pass=None, 

941 high_pass=high_pass, 

942 t_r=t_r, 

943 ) 

944 

945 # cosine high pass filter 

946 res_cos = clean( 

947 sx, 

948 detrend=False, 

949 standardize=False, 

950 filter="cosine", 

951 low_pass=None, 

952 high_pass=high_pass, 

953 t_r=t_r, 

954 ) 

955 

956 # Compute power spectrum density for both test 

957 f, Pxx_den_low = scipy.signal.welch(np.mean(res_low.T, axis=0), fs=t_r) 

958 f, Pxx_den_high = scipy.signal.welch(np.mean(res_high.T, axis=0), fs=t_r) 

959 f, Pxx_den_cos = scipy.signal.welch(np.mean(res_cos.T, axis=0), fs=t_r) 

960 

961 # Verify that the filtered frequencies are removed 

962 assert np.sum(Pxx_den_low[f >= low_pass * 2.0]) <= 1e-4 

963 assert np.sum(Pxx_den_high[f <= high_pass / 2.0]) <= 1e-4 

964 assert np.sum(Pxx_den_cos[f <= high_pass / 2.0]) <= 1e-4 

965 

966 

967@pytest.mark.parametrize("t_r", [1, 1.0]) 

968@pytest.mark.parametrize("high_pass", [1, 1.0]) 

969def test_clean_t_r_highpass_float_int(t_r, high_pass): 

970 """Make sure t_r and high_pass can be int. 

971 

972 Regression test for: https://github.com/nilearn/nilearn/issues/4803 

973 """ 

974 # Create signal 

975 sx = np.array( 

976 [ 

977 np.sin(np.linspace(0, 100, 100) * 1.5), 

978 np.sin(np.linspace(0, 100, 100) * 3.0), 

979 np.sin(np.linspace(0, 100, 100) / 8.0), 

980 ] 

981 ).T 

982 

983 # Create confound 

984 _, _, confounds = generate_signals( 

985 n_features=10, n_confounds=10, length=100 

986 ) 

987 clean( 

988 sx, 

989 detrend=False, 

990 standardize=False, 

991 filter="cosine", 

992 low_pass=None, 

993 high_pass=high_pass, 

994 t_r=t_r, 

995 ) 

996 

997 

998def test_clean_finite_no_inplace_mod(): 

999 """Test for verifying that the passed in signal array is not modified. 

1000 

1001 For PR #2125 . This test is failing on main, passing in this PR. 

1002 """ 

1003 n_samples = 2 

1004 # n_features Must be higher than 500 

1005 n_features = 501 

1006 x_orig, _, _ = generate_signals(n_features=n_features, length=n_samples) 

1007 x_orig_inital_copy = x_orig.copy() 

1008 

1009 x_orig_with_nans = x_orig.copy() 

1010 x_orig_with_nans[0, 0] = np.nan 

1011 x_orig_with_nans_initial_copy = x_orig_with_nans.copy() 

1012 

1013 _ = clean(x_orig) 

1014 assert array_equal(x_orig, x_orig_inital_copy) 

1015 

1016 _ = clean(x_orig_with_nans, ensure_finite=True) 

1017 assert np.isnan(x_orig_with_nans_initial_copy[0, 0]) 

1018 assert np.isnan(x_orig_with_nans[0, 0]) 

1019 

1020 

1021def test_high_variance_confounds_c_f(): 

1022 """Check C and F order give same result. 

1023 

1024 They might take different paths in the function. 

1025 """ 

1026 n_features = 1001 

1027 length = 20 

1028 n_confounds = 5 

1029 

1030 seriesC, _, _ = generate_signals( 

1031 n_features=n_features, length=length, order="C" 

1032 ) 

1033 seriesF, _, _ = generate_signals( 

1034 n_features=n_features, length=length, order="F" 

1035 ) 

1036 

1037 assert_almost_equal(seriesC, seriesF, decimal=13) 

1038 

1039 outC = high_variance_confounds( 

1040 seriesC, n_confounds=n_confounds, detrend=False 

1041 ) 

1042 outF = high_variance_confounds( 

1043 seriesF, n_confounds=n_confounds, detrend=False 

1044 ) 

1045 

1046 assert_almost_equal(outC, outF, decimal=13) 

1047 

1048 

1049def test_high_variance_confounds_scaling(): 

1050 """Check result not be influenced by global scaling.""" 

1051 n_features = 1001 

1052 length = 20 

1053 n_confounds = 5 

1054 

1055 seriesC, _, _ = generate_signals( 

1056 n_features=n_features, length=length, order="C" 

1057 ) 

1058 

1059 seriesG = 2 * seriesC 

1060 outG = high_variance_confounds( 

1061 seriesG, n_confounds=n_confounds, detrend=False 

1062 ) 

1063 

1064 outC = high_variance_confounds( 

1065 seriesC, n_confounds=n_confounds, detrend=False 

1066 ) 

1067 

1068 assert_almost_equal(outC, outG, decimal=13) 

1069 assert outG.shape == (length, n_confounds) 

1070 

1071 

1072def test_high_variance_confounds_percentile(): 

1073 """Check changing percentile changes the result.""" 

1074 n_features = 1001 

1075 length = 20 

1076 n_confounds = 5 

1077 

1078 seriesC, _, _ = generate_signals( 

1079 n_features=n_features, length=length, order="C" 

1080 ) 

1081 seriesG = seriesC 

1082 outG = high_variance_confounds( 

1083 seriesG, percentile=1.0, n_confounds=n_confounds, detrend=False 

1084 ) 

1085 

1086 outC = high_variance_confounds( 

1087 seriesC, n_confounds=n_confounds, detrend=False 

1088 ) 

1089 

1090 with pytest.raises(AssertionError): 

1091 assert_almost_equal(outC, outG, decimal=13) 

1092 assert outG.shape == (length, n_confounds) 

1093 

1094 

1095def test_high_variance_confounds_detrend(): 

1096 """Check adding a trend and detrending give same results as no trend.""" 

1097 n_features = 1001 

1098 length = 20 

1099 n_confounds = 5 

1100 

1101 seriesC, _, _ = generate_signals( 

1102 n_features=n_features, length=length, order="C" 

1103 ) 

1104 seriesG = seriesC 

1105 

1106 # Check shape of output 

1107 out = high_variance_confounds(seriesG, n_confounds=7, detrend=False) 

1108 

1109 assert out.shape == (length, 7) 

1110 

1111 trends = generate_trends(n_features=n_features, length=length) 

1112 seriesGt = seriesG + trends 

1113 

1114 outG = high_variance_confounds( 

1115 seriesG, detrend=False, n_confounds=n_confounds 

1116 ) 

1117 outGt = high_variance_confounds( 

1118 seriesGt, detrend=True, n_confounds=n_confounds 

1119 ) 

1120 # Since sign flips could occur, we look at the absolute values of the 

1121 # covariance, rather than the absolute difference, and compare this to 

1122 # the identity matrix 

1123 assert_almost_equal( 

1124 np.abs(outG.T.dot(outG)), np.identity(outG.shape[1]), decimal=13 

1125 ) 

1126 # Control for sign flips by taking the min of both possibilities 

1127 assert_almost_equal( 

1128 np.min(np.abs(np.dstack([outG - outGt, outG + outGt])), axis=2), 

1129 np.zeros(outG.shape), 

1130 ) 

1131 

1132 

1133def test_high_variance_confounds_nan(): 

1134 """Control robustness to NaNs.""" 

1135 n_features = 1001 

1136 length = 20 

1137 n_confounds = 5 

1138 seriesC, _, _ = generate_signals( 

1139 n_features=n_features, length=length, order="C" 

1140 ) 

1141 

1142 seriesC[:, 0] = 0 

1143 out1 = high_variance_confounds(seriesC, n_confounds=n_confounds) 

1144 

1145 seriesC[:, 0] = np.nan 

1146 out2 = high_variance_confounds(seriesC, n_confounds=n_confounds) 

1147 

1148 assert_almost_equal(out1, out2, decimal=13) 

1149 

1150 

1151def test_clean_standardize_false(): 

1152 """Check output cleaning butterworth filter and no standardization.""" 

1153 n_samples = 500 

1154 n_features = 5 

1155 t_r = 2 

1156 

1157 signals, _, _ = generate_signals(n_features=n_features, length=n_samples) 

1158 

1159 cleaned_signals = clean(signals, standardize=False, detrend=False) 

1160 

1161 assert_almost_equal(cleaned_signals, signals) 

1162 

1163 # these show return the same results 

1164 cleaned_butterworth_signals = clean( 

1165 signals, 

1166 detrend=False, 

1167 standardize=False, 

1168 filter="butterworth", 

1169 high_pass=0.01, 

1170 t_r=t_r, 

1171 ) 

1172 butterworth_signals = butterworth( 

1173 signals, 

1174 sampling_rate=1 / t_r, 

1175 high_pass=0.01, 

1176 ) 

1177 

1178 assert_equal(cleaned_butterworth_signals, butterworth_signals) 

1179 

1180 

1181def test_clean_psc(rng): 

1182 """Test clean with percent signal change.""" 

1183 n_samples = 500 

1184 n_features = 5 

1185 

1186 signals = generate_signals_plus_trends( 

1187 n_features=n_features, n_samples=n_samples 

1188 ) 

1189 # positive mean signal 

1190 means = rng.standard_normal((1, n_features)) 

1191 signals_pos_mean = signals + means 

1192 

1193 # a mix of pos and neg mean signal 

1194 signals_mixed_mean = signals + np.append(means[:, :-3], -1 * means[:, -3:]) 

1195 

1196 # both types should pass 

1197 for s in [signals_pos_mean, signals_mixed_mean]: 

1198 # no detrend 

1199 cleaned_signals = clean(s, standardize="psc", detrend=False) 

1200 

1201 ss_signals = standardize_signal(s, detrend=False, standardize="psc") 

1202 assert_almost_equal(cleaned_signals.mean(0), 0) 

1203 assert_almost_equal(cleaned_signals, ss_signals) 

1204 

1205 # psc signal should correlate with z score, since it's just difference 

1206 # in scaling 

1207 z_signals = clean(s, standardize="zscore_sample", detrend=False) 

1208 

1209 _assert_correlation_almost_1(z_signals, cleaned_signals) 

1210 

1211 cleaned_signals = clean(s, standardize="psc", detrend=True) 

1212 z_signals = clean(s, standardize="zscore_sample", detrend=True) 

1213 

1214 assert_almost_equal(cleaned_signals.mean(0), 0) 

1215 _assert_correlation_almost_1(z_signals, cleaned_signals) 

1216 

1217 

1218def test_clean_psc_butterworth(rng): 

1219 """Test clean with percent signal change and a butterworth filter.""" 

1220 n_samples = 500 

1221 n_features = 5 

1222 

1223 signals = generate_signals_plus_trends( 

1224 n_features=n_features, n_samples=n_samples 

1225 ) 

1226 # positive mean signal 

1227 means = rng.standard_normal((1, n_features)) 

1228 signals_pos_mean = signals + means 

1229 

1230 # a mix of pos and neg mean signal 

1231 signals_mixed_mean = signals + np.append(means[:, :-3], -1 * means[:, -3:]) 

1232 

1233 # both types should pass 

1234 for s in [signals_pos_mean, signals_mixed_mean]: 

1235 # test with high pass with butterworth 

1236 hp_butterworth_signals = clean( 

1237 s, 

1238 detrend=False, 

1239 filter="butterworth", 

1240 high_pass=0.01, 

1241 t_r=2, 

1242 standardize="psc", 

1243 ) 

1244 z_butterworth_signals = clean( 

1245 s, 

1246 detrend=False, 

1247 filter="butterworth", 

1248 high_pass=0.01, 

1249 t_r=2, 

1250 standardize="zscore_sample", 

1251 ) 

1252 

1253 assert_almost_equal(hp_butterworth_signals.mean(0), 0) 

1254 _assert_correlation_almost_1( 

1255 z_butterworth_signals, hp_butterworth_signals 

1256 ) 

1257 

1258 

1259def _assert_correlation_almost_1(signal_1, signal_2): 

1260 """Check that correlation between 2 signals equal to 1.""" 

1261 assert_almost_equal( 

1262 np.corrcoef(signal_1[:, 0], signal_2[:, 0])[0, 1], 

1263 0.99999, 

1264 decimal=5, 

1265 ) 

1266 

1267 

1268def test_clean_psc_warning(rng): 

1269 """Leave out the last 3 columns with a mean of zero \ 

1270 to test user warning positive mean signal. 

1271 """ 

1272 n_samples = 500 

1273 n_features = 5 

1274 

1275 signals = generate_signals_plus_trends( 

1276 n_features=n_features, n_samples=n_samples 

1277 ) 

1278 

1279 means = rng.standard_normal((1, n_features)) 

1280 

1281 signals_w_zero = signals + np.append(means[:, :-3], np.zeros((1, 3))) 

1282 

1283 with pytest.warns(UserWarning) as records: 

1284 cleaned_w_zero = clean(signals_w_zero, standardize="psc") 

1285 

1286 psc_warning = sum( 

1287 "psc standardization strategy" in str(r.message) for r in records 

1288 ) 

1289 assert psc_warning == 1 

1290 assert_equal(cleaned_w_zero[:, -3:].mean(0), 0) 

1291 

1292 

1293def test_clean_zscore(rng): 

1294 """Check that cleaning with Z scoring gives expected results. 

1295 

1296 - mean of 0 

1297 - std of 1 

1298 - difference between and sample and population z-scoring. 

1299 """ 

1300 n_samples = 500 

1301 n_features = 5 

1302 

1303 signals, _, _ = generate_signals(n_features=n_features, length=n_samples) 

1304 

1305 signals += rng.standard_normal(size=(1, n_features)) 

1306 

1307 cleaned_signals_ = clean(signals, standardize="zscore") 

1308 

1309 assert_almost_equal(cleaned_signals_.mean(0), 0) 

1310 assert_almost_equal(cleaned_signals_.std(0), 1) 

1311 

1312 # Repeating test above but for new correct strategy 

1313 cleaned_signals = clean(signals, standardize="zscore_sample") 

1314 

1315 assert_almost_equal(cleaned_signals.mean(0), 0) 

1316 assert_almost_equal(cleaned_signals.std(0), 1, decimal=3) 

1317 

1318 # Show outcome from two zscore strategies is not equal 

1319 with pytest.raises(AssertionError): 

1320 assert_array_equal(cleaned_signals_, cleaned_signals) 

1321 

1322 

1323def test_create_cosine_drift_terms(): 

1324 """Testing cosine filter interface and output.""" 

1325 # fmriprep high pass cutoff is 128s, it's around 0.008 hz 

1326 t_r, high_pass = 2.5, 0.008 

1327 signals, _, confounds = generate_signals( 

1328 n_features=41, n_confounds=5, length=45 

1329 ) 

1330 

1331 # Not passing confounds it will return drift terms only 

1332 frame_times = np.arange(signals.shape[0]) * t_r 

1333 cosine_drift = create_cosine_drift(high_pass, frame_times)[:, :-1] 

1334 confounds_with_drift = np.hstack((confounds, cosine_drift)) 

1335 

1336 cosine_confounds = _create_cosine_drift_terms( 

1337 signals, confounds, high_pass, t_r 

1338 ) 

1339 assert_almost_equal(cosine_confounds, np.hstack((confounds, cosine_drift))) 

1340 

1341 # Not passing confounds it will return drift terms only 

1342 drift_terms_only = _create_cosine_drift_terms( 

1343 signals, None, high_pass, t_r 

1344 ) 

1345 assert_almost_equal(drift_terms_only, cosine_drift) 

1346 

1347 # drift terms in confounds will create warning and no change to confounds 

1348 with pytest.warns(UserWarning, match="user supplied confounds"): 

1349 cosine_confounds = _create_cosine_drift_terms( 

1350 signals, confounds_with_drift, high_pass, t_r 

1351 ) 

1352 assert_array_equal(cosine_confounds, confounds_with_drift) 

1353 

1354 # raise warning if cosine drift term is not created 

1355 high_pass_fail = 0.002 

1356 with pytest.warns(UserWarning, match="Cosine filter was not created"): 

1357 cosine_confounds = _create_cosine_drift_terms( 

1358 signals, confounds, high_pass_fail, t_r 

1359 ) 

1360 assert_array_equal(cosine_confounds, confounds) 

1361 

1362 

1363def test_clean_sample_mask(): 

1364 """Test sample_mask related feature.""" 

1365 signals, _, confounds = generate_signals( 

1366 n_features=11, n_confounds=5, length=40 

1367 ) 

1368 

1369 sample_mask = np.arange(signals.shape[0]) 

1370 scrub_index = [2, 3, 6, 7, 8, 30, 31, 32] 

1371 sample_mask = np.delete(sample_mask, scrub_index) 

1372 

1373 sample_mask_binary = np.full(signals.shape[0], True) 

1374 sample_mask_binary[scrub_index] = False 

1375 

1376 scrub_clean = clean(signals, confounds=confounds, sample_mask=sample_mask) 

1377 

1378 assert scrub_clean.shape[0] == sample_mask.shape[0] 

1379 

1380 # test the binary mask 

1381 scrub_clean_bin = clean( 

1382 signals, confounds=confounds, sample_mask=sample_mask_binary 

1383 ) 

1384 assert_equal(scrub_clean_bin, scrub_clean) 

1385 

1386 

1387def test_sample_mask_across_runs(): 

1388 """Test sample_mask related feature but with several runs.""" 

1389 # list of sample_mask for each run 

1390 signals, _, confounds = generate_signals( 

1391 n_features=11, n_confounds=5, length=40 

1392 ) 

1393 

1394 runs = np.ones(signals.shape[0]) 

1395 runs[: signals.shape[0] // 2] = 0 

1396 

1397 sample_mask_sep = [np.arange(20), np.arange(20)] 

1398 scrub_index = [[6, 7, 8], [10, 11, 12]] 

1399 sample_mask_sep = [ 

1400 np.delete(sm, si) for sm, si in zip(sample_mask_sep, scrub_index) 

1401 ] 

1402 

1403 scrub_sep_mask = clean( 

1404 signals, confounds=confounds, sample_mask=sample_mask_sep, runs=runs 

1405 ) 

1406 

1407 assert scrub_sep_mask.shape[0] == signals.shape[0] - 6 

1408 

1409 # test for binary mask per run 

1410 sample_mask_sep_binary = [ 

1411 np.full(signals.shape[0] // 2, True), 

1412 np.full(signals.shape[0] // 2, True), 

1413 ] 

1414 sample_mask_sep_binary[0][scrub_index[0]] = False 

1415 sample_mask_sep_binary[1][scrub_index[1]] = False 

1416 scrub_sep_mask = clean( 

1417 signals, 

1418 confounds=confounds, 

1419 sample_mask=sample_mask_sep_binary, 

1420 runs=runs, 

1421 ) 

1422 

1423 assert scrub_sep_mask.shape[0] == signals.shape[0] - 6 

1424 

1425 

1426def test_clean_sample_mask_error(): 

1427 """Check proper errors are thrown when using clean with sample_mask.""" 

1428 signals, _, _ = generate_signals(n_features=11, n_confounds=5, length=40) 

1429 

1430 sample_mask = np.arange(signals.shape[0]) 

1431 scrub_index = [2, 3, 6, 7, 8, 30, 31, 32] 

1432 sample_mask = np.delete(sample_mask, scrub_index) 

1433 

1434 # list of sample_mask for each run 

1435 runs = np.ones(signals.shape[0]) 

1436 runs[: signals.shape[0] // 2] = 0 

1437 

1438 sample_mask_sep = [np.arange(20), np.arange(20)] 

1439 scrub_index = [[6, 7, 8], [10, 11, 12]] 

1440 sample_mask_sep = [ 

1441 np.delete(sm, si) for sm, si in zip(sample_mask_sep, scrub_index) 

1442 ] 

1443 

1444 # 1D sample mask with runs labels 

1445 with pytest.raises( 

1446 ValueError, match=r"Number of sample_mask \(\d\) not matching" 

1447 ): 

1448 clean(signals, sample_mask=sample_mask, runs=runs) 

1449 

1450 # invalid input for sample_mask 

1451 with pytest.raises(TypeError, match="unhandled type"): 

1452 clean(signals, sample_mask="not_supported") 

1453 

1454 # sample_mask too long 

1455 with pytest.raises( 

1456 IndexError, match="more timepoints than the current run" 

1457 ): 

1458 clean(signals, sample_mask=np.hstack((sample_mask, sample_mask))) 

1459 

1460 # list of sample_mask with one that's too long 

1461 invalid_sample_mask_sep = [np.arange(10), np.arange(30)] 

1462 with pytest.raises( 

1463 IndexError, match="more timepoints than the current run" 

1464 ): 

1465 clean(signals, sample_mask=invalid_sample_mask_sep, runs=runs) 

1466 

1467 # list of sample_mask with invalid indexing in one 

1468 sample_mask_sep[-1][-1] = 100 

1469 with pytest.raises(IndexError, match="invalid index"): 

1470 clean(signals, sample_mask=sample_mask_sep, runs=runs) 

1471 

1472 # invalid index in 1D sample_mask 

1473 sample_mask[-1] = 999 

1474 with pytest.raises(IndexError, match=r"invalid index \[\d*\]"): 

1475 clean(signals, sample_mask=sample_mask) 

1476 

1477 

1478def test_handle_scrubbed_volumes(): 

1479 """Check interpolation/censoring of signals based on filter type.""" 

1480 signals, _, confounds = generate_signals( 

1481 n_features=11, n_confounds=5, length=40 

1482 ) 

1483 

1484 sample_mask = np.arange(signals.shape[0]) 

1485 scrub_index = np.array([2, 3, 6, 7, 8, 30, 31, 32]) 

1486 sample_mask = np.delete(sample_mask, scrub_index) 

1487 

1488 ( 

1489 interpolated_signals, 

1490 interpolated_confounds, 

1491 sample_mask, 

1492 ) = _handle_scrubbed_volumes( 

1493 signals, confounds, sample_mask, "butterworth", 2.5, True 

1494 ) 

1495 

1496 assert_equal(interpolated_signals[sample_mask, :], signals[sample_mask, :]) 

1497 assert_equal( 

1498 interpolated_confounds[sample_mask, :], confounds[sample_mask, :] 

1499 ) 

1500 

1501 ( 

1502 scrubbed_signals, 

1503 scrubbed_confounds, 

1504 sample_mask, 

1505 ) = _handle_scrubbed_volumes( 

1506 signals, confounds, sample_mask, "cosine", 2.5, True 

1507 ) 

1508 

1509 assert_equal(scrubbed_signals, signals[sample_mask, :]) 

1510 assert_equal(scrubbed_confounds, confounds[sample_mask, :]) 

1511 

1512 

1513def test_handle_scrubbed_volumes_with_extrapolation(): 

1514 """Check interpolation of signals with extrapolation.""" 

1515 signals, _, confounds = generate_signals( 

1516 n_features=11, n_confounds=5, length=40 

1517 ) 

1518 

1519 sample_mask = np.arange(signals.shape[0]) 

1520 scrub_index = np.concatenate((np.arange(5), [10, 20, 30])) 

1521 sample_mask = np.delete(sample_mask, scrub_index) 

1522 

1523 # Test cubic spline interpolation (enabled extrapolation) in the 

1524 # very first n=5 samples of generated signal 

1525 extrapolate_warning = ( 

1526 "By default the cubic spline interpolator extrapolates " 

1527 "the out-of-bounds censored volumes in the data run. This " 

1528 "can lead to undesired filtered signal results. Starting in " 

1529 "version 0.13, the default strategy will be not to extrapolate " 

1530 "but to discard those volumes at filtering." 

1531 ) 

1532 with pytest.warns(FutureWarning, match=extrapolate_warning): 

1533 ( 

1534 extrapolated_signals, 

1535 extrapolated_confounds, 

1536 extrapolated_sample_mask, 

1537 ) = _handle_scrubbed_volumes( 

1538 signals, confounds, sample_mask, "butterworth", 2.5, True 

1539 ) 

1540 assert_equal(signals.shape[0], extrapolated_signals.shape[0]) 

1541 assert_equal(confounds.shape[0], extrapolated_confounds.shape[0]) 

1542 assert_equal(sample_mask, extrapolated_sample_mask) 

1543 

1544 

1545def test_handle_scrubbed_volumes_without_extrapolation(): 

1546 """Check interpolation of signals disabling extrapolation.""" 

1547 signals, _, confounds = generate_signals( 

1548 n_features=11, n_confounds=5, length=40 

1549 ) 

1550 

1551 outer_samples = [0, 1, 2, 3, 4] 

1552 inner_samples = [10, 20, 30] 

1553 total_samples = len(outer_samples) + len(inner_samples) 

1554 sample_mask = np.arange(signals.shape[0]) 

1555 scrub_index = np.concatenate((outer_samples, inner_samples)) 

1556 sample_mask = np.delete(sample_mask, scrub_index) 

1557 

1558 # Test cubic spline interpolation without predicting values outside 

1559 # the range of the signal available (disabled extrapolation), discarding 

1560 # the first n censored samples of generated signal 

1561 ( 

1562 interpolated_signals, 

1563 interpolated_confounds, 

1564 interpolated_sample_mask, 

1565 ) = _handle_scrubbed_volumes( 

1566 signals, confounds, sample_mask, "butterworth", 2.5, False 

1567 ) 

1568 assert_equal( 

1569 signals.shape[0], interpolated_signals.shape[0] + len(outer_samples) 

1570 ) 

1571 assert_equal( 

1572 confounds.shape[0], 

1573 interpolated_confounds.shape[0] + len(outer_samples), 

1574 ) 

1575 assert_equal(sample_mask - sample_mask[0], interpolated_sample_mask) 

1576 

1577 # Assert that the modified sample mask (interpolated_sample_mask) 

1578 # can be applied to the interpolated signals and confounds 

1579 ( 

1580 censored_signals, 

1581 censored_confounds, 

1582 ) = _censor_signals( 

1583 interpolated_signals, interpolated_confounds, interpolated_sample_mask 

1584 ) 

1585 assert_equal(signals.shape[0], censored_signals.shape[0] + total_samples) 

1586 assert_equal( 

1587 confounds.shape[0], censored_confounds.shape[0] + total_samples 

1588 ) 

1589 

1590 

1591def test_handle_scrubbed_volumes_exception(): 

1592 """Check if an exception is raised when the sample mask is empty.""" 

1593 signals, _, confounds = generate_signals( 

1594 n_features=11, n_confounds=5, length=40 

1595 ) 

1596 

1597 sample_mask = np.arange(signals.shape[0]) 

1598 scrub_index = np.arange(signals.shape[0]) 

1599 sample_mask = np.delete(sample_mask, scrub_index) 

1600 

1601 with pytest.raises( 

1602 AllVolumesRemovedError, 

1603 match="The size of the sample mask is 0. " 

1604 "All volumes were marked as motion outliers " 

1605 "can not proceed. ", 

1606 ): 

1607 _handle_scrubbed_volumes( 

1608 signals, confounds, sample_mask, "butterworth", 2.5, True 

1609 )