diff --git a/test/handler/test_elmmake_handler.vader b/test/handler/test_elmmake_handler.vader
index cbd7ac9f..f3424b4b 100644
--- a/test/handler/test_elmmake_handler.vader
+++ b/test/handler/test_elmmake_handler.vader
@@ -1,6 +1,14 @@
 Before:
+  let b:tmp = has('win32') ? substitute($TMP, '\\', '\\\\', 'g') : $TMPDIR
+
   runtime ale_linters/elm/make.vim
 
+After:
+  unlet! b:tmp
+  unlet! g:config_error_lines
+
+  call ale#linter#Reset()
+
 Execute(The elm-make handler should parse lines correctly):
   AssertEqual
   \ [
@@ -42,8 +50,8 @@ Execute(The elm-make handler should parse lines correctly):
   \   },
   \ ],
   \ ale_linters#elm#make#Handle(347, [
-  \   '[{"tag":"unused import","overview":"warning overview","details":"warning details","region":{"start":{"line":33,"column":1},"end":{"line":33,"column":19}},"type":"warning","file":"' . $TMPDIR . 'Module.elm"}]',
-  \   '[{"tag":"TYPE MISMATCH","overview":"error overview 1","subregion":{"start":{"line":406,"column":5},"end":{"line":408,"column":18}},"details":"error details 1","region":{"start":{"line":404,"column":1},"end":{"line":408,"column":18}},"type":"error","file":"' . $TMPDIR . 'Module.elm"},{"tag":"TYPE MISMATCH","overview":"error overview 2","subregion":{"start":{"line":407,"column":12},"end":{"line":407,"column":17}},"details":"error details 2","region":{"start":{"line":406,"column":5},"end":{"line":407,"column":17}},"type":"error","file":"' . $TMPDIR . 'Module.elm"},{"tag":"TYPE MISMATCH","overview":"error overview 3","subregion":{"start":{"line":406,"column":88},"end":{"line":406,"column":93}},"details":"error details 3","region":{"start":{"line":406,"column":5},"end":{"line":406,"column":93}},"type":"error","file":"' . $TMPDIR . 'Module.elm"}]'
+  \   '[{"tag":"unused import","overview":"warning overview","details":"warning details","region":{"start":{"line":33,"column":1},"end":{"line":33,"column":19}},"type":"warning","file":"' . b:tmp . 'Module.elm"}]',
+  \   '[{"tag":"TYPE MISMATCH","overview":"error overview 1","subregion":{"start":{"line":406,"column":5},"end":{"line":408,"column":18}},"details":"error details 1","region":{"start":{"line":404,"column":1},"end":{"line":408,"column":18}},"type":"error","file":"' . b:tmp . 'Module.elm"},{"tag":"TYPE MISMATCH","overview":"error overview 2","subregion":{"start":{"line":407,"column":12},"end":{"line":407,"column":17}},"details":"error details 2","region":{"start":{"line":406,"column":5},"end":{"line":407,"column":17}},"type":"error","file":"' . b:tmp . 'Module.elm"},{"tag":"TYPE MISMATCH","overview":"error overview 3","subregion":{"start":{"line":406,"column":88},"end":{"line":406,"column":93}},"details":"error details 3","region":{"start":{"line":406,"column":5},"end":{"line":406,"column":93}},"type":"error","file":"' . b:tmp . 'Module.elm"}]'
   \ ])
 
 Execute(The elm-make handler should put an error on the first line if a line cannot be parsed):
@@ -66,11 +74,7 @@ Execute(The elm-make handler should put an error on the first line if a line can
   \   },
   \ ],
   \ ale_linters#elm#make#Handle(347, [
-  \   '[{"tag":"unused import","overview":"warning overview","details":"warning details","region":{"start":{"line":33,"column":1},"end":{"line":33,"column":19}},"type":"warning","file":"' . $TMPDIR . 'Module.elm"}]',
+  \   '[{"tag":"unused import","overview":"warning overview","details":"warning details","region":{"start":{"line":33,"column":1},"end":{"line":33,"column":19}},"type":"warning","file":"' . b:tmp . 'Module.elm"}]',
   \   "Not JSON",
   \   "Also not JSON",
   \ ])
-
-After:
-  unlet! g:config_error_lines
-  call ale#linter#Reset()
diff --git a/test/handler/test_ghc_handler.vader b/test/handler/test_ghc_handler.vader
index bf54386e..b76046ca 100644
--- a/test/handler/test_ghc_handler.vader
+++ b/test/handler/test_ghc_handler.vader
@@ -36,11 +36,11 @@ Execute(The ghc handler should handle ghc 8 output):
   \ ],
   \ ale#handlers#haskell#HandleGHCFormat(bufnr(''), [
   \ '',
-  \ 'src/Appoint/Lib.hs:6:1: error:',
+  \ ale#path#Winify('src/Appoint/Lib.hs') . ':6:1: error:',
   \ '    Failed to load interface for ‘GitHub.Data’',
   \ '    Use -v to see a list of the files searched for.',
   \ '',
-  \ 'src/Appoint/Lib.hs:7:1: warning:',
+  \ ale#path#Winify('src/Appoint/Lib.hs') . ':7:1: warning:',
   \ '    Failed to load interface for ‘GitHub.Endpoints.PullRequests’',
   \ '    Use -v to see a list of the files searched for.',
   \ ])
@@ -70,9 +70,9 @@ Execute(The ghc handler should handle ghc 7 output):
   \   },
   \ ],
   \ ale#handlers#haskell#HandleGHCFormat(bufnr(''), [
-  \ 'src/Main.hs:168:1:',
+  \ ale#path#Winify('src/Main.hs') . ':168:1:',
   \ '    parse error (possibly incorrect indentation or mismatched brackets)',
-  \ 'src/Main.hs:84:1:Warning: Top-level binding with no type signature:^@  myLayout :: Choose Tall (Choose (Mirror Tall) Full) a',
-  \ 'src/Main.hs:94:5:Error:',
+  \ ale#path#Winify('src/Main.hs') . ':84:1:Warning: Top-level binding with no type signature:^@  myLayout :: Choose Tall (Choose (Mirror Tall) Full) a',
+  \ ale#path#Winify('src/Main.hs') . ':94:5:Error:',
   \ '  Some other error',
   \ ])
diff --git a/test/handler/test_idris_handler.vader b/test/handler/test_idris_handler.vader
index 1c20be7b..b1fb2a01 100644
--- a/test/handler/test_idris_handler.vader
+++ b/test/handler/test_idris_handler.vader
@@ -1,6 +1,16 @@
 Before:
+  Save $TMPDIR
+
+  " Set TMPDIR so the temporary file checks work.
+  let $TMPDIR = '/tmp'
+
   runtime ale_linters/idris/idris.vim
 
+After:
+  Restore
+
+  call ale#linter#Reset()
+
 Execute(The idris handler should parse messages that reference a single column):
   call ale#test#SetFilename('/tmp/foo.idr')
 
diff --git a/test/handler/test_mypy_handler.vader b/test/handler/test_mypy_handler.vader
index d0cf91e2..a3e224f3 100644
--- a/test/handler/test_mypy_handler.vader
+++ b/test/handler/test_mypy_handler.vader
@@ -1,64 +1,66 @@
 Before:
   runtime ale_linters/python/mypy.vim
 
+  call ale#test#SetDirectory('/testplugin/test/handler')
+
 After:
+  call ale#test#RestoreDirectory()
   call ale#linter#Reset()
-  silent file something_else.py
 
 Execute(The mypy handler should parse lines correctly):
-  silent file foo/bar/__init__.py
+  call ale#test#SetFilename('__init__.py')
 
   AssertEqual
   \ [
   \   {
   \     'lnum': 768,
   \     'col': 38,
-  \     'filename': 'foo/bar/foo/bar/baz.py',
+  \     'filename': ale#path#Winify(g:dir . '/baz.py'),
   \     'type': 'E',
   \     'text': 'Cannot determine type of ''SOME_SYMBOL''',
   \   },
   \   {
   \     'lnum': 821,
   \     'col': 38,
-  \     'filename': 'foo/bar/foo/bar/baz.py',
+  \     'filename': ale#path#Winify(g:dir . '/baz.py'),
   \     'type': 'E',
   \     'text': 'Cannot determine type of ''SOME_SYMBOL''',
   \   },
   \   {
   \     'lnum': 38,
   \     'col': 44,
-  \     'filename': 'foo/bar/foo/bar/other.py',
+  \     'filename': ale#path#Winify(g:dir . '/other.py'),
   \     'type': 'E',
   \     'text': 'Cannot determine type of ''ANOTHER_SYMBOL''',
   \   },
   \   {
   \     'lnum': 15,
   \     'col': 3,
-  \     'filename': 'foo/bar/foo/bar/__init__.py',
+  \     'filename': ale#path#Winify(g:dir . '/__init__.py'),
   \     'type': 'E',
   \     'text': 'Argument 1 to "somefunc" has incompatible type "int"; expected "str"'
   \   },
   \   {
   \     'lnum': 72,
   \     'col': 1,
-  \     'filename': 'foo/bar/foo/bar/__init__.py',
+  \     'filename': ale#path#Winify(g:dir . '/__init__.py'),
   \     'type': 'W',
   \     'text': 'Some warning',
   \   },
   \ ],
   \ ale_linters#python#mypy#Handle(bufnr(''), [
-  \ 'foo/bar/baz.py: note: In class "SomeClass":',
-  \ 'foo/bar/baz.py:768:38: error: Cannot determine type of ''SOME_SYMBOL''',
-  \ 'foo/bar/baz.py: note: In class "AnotherClass":',
-  \ 'foo/bar/baz.py:821:38: error: Cannot determine type of ''SOME_SYMBOL''',
-  \ 'foo/bar/__init__.py:92: note: In module imported here:',
-  \ 'foo/bar/other.py: note: In class "YetAnotherClass":',
-  \ 'foo/bar/other.py:38:44: error: Cannot determine type of ''ANOTHER_SYMBOL''',
-  \ 'foo/bar/__init__.py: note: At top level:',
-  \ 'foo/bar/__init__.py:15:3: error: Argument 1 to "somefunc" has incompatible type "int"; expected "str"',
+  \ 'baz.py: note: In class "SomeClass":',
+  \ 'baz.py:768:38: error: Cannot determine type of ''SOME_SYMBOL''',
+  \ 'baz.py: note: In class "AnotherClass":',
+  \ 'baz.py:821:38: error: Cannot determine type of ''SOME_SYMBOL''',
+  \ '__init__.py:92: note: In module imported here:',
+  \ 'other.py: note: In class "YetAnotherClass":',
+  \ 'other.py:38:44: error: Cannot determine type of ''ANOTHER_SYMBOL''',
+  \ '__init__.py: note: At top level:',
+  \ '__init__.py:15:3: error: Argument 1 to "somefunc" has incompatible type "int"; expected "str"',
   \ 'another_module/bar.py:14: note: In module imported here,',
   \ 'another_module/__init__.py:16: note: ... from here,',
-  \ 'foo/bar/__init__.py:72:1: warning: Some warning',
+  \ '__init__.py:72:1: warning: Some warning',
   \ ])
 
 Execute(The mypy handler should handle Windows names with spaces):
diff --git a/test/handler/test_perl_handler.vader b/test/handler/test_perl_handler.vader
index 18c5d706..9e1c5203 100644
--- a/test/handler/test_perl_handler.vader
+++ b/test/handler/test_perl_handler.vader
@@ -15,9 +15,9 @@ Execute(The Perl linter should ignore errors from other files):
   \    {'lnum': '2', 'type': 'E', 'text': 'Compilation failed in require'},
   \ ],
   \ ale_linters#perl#perl#Handle(bufnr(''), [
-  \   'syntax error at ' . g:dir . '/foo.pm line 4, near "aklsdfjmy "',
-  \   'Compilation failed in require at ' . g:dir . '/bar.pl line 2.',
-  \   'BEGIN failed--compilation aborted at ' . g:dir . '/bar.pl line 2.',
+  \   'syntax error at ' . ale#path#Winify(g:dir . '/foo.pm') . ' line 4, near "aklsdfjmy "',
+  \   'Compilation failed in require at ' . ale#path#Winify(g:dir . '/bar.pl') . ' line 2.',
+  \   'BEGIN failed--compilation aborted at ' . ale#path#Winify(g:dir . '/bar.pl') . ' line 2.',
   \ ])
 
 Execute(The Perl linter should complain about failing to locate modules):
diff --git a/test/handler/test_rails_best_practices_handler.vader b/test/handler/test_rails_best_practices_handler.vader
index 11875cbd..9875e977 100644
--- a/test/handler/test_rails_best_practices_handler.vader
+++ b/test/handler/test_rails_best_practices_handler.vader
@@ -29,11 +29,11 @@ Execute(The rails_best_practices handler should parse JSON correctly):
   \    '{',
   \      '"message": "use local variable",',
   \      '"line_number": "5",',
-  \      '"filename": "' . g:dir . '/ruby_fixtures/valid_rails_app/app/models/thing.rb"',
+  \      printf('"filename": "%s"', substitute(expand('%:p'), '\\', '\\\\', 'g')),
   \    '},{',
   \      '"message": "other advice",',
   \      '"line_number": "10",',
-  \      '"filename": "' . g:dir . '/ruby_fixtures/valid_rails_app/app/models/thing.rb"',
+  \      printf('"filename": "%s"', substitute(expand('%:p'), '\\', '\\\\', 'g')),
   \    '}',
   \  ']'
   \  ])