1package main
2
3import (
4 "bytes"
5 "context"
6 "errors"
7 "fmt"
8 "io/fs"
9 "os"
10 "os/exec"
11 "path/filepath"
12 "regexp"
13 "sort"
14 "strings"
15 "syscall"
16 "time"
17)
18
19func runcase(cfg suiteconfig, tc testcase) caseresult {
20 result := caseresult{Test: tc}
21
22 workdir := filepath.Join(cfg.temproot, filepath.FromSlash(tc.RelPath))
23 if err := os.MkdirAll(workdir, 0o755); err != nil {
24 return failedcaseresult(cfg, tc, result, workdir, fmt.Sprintf("create workdir: %v", err))
25 }
26
27 if err := materializesetup(workdir, tc.Meta.Setup); err != nil {
28 return failedcaseresult(cfg, tc, result, workdir, fmt.Sprintf("prepare setup: %v", err))
29 }
30
31 execdir := workdir
32 if tc.Meta.Cwd != "" {
33 execdir = filepath.Join(workdir, filepath.FromSlash(tc.Meta.Cwd))
34 if err := os.MkdirAll(execdir, 0o755); err != nil {
35 return failedcaseresult(cfg, tc, result, workdir, fmt.Sprintf("prepare cwd: %v", err))
36 }
37 }
38
39 makename, aliases := mkfiles(tc)
40 for _, alias := range aliases {
41 aliaspath := filepath.Join(workdir, filepath.FromSlash(alias))
42 if err := os.MkdirAll(filepath.Dir(aliaspath), 0o755); err != nil {
43 return failedcaseresult(cfg, tc, result, workdir, fmt.Sprintf("prepare makefile dir: %v", err))
44 }
45 if err := os.WriteFile(aliaspath, []byte(tc.MakeContents), 0o644); err != nil {
46 return failedcaseresult(cfg, tc, result, workdir, fmt.Sprintf("write makefile: %v", err))
47 }
48 }
49
50 logpath := filepath.Join(workdir, tc.Meta.Case+".log")
51 runpath := filepath.Join(workdir, tc.Meta.Case+".run")
52 basepath := filepath.Join(workdir, tc.Meta.Case+".base")
53 diffpath := filepath.Join(workdir, tc.Meta.Case+".diff")
54
55 command, commandstring, err := buildcommand(cfg.makepath, makename, tc)
56 if err != nil {
57 return failedcaseresult(cfg, tc, result, workdir, fmt.Sprintf("build command: %v", err))
58 }
59
60 env := buildenv(cfg, execdir, tc.Meta.Env)
61 logdata, exitstatus, timedout, err := executecase(command, execdir, env, tc.Meta.Stdin, tc.Meta.TimeoutSeconds)
62 if err != nil {
63 return failedcaseresult(cfg, tc, result, workdir, fmt.Sprintf("run case: %v", err))
64 }
65 if timedout {
66 logdata = append(logdata, []byte(fmt.Sprintf("\ntest timed out after %d seconds\n", tc.Meta.TimeoutSeconds))...)
67 }
68
69 extratmp, extratext, err := leftovertempfiles(filepath.Join(workdir, "_tmp"))
70 if err != nil {
71 return failedcaseresult(cfg, tc, result, workdir, fmt.Sprintf("check temp files: %v", err))
72 }
73 if extratmp {
74 logdata = append(logdata, []byte(extratext)...)
75 }
76
77 if err := os.WriteFile(logpath, logdata, 0o644); err != nil {
78 return failedcaseresult(cfg, tc, result, workdir, fmt.Sprintf("write log: %v", err))
79 }
80
81 result.ExitMatched = exitstatus == tc.Meta.ExpectedExit
82 result.OutputMatched = compareoutput(tc, cfg.makepath, workdir, env, string(logdata))
83 result.ExtraTmp = extratmp
84 result.Passed = result.ExitMatched && result.OutputMatched && !result.ExtraTmp
85
86 if result.Passed {
87 if cfg.keep {
88 _, preserveerr := preservecasedir(cfg, tc.RelPath, workdir)
89 if preserveerr != nil {
90 return failedcaseresult(cfg, tc, result, workdir, fmt.Sprintf("preserve case: %v", preserveerr))
91 }
92 }
93 return result
94 }
95
96 if err := os.WriteFile(runpath, []byte(commandstring), 0o644); err != nil {
97 return failedcaseresult(cfg, tc, result, workdir, fmt.Sprintf("write run file: %v", err))
98 }
99 if tc.Meta.CompareOutput && !result.OutputMatched {
100 if err := os.WriteFile(basepath, []byte(tc.ExpectedOut), 0o644); err != nil {
101 return failedcaseresult(cfg, tc, result, workdir, fmt.Sprintf("write base file: %v", err))
102 }
103 if err := writediff(basepath, logpath, diffpath); err != nil {
104 return failedcaseresult(cfg, tc, result, workdir, fmt.Sprintf("write diff: %v", err))
105 }
106 }
107
108 _, err = preservecasedir(cfg, tc.RelPath, workdir)
109 if err != nil {
110 return failedcaseresult(cfg, tc, result, workdir, fmt.Sprintf("preserve failure: %v", err))
111 }
112 return result
113}
114
115func failedcaseresult(cfg suiteconfig, tc testcase, result caseresult, workdir, message string) caseresult {
116 errorfile := filepath.Join(workdir, "runner-error.txt")
117 if workdir != "" {
118 _ = os.MkdirAll(workdir, 0o755)
119 _ = os.WriteFile(errorfile, []byte(message+"\n"), 0o644)
120 _, _ = preservecasedir(cfg, tc.RelPath, workdir)
121 }
122 return result
123}
124
125func materializesetup(root string, entries []setupentry) error {
126 for _, entry := range entries {
127 fullpath := filepath.Join(root, filepath.FromSlash(entry.Path))
128 switch entry.Kind {
129 case "dir":
130 if err := os.MkdirAll(fullpath, 0o755); err != nil {
131 return err
132 }
133 if mode, ok := parsemode(entry.Mode); ok {
134 if err := os.Chmod(fullpath, mode); err != nil {
135 return err
136 }
137 }
138 if err := settimes(fullpath, entry.MTime); err != nil {
139 return err
140 }
141 case "file":
142 if err := os.MkdirAll(filepath.Dir(fullpath), 0o755); err != nil {
143 return err
144 }
145 mode := fs.FileMode(0o644)
146 if parsed, ok := parsemode(entry.Mode); ok {
147 mode = parsed
148 }
149 if err := os.WriteFile(fullpath, []byte(entry.Content), mode); err != nil {
150 return err
151 }
152 if err := settimes(fullpath, entry.MTime); err != nil {
153 return err
154 }
155 case "symlink":
156 if err := os.MkdirAll(filepath.Dir(fullpath), 0o755); err != nil {
157 return err
158 }
159 if err := os.Symlink(entry.Target, fullpath); err != nil {
160 return err
161 }
162 default:
163 return fmt.Errorf("unsupported setup entry kind %q", entry.Kind)
164 }
165 }
166 return nil
167}
168
169func parsemode(s string) (fs.FileMode, bool) {
170 if s == "" {
171 return 0, false
172 }
173 var mode uint32
174 _, err := fmt.Sscanf(s, "%o", &mode)
175 if err != nil {
176 return 0, false
177 }
178 return fs.FileMode(mode), true
179}
180
181func settimes(path string, mtime int64) error {
182 if mtime == 0 {
183 return nil
184 }
185 ts := time.Unix(mtime, 0)
186 return os.Chtimes(path, ts, ts)
187}
188
189func mkfiles(tc testcase) (string, []string) {
190 if tc.Meta.Command != "" {
191 return "", nil
192 }
193 seen := map[string]bool{}
194 var names []string
195 add := func(name string) {
196 if name == "" || seen[name] {
197 return
198 }
199 seen[name] = true
200 names = append(names, name)
201 }
202
203 add(tc.Meta.MakefileName)
204
205 for _, match := range makefilenamepattern.FindAllString(tc.MakeContents, -1) {
206 add(match)
207 }
208 for _, match := range makefilenamepattern.FindAllString(tc.ExpectedOut, -1) {
209 add(match)
210 }
211 for _, entry := range tc.Meta.Setup {
212 for _, match := range makefilenamepattern.FindAllString(entry.Path, -1) {
213 add(match)
214 }
215 for _, match := range makefilenamepattern.FindAllString(entry.Content, -1) {
216 add(match)
217 }
218 }
219 if tc.Meta.Options.Shell != "" {
220 for _, match := range makefilenamepattern.FindAllString(tc.Meta.Options.Shell, -1) {
221 add(match)
222 }
223 }
224 for _, arg := range tc.Meta.Options.Args {
225 for _, match := range makefilenamepattern.FindAllString(arg, -1) {
226 add(match)
227 }
228 }
229
230 primary := ""
231 if tc.MakeContents != "" {
232 defaultname := tc.Meta.Case + ".mk"
233 add(defaultname)
234 primary = defaultname
235 }
236 if len(names) > 0 {
237 primary = names[0]
238 }
239 return primary, names
240}
241
242func buildcommand(makepath, makefilename string, tc testcase) (*exec.Cmd, string, error) {
243 if tc.Meta.Command != "" {
244 cmd := exec.Command("/bin/sh", "-c", tc.Meta.Command)
245 return cmd, formatruncommand([]string{tc.Meta.Command}), nil
246 }
247
248 quotedmake := shellquote(makepath)
249
250 switch tc.Meta.OptionsMode {
251 case "", "argv":
252 args := []string{makepath}
253 if makefilename != "" {
254 args = append(args, "-f", makefilename)
255 }
256 if tc.Meta.Options.Shell != "" && strings.TrimSpace(tc.Meta.Options.Shell) != "" {
257 args = append(args, tc.Meta.Options.Shell)
258 }
259 args = append(args, tc.Meta.Options.Args...)
260 cmd := exec.Command(args[0], args[1:]...)
261 return cmd, formatruncommand(args), nil
262 case "shell":
263 shellcommand := quotedmake
264 if makefilename != "" {
265 shellcommand += " -f " + shellquote(makefilename)
266 }
267 if tc.Meta.Options.Shell != "" {
268 shellcommand += " " + tc.Meta.Options.Shell
269 }
270 cmd := exec.Command("/bin/sh", "-c", shellcommand)
271 return cmd, formatruncommand([]string{shellcommand}), nil
272 default:
273 return nil, "", fmt.Errorf("unsupported options_mode %q", tc.Meta.OptionsMode)
274 }
275}
276
277func formatruncommand(args []string) string {
278 if len(args) == 1 {
279 return args[0] + "\n"
280 }
281 quoted := make([]string, 0, len(args))
282 for _, arg := range args {
283 quoted = append(quoted, shellquote(arg))
284 }
285 return strings.Join(quoted, " ") + "\n"
286}
287
288func shellquote(s string) string {
289 if s == "" {
290 return "''"
291 }
292 if !strings.ContainsAny(s, " \t\n'\"#$&;|*?()<>[]{}!~`") {
293 return s
294 }
295 return "'" + strings.ReplaceAll(s, "'", "'\\''") + "'"
296}
297
298// had some issues, so create a stable env and prepends tests/ to path so
299// we always get our make and testhelp wrappers
300func buildenv(cfg suiteconfig, workdir string, deltas map[string]*string) []string {
301 envmap := make(map[string]string)
302 for _, entry := range os.Environ() {
303 parts := strings.SplitN(entry, "=", 2)
304 if len(parts) != 2 {
305 continue
306 }
307 envmap[parts[0]] = parts[1]
308 }
309 envmap["LC_ALL"] = "C"
310 envmap["LANG"] = "C"
311 envmap["LANGUAGE"] = "C"
312 tmpdir := filepath.Join(workdir, "_tmp")
313 envmap["TMPDIR"] = tmpdir
314 envmap["TMP"] = tmpdir
315 envmap["TEMP"] = tmpdir
316 path := filepath.Join(cfg.reporoot, "tests")
317 if current, ok := envmap["PATH"]; ok && current != "" {
318 path += string(os.PathListSeparator) + current
319 }
320 envmap["PATH"] = path
321 for key, value := range deltas {
322 if value == nil {
323 delete(envmap, key)
324 continue
325 }
326 envmap[key] = expandenvplaceholders(*value, workdir, envmap)
327 }
328 keys := make([]string, 0, len(envmap))
329 for key := range envmap {
330 keys = append(keys, key)
331 }
332 sort.Strings(keys)
333 env := make([]string, 0, len(keys))
334 for _, key := range keys {
335 env = append(env, key+"="+envmap[key])
336 }
337 return env
338}
339
340func expandenvplaceholders(value, workdir string, envmap map[string]string) string {
341 value = strings.ReplaceAll(value, "#PWD#", filepath.ToSlash(workdir))
342 value = strings.ReplaceAll(value, "#PATH#", envmap["PATH"])
343 return value
344}
345
346func executecase(cmd *exec.Cmd, workdir string, env []string, stdin string, timeoutseconds int) ([]byte, int, bool, error) {
347 if err := os.MkdirAll(filepath.Join(workdir, "_tmp"), 0o755); err != nil {
348 return nil, 0, false, err
349 }
350
351 ctx := context.Background()
352 var cancel context.CancelFunc
353 if timeoutseconds > 0 {
354 ctx, cancel = context.WithTimeout(ctx, time.Duration(timeoutseconds)*time.Second)
355 defer cancel()
356 }
357
358 wrapped := exec.CommandContext(ctx, cmd.Path, cmd.Args[1:]...)
359 wrapped.Dir = workdir
360 wrapped.Env = env
361 var combined bytes.Buffer
362 wrapped.Stderr = &combined
363 wrapped.Stdout = &combined
364 if stdin != "" {
365 wrapped.Stdin = strings.NewReader(stdin)
366 }
367
368 err := wrapped.Run()
369 output := combined.Bytes()
370
371 timedout := errors.Is(ctx.Err(), context.DeadlineExceeded)
372 if timedout {
373 return output, 14, true, nil
374 }
375
376 if err == nil {
377 return output, 0, false, nil
378 }
379
380 var exiterr *exec.ExitError
381 if errors.As(err, &exiterr) {
382 if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
383 return output, int(status), false, nil
384 }
385 return output, exiterr.ExitCode() << 8, false, nil
386 }
387
388 return output, 0, false, err
389}
390
391func leftovertempfiles(tmproot string) (bool, string, error) {
392 entries, err := os.ReadDir(tmproot)
393 if err != nil {
394 if errors.Is(err, os.ErrNotExist) {
395 return false, "", nil
396 }
397 return false, "", err
398 }
399 if len(entries) == 0 {
400 return false, "", nil
401 }
402 names := make([]string, 0, len(entries))
403 for _, entry := range entries {
404 names = append(names, filepath.ToSlash(filepath.Join("_tmp", entry.Name())))
405 }
406 sort.Strings(names)
407 return true, "Leftover temporary files: " + strings.Join(names, " ") + "\n", nil
408}
409
410// we expand #PWD# because the gnu make cases use that
411// placeholder from the old perl suite runner for per run work directories
412// also strips clock skew issues, normalizes newlines, and optionally applies regex
413// matching for some cases using that (the gnu suite, mostly in makeflags and some
414// options tests).
415func compareoutput(tc testcase, makepath, workdir string, env []string, actual string) bool {
416 if !tc.Meta.CompareOutput || tc.Meta.OutputMode == "ignore" {
417 return true
418 }
419
420 expected := expandexpectedoutput(tc.ExpectedOut, makepath, workdir, envpath(env))
421 filteredactual := stripskewlines(actual)
422 if filteredactual == expected {
423 return true
424 }
425
426 normalizedexpected := normalizenewlines(expected)
427 normalizedactual := normalizenewlines(filteredactual)
428 if normalizedactual == normalizedexpected {
429 return true
430 }
431
432 if tc.Meta.OutputMode != "regex" {
433 return false
434 }
435
436 patterntext := strings.TrimSuffix(normalizedexpected, "\n")
437 if len(patterntext) < 2 || patterntext[0] != '/' || patterntext[len(patterntext)-1] != '/' {
438 return false
439 }
440 pattern := patterntext[1 : len(patterntext)-1]
441 re, err := regexp.Compile(pattern)
442 if err != nil {
443 return false
444 }
445 return re.MatchString(actual) || re.MatchString(normalizedactual)
446}
447
448func expandexpectedoutput(expected, makepath, workdir, path string) string {
449 expected = strings.ReplaceAll(expected, "#MAKE#", filepath.ToSlash(makepath))
450 expected = strings.ReplaceAll(expected, "#PWD#", filepath.ToSlash(workdir))
451 expected = strings.ReplaceAll(expected, "#PATH#", path)
452 return expected
453}
454
455func envpath(env []string) string {
456 for _, entry := range env {
457 if strings.HasPrefix(entry, "PATH=") {
458 return strings.TrimPrefix(entry, "PATH=")
459 }
460 }
461 return ""
462}
463
464func stripskewlines(s string) string {
465 var kept []string
466 for _, line := range strings.SplitAfter(s, "\n") {
467 lower := strings.ToLower(line)
468 if strings.Contains(lower, "modification time") && strings.Contains(lower, "in the future") {
469 continue
470 }
471 if strings.Contains(lower, "clock skew detected") {
472 continue
473 }
474 kept = append(kept, line)
475 }
476 return strings.Join(kept, "")
477}
478
479func normalizenewlines(s string) string {
480 return strings.ReplaceAll(s, "\r\n", "\n")
481}
482
483func writediff(basepath, logpath, diffpath string) error {
484 diffbin, err := exec.LookPath("diff")
485 if err != nil {
486 message := fmt.Sprintf("Log file %s differs from base file %s\n", logpath, basepath)
487 return os.WriteFile(diffpath, []byte(message), 0o644)
488 }
489
490 cmd := exec.Command(diffbin, "-u", basepath, logpath)
491 output, err := cmd.CombinedOutput()
492 if err != nil {
493 var exiterr *exec.ExitError
494 if !errors.As(err, &exiterr) {
495 return err
496 }
497 }
498 return os.WriteFile(diffpath, output, 0o644)
499}
