1// Copyright 2020 Google Inc. All rights reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package bazel 16 17import ( 18 "fmt" 19 "reflect" 20 "testing" 21) 22 23func TestAqueryMultiArchGenrule(t *testing.T) { 24 // This input string is retrieved from a real build of bionic-related genrules. 25 const inputString = ` 26{ 27 "artifacts": [{ 28 "id": 1, 29 "pathFragmentId": 1 30 }, { 31 "id": 2, 32 "pathFragmentId": 6 33 }, { 34 "id": 3, 35 "pathFragmentId": 8 36 }, { 37 "id": 4, 38 "pathFragmentId": 12 39 }, { 40 "id": 5, 41 "pathFragmentId": 19 42 }, { 43 "id": 6, 44 "pathFragmentId": 20 45 }, { 46 "id": 7, 47 "pathFragmentId": 21 48 }], 49 "actions": [{ 50 "targetId": 1, 51 "actionKey": "ab53f6ecbdc2ee8cb8812613b63205464f1f5083f6dca87081a0a398c0f1ecf7", 52 "mnemonic": "Genrule", 53 "configurationId": 1, 54 "arguments": ["/bin/bash", "-c", "source ../bazel_tools/tools/genrule/genrule-setup.sh; ../sourceroot/bionic/libc/tools/gensyscalls.py arm ../sourceroot/bionic/libc/SYSCALLS.TXT \u003e bazel-out/sourceroot/k8-fastbuild/bin/bionic/libc/syscalls-arm.S"], 55 "environmentVariables": [{ 56 "key": "PATH", 57 "value": "/bin:/usr/bin:/usr/local/bin" 58 }], 59 "inputDepSetIds": [1], 60 "outputIds": [4], 61 "primaryOutputId": 4 62 }, { 63 "targetId": 2, 64 "actionKey": "9f4309ce165dac458498cb92811c18b0b7919782cc37b82a42d2141b8cc90826", 65 "mnemonic": "Genrule", 66 "configurationId": 1, 67 "arguments": ["/bin/bash", "-c", "source ../bazel_tools/tools/genrule/genrule-setup.sh; ../sourceroot/bionic/libc/tools/gensyscalls.py x86 ../sourceroot/bionic/libc/SYSCALLS.TXT \u003e bazel-out/sourceroot/k8-fastbuild/bin/bionic/libc/syscalls-x86.S"], 68 "environmentVariables": [{ 69 "key": "PATH", 70 "value": "/bin:/usr/bin:/usr/local/bin" 71 }], 72 "inputDepSetIds": [2], 73 "outputIds": [5], 74 "primaryOutputId": 5 75 }, { 76 "targetId": 3, 77 "actionKey": "50d6c586103ebeed3a218195540bcc30d329464eae36377eb82f8ce7c36ac342", 78 "mnemonic": "Genrule", 79 "configurationId": 1, 80 "arguments": ["/bin/bash", "-c", "source ../bazel_tools/tools/genrule/genrule-setup.sh; ../sourceroot/bionic/libc/tools/gensyscalls.py x86_64 ../sourceroot/bionic/libc/SYSCALLS.TXT \u003e bazel-out/sourceroot/k8-fastbuild/bin/bionic/libc/syscalls-x86_64.S"], 81 "environmentVariables": [{ 82 "key": "PATH", 83 "value": "/bin:/usr/bin:/usr/local/bin" 84 }], 85 "inputDepSetIds": [3], 86 "outputIds": [6], 87 "primaryOutputId": 6 88 }, { 89 "targetId": 4, 90 "actionKey": "f30cbe442f5216f4223cf16a39112cad4ec56f31f49290d85cff587e48647ffa", 91 "mnemonic": "Genrule", 92 "configurationId": 1, 93 "arguments": ["/bin/bash", "-c", "source ../bazel_tools/tools/genrule/genrule-setup.sh; ../sourceroot/bionic/libc/tools/gensyscalls.py arm64 ../sourceroot/bionic/libc/SYSCALLS.TXT \u003e bazel-out/sourceroot/k8-fastbuild/bin/bionic/libc/syscalls-arm64.S"], 94 "environmentVariables": [{ 95 "key": "PATH", 96 "value": "/bin:/usr/bin:/usr/local/bin" 97 }], 98 "inputDepSetIds": [4], 99 "outputIds": [7], 100 "primaryOutputId": 7 101 }], 102 "targets": [{ 103 "id": 1, 104 "label": "@sourceroot//bionic/libc:syscalls-arm", 105 "ruleClassId": 1 106 }, { 107 "id": 2, 108 "label": "@sourceroot//bionic/libc:syscalls-x86", 109 "ruleClassId": 1 110 }, { 111 "id": 3, 112 "label": "@sourceroot//bionic/libc:syscalls-x86_64", 113 "ruleClassId": 1 114 }, { 115 "id": 4, 116 "label": "@sourceroot//bionic/libc:syscalls-arm64", 117 "ruleClassId": 1 118 }], 119 "depSetOfFiles": [{ 120 "id": 1, 121 "directArtifactIds": [1, 2, 3] 122 }, { 123 "id": 2, 124 "directArtifactIds": [1, 2, 3] 125 }, { 126 "id": 3, 127 "directArtifactIds": [1, 2, 3] 128 }, { 129 "id": 4, 130 "directArtifactIds": [1, 2, 3] 131 }], 132 "configuration": [{ 133 "id": 1, 134 "mnemonic": "k8-fastbuild", 135 "platformName": "k8", 136 "checksum": "485c362832c178e367d972177f68e69e0981e51e67ef1c160944473db53fe046" 137 }], 138 "ruleClasses": [{ 139 "id": 1, 140 "name": "genrule" 141 }], 142 "pathFragments": [{ 143 "id": 5, 144 "label": ".." 145 }, { 146 "id": 4, 147 "label": "sourceroot", 148 "parentId": 5 149 }, { 150 "id": 3, 151 "label": "bionic", 152 "parentId": 4 153 }, { 154 "id": 2, 155 "label": "libc", 156 "parentId": 3 157 }, { 158 "id": 1, 159 "label": "SYSCALLS.TXT", 160 "parentId": 2 161 }, { 162 "id": 7, 163 "label": "tools", 164 "parentId": 2 165 }, { 166 "id": 6, 167 "label": "gensyscalls.py", 168 "parentId": 7 169 }, { 170 "id": 11, 171 "label": "bazel_tools", 172 "parentId": 5 173 }, { 174 "id": 10, 175 "label": "tools", 176 "parentId": 11 177 }, { 178 "id": 9, 179 "label": "genrule", 180 "parentId": 10 181 }, { 182 "id": 8, 183 "label": "genrule-setup.sh", 184 "parentId": 9 185 }, { 186 "id": 18, 187 "label": "bazel-out" 188 }, { 189 "id": 17, 190 "label": "sourceroot", 191 "parentId": 18 192 }, { 193 "id": 16, 194 "label": "k8-fastbuild", 195 "parentId": 17 196 }, { 197 "id": 15, 198 "label": "bin", 199 "parentId": 16 200 }, { 201 "id": 14, 202 "label": "bionic", 203 "parentId": 15 204 }, { 205 "id": 13, 206 "label": "libc", 207 "parentId": 14 208 }, { 209 "id": 12, 210 "label": "syscalls-arm.S", 211 "parentId": 13 212 }, { 213 "id": 19, 214 "label": "syscalls-x86.S", 215 "parentId": 13 216 }, { 217 "id": 20, 218 "label": "syscalls-x86_64.S", 219 "parentId": 13 220 }, { 221 "id": 21, 222 "label": "syscalls-arm64.S", 223 "parentId": 13 224 }] 225}` 226 actualbuildStatements, _ := AqueryBuildStatements([]byte(inputString)) 227 expectedBuildStatements := []BuildStatement{} 228 for _, arch := range []string{"arm", "arm64", "x86", "x86_64"} { 229 expectedBuildStatements = append(expectedBuildStatements, 230 BuildStatement{ 231 Command: fmt.Sprintf( 232 "/bin/bash -c 'source ../bazel_tools/tools/genrule/genrule-setup.sh; ../sourceroot/bionic/libc/tools/gensyscalls.py %s ../sourceroot/bionic/libc/SYSCALLS.TXT > bazel-out/sourceroot/k8-fastbuild/bin/bionic/libc/syscalls-%s.S'", 233 arch, arch), 234 OutputPaths: []string{ 235 fmt.Sprintf("bazel-out/sourceroot/k8-fastbuild/bin/bionic/libc/syscalls-%s.S", arch), 236 }, 237 InputPaths: []string{ 238 "../sourceroot/bionic/libc/SYSCALLS.TXT", 239 "../sourceroot/bionic/libc/tools/gensyscalls.py", 240 "../bazel_tools/tools/genrule/genrule-setup.sh", 241 }, 242 Env: []KeyValuePair{ 243 KeyValuePair{Key: "PATH", Value: "/bin:/usr/bin:/usr/local/bin"}, 244 }, 245 Mnemonic: "Genrule", 246 }) 247 } 248 assertBuildStatements(t, expectedBuildStatements, actualbuildStatements) 249} 250 251func TestInvalidOutputId(t *testing.T) { 252 const inputString = ` 253{ 254 "artifacts": [{ 255 "id": 1, 256 "pathFragmentId": 1 257 }, { 258 "id": 2, 259 "pathFragmentId": 2 260 }], 261 "actions": [{ 262 "targetId": 1, 263 "actionKey": "x", 264 "mnemonic": "x", 265 "arguments": ["touch", "foo"], 266 "inputDepSetIds": [1], 267 "outputIds": [3], 268 "primaryOutputId": 3 269 }], 270 "depSetOfFiles": [{ 271 "id": 1, 272 "directArtifactIds": [1, 2] 273 }], 274 "pathFragments": [{ 275 "id": 1, 276 "label": "one" 277 }, { 278 "id": 2, 279 "label": "two" 280 }] 281}` 282 283 _, err := AqueryBuildStatements([]byte(inputString)) 284 assertError(t, err, "undefined outputId 3") 285} 286 287func TestInvalidInputDepsetId(t *testing.T) { 288 const inputString = ` 289{ 290 "artifacts": [{ 291 "id": 1, 292 "pathFragmentId": 1 293 }, { 294 "id": 2, 295 "pathFragmentId": 2 296 }], 297 "actions": [{ 298 "targetId": 1, 299 "actionKey": "x", 300 "mnemonic": "x", 301 "arguments": ["touch", "foo"], 302 "inputDepSetIds": [2], 303 "outputIds": [1], 304 "primaryOutputId": 1 305 }], 306 "depSetOfFiles": [{ 307 "id": 1, 308 "directArtifactIds": [1, 2] 309 }], 310 "pathFragments": [{ 311 "id": 1, 312 "label": "one" 313 }, { 314 "id": 2, 315 "label": "two" 316 }] 317}` 318 319 _, err := AqueryBuildStatements([]byte(inputString)) 320 assertError(t, err, "undefined input depsetId 2") 321} 322 323func TestInvalidInputArtifactId(t *testing.T) { 324 const inputString = ` 325{ 326 "artifacts": [{ 327 "id": 1, 328 "pathFragmentId": 1 329 }, { 330 "id": 2, 331 "pathFragmentId": 2 332 }], 333 "actions": [{ 334 "targetId": 1, 335 "actionKey": "x", 336 "mnemonic": "x", 337 "arguments": ["touch", "foo"], 338 "inputDepSetIds": [1], 339 "outputIds": [1], 340 "primaryOutputId": 1 341 }], 342 "depSetOfFiles": [{ 343 "id": 1, 344 "directArtifactIds": [1, 3] 345 }], 346 "pathFragments": [{ 347 "id": 1, 348 "label": "one" 349 }, { 350 "id": 2, 351 "label": "two" 352 }] 353}` 354 355 _, err := AqueryBuildStatements([]byte(inputString)) 356 assertError(t, err, "undefined input artifactId 3") 357} 358 359func TestInvalidPathFragmentId(t *testing.T) { 360 const inputString = ` 361{ 362 "artifacts": [{ 363 "id": 1, 364 "pathFragmentId": 1 365 }, { 366 "id": 2, 367 "pathFragmentId": 2 368 }], 369 "actions": [{ 370 "targetId": 1, 371 "actionKey": "x", 372 "mnemonic": "x", 373 "arguments": ["touch", "foo"], 374 "inputDepSetIds": [1], 375 "outputIds": [1], 376 "primaryOutputId": 1 377 }], 378 "depSetOfFiles": [{ 379 "id": 1, 380 "directArtifactIds": [1, 2] 381 }], 382 "pathFragments": [{ 383 "id": 1, 384 "label": "one" 385 }, { 386 "id": 2, 387 "label": "two", 388 "parentId": 3 389 }] 390}` 391 392 _, err := AqueryBuildStatements([]byte(inputString)) 393 assertError(t, err, "undefined path fragment id 3") 394} 395 396func TestDepfiles(t *testing.T) { 397 const inputString = ` 398{ 399 "artifacts": [{ 400 "id": 1, 401 "pathFragmentId": 1 402 }, { 403 "id": 2, 404 "pathFragmentId": 2 405 }, { 406 "id": 3, 407 "pathFragmentId": 3 408 }], 409 "actions": [{ 410 "targetId": 1, 411 "actionKey": "x", 412 "mnemonic": "x", 413 "arguments": ["touch", "foo"], 414 "inputDepSetIds": [1], 415 "outputIds": [2, 3], 416 "primaryOutputId": 2 417 }], 418 "depSetOfFiles": [{ 419 "id": 1, 420 "directArtifactIds": [1, 2, 3] 421 }], 422 "pathFragments": [{ 423 "id": 1, 424 "label": "one" 425 }, { 426 "id": 2, 427 "label": "two" 428 }, { 429 "id": 3, 430 "label": "two.d" 431 }] 432}` 433 434 actual, err := AqueryBuildStatements([]byte(inputString)) 435 if err != nil { 436 t.Errorf("Unexpected error %q", err) 437 } 438 if expected := 1; len(actual) != expected { 439 t.Fatalf("Expected %d build statements, got %d", expected, len(actual)) 440 } 441 442 bs := actual[0] 443 expectedDepfile := "two.d" 444 if bs.Depfile == nil { 445 t.Errorf("Expected depfile %q, but there was none found", expectedDepfile) 446 } else if *bs.Depfile != expectedDepfile { 447 t.Errorf("Expected depfile %q, but got %q", expectedDepfile, *bs.Depfile) 448 } 449} 450 451func TestMultipleDepfiles(t *testing.T) { 452 const inputString = ` 453{ 454 "artifacts": [{ 455 "id": 1, 456 "pathFragmentId": 1 457 }, { 458 "id": 2, 459 "pathFragmentId": 2 460 }, { 461 "id": 3, 462 "pathFragmentId": 3 463 }, { 464 "id": 4, 465 "pathFragmentId": 4 466 }], 467 "actions": [{ 468 "targetId": 1, 469 "actionKey": "x", 470 "mnemonic": "x", 471 "arguments": ["touch", "foo"], 472 "inputDepSetIds": [1], 473 "outputIds": [2,3,4], 474 "primaryOutputId": 2 475 }], 476 "depSetOfFiles": [{ 477 "id": 1, 478 "directArtifactIds": [1, 2, 3, 4] 479 }], 480 "pathFragments": [{ 481 "id": 1, 482 "label": "one" 483 }, { 484 "id": 2, 485 "label": "two" 486 }, { 487 "id": 3, 488 "label": "two.d" 489 }, { 490 "id": 4, 491 "label": "other.d" 492 }] 493}` 494 495 _, err := AqueryBuildStatements([]byte(inputString)) 496 assertError(t, err, `found multiple potential depfiles "two.d", "other.d"`) 497} 498 499func TestTransitiveInputDepsets(t *testing.T) { 500 // The input aquery for this test comes from a proof-of-concept starlark rule which registers 501 // a single action with many inputs given via a deep depset. 502 const inputString = ` 503{ 504 "artifacts": [{ 505 "id": 1, 506 "pathFragmentId": 1 507 }, { 508 "id": 2, 509 "pathFragmentId": 7 510 }, { 511 "id": 3, 512 "pathFragmentId": 8 513 }, { 514 "id": 4, 515 "pathFragmentId": 9 516 }, { 517 "id": 5, 518 "pathFragmentId": 10 519 }, { 520 "id": 6, 521 "pathFragmentId": 11 522 }, { 523 "id": 7, 524 "pathFragmentId": 12 525 }, { 526 "id": 8, 527 "pathFragmentId": 13 528 }, { 529 "id": 9, 530 "pathFragmentId": 14 531 }, { 532 "id": 10, 533 "pathFragmentId": 15 534 }, { 535 "id": 11, 536 "pathFragmentId": 16 537 }, { 538 "id": 12, 539 "pathFragmentId": 17 540 }, { 541 "id": 13, 542 "pathFragmentId": 18 543 }, { 544 "id": 14, 545 "pathFragmentId": 19 546 }, { 547 "id": 15, 548 "pathFragmentId": 20 549 }, { 550 "id": 16, 551 "pathFragmentId": 21 552 }, { 553 "id": 17, 554 "pathFragmentId": 22 555 }, { 556 "id": 18, 557 "pathFragmentId": 23 558 }, { 559 "id": 19, 560 "pathFragmentId": 24 561 }, { 562 "id": 20, 563 "pathFragmentId": 25 564 }, { 565 "id": 21, 566 "pathFragmentId": 26 567 }], 568 "actions": [{ 569 "targetId": 1, 570 "actionKey": "3b826d17fadbbbcd8313e456b90ec47c078c438088891dd45b4adbcd8889dc50", 571 "mnemonic": "Action", 572 "configurationId": 1, 573 "arguments": ["/bin/bash", "-c", "touch bazel-out/sourceroot/k8-fastbuild/bin/testpkg/test_out"], 574 "inputDepSetIds": [1], 575 "outputIds": [21], 576 "primaryOutputId": 21 577 }], 578 "depSetOfFiles": [{ 579 "id": 3, 580 "directArtifactIds": [1, 2, 3, 4, 5] 581 }, { 582 "id": 4, 583 "directArtifactIds": [6, 7, 8, 9, 10] 584 }, { 585 "id": 2, 586 "transitiveDepSetIds": [3, 4], 587 "directArtifactIds": [11, 12, 13, 14, 15] 588 }, { 589 "id": 5, 590 "directArtifactIds": [16, 17, 18, 19] 591 }, { 592 "id": 1, 593 "transitiveDepSetIds": [2, 5], 594 "directArtifactIds": [20] 595 }], 596 "pathFragments": [{ 597 "id": 6, 598 "label": "bazel-out" 599 }, { 600 "id": 5, 601 "label": "sourceroot", 602 "parentId": 6 603 }, { 604 "id": 4, 605 "label": "k8-fastbuild", 606 "parentId": 5 607 }, { 608 "id": 3, 609 "label": "bin", 610 "parentId": 4 611 }, { 612 "id": 2, 613 "label": "testpkg", 614 "parentId": 3 615 }, { 616 "id": 1, 617 "label": "test_1", 618 "parentId": 2 619 }, { 620 "id": 7, 621 "label": "test_2", 622 "parentId": 2 623 }, { 624 "id": 8, 625 "label": "test_3", 626 "parentId": 2 627 }, { 628 "id": 9, 629 "label": "test_4", 630 "parentId": 2 631 }, { 632 "id": 10, 633 "label": "test_5", 634 "parentId": 2 635 }, { 636 "id": 11, 637 "label": "test_6", 638 "parentId": 2 639 }, { 640 "id": 12, 641 "label": "test_7", 642 "parentId": 2 643 }, { 644 "id": 13, 645 "label": "test_8", 646 "parentId": 2 647 }, { 648 "id": 14, 649 "label": "test_9", 650 "parentId": 2 651 }, { 652 "id": 15, 653 "label": "test_10", 654 "parentId": 2 655 }, { 656 "id": 16, 657 "label": "test_11", 658 "parentId": 2 659 }, { 660 "id": 17, 661 "label": "test_12", 662 "parentId": 2 663 }, { 664 "id": 18, 665 "label": "test_13", 666 "parentId": 2 667 }, { 668 "id": 19, 669 "label": "test_14", 670 "parentId": 2 671 }, { 672 "id": 20, 673 "label": "test_15", 674 "parentId": 2 675 }, { 676 "id": 21, 677 "label": "test_16", 678 "parentId": 2 679 }, { 680 "id": 22, 681 "label": "test_17", 682 "parentId": 2 683 }, { 684 "id": 23, 685 "label": "test_18", 686 "parentId": 2 687 }, { 688 "id": 24, 689 "label": "test_19", 690 "parentId": 2 691 }, { 692 "id": 25, 693 "label": "test_root", 694 "parentId": 2 695 }, { 696 "id": 26, 697 "label": "test_out", 698 "parentId": 2 699 }] 700}` 701 702 actualbuildStatements, _ := AqueryBuildStatements([]byte(inputString)) 703 // Inputs for the action are test_{i} from 1 to 20, and test_root. These inputs 704 // are given via a deep depset, but the depset is flattened when returned as a 705 // BuildStatement slice. 706 inputPaths := []string{"bazel-out/sourceroot/k8-fastbuild/bin/testpkg/test_root"} 707 for i := 1; i < 20; i++ { 708 inputPaths = append(inputPaths, fmt.Sprintf("bazel-out/sourceroot/k8-fastbuild/bin/testpkg/test_%d", i)) 709 } 710 expectedBuildStatements := []BuildStatement{ 711 BuildStatement{ 712 Command: "/bin/bash -c touch bazel-out/sourceroot/k8-fastbuild/bin/testpkg/test_out", 713 OutputPaths: []string{"bazel-out/sourceroot/k8-fastbuild/bin/testpkg/test_out"}, 714 InputPaths: inputPaths, 715 Mnemonic: "Action", 716 }, 717 } 718 assertBuildStatements(t, expectedBuildStatements, actualbuildStatements) 719} 720 721func assertError(t *testing.T, err error, expected string) { 722 if err == nil { 723 t.Errorf("expected error '%s', but got no error", expected) 724 } else if err.Error() != expected { 725 t.Errorf("expected error '%s', but got: %s", expected, err.Error()) 726 } 727} 728 729// Asserts that the given actual build statements match the given expected build statements. 730// Build statement equivalence is determined using buildStatementEquals. 731func assertBuildStatements(t *testing.T, expected []BuildStatement, actual []BuildStatement) { 732 if len(expected) != len(actual) { 733 t.Errorf("expected %d build statements, but got %d,\n expected: %v,\n actual: %v", 734 len(expected), len(actual), expected, actual) 735 return 736 } 737ACTUAL_LOOP: 738 for _, actualStatement := range actual { 739 for _, expectedStatement := range expected { 740 if buildStatementEquals(actualStatement, expectedStatement) { 741 continue ACTUAL_LOOP 742 } 743 } 744 t.Errorf("unexpected build statement %v.\n expected: %v", 745 actualStatement, expected) 746 return 747 } 748} 749 750func buildStatementEquals(first BuildStatement, second BuildStatement) bool { 751 if first.Mnemonic != second.Mnemonic { 752 return false 753 } 754 if first.Command != second.Command { 755 return false 756 } 757 // Ordering is significant for environment variables. 758 if !reflect.DeepEqual(first.Env, second.Env) { 759 return false 760 } 761 // Ordering is irrelevant for input and output paths, so compare sets. 762 if !reflect.DeepEqual(stringSet(first.InputPaths), stringSet(second.InputPaths)) { 763 return false 764 } 765 if !reflect.DeepEqual(stringSet(first.OutputPaths), stringSet(second.OutputPaths)) { 766 return false 767 } 768 return true 769} 770 771func stringSet(stringSlice []string) map[string]struct{} { 772 stringMap := make(map[string]struct{}) 773 for _, s := range stringSlice { 774 stringMap[s] = struct{}{} 775 } 776 return stringMap 777} 778