From e22e4feae2cd746a422732f81b9f3d02e91978a5 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Wed, 20 Oct 2021 07:07:00 -0500 Subject: [PATCH 1/2] Rephrase instructions for local usage (#327) The rewords some of the instructions in the README for local coverage measurement. The major changes & their motivations are: - indicate the purpose of each step before describing the mechanics of the step. This also makes it easier to indicate that one does not necessarily need to measure summary statistics if one's only goal is to find lines lacking coverage. - prioritize `Pkg.test(; coverage=true)` above manual measurement. Most users will want the simpler option. - clarify the role of the `.info` tracefile. This allows users to determine when they need to supply the more complex set of options. By presenting it as a choice it also makes it clearer that you don't need to execute both lines. - acknowledge limitations in the accuracy of Julia's coverage analysis --- README.md | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index ab4e54a..b532d8f 100644 --- a/README.md +++ b/README.md @@ -22,22 +22,25 @@ Most users will want to use [Coverage.jl](https://github.com/JuliaCI/Coverage.jl ### Code coverage -*Step 1:* Navigate to your test directory, and run julia with the `--code-coverage` option: +*Step 1: collect coverage data.* If you are using your default test suite, you can collect coverage data with `Pkg.test("MyPkg"; coverage=true)`. Alternatively, you can collect coverage data manually: in the terminal, navigate to whatever directory you want to start from as the working directory, and run julia with the `--code-coverage` option: + ```sh julia --code-coverage=user +``` +or more comprehensively (if you're interested in getting coverage for Julia's standard libraries) +```sh julia --code-coverage=tracefile-%p.info --code-coverage=user # available in Julia v1.1+ ``` +You can add other options (e.g., `--project`) as needed. After the REPL starts, execute whatever commands you wish, and then quit Julia. Coverage data are written to files when Julia exits. -*Step 2:* Run your tests (e.g., `include("runtests.jl")`) and quit Julia. (If you use `Pkg.test` to run your tests, set the `coverage` keyword argument to `true`, i.e. `Pkg.test("MyPkg"; coverage=true)`.) - -*Step 3:* Navigate to the top-level directory of your package, restart Julia (with no special flags) and analyze your code coverage: +*Step 2: collect summary statistics (optional).* Navigate to the top-level directory of your package, restart Julia (with no special flags) and analyze your code coverage: ```julia using Coverage # process '*.cov' files coverage = process_folder() # defaults to src/; alternatively, supply the folder name as argument -coverage = append!(coverage, process_folder("deps")) -# process '*.info' files +coverage = append!(coverage, process_folder("deps")) # useful if you want to analyze more than just src/ +# process '*.info' files, if you collected them coverage = merge_coverage_counts(coverage, filter!( let prefixes = (joinpath(pwd(), "src", ""), joinpath(pwd(), "deps", "")) @@ -51,10 +54,15 @@ covered_lines, total_lines = get_summary(coverage) ``` The fraction of total coverage is equal to `covered_lines/total_lines`. -To discover which functions lack testing, browse through the `*.cov` files in your `src/` -directory and look for lines starting with `-` or `0` - those lines were never executed. +*Step 3: identify uncovered lines (optional).* To discover which functions lack testing, browse through the `*.cov` files in your `src/` +directory and look for lines starting with `-` or `0`, which mark lines that were never executed. Numbers larger than 0 are counts of the number of times the respective line was executed. +Note that blank lines, comments, lines with `end` statements, etc. are marked with `-` but do not count against your coverage. + +Be aware of a few limitations: +- a line that can take one of two branches gets marked as covered even if only one branch is tested +- currently, code run by Julia's internal interpreter [is not marked as covered](https://github.com/JuliaLang/julia/issues/37059). ### Memory allocation From 5872cf79280f2791c0fc0bf9d598147f44b63234 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Galy-Fajou?= Date: Tue, 24 May 2022 23:22:13 +0200 Subject: [PATCH 2/2] Add Gitlab CI to Coverage (#331) --- Project.toml | 2 +- src/codecovio.jl | 14 ++++++++ src/coveralls.jl | 12 ++++++- test/runtests.jl | 88 ++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 114 insertions(+), 2 deletions(-) diff --git a/Project.toml b/Project.toml index 8223321..a11430c 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "Coverage" uuid = "a2441757-f6aa-5fb2-8edb-039e3f45d037" authors = ["Iain Dunning ", "contributors"] -version = "1.4.0" +version = "1.5.0" [deps] CoverageTools = "c36e975a-824b-4404-a568-ef97ca766997" diff --git a/src/codecovio.jl b/src/codecovio.jl index c6f93cb..0cf2129 100644 --- a/src/codecovio.jl +++ b/src/codecovio.jl @@ -71,6 +71,7 @@ end add_ci_to_kwargs(; kwargs...) = add_ci_to_kwargs(Dict{Symbol,Any}(kwargs)) function add_ci_to_kwargs(kwargs::Dict) + # https://docs.codecov.com/reference/upload if lowercase(get(ENV, "APPVEYOR", "false")) == "true" appveyor_pr = get(ENV, "APPVEYOR_PULL_REQUEST_NUMBER", "") appveyor_job = join( @@ -175,6 +176,19 @@ function add_ci_to_kwargs(kwargs::Dict) if ENV["BUILDKITE_PULL_REQUEST"] != "false" kwargs = set_defaults(kwargs, pr = ENV["BUILDKITE_PULL_REQUEST"]) end + elseif lowercase(get(ENV, "GITLAB_CI", "false")) == "true" + # Gitlab API: https://docs.gitlab.com/ee/ci/variables/predefined_variables.html + branch = ENV["CI_COMMIT_REF_NAME"] + num_mr = branch == ENV["CI_DEFAULT_BRANCH"] ? "false" : ENV["CI_MERGE_REQUEST_IID"] + kwargs = set_defaults(kwargs, + service = "gitlab", + branch = branch, + commit = ENV["CI_COMMIT_SHA"], + job = ENV["CI_JOB_ID"], + build_url = ENV["CI_PIPELINE_URL"], + build = ENV["CI_PIPELINE_IID"], + pr = num_mr, + ) else error("No compatible CI platform detected") end diff --git a/src/coveralls.jl b/src/coveralls.jl index 77eb00b..19cd14f 100644 --- a/src/coveralls.jl +++ b/src/coveralls.jl @@ -69,7 +69,7 @@ end function prepare_request(fcs::Vector{FileCoverage}, local_env::Bool, git_info=query_git_info) data = Dict{String,Any}("source_files" => map(to_json, fcs)) - + # Coveralls API : https://docs.coveralls.io/api-reference if local_env # Attempt to parse git info via git_info, unless the user explicitly disables it by setting git_info to nothing data["service_name"] = "local" @@ -105,6 +105,16 @@ function prepare_request(fcs::Vector{FileCoverage}, local_env::Bool, git_info=qu github_pr = get(github_pr_info, "number", "") github_pr::Union{AbstractString, Integer} ((github_pr isa Integer) || (!isempty(github_pr))) && (data["service_pull_request"] = strip(string(github_pr))) + elseif lowercase(get(ENV, "GITLAB_CI", "false")) == "true" + # Gitlab API: https://docs.gitlab.com/ee/ci/variables/predefined_variables.html + branch = ENV["CI_COMMIT_REF_NAME"] + num_mr = branch == ENV["CI_DEFAULT_BRANCH"] ? "false" : ENV["CI_MERGE_REQUEST_IID"] + data["service_pull_request"] = num_mr + data["service_number"] = ENV["CI_PIPELINE_IID"] + data["service_job_id"] = ENV["CI_JOB_ID"] + data["service_name"] = "gitlab" + data["git"] = parse_git_info(git_info) + data["git"]["branch"] = branch else data["git"] = parse_git_info(git_info) end diff --git a/test/runtests.jl b/test/runtests.jl index b81a76f..530f1bf 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -482,6 +482,79 @@ withenv( end end + # test Gitlab ci submission process + + # set up Gitlab ci env + withenv( + "GITLAB_CI" => "true", + "CI_MERGE_REQUEST_IID" => "t_pr", + "CI_JOB_ID" => "t_proj", + "CI_COMMIT_REF_NAME" => "t_branch", + "CI_COMMIT_SHA" => "t_commit", + "CI_PROJECT_NAME" => "t_repo", + "CI_PIPELINE_URL" => "t_url", + "CI_PIPELINE_IID" => "t_num", + "CI_DEFAULT_BRANCH" => "master", + ) do + + # default values + codecov_url = construct_uri_string_ci() + @test occursin("codecov.io", codecov_url) + @test occursin("service=gitlab", codecov_url) + @test occursin("branch=t_branch", codecov_url) + @test occursin("commit=t_commit", codecov_url) + @test occursin("pr=t_pr", codecov_url) + @test occursin("build_url=t_url", codecov_url) + @test occursin("build=t_num", codecov_url) + + # env var url override + withenv( "CODECOV_URL" => "https://enterprise-codecov-1.com" ) do + + codecov_url = construct_uri_string_ci() + @test occursin("enterprise-codecov-1.com", codecov_url) + @test occursin("service=gitlab", codecov_url) + @test occursin("branch=t_branch", codecov_url) + @test occursin("commit=t_commit", codecov_url) + @test occursin("pr=t_pr", codecov_url) + @test occursin("build_url=t_url", codecov_url) + @test occursin("build=t_num", codecov_url) + + # function argument url override + codecov_url = construct_uri_string_ci(codecov_url="https://enterprise-codecov-2.com") + @test occursin("enterprise-codecov-2.com", codecov_url) + @test occursin("service=gitlab", codecov_url) + @test occursin("branch=t_branch", codecov_url) + @test occursin("commit=t_commit", codecov_url) + @test occursin("pr=t_pr", codecov_url) + @test occursin("build_url=t_url", codecov_url) + @test occursin("build=t_num", codecov_url) + + # env var token + withenv( "CODECOV_TOKEN" => "token_name_1" ) do + + codecov_url = construct_uri_string_ci() + @test occursin("enterprise-codecov-1.com", codecov_url) + @test occursin("token=token_name_1", codecov_url) + @test occursin("service=gitlab", codecov_url) + @test occursin("branch=t_branch", codecov_url) + @test occursin("commit=t_commit", codecov_url) + @test occursin("pr=t_pr", codecov_url) + @test occursin("build_url=t_url", codecov_url) + @test occursin("build=t_num", codecov_url) + + # function argument token url override + codecov_url = construct_uri_string_ci(token="token_name_2") + @test occursin("enterprise-codecov-1.com", codecov_url) + @test occursin("service=gitlab", codecov_url) + @test occursin("branch=t_branch", codecov_url) + @test occursin("commit=t_commit", codecov_url) + @test occursin("pr=t_pr", codecov_url) + @test occursin("build_url=t_url", codecov_url) + @test occursin("build=t_num", codecov_url) + end + end + end + # test codecov token masking withenv( "APPVEYOR" => "true", @@ -637,6 +710,21 @@ withenv( end end + # test Gitlab see https://docs.coveralls.io/api-reference + withenv("GITLAB_CI" => "true", + "CI_PIPELINE_IID" => "my_job_num", + "CI_JOB_ID" => "my_job_id", + "CI_COMMIT_REF_NAME" => "test", + "CI_DEFAULT_BRANCH" => "master", + "CI_MERGE_REQUEST_IID" => "t_pr") do + request = Coverage.Coveralls.prepare_request(fcs, false) + @test request["repo_token"] == "token_name_1" + @test request["service_number"] == "my_job_num" + @test request["service_job_id"] == "my_job_id" + @test request["service_name"] == "gitlab" + @test request["service_pull_request"] == "t_pr" + end + # test git_info (only works with Jenkins & local at the moment) withenv("JENKINS" => "true", "BUILD_ID" => "my_job_id",