diff --git a/.formatter.exs b/.formatter.exs new file mode 100644 index 00000000000..a7df4e3d63d --- /dev/null +++ b/.formatter.exs @@ -0,0 +1,25 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +[ + inputs: [ + "lib/*/{lib,scripts,unicode,test}/**/*.{ex,exs}", + "lib/*/*.exs", + "lib/ex_unit/examples/*.exs", + ".formatter.exs" + ], + locals_without_parens: [ + # Formatter tests + assert_format: 2, + assert_format: 3, + assert_same: 1, + assert_same: 2, + + # Errors tests + assert_eval_raise: 3, + + # Float tests + float_assert: 1 + ] +] diff --git a/.gitattributes b/.gitattributes index 1a79d0b894c..ebff1ea6a4e 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,7 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + lib/elixir/test/elixir/fixtures/*.txt text eol=lf +*.ex diff=elixir +*.exs diff=elixir diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 00000000000..71572d44a19 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team + +--- +blank_issues_enabled: true + +contact_links: + - name: Ask questions, support, and general discussions + url: https://elixirforum.com/ + about: Ask questions, provide support, and more on Elixir Forum + + - name: Propose new features + url: https://github.com/elixir-lang/elixir/#proposing-new-features + about: Propose new features in our mailing list diff --git a/.github/ISSUE_TEMPLATE/issue.yml b/.github/ISSUE_TEMPLATE/issue.yml new file mode 100644 index 00000000000..cd1ba44e31e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/issue.yml @@ -0,0 +1,56 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team + +--- +name: Report an issue +description: + Tell us about something that is not working the way we (probably) intend +body: + - type: markdown + attributes: + value: > + Thank you for contributing to Elixir! :heart: + + + Please, do not use this form for guidance, questions or support. + Try instead in [Elixir Forum](https://elixirforum.com), + the [IRC Chat](https://web.libera.chat/#elixir), + [Stack Overflow](https://stackoverflow.com/questions/tagged/elixir), + [Slack](https://elixir-slackin.herokuapp.com), + [Discord](https://discord.gg/elixir) or in other online communities. + + - type: textarea + id: elixir-and-otp-version + attributes: + label: Elixir and Erlang/OTP versions + description: Paste the output of `elixir --version` here. + validations: + required: true + + - type: input + id: os + attributes: + label: Operating system + description: The operating system that this issue is happening on. + validations: + required: true + + - type: textarea + id: current-behavior + attributes: + label: Current behavior + description: > + Include code samples, errors, and stacktraces if appropriate. + + + If reporting a bug, please include the reproducing steps. + validations: + required: true + + - type: textarea + id: expected-behavior + attributes: + label: Expected behavior + description: A short description on how you expect the code to behave. + validations: + required: true diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000000..ea242ad8eef --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team + +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000000..b99d2ee667c --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,178 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +name: CI + +on: + push: + paths-ignore: + - "lib/**/*.md" + + pull_request: + paths-ignore: + - "lib/**/*.md" + +env: + ELIXIR_ASSERT_TIMEOUT: 2000 + ELIXIRC_OPTS: "--warnings-as-errors" + LANG: C.UTF-8 + +permissions: + contents: read + +jobs: + test_linux: + name: Ubuntu 24.04, OTP ${{ matrix.otp_version }}${{ matrix.deterministic && ' (deterministic)' || '' }}${{ matrix.coverage && ' (coverage)' || '' }} + runs-on: ubuntu-24.04 + + strategy: + fail-fast: false + matrix: + include: + - otp_version: "28.1" + deterministic: true + - otp_version: "28.1" + erlc_opts: "warnings_as_errors" + docs: true + coverage: true + - otp_version: "27.3" + erlc_opts: "warnings_as_errors" + - otp_version: "27.0" + erlc_opts: "warnings_as_errors" + - otp_version: "26.0" + - otp_version: master + development: true + - otp_version: maint + development: true + + # Earlier Erlang/OTP versions ignored compiler directives + # when using warnings_as_errors. So we only set ERLC_OPTS + # from Erlang/OTP 27+. + env: + ERLC_OPTS: ${{ matrix.erlc_opts || '' }} + steps: + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + + - uses: erlef/setup-beam@e6d7c94229049569db56a7ad5a540c051a010af9 # v1.20.4 + with: + otp-version: ${{ matrix.otp_version }} + + - name: Set ERL_COMPILER_OPTIONS + if: ${{ matrix.deterministic }} + run: echo "ERL_COMPILER_OPTIONS=deterministic" >> $GITHUB_ENV + + - name: Compile Elixir + run: | + make compile + echo "$PWD/bin" >> $GITHUB_PATH + + - name: Build info + run: bin/elixir --version + + - name: Check format + run: make test_formatted && echo "All Elixir source code files are properly formatted." + + - name: Erlang test suite + run: make test_erlang + continue-on-error: ${{ matrix.development == true }} + + - name: Elixir test suite + run: make test_elixir + continue-on-error: ${{ matrix.development == true }} + env: + COVER: "${{ matrix.coverage }}" + + - name: Build docs (ExDoc main) + if: ${{ matrix.docs }} + run: | + cd .. + git clone https://github.com/elixir-lang/ex_doc.git --depth 1 + cd ex_doc + ../elixir/bin/mix do local.rebar --force + local.hex --force + deps.get + compile + cd ../elixir/ + git fetch --tags + DOCS_OPTIONS="--warnings-as-errors" make docs + + - name: "Calculate Coverage" + if: ${{ matrix.coverage }} + run: make cover | tee "$GITHUB_STEP_SUMMARY" + + - name: "Upload Coverage Artifact" + if: ${{ matrix.coverage }} + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + with: + name: TestCoverage + path: cover/* + + - name: Check reproducible builds + if: ${{ matrix.deterministic }} + run: | + rm -rf .git + # Recompile System without .git + cd lib/elixir && ../../bin/elixirc -o ebin lib/system.ex && cd - + taskset 1 make check_reproducible + + test_windows: + name: Windows Server 2022, OTP ${{ matrix.otp_version }} + runs-on: windows-2022 + + strategy: + matrix: + otp_version: + - "28.1" + - "27.3" + - "26.2" + + steps: + - name: Configure Git + run: git config --global core.autocrlf input + + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + + - uses: erlef/setup-beam@e6d7c94229049569db56a7ad5a540c051a010af9 # v1.20.4 + with: + otp-version: ${{ matrix.otp_version }} + + - name: Compile Elixir + run: | + Remove-Item -Recurse -Force '.git' + make compile + + - name: Build info + run: bin/elixir --version + + - name: Check format + run: make test_formatted && echo "All Elixir source code files are properly formatted." + + - name: Erlang test suite + run: make test_erlang + + - name: Elixir test suite + run: | + Remove-Item 'c:/Windows/System32/drivers/etc/hosts' + make test_elixir + + license_compliance: + name: Check Licence Compliance + + runs-on: ubuntu-24.04 + + steps: + - name: Use HTTPS instead of SSH for Git cloning + id: git-config + shell: bash + run: git config --global url.https://github.com/.insteadOf ssh://git@github.com/ + + - name: Checkout project + id: checkout + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + + - name: "Run OSS Review Toolkit" + id: ort + uses: ./.github/workflows/ort + with: + upload-reports: true + fail-on-violation: true + report-formats: "WebApp" + version: "${{ github.sha }}" diff --git a/.github/workflows/markdown.yml b/.github/workflows/markdown.yml new file mode 100644 index 00000000000..23a2dc9ab65 --- /dev/null +++ b/.github/workflows/markdown.yml @@ -0,0 +1,39 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team + +name: Markdown Content + +on: + push: + branches: + - "main" + + paths: &paths-filter + - "**/*.md" + - .github/workflows/markdown.yml + - .markdownlint-cli2.jsonc + + pull_request: + paths: *paths-filter + + workflow_dispatch: + +permissions: + contents: read + +env: + LANG: C.UTF-8 + +jobs: + lint: + name: Lint Markdown content + runs-on: ubuntu-latest + + strategy: + fail-fast: false + + steps: + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + + - name: Run markdownlint-cli2 + uses: DavidAnson/markdownlint-cli2-action@30a0e04f1870d58f8d717450cc6134995f993c63 # v21.0.0 diff --git a/.github/workflows/notify.exs b/.github/workflows/notify.exs new file mode 100644 index 00000000000..77251132a3d --- /dev/null +++ b/.github/workflows/notify.exs @@ -0,0 +1,79 @@ +# #!/usr/bin/env elixir + +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team + +[tag] = System.argv() + +Mix.install([ + {:req, "~> 0.2.1"}, + {:jason, "~> 1.0"} +]) + +%{status: 200, body: release} = + Req.get!("https://api.github.com/repos/elixir-lang/elixir/releases/tags/#{tag}") + +if release["draft"] do + raise "cannot notify a draft release" +end + +## Notify on elixir-lang-ann + +names_and_checksums = + for asset <- release["assets"], + name = asset["name"], + name =~ ~r/.sha\d+sum$/, + do: {name, Req.get!(asset["browser_download_url"]).body} + +line_items = + for {name, checksum_and_name} <- Enum.sort(names_and_checksums) do + [checksum | _] = String.split(checksum_and_name, " ") + root = Path.rootname(name) + "." <> type = Path.extname(name) + " * #{root} - #{type} - #{checksum}\n" + end + +body = "https://github.com/elixir-lang/elixir/releases/tag/#{tag}\n\n#{line_items}" + +IO.puts([ + "========================================\n", + body, + "\n========================================" +]) + +mail = %{ + # The email must have access to post + "From" => "jose.valim@dashbit.co", + "To" => "elixir-lang-ann@googlegroups.com", + "Subject" => "Elixir #{tag} released", + "HtmlBody" => body, + "MessageStream" => "outbound" +} + +unless System.get_env("DRYRUN") do + headers = %{ + "X-Postmark-Server-Token" => System.fetch_env!("ELIXIR_LANG_ANN_TOKEN") + } + + resp = Req.post!("https://api.postmarkapp.com/email", {:json, mail}, headers: headers) + IO.puts("#{resp.status} elixir-lang-ann\n#{inspect(resp.body)}") +end + +## Notify on Elixir Forum + +post = %{ + "title" => "Elixir #{tag} released", + "raw" => "https://github.com/elixir-lang/elixir/releases/tag/#{tag}\n\n#{release["body"]}", + # Elixir News + "category" => 28 +} + +unless System.get_env("DRYRUN") do + headers = %{ + "api-key" => System.fetch_env!("ELIXIR_FORUM_TOKEN"), + "api-username" => "Elixir" + } + + resp = Req.post!("https://elixirforum.com/posts.json", {:json, post}, headers: headers) + IO.puts("#{resp.status} Elixir Forum\n#{inspect(resp.body)}") +end diff --git a/.github/workflows/ort/action.yml b/.github/workflows/ort/action.yml new file mode 100644 index 00000000000..99cf2e7063e --- /dev/null +++ b/.github/workflows/ort/action.yml @@ -0,0 +1,109 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team + +name: "Run OSS Review Toolkit" +description: "Runs OSS Review Toolkit & generates SBoMs" +inputs: + report-formats: + description: "ORT Report Formats" + required: true + fail-on-violation: + description: "Whether to fail on violation." + required: false + default: false + upload-reports: + description: "Whether to upload all reports" + required: false + default: false + version: + description: "Elixir Version (Tag / SHA)" + required: true + +outputs: + results-path: + description: "See oss-review-toolkit/ort-ci-github-action action" + value: "${{ steps.ort.outputs.results-path }}" + results-sbom-cyclonedx-xml-path: + description: "See oss-review-toolkit/ort-ci-github-action action" + value: "${{ steps.ort.outputs.results-sbom-cyclonedx-xml-path }}" + results-sbom-cyclonedx-json-path: + description: "See oss-review-toolkit/ort-ci-github-action action" + value: "${{ steps.ort.outputs.results-sbom-cyclonedx-json-path }}" + results-sbom-spdx-yml-path: + description: "See oss-review-toolkit/ort-ci-github-action action" + value: "${{ steps.ort.outputs.results-sbom-spdx-yml-path }}" + results-sbom-spdx-json-path: + description: "See oss-review-toolkit/ort-ci-github-action action" + value: "${{ steps.ort.outputs.results-sbom-spdx-json-path }}" + +runs: + using: "composite" + steps: + - name: Fetch Default ORT Config + id: fetch-default-ort-config + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: oss-review-toolkit/ort-config + ref: "main" + path: ".ort-config" + + - name: Setup ORT Config + id: setup-ort-config + shell: bash + run: | + mkdir -p "/$HOME/.ort/" + + # Move Fetched Default Config into Place + mv .ort-config "$HOME/.ort/config" + + # Append Global ORT Config + cat .ort/config/config.yml >> "$HOME/.ort/config/config.yml" + + # Override Default Evaluator Rules + cp .ort/config/evaluator.rules.kts "$HOME/.ort/config/evaluator.rules.kts" + + # Add Package Configurations + mkdir -p "$HOME/.ort/config/package-configurations/SpdxDocumentFile/The Elixir Team" + for FILE in .ort/package-configurations/*.yml; do + COMPONENT="$(basename "$FILE")" + cp "$FILE" "$HOME/.ort/config/package-configurations/SpdxDocumentFile/The Elixir Team/$COMPONENT" + sed -i -E \ + "s/(\"SpdxDocumentFile:The Elixir Team:.+:)\"/\1${ELIXIR_VERSION}\"/" \ + "$HOME/.ort/config/package-configurations/SpdxDocumentFile/The Elixir Team/$COMPONENT" + done + + # Set Version in SPDX & Config + sed -i "s/# elixir-version-insert/versionInfo: '${ELIXIR_VERSION}'/" project.spdx.yml + sed -i -E "s/(\"SpdxDocumentFile:The Elixir Team:.+:)\"/\1${ELIXIR_VERSION}\"/" .ort.yml + sed -i "s|https://github.com/elixir-lang/elixir.git|${ELIXIR_REPO}@${ELIXIR_VERSION}|" project.spdx.yml + env: + ELIXIR_VERSION: "${{ inputs.version }}" + ELIXIR_REPO: "${{ github.server_url }}/${{ github.repository }}.git" + + - name: "Cache ScanCode" + uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 + with: + path: "~/.cache/scancode-tk" + key: ${{ runner.os }}-scancode + + - name: Run OSS Review Toolkit + id: ort + uses: oss-review-toolkit/ort-ci-github-action@1805edcf1f4f55f35ae6e4d2d9795ccfb29b6021 # v1.1.0 + with: + image: ghcr.io/oss-review-toolkit/ort-minimal:65.0.0 + run: >- + labels, + cache-dependencies, + cache-scan-results, + analyzer, + scanner, + advisor, + evaluator, + reporter, + ${{ inputs.upload-reports == 'true' && 'upload-results' || '' }} + fail-on: "${{ inputs.fail-on-violation == 'true' && 'violations,issues' || '' }}" + report-formats: "${{ inputs.report-formats }}" + ort-cli-report-args: >- + -O CycloneDX=output.file.formats=json,xml + -O SpdxDocument=outputFileFormats=JSON,YAML + sw-version: "${{ inputs.version }}" diff --git a/.github/workflows/posix_compliance.yml b/.github/workflows/posix_compliance.yml new file mode 100644 index 00000000000..16d10ee94d1 --- /dev/null +++ b/.github/workflows/posix_compliance.yml @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +name: POSIX Compliance + +on: + push: + paths: &paths-filter + - .github/workflows/posix_compliance.yml + - bin/elixir + - bin/elixirc + - bin/iex + + pull_request: + paths: *paths-filter + + workflow_dispatch: + +permissions: + contents: read + +env: + LANG: C.UTF-8 + +jobs: + check_posix_compliance: + name: Check POSIX compliance + runs-on: ubuntu-latest + + strategy: + fail-fast: false + + steps: + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + + - name: Install ShellCheck + run: | + sudo apt update + sudo apt install -y shellcheck + + - name: Run ShellCheck on bin/ dir + run: | + shellcheck -e SC2039,2086 bin/elixir && \ + echo "bin/elixir is POSIX compliant" + + shellcheck bin/elixirc && \ + echo "bin/elixirc is POSIX compliant" + + shellcheck bin/iex && \ + echo "bin/iex is POSIX compliant" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000000..59c1aa80351 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,439 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team + +name: Releases + +on: + push: + branches: + - main + - v*.* + + tags: + - v* + + workflow_dispatch: + +env: + ELIXIR_OPTS: "--warnings-as-errors" + LANG: C.UTF-8 + +permissions: + contents: read + +jobs: + create_draft_release: + name: Create draft release + runs-on: ubuntu-24.04 + + permissions: + contents: write + + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + steps: + - name: Create draft release + if: github.ref_type != 'branch' + run: | + gh release create \ + --repo ${{ github.repository }} \ + --title ${{ github.ref_name }} \ + --notes '' \ + --draft \ + ${{ github.ref_name }} + + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + if: github.ref_type == 'branch' + + - name: Update ${{ github.ref_name }}-latest + if: github.ref_type == 'branch' + run: | + ref_name=${{ github.ref_name }}-latest + + if ! gh release view $ref_name; then + gh release create \ + --latest=false \ + --title $ref_name \ + --notes "Automated release for latest ${{ github.ref_name }}." \ + $ref_name + fi + + git tag $ref_name --force + git push origin $ref_name --force + + build: + name: Ubuntu 24.04, OTP ${{ matrix.otp_version }}${{ matrix.build_docs && ' (build docs)' || '' }} + runs-on: ubuntu-24.04 + + strategy: + fail-fast: true + matrix: + include: + - otp: 26 + otp_version: "26.0" + + - otp: 27 + otp_version: "27.0" + + - otp: 28 + otp_version: "28.0" + build_docs: build_docs + + steps: + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + + - name: "Build Release" + uses: ./.github/workflows/release_pre_built + with: + otp_version: ${{ matrix.otp_version }} + otp: ${{ matrix.otp }} + build_docs: ${{ matrix.build_docs }} + + - name: Create Docs Hashes + if: matrix.build_docs + run: | + shasum -a 1 Docs.zip > Docs.zip.sha1sum + shasum -a 256 Docs.zip > Docs.zip.sha256sum + + - name: "Upload Linux release artifacts" + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + with: + name: build-linux-elixir-otp-${{ matrix.otp }} + path: elixir-otp-${{ matrix.otp }}.zip + + - name: "Upload Windows release artifacts" + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + with: + name: build-windows-elixir-otp-${{ matrix.otp }} + path: elixir-otp-${{ matrix.otp }}.exe + + - name: "Upload doc artifacts" + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + if: matrix.build_docs + with: + name: Docs + path: Docs.zip* + + sign: + name: Sign files, ${{ matrix.flavor == 'windows' && 'Windows' || matrix.flavor == 'linux' && 'Linux' || matrix.flavor }}, OTP ${{ matrix.otp }} + needs: [build] + environment: release + strategy: + fail-fast: true + matrix: + otp: [26, 27, 28] + flavor: [windows, linux] + + env: + RELEASE_FILE: elixir-otp-${{ matrix.otp }}.${{ matrix.flavor == 'linux' && 'zip' || 'exe' }} + + runs-on: ${{ matrix.flavor == 'linux' && 'ubuntu-24.04' || 'windows-2022' }} + + permissions: + contents: write + id-token: write + + steps: + - name: "Download build" + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + with: + name: build-${{ matrix.flavor }}-elixir-otp-${{ matrix.otp }} + + - name: Log in to Azure + if: ${{ matrix.flavor == 'windows' && vars.AZURE_TRUSTED_SIGNING_ACCOUNT_NAME }} + uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2.3.0 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + - name: "Sign files with Trusted Signing" + uses: azure/trusted-signing-action@fc390cf8ed0f14e248a542af1d838388a47c7a7c # v0.5.10 + if: ${{ matrix.flavor == 'windows' && vars.AZURE_TRUSTED_SIGNING_ACCOUNT_NAME }} + with: + endpoint: https://eus.codesigning.azure.net/ + trusted-signing-account-name: ${{ vars.AZURE_TRUSTED_SIGNING_ACCOUNT_NAME }} + certificate-profile-name: ${{ vars.AZURE_CERTIFICATE_PROFILE_NAME }} + files-folder: ${{ github.workspace }} + files-folder-filter: exe + file-digest: SHA256 + timestamp-rfc3161: http://timestamp.acs.microsoft.com + timestamp-digest: SHA256 + + - name: Create Release Hashes + if: matrix.flavor == 'windows' + shell: pwsh + run: | + $sha1 = Get-FileHash "$env:RELEASE_FILE" -Algorithm SHA1 + $sha1.Hash.ToLower() + " " + $env:RELEASE_FILE | Out-File "$env:RELEASE_FILE.sha1sum" + + $sha256 = Get-FileHash "$env:RELEASE_FILE" -Algorithm SHA256 + $sha256.Hash.ToLower() + " " + $env:RELEASE_FILE | Out-File "$env:RELEASE_FILE.sha256sum" + + - name: Create Release Hashes + if: matrix.flavor == 'linux' + shell: bash + run: | + shasum -a 1 "$RELEASE_FILE" > "${RELEASE_FILE}.sha1sum" + shasum -a 256 "$RELEASE_FILE" > "${RELEASE_FILE}.sha256sum" + + - name: "Upload Linux release artifacts" + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + with: + name: sign-${{ matrix.flavor }}-elixir-otp-${{ matrix.otp }} + path: ${{ env.RELEASE_FILE }}* + + sbom: + name: Generate SBoM + needs: [build, sign] + runs-on: ubuntu-24.04 + + permissions: + contents: write + id-token: write + attestations: write + + steps: + - name: Use HTTPS instead of SSH for Git cloning + id: git-config + shell: bash + run: git config --global url.https://github.com/.insteadOf ssh://git@github.com/ + + - name: Checkout project + id: checkout + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + + - name: "Download Build Artifacts" + id: download-build-artifacts + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + with: + pattern: "{sign-*-elixir-otp-*,Docs}" + merge-multiple: true + path: /tmp/build-artifacts/ + + - name: "Run OSS Review Toolkit" + id: ort + uses: ./.github/workflows/ort + with: + report-formats: "CycloneDx,SpdxDocument" + version: "${{ github.ref_type == 'tag' && github.ref_name || github.sha }}" + + - name: Attest Distribution Assets with SBoM + id: attest-sbom + uses: actions/attest-sbom@4651f806c01d8637787e274ac3bdf724ef169f34 # v3.0.0 + with: + subject-path: | + /tmp/build-artifacts/{elixir-otp-*.*,Docs.zip} + ${{ steps.ort.outputs.results-sbom-cyclonedx-xml-path }} + ${{ steps.ort.outputs.results-sbom-cyclonedx-json-path }} + ${{ steps.ort.outputs.results-sbom-spdx-yml-path }} + ${{ steps.ort.outputs.results-sbom-spdx-json-path }} + sbom-path: "${{ steps.ort.outputs.results-sbom-spdx-json-path }}" + + - name: "Copy SBoM provenance" + id: sbom-provenance + shell: bash + run: | + mkdir attestations + + for FILE in /tmp/build-artifacts/{elixir-otp-*.*,Docs.zip}; do + cp "$ATTESTATION" "attestations/$(basename "$FILE").sigstore" + done + + cp "$ATTESTATION" "attestations/$(basename "${{ steps.ort.outputs.results-sbom-cyclonedx-xml-path }}").sigstore" + cp "$ATTESTATION" "attestations/$(basename "${{ steps.ort.outputs.results-sbom-cyclonedx-json-path }}").sigstore" + cp "$ATTESTATION" "attestations/$(basename "${{ steps.ort.outputs.results-sbom-spdx-yml-path }}").sigstore" + cp "$ATTESTATION" "attestations/$(basename "${{ steps.ort.outputs.results-sbom-spdx-json-path }}").sigstore" + env: + ATTESTATION: "${{ steps.attest-sbom.outputs.bundle-path }}" + + - name: "Assemble Release SBoM Artifacts" + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + with: + name: "SBoM" + path: | + ${{ steps.ort.outputs.results-sbom-cyclonedx-xml-path }} + ${{ steps.ort.outputs.results-sbom-cyclonedx-json-path }} + ${{ steps.ort.outputs.results-sbom-spdx-yml-path }} + ${{ steps.ort.outputs.results-sbom-spdx-json-path }} + + - name: "Assemble Distribution Attestations" + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + with: + name: "Attestations" + path: "attestations/*.sigstore" + + upload-release: + name: Upload release + needs: [create_draft_release, build, sign, sbom] + runs-on: ubuntu-24.04 + + permissions: + contents: write + + steps: + - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + with: + pattern: "{sign-*-elixir-otp-*,Docs,SBoM,Attestations}" + merge-multiple: true + + - name: Upload Pre-build + shell: bash + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + if [ "${{ github.ref_type }}" == "branch" ]; then + tag=${{ github.ref_name }}-latest + else + tag="${{ github.ref_name }}" + fi + + gh release upload \ + --repo ${{ github.repository }} \ + --clobber \ + "$tag" \ + elixir-otp-*.zip \ + elixir-otp-*.zip.sha{1,256}sum \ + elixir-otp-*.zip.sigstore \ + elixir-otp-*.exe \ + elixir-otp-*.exe.sha{1,256}sum \ + elixir-otp-*.exe.sigstore \ + Docs.zip \ + Docs.zip.sha{1,256}sum \ + Docs.zip.sigstore \ + bom.* + + upload-builds-hex-pm: + name: Upload builds to hex.pm + runs-on: ubuntu-24.04 + needs: [build, sign] + concurrency: builds-hex-pm + environment: release + + env: + AWS_ACCESS_KEY_ID: ${{ secrets.HEX_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.HEX_AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ vars.HEX_AWS_REGION }} + AWS_S3_BUCKET: ${{ vars.HEX_AWS_S3_BUCKET }} + + steps: + - name: "Check if variables are set up" + if: "${{ ! vars.HEX_AWS_REGION }}" + run: | + echo "Required variables for uploading to hex.pm are not set up, skipping..." + exit 1 + + - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + with: + pattern: "{sign-*-elixir-otp-*,Docs}" + merge-multiple: true + + - name: Init purge keys file + run: | + touch purge_keys.txt + + - name: Upload Precompiled to S3 + run: | + ref_name=${{ github.ref_name }} + + oldest_otp=$(find . -type f -name 'elixir-otp-*.zip' | sed -r 's/^.*elixir-otp-([[:digit:]]+)\.zip$/\1/' | sort -n | head -n 1) + + for zip in $(find . -type f -name 'elixir-otp-*.zip' | sed 's/^\.\///'); do + dest=${zip/elixir/${ref_name}} + surrogate_key=${dest/.zip$/} + + aws s3 cp "${zip}" "s3://${AWS_S3_BUCKET}/builds/elixir/${dest}" \ + --cache-control "public,max-age=3600" \ + --metadata "{\"surrogate-key\":\"builds builds/elixir builds/elixir/${surrogate_key}\",\"surrogate-control\":\"public,max-age=604800\"}" + echo "builds/elixir/${surrogate_key}" >> purge_keys.txt + + if [ "$zip" == "elixir-otp-${oldest_otp}.zip" ]; then + aws s3 cp "${zip}" "s3://${AWS_S3_BUCKET}/builds/elixir/${ref_name}.zip" \ + --cache-control "public,max-age=3600" \ + --metadata "{\"surrogate-key\":\"builds builds/elixir builds/elixir/${ref_name}\",\"surrogate-control\":\"public,max-age=604800\"}" + echo builds/elixir/${ref_name} >> purge_keys.txt + fi + done + + - name: Upload Docs to S3 + run: | + version=$(echo ${{ github.ref_name }} | sed -e 's/^v//g') + + unzip Docs.zip + + for f in doc/*; do + if [ -d "$f" ]; then + app=$(echo "$f" | sed s/"doc\/"//) + tarball="${app}-${version}.tar.gz" + surrogate_key="docs/${app}-${version}" + + tar -czf "${tarball}" -C "doc/${app}" . + aws s3 cp "${tarball}" "s3://${AWS_S3_BUCKET}/docs/${tarball}" \ + --cache-control "public,max-age=3600" \ + --metadata "{\"surrogate-key\":\"${surrogate_key}\",\"surrogate-control\":\"public,max-age=604800\"}" + echo "${surrogate_key}" >> ../purge_keys.txt + fi + done + + - name: Update builds txt + run: | + date="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" + ref_name=${{ github.ref_name }} + + oldest_otp=$(find . -name 'elixir-otp-*.zip.sha256sum' | sed -r 's/^.*elixir-otp-([[:digit:]]+)\.zip\.sha256sum$/\1/' | sort -n | head -n 1) + + aws s3 cp "s3://${AWS_S3_BUCKET}/builds/elixir/builds.txt" builds.txt || true + touch builds.txt + + for sha256_file in $(find . -name 'elixir-otp-*.zip.sha256sum' | sed 's/^\.\///'); do + otp_version=$(echo "${sha256_file}" | sed -r 's/^elixir-otp-([[:digit:]]+)\.zip\.sha256sum/otp-\1/') + build_sha256=$(cut -d ' ' -f 1 "${sha256_file}") + + sed -i "/^${ref_name}-${otp_version} /d" builds.txt + echo -e "${ref_name}-${otp_version} ${{ github.sha }} ${date} ${build_sha256} \n$(cat builds.txt)" > builds.txt + + if [ "${otp_version}" == "otp-${oldest_otp}" ]; then + sed -i "/^${ref_name} /d" builds.txt + echo -e "${ref_name} ${{ github.sha }} ${date} ${build_sha256} \n$(cat builds.txt)" > builds.txt + fi + done + + sort -u -k1,1 -o builds.txt builds.txt + aws s3 cp builds.txt "s3://${AWS_S3_BUCKET}/builds/elixir/builds.txt" \ + --cache-control "public,max-age=3600" \ + --metadata '{"surrogate-key":"builds builds/elixir builds/elixir/txt","surrogate-control":"public,max-age=604800"}' + + echo 'builds/elixir/txt' >> purge_keys.txt + + - name: Flush cache + if: github.repository == 'elixir-lang/elixir' + run: | + function purge_key() { + curl \ + -X POST \ + -H "Fastly-Key: ${FASTLY_KEY}" \ + -H "Accept: application/json" \ + -H "Content-Length: 0" \ + "https://api.fastly.com/service/$1/purge/$2" + } + + function purge() { + purge_key ${FASTLY_REPO_SERVICE_ID} $1 + purge_key ${FASTLY_BUILDS_SERVICE_ID} $1 + sleep 2 + purge_key ${FASTLY_REPO_SERVICE_ID} $1 + purge_key ${FASTLY_BUILDS_SERVICE_ID} $1 + sleep 2 + purge_key ${FASTLY_REPO_SERVICE_ID} $1 + purge_key ${FASTLY_BUILDS_SERVICE_ID} $1 + } + + for key in $(cat purge_keys.txt); do + purge "${key}" + done + + env: + FASTLY_REPO_SERVICE_ID: ${{ secrets.HEX_FASTLY_REPO_SERVICE_ID }} + FASTLY_BUILDS_SERVICE_ID: ${{ secrets.HEX_FASTLY_BUILDS_SERVICE_ID }} + FASTLY_KEY: ${{ secrets.HEX_FASTLY_KEY }} diff --git a/.github/workflows/release_notifications.yml b/.github/workflows/release_notifications.yml new file mode 100644 index 00000000000..630d16f5294 --- /dev/null +++ b/.github/workflows/release_notifications.yml @@ -0,0 +1,32 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team + +name: Release Notifications + +on: + release: + types: + - published + +permissions: + contents: read + +jobs: + notify: + runs-on: ubuntu-latest + name: Notify + + steps: + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + + - uses: erlef/setup-beam@e6d7c94229049569db56a7ad5a540c051a010af9 # v1.20.4 + with: + otp-version: "27.3" + elixir-version: "1.18.3" + + - name: Run Elixir script + env: + ELIXIR_FORUM_TOKEN: ${{ secrets.ELIXIR_FORUM_TOKEN }} + ELIXIR_LANG_ANN_TOKEN: ${{ secrets.ELIXIR_LANG_ANN_TOKEN }} + run: | + elixir .github/workflows/notify.exs ${{ github.ref_name }} diff --git a/.github/workflows/release_pre_built/action.yml b/.github/workflows/release_pre_built/action.yml new file mode 100644 index 00000000000..22fc1b04729 --- /dev/null +++ b/.github/workflows/release_pre_built/action.yml @@ -0,0 +1,75 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team + +name: Release Pre-build +description: "Builds Elixir release, ExDoc and generates docs" + +inputs: + otp: + description: "The major OTP version" + + otp_version: + description: "The exact OTP version (major.minor[.patch])" + + build_docs: + description: "Whether docs have to be built" + +runs: + using: "composite" + + steps: + - uses: erlef/setup-beam@5304e04ea2b355f03681464e683d92e3b2f18451 # v1.18.2 + with: + otp-version: ${{ inputs.otp_version }} + version-type: strict + + - name: Build Elixir Release + shell: bash + run: | + make Precompiled.zip + mv Precompiled.zip elixir-otp-${{ inputs.otp }}.zip + echo "$PWD/bin" >> $GITHUB_PATH + + - name: Install NSIS + shell: bash + run: | + sudo apt update + sudo apt install -y nsis + + - name: Build Elixir Windows Installer + shell: bash + run: | + export OTP_VERSION=${{ inputs.otp_version }} + export ELIXIR_ZIP=$PWD/elixir-otp-${{ inputs.otp }}.zip + (cd lib/elixir/scripts/windows_installer && ./build.sh) + mv lib/elixir/scripts/windows_installer/tmp/elixir-otp-${{ inputs.otp }}.exe . + - name: Get ExDoc ref + if: ${{ inputs.build_docs }} + shell: bash + run: | + if [ "${{ github.ref_name }}" = "main" ]; then + ref=main + else + ref=v$(curl -s https://hex.pm/api/packages/ex_doc | jq --raw-output '.latest_stable_version') + fi + echo "EX_DOC_REF=$ref" >> $GITHUB_ENV + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + if: ${{ inputs.build_docs }} + with: + repository: elixir-lang/ex_doc + ref: ${{ env.EX_DOC_REF }} + path: ex_doc + - name: Build ex_doc + if: ${{ inputs.build_docs }} + shell: bash + run: | + mv ex_doc ../ex_doc + cd ../ex_doc + ../elixir/bin/mix do local.rebar --force + local.hex --force + deps.get + compile + cd ../elixir + - name: Build Docs + if: ${{ inputs.build_docs }} + shell: bash + run: | + git fetch --tags + make Docs.zip diff --git a/.gitignore b/.gitignore index 9f5a6a2479e..ea78a647a17 100644 --- a/.gitignore +++ b/.gitignore @@ -1,14 +1,19 @@ -/.eunit -/.release -/docs -/ebin -/lib/*/ebin/* -/lib/*/tmp -/lib/elixir/src/elixir.app.src -/lib/elixir/src/*_lexer.erl +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +/doc/ +/lib/*/ebin/ +/lib/*/_build/ +/lib/*/tmp/ /lib/elixir/src/*_parser.erl -/lib/elixir/test/ebin -/rel/elixir +/lib/elixir/test/ebin/ +/man/elixir.1 +/man/iex.1 +/Docs.zip +/Precompiled.zip +/.eunit +.elixir.plt erl_crash.dump -.dialyzer_plt -.dialyzer.base_plt +/cover/ +.tool-versions diff --git a/.markdownlint-cli2.jsonc b/.markdownlint-cli2.jsonc new file mode 100644 index 00000000000..4b35e0693b0 --- /dev/null +++ b/.markdownlint-cli2.jsonc @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021 The Elixir Team +{ + "globs": [ + "**/*.md" + ], + "ignores": [ + ".git/**" + ], + "gitignore": true, + "config": { + // Consecutive header levels (h1 -> h2 -> h3). + "MD001": false, + // Header style. We use #s. + "MD003": { + "style": "atx" + }, + // Style of unordered lists.. + "MD007": { + "indent": 2, + "start_indented": true + }, + // Line length. Who cares. + "MD013": false, + // This warns if you have "console" or "shell" code blocks with a dollar sign $ that + // don't show output. We use those a lot, so this is fine for us. + "MD014": false, + // Multiple headings with the same content. + "MD024": { + // Duplication is allowed for headings with different parents. + "siblings_only": true + }, + // Trailing punctuation in heading. + // Some headers finish with ! because it refers to a function name. Therefore we remove ! from + // the default values. + "MD026": { + "punctuation": ".,;:。,;:!" + }, + // Allow empty line between block quotes. Used by contiguous admonition blocks. + "MD028": false, + // Allowed HTML inline elements. + "MD033": { + "allowed_elements": [ + "h1", + "a", + "br", + "img", + "picture", + "source", + "noscript", + "p", + "script" + ] + }, + // This warns if you have spaces in code blocks. Sometimes, that's fine. + "MD038": false, + // Code block style. We don't care if it's fenced or indented. + "MD046": false, + // Our tables are too large to align. + "MD060": false + } +} \ No newline at end of file diff --git a/.ort.yml b/.ort.yml new file mode 100644 index 00000000000..36153b73e30 --- /dev/null +++ b/.ort.yml @@ -0,0 +1,118 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team + +excludes: + paths: + - pattern: "man/*" + reason: "DOCUMENTATION_OF" + comment: "Documentation" + - pattern: ".github/**/*" + reason: "BUILD_TOOL_OF" + comment: "Documentation" + - pattern: ".ort/**/*" + reason: "BUILD_TOOL_OF" + comment: "Documentation" + + # Unfortunately we'll have to repeat all package level excludes here + # Make sure to keep them in sync with the package configuration in + # .ort/package-configurations + - pattern: "lib/*/pages/**/*" + reason: "DOCUMENTATION_OF" + comment: "Documentation" + - pattern: "lib/*/test/**/*" + reason: "TEST_OF" + comment: "Tests" + - pattern: "lib/*/scripts/**/*" + reason: "BUILD_TOOL_OF" + comment: "Build Tool" + - pattern: "lib/*/examples/**/*" + reason: "EXAMPLE_OF" + comment: "Example" + +curations: + license_findings: + # Version File + - path: "VERSION" + reason: "NOT_DETECTED" + comment: "Apply Trademark Policy to VERSION file" + detected_license: "NONE" + concluded_license: "Apache-2.0" + + # Wrongly Identified + - path: ".gitignore" + reason: "INCORRECT" + comment: "Ignored by ScanCode" + detected_license: "NONE" + concluded_license: "Apache-2.0" + - path: ".gitattributes" + reason: "INCORRECT" + comment: "Ignored by ScanCode" + detected_license: "NONE" + concluded_license: "Apache-2.0" + - path: "CONTRIBUTING.md" + reason: "INCORRECT" + comment: "Wrongly identified TSL license" + detected_license: "Apache-2.0 OR NOASSERTION OR LicenseRef-scancode-tsl-2020" + concluded_license: "Apache-2.0" + - path: "OPEN_SOURCE_POLICY.md" + reason: "INCORRECT" + comment: "Wrongly identified NOASSERTION" + detected_license: "NOASSERTION" + concluded_license: "Apache-2.0" + + # Unfortunately we'll have to repeat all package level license curations here + # Make sure to keep them in sync with the package configuration in + # .ort/package-configurations + + # Test Fixtures + - path: "lib/*/test/fixtures/**/*" + reason: "NOT_DETECTED" + comment: "Apply default license to test fixtures" + detected_license: "NONE" + concluded_license: "Apache-2.0" + + # Logos + - path: "lib/elixir/pages/images/logo.png" + reason: "NOT_DETECTED" + comment: "Apply Trademark Policy to Elixir Logo" + detected_license: "NONE" + concluded_license: "LicenseRef-elixir-trademark-policy" + - path: "lib/elixir/scripts/windows_installer/assets/Elixir.ico" + reason: "NOT_DETECTED" + comment: "Apply Trademark Policy to Elixir Logo" + detected_license: "NONE" + concluded_license: "LicenseRef-elixir-trademark-policy" + + # Documentation Images + - path: "lib/elixir/pages/images/**/*.png" + reason: "NOT_DETECTED" + comment: "Apply default license to all images" + detected_license: "NONE" + concluded_license: "Apache-2.0" + + # Test Fixtures + - path: "lib/elixir/test/elixir/fixtures/**/*" + reason: "NOT_DETECTED" + comment: "Apply default license to test fixtures" + detected_license: "NONE" + concluded_license: "Apache-2.0" + + # Unicode + - path: "lib/elixir/unicode/*.txt" + reason: "NOT_DETECTED" + comment: "Apply default license to unicode files" + detected_license: "NONE" + concluded_license: "LicenseRef-scancode-unicode" + + # Wrongly Identified + - path: "lib/elixir/pages/references/library-guidelines.md" + reason: "INCORRECT" + comment: | + The guide mentions multiple licenses for users to choose from. + It however is not licensed itself by the mentioned licenses. + concluded_license: "Apache-2.0" + - path: "lib/elixir/scripts/windows_installer/.gitignore" + reason: "INCORRECT" + comment: "Ignored by ScanCode" + detected_license: "NONE" + concluded_license: "Apache-2.0" diff --git a/.ort/config/config.yml b/.ort/config/config.yml new file mode 100644 index 00000000000..3d72346ad8e --- /dev/null +++ b/.ort/config/config.yml @@ -0,0 +1,21 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team + +ort: + enableRepositoryPackageCurations: true + enableRepositoryPackageConfigurations: true + + scanner: + skipConcluded: false + includeFilesWithoutFindings: true + + analyzer: + allowDynamicVersions: true + enabledPackageManagers: [SpdxDocumentFile] + + reporter: + reporters: + SpdxDocument: + options: + creationInfoOrganization: The Elixir Team + documentName: "Elixir Source SPDX Document" diff --git a/.ort/config/evaluator.rules.kts b/.ort/config/evaluator.rules.kts new file mode 100644 index 00000000000..1afc2b50e80 --- /dev/null +++ b/.ort/config/evaluator.rules.kts @@ -0,0 +1,88 @@ +/* + * Copyright (C) 2019 The ORT Project Authors (see ) + * Copyright (c) 2021 The Elixir Team + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + + // Docs: https://oss-review-toolkit.org/ort/docs/configuration/evaluator-rules + +val whitelistedLicenses = listOf( + // License for Elixir & Imported Erlang Projects + "Apache-2.0", + // License for the Elixir Logo + "LicenseRef-elixir-trademark-policy", + "LicenseRef-scancode-elixir-trademark-policy", + // License for included Unicode Files + "LicenseRef-scancode-unicode", + // DCO for committers + "LicenseRef-scancode-dco-1.1" +).map { SpdxSingleLicenseExpression.parse(it) }.toSet() + +fun PackageRule.howToFixDefault() = """ + * Check if this license violation is intended + * Adjust evaluation rules in `.ort/config/evaluator.rules.kts` + """.trimIndent() + +fun PackageRule.LicenseRule.isHandled() = + object : RuleMatcher { + override val description = "isHandled($license)" + + override fun matches() = license in whitelistedLicenses + } + +fun RuleSet.unhandledLicenseRule() = packageRule("UNHANDLED_LICENSE") { + // Do not trigger this rule on packages that have been excluded in the .ort.yml. + require { + -isExcluded() + } + + // Define a rule that is executed for each license of the package. + licenseRule("UNHANDLED_LICENSE", LicenseView.CONCLUDED_OR_DECLARED_AND_DETECTED) { + require { + -isExcluded() + -isHandled() + } + + // Throw an error message including guidance how to fix the issue. + error( + "The license $license is currently not covered by policy rules. " + + "The license was ${licenseSource.name.lowercase()} in package " + + "${pkg.metadata.id.toCoordinates()}.", + howToFixDefault() + ) + } +} + +fun RuleSet.unmappedDeclaredLicenseRule() = packageRule("UNMAPPED_DECLARED_LICENSE") { + require { + -isExcluded() + } + + resolvedLicenseInfo.licenseInfo.declaredLicenseInfo.processed.unmapped.forEach { unmappedLicense -> + warning( + "The declared license '$unmappedLicense' could not be mapped to a valid license or parsed as an SPDX " + + "expression. The license was found in package ${pkg.metadata.id.toCoordinates()}.", + howToFixDefault() + ) + } +} + +val ruleSet = ruleSet(ortResult, licenseInfoResolver, resolutionProvider) { + unhandledLicenseRule() + unmappedDeclaredLicenseRule() +} + +ruleViolations += ruleSet.violations diff --git a/.ort/package-configurations/eex.yml b/.ort/package-configurations/eex.yml new file mode 100644 index 00000000000..b205a6f65e0 --- /dev/null +++ b/.ort/package-configurations/eex.yml @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team + +id: "SpdxDocumentFile:The Elixir Team:eex:" +path_excludes: + - pattern: "lib/eex/test/**/*" + reason: "TEST_OF" + comment: "Tests" +license_finding_curations: + # Test Fixtures + - path: "lib/eex/test/fixtures/**/*" + reason: "NOT_DETECTED" + comment: "Apply default license to test fixtures" + detected_license: "NONE" + concluded_license: "Apache-2.0" diff --git a/.ort/package-configurations/elixir.yml b/.ort/package-configurations/elixir.yml new file mode 100644 index 00000000000..a9324799d69 --- /dev/null +++ b/.ort/package-configurations/elixir.yml @@ -0,0 +1,60 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team + +id: "SpdxDocumentFile:The Elixir Team:elixir:" +path_excludes: + - pattern: "lib/elixir/pages/**/*" + reason: "DOCUMENTATION_OF" + comment: "Documentation" + - pattern: "lib/elixir/scripts/**/*" + reason: "BUILD_TOOL_OF" + comment: "Build Tool" + - pattern: "lib/elixir/test/**/*" + reason: "TEST_OF" + comment: "Tests" +license_finding_curations: + # Logos + - path: "lib/elixir/pages/images/logo.png" + reason: "NOT_DETECTED" + comment: "Apply Trademark Policy to Elixir Logo" + detected_license: "NONE" + concluded_license: "LicenseRef-elixir-trademark-policy" + - path: "lib/elixir/scripts/windows_installer/assets/Elixir.ico" + reason: "NOT_DETECTED" + comment: "Apply Trademark Policy to Elixir Logo" + detected_license: "NONE" + concluded_license: "LicenseRef-elixir-trademark-policy" + + # Documentation Images + - path: "lib/elixir/pages/images/**/*.png" + reason: "NOT_DETECTED" + comment: "Apply default license to all images" + detected_license: "NONE" + concluded_license: "Apache-2.0" + + # Test Fixtures + - path: "lib/elixir/test/elixir/fixtures/**/*" + reason: "NOT_DETECTED" + comment: "Apply default license to test fixtures" + detected_license: "NONE" + concluded_license: "Apache-2.0" + + # Unicode + - path: "lib/elixir/unicode/*.txt" + reason: "NOT_DETECTED" + comment: "Apply default license to unicode files" + detected_license: "NONE" + concluded_license: "LicenseRef-scancode-unicode" + + # Wrongly Identified + - path: "lib/elixir/pages/references/library-guidelines.md" + reason: "INCORRECT" + comment: | + The guide mentions multiple licenses for users to choose from. + It however is not licensed itself by the mentioned licenses. + concluded_license: "Apache-2.0" + - path: "lib/elixir/scripts/windows_installer/.gitignore" + reason: "INCORRECT" + comment: "Ignored by ScanCode" + detected_license: "NONE" + concluded_license: "Apache-2.0" diff --git a/.ort/package-configurations/ex_unit.yml b/.ort/package-configurations/ex_unit.yml new file mode 100644 index 00000000000..57c6bf498d8 --- /dev/null +++ b/.ort/package-configurations/ex_unit.yml @@ -0,0 +1,18 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team + +id: "SpdxDocumentFile:The Elixir Team:exunit:" +path_excludes: + - pattern: "lib/ex_unit/examples/**/*" + reason: "EXAMPLE_OF" + comment: "Example" + - pattern: "lib/ex_unit/test/**/*" + reason: "TEST_OF" + comment: "Tests" +license_finding_curations: + # Test Fixtures + - path: "lib/ex_unit/test/fixtures/**/*" + reason: "NOT_DETECTED" + comment: "Apply default license to test fixtures" + detected_license: "NONE" + concluded_license: "Apache-2.0" diff --git a/.ort/package-configurations/logger.yml b/.ort/package-configurations/logger.yml new file mode 100644 index 00000000000..4005427948d --- /dev/null +++ b/.ort/package-configurations/logger.yml @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team + +id: "SpdxDocumentFile:The Elixir Team:logger:" +path_excludes: + - pattern: "lib/logger/test/**/*" + reason: "TEST_OF" + comment: "Tests" diff --git a/.ort/package-configurations/mix.yml b/.ort/package-configurations/mix.yml new file mode 100644 index 00000000000..a45db2bc6c0 --- /dev/null +++ b/.ort/package-configurations/mix.yml @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team + +id: "SpdxDocumentFile:The Elixir Team:mix:" +path_excludes: + - pattern: "lib/mix/test/**/*" + reason: "TEST_OF" + comment: "Tests" +license_finding_curations: + # Test Fixtures + - path: "lib/mix/test/fixtures/**/*" + reason: "NOT_DETECTED" + comment: "Apply default license to test fixtures" + detected_license: "NONE" + concluded_license: "Apache-2.0" diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index dcb0d1f96b1..00000000000 --- a/.travis.yml +++ /dev/null @@ -1,9 +0,0 @@ -language: erlang -script: "make compile && rm -rf .git && make test" -notifications: - irc: "irc.freenode.org#elixir-lang" - recipients: - - jose.valim@plataformatec.com.br - - eric.meadows.jonsson@gmail.com -otp_release: - - 17.0 diff --git a/CHANGELOG.md b/CHANGELOG.md index 104d62bc78f..34f9504f316 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,1133 +1,76 @@ -# Changelog + -## v0.14.3-dev +# Changelog for Elixir v1.20 -* Enhancements +## Type system improvements -* Bug fixes - * [Kernel] `|>`, `<<<`, `>>>` and `^^^` were made left associative in operator table - * [Kernel] `<`, `>`, `<=`, `>=` were given higher precedence than comparison ones (`==`, `!=`, etc) in operator table +### Full type inference -* Soft deprecations (no warnings emitted) +Elixir now performs inference of whole functions. The best way to show the new capabilities are with examples. Take the following code: -* Deprecations +```elixir +def add_foo_and_bar(data) do + data.foo + data.bar +end +``` -* Backwards incompatible changes +Elixir now infers that the function expects a `map` as first argument, and the map must have the keys `.foo` and `.bar` whose values are either `integer()` or `float()`. The return type will be either `integer()` or `float()`. -## v0.14.2 (2014-06-29) +Here is another example: -* Enhancements - * [Enum] Improve performance of `Enum.join/2` and `Enum.map_join/3` by using iolists - * [Kernel] Ensure compatibility with Erlang 17.1 - * [Kernel] Support `@external_resource` attribute to external dependencies to a module - * [Mix] Allow built Git dependencies to run on a system without Git by passing `--no-deps-check` - * [Mix] Add `MIX_ARCHIVES` env variable (it is recommended for Elixir build tools to swap this environment) - * [Task] Set `:proc_lib` initial call on task to aid debugging - * [Typespec] Delay typespec compilation to after expansion - * [URI] Allow `parse/1` now accepts `%URI{}` as argument and return the uri itself +```elixir +def sum_to_string(a, b) do + Integer.to_string(a + b) +end +``` -* Bug fixes - * [CLI] Support paths inside archives in `-pa` and `-pz` options - * [IEx] Remove delay when printing data from the an application start callback - * [IEx] Ensure we show a consistent error when we cannot evaluate `.iex.exs` - * [Kernel] Ensure derived protocols are defined with a file - * [Kernel] Change precedence of `&` to not special case `/` - * [Kernel] Ensure we can only use variables and `\\` as arguments of bodyless clause +Even though the `+` operator works with both integers and floats, Elixir infers that `a` and `b` must be both integers, as the result of `+` is given to a function that expects an integer. The inferred type information is then used during type checking to find possible typing errors. -* Soft deprecations (no warnings emitted) - * [EEx] Using `EEx.TransformerEngine` and `EEx.AssignsEngine` are deprecated in favor of function composition with `Macro.prewalk/1` (see `EEx.SmartEngine` for an example) - * [Kernel] `Kernel.xor/2` is deprecated - * [Mix] `Mix.Generator.from_file/1` is deprecated in favor of passing `from_file: file` option to `embed_text/2` and `embed_template/2` (note though that `from_file/1` expects a path relative to the current file while the `from_file: file` expects one relative to the current working directory) +### Acknowledgements -* Deprecations - * [Kernel] `size/1` is deprecated in favor of `byte_size/1` and `tuple_size/1` (this change was soft deprecated two releases ago) +The type system was made possible thanks to a partnership between [CNRS](https://www.cnrs.fr/) and [Remote](https://remote.com/). The development work is currently sponsored by [Fresha](https://www.fresha.com/) and [Dashbit](https://dashbit.co/). -* Backwards incompatible changes - * [CLI] Remove support for the `--gen-debug` option as its usage is not documented by OTP - * [Kernel] Sigils no longer balance start and end tokens, e.g. the sigil `~s(f(o)o)` is no longer valid as it finishes in the first closing `)` - * [Kernel] Variables set in `cond` clause heads are no longer available outside of that particular `cond` clause (this is the behaviour also found in `case`, `receive` and friends) - * [System] `build_info/0` now returns a map +## v1.20.0-dev -## v0.14.1 (2014-06-18) +### 1. Enhancements -* Enhancements - * [Base] Decoding and encoding functions now accept the `:case` as an option - * [ExUnit] The test process now exits with `:shutdown` reason - * [GenEvent] `GenEvent.stream/2` now accepts `:sync` and `:async` modes - * [Node] Add `Node.start/3` and `Node.stop/0` - * [String] Updated Unicode database to 7.0 - * [Task] Log when tasks crash +#### Elixir -* Bug fixes - * [Enum] `Enum.slice/2` and `Enum.slice/3` always returns a list (and never nil) - * [Kernel] Disambiguate (w)erl to (w)erl.exe - * [Mix] Ensure umbrella project is recompiled when a dependency inside an umbrella child changes - * [OptionParser] Do not allow underscores in option names - * [Path] Fix path expansion of `"/.."` - * [Path] Do not match files starting with `.` in `Path.wildcard/2` by default - * [Process] `Process.info(pid, :registered_name)` returns `{:registered_name, nil}` if there is no registered name - * [String] `String.slice/2` and `String.slice/3` always returns a list (and never nil) - * [URI] `encode/1` does not escape reserved/unreserved characters by default nor encodes whitespace as `+` (check `URI.encode_www_form/1` and `URI.decode_www_form/1` for previous behaviour) + * [Enum] Add `Enum.min_max` sorter + * [Integer] Add `Integer.ceil_div/2` + * [Kernel] Print intermediate results of `dbg` for pipes + * [Kernel] Warn on unused requires + * [Regex] Add `Regex.import/1` to import regexes defined with `/E` -* Deprecations - * [Mix] `:escript_*` options were moved into a single `:escript` group +#### ExUnit -* Backwards incompatible changes - * [GenEvent] `GenEvent.stream/2` defaults to `:sync` mode - * [Kernel] Remove `get_in/1` + * [ExUnit.CaptureLog] Add `:formatter` option for custom log formatting -## v0.14.0 (2014-06-08) +#### Mix -* Enhancements - * [ExUnit] Add `on_exit/1` callbacks that are guaranteed to run once the test process exits and always in another process - * [Kernel] Store documentation in the abstract code to avoid loading them when the module is loaded - * [Kernel] Add `get_in/2`, `put_in/3`, `update_in/3` and `get_and_update_in/3` to handle nested data structure operations - * [Kernel] Add `get_in/1`, `put_in/2`, `update_in/2` and `get_and_update_in/2` to handle nested data structure operations via paths - * [Mix] Add `Mix.Config` to ease definition of configuration files - * [Mix] Add `mix loadconfig` task that can be called multiple times to load external configs - * [Mix] Support `--config` option on `mix run` - * [Mix] Support `HTTP_PROXY` and `HTTPS_PROXY` on Mix url commands - * [Mix] Support `--names` options in `mix help` which emit only names (useful for autocompletion) - * [Protocol] Add `Protocol.consolidate/2`, `Protocol.consolidated?/1` and a `mix compile.protocols` task for protocol consolidation - * [Protocol] Add `Protocol.derive/3` for runtime deriving of a struct - * [String] Add `String.chunk/2` - * [Struct] Add support for `@derive` before `defstruct/2` definitions + * [mix test] Add `mix test --dry-run` -* Bug fixes - * [File] `File.rm` now consistently deletes read-only across operating systems - * [Kernel] Ensure Mix `_build` structure works on Windows when copying projects - * [Kernel] Ensure `1.0E10` (with uppercase E) is also valid syntax - * [Mix] Fix `mix do` task for Windows' powershell users - * [Path] Fix `Path.absname("/")` and `Path.expand("/")` to return the absolute path `"/"`. +### 2. Bug fixes -* Soft deprecations (no warnings emitted) - * [Kernel] `size/1` is deprecated, please use `byte_size/1` or `tuple_size/1` instead - * [ExUnit] `teardown/2` and `teardown_all/2` are deprecated in favor of `on_exit/1` callbacks +### 3. Soft deprecations (no warnings emitted) -* Deprecations - * [Access] `Access.access/2` is deprecated in favor of `Access.get/2` - * [Dict] `Dict.Behaviour` is deprecated in favor of `Dict` - * [Kernel] `Application.Behaviour`, `GenEvent.Behaviour`, `GenServer.Behaviour` and `Supervisor.Behaviour` are deprecated in favor of `Application`, `GenEvent`, `GenServer` and `Supervisor` - * [Kernel] `defexception/3` is deprecated in favor of `defexception/1` - * [Kernel] `raise/3` is deprecated in favor of `reraise/2` - * [Kernel] `set_elem/3` is deprecated in favor of `put_elem/3` - * [Kernel] Passing an atom `var!/1` is deprecated, variables can be built dynamically with `Macro.var/2` - * [Mix] Exceptions that define a `:mix_error` field to be compatible with Mix are no longer supported. Instead please provide a `:mix` field and use `Mix.raise/1` and `Mix.raise/2` +### 4. Hard deprecations -* Backwards incompatible changes - * [Access] `Kernel.access/2` no longer exists and the `Access` protocol now requires `get/2` (instead of `access/2`) and `get_and_update/3` to be implemented - * [Kernel] Retrieving docs as `module.__info__(:docs)` is no longer supported, please use `Code.get_docs/2` instead - * [Kernel] `Code.compiler_options/1` no longer accepts custom options, only the ones specified by Elixir (use mix config instead) - * [Mix] `mix new` no longer generates a supevision tree by default, please pass `--sup` instead - * [Task] Tasks are automatically linked to callers and a failure in the task will crash the caller directly +#### Elixir -## v0.13.3 (2014-05-24) + * [File] `File.stream!(path, modes, lines_or_bytes)` is deprecated in favor of `File.stream!(path, lines_or_bytes, modes)` + * [Kernel] Matching on the size inside a bit pattern now requires the pin operator for consistency, such as `<>` + * [Kernel.ParallelCompiler] `Kernel.ParallelCompiler.async/1` is deprecated in favor of `Kernel.ParallelCompiler.pmap/2`, which is more performant and addresses known limitations -* Enhancements - * [OptionParser] Add `:strict` option that only parses known switches - * [OptionParser] Add `next/2` useful for manual parsing of options - * [Macro] Add `Macro.prewalk/2/3` and `Macro.postwalk/2/3` - * [Kernel] `GenEvent`, `GenServer`, `Supervisor`, `Agent` and `Task` modules added - * [Kernel] Make deprecations compiler warnings to avoid the same deprecation being printed multiple times +#### Logger -* Bug fixes - * [Enum] Fix `Enum.join/2` and `Enum.map_join/3` for empty binaries at the beginning of the collection - * [ExUnit] Ensure the formatter doesn't error when printing :EXITs - * [Kernel] Rename `ELIXIR_ERL_OPTS` to `ELIXIR_ERL_OPTIONS` for consistency with `ERL_COMPILER_OPTIONS` - * [OptionParser] Parse `-` as a plain argument - * [OptionParser] `--` is always removed from argument list on `parse/2` and when it is the leading entry on `parse_head/2` - * [Regex] Properly escape regex (previously regex controls were double escaped) + * [Logger] `Logger.*_backend` functions are deprecated in favor of handlers. If you really want to keep on using backends, see the `:logger_backends` package + * [Logger] `Logger.enable/1` and `Logger.disable/1` have been deprecated in favor of `Logger.put_process_level/2` and `Logger.delete_process_level/1` -* Soft deprecations (no warnings emitted) - * [Dict] `Dict.Behaviour` is deprecated in favor of `Dict` - * [Kernel] `Application.Behaviour`, `GenEvent.Behaviour`, `GenServer.Behaviour` and `Supervisor.Behaviour` are deprecated in favor of `Application`, `GenEvent`, `GenServer` and `Supervisor` - * [Kernel] `defexception/3` is deprecated in favor of `defexception/1` - * [Kernel] `raise/3` is deprecated in favor of `reraise/2` - * [Kernel] `set_elem/3` is deprecated in favor of `put_elem/3` +## v1.19 -* Soft deprecations for conversions (no warnings emitted) - * [Kernel] `atom_to_binary/1` and `atom_to_list/1` are deprecated in favor of `Atom.to_string/1` and `Atom.to_char_list/1` - * [Kernel] `bitstring_to_list/1` and `list_to_bitstring/1` are deprecated in favor of the `:erlang` ones - * [Kernel] `binary_to_atom/1`, `binary_to_existing_atom/1`, `binary_to_float/1`, `binary_to_integer/1` and `binary_to_integer/2` are deprecated in favor of conversion functions in `String` - * [Kernel] `float_to_binary/*` and `float_to_list/*` are deprecated in favor of `Float.to_string/*` and `Float.to_char_list/*` - * [Kernel] `integer_to_binary/*` and `integer_to_list/*` are deprecated in favor of `Integer.to_string/*` and `Integer.to_char_list/*` - * [Kernel] `iodata_to_binary/1` and `iodata_length/1` are deprecated `IO.iodata_to_binary/1` and `IO.iodata_length/1` - * [Kernel] `list_to_atom/1`, `list_to_existing_atom/1`, `list_to_float/1`, `list_to_integer/1`, `list_to_integer/2` and `list_to_tuple/1` are deprecated in favor of conversion functions in `List` - * [Kernel] `tuple_to_list/1` is deprecated in favor of `Tuple.to_list/1` - * [List] `List.from_char_data/1` and `List.from_char_data!/1` deprecated in favor of `String.to_char_list/1` - * [String] `String.from_char_data/1` and `String.from_char_data!/1` deprecated in favor of `List.to_string/1` - -* Deprecations - * [Kernel] `is_exception/1`, `is_record/1` and `is_record/2` are deprecated in favor of `Exception.exception?1`, `Record.record?/1` and `Record.record?/2` - * [Kernel] `defrecord/3` is deprecated in favor of structs - * [Kernel] `:hygiene` in `quote` is deprecated - * [Mix] `Mix.project/0` is deprecated in favor of `Mix.Project.config/0` - * [Process] `Process.spawn/1`, `Process.spawn/3`, `Process.spawn_link/1`, `Process.spawn_link/3`, `Process.spawn_monitor/1`, `Process.spawn_monitor/3`, `Process.send/2` and `Process.self/0` are deprecated in favor of the ones in `Kernel` - -* Backwards incompatible changes - * [Exception] Exceptions now generate structs instead of records - * [OptionParser] Errors on parsing returns the switch and value as binaries (unparsed) - * [String] `String.to_char_list/1` (previously deprecated) no longer returns a tuple but the char list only and raises in case of failure - -## v0.13.2 (2014-05-11) - -* Enhancements - * [Application] Add an Application module with common functions to work with OTP applications - * [Exception] Add `Exception.message/1`, `Exception.format_banner/1`, `Exception.format_exit/1` and `Exception.format/1` - * [File] Add `File.ln_s/1` - * [Mix] `mix deps.clean` now works accross environments - * [Mix] Support line numbers in `mix test`, e.g. test/some/file_test.exs:12 - * [Mix] Use `@file` attributes to detect dependencies in between `.ex` and external files. This means changing an `.eex` file will no longer recompile the whole project only the files that depend directly on it - * [Mix] Support application configurations in `config/config.exs` - * [Mix] Support user-wide configuration with `~/.mix/config.exs` - * [Mix] `mix help` now uses ANSI formatting to print guides - * [Regex] Support functions in `Regex.replace/4` - * [String] Support `:parts` in `String.split/3` - -* Bug fixes - * [Code] Ensure we don't lose the caller stacktrace on code evaluation - * [IEx] Exit signals now exits the IEx evaluator and a new one is spawned on its place - * [IEx] Ensure we don't prune too much stacktrace when reporting failures - * [IEx] Fix an issue where `iex.bat` on Windows was not passing the proper parameters forward - * [Kernel] Ensure modules defined on root respect defined aliases - * [Kernel] Do not wrap single lists in `:__block__` - * [Kernel] Ensure emitted beam code works nicely with dialyzer - * [Kernel] Do not allow a module named `Elixir` to be defined - * [Kernel] Create remote funs even if mod is a variable in capture `&mod.fun/arity` - * [Kernel] Improve compiler message when duplicated modules are detected - * [Mix] Generate `.gitignore` for `--umbrella` projects - * [Mix] Verify if a git dependency in deps has a proper git checkout and clean it automatically when it doesn't - * [Mix] Ensure `mix test` works with `IEx.pry/0` - * [System] Convert remaining functions in System to rely on char data - -* Soft deprecations (no warnings emitted) - * [Exception] `exception.message` is deprecated in favor `Exception.message/1` for retrieving exception messages - * [Kernel] `is_exception/1`, `is_record/1` and `is_record/2` are deprecated in favor of `Exception.exception?1`, `Record.record?/1` and `Record.record?/2` - * [Mix] `Mix.project/0` is deprecated in favor of `Mix.Project.config/0` - * [Process] `Process.spawn/1`, `Process.spawn/3`, `Process.spawn_link/1`, `Process.spawn_link/3`, `Process.spawn_monitor/1`, `Process.spawn_monitor/3`, `Process.send/2` and `Process.self/0` are deprecated in favor of the ones in `Kernel` - -* Deprecations - * [IEx] IEx.Options is deprecated in favor of `IEx.configure/1` and `IEx.configuration/0` - * [Kernel] `lc` and `bc` comprehensions are deprecated in favor of `for` - * [Macro] `Macro.safe_terms/1` is deprecated - * [Process] `Process.delete/0` is deprecated - * [Regex] Deprecate `:global` option in `Regex.split/3` in favor of `parts: :infinity` - * [String] Deprecate `:global` option in `String.split/3` in favor of `parts: :infinity` - -* Backwards incompatible changes - * [ExUnit] `ExUnit.Test` and `ExUnit.TestCase` has been converted to structs - * [ExUnit] The test and callback context has been converted to maps - * [Kernel] `File.Stat`, `HashDict`, `HashSet`, `Inspect.Opts`, `Macro.Env`, `Range`, `Regex` and `Version.Requirement` have been converted to structs. This means `is_record/2` checks will no longer work, instead, you can pattern match on them using `%Range{}` and similar - * [URI] The `URI.Info` record has now become the `URI` struct - * [Version] The `Version.Schema` record has now become the `Version` struct - -## v0.13.1 (2014-04-27) - -* Enhancements - * [Mix] Support `MIX_EXS` as configuration for running the current mix.exs file - * [Mix] Support Hex out of the box. This means users do not need to install Hex directly, instead, Mix will prompt whenever there is a need to have Hex installed - -* Bug fixes - * [ExUnit] Ensure doctest failures are properly reported - * [Kernel] Fix a bug where comprehensions arguments were not properly take into account in the variable scope - * [Mix] Fix issue on rebar install when the endpoint was redirecting to a relative uri - -* Soft deprecations (no warnings emitted) - * [Kernel] `iolist_size` and `iolist_to_binary` are deprecated in favor of `iodata_length` and `iodata_to_binary` - * [String] `String.to_char_list/1` is deprecated in favor of `List.from_char_data/1` - * [String] `String.from_char_list/1` is deprecated in favor of `String.from_char_data/1` - -* Deprecations - * [Mix] `:env` key in project configuration is deprecated - * [Regex] `Regex.groups/1` is deprecated in favor of `Regex.names/1` - -* Backwards incompatible changes - * [Macro] `Macro.unpipe/1` now returns tuples and `Macro.pipe/2` was removed in favor of `Macro.pipe/3` which explicitly expects the second element of the tuple returned by the new `Macro.unpipe/1` - * [Path] The functions in Path now only emit strings as result, regardless if the input was a char list or a string - * [Path] Atoms are no longer supported in Path functions - * [Regex] Regexes are no longer unicode by default. Instead, they must be explicitly marked with the `u` option - -## v0.13.0 (2014-04-20) - -* Enhancements - * [Base] Add `Base` module which does conversions to bases 16, 32, hex32, 64 and url64 - * [Code] Add `Code.eval_file/2` - * [Collectable] Add the `Collectable` protocol that empowers `Enum.into/2` and `Stream.into/2` and the `:into` option in comprehensions - * [Collectable] Implement `Collectable` for lists, dicts, bitstrings, functions and provide both `File.Stream` and `IO.Stream` - * [EEx] Add `handle_body/1` callback to `EEx.Engine` - * [Enum] Add `Enum.group_by/2`, `Enum.into/2`, `Enum.into/3`, `Enum.traverse/2` and `Enum.sum/2` - * [ExUnit] Randomize cases and tests suite runs, allow seed configuration and the `--seed` flag via `mix test` - * [ExUnit] Support `--only` for filtering when running tests with `mix test` - * [ExUnit] Raise an error if another `capture_io` process already captured the device - * [ExUnit] Improve formatter to show source code and rely on lhs and rhs (instead of expected and actual) - * [IEx] Allow prompt configuration with the `:prompt` option - * [IEx] Use werl on Windows - * [Kernel] Support `ERL_PATH` in `bin/elixir` - * [Kernel] Support interpolation in keyword syntax - * [Map] Add a Map module and support 17.0 maps and structs - * [Mix] Add dependency option `:only` to specify the dependency environment. `mix deps.get` and `mix deps.update` works accross all environment unless `--only` is specified - * [Mix] Add `Mix.Shell.prompt/1` - * [Mix] Ensure the project is compiled in case Mix' CLI cannot find a task - * [Node] Add `Node.ping/1` - * [Process] Include `Process.send/3` and support the `--gen-debug` option - * [Regex] Regexes no longer need the "g" option when there is a need to use named captures - * [Stream] Add `Stream.into/2` and `Stream.into/3` - * [StringIO] Add a `StringIO` module that allows a String to be used as IO device - * [System] Add `System.delete_env/1` to remove a variable from the environment - -* Bug fixes - * [CLI] Ensure `--app` is handled as an atom before processing - * [ExUnit] Ensure `ExUnit.Assertions` does not emit compiler warnings for `assert_receive` - * [Kernel] Ensure the same pid is not queued twice in the parallel compiler - * [Macro] `Macro.to_string/2` considers proper precedence when translating `!(foo > bar)` into a string - * [Mix] Automatically recompile on outdated Elixir version and show proper error messages - * [Mix] Ensure generated `.app` file includes core dependencies - * [Mix] Allow a dependency with no SCM to be overridden - * [Mix] Allow queries in `mix local.install` URL - * [OptionParser] Do not recognize undefined aliases as switches - -* Soft deprecations (no warnings emitted) - * [Kernel] `lc` and `bc` comprehensions are deprecated in favor of `for` - * [ListDict] `ListDict` is deprecated in favor of `Map` - * [Record] `defrecord/2`, `defrecordp/3`, `is_record/1` and `is_record/2` macros in Kernel are deprecated. Instead, use the new macros and API defined in the `Record` module - -* Deprecations - * [Dict] `Dict.empty/1`, `Dict.new/1` and `Dict.new/2` are deprecated - * [Exception] `Exception.normalize/1` is deprecated in favor of `Exception.normalize/2` - -* Backwards incompatible changes - * [ExUnit] Formatters are now required to be a GenEvent and `ExUnit.run/2` returns a map with results - -## v0.12.5 (2014-03-09) - -* Bug fixes - * [Kernel] Ensure `try` does not generate an after clause. Generating an after clause forbade clauses in the `else` part from being tail recursive. This should improve performance and memory consumption of `Stream` functions - * [Mix] Automatically recompile on outdated Elixir version and show proper error messages - -* Deprecations - * [File] `File.stream_to!/3` is deprecated - * [GenFSM] `GenFSM` is deprecated - * [Kernel] `%` for sigils is deprecated in favor of `~` - * [Kernel] `is_range/1` and `is_regex/1` are deprecated in favor of `Range.range?/1` and `Regex.regex?/1` - * [Stream] `Stream.after/1` is deprecated - * [URI] `URI.decode_query/1` is deprecated in favor of `URI.decode_query/2` with explicit dict argument - * [URI] Passing lists as key or values in `URI.encode_query/1` is deprecated - -* Backwards incompatible changes - * [Mix] Remove `MIX_GIT_FORCE_HTTPS` as Git itself already provides mechanisms for doing so - -## v0.12.4 (2014-02-12) - -* Enhancements - * [Mix] `mix deps.get` and `mix deps.update` no longer compile dependencies afterwards. Instead, they mark the dependencies which are going to be automatically compiled next time `deps.check` is invoked (which is done automatically by most mix tasks). This means users should have a better workflow when migrating in between environments - -* Deprecations - * [Kernel] `//` for default arguments is deprecated in favor of `\\` - * [Kernel] Using `%` for sigils is deprecated in favor of `~`. This is a soft deprecation, no warnings will be emitted for it in this release - * [Kernel] Using `^` inside function clause heads is deprecated, please use a guard instead - -* Backwards incompatible changes - * [ExUnit] `CaptureIO` returns an empty string instead of nil when there is no capture - * [Version] The `Version` module now only works with SemVer. The functions `Version.parse/1` and `Version.parse_requirement/1` now return `{:ok,res} | :error` for the cases you want to handle non SemVer cases manually. All other functions will trigger errors on non semantics versions - -## v0.12.3 (2014-02-02) - -* Enhancements - * [Kernel] Warnings now are explicitly tagged with "warning:" in messages - * [Kernel] Explicit functions inlined by the compiler, including operators. This means that `Kernel.+/2` will now expand to `:erlang.+/2` and so on - * [Mix] Do not fail if a Mix dependency relies on an outdated Elixir version - * [Process] Add `Process.send/2` and `Process.send_after/3` - * [Version] Add `Version.compare/2` - -* Bug fixes - * [Atom] Inspect `:...` and `:foo@bar` without quoting - * [Keyword] The list `[1, 2, three: :four]` now correctly expands to `[1, 2, {:three, :four}]` - * [Kernel] Ensure undefined `@attributes` shows proper stacktrace in warnings - * [Kernel] Guarantee nullary funs/macros are allowed in guards - * [Process] Ensure monitoring functions are inlined by the compiler - -* Deprecations - * [IEx] The helper `m/0` has been deprecated. The goal is to group all runtime statistic related helpers into a single module - * [Kernel] `binary_to_term/1`, `binary_to_term/2`, `term_to_binary/1` and `term_to_binary/2` are deprecated in favor of their counterparts in the `:erlang` module - * [Kernel] `//` for default arguments is deprecated in favor of `\\`. This is a soft deprecation, no warnings will be emitted for it in this release - * [Kernel] Deprecated `@behavior` in favor of `@behaviour` - * [Record] `to_keywords`, `getter` and `list getter` functionalities in `defrecordp` are deprecated - * [Record] `Record.import/2` is deprecated - -* Backwards incompatible changes - * [Dict] Implementations of `equal?/2` and `merge/2` in `HashDict` and `ListDict` are no longer polymorphic. To get polymorphism, use the functions in `Dict` instead - * [File] `File.cp/3` and `File.cp_r/3` no longer carry Unix semantics where the function behaves differently if the destination is an existing previous directory or not. It now always copies source to destination, doing it recursively in the latter - * [IEx] IEx now loads the `.iex.exs` file instead of `.iex` - * [Kernel] Remove `**` from the list of allowed operators - * [Kernel] Limit sigils delimiters to one of the following: `<>`, `{}`, `[]`, `()`, `||`, `//`, `"` and `'` - * [Range] `Range` is no longer a record, instead use `first .. last` if you need pattern matching - * [Set] Implementations of `difference/2`, `disjoint?/2`, `equal?/2`, `intersection/2`, `subset?/2` and `union/2` in `HashSet` are no longer polymorphic. To get polymorphism, use the functions in `Set` instead - -## v0.12.2 (2014-01-15) - -* Enhancements - * [EEx] Allow `EEx.AssignsEngine` to accept any Dict - * [Enum] Add `Enum.flat_map_reduce/3` - * [ExUnit] Support `@moduletag` in ExUnit cases - * [Kernel] Improve stacktraces to be relative to the compilation path and include the related application - * [Stream] Add `Stream.transform/3` - -* Bug fixes - * [ExUnit] `:include` in ExUnit only has effect if a test was previously excluded with `:exclude` - * [ExUnit] Only run `setup_all` and `teardown_all` if there are tests in the case - * [Kernel] Ensure bitstring modifier arguments are expanded - * [Kernel] Ensure compiler does not block on missing modules - * [Kernel] Ensure `<>/2` works only with binaries - * [Kernel] Fix usage of string literals inside `<<>>` when `utf8`/`utf16`/`utf32` is used as specifier - * [Mix] Ensure mix properly copies _build dependencies on Windows - -* Deprecations - * [Enum] Deprecate `Enum.first/1` in favor of `Enum.at/2` and `List.first/1` - * [Kernel] Deprecate continuable heredocs. In previous versions, Elixir would continue parsing on the same line the heredoc started, this behaviour has been deprecated - * [Kernel] `is_alive/0` is deprecated in favor of `Node.alive?` - * [Kernel] `Kernel.inspect/2` with `Inspect.Opts[]` is deprecated in favor of `Inspect.Algebra.to_doc/2` - * [Kernel] `Kernel.inspect/2` with `:raw` option is deprecated, use `:records` option instead - * [Kernel] Deprecate `<-/2` in favor of `send/2` - -* Backwards incompatible changes - * [String] Change `String.next_grapheme/1` and `String.next_codepoint/1` to return `nil` on string end - -## v0.12.1 (2014-01-04) - -* Enhancements - * [ExUnit] Support `:include` and `:exclude` configuration options to filter which tests should run based on their tags. Those options are also supported via `mix test` as `--include` and `--exclude` - * [ExUnit] Allow doctests to match against `#MyModule<>` - -* Bug fixes - * [CLI] Abort when a pattern given to elixirc does not match any file - * [Float] Fix `Float.parse/1` to handle numbers of the form "-0.x" - * [IEx] Improve error message for `IEx.Helpers.r` when module does not exist - * [Mix] Ensure `deps.get` updates origin if lock origin and dep origin do not match - * [Mix] Use relative symlinks in _build - * [Typespec] Fix conversion of unary ops from typespec format to ast - * [Typespec] Fix handling of `tuple()` and `{}` - -* Deprecations - * [Kernel] Do not leak clause heads. Previously, a variable defined in a case/receive head clauses would leak to the outer scope. This behaviour is deprecated and will be removed in the next release. - * [Kernel] Deprecate `__FILE__` in favor of `__DIR__` or `__ENV__.file` - -* Backwards incompatible changes - * [GenFSM] GenServer now stops on unknown event/sync_event requests - * [GenServer] GenServer now stops on unknown call/cast requests - * [Kernel] Change how `->` is represented in AST. Now each clause is represented by its own AST node which makes composition easier. See commit 51aef55 for more information. - -## v0.12.0 (2013-12-15) - -* Enhancements - * [Exception] Allow `exception/1` to be overridden and promote it as the main mechanism to customize exceptions - * [File] Add `File.stream_to!/3` - * [Float] Add `Float.floor/1`, `Float.ceil/1` and `Float.round/3` - * [Kernel] Add `List.delete_at/2` and `List.updated_at/3` - * [Kernel] Add `Enum.reverse/2` - * [Kernel] Implement `defmodule/2`, `@/1`, `def/2` and friends in Elixir itself. `case/2`, `try/2` and `receive/1` have been made special forms. `var!/1`, `var!/2` and `alias!/1` have also been implemented in Elixir and demoted from special forms - * [Record] Support dynamic fields in `defrecordp` - * [Stream] Add `Stream.resource/3` - * [Stream] Add `Stream.zip/2`, `Stream.filter_map/3`, `Stream.each/2`, `Stream.take_every/2`, `Stream.chunk/2`, `Stream.chunk/3`, `Stream.chunk/4`, `Stream.chunk_by/2`, `Stream.scan/2`, `Stream.scan/3`, `Stream.uniq/2`, `Stream.after/2` and `Stream.run/1` - * [Stream] Support `Stream.take/2` and `Stream.drop/2` with negative counts - -* Bug fixes - * [HashDict] Ensure a `HashDict` stored in an attribute can be accessed via the attribute - * [Enum] Fix bug in `Enum.chunk/4` where you'd get an extra element when the enumerable was a multiple of the counter and a pad was given - * [IEx] Ensure `c/2` helper works with full paths - * [Kernel] `quote location: :keep` now only affects definitions in order to keep the proper trace in definition exceptions - * [Mix] Also symlink `include` directories in _build dependencies - * [Version] Fix `Version.match?/2` with `~>` and versions with alphanumeric build info (like `-dev`) - -* Deprecations - * [Enum] `Enumerable.count/1` and `Enumerable.member?/2` should now return tagged tuples. Please see `Enumerable` docs for more info - * [Enum] Deprecate `Enum.chunks/2`, `Enum.chunks/4` and `Enum.chunks_by/2` in favor of `Enum.chunk/2`, `Enum.chunk/4` and `Enum.chunk_by/2` - * [File] `File.binstream!/3` is deprecated. Simply use `File.stream!/3` which is able to figure out if `stream` or `binstream` operations should be used - * [Macro] `Macro.extract_args/1` is deprecated in favor of `Macro.decompose_call/1` - -* Backwards incompatible changes - * [Enum] Behaviour of `Enum.drop/2` and `Enum.take/2` has been switched when given negative counts - * [Enum] Behaviour of `Enum.zip/2` has been changed to stop as soon as the first enumerable finishes - * [Enum] `Enumerable.reduce/3` protocol has changed to support suspension. Please see `Enumerable` docs for more info - * [Mix] Require `:escript_main_module` to be set before generating escripts - * [Range] `Range.Iterator` protocol has changed in order to work with the new `Enumerable.reduce/3`. Please see `Range.Iterator` docs for more info - * [Stream] The `Stream.Lazy` structure has changed to accumulate functions and accumulators as we go (its inspected representation has also changed) - * [Typespec] `when` clauses were moved to the outer part of the spec and should be in the keywords format. So `add(a, b) when is_subtype(a, integer) and is_subtype(b, integer) :: integer` should now be written as `add(a, b) :: integer when a: integer, b: integer` - -## v0.11.2 (2013-11-14) - -* Enhancements - * [Mix] Add `mix iex` that redirects users to the proper `iex -S mix` command - * [Mix] Support `build_per_environment: true` in project configuration that manages a separete build per environment, useful when you have per-environment behaviour/compilation - -* Backwards incompatible changes - * [Mix] Mix now compiles files to `_build`. Projects should update just fine, however documentation and books may want to update to the latest information - -## v0.11.1 (2013-11-07) - -* Enhancements - * [Mix] Improve dependency convergence by explicitly checking each requirement instead of expecting all requirements to be equal - * [Mix] Support optional dependencies with `optional: true`. Optional dependencies are downloaded for the current project but they are automatically skipped when such project is used as a dependency - -* Bug fixes - * [Kernel] Set compilation status per ParallelCompiler and not globally - * [Mix] Ensure Mix does not load previous dependencies versions before `deps.get`/`deps.update` - * [Mix] Ensure umbrella apps are sorted before running recursive commands - * [Mix] Ensure umbrella apps run in the same environment as the parent project - * [Mix] Ensure dependency tree is topsorted before compiling - * [Mix] Raise error when duplicated projects are pushed into the stack - * [URI] Allow lowercase escapes in URI - -* Backwards incompatible changes - * [Mix] Setting `:load_paths` in your project configuration is deprecated - -## v0.11.0 (2013-11-02) - -* Enhancements - * [Code] Eval now returns variables from other contexts - * [Dict] Document and enforce all dicts use the match operator (`===`) when checking for keys - * [Enum] Add `Enum.slice/2` with a range - * [Enum] Document and enforce `Enum.member?/2` to use the match operator (`===`) - * [IEx] Split `IEx.Evaluator` from `IEx.Server` to allow custom evaluators - * [IEx] Add support for `IEx.pry` which halts a given process for inspection - * [IO] Add specs and allow some IO APIs to receive any data that implements `String.Chars` - * [Kernel] Improve stacktraces on command line interfaces - * [Kernel] Sigils can now handle balanced tokens as in `%s(f(o)o)` - * [Kernel] Emit warnings when an alias is not used - * [Macro] Add `Macro.pipe/3` and `Macro.unpipe/1` for building pipelines - * [Mix] Allow umbrella children to share dependencies between them - * [Mix] Allow mix to be escriptize'd - * [Mix] Speed mix projects compilation by relying on more manifests information - * [Protocol] Protocols now provide `impl_for/1` and `impl_for!/1` functions which receive a structure and returns its respective implementation, otherwise returns nil or an error - * [Set] Document and enforce all sets use the match operator (`===`) when checking for keys - * [String] Update to Unicode 6.3.0 - * [String] Add `String.slice/2` with a range - -* Bug fixes - * [Exception] Ensure `defexception` fields can be set dynamically - * [Kernel] Guarantee aliases hygiene is respected when the current module name is not known upfront - * [Kernel] `Kernel.access/2` no longer flattens lists - * [Mix] Ensure cyclic dependencies are properly handled - * [String] Implement the extended grapheme cluster algorithm for `String` operations - -* Deprecations - * [Kernel] `pid_to_list/1`, `list_to_pid/1`, `binary_to_atom/2`, `binary_to_existing_atom/2` and `atom_to_binary/2` are deprecated in favor of their counterparts in the `:erlang` module - * [Kernel] `insert_elem/3` and `delete_elem/2` are deprecated in favor of `Tuple.insert_at/3` and `Tuple.delete_at/2` - * [Kernel] Use of `in` inside matches (as in `x in [1,2,3] -> x`) is deprecated in favor of the guard syntax (`x when x in [1,2,3]`) - * [Macro] `Macro.expand_all/2` is deprecated - * [Protocol] `@only` and `@except` in protocols are now deprecated - * [Protocol] Protocols no longer fallback to `Any` out of the box (this functionality needs to be explicitly enabled by setting `@fallback_to_any` to true) - * [String] `String.to_integer/1` and `String.to_float/1` are deprecated in favor of `Integer.parse/1` and `Float.parse/1` - -* Backwards incompatible changes - * [CLI] Reading `.elixirrc` has been dropped in favor of setting env vars - * [Kernel] `Kernel.access/2` now expects the second argument to be a compile time list - * [Kernel] `fn -> end` quoted expression is no longer wrapped in a `do` keyword - * [Kernel] Quoted variables from the same module must be explicitly shared. Previously, if a function returned `quote do: a = 1`, another function from the same module could access it as `quote do: a`. This has been fixed and the variables must be explicitly shared with `var!(a, __MODULE__)` - * [Mix] Umbrella apps now treat children apps as dependencies. This means all dependencies will be checked out in the umbrela `deps` directory. On upgrade, child apps need to point to the umbrella project by setting `deps_path: "../../deps_path", lockfile: "../../mix.lock"` in their project config - * [Process] `Process.group_leader/2` args have been reversed so the "subject" comes first - * [Protocol] Protocol no longer dispatches to `Number`, but to `Integer` and `Float` - -## v0.10.3 (2013-10-02) - -* Enhancements - * [Enum] Add `Enum.take_every/2` - * [IEx] IEx now respects signals sent from the Ctrl+G menu - * [Kernel] Allow documentation for types with `@typedoc` - * [Mix] Allow apps to be selected in umbrella projects - * [Record] Generated record functions `new` and `update` also take options with strings as keys - * [Stream] Add `Stream.unfold/1` - -* Bug fixes - * [Dict] Fix a bug when a HashDict was marked as equal when one was actually a subset of the other - * [EEx] Solve issue where `do` blocks inside templates were not properly aligned - * [ExUnit] Improve checks and have better error reports on poorly aligned doctests - * [Kernel] Fix handling of multiple heredocs on the same line - * [Kernel] Provide better error messages for match, guard and quoting errors - * [Kernel] Make `Kernel.raise/2` a macro to avoid messing up stacktraces - * [Kernel] Ensure `&()` works on quoted blocks with only one expression - * [Mix] Address an issue where a dependency was not compiled in the proper order when specified in different projects - * [Mix] Ensure `compile: false` is a valid mechanism for disabling the compilation of dependencies - * [Regex] Fix bug on `Regex.scan/3` when capturing groups and the regex has no groups - * [String] Fix a bug with `String.split/2` when given an empty pattern - * [Typespec] Guarantee typespecs error reports point to the proper line - -* Deprecations - * [Kernel] The previous partial application syntax (without the `&` operator) has now been deprecated - * [Regex] `Regex.captures/3` is deprecated in favor of `Regex.named_captures/3` - * [String] `String.valid_codepoint?/1` is deprecated in favor of pattern matching with `<<_ :: utf8 >>` - -* Backwards incompatible changes - * [IEx] The `r/0` helper has been removed as it caused surprising behaviour when many modules with dependencies were accumulated - * [Mix] `Mix.Version` was renamed to `Version` - * [Mix] `File.IteratorError` was renamed to `IO.StreamError` - * [Mix] `mix new` now defaults to the `--sup` option, use `--bare` to get the previous behaviour - -## v0.10.2 (2013-09-03) - -* Enhancements - * [CLI] Add `--verbose` to elixirc, which now is non-verbose by default - * [Dict] Add `Dict.Behaviour` as a convenience to create your own dictionaries - * [Enum] Add `Enum.split/2`, `Enum.reduce/2`, `Enum.flat_map/2`, `Enum.chunk/2`, `Enum.chunk/4`, `Enum.chunk_by/2`, `Enum.concat/1` and `Enum.concat/2` - * [Enum] Support negative indices in `Enum.at/fetch/fetch!` - * [ExUnit] Show failures on CLIFormatter as soon as they pop up - * [IEx] Allow for strings in `h` helper - * [IEx] Helpers `r` and `c` can handle erlang sources - * [Integer] Add `odd?/1` and `even?/1` - * [IO] Added support to specifying a number of bytes to stream to `IO.stream`, `IO.binstream`, `File.stream!` and `File.binstream!` - * [Kernel] Include file and line on error report for overriding an existing function/macro - * [Kernel] Convert external functions into quoted expressions. This allows record fields to contain functions as long as they point to an `&Mod.fun/arity` - * [Kernel] Allow `foo?` and `bar!` as valid variable names - * [List] Add `List.replace_at/3` - * [Macro] Improve printing of the access protocol on `Macro.to_string/1` - * [Macro] Add `Macro.to_string/2` to support annotations on the converted string - * [Mix] Automatically recompile a project if the Elixir version changes - * [Path] Add `Path.relative_to_cwd/2` - * [Regex] Allow erlang `re` options when compiling Elixir regexes - * [Stream] Add `Stream.concat/1`, `Stream.concat/2` and `Stream.flat_map/2` - * [String] Add regex pattern support to `String.replace/3` - * [String] Add `String.ljust/2`, `String.rjust/2`, `String.ljust/3` and `String.rjust/3` - * [URI] `URI.parse/1` supports IPv6 addresses - -* Bug fixes - * [Behaviour] Do not compile behaviour docs if docs are disabled on compilation - * [ExUnit] Doctests no longer eat too much space and provides detailed reports for poorly indented lines - * [File] Fix a bug where `File.touch(file, datetime)` was not setting the proper datetime when the file did not exist - * [Kernel] Limit `inspect` results to 50 items by default to avoid printing too much data - * [Kernel] Return a readable error on oversized atoms - * [Kernel] Allow functions ending with `?` or `!` to be captured - * [Kernel] Fix default shutdown of child supervisors to `:infinity` - * [Kernel] Fix regression when calling a function/macro ending with bang, followed by `do/end` blocks - * [List] Fix bug on `List.insert_at/3` that added the item at the wrong position for negative indexes - * [Macro] `Macro.escape/2` can now escape improper lists - * [Mix] Fix `Mix.Version` matching on pre-release info - * [Mix] Ensure `watch_exts` trigger full recompilation on change with `mix compile` - * [Mix] Fix regression on `mix clean --all` - * [String] `String.strip/2` now supports removing unicode characters - * [String] `String.slice/3` still returns the proper result when there is no length to be extracted - * [System] `System.get_env/0` now returns a list of tuples as previously advertised - -* Deprecations - * [Dict] `Dict.update/3` is deprecated in favor of `Dict.update!/3` - * [Enum] `Enum.min/2` and `Enum.max/2` are deprecated in favor of `Enum.min_by/2` and `Enum.max_by/2` - * [Enum] `Enum.join/2` and `Enum.map_join/3` with a char list are deprecated - * [IO] `IO.stream(device)` and `IO.binstream(device)` are deprecated in favor of `IO.stream(device, :line)` and `IO.binstream(device, :line)` - * [Kernel] `list_to_binary/1`, `binary_to_list/1` and `binary_to_list/3` are deprecated in favor of `String.from_char_list!/1` and `String.to_char_list!/1` for characters and `:binary.list_to_bin/1`, `:binary.bin_to_list/1` and `:binary.bin_to_list/3` for bytes - * [Kernel] `to_binary/1` is deprecated in favor of `to_string/1` - * [Kernel] Deprecate `def/4` and friends in favor of `def/2` with unquote and friends - * [Kernel] Deprecate `%b` and `%B` in favor of `%s` and `%S` - * [List] `List.concat/2` is deprecated in favor of `Enum.concat/2` - * [Macro] `Macro.unescape_binary/1` and `Macro.unescape_binary/2` are deprecated in favor of `Macro.unescape_string/1` and `Macro.unescape_string/2` - * [Mix] `:umbrella` option for umbrella paths has been deprecated in favor of `:in_umbrella` - -* Backwards incompatible changes - * [IO] IO functions now only accept iolists as arguments - * [Kernel] `Binary.Chars` was renamed to `String.Chars` - * [Kernel] The previous ambiguous import syntax `import :functions, Foo` was removed in favor of `import Foo, only: :functions` - * [OptionParser] `parse` and `parse_head` now returns a tuple with three elements instead of two - -## v0.10.1 (2013-08-03) - -* Enhancements - * [Behaviour] Add support for `defmacrocallback/1` - * [Enum] Add `Enum.shuffle/1` - * [ExUnit] The `:trace` option now also reports run time for each test - * [ExUnit] Add support for `:color` to enable/disable ANSI coloring - * [IEx] Add the `clear` helper to clear the screen. - * [Kernel] Add the capture operator `&` - * [Kernel] Add support for `GenFSM.Behaviour` - * [Kernel] Functions now points to the module and function they were defined when inspected - * [Kernel] A documentation attached to a function that is never defined now prints warnings - * [List] Add `List.keysort/2` - * [Mix] `:test_helper` project configuration did not affect `mix test` and was therefore removed. A `test/test_helper.exs` file is still necessary albeit it doesn't need to be automatically required in each test file - * [Mix] Add manifests for yecc, leex and Erlang compilers, making it easier to detect dependencies in between compilers and providing a more useful clean behaviour - * [Mix] `mix help` now outputs information about the default mix task - * [Mix] Add `--no-deps-check` option to `mix run`, `mix compile` and friends to not check dependency status - * [Mix] Add support for `MIX_GIT_FORCE_HTTPS` system environment that forces HTTPS for known providers, useful when the regular git port is blocked. This configuration does not affect the `mix.lock` results - * [Mix] Allow coverage tool to be pluggable via the `:test_coverage` configuration - * [Mix] Add `mix cmd` as a convenience to run a command recursively in child apps in an umbrella application - * [Mix] Support `umbrella: true` in dependencies as a convenience for setting up umbrella path deps - * [Mix] `mix run` now behaves closer to the `elixir` command and properly mangles the ARGV - * [String] Add `Regex.scan/3` now supports capturing groups - * [String] Add `String.reverse/1` - -* Bug fixes - * [Behaviour] Ensure callbacks are stored in the definition order - * [CLI] Speed up boot time on Elixir .bat files - * [IEx] Reduce cases where IEx parser can get stuck - * [Kernel] Improve error messages when the use of an operator has no effect - * [Kernel] Fix a bug where warnings were not being generated when imported macros conflicted with local functions or macros - * [Kernel] Document that `on_definition` can only be a function as it is evaluated inside the function context - * [Kernel] Ensure `%w` sigils with no interpolation are fully expanded at compile time - * [Mix] `mix deps.update`, `mix deps.clean` and `mix deps.unlock` no longer change all dependencies unless `--all` is given - * [Mix] Always run ` mix loadpaths` on `mix app.start`, even if `--no-compile` is given - * [OptionParser] Do not add boolean flags to the end result if they were not given - * [OptionParser] Do not parse non-boolean flags as booleans when true or false are given - * [OptionParser] Ensure `:keep` and `:integer`|`:float` can be given together as options - * [OptionParser] Ensure `--no-flag` sets `:flag` to false when `:flag` is a registered boolean switch - -* Deprecations - * [Kernel] `function(Mod.fun/arity)` and `function(fun/arity)` are deprecated in favor of `&Mod.fun/arity` and `&fun/arity` - * [Kernel] `function/3` is deprecated in favor of `Module.function/3` - * [Kernel] `Kernel.ParallelCompiler` now receives a set of callbacks instead of a single one - * [Mix] `:test_coverage` option now expect keywords arguments and the `--cover` flag is now treated as a boolean - -* Backwards incompatible changes - * [Regex] `Regex.scan/3` now always returns a list of lists, normalizing the result, instead of list with mixed lists and binaries - * [System] `System.halt/2` was removed since the current Erlang implementation of such function is bugged - -## v0.10.0 (2013-07-15) - -* Enhancements - * [ExUnit] Support `trace: true` option which gives detailed reporting on test runs - * [HashDict] Optimize `HashDict` to store pairs in a cons cell reducing storage per key by half - * [Kernel] Add pretty printing support for inspect - * [Kernel] Add document algebra library used as the foundation for pretty printing - * [Kernel] Add `defrecordp/3` that enables specifying the first element of the tuple - * [Kernel] Add the `Set` API and a hash based implementation via `HashSet` - * [Kernel] Add `Stream` as composable, lazy-enumerables - * [Mix] `mix archive` now includes the version of the generated archive - * [Mix] Mix now requires explicit dependency overriding to be given with `override: true` - * [Mix] Projects can now define an `:elixir` key to outline supported Elixir versions - * [Typespec] Improve error messages to contain file, line and the typespec itself - -* Bug fixes - * [CLI] Elixir can now run on Unix directories with `:` in its path - * [Kernel] `match?/2` does not leak variables to outer scope - * [Kernel] Keep `head|tail` format when splicing at the tail - * [Kernel] Ensure variables defined in the module body are not passed to callbacks - * [Mix] On dependencies conflict, show from where each source is coming from - * [Mix] Empty projects no longer leave empty ebin files on `mix compile` - * [Module] Calling `Module.register_attribute/3` no longer automatically changes it to persisted or accumulated - -* Deprecations - * [Enum] Receiving the index of iteration in `Enum.map/2` and `Enum.each/2` is deprecated in favor of `Stream.with_index/1` - * [File] `File.iterator/1` and `File.biniterator/1` are deprecated in favor of `IO.stream/1` and `IO.binstream/1` - * [File] `File.iterator!/2` and `File.biniterator!/2` are deprecated in favor of `File.stream!/2` and `File.binstream!/2` - * [Kernel] Deprecate recently added `quote binding: ...` in favor of the clearer `quote bind_quoted: ...` - * [Kernel] Deprecate `Kernel.float/1` in favor of a explicit conversion - * [Mix] Deprecate `mix run EXPR` in favor of `mix run -e EXPR` - * [Record] `Record.__index__/2` deprecated in favor of `Record.__record__(:index, key)` - -* Backwards incompatible changes - * [Kernel] The `Binary.Inspect` protocol has been renamed to `Inspect` - * [Kernel] Tighten up the grammar rules regarding parentheses omission, previously the examples below would compile but now they raise an error message: - - do_something 1, is_list [], 3 - [1, is_atom :foo, 3] - - * [Module] Calling `Module.register_attribute/3` no longer automatically changes it to persisted or accumulated - * [Record] First element of a record via `defrecordp` is now the `defrecordp` name and no longer the current atom - * [URI] Remove custom URI parsers in favor of `URI.default_port/2` - -## v0.9.3 (2013-06-23) - -* Enhancements - * [File] Add `File.chgrp`, `File.chmod` and `File.chown` - * [Kernel] Add `--warnings-as-errors` to Elixir's compiler options - * [Kernel] Print warnings to stderr - * [Kernel] Warn on undefined module attributes - * [Kernel] Emit warning for `x in []` in guards - * [Kernel] Add `binding/0` and `binding/1` for retrieving bindings - * [Kernel] `quote` now allows a binding as an option - * [Macro] Add `Macro.expand_once/2` and `Macro.expand_all/2` - * [Mix] Implement `Mix.Version` for basic versioning semantics - * [Mix] Support creation and installation of archives (.ez files) - * [Mix] `github: ...` shortcut now uses the faster `git` schema instead of `https` - * [Record] Allow types to be given to `defrecordp` - -* Bug fixes - * [Kernel] The elixir executable on Windows now supports the same options as the UNIX one - * [Kernel] Improve error messages on default clauses clash - * [Kernel] `__MODULE__.Foo` now returns `Foo` when outside of a Module - * [Kernel] Improve error messages when default clauses from different definitions collide - * [Kernel] `^x` variables should always refer to the value before the expression - * [Kernel] Allow `(x, y) when z` in function clauses and try expressions - * [Mix] Mix now properly evaluates rebar scripts - -* Deprecations - * [Code] `Code.string_to_ast/1` has been deprecated in favor of `Code.string_to_quoted/1` - * [Macro] `Macro.to_binary/1` has been deprecated in favor of `Macro.to_string/1` - * [Typespec] Deprecate `(fun(...) -> ...)` in favor of `(... -> ...)` - -* Backwards incompatible changes - * [Bitwise] Precedence of operators used by the Bitwise module were changed, check `elixir_parser.yrl` for more information - * [File] `rm_rf` and `cp_r` now returns a tuple with three elements on failures - * [Kernel] The quoted representation for `->` clauses changed from a tuple with two elements to a tuple with three elements to support metadata - * [Kernel] Sigils now dispatch to `sigil_$` instead of `__$__` where `$` is the sigil character - * [Macro] `Macro.expand/2` now expands until final form. Although this is backwards incompatible, it is very likely you do not need to change your code, since expansion until its final form is recommended, particularly if you are expecting an atom out of it - * [Mix] No longer support beam files on `mix local` - -## v0.9.2 (2013-06-13) - -* Enhancements - * [ExUnit] `capture_io` now captures prompt by default - * [Mix] Automatically import git dependencies from Rebar - * [Mix] Support for dependencies directly from the umbrella application - * [Regex] Add `Regex.escape` - * [String] Add `String.contains?` - * [URI] Implement `Binary.Chars` (aka `to_binary`) for `URI.Info` - -* Bug fixes - * [HashDict] Ensure HashDict uses exact match throughout its implementation - * [IEx] Do not interpret ANSI codes in IEx results - * [IEx] Ensure `--cookie` is set before accessing remote shell - * [Kernel] Do not ignore nil when dispatching protocols to avoid infinite loops - * [Mix] Fix usage of shell expressions in `Mix.Shell.cmd` - * [Mix] Start the application by default on escripts - -* Deprecations - * [Regex] `Regex.index/2` is deprecated in favor `Regex.run/3` - * [Kernel] `super` no longer supports implicit arguments - -* Backwards incompatible changes - * [Kernel] The `=~` operator now returns true or false instead of an index - -## v0.9.1 (2013-05-30) - -* Enhancements - * [IEx] Limit the number of entries kept in history and allow it to be configured - * [Kernel] Add `String.start_with?` and `String.end_with?` - * [Typespec] Allow keywords, e.g. `[foo: integer, bar: boolean | module]`, in typespecs - -* Bug fixes - * [Dict] `Enum.to_list` and `Dict.to_list` now return the same results for dicts - * [IEx] Enable shell customization via the `IEx.Options` module - * [Kernel] Fix a bug where `unquote_splicing` did not work on the left side of a stab op - * [Kernel] Unused functions with cyclic dependencies are now also warned as unused - * [Mix] Fix a bug where `mix deps.get` was not retrieving nested dependencies - * [Record] Fix a bug where nested records cannot be defined - * [Record] Fix a bug where a record named Record cannot be defined - -## v0.9.0 (2013-05-23) - -* Enhancements - * [ExUnit] `ExUnit.CaptureIO` now accepts an input to be used during capture - * [IEx] Add support for .iex files that are loaded during shell's boot process - * [IEx] Add `import_file/1` helper - -* Backwards incompatible changes - * [Enum] `Enum.Iterator` was replaced by the more composable and functional `Enumerable` protocol which supports reductions - * [File] `File.iterator/1` and `File.biniterator/1` have been removed in favor of the safe `File.iterator!/1` and `File.biniterator!/1` ones - * [Kernel] Erlang R15 is no longer supported - * [Kernel] Elixir modules are now represented as `Elixir.ModuleName` (using `.` instead of `-` as separator) - -## v0.8.3 (2013-05-22) - -* Enhancements - * [CLI] Flags `-p` and `-pr` fails if pattern match no files - * [CLI] Support `--hidden` and `--cookie` flags for distributed Erlang - * [Enum] Add `Enum.to_list/1`, `Enum.member?/2`, `Enum.uniq/2`, `Enum.max/1`, `Enum.max/2`, `Enum.min/1` and `Enum.min/2` - * [ExUnit] Add `ExUnit.CaptureIO` for IO capturing during tests - * [ExUnit] Consider load time on ExUnit time reports - * [IEx] Support `ls` with colored output - * [IEx] Add `#iex:break` to break incomplete expressions - * [Kernel] Add `Enum.at`, `Enum.fetch` and `Enum.fetch!` - * [Kernel] Add `String.to_integer` and `String.to_float` - * [Kernel] Add `Dict.take`, `Dict.drop`, `Dict.split`, `Dict.pop` and `Dict.fetch!` - * [Kernel] Many optimizations for code compilation - * [Kernel] `in` can be used with right side expression outside guards - * [Kernel] Add `Node.get_cookie/0` and `Node.set_cookie/2` - * [Kernel] Add `__DIR__` - * [Kernel] Expand macros and attributes on quote, import, alias and require - * [Kernel] Improve warnings related to default arguments - * [Keyword] Add `Keyword.delete_first/2` - * [Mix] Add `local.rebar` to download a local copy of rebar, and change `deps.compile` to use it if needed - * [Mix] Support umbrella applications - * [Mix] Load beam files available at `MIX_PATH` on CLI usage - * [String] Add `String.valid?` and `String.valid_character?` - -* Bug fixes - * [ExUnit] Handle exit messages from in ExUnit - * [ExUnit] Failures on ExUnit's setup_all now invalidates all tests - * [Kernel] Ensure we don't splice keyword args unecessarily - * [Kernel] Private functions used by private macros no longer emit an unused warning - * [Kernel] Ensure Elixir won't trip on empty receive blocks - * [Kernel] `String.slice` now returns an empty string when out of range by 1 - * [Mix] Generate manifest files after compilation to avoid depending on directory timestamps and to remove unused .beam files - * [Path] `Path.expand/2` now correctly expands `~` in the second argument - * [Regex] Fix badmatch with `Regex.captures(%r/(.)/g, "cat")` - * [URI] Downcase host and scheme and URIs - -* Deprecations - * [Code] `Code.eval` is deprecated in favor of `Code.eval_string` - * [Exception] `Exception.format_entry` is deprecated in favor of `Exception.format_stacktrace_entry` - * [ExUnit] `assert left inlist right` is deprecated in favor of `assert left in right` - * [IO] `IO.getb` is deprecated in favor of `IO.getn` - * [List] `List.member?/2` is deprecated in favor of `Enum.member?/2` - * [Kernel] `var_context` in quote was deprecated in favor of `context` - * [Kernel] `Enum.at!` and `Dict.get!` is deprecated in favor of `Enum.fetch!` and `Dict.fetch!` - -* Backwards incompatible changes - * [Dict] `List.Dict` was moved to `ListDict` - * [IO] `IO.gets`, `IO.getn` and friends now return binaries when reading from stdio - * [Kernel] Precedence of `|>` has changed to lower to support constructs like `1..5 |> Enum.to_list` - * [Mix] `mix escriptize` now receives arguments as binaries - -## v0.8.2 (2013-04-20) - -* Enhancements - * [ExUnit] Use ANSI escape codes in CLI output - * [ExUnit] Include suite run time on CLI results - * [ExUnit] Add support to doctests, allowing test cases to be generated from code samples - * [File] Add `File.ls` and `File.ls!` - * [IEx] Support `pwd` and `cd` helpers - * [Kernel] Better error reporting for invalid bitstring generators - * [Kernel] Improve meta-programming by allowing `unquote` on `def/2`, `defp/2`, `defmacro/2` and `defmacrop/2` - * [Kernel] Add support to R16B new functions: `insert_elem/3` and `delete_elem/2` - * [Kernel] Import conflicts are now lazily handled. If two modules import the same functions, it will fail only if the function is invoked - * [Mix] Support `--cover` on mix test and `test_coverage` on Mixfiles - * [Record] Each record now provides `Record.options` with the options supported by its `new` and `update` functions - -* Bug fixes - * [Binary] inspect no longer escapes standalone hash `#` - * [IEx] The `h` helper can now retrieve docs for special forms - * [Kernel] Record optimizations were not being triggered in functions inside the record module - * [Kernel] Aliases defined inside macros should be carried over - * [Kernel] Fix a bug where nested records could not use the Record[] syntax - * [Path] Fix a bug on `Path.expand` when expanding paths starting with `~` - -* Deprecations - * [Kernel] `setelem/3` is deprecated in favor of `set_elem/3` - * [Kernel] `function(:is_atom, 1)` is deprecated in favor of `function(is_atom/1)` - -* Backwards incompatible changes - * [Kernel] `unquote` now only applies to the closest quote. If your code contains a quote that contains another quote that calls unquote, it will no longer work. Use `Macro.escape` instead and pass your quoted contents up in steps, for example: - - quote do - quote do: unquote(x) - end - - should become: - - quote do - unquote(Macro.escape(x)) - end - -## v0.8.1 (2013-02-17) - -* Enhancements - * [ExUnit] Tests can now receive metadata set on setup/teardown callbacks - * [ExUnit] Add support to ExUnit.CaseTemplate to share callbacks in between test cases - * [IO] Add `IO.ANSI` to make it easy to write ANSI escape codes - * [Kernel] Better support for Unicode lists - * [Kernel] Reduce variables footprint in `case`/`receive` clauses - * [Kernel] Disable native compilation when on_load attributes is present to work around an Erlang bug - * [Macro] `Macro.expand` also considers macros from the current `__ENV__` module - * [Mix] Improve support for compilation of `.erl` files - * [Mix] Add support for compilation of `.yrl` and `.xrl` files - * [OptionParser] Switches are now overridden by default but can be kept in order if chosen - * [Typespec] Better error reporting for invalid typespecs - -* Bug fixes - * [Mix] Allow Mix projects to be generated with just one letter - -* Backwards incompatible changes - * [Kernel] `before_compile` and `after_compile` callbacks now receive the environment as first argument instead of the module - -* Deprecations - * [ExUnit] Explicitly defined test/setup/teardown functions are deprecated - * [Kernel] Tidy up and clean `quote` API - * [Kernel] Old `:local.(args)` syntax is deprecated - * [Process] `Process.self` is deprecated in favor `Kernel.self` - -## v0.8.0 (2013-01-28) - -* Enhancements - * [Binary] Support `<< "string" :: utf8 >>` as in Erlang - * [Binary] Support `\a` escape character in binaries - * [Binary] Support syntax shortcut for specifying size in bit syntax - * [CLI] Support `--app` option to start an application and its dependencies - * [Dict] Support `put_new` in `Dict` and `Keyword` - * [Dict] Add `ListDict` and a faster `HashDict` implementation - * [ExUnit] ExUnit now supports multiple runs in the same process - * [ExUnit] Failures in ExUnit now shows a tailored stacktrace - * [ExUnit] Introduce `ExUnit.ExpectationError` to provide better error messages - * [Kernel] Introduce `Application.Behaviour` to define application module callbacks - * [Kernel] Introduce `Supervisor.Behaviour` to define supervisors callbacks - * [Kernel] More optimizations were added to Record handling - * [Kernel] `?\x` and `?\` are now supported ways to retrieve a codepoint - * [Kernel] Octal numbers can now be defined as `0777` - * [Kernel] Improve macros hygiene regarding variables, aliases and imports - * [Mix] Mix now starts the current application before run, iex, test and friends - * [Mix] Mix now provides basic support for compiling `.erl` files - * [Mix] `mix escriptize` only generates escript if necessary and accept `--force` and `--no-compile` as options - * [Path] Introduce `Path` module to hold filesystem paths related functions - * [String] Add `String.capitalize` and `String.slice` - * [System] Add `System.tmp_dir`, `System.cwd` and `System.user_home` - -* Bug fixes - * [Kernel] `import` with `only` accepts functions starting with underscore - * [String] `String.first` and `String.last` return nil for empty binaries - * [String] `String.rstrip` and `String.lstrip` now verify if argument is a binary - * [Typespec] Support `...` inside typespec's lists - -* Backwards incompatible changes - * [Kernel] The AST now allows metadata to be attached to each node. This means the second item in the AST is no longer an integer (representing the line), but a keywords list. Code that relies on the line information from AST or that manually generate AST nodes need to be properly updated - -* Deprecations - * [Dict] Deprecate `Binary.Dict` and `OrdDict` in favor of `HashDict` and `ListDict` - * [File] Deprecate path related functions in favor of the module `Path` - * [Kernel] The `/>` operator has been deprecated in favor of `|>` - * [Mix] `Mix.Project.sources` is deprecated in favor of `Mix.Project.config_files` - * [Mix] `mix iex` is no longer functional, please use `iex -S mix` - * [OptionParser] `:flags` option was deprecated in favor of `:switches` to support many types - -## v0.7.2 (2012-12-04) - -* Enhancements - * [CLI] `--debug-info` is now true by default - * [ExUnit] Make ExUnit exit happen in two steps allowing developers to add custom `at_exit` hooks - * [IEx] Many improvements to helpers functions `h/1`, `s/1` and others - * [Kernel] Functions defined with `fn` can now handle many clauses - * [Kernel] Raise an error if clauses with different arities are defined in the same function - * [Kernel] `function` macro now accepts arguments in `M.f/a` and `f/a` formats - * [Macro] Improvements to `Macro.to_binary` - * [Mix] Mix now echoes the output as it comes when executing external commands such as git or rebar - * [Mix] Mix now validates `application` callback's values - * [Record] Record accessors are now optimized and can be up to 6x faster in some cases - * [String] Support `\xXX` and `\x{HEX}` escape sequences in strings, char lists and regexes - -* Bug fixes - * [Bootstrap] Compiling Elixir source no longer fails if environment variables contain utf-8 entries - * [IEx] IEx will now wait for all command line options to be processed before starting - * [Kernel] Ensure proper stacktraces when showing deprecations - -* Deprecations - * [Enum] `Enum.qsort` is deprecated in favor of `Enum.sort` - * [List] `List.sort` and `List.uniq` have been deprecated in favor of their `Enum` counterparts - * [Record] Default-based generated functions are deprecated - * [Typespec] Enhancements and deprecations to the `@spec/@callback` and the fun type syntax - -## v0.7.1 (2012-11-18) - -* Enhancements - * [IEx] Only show documented functions and also show docs for default generated functions - * [IO] Add `IO.binread`, `IO.binwrite` and `IO.binreadline` to handle raw binary file operations - * [ExUnit] Add support for user configuration at `HOME/.ex_unit.exs` - * [ExUnit] Add support for custom formatters via a well-defined behaviour - * [Kernel] Add support for `defrecordp` - * [Kernel] Improved dialyzer support - * [Kernel] Improved error messages when creating functions with aliases names - * [Mix] Improve SCM behaviour to allow more robust integration - * [Mix] Changing deps information on `mix.exs` forces users to fetch new dependencies - * [Mix] Support (parallel) requires on mix run - * [Mix] Support `-q` when running tests to compile only changed files - * [String] Support `String.downcase` and `String.upcase` according to Unicode 6.2.0 - * [String] Add support for graphemes in `String.length`, `String.at` and others - * [Typespec] Support `@opaque` as attribute - * [Typespec] Define a default type `t` for protocols and records - * [Typespec] Add support for the access protocol in typespecs - -* Bug fixes - * [Kernel] Fix an issue where variables inside clauses remained unassigned - * [Kernel] Ensure `defoverridable` functions can be referred in many clauses - * [Kernel] Allow keywords as function names when following a dot (useful when integrating with erlang libraries) - * [File] File is opened by default on binary mode instead of utf-8 - -* Deprecations - * [Behaviour] `defcallback/1` is deprecated in favor of `defcallback/2` which matches erlang `@callbacks` - * [Enum] `Enum.times` is deprecated in favor of using ranges - * [System] `halt` moved to `System` module - -## v0.7.0 (2012-10-20) - -* Enhancements - * [Behaviour] Add Behaviour with a simple callback DSL to define callbacks - * [Binary] Add a Dict binary that converts its keys to binaries on insertion - * [Binary] Optimize `Binary.Inspect` and improve inspect for floats - * [CLI] Support `--detached` option - * [Code] `Code.string_to_ast` supports `:existing_atoms_only` as an option in order to guarantee no new atoms is generated when parsing the code - * [EEx] Support `<%%` and `<%#` tags - * [ExUnit] Support `after_spawn` callbacks which are invoked after each process is spawned - * [ExUnit] Support context data in `setup_all`, `setup`, `teardown` and `teardown_all` callbacks - * [IEx] Support `after_spawn` callbacks which are invoked after each process is spawned - * [Kernel] Better error messages when invalid options are given to `import`, `alias` or `require` - * [Kernel] Allow partial application on literals, for example: `{&1, &2}` to build tuples or `[&1|&2]` to build cons cells - * [Kernel] Added `integer_to_binary` and `binary_to_integer` - * [Kernel] Added `float_to_binary` and `binary_to_float` - * [Kernel] Many improvements to `unquote` and `unquote_splicing`. For example, `unquote(foo).unquote(bar)(args)` is supported and no longer need to be written via `apply` - * [Keyword] Keyword list is no longer ordered according to Erlang terms but the order in which they are specified - * [List] Add `List.keyreplace` and `List.keystore` - * [Macro] Support `Macro.safe_term` which returns `:ok` if an expression does not execute code and is made only of raw data types - * [Mix] Add support for environments - the current environment can be set via `MIX_ENV` - * [Mix] Add support for handling and fetching dependencies' dependencies - * [Module] Support module creation via `Module.create` - * [Range] Support decreasing ranges - * [Record] Improvements to the Record API, added `Record.defmacros` - * [Regex] Add `:return` option to `Regex.run` and `Regex.scan` - * [String] Add a String module responsible for handling UTf-8 binaries - -* Bug fixes - * [File] `File.cp` and `File.cp_r` now preserves the file's mode - * [IEx] Fix a bug where printing to `:stdio` on `IEx` was causing it to hang - * [Macro] Fix a bug where quoted expressions were not behaving the same as their non-quoted counterparts - * [Mix] `mix deps.get [DEPS]` now only gets the specified dependencies - * [Mix] Mix now exits with status 1 in case of failures - * [Protocol] Avoid false positives on protocol dispatch (a bug caused the dispatch to be triggered to an invalid protocol) - -* Backwards incompatible changes - * [ExUnit] `setup` and `teardown` callbacks now receives the test name as second argument - * [Kernel] Raw function definition with `def/4`, `defp/4`, `defmacro/4`, `defmacrop/4` now evaluates all arguments. The previous behaviour was accidental and did not properly evaluate all arguments - * [Kernel] Change tuple-related (`elem` and `setelem`), Enum functions (`find_index`, `nth!` and `times`) and List functions (List.key*) to zero-index - -* Deprecations - * [Code] `Code.require_file` and `Code.load_file` now expect the full name as argument - * [Enum] `List.reverse/1` and `List.zip/2` were moved to `Enum` - * [GenServer] Rename `GenServer.Behavior` to `GenServer.Behaviour` - * [Kernel] Bitstring syntax now uses `::` instead of `|` - * [Kernel] `Erlang.` syntax is deprecated in favor of simply using atoms - * [Module] `Module.read_attribute` and `Module.add_attribute` deprecated in favor of `Module.get_attribute` and `Module.put_attribute` which mimics Dict API - -## v0.6.0 (2012-08-01) - -* Backwards incompatible changes - * [Kernel] Compile files now follow `Elixir-ModuleName` convention to solve issues with Erlang embedded mode. This removes the `__MAIN__` pseudo-variable as modules are now located inside `Elixir` namespace - * [Kernel] `__using__` callback triggered by `use` now receives just one argument. Caller information can be accessed via macros using `__CALLER__` - * [Kernel] Comprehensions syntax changed to be more compatible with Erlang behaviour - * [Kernel] loop and recur are removed in favor of recursion with named functions - * [Module] Removed data functions in favor of unifying the attributes API - -* Deprecations - * [Access] The semantics of the access protocol were reduced from a broad query API to simple data structure key-based access - * [ExUnit] Some assertions are deprecated in favor of simply using `assert()` - * [File] `File.read_info` is deprecated in favor of `File.stat` - * [IO] `IO.print` is deprecated in favor of `IO.write` - * [Kernel] Deprecate `__LINE__` and `__FUNCTION__` in favor of `__ENV__.line` and `__ENV__.function` - * [Kernel] Deprecate `in_guard` in favor of `__CALLER__.in_guard?` - * [Kernel] `refer` is deprecated in favor of `alias` - * [Module] `Module.add_compile_callback(module, target, callback)` is deprecated in favor of `Module.put_attribute(module, :before_compile, {target, callback})` - * [Module] `Module.function_defined?` is deprecated in favor of `Module.defines?` - * [Module] `Module.defined_functions` is deprecated in favor of `Module.definitions_in` - -* Enhancements - * [Enum] Enhance Enum protocol to support `Enum.count` - * [Enum] Optimize functions when a list is given as collection - * [Enum] Add `find_index`, `nth!` and others - * [ExUnit] Support setup and teardown callbacks - * [IEx] IEx now provides autocomplete if the OS supports tty - * [IEx] IEx now supports remsh - * [IEx] Elixir now defaults to compile with documentation and `d` can be used in IEx to print modules and functions documentation - * [IEx] Functions `c` and `m` are available in IEx to compile and print available module information. Functions `h` and `v` are available to show history and print previous commands values - * [IO/File] Many improvements to `File` and `IO` modules - * [Kernel] Operator `!` is now allowed in guard clauses - * [Kernel] Introduce operator `=~` for regular expression matches - * [Kernel] Compiled docs now include the function signature - * [Kernel] `defmodule` do not start a new variable scope, this improves meta-programming capabilities - * [Kernel] quote special form now supports line and unquote as options - * [Kernel] Document the macro `@` and allow attributes to be read inside functions - * [Kernel] Add support to the `%R` sigil. The same as `%r`, but without interpolation or escaping. Both implementations were also optimized to generate the regex at compilation time - * [Kernel] Add `__ENV__` which returns a `Macro.Env` record with information about the compilation environment - * [Kernel] Add `__CALLER__` inside macros which returns a `Macro.Env` record with information about the calling site - * [Macro] Add `Macro.expand`, useful for debugging what a macro expands to - * [Mix] First Mix public release - * [Module] Add support to `@before_compile` and `@after_compile` callbacks. The first receives the module name while the latter receives the module name and its object code - * [OptionParser] Make OptionParser public, add support to flags and improved switch parsing - * [Range] Add a Range module with support to `in` operator (`x in 1..3`) and iterators - * [Record] Allow `Record[_: value]` to set a default value to all records fields, as in Erlang - * [Record] Records now provide a `to_keywords` function - * [Regex] Back references are now properly supported - * [System] Add `System.find_executable` - -## v0.5.0 (2012-05-24) - -* First official release +The CHANGELOG for v1.19 releases can be found [in the v1.19 branch](https://github.com/elixir-lang/elixir/blob/v1.19/CHANGELOG.md). diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 00000000000..e1505186946 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,72 @@ + + +# Code of Conduct + +Contact: + +## Why have a Code of Conduct? + +As contributors and maintainers of this project, we are committed to providing a friendly, safe and welcoming environment for all, regardless of age, disability, gender, nationality, race, religion, sexuality, or similar personal characteristic. + +The goal of the Code of Conduct is to specify a baseline standard of behavior so that people with different social values and communication styles can talk about Elixir effectively, productively, and respectfully, even in face of disagreements. The Code of Conduct also provides a mechanism for resolving conflicts in the community when they arise. + +## Our Values + +These are the values Elixir developers should aspire to: + + * Be friendly and welcoming + * Be kind + * Remember that people have varying communication styles and that not everyone is using their native language. (Meaning and tone can be lost in translation.) + * Interpret the arguments of others in good faith, do not seek to disagree. + * When we do disagree, try to understand why. + * Be thoughtful + * Productive communication requires effort. Think about how your words will be interpreted. + * Remember that sometimes it is best to refrain entirely from commenting. + * Be respectful + * In particular, respect differences of opinion. It is important that we resolve disagreements and differing views constructively. + * Be constructive + * Avoid derailing: stay on topic; if you want to talk about something else, start a new conversation. + * Avoid unconstructive criticism: don't merely decry the current state of affairs; offer — or at least solicit — suggestions as to how things may be improved. + * Avoid harsh words and stern tone: we are all aligned towards the well-being of the community and the progress of the ecosystem. Harsh words exclude, demotivate, and lead to unnecessary conflict. + * Avoid snarking (pithy, unproductive, sniping comments). + * Avoid microaggressions (brief and commonplace verbal, behavioral and environmental indignities that communicate hostile, derogatory or negative slights and insults towards a project, person or group). + * Be responsible + * What you say and do matters. Take responsibility for your words and actions, including their consequences, whether intended or otherwise. + +The following actions are explicitly forbidden: + + * Insulting, demeaning, hateful, or threatening remarks. + * Discrimination based on age, disability, gender, nationality, race, religion, sexuality, or similar personal characteristic. + * Bullying or systematic harassment. + * Unwelcome sexual advances. + * Incitement to any of these. + +## Where does the Code of Conduct apply? + +If you participate in or contribute to the Elixir ecosystem in any way, you are encouraged to follow the Code of Conduct while doing so. + +Explicit enforcement of the Code of Conduct applies to the official mediums operated by the Elixir project: + + * The [official GitHub projects][1] and code reviews. + * The official elixir-lang mailing lists. + * The **[#elixir][2]** IRC channel on [Libera.Chat][3]. + +Other Elixir activities (such as conferences, meetups, and unofficial forums) are encouraged to adopt this Code of Conduct. Such groups must provide their own contact information. + +Project maintainers may block, remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct. + +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by emailing: . All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. **All reports will be kept confidential**. + +**The goal of the Code of Conduct is to resolve conflicts in the most harmonious way possible**. We hope that in most cases issues may be resolved through polite discussion and mutual agreement. Bannings and other forceful measures are to be employed only as a last resort. **Do not** post about the issue publicly or try to rally sentiment against a particular individual or group. + +## Acknowledgements + +This document was based on the Code of Conduct from the Go project (dated Sep/2021) and the Contributor Covenant (v1.4). + +[1]: https://github.com/elixir-lang/ +[2]: https://web.libera.chat/#elixir +[3]: https://libera.chat/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7497c70c66a..c76fdfd97f7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,286 +1,205 @@ -# Contributing to Elixir - -Please take a moment to review this document in order to make the contribution -process easy and effective for everyone involved! - -## Using the issue tracker - -Use the issues tracker for: - -* [bug reports](#bugs-reports) -* [submitting pull requests](#pull-requests) - -Please **do not** use the issues tracker for personal support requests nor feature requests. Support requests should be send to: - -* [the elixir-talk mailing list](http://groups.google.com/group/elixir-lang-talk) -* [Stack Overflow](http://stackoverflow.com/questions/ask?tags=elixir) -* [#elixir-lang](irc://chat.freenode.net/elixir-lang) - -Feature requests can be discussed on [the elixir-core mailing list](http://groups.google.com/group/elixir-lang-core). - -We do our best to keep the issues tracker tidy and organized, making it useful -for everyone. For example, we classify open issues per application and perceived -difficulty of the issue, making it easier for developers to -[contribute to Elixir](#contributing). - -## Bug reports - -A bug is a _demonstrable problem_ that is caused by the code in the repository. -Good bug reports are extremely helpful - thank you! - -Guidelines for bug reports: - -1. **Use the GitHub issue search** — check if the issue has already been - reported. - -2. **Check if the issue has been fixed** — try to reproduce it using the - `master` branch in the repository. - -3. **Isolate and report the problem** — ideally create a reduced test - case. - -Please try to be as detailed as possible in your report. Include information about -your Operating System, your Erlang and Elixir versions. Please provide steps to -reproduce the issue as well as the outcome you were expecting! All these details -will help developers to fix any potential bugs. - -Example: + -> Short and descriptive example bug report title -> -> A summary of the issue and the environment in which it occurs. If suitable, -> include the steps required to reproduce the bug. -> -> 1. This is the first step -> 2. This is the second step -> 3. Further steps, etc. -> -> `` - a link to the reduced test case (e.g. a GitHub Gist) -> -> Any other information you want to share that is relevant to the issue being -> reported. This might include the lines of code that you have identified as -> causing the bug, and potential solutions (and your opinions on their -> merits). - -## Feature requests - -Feature requests are welcome and should be discussed on [the elixir-core mailing list](http://groups.google.com/group/elixir-lang-core). But take a moment to find -out whether your idea fits with the scope and aims of the project. It's up to *you* -to make a strong case to convince the community of the merits of this feature. -Please provide as much detail and context as possible. +# Contributing to Elixir -## Contributing +We invite contributions to Elixir. To contribute, there are a few +things you need to know about the code. First, Elixir code is divided +by each application inside the `lib` folder: -We incentivize everyone to contribute to Elixir and help us tackle -existing issues! To do so, there are a few things you need to know -about the code. First, Elixir code is divided in applications inside -the `lib` folder: + * `elixir` - Elixir's kernel and standard library -* `elixir` - Contains Elixir's kernel and stdlib + * `eex` - EEx is the template engine that allows you to embed Elixir -* `eex` - Template engine that allows you to embed Elixir + * `ex_unit` - ExUnit is a simple test framework that ships with Elixir -* `ex_unit` - Simple test framework that ships with Elixir + * `iex` - IEx stands for Interactive Elixir: Elixir's interactive shell -* `iex` — IEx, Elixir's interactive shell + * `logger` - Logger is the built-in logger -* `mix` — Elixir's build tool + * `mix` - Mix is Elixir's build tool -You can run all tests in the root directory with `make test` and you can -also run tests for a specific framework `make test_#{NAME}`, for example, -`make test_ex_unit`. +You can run all tests in the root directory with `make test`. You can +also run tests for a specific framework with `make test_#{APPLICATION}`, for example, +`make test_ex_unit`. If you just changed something in Elixir's standard +library, you can run only that portion through `make test_stdlib`. -In case you are changing a single file, you can compile and run tests only -for that particular file for fast development cycles. For example, if you +If you are only changing one file, you can choose to compile and run tests +for that specific file for faster development cycles. For example, if you are changing the String module, you can compile it and run its tests as: - $ bin/elixirc lib/elixir/lib/string.ex -o lib/elixir/ebin - $ bin/elixir lib/elixir/test/elixir/string_test.exs - -After your changes are done, please remember to run the full suite with -`make test`. - -From time to time, your tests may fail in an existing Elixir checkout and -may require a clean start by running `make clean compile`. You can always -check [the official build status on Travis-CI](https://travis-ci.org/elixir-lang/elixir). - -With tests running and passing, you are ready to contribute to Elixir and -send your pull requests. +```sh +bin/elixirc lib/elixir/lib/string.ex -o lib/elixir/ebin +bin/elixir lib/elixir/test/elixir/string_test.exs +``` -### Building on Windows +Some test files need their `test_helper.exs` to be explicitly required +before, such as: -There are a few extra steps you'll need to take for contributing from Windows. -Basically, once you have Erlang 17, Git, and MSYS from MinGW on your system, -you're all set. Specifically, here's what you need to do to get up and running: +```sh +bin/elixir -r lib/logger/test/test_helper.exs lib/logger/test/logger_test.exs +``` -1. Install [Git](http://www.git-scm.com/download/win), -[Erlang](http://www.erlang.org/download.html), and the -[MinGW Installation Manager](http://sourceforge.net/projects/mingw/files/latest/download?source=files). -2. Use the MinGW Installation Manager to install the msys-bash, msys-make, and -msys-grep packages. -3. Add `;C:\Program Files (x86)\Git\bin;C:\Program Files\erl6.0\bin;C:\Program Files\erl6.0\erts-6.0\bin;C:\MinGW\msys\1.0\bin` -to your "Path" environment variable . (This is under Control Panel > System -and Security > System > Advanced system settings > Environment Variables > -System variables) +You can also use the `LINE` env var to run a single test: -You can now work in the Command Prompt similar to how you would on other OS'es, -except for some things (like creating symlinks) you'll need to run the Command -Prompt as an Administrator. +```sh +LINE=123 bin/elixir lib/elixir/test/elixir/string_test.exs +```` -## Contributing Documentation +To recompile all (including Erlang modules): -Code documentation (`@doc`, `@moduledoc`, `@typedoc`) has a special convention: -the first paragraph is considered to be a short summary. +```sh +make compile +``` -For functions, macros and callbacks say what it will do. For example write -something like: +After your changes are done, please remember to run `make format` to guarantee +all files are properly formatted, then run the full suite with +`make test`. -```elixir -@doc """ -Returns only those elements for which `fun` is true. +If your contribution fails during the bootstrapping of the language, +you can rebuild the language from scratch with: -... -""" -def filter(collection, fun) ... +```sh +make clean_elixir compile ``` -For modules, protocols and types say what it is. For example write -something like: - -```elixir -defmodule File.Stat do - @moduledoc """ - Information about a file. +Similarly, if you can not get Elixir to compile or the tests to pass after +updating an existing checkout, run `make clean compile`. You can check +[the official build status](https://github.com/elixir-lang/elixir/actions/workflows/ci.yml). +More tasks can be found by reading the [Makefile](Makefile). - ... - """ +We encourage contributors to write tests that capture both existing and newly +introduced behavior, especially for bug fixes and major changes: - defstruct [...] -end -``` + * **Bug Fixes:** If you are fixing a bug, please try to include a test that + *fails* before your change and *passes* afterward. This makes it easier to + confirm that the fix addresses the underlying issue and helps prevent + regressions in the future. -Keep in mind that the first paragraph might show up in a summary somewhere, long -texts in the first paragraph create very ugly summaries. As a rule of thumb -anything longer than 80 characters is too long. + * **New Features or Major Changes:** If you are adding a new feature or making + major changes to existing functionality, please add tests that cover the + major parts of that functionality. Aim to have the best code coverage possible. -Try to keep unneccesary details out of the first paragraph, it's only there to -give a user a quick idea of what the documented "thing" does/is. The rest of the -documentation string can contain the details, for example when a value and when -`nil` is returned. +With tests running and passing, you are ready to contribute to Elixir and +[send a pull request](https://help.github.com/articles/using-pull-requests/). +We have saved some excellent pull requests we have received in the past in +case you are looking for some examples: -If possible include examples, preferably in a form that works with doctests. For -example: + * [Implement Enum.member? - Pull request](https://github.com/elixir-lang/elixir/pull/992) -```elixir -@doc """ -Return only those elements for which `fun` is true. + * [Add String.valid? - Pull request](https://github.com/elixir-lang/elixir/pull/1058) -## Examples + * [Implement capture_io for ExUnit - Pull request](https://github.com/elixir-lang/elixir/pull/1059) - iex> Enum.filter([1, 2, 3], fn(x) -> rem(x, 2) == 0 end) - [2] +## Reviewing changes -""" -def filter(collection, fun) ... -``` +Once a pull request is sent, the Elixir team will review your changes. +We outline our process below to clarify the roles of everyone involved. -This makes it easy to test the examples so that they don't go stale and examples -are often a great help in explaining what a function does. +All pull requests must be approved by two committers before being merged into +the repository. If changes are necessary, the team will leave appropriate +comments requesting changes to the code. Unfortunately, we cannot guarantee a +pull request will be merged, even when modifications are requested, as the Elixir +team will re-evaluate the contribution as it changes. -## Pull requests +Committers may also push style changes directly to your branch. If you would +rather manage all changes yourself, you can disable the "Allow edits from maintainers" +feature when submitting your pull request. -Good pull requests - patches, improvements, new features - are a fantastic -help. They should remain focused in scope and avoid containing unrelated -commits. +The Elixir team may optionally assign someone to review a pull request. +If someone is assigned, they must explicitly approve the code before +another team member can merge it. -**IMPORTANT**: By submitting a patch, you agree that your work will be -licensed under the license used by the project. +When the review finishes, your pull request will be squashed and merged +into the repository. If you have carefully organized your commits and +believe they should be merged without squashing, please mention it in +a comment. -If you have any large pull request in mind (e.g. implementing features, -refactoring code, etc), **please ask first** otherwise you risk spending -a lot of time working on something that the project's developers might -not want to merge into the project. +## Licensing and Compliance Requirements -Please adhere to the coding conventions in the project (indentation, -accurate comments, etc.) and don't forget to add your own tests and -documentation. When working with git, we recommend the following process -in order to craft an excellent pull request: +Please review our [Open Source Policy](OPEN_SOURCE_POLICY.md) for complete +guidelines on licensing and compliance. Below is a summary of the key points +affecting **all external contributors**: -1. [Fork](http://help.github.com/fork-a-repo/) the project, clone your fork, - and configure the remotes: + * Accepted Licenses: Any code contributed must be licensed under the + `Apache-2.0` license. - ```bash - # Clone your fork of the repo into the current directory - git clone https://github.com//elixir - # Navigate to the newly cloned directory - cd elixir - # Assign the original repo to a remote called "upstream" - git remote add upstream https://github.com/elixir-lang/elixir - ``` + * SPDX License Headers: With the exception of approved test fixture files, + all new or modified files in a pull request must include correct SPDX + headers. If you are creating a new file under the `Apache-2.0` license, for + instance, please use: -2. If you cloned a while ago, get the latest changes from upstream: + ```elixir + # SPDX-License-Identifier: Apache-2.0 + # SPDX-FileCopyrightText: 2021 The Elixir Team + ``` - ```bash - git checkout master - git pull upstream master - ``` + * No Executable Binaries: Contributions must **not** include any executable + binary files. If you require an exception (for example, certain test artifacts), + please see the policy on how to request approval and document exceptions. -3. Create a new topic branch (off of `master`) to contain your feature, change, - or fix. + * Preserving Copyright and License Info: If you copy code from elsewhere, + ensure that **all original copyright and license notices remain intact**. If + they are missing or incomplete, you must add them. - **IMPORTANT**: Making changes in `master` is discouraged. You should always - keep your local `master` in sync with upstream `master` and make your - changes in topic branches. + * Failure to Comply: Pull requests that do not meet these licensing and + compliance standards will be rejected or require modifications before merging. - ```bash - git checkout -b - ``` + * Developer Certificate of Origin: All contributions are subject to the + Developer Certificate of Origin. -4. Commit your changes in logical chunks. Keep your commit messages organized, - with a short description in the first line and more detailed information on - the following lines. Feel free to use Git's - [interactive rebase](https://help.github.com/articles/interactive-rebase) - feature to tidy up your commits before making them public. + ```text + By making a contribution to this project, I certify that: -5. Make sure all the tests are still passing. + (a) The contribution was created in whole or in part by me and I + have the right to submit it under the open source license + indicated in the file; or - ```bash - make test - ``` + (b) The contribution is based upon previous work that, to the + best of my knowledge, is covered under an appropriate open + source license and I have the right under that license to + submit that work with modifications, whether created in whole + or in part by me, under the same open source license (unless + I am permitted to submit under a different license), as + Indicated in the file; or - This command will compile the code in your branch and use that - version of Elixir to run the tests. This is needed to ensure your changes can - pass all the tests. + (c) The contribution was provided directly to me by some other + person who certified (a), (b) or (c) and I have not modified + it. -6. Push your topic branch up to your fork: + (d) I understand and agree that this project and the contribution + are public and that a record of the contribution (including + all personal information I submit with it, including my + sign-off) is maintained indefinitely and may be redistributed + consistent with this project or the open source license(s) + involved. + ``` - ```bash - git push origin - ``` + See for a copy of the Developer Certificate + of Origin license. -7. [Open a Pull Request](https://help.github.com/articles/using-pull-requests/) - with a clear title and description. +## Building documentation -8. If you haven't updated your pull request for a while, you should consider - rebasing on master and resolving any conflicts. +Building the documentation requires that [ExDoc](https://github.com/elixir-lang/ex_doc) +is installed and built alongside Elixir. - **IMPORTANT**: _Never ever_ merge upstream `master` into your branches. You - should always `git rebase` on `master` to bring your changes up to date when - necessary. +After cloning and compiling Elixir, run: - ```bash - git checkout master - git pull upstream master - git checkout - git rebase master - ``` +```sh +elixir_dir=$(pwd) +cd .. && git clone https://github.com/elixir-lang/ex_doc.git +cd ex_doc && "${elixir_dir}/bin/elixir" "${elixir_dir}/bin/mix" do deps.get + compile -We have saved some excellent pull requests we have received in the past in case -you are looking for some examples: +# Now we will go back to Elixir's root directory, +cd "${elixir_dir}" -* https://github.com/elixir-lang/elixir/pull/992 -* https://github.com/elixir-lang/elixir/pull/1041 -* https://github.com/elixir-lang/elixir/pull/1058 -* https://github.com/elixir-lang/elixir/pull/1059 +# and generate HTML and EPUB documents: +make docs +``` -Thank you for your contributions! +This will produce documentation sets for `elixir`, `eex`, `ex_unit`, `iex`, `logger`, +and `mix` under the `doc` directory. If you are planning to contribute documentation, +[please check our best practices for writing documentation](https://hexdocs.pm/elixir/writing-documentation.html). diff --git a/LEGAL b/LEGAL deleted file mode 100644 index 8948b8ad2f0..00000000000 --- a/LEGAL +++ /dev/null @@ -1,8 +0,0 @@ -LEGAL NOTICE INFORMATION ------------------------- - -All the files in this distribution are covered under either Elixir's -license (see the file LICENSE) except the files mentioned below that -contains sections that are under Erlang's License (EPL): - -lib/elixir/src/elixir_parser.erl (generated by build scripts) diff --git a/LICENSE b/LICENSE index d3a92d217c9..d9a10c0d8e8 100644 --- a/LICENSE +++ b/LICENSE @@ -1,13 +1,176 @@ -Copyright 2012-2013 Plataformatec. + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - http://www.apache.org/licenses/LICENSE-2.0 + 1. Definitions. - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. \ No newline at end of file + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/LICENSES/Apache-2.0.txt b/LICENSES/Apache-2.0.txt new file mode 100644 index 00000000000..137069b8238 --- /dev/null +++ b/LICENSES/Apache-2.0.txt @@ -0,0 +1,73 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. + +"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: + + (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. + + You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + +To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/LICENSES/LicenseRef-elixir-trademark-policy.txt b/LICENSES/LicenseRef-elixir-trademark-policy.txt new file mode 100644 index 00000000000..b94911196dc --- /dev/null +++ b/LICENSES/LicenseRef-elixir-trademark-policy.txt @@ -0,0 +1,98 @@ +ELIXIR TEAM TRADEMARKS POLICY + +This document outlines the policy for allowed usage of the “Elixir” word and the +Elixir logo by other parties. + +“Elixir” and the Elixir logo are registered trademarks of the Elixir Team. The +Elixir Team believes in a decentralized approach to growing the community and +the ecosystem, independent of the Elixir project and the Elixir Team. + +Anyone can use the Elixir trademarks if that use of the trademark is nominative. +The trademarks must not be used to disparage the project and its community, nor +be used in any way to imply ownership, endorsement, or association with the +Elixir project and the Elixir Team. + +You must not visually combine the Elixir logo with any other images, or change +the logo in any way other than ways required by printing restrictions. If you +want to create your own visual identity in relation to Elixir, you might use the +shape of an unrelated “water drop” as part of your design, as seen in many +community projects and initiatives. You must not combine or modify the Elixir +logo. + +The Elixir logo is available in our repository in both vertical and horizontal +versions. + +Nominative use +The “nominative use” (or “nominative fair use”) is a legal doctrine that +authorizes everyone (even commercial companies) to use or refer to the trademark +of another if: + +The product or service in question must be one not readily identifiable without +use of the trademark. + +Only so much of the mark or marks may be used as is reasonably necessary to +identify the product or service. + +The organization using the mark must do nothing that would, in conjunction with +the mark, suggest sponsorship or endorsement by the trademark holder. + +Our trademarks must be used to refer to the Elixir programming language. + +Examples of permitted use +All examples listed next must strictly adhere to the terms outlined in the +previous sections: + +Usage of the Elixir logo to say a technology is “powered by Elixir” under +nominative use. Linking back to the Elixir website, if possible, is appreciated. + +Usage of the Elixir logo to display it as a supported technology in a service or +platform. For instance, you may say “we support Elixir” and use the Elixir logo, +but you may not refer to yourself as “the Elixir platform” nor imply any form of +endorsement or association with Elixir. + +Usage of the Elixir logo in non-commercial community meetups, in presentations, +and in courses when referring to the language and its ecosystem under nominative +use. + +Usage of the Elixir logo in non-commercial swag (stickers, t-shirts, mugs, etc) +to promote the Elixir programming language. The Elixir marks must be the only +marks featured in the product. You need permission to make swag that include +Elixir and other third party marks in them. + +Inclusion of the Elixir logo in non-commercial icon sets. Use of the Elixir +icons must still adhere to Elixir’s trademark policies. + +Usage of the “Elixir” word in book titles, meetups, conferences, and podcasts. +You must not use the word to imply uniqueness or endorsement from the Elixir +team. “The Elixir book” and “The Elixir podcast” are not permitted. +“Elixir in Action”, “Thinking Elixir”, and “Kraków Elixir User Group” are valid +examples already in use today. + +Usage of the “Elixir” word in the names of freely distributed software and +hardware products is allowed when referring to use with or suitability for the +Elixir programming language, such as wxElixir, Elixirsense, etc. If the product +includes the Elixir programming language itself, then you must also respect its +license. + +Examples of not permitted use +Here is a non-exhaustive list of non permitted uses of the marks: + +Usage of the Elixir logo in book covers, conferences, and podcasts. + +Usage of the Elixir logo as the mark of third party projects, even in combination +with other marks. + +Naming any company or product after Elixir, such as “The Elixir Hosting”, +“The Elixir Consultants”, etc. + +Examples that require permission +Here are some examples that may be granted permission upon request: + +Selling merchandise (stickers, t-shirts, mugs, etc). +You can request permission by emailing trademarks@elixir-lang.org. + +Important note +Nothing in this page shall be interpreted to allow any third party to claim any +association with the Elixir project and the Elixir Team, or to imply any +approval or support by the Elixir project and the Elixir Team for any third +party products, services, or events. diff --git a/LICENSES/LicenseRef-scancode-unicode.txt b/LICENSES/LicenseRef-scancode-unicode.txt new file mode 100644 index 00000000000..24438a47d73 --- /dev/null +++ b/LICENSES/LicenseRef-scancode-unicode.txt @@ -0,0 +1,58 @@ +UNICODE, INC. LICENSE AGREEMENT - DATA FILES AND SOFTWARE + +Unicode Data Files include all data files under the directories +http://www.unicode.org/Public/, http://www.unicode.org/reports/, and +http://www.unicode.org/cldr/data/ . Unicode Software includes any source +code published in the Unicode Standard or under the directories +http://www.unicode.org/Public/, http://www.unicode.org/reports/, and +http://www.unicode.org/cldr/data/. + +NOTICE TO USER: Carefully read the following legal agreement. BY +DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING UNICODE INC.'S DATA +FILES ("DATA FILES"), AND/OR SOFTWARE ("SOFTWARE"), YOU UNEQUIVOCALLY +ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE TERMS AND CONDITIONS OF THIS +AGREEMENT. IF YOU DO NOT AGREE, DO NOT DOWNLOAD, INSTALL, COPY, DISTRIBUTE +OR USE THE DATA FILES OR SOFTWARE. + +COPYRIGHT AND PERMISSION NOTICE + +Copyright © Unicode, Inc. All rights reserved. Distributed under +the Terms of Use in http://www.unicode.org/copyright.html. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of the Unicode data files and any associated documentation (the +"Data Files") or Unicode software and any associated documentation (the +"Software") to deal in the Data Files or Software without restriction, +including without limitation the rights to use, copy, modify, merge, +publish, distribute, and/or sell copies of the Data Files or Software, +and to permit persons to whom the Data Files or Software are furnished +to do so, provided that + +(a) the above copyright notice(s) and this permission notice appear with +all copies of the Data Files or Software, + +(b) both the above copyright notice(s) and this permission notice appear +in associated documentation, and + +(c) there is clear notice in each modified Data File or in the Software +as well as in the documentation associated with the Data File(s) or +Software that the data or software has been modified. + +THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR HOLDERS INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR +ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER +RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +CONNECTION WITH THE USE OR PERFORMANCE OF THE DATA FILES OR SOFTWARE. + +Except as contained in this notice, the name of a copyright holder shall +not be used in advertising or otherwise to promote the sale, use or +other dealings in these Data Files or Software without prior written +authorization of the copyright holder. + +Unicode and the Unicode logo are trademarks of Unicode, Inc., and may be +registered in some jurisdictions. All other trademarks and registered +trademarks mentioned herein are the property of their respective owners. diff --git a/Makefile b/Makefile index b660c202c21..24d02701acc 100644 --- a/Makefile +++ b/Makefile @@ -1,95 +1,125 @@ -REBAR := rebar -ELIXIRC := bin/elixirc --verbose --ignore-module-conflict +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +PREFIX ?= /usr/local +TEST_FILES ?= "*_test.exs" +SHARE_PREFIX ?= $(PREFIX)/share +MAN_PREFIX ?= $(SHARE_PREFIX)/man +CANONICAL := main/ +ELIXIRC := bin/elixirc --ignore-module-conflict $(ELIXIRC_OPTS) +ELIXIRC_MIN_SIG := $(ELIXIRC) -e 'Code.put_compiler_option :infer_signatures, []' ERLC := erlc -I lib/elixir/include +ERL_MAKE := erl -make ERL := erl -I lib/elixir/include -noshell -pa lib/elixir/ebin +GENERATE_APP := $(CURDIR)/lib/elixir/scripts/generate_app.escript VERSION := $(strip $(shell cat VERSION)) Q := @ -PREFIX := /usr/local LIBDIR := lib +BINDIR := bin INSTALL = install INSTALL_DIR = $(INSTALL) -m755 -d INSTALL_DATA = $(INSTALL) -m644 INSTALL_PROGRAM = $(INSTALL) -m755 +GIT_REVISION = $(strip $(shell git rev-parse HEAD 2> /dev/null )) +GIT_TAG = $(strip $(shell head="$(call GIT_REVISION)"; git tag --points-at $$head 2> /dev/null | grep -v latest | tail -1)) +SOURCE_DATE_EPOCH_PATH = lib/elixir/tmp/ebin_reproducible +SOURCE_DATE_EPOCH_FILE = $(SOURCE_DATE_EPOCH_PATH)/SOURCE_DATE_EPOCH -.PHONY: install compile erlang elixir dialyze test clean docs release_docs release_zip check_erlang_release -.NOTPARALLEL: compile +.PHONY: cover install install_man build_plt clean_plt dialyze test check_reproducible clean clean_elixir clean_man format docs Docs.zip Precompiled.zip zips +.NOTPARALLEL: #==> Functions -# This check should work for older versions like R16B -# as well as new verions like 17.1 and 18 define CHECK_ERLANG_RELEASE - $(Q) erl -noshell -eval 'io:fwrite("~s", [erlang:system_info(otp_release)])' -s erlang halt | grep -q '^1[789]'; \ - if [ $$? != 0 ]; then \ - echo "At least Erlang 17.0 is required to build Elixir"; \ - exit 1; \ - fi; + erl -noshell -eval '{V,_} = string:to_integer(erlang:system_info(otp_release)), io:fwrite("~s", [is_integer(V) and (V >= 26)])' -s erlang halt | grep -q '^true'; \ + if [ $$? != 0 ]; then \ + echo "At least Erlang/OTP 26.0 is required to build Elixir"; \ + exit 1; \ + fi endef define APP_TEMPLATE $(1): lib/$(1)/ebin/Elixir.$(2).beam lib/$(1)/ebin/$(1).app lib/$(1)/ebin/$(1).app: lib/$(1)/mix.exs - $(Q) mkdir -p lib/$(1)/_build/shared/lib/$(1) - $(Q) cp -R lib/$(1)/ebin lib/$(1)/_build/shared/lib/$(1)/ - $(Q) cd lib/$(1) && ../../bin/elixir -e "Mix.start(:permanent, [])" -r mix.exs -e "Mix.Task.run('compile.app')" - $(Q) cp lib/$(1)/_build/shared/lib/$(1)/ebin/$(1).app lib/$(1)/ebin/$(1).app - $(Q) rm -rf lib/$(1)/_build + $(Q) cd lib/$(1) && ../../bin/elixir -e 'Mix.start(:permanent, [])' -r mix.exs -e 'Mix.Task.run("compile.app", ~w[--compile-path ebin])' lib/$(1)/ebin/Elixir.$(2).beam: $(wildcard lib/$(1)/lib/*.ex) $(wildcard lib/$(1)/lib/*/*.ex) $(wildcard lib/$(1)/lib/*/*/*.ex) @ echo "==> $(1) (compile)" @ rm -rf lib/$(1)/ebin $(Q) cd lib/$(1) && ../../$$(ELIXIRC) "lib/**/*.ex" -o ebin -test_$(1): $(1) - @ echo "==> $(1) (exunit)" - $(Q) cd lib/$(1) && ../../bin/elixir -r "test/test_helper.exs" -pr "test/**/*_test.exs"; +test_$(1): test_formatted $(1) + @ echo "==> $(1) (ex_unit)" + $(Q) cd lib/$(1) && ../../bin/elixir -r "test/test_helper.exs" -pr "test/**/$(TEST_FILES)"; + +cover/ex_unit_$(1).coverdata: + $(Q) COVER="1" $(MAKE) test_$(1) +cover/combined.coverdata: cover/ex_unit_$(1).coverdata +endef + +define WRITE_SOURCE_DATE_EPOCH +$(shell mkdir -p $(SOURCE_DATE_EPOCH_PATH) && bin/elixir -e \ + 'IO.puts System.build_info()[:date] \ + |> DateTime.from_iso8601() \ + |> elem(1) \ + |> DateTime.to_unix()' > $(SOURCE_DATE_EPOCH_FILE)) +endef + +define READ_SOURCE_DATE_EPOCH +$(strip $(shell cat $(SOURCE_DATE_EPOCH_FILE))) endef #==> Compilation tasks -KERNEL:=lib/elixir/ebin/Elixir.Kernel.beam -UNICODE:=lib/elixir/ebin/Elixir.String.Unicode.beam +APP := lib/elixir/ebin/elixir.app +EEX := lib/eex/ebin/Elixir.EEx.beam +ELIXIR := lib/elixir/ebin/elixir.beam +PARSER := lib/elixir/src/elixir_parser.erl +KERNEL := lib/elixir/ebin/Elixir.Kernel.beam +UNICODE := lib/elixir/ebin/Elixir.String.Unicode.beam default: compile -compile: lib/elixir/src/elixir.app.src erlang elixir +compile: erlang elixir -lib/elixir/src/elixir.app.src: src/elixir.app.src - $(Q) $(call CHECK_ERLANG_RELEASE) - $(Q) rm -rf lib/elixir/src/elixir.app.src - $(Q) echo "%% This file is automatically generated from /src/elixir.app.src" \ - >lib/elixir/src/elixir.app.src - $(Q) cat src/elixir.app.src >>lib/elixir/src/elixir.app.src +erlang: $(ELIXIR) +$(ELIXIR): $(PARSER) lib/elixir/src/* + $(Q) if [ ! -f $(APP) ]; then $(call CHECK_ERLANG_RELEASE); fi + $(Q) cd lib/elixir && mkdir -p ebin && $(ERL_MAKE) + $(Q) $(GENERATE_APP) $(VERSION) -erlang: - $(Q) cd lib/elixir && ../../$(REBAR) compile +$(PARSER): lib/elixir/src/elixir_parser.yrl + $(Q) erlc -o $@ +'{verbose,true}' +'{report,true}' $< -# Since Mix depends on EEx and EEx depends on -# Mix, we first compile EEx without the .app -# file, then mix and then compile EEx fully -elixir: stdlib lib/eex/ebin/Elixir.EEx.beam mix ex_unit eex iex +# Since Mix depends on EEx and EEx depends on Mix, +# we first compile EEx without the .app file, +# then Mix, and then compile EEx fully +elixir: stdlib $(EEX) mix ex_unit logger eex iex +stdlib: $(KERNEL) $(UNICODE) $(APP) -stdlib: $(KERNEL) VERSION -$(KERNEL): lib/elixir/lib/*.ex lib/elixir/lib/*/*.ex - $(Q) if [ ! -f $(KERNEL) ]; then \ - echo "==> bootstrap (compile)"; \ - $(ERL) -s elixir_compiler core -s erlang halt; \ +$(KERNEL): lib/elixir/src/* lib/elixir/lib/*.ex lib/elixir/lib/*/*.ex lib/elixir/lib/*/*/*.ex VERSION + $(Q) if [ ! -f $(KERNEL) ]; then \ + echo "==> bootstrap (compile)"; \ + $(ERL) -s elixir_compiler bootstrap -s erlang halt; \ + "$(MAKE)" unicode; \ fi @ echo "==> elixir (compile)"; - $(Q) cd lib/elixir && ../../$(ELIXIRC) "lib/kernel.ex" -o ebin; - $(Q) cd lib/elixir && ../../$(ELIXIRC) "lib/**/*.ex" -o ebin; - $(Q) $(MAKE) unicode - $(Q) rm -rf lib/elixir/ebin/elixir.app - $(Q) cd lib/elixir && ../../$(REBAR) compile + $(Q) cd lib/elixir && ../../$(ELIXIRC_MIN_SIG) "lib/**/*.ex" -o ebin; + +$(APP): lib/elixir/src/elixir.app.src lib/elixir/ebin VERSION $(GENERATE_APP) + $(Q) $(GENERATE_APP) $(VERSION) unicode: $(UNICODE) $(UNICODE): lib/elixir/unicode/* @ echo "==> unicode (compile)"; - @ echo "This step can take up to a minute to compile in order to embed the Unicode database" - $(Q) cd lib/elixir && ../../$(ELIXIRC) unicode/unicode.ex -o ebin; + $(Q) $(ELIXIRC_MIN_SIG) lib/elixir/unicode/unicode.ex -o lib/elixir/ebin; + $(Q) $(ELIXIRC_MIN_SIG) lib/elixir/unicode/tokenizer.ex -o lib/elixir/ebin; + $(Q) $(ELIXIRC_MIN_SIG) lib/elixir/unicode/security.ex -o lib/elixir/ebin; $(eval $(call APP_TEMPLATE,ex_unit,ExUnit)) +$(eval $(call APP_TEMPLATE,logger,Logger)) $(eval $(call APP_TEMPLATE,eex,EEx)) $(eval $(call APP_TEMPLATE,mix,Mix)) $(eval $(call APP_TEMPLATE,iex,IEx)) @@ -97,64 +127,151 @@ $(eval $(call APP_TEMPLATE,iex,IEx)) install: compile @ echo "==> elixir (install)" $(Q) for dir in lib/*; do \ + rm -rf $(DESTDIR)$(PREFIX)/$(LIBDIR)/elixir/$$dir/ebin; \ $(INSTALL_DIR) "$(DESTDIR)$(PREFIX)/$(LIBDIR)/elixir/$$dir/ebin"; \ $(INSTALL_DATA) $$dir/ebin/* "$(DESTDIR)$(PREFIX)/$(LIBDIR)/elixir/$$dir/ebin"; \ done $(Q) $(INSTALL_DIR) "$(DESTDIR)$(PREFIX)/$(LIBDIR)/elixir/bin" $(Q) $(INSTALL_PROGRAM) $(filter-out %.ps1, $(filter-out %.bat, $(wildcard bin/*))) "$(DESTDIR)$(PREFIX)/$(LIBDIR)/elixir/bin" - $(Q) $(INSTALL_DIR) "$(DESTDIR)$(PREFIX)/bin" - $(Q) for file in "$(DESTDIR)$(PREFIX)"/$(LIBDIR)/elixir/bin/* ; do \ - ln -sf "../$(LIBDIR)/elixir/bin/$${file##*/}" "$(DESTDIR)$(PREFIX)/bin/" ; \ + $(Q) $(INSTALL_DIR) "$(DESTDIR)$(PREFIX)/$(BINDIR)" + $(Q) for file in "$(DESTDIR)$(PREFIX)"/$(LIBDIR)/elixir/bin/*; do \ + ln -sf "../$(LIBDIR)/elixir/bin/$${file##*/}" "$(DESTDIR)$(PREFIX)/$(BINDIR)/"; \ done - -clean: - cd lib/elixir && ../../$(REBAR) clean + "$(MAKE)" install_man + +check_reproducible: compile + $(Q) echo "==> Checking for reproducible builds..." + $(Q) rm -rf lib/*/tmp/ebin_reproducible/ + $(call WRITE_SOURCE_DATE_EPOCH) + $(Q) mkdir -p lib/elixir/tmp/ebin_reproducible/ \ + lib/eex/tmp/ebin_reproducible/ \ + lib/ex_unit/tmp/ebin_reproducible/ \ + lib/iex/tmp/ebin_reproducible/ \ + lib/logger/tmp/ebin_reproducible/ \ + lib/mix/tmp/ebin_reproducible/ + $(Q) mv lib/elixir/ebin/* lib/elixir/tmp/ebin_reproducible/ + $(Q) mv lib/eex/ebin/* lib/eex/tmp/ebin_reproducible/ + $(Q) mv lib/ex_unit/ebin/* lib/ex_unit/tmp/ebin_reproducible/ + $(Q) mv lib/iex/ebin/* lib/iex/tmp/ebin_reproducible/ + $(Q) mv lib/logger/ebin/* lib/logger/tmp/ebin_reproducible/ + $(Q) mv lib/mix/ebin/* lib/mix/tmp/ebin_reproducible/ + $(Q) rm -rf lib/*/ebin + SOURCE_DATE_EPOCH=$(call READ_SOURCE_DATE_EPOCH) "$(MAKE)" compile + $(Q) echo "Diffing..." + $(Q) bin/elixir lib/elixir/scripts/diff.exs lib/elixir/ebin/ lib/elixir/tmp/ebin_reproducible/ + $(Q) bin/elixir lib/elixir/scripts/diff.exs lib/eex/ebin/ lib/eex/tmp/ebin_reproducible/ + $(Q) bin/elixir lib/elixir/scripts/diff.exs lib/ex_unit/ebin/ lib/ex_unit/tmp/ebin_reproducible/ + $(Q) bin/elixir lib/elixir/scripts/diff.exs lib/iex/ebin/ lib/iex/tmp/ebin_reproducible/ + $(Q) bin/elixir lib/elixir/scripts/diff.exs lib/logger/ebin/ lib/logger/tmp/ebin_reproducible/ + $(Q) bin/elixir lib/elixir/scripts/diff.exs lib/mix/ebin/ lib/mix/tmp/ebin_reproducible/ + $(Q) echo "Builds are reproducible" + +clean: clean_man rm -rf ebin rm -rf lib/*/ebin - rm -rf lib/elixir/test/ebin - rm -rf lib/*/tmp - rm -rf lib/mix/test/fixtures/git_repo - rm -rf lib/mix/test/fixtures/deps_on_git_repo - rm -rf lib/mix/test/fixtures/git_rebar - rm -rf lib/elixir/src/elixir.app.src - -clean_exbeam: + rm -rf $(PARSER) + rm -rf lib/*/_build/ + rm -rf lib/*/tmp/ + rm -rf lib/elixir/test/ebin/ + rm -rf lib/mix/test/fixtures/deps_on_git_repo/ + rm -rf lib/mix/test/fixtures/git_rebar/ + rm -rf lib/mix/test/fixtures/git_repo/ + rm -rf lib/mix/test/fixtures/git_sparse_repo/ + rm -rf lib/mix/test/fixtures/archive/ebin/ + rm -f erl_crash.dump + rm -rf cover + +clean_elixir: $(Q) rm -f lib/*/ebin/Elixir.*.beam -#==> Release tasks - -SOURCE_REF = $(shell head="$$(git rev-parse HEAD)" tag="$$(git tag --points-at $$head | tail -1)" ; echo "$${tag:-$$head}\c") -DOCS = bin/elixir ../ex_doc/bin/ex_doc "$(1)" "$(VERSION)" "lib/$(2)/ebin" -m "$(3)" -u "https://github.com/elixir-lang/elixir" --source-ref "$(call SOURCE_REF)" -o docs/$(2) -p http://elixir-lang.org/docs.html - -docs: compile ../ex_doc/bin/ex_doc - $(Q) rm -rf docs - $(call DOCS,Elixir,elixir,Kernel) - $(call DOCS,EEx,eex,EEx) - $(call DOCS,Mix,mix,Mix) - $(call DOCS,IEx,iex,IEx) - $(call DOCS,ExUnit,ex_unit,ExUnit) +#==> Documentation tasks + +SOURCE_REF = $(shell tag="$(call GIT_TAG)" revision="$(call GIT_REVISION)"; echo "$${tag:-$$revision}") +DOCS_COMPILE = CANONICAL=$(CANONICAL) bin/elixir ../ex_doc/bin/ex_doc "$(1)" "$(VERSION)" "lib/$(2)/ebin" --main "$(3)" --source-url "https://github.com/elixir-lang/elixir" --source-ref "$(call SOURCE_REF)" --logo lib/elixir/pages/images/logo.png --output doc/$(2) --canonical "https://hexdocs.pm/$(2)/$(CANONICAL)" --homepage-url "https://elixir-lang.org/docs.html" $(DOCS_OPTIONS) $(4) +DOCS_CONFIG = bin/elixir lib/elixir/scripts/docs_config.exs "$(1)" + +docs: compile ../ex_doc/bin/ex_doc docs_elixir docs_eex docs_mix docs_iex docs_ex_unit docs_logger + +docs_elixir: compile ../ex_doc/bin/ex_doc + @ echo "==> ex_doc (elixir)" + $(Q) rm -rf doc/elixir + $(call DOCS_COMPILE,Elixir,elixir,Kernel,--config "lib/elixir/scripts/elixir_docs.exs") + $(call DOCS_CONFIG,elixir) + +docs_eex: compile ../ex_doc/bin/ex_doc + @ echo "==> ex_doc (eex)" + $(Q) rm -rf doc/eex + $(call DOCS_COMPILE,EEx,eex,EEx,--config "lib/elixir/scripts/mix_docs.exs") + $(call DOCS_CONFIG,eex) + +docs_mix: compile ../ex_doc/bin/ex_doc + @ echo "==> ex_doc (mix)" + $(Q) rm -rf doc/mix + $(call DOCS_COMPILE,Mix,mix,Mix,--config "lib/elixir/scripts/mix_docs.exs") + $(call DOCS_CONFIG,mix) + +docs_iex: compile ../ex_doc/bin/ex_doc + @ echo "==> ex_doc (iex)" + $(Q) rm -rf doc/iex + $(call DOCS_COMPILE,IEx,iex,IEx,--config "lib/elixir/scripts/mix_docs.exs") + $(call DOCS_CONFIG,iex) + +docs_ex_unit: compile ../ex_doc/bin/ex_doc + @ echo "==> ex_doc (ex_unit)" + $(Q) rm -rf doc/ex_unit + $(call DOCS_COMPILE,ExUnit,ex_unit,ExUnit,--config "lib/elixir/scripts/mix_docs.exs") + $(call DOCS_CONFIG,ex_unit) + +docs_logger: compile ../ex_doc/bin/ex_doc + @ echo "==> ex_doc (logger)" + $(Q) rm -rf doc/logger + $(call DOCS_COMPILE,Logger,logger,Logger,--config "lib/elixir/scripts/mix_docs.exs") + $(call DOCS_CONFIG,logger) ../ex_doc/bin/ex_doc: - @ echo "ex_doc is not found in ../ex_doc as expected. See README for more information." + @ echo "ex_doc is not found in ../ex_doc as expected. See CONTRIBUTING.md for more information." @ false -release_zip: compile - rm -rf v$(VERSION).zip - zip -9 -r v$(VERSION).zip bin CHANGELOG.md LEGAL lib/*/ebin LICENSE README.md VERSION +#==> Zip tasks + +Docs.zip: docs + rm -f Docs.zip + zip -9 -r Docs.zip CHANGELOG.md doc LICENSE README.md + @ echo "Docs file created $(CURDIR)/Docs.zip" + +Precompiled.zip: build_man compile + rm -f Precompiled.zip + zip -9 -r Precompiled.zip bin CHANGELOG.md lib/*/ebin lib/*/lib LICENSE Makefile man README.md VERSION + @ echo "Precompiled file created $(CURDIR)/Precompiled.zip" + +#==> Test tasks -release_docs: docs - cd ../docs - rm -rf ../docs/master - mv docs ../docs/master +test: test_formatted test_erlang test_elixir -#==> Tests tasks +test_windows: test test_taskkill -test: test_erlang test_elixir +test_taskkill: + taskkill //IM erl.exe //F //T //FI "MEMUSAGE gt 0" + taskkill //IM epmd.exe //F //T //FI "MEMUSAGE gt 0" TEST_ERL = lib/elixir/test/erlang TEST_EBIN = lib/elixir/test/ebin TEST_ERLS = $(addprefix $(TEST_EBIN)/, $(addsuffix .beam, $(basename $(notdir $(wildcard $(TEST_ERL)/*.erl))))) +define FORMAT + $(Q) if [ "$(OS)" = "Windows_NT" ]; then \ + cmd //C call ./bin/mix.bat format $(1); \ + else \ + bin/elixir bin/mix format $(1); \ + fi +endef + +format: compile + $(call FORMAT) + +test_formatted: compile + $(call FORMAT,--check-formatted) + test_erlang: compile $(TEST_ERLS) @ echo "==> elixir (eunit)" $(Q) $(ERL) -pa $(TEST_EBIN) -s test_helper test; @@ -164,25 +281,68 @@ $(TEST_EBIN)/%.beam: $(TEST_ERL)/%.erl $(Q) mkdir -p $(TEST_EBIN) $(Q) $(ERLC) -o $(TEST_EBIN) $< -test_elixir: test_stdlib test_ex_unit test_doc_test test_mix test_eex test_iex - -test_doc_test: compile - @ echo "==> doctest (exunit)" - $(Q) cd lib/elixir && ../../bin/elixir -r "test/doc_test.exs"; +test_elixir: test_stdlib test_ex_unit test_logger test_eex test_iex test_mix test_stdlib: compile - @ echo "==> elixir (exunit)" + @ echo "==> elixir (ex_unit)" $(Q) exec epmd & exit - $(Q) cd lib/elixir && ../../bin/elixir -r "test/elixir/test_helper.exs" -pr "test/elixir/**/*_test.exs"; + $(Q) if [ "$(OS)" = "Windows_NT" ]; then \ + cd lib/elixir && cmd //C call ../../bin/elixir.bat --sname primary -r "test/elixir/test_helper.exs" -pr "test/elixir/**/$(TEST_FILES)"; \ + else \ + cd lib/elixir && ../../bin/elixir --sname primary -r "test/elixir/test_helper.exs" -pr "test/elixir/**/$(TEST_FILES)"; \ + fi + +cover/ex_unit_elixir.coverdata: + $(Q) COVER="1" $(MAKE) test_stdlib +cover/combined.coverdata: cover/ex_unit_elixir.coverdata + +cover/combined.coverdata: + bin/elixir ./lib/elixir/scripts/cover.exs + +cover: cover/combined.coverdata + +#==> Dialyzer tasks + +DIALYZER_OPTS = --no_check_plt --fullpath -Werror_handling -Wunmatched_returns -Wunderspecs +PLT = .elixir.plt -.dialyzer.base_plt: - @ echo "==> Adding Erlang/OTP basic applications to a new base PLT" - $(Q) dialyzer --output_plt .dialyzer.base_plt --build_plt --apps erts kernel stdlib compiler tools syntax_tools parsetools +$(PLT): + @ echo "==> Building PLT with Elixir's dependencies..." + $(Q) dialyzer --output_plt $(PLT) --build_plt --apps erts kernel stdlib compiler syntax_tools parsetools tools ssl inets crypto runtime_tools ftp tftp mnesia public_key asn1 sasl -dialyze: .dialyzer.base_plt - $(Q) rm -f .dialyzer_plt - $(Q) cp .dialyzer.base_plt .dialyzer_plt - @ echo "==> Adding Elixir to PLT..." - $(Q) dialyzer --plt .dialyzer_plt --add_to_plt -r lib/elixir/ebin lib/ex_unit/ebin lib/eex/ebin lib/iex/ebin lib/mix/ebin +clean_plt: + $(Q) rm -f $(PLT) + +build_plt: clean_plt $(PLT) + +dialyze: compile $(PLT) @ echo "==> Dialyzing Elixir..." - $(Q) dialyzer --plt .dialyzer_plt -r lib/elixir/ebin lib/ex_unit/ebin lib/eex/ebin lib/iex/ebin lib/mix/ebin + $(Q) dialyzer -pa lib/elixir/ebin --plt $(PLT) $(DIALYZER_OPTS) lib/*/ebin + +#==> Man page tasks + +build_man: man/iex.1 man/elixir.1 + +define BUILD_MANPAGES +man/$(APP).1: + $(Q) cp man/$(APP).1.in man/$(APP).1 + $(Q) sed -i.bak "/{COMMON}/r man/common" man/$(APP).1 + $(Q) sed -i.bak "/{COMMON}/d" man/$(APP).1 + $(Q) rm -f man/$(APP).1.bak +endef + +$(foreach APP, elixir iex, $(eval $(BUILD_MANPAGES))) + +clean_man: + rm -f man/elixir.1 + rm -f man/elixir.1.bak + rm -f man/iex.1 + rm -f man/iex.1.bak + +install_man: build_man + $(Q) mkdir -p $(DESTDIR)$(MAN_PREFIX)/man1 + $(Q) $(INSTALL_DATA) man/elixir.1 $(DESTDIR)$(MAN_PREFIX)/man1 + $(Q) $(INSTALL_DATA) man/elixirc.1 $(DESTDIR)$(MAN_PREFIX)/man1 + $(Q) $(INSTALL_DATA) man/iex.1 $(DESTDIR)$(MAN_PREFIX)/man1 + $(Q) $(INSTALL_DATA) man/mix.1 $(DESTDIR)$(MAN_PREFIX)/man1 + "$(MAKE)" clean_man diff --git a/OPEN_SOURCE_POLICY.md b/OPEN_SOURCE_POLICY.md new file mode 100644 index 00000000000..381975f0db0 --- /dev/null +++ b/OPEN_SOURCE_POLICY.md @@ -0,0 +1,165 @@ + + +# Open Source Policy + +## 1. Introduction + +This Open Source Policy outlines the licensing, contribution, and compliance +requirements for all code released under the Elixir project. By adhering to +these guidelines, we ensure that our community, maintainers, and contributors +uphold both legal and ethical standards while fostering a collaborative, +transparent environment. + +This policy exists to support and protect the Elixir community. It aims to +balance openness, collaboration, and respect for all contributors’ rights, +ensuring that Elixir remains a trusted and innovative open source project. + +## 2. Scope + +This policy applies to the Elixir Programming language, located at +. It covers every file, and contribution +made, including documentation and any associated assets. + +## 3. Licensing + +All code released by the Elixir team is licensed under the +[Apache-2.0](./LICENSES/Apache-2.0.txt) license. Additionally, the following +licenses are recognized as permissible in this project: + + - The Unicode license, as documented at + [LicenseRef-scancode-unicode](./LICENSES/LicenseRef-scancode-unicode.txt) + + - The Elixir Trademark Policy, as documented at + [LicenseRef-elixir-trademark-policy](./LICENSES/LicenseRef-elixir-trademark-policy.txt) + +These licenses are considered acceptable for any files or code that form part of +an Elixir repository. If a contribution requires a different license, it must +either be rejected or prompt an update to this policy. + +## 4. Contributing to the Elixir repository + +Any code contributed to the Elixir repository must fall under one of the accepted +licenses (Apache-2.0, Unicode, or Elixir Trademark). Contributions under any +other license will be rejected unless this policy is formally revised to include +that license. All files except those specifically exempted (e.g., certain test +fixture files) must contain SPDX license and copyright headers +(`SPDX-License-Identifier` and `SPDX-FileCopyrightText`). If a file qualifies +for an exception, this must be configured in the ORT (Open Source Review Toolkit) +configuration and undergo review. + +Contributions must not introduce executable binary files into the codebase. + +## 5. Preservation of Copyright and License Information + +Any third-party code incorporated into the Elixir repository must retain original +copyright and license headers. If no such headers exist in the source, they must +be added. This practice ensures that original authors receive proper credit and +that the licensing lineage is preserved. + +## 6. Objectives + +The Elixir project aims to promote a culture of responsible open source usage. +Specifically, our objectives include: + +### 6.1 Clearly Define and Communicate Licensing & Compliance Policies + +We will identify and document all third-party dependencies, ensure that license +information is communicated clearly, and maintain a project-wide license policy +or compliance handbook. + +### 6.2 Implement Clear Processes for Reviewing Contributions + +We will provide well-defined contribution guidelines. We implement the +Developer Certificate of Origin (DCO) for additional clarity regarding +contributor rights and obligations. + +### 6.3 Track and Audit Third-Party Code Usage + +All projects will implement a Software Bill of Materials (SBoM) strategy and +regularly verify license compliance for direct and transitive dependencies. + +### 6.4 Monitor and Continuously Improve Open Source Compliance + +We will conduct periodic internal audits, integrate compliance checks into +continuous integration (CI/CD) pipelines, and regularly review and refine these +objectives to align with best practices. + +## 7. Roles and Responsibilities + +### 7.1 Core Team Member + +Core Team Members are responsible for being familiar with this policy and +ensuring it is consistently enforced. They must demonstrate sufficient +competencies to understand the policy requirements and must reject or request +changes to any pull requests that violate these standards. + +### 7.2 Contributor + +Contributors are expected to follow this policy when submitting code. If a +contributor submits a pull request that does not comply with the policy +(e.g., introduces a disallowed license), Core Team Members have the authority to +reject it or request changes. No special competencies are required for +contributors beyond awareness and adherence to the policy. + +### 7.3 EEF CISO + +The CISO designated by the Erlang Ecosystem Foundation (EEF) provides oversight +on queries and guidance regarding open source compliance or legal matters for +Elixir. The CISO is responsible for checking ongoing compliance with the policy, +escalating potential violations to the Core Team, and involving legal counsel if +necessary. This role does not require legal expertise but does involve +initiating legal or community discussions when needed. + +## 8. Implications of Failing to Follow the Program Requirements + +If a violation of this policy is identified, the Elixir Core Team will undertake +the following actions: + +## 8.1 Review the Codebase for Additional Violations + +We will investigate the codebase thoroughly to detect any similar instances of +non-compliance. + +## 8.2 Review and Update the Process or Policy + +In collaboration with the EEF CISO, the Elixir Core Team will assess the policy +and our internal workflows, making any necessary clarifications or amendments to +reduce the likelihood of recurrence. + +## 8.3 Notify and Train Core Team Members + +We will ensure that all active Core Team Members are informed about any policy +changes and understand how to apply them in everyday development. + +## 8.4 Remove or Replace the Offending Code + +If required, we will remove or replace the non-compliant code. + +## 9. Contact + +The project maintains a private mailing list at +[policy@elixir-lang.org](mailto:policy@elixir-lang.org) for handling licensing +and policy-related queries. Email is the preferred communication channel, and +the EEF CISO will be included on this list to provide assistance and ensure +timely responses. While solutions may take longer to implement, the project +commits to acknowledging all queries within five business days. + +## 10. External Contributions of Core Team Members + +When Core Team Members contribute to repositories outside Elixir, they do so in +a personal capacity or via their employer. They will not act as official +representatives of the Elixir team in those external contexts. + +## 11. Policy Review and Amendments + +This policy will be revisited annually to address new concerns, accommodate +changes in community standards, or adjust to emerging legal or technical +requirements. Proposed amendments must be reviewed by the Core Team and, if +necessary, by the EEF CISO. Any significant changes will be communicated to +contributors and made publicly available. + +*Effective Date: 2025-02-20* +*Last Reviewed: 2025-11-20* diff --git a/README.md b/README.md index 5bfeb70f5ff..e82eb0e4341 100644 --- a/README.md +++ b/README.md @@ -1,57 +1,158 @@ -![Elixir](https://github.com/elixir-lang/elixir-lang.github.com/raw/master/images/logo/logo.png) -========= -[![Build Status](https://secure.travis-ci.org/elixir-lang/elixir.svg?branch=master "Build Status")](http://travis-ci.org/elixir-lang/elixir) + -For more about Elixir, installation and documentation, [check Elixir's website](http://elixir-lang.org/). +

+ + + Elixir logo + +

-## Usage +[![CI](https://github.com/elixir-lang/elixir/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/elixir-lang/elixir/actions/workflows/ci.yml?query=branch%3Amain) +[![OpenSSF Best Practices](https://www.bestpractices.dev/projects/10187/badge)](https://www.bestpractices.dev/projects/10187) -If you want to contribute to Elixir or run it from source, clone this repository to your machine, compile and test it: +Elixir is a dynamic, functional language designed for building scalable +and maintainable applications. - $ git clone https://github.com/elixir-lang/elixir.git - $ cd elixir - $ make clean test +For more about Elixir, installation and documentation, +[check Elixir's website](https://elixir-lang.org/). -If Elixir fails to build (specifically when pulling in a new version via git), be sure to remove any previous build artifacts by running `make clean`, then `make test`. +## Policies -If tests pass, you are ready to move on to the [Getting Started guide][1] or to try Interactive Elixir by running: `bin/iex` in your terminal. +New releases are announced in the [announcement mailing list][8]. +You can subscribe by sending an email to +and replying to the confirmation email. -However, if tests fail, it is likely you have an outdated Erlang version (Elixir requires Erlang 17.0 or later). You can check your Erlang version by calling `erl` in the command line. You will see some information as follows: +All security releases [will be tagged with `[security]`][10]. For more +information, please read our [Security Policy][9]. - Erlang/OTP 17 [erts-6.0] [source-07b8f44] [64-bit] [smp:4:4] [async-threads:10] [hipe] [kernel-poll:false] +All interactions in our official communication channels follow our +[Code of Conduct][1]. -If you have the correct version and tests still fail, feel free to [open an issue][2]. +All contributions are required to conform to our [Open Source Policy][11]. -## Building documentation +## Bug reports -Building the documentation requires [ex_doc](https://github.com/elixir-lang/ex_doc) to be installed and built in the same containing folder as elixir. +For reporting bugs, [visit our issue tracker][2] and follow the steps +for reporting a new issue. **Please disclose security vulnerabilities +privately [in our Security page](https://github.com/elixir-lang/elixir/security)**. - # After cloning and compiling Elixir - $ git clone git://github.com/elixir-lang/ex_doc.git - $ cd ex_doc && ../elixir/bin/mix compile - $ cd ../elixir && make docs +All currently open bugs related to Elixir are listed in the issues tracker. +The Elixir team uses the issues tracker to focus on *actionable items*, +including planned enhancements in the short and medium term. We also do +our best to label entries for clarity and to ease collaboration. -## Contributing +Our *actionable item policy* has some important consequences, such as: + + * Proposing new features as well as requests for support, help, and + guidance must be done in their own spaces, detailed next. + + * Issues we have identified to be outside of Elixir's scope, + such as an upstream bug, will be closed (and requested to be moved + elsewhere if appropriate). + + * We actively close unrelated and non-actionable issues to keep the + issues tracker tidy. If you believe we got something wrong, drop a + comment and we can always reopen the issue. + +By keeping the overall issues tracker tidy and organized, the community +can easily peek at what is coming in new releases and also get involved +by commenting on existing issues and submitting pull requests. Please +remember to keep the tone positive and be kind! For more information, +see the [Code of Conduct][1]. + +## Discussions, support, and help + +For general discussions, support, and help, please use the community +spaces [listed on the sidebar of the Elixir website](https://elixir-lang.org/), +such as forums, chat platforms, etc, where the wider community will be available +to help you. + +## Proposing new features + +We encourage you to first propose new features in the community spaces +listed above. These discussions help refine ideas and gather feedback before +submission. Our website also includes [a general outline of the language +history and its current development focus](https://elixir-lang.org/development.html). -We appreciate any contribution to Elixir, so check out our [CONTRIBUTING.md](CONTRIBUTING.md) guide for more information. We usually keep a list of features and bugs [in the issue tracker][2]. +Once you are ready, you can submit your proposal to the [Elixir Core +mailing list][3], either through the web interface or by subscribing to +it at . Remember to include +a clear problem description, compare the proposed solution to existing +alternatives in the Elixir ecosystem (and in other languages if possible), +and consider the potential impact your changes will have on the codebase and +community. -## Important links +Once a proposal is accepted, it will be added to [the issue tracker][2]. +Features and bug fixes that have already been merged and will be included +in the next release are then "closed" and added to the [changelog][7] +before release. -* \#elixir-lang on freenode IRC -* [Website][1] -* [Issue tracker][2] -* [elixir-talk Mailing list (questions)][3] -* [elixir-core Mailing list (development)][4] +## Compiling from source + +For the many different ways to install Elixir, +[see our installation instructions on the website](https://elixir-lang.org/install.html). +However, if you want to contribute to Elixir, you will need to compile from source. + +First, [install Erlang](https://elixir-lang.org/install.html#installing-erlang). +After that, clone this repository to your machine, compile and test it: + +```sh +git clone https://github.com/elixir-lang/elixir.git +cd elixir +make +``` + +> Note: if you are running on Windows, +[this article includes important notes for compiling Elixir from source +on Windows](https://github.com/elixir-lang/elixir/wiki/Windows). + +In case you want to use this Elixir version as your system version, +you need to add the `bin` directory to [your PATH environment variable](https://elixir-lang.org/install.html#setting-path-environment-variable). + +When updating the repository, you may want to run `make clean` before +recompiling. For deterministic builds, you should set the environment +variable `ERL_COMPILER_OPTIONS=deterministic`. + +## Contributing - [1]: http://elixir-lang.org +Contributions to Elixir are always welcome! Before you get started, please check +out our [CONTRIBUTING.md](CONTRIBUTING.md) file. There you will find detailed +guidelines on how to set up your environment, run the test suite, format your +code, and submit pull requests. We also include information on our review +process, licensing requirements, and helpful tips to ensure a smooth +contribution experience. + +## Development links + + * [Elixir Documentation][6] + * [Elixir Core Mailing list (development)][3] + * [Announcement mailing list][8] + * [Code of Conduct][1] + * [Issue tracker][2] + * [Changelog][7] + * [Security Policy][9] + * **[#elixir][4]** on [Libera.Chat][5] IRC + + [1]: CODE_OF_CONDUCT.md [2]: https://github.com/elixir-lang/elixir/issues - [3]: http://groups.google.com/group/elixir-lang-talk - [4]: http://groups.google.com/group/elixir-lang-core + [3]: https://groups.google.com/group/elixir-lang-core + [4]: https://web.libera.chat/#elixir + [5]: https://libera.chat + [6]: https://elixir-lang.org/docs.html + [7]: CHANGELOG.md + [8]: https://groups.google.com/group/elixir-lang-ann + [9]: SECURITY.md + [10]: https://groups.google.com/forum/#!searchin/elixir-lang-ann/%5Bsecurity%5D%7Csort:date + [11]: OPEN_SOURCE_POLICY.md ## License -"Elixir" and the Elixir logo are copyright (c) 2012 Plataformatec. +"Elixir" and the Elixir logo are registered trademarks of The Elixir Team. -Elixir source code is released under Apache 2 License with some parts under Erlang's license (EPL). +Elixir source code is released under Apache License 2.0. -Check [LEGAL](LEGAL) and [LICENSE](LICENSE) files for more information. +Check [LICENSE](LICENSE) file for more information. diff --git a/RELEASE.md b/RELEASE.md index 965ace69e87..b93f1b68932 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,31 +1,55 @@ + + # Release process -This document simply outlines the release process: +## Shipping a new version + +1. Update version in /VERSION, bin/elixir, and bin/elixir.bat + +2. Ensure /CHANGELOG.md is updated, versioned and add the current date + +3. Update "Compatibility and Deprecations" if a new OTP version is supported + +4. Commit changes above with title "Release vVERSION" and push it + +5. Once GitHub actions completes, generate a new tag, and push it + +6. Wait until GitHub Actions publish artifacts to the draft release + +7. Copy the relevant bits from /CHANGELOG.md to the GitHub release and publish it (link to the announcement if there is one) + +8. Update `_data/elixir-versions.yml` (except for RCs) in `elixir-lang/elixir-lang.github.com` + +## Creating a new vMAJOR.MINOR branch (before first rc) + +### In the new branch -1. Remove `-dev` extension from VERSION +1. Comment out `CANONICAL := main/` in /Makefile -2. Ensure CHANGELOG is updated and timestamp +2. Update tables in /SECURITY.md and "Compatibility and Deprecations" -3. Commit changes above with title "Release vVERSION" and generate new tag +3. Commit "Branch out vMAJOR.MINOR" -4. Run `make clean test` to ensure all tests pass from scratch and the CI is green +### Back in main -5. Push master and tags +1. Bump /VERSION file, bin/elixir, and bin/elixir.bat -6. Release new docs with `make release_docs`, move docs to `docs/stable` +2. Start new /CHANGELOG.md -7. Release new zip with `make release_zip`, push new zip to GitHub Releases +3. Update tables in /SECURITY.md and in "Compatibility and Deprecations" -8. Merge master into stable branch and push it +4. Commit "Start vMAJOR.MINOR+1" -9. After release, bump versions, add `-dev` back and commit +## Changing supported Erlang/OTP versions -10. `make release_docs` once again and push it to `elixir-lang.org` +1. Update the table in Compatibility and Deprecations -11. Also update `release` file in `elixir-lang.org` +2. Update `otp_release` checks in `/Makefile` and `/lib/elixir/src/elixir.erl` -## Places where version is mentioned +3. Update relevant CI workflows in `/.github/workflows/*.yml` - for release workflows, outdated/recently added Erlang/OTP versions must run conditionally -* VERSION -* CHANGELOG -* src/elixir.app.src +4. Remove `otp_release` version checks that are no longer needed diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000000..d132954cc5b --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,30 @@ + + +# Security Policy + +## Supported versions + +Elixir applies bug fixes only to the latest minor branch. Security patches are available for the last 5 minor branches: + +Elixir version | Support +:------------- | :----------------------------- +1.20 | Development +1.19 | Bug fixes and security patches +1.18 | Security patches only +1.17 | Security patches only +1.16 | Security patches only +1.15 | Security patches only + +## Announcements + +New releases are announced in the read-only [announcements mailing list](https://groups.google.com/group/elixir-lang-ann). You can subscribe by sending an email to and replying to the confirmation email. Security notifications [will be tagged with `[security]`](https://groups.google.com/forum/#!searchin/elixir-lang-ann/%5Bsecurity%5D%7Csort:date). + +You may also see [all releases](https://github.com/elixir-lang/elixir/releases) and [consult all disclosed vulnerabilities](https://github.com/elixir-lang/elixir/security) on GitHub. + +## Reporting a vulnerability + +[Please disclose security vulnerabilities privately via GitHub](https://github.com/elixir-lang/elixir/security). diff --git a/VERSION b/VERSION index 49ccc4f4b86..734375f897d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.14.3-dev \ No newline at end of file +1.20.0-dev diff --git a/bin/elixir b/bin/elixir index d63c1d206fc..40d3a6d8c4c 100755 --- a/bin/elixir +++ b/bin/elixir @@ -1,26 +1,68 @@ #!/bin/sh -if [ $# -eq 0 ] || [ "$1" = "--help" ] || [ "$1" = "-h" ]; then - echo "Usage: `basename $0` [options] [.exs file] [data] - - -v Prints version and exit - -e \"command\" Evaluates the given command (*) - -r \"file\" Requires the given files/patterns (*) - -S \"script\"   Finds and executes the given script - -pr \"file\" Requires the given files/patterns in parallel (*) - -pa \"path\" Prepends the given path to Erlang code path (*) - -pz \"path\" Appends the given path to Erlang code path (*) - --app \"app\" Start the given app and its dependencies (*) - --erl \"switches\" Switches to be passed down to erlang (*) - --name \"name\" Makes and assigns a name to the distributed node - --sname \"name\" Makes and assigns a short name to the distributed node - --cookie \"cookie\" Sets a cookie for this distributed node - --hidden Makes a hidden node - --detached Starts the Erlang VM detached from console - --no-halt Does not halt the Erlang VM after execution - -** Options marked with (*) can be given more than once -** Options given after the .exs file or -- are passed down to the executed code -** Options can be passed to the erlang runtime using ELIXIR_ERL_OPTIONS or --erl" >&2 + +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +set -e + +ELIXIR_VERSION=1.20.0-dev + +if [ $# -eq 0 ] || { [ $# -eq 1 ] && { [ "$1" = "--help" ] || [ "$1" = "-h" ]; }; }; then + cat <&2 +Usage: $(basename "$0") [options] [.exs file] [data] + +## General options + + -e "COMMAND" Evaluates the given command (*) + -h, --help Prints this message (standalone) + -r "FILE" Requires the given files/patterns (*) + -S SCRIPT Finds and executes the given script in \$PATH + -pr "FILE" Requires the given files/patterns in parallel (*) + -pa "PATH" Prepends the given path to Erlang code path (*) + -pz "PATH" Appends the given path to Erlang code path (*) + -v, --version Prints Erlang/OTP and Elixir versions (standalone) + + --color, --no-color Enables or disables ANSI coloring + --erl "SWITCHES" Switches to be passed down to Erlang (*) + --eval "COMMAND" Evaluates the given command, same as -e (*) + --logger-otp-reports BOOL Enables or disables OTP reporting + --logger-sasl-reports BOOL Enables or disables SASL reporting + --no-halt Does not halt the Erlang VM after execution + --short-version Prints Elixir version (standalone) + +Options given after the .exs file or -- are passed down to the executed code. +Options can be passed to the Erlang runtime using \$ELIXIR_ERL_OPTIONS or --erl. + +## Distribution options + +The following options are related to node distribution. + + --cookie COOKIE Sets a cookie for this distributed node + --hidden Makes a hidden node + --name NAME Makes and assigns a name to the distributed node + --rpc-eval NODE "COMMAND" Evaluates the given command on the given remote node (*) + --sname NAME Makes and assigns a short name to the distributed node + +--name and --sname may be set to undefined so one is automatically generated. + +## Release options + +The following options are generally used under releases. + + --boot "FILE" Uses the given FILE.boot to start the system + --boot-var VAR "VALUE" Makes \$VAR available as VALUE to FILE.boot (*) + --erl-config "FILE" Loads configuration in FILE.config written in Erlang (*) + --pipe-to "PIPEDIR" "LOGDIR" Starts the Erlang VM as a named PIPEDIR and LOGDIR + --vm-args "FILE" Passes the contents in file as arguments to the VM + +--pipe-to starts Elixir detached from console (Unix-like only). +It will attempt to create PIPEDIR and LOGDIR if they don't exist. +See run_erl to learn more. To reattach, run: to_erl PIPEDIR. + +** Options marked with (*) can be given more than once. +** Standalone options can't be combined with other options. +USAGE exit 1 fi @@ -30,64 +72,175 @@ readlink_f () { if [ -h "$filename" ]; then readlink_f "$(readlink "$filename")" else - echo "`pwd -P`/$filename" + echo "$(pwd -P)/$filename" fi } -MODE="elixir" +if [ $# -eq 1 ] && [ "$1" = "--short-version" ]; then + echo "$ELIXIR_VERSION" + exit 0 +fi + +# Stores static Erlang arguments and --erl (which is passed as is) ERL="" + +# Stores erl arguments preserving spaces/quotes (mimics an array) +erl_set () { + eval "E${E}=\$1" + E=$((E + 1)) +} + +# Checks if a string starts with prefix. Usage: starts_with "$STRING" "$PREFIX" +starts_with () { + case $1 in + "$2"*) true;; + *) false;; + esac +} + +ERL_EXEC="erl" +MODE="cli" I=1 +E=0 +LENGTH=$# +set -- "$@" -extra -while [ $I -le $# ]; do - S=1 - eval "PEEK=\${$I}" - case "$PEEK" in +while [ $I -le $LENGTH ]; do + # S counts to be shifted, C counts to be copied + S=0 + C=0 + case "$1" in + +elixirc) + C=1 + ;; +iex) + C=1 MODE="iex" ;; - +elixirc) - MODE="elixirc" + -v|--no-halt|--color|--no-color) + C=1 + ;; + -e|-r|-pr|-pa|-pz|--eval|--remsh|--dot-iex|--dbg) + C=2 ;; - -v|--compile|--no-halt) + --rpc-eval) + C=3 ;; - -e|-r|-pr|-pa|-pz|--remsh|--app) + --hidden) + S=1 + ERL="$ERL -hidden" + ;; + --logger-otp-reports) S=2 + if [ "$2" = 'true' ] || [ "$2" = 'false' ]; then + ERL="$ERL -logger handle_otp_reports $2" + fi ;; - --detached|--hidden) - ERL="$ERL `echo $PEEK | cut -c 2-`" + --logger-sasl-reports) + S=2 + if [ "$2" = 'true' ] || [ "$2" = 'false' ]; then + ERL="$ERL -logger handle_sasl_reports $2" + fi + ;; + --erl) + S=2 + ERL="$ERL $2" ;; --cookie) - I=$(expr $I + 1) - eval "VAL=\${$I}" - ERL="$ERL -setcookie "$VAL"" + S=2 + erl_set "-setcookie" + erl_set "$2" ;; --sname|--name) - I=$(expr $I + 1) - eval "VAL=\${$I}" - ERL="$ERL `echo $PEEK | cut -c 2-` "$VAL"" + S=2 + erl_set "$(echo "$1" | cut -c 2-)" + erl_set "$2" ;; - --erl) - I=$(expr $I + 1) - eval "VAL=\${$I}" - ERL="$ERL "$VAL"" + --erl-config) + S=2 + erl_set "-config" + erl_set "$2" + ;; + --vm-args) + S=2 + erl_set "-args_file" + erl_set "$2" + ;; + --boot) + S=2 + erl_set "-boot" + erl_set "$2" + ;; + --boot-var) + S=3 + erl_set "-boot_var" + erl_set "$2" + erl_set "$3" + ;; + --pipe-to) + S=3 + RUN_ERL_PIPE="$2" + RUN_ERL_LOG="$3" + if [ "$(starts_with "$RUN_ERL_PIPE" "-")" ]; then + echo "--pipe-to : PIPEDIR cannot be a switch" >&2 && exit 1 + elif [ "$(starts_with "$RUN_ERL_LOG" "-")" ]; then + echo "--pipe-to : LOGDIR cannot be a switch" >&2 && exit 1 + fi ;; *) + while [ $I -le $LENGTH ]; do + I=$((I + 1)) + set -- "$@" "$1" + shift + done break ;; esac - I=$(expr $I + $S) + + while [ $I -le $LENGTH ] && [ $C -gt 0 ]; do + C=$((C - 1)) + I=$((I + 1)) + set -- "$@" "$1" + shift + done + + I=$((I + S)) + shift $S +done + +I=$((E - 1)) +while [ $I -ge 0 ]; do + eval "VAL=\$E$I" + set -- "$VAL" "$@" + I=$((I - 1)) done SELF=$(readlink_f "$0") SCRIPT_PATH=$(dirname "$SELF") -if [ "$MODE" != "iex" ]; then ERL="$ERL -noshell -s elixir start_cli"; fi -if [ -z "$ERL_PATH" ]; then - if [ -f "$SCRIPT_PATH/../releases/RELEASES" ] && [ -f "$SCRIPT_PATH/erl" ]; then - ERL_PATH="$SCRIPT_PATH"/erl - else - ERL_PATH=erl - fi +if [ "$OSTYPE" = "cygwin" ]; then SCRIPT_PATH=$(cygpath -m "$SCRIPT_PATH"); fi +if [ "$MODE" != "iex" ]; then ERL="-s elixir start_cli $ERL"; fi + +# One MAY change ERTS_BIN= but you MUST NOT change +# ERTS_BIN=$ERTS_BIN as it is handled by Elixir releases. +ERTS_BIN= +ERTS_BIN="$ERTS_BIN" + +set -- "$ERTS_BIN$ERL_EXEC" -noshell -elixir_root "$SCRIPT_PATH"/../lib -pa "$SCRIPT_PATH"/../lib/elixir/ebin $ELIXIR_ERL_OPTIONS $ERL "$@" + +if [ -n "$RUN_ERL_PIPE" ]; then + ESCAPED="" + for PART in "$@"; do + ESCAPED="$ESCAPED $(printf '%s' "$PART" | sed 's@[^a-zA-Z0-9_/-]@\\&@g')" + done + mkdir -p "$RUN_ERL_PIPE" + mkdir -p "$RUN_ERL_LOG" + ERL_EXEC="run_erl" + set -- "$ERTS_BIN$ERL_EXEC" -daemon "$RUN_ERL_PIPE/" "$RUN_ERL_LOG/" "$ESCAPED" fi -exec "$ERL_PATH" -pa "$SCRIPT_PATH"/../lib/*/ebin $ELIXIR_ERL_OPTIONS $ERL -extra "$@" +if [ -n "$ELIXIR_CLI_DRY_RUN" ]; then + echo "$@" +else + exec "$@" +fi diff --git a/bin/elixir.bat b/bin/elixir.bat index 3ed6dda1150..3b4fb11721f 100644 --- a/bin/elixir.bat +++ b/bin/elixir.bat @@ -1,95 +1,149 @@ @echo off -if "%1"=="" goto documentation -if "%1"=="--help" goto documentation -if "%1"=="-h" goto documentation -if "%1"=="/h" goto documentation + +:: SPDX-License-Identifier: Apache-2.0 +:: SPDX-FileCopyrightText: 2021 The Elixir Team +:: SPDX-FileCopyrightText: 2012 Plataformatec + +set ELIXIR_VERSION=1.20.0-dev + +if ""%1""=="""" if ""%2""=="""" goto documentation +if /I ""%1""==""--help"" if ""%2""=="""" goto documentation +if /I ""%1""==""-h"" if ""%2""=="""" goto documentation +if /I ""%1""==""/h"" if ""%2""=="""" goto documentation +if ""%1""==""/?"" if ""%2""=="""" goto documentation +if /I ""%1""==""--short-version"" if ""%2""=="""" goto shortversion goto parseopts :documentation echo Usage: %~nx0 [options] [.exs file] [data] echo. -echo -v Prints version and exit -echo -e command Evaluates the given command (*) -echo -r file Requires the given files/patterns (*) -echo -S script Finds and executes the given script -echo -pr file Requires the given files/patterns in parallel (*) -echo -pa path Prepends the given path to Erlang code path (*) -echo -pz path Appends the given path to Erlang code path (*) -echo --app app Start the given app and its dependencies (*) -echo --erl switches Switches to be passed down to erlang (*) -echo --name name Makes and assigns a name to the distributed node -echo --sname name Makes and assigns a short name to the distributed node -echo --cookie cookie Sets a cookie for this distributed node -echo --hidden Makes a hidden node -echo --detached Starts the Erlang VM detached from console -echo --no-halt Does not halt the Erlang VM after execution +echo ## General options +echo. +echo -e "COMMAND" Evaluates the given command (*) +echo -h, --help Prints this message (standalone) +echo -r "FILE" Requires the given files/patterns (*) +echo -S SCRIPT Finds and executes the given script in $PATH +echo -pr "FILE" Requires the given files/patterns in parallel (*) +echo -pa "PATH" Prepends the given path to Erlang code path (*) +echo -pz "PATH" Appends the given path to Erlang code path (*) +echo -v, --version Prints Erlang/OTP and Elixir versions (standalone) +echo. +echo --color, --no-color Enables or disables ANSI coloring +echo --erl "SWITCHES" Switches to be passed down to Erlang (*) +echo --eval "COMMAND" Evaluates the given command, same as -e (*) +echo --logger-otp-reports BOOL Enables or disables OTP reporting +echo --logger-sasl-reports BOOL Enables or disables SASL reporting +echo --no-halt Does not halt the Erlang VM after execution +echo --short-version Prints Elixir version (standalone) +echo. +echo Options given after the .exs file or -- are passed down to the executed code. +echo Options can be passed to the Erlang runtime using $ELIXIR_ERL_OPTIONS or --erl. +echo. +echo ## Distribution options +echo. +echo The following options are related to node distribution. +echo. +echo --cookie COOKIE Sets a cookie for this distributed node +echo --hidden Makes a hidden node +echo --name NAME Makes and assigns a name to the distributed node +echo --rpc-eval NODE "COMMAND" Evaluates the given command on the given remote node (*) +echo --sname NAME Makes and assigns a short name to the distributed node +echo. +echo --name and --sname may be set to undefined so one is automatically generated. +echo. +echo ## Release options +echo. +echo The following options are generally used under releases. +echo. +echo --boot "FILE" Uses the given FILE.boot to start the system +echo --boot-var VAR "VALUE" Makes $VAR available as VALUE to FILE.boot (*) +echo --erl-config "FILE" Loads configuration in FILE.config written in Erlang (*) +echo --vm-args "FILE" Passes the contents in file as arguments to the VM echo. -echo ** Options marked with (*) can be given more than once -echo ** Options given after the .exs file or -- are passed down to the executed code -echo ** Options can be passed to the erlang runtime using ELIXIR_ERL_OPTIONS or --erl -goto :EOF +echo --pipe-to is not supported on Windows. If set, Elixir won't boot. +echo. +echo ** Options marked with (*) can be given more than once. +echo ** Standalone options can't be combined with other options. +goto end + +:shortversion +echo %ELIXIR_VERSION% +goto end :parseopts +setlocal enabledelayedexpansion rem Parameters for Erlang set parsErlang= -rem Make sure we keep a copy of all parameters -set allPars=%* - -rem Get the original path name from the batch file -set originPath=%~dp0 - rem Optional parameters before the "-extra" parameter set beforeExtra= -rem Flag which determines whether or not to use werl vs erl -set useWerl=0 +rem Option which determines whether the loop is over +set endLoop=0 + +rem Designates the path to the current script +set SCRIPT_PATH=%~dp0 + +rem Designates the path to the ERTS system +set ERTS_BIN= +set ERTS_BIN=!ERTS_BIN! rem Recursive loop called for each parameter that parses the cmd line parameters :startloop -set par="%1" -shift -if "%par%"=="" ( - rem if no parameters defined - goto :expand_erl_libs -) -if "%par%"=="""" ( - rem if no parameters defined - special case for parameter that is already quoted - goto :expand_erl_libs +set "par=%~1" +if "!par!"=="" ( + rem skip if no parameter + goto run ) +shift +set par="!par:"=\"!" rem ******* EXECUTION OPTIONS ********************** -IF "%par%"==""+iex"" (Set useWerl=1) +if !par!=="+iex" (set useIEx=1 && goto startloop) +if !par!=="+elixirc" (goto startloop) +rem ******* ELIXIR PARAMETERS ********************** +if ""==!par:-e=! (shift && goto startloop) +if ""==!par:--eval=! (shift && goto startloop) +if ""==!par:--rpc-eval=! (shift && shift && goto startloop) +if ""==!par:-r=! (shift && goto startloop) +if ""==!par:-pr=! (shift && goto startloop) +if ""==!par:-pa=! (shift && goto startloop) +if ""==!par:-pz=! (shift && goto startloop) +if ""==!par:-v=! (goto startloop) +if ""==!par:--version=! (goto startloop) +if ""==!par:--no-halt=! (goto startloop) +if ""==!par:--color=! (goto startloop) +if ""==!par:--no-color=! (goto startloop) +if ""==!par:--remsh=! (shift && goto startloop) +if ""==!par:--dot-iex=! (shift && goto startloop) +if ""==!par:--dbg=! (shift && goto startloop) rem ******* ERLANG PARAMETERS ********************** -IF NOT "%par%"=="%par:--detached=%" (Set parsErlang=%parsErlang% -detached) -IF NOT "%par%"=="%par:--hidden=%" (Set parsErlang=%parsErlang% -hidden) -IF NOT "%par%"=="%par:--cookie=%" (Set parsErlang=%parsErlang% -setcookie %1 && shift) -IF NOT "%par%"=="%par:--sname=%" (Set parsErlang=%parsErlang% -sname %1 && shift) -IF NOT "%par%"=="%par:--name=%" (Set parsErlang=%parsErlang% -name %1 && shift) -IF NOT "%par%"=="%par:--erl=%" (Set beforeExtra=%beforeExtra% %~1 && shift) -rem ******* elixir parameters ********************** -rem Note: we don't have to do anything with options that don't take an argument -IF NOT "%par%"=="%par:-e=%" (shift) -IF NOT "%par%"=="%par:-r=%" (shift) -IF NOT "%par%"=="%par:-pr=%" (shift) -IF NOT "%par%"=="%par:-pa=%" (shift) -IF NOT "%par%"=="%par:-pz=%" (shift) -IF NOT "%par%"=="%par:--app=%" (shift) -IF NOT "%par%"=="%par:--remsh=%" (shift) -goto:startloop +if ""==!par:--boot=! (set "parsErlang=!parsErlang! -boot "%~1"" && shift && goto startloop) +if ""==!par:--boot-var=! (set "parsErlang=!parsErlang! -boot_var "%~1" "%~2"" && shift && shift && goto startloop) +if ""==!par:--cookie=! (set "parsErlang=!parsErlang! -setcookie "%~1"" && shift && goto startloop) +if ""==!par:--hidden=! (set "parsErlang=!parsErlang! -hidden" && goto startloop) +if ""==!par:--erl-config=! (set "parsErlang=!parsErlang! -config "%~1"" && shift && goto startloop) +if ""==!par:--logger-otp-reports=! (set "parsErlang=!parsErlang! -logger handle_otp_reports %1" && shift && goto startloop) +if ""==!par:--logger-sasl-reports=! (set "parsErlang=!parsErlang! -logger handle_sasl_reports %1" && shift && goto startloop) +if ""==!par:--name=! (set "parsErlang=!parsErlang! -name "%~1"" && shift && goto startloop) +if ""==!par:--sname=! (set "parsErlang=!parsErlang! -sname "%~1"" && shift && goto startloop) +if ""==!par:--vm-args=! (set "parsErlang=!parsErlang! -args_file "%~1"" && shift && goto startloop) +if ""==!par:--erl=! (set "beforeExtra=!beforeExtra! %~1" && shift && goto startloop) +if ""==!par:--pipe-to=! (echo --pipe-to : Option is not supported on Windows && goto end) -rem ******* assume all pre-params are parsed ******************** -:expand_erl_libs -rem ******* expand all ebin paths as Windows does not support the ..\*\ebin wildcard ******************** -SETLOCAL enabledelayedexpansion -set ext_libs= -for /d %%d in ("%originPath%..\lib\*.") do ( - set ext_libs=!ext_libs! -pa "%%~fd\ebin" -) -SETLOCAL disabledelayedexpansion :run -IF %useWerl% EQU 1 ( - werl.exe %ext_libs% %ELIXIR_ERL_OPTIONS% %parsErlang% -s elixir start_cli %beforeExtra% -extra %* -) ELSE ( - erl.exe %ext_libs% -noshell %ELIXIR_ERL_OPTIONS% %parsErlang% -s elixir start_cli %beforeExtra% -extra %* +setlocal disabledelayedexpansion +if not defined useIEx ( + set beforeExtra=-s elixir start_cli %beforeExtra% +) + +set beforeExtra=-noshell -elixir_root "%SCRIPT_PATH%..\lib" -pa "%SCRIPT_PATH%..\lib\elixir\ebin" %beforeExtra% + +if defined ELIXIR_CLI_DRY_RUN ( + echo "%ERTS_BIN%erl.exe" %ELIXIR_ERL_OPTIONS% %parsErlang% %beforeExtra% -extra %* +) else ( + "%ERTS_BIN%erl.exe" %ELIXIR_ERL_OPTIONS% %parsErlang% %beforeExtra% -extra %* ) +exit /B %ERRORLEVEL% +:end +endlocal diff --git a/bin/elixirc b/bin/elixirc index 109eca106cc..5c07d5d76ee 100755 --- a/bin/elixirc +++ b/bin/elixirc @@ -1,17 +1,30 @@ #!/bin/sh + +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +set -e + if [ $# -eq 0 ] || [ "$1" = "--help" ] || [ "$1" = "-h" ]; then - echo "Usage: `basename $0` [elixir switches] [compiler switches] [.ex files] - - -o The directory to output compiled files - --no-docs Do not attach documentation to compiled modules - --no-debug-info Do not attach debug info to compiled modules - --ignore-module-conflict - --warnings-as-errors Treat warnings as errors and return non-zero exit code - --verbose Print informational messages. - -** Options given after -- are passed down to the executed code -** Options can be passed to the erlang runtime using ELIXIR_ERL_OPTIONS -** Options can be passed to the erlang compiler using ERL_COMPILER_OPTIONS" >&2 + cat <&2 +Usage: $(basename "$0") [elixir switches] [compiler switches] [.ex files] + + -h, --help Prints this message and exits + -o The directory to output compiled files + -v, --version Prints Elixir version and exits (standalone) + + --ignore-module-conflict Does not emit warnings if a module was previously defined + --no-debug-info Does not attach debug info to compiled modules + --no-docs Does not attach documentation to compiled modules + --profile time Profile the time to compile modules + --verbose Prints compilation status + --warnings-as-errors Treats warnings as errors and return non-zero exit status + +Options given after -- are passed down to the executed code. +Options can be passed to the Erlang runtime using \$ELIXIR_ERL_OPTIONS. +Options can be passed to the Erlang compiler using \$ERL_COMPILER_OPTIONS. +USAGE exit 1 fi @@ -21,7 +34,7 @@ readlink_f () { if [ -h "$filename" ]; then readlink_f "$(readlink "$filename")" else - echo "`pwd -P`/$filename" + echo "$(pwd -P)/$filename" fi } diff --git a/bin/elixirc.bat b/bin/elixirc.bat index 9d118975d25..d1b0599f818 100644 --- a/bin/elixirc.bat +++ b/bin/elixirc.bat @@ -1,10 +1,17 @@ @echo off + +:: SPDX-License-Identifier: Apache-2.0 +:: SPDX-FileCopyrightText: 2021 The Elixir Team +:: SPDX-FileCopyrightText: 2012 Plataformatec + +setlocal set argc=0 for %%A in (%*) do ( - if "%%A"=="--help" goto documentation - if "%%A"=="-h" goto documentation - if "%%A"=="/h" goto documentation - set /A argc+=1 + if /I "%%A"=="--help" goto documentation + if /I "%%A"=="-h" goto documentation + if /I "%%A"=="/h" goto documentation + if "%%A"=="/?" goto documentation + set /A argc+=1 ) if %argc%==0 goto documentation goto run @@ -12,15 +19,24 @@ goto run :documentation echo Usage: %~nx0 [elixir switches] [compiler switches] [.ex files] echo. -echo -o The directory to output compiled files -echo --no-docs Do not attach documentation to compiled modules -echo --no-debug-info Do not attach debug info to compiled modules -echo --ignore-module-conflict -echo --warnings-as-errors Treat warnings as errors and return non-zero exit code -echo --verbose Print informational messages. +echo -h, --help Prints this message and exits +echo -o The directory to output compiled files +echo -v, --version Prints Elixir version and exits (standalone) +echo. +echo --ignore-module-conflict Does not emit warnings if a module was previously defined +echo --no-debug-info Does not attach debug info to compiled modules +echo --no-docs Does not attach documentation to compiled modules +echo --profile time Profile the time to compile modules +echo --verbose Prints compilation status +echo --warnings-as-errors Treats warnings as errors and returns non-zero exit status echo. echo ** Options given after -- are passed down to the executed code -echo ** Options can be passed to the erlang runtime using ELIXIR_ERL_OPTIONS -echo ** Options can be passed to the erlang compiler using ERL_COMPILER_OPTIONS >&2 +echo ** Options can be passed to the Erlang runtime using ELIXIR_ERL_OPTIONS +echo ** Options can be passed to the Erlang compiler using ERL_COMPILER_OPTIONS +goto end + :run call "%~dp0\elixir.bat" +elixirc %* + +:end +endlocal diff --git a/bin/iex b/bin/iex index e5187f2b08d..ba433a3ce0b 100755 --- a/bin/iex +++ b/bin/iex @@ -1,28 +1,25 @@ #!/bin/sh -if [ $# -gt 0 ] && ([ "$1" = "--help" ] || [ "$1" = "-h" ]); then - echo "Usage: `basename $0` [options] [.exs file] [data] - - -v Prints version - -e \"command\" Evaluates the given command (*) - -r \"file\" Requires the given files/patterns (*) - -S \"script\"   Finds and executes the given script - -pr \"file\" Requires the given files/patterns in parallel (*) - -pa \"path\" Prepends the given path to Erlang code path (*) - -pz \"path\" Appends the given path to Erlang code path (*) - --app \"app\" Start the given app and its dependencies (*) - --erl \"switches\" Switches to be passed down to erlang (*) - --name \"name\" Makes and assigns a name to the distributed node - --sname \"name\" Makes and assigns a short name to the distributed node - --cookie \"cookie\" Sets a cookie for this distributed node - --hidden Makes a hidden node - --detached Starts the Erlang VM detached from console - --remsh \"name\" Connects to a node using a remote shell - --dot-iex \"path\" Overrides default .iex.exs file and uses path instead; - path can be empty, then no file will be loaded - -** Options marked with (*) can be given more than once -** Options given after the .exs file or -- are passed down to the executed code -** Options can be passed to the VM using ELIXIR_ERL_OPTIONS or --erl" >&2 + +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +set -e + +if [ "$1" = "--help" ] || [ "$1" = "-h" ]; then + cat <&2 +Usage: $(basename "$0") [options] [.exs file] [data] + +The following options are exclusive to IEx: + + --dbg pry Sets the backend for Kernel.dbg/2 to IEx.pry/0 + --dot-iex "FILE" Evaluates FILE, line by line, to set up IEx' environment. + Defaults to evaluating .iex.exs or ~/.iex.exs, if any exists. + If FILE is empty, then no file will be loaded. + --remsh NAME Connects to a node using a remote shell. + +It accepts all other options listed by "elixir --help". +USAGE exit 1 fi @@ -32,10 +29,10 @@ readlink_f () { if [ -h "$filename" ]; then readlink_f "$(readlink "$filename")" else - echo "`pwd -P`/$filename" + echo "$(pwd -P)/$filename" fi } SELF=$(readlink_f "$0") SCRIPT_PATH=$(dirname "$SELF") -exec "$SCRIPT_PATH"/elixir --no-halt --erl "-user Elixir.IEx.CLI" +iex "$@" +exec "$SCRIPT_PATH"/elixir --no-halt --erl "-user elixir" +iex "$@" diff --git a/bin/iex.bat b/bin/iex.bat index 717714c3605..5739f66081b 100644 --- a/bin/iex.bat +++ b/bin/iex.bat @@ -1,2 +1,31 @@ @echo off -call "%~dp0\elixir.bat" +iex --erl "-user Elixir.IEx.CLI" --no-halt %* + +:: SPDX-License-Identifier: Apache-2.0 +:: SPDX-FileCopyrightText: 2021 The Elixir Team +:: SPDX-FileCopyrightText: 2012 Plataformatec + +setlocal +if /I ""%1""==""--help"" goto documentation +if /I ""%1""==""-h"" goto documentation +if /I ""%1""==""/h"" goto documentation +if ""%1""==""/?"" goto documentation +goto run + +:documentation +echo Usage: %~nx0 [options] [.exs file] [data] +echo. +echo The following options are exclusive to IEx: +echo. +echo --dbg pry Sets the backend for Kernel.dbg/2 to IEx.pry/0 +echo --dot-iex "FILE" Evaluates FILE, line by line, to set up IEx' environment. +echo Defaults to evaluating .iex.exs or ~/.iex.exs, if any exists. +echo If FILE is empty, then no file will be loaded. +echo --remsh NAME Connects to a node using a remote shell +echo. +echo It accepts all other options listed by "elixir --help". +goto end + +:run +call "%~dp0\elixir.bat" --no-halt --erl "-user elixir" +iex %* +:end +endlocal diff --git a/bin/mix b/bin/mix index 9dab2afd5ce..ef962748ae0 100755 --- a/bin/mix +++ b/bin/mix @@ -1,3 +1,7 @@ #!/usr/bin/env elixir -Mix.start -Mix.CLI.main + +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Mix.CLI.main() diff --git a/bin/mix.bat b/bin/mix.bat index 435a5257340..b0fde891d57 100644 --- a/bin/mix.bat +++ b/bin/mix.bat @@ -1,2 +1,7 @@ @echo off + +:: SPDX-License-Identifier: Apache-2.0 +:: SPDX-FileCopyrightText: 2021 The Elixir Team +:: SPDX-FileCopyrightText: 2012 Plataformatec + call "%~dp0\elixir.bat" "%~dp0\mix" %* diff --git a/bin/mix.ps1 b/bin/mix.ps1 old mode 100644 new mode 100755 index 9a4a36005cf..1bee46dc8d7 --- a/bin/mix.ps1 +++ b/bin/mix.ps1 @@ -1,27 +1,27 @@ -# Initialize with path to mix.bat relative to caller's working directory -$toCmd = '' + (Resolve-Path -relative (Split-Path $MyInvocation.MyCommand.Path)) + '\mix.bat' +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec -foreach ($arg in $args) +# Store path to mix.bat as a FileInfo object +$mixBatPath = (Get-ChildItem (((Get-ChildItem $MyInvocation.MyCommand.Path).Directory.FullName) + '\mix.bat')) +$newArgs = @() + +for ($i = 0; $i -lt $args.length; $i++) { - $toCmd += ' ' - - if ($arg -is [array]) + if ($args[$i] -is [array]) { # Commas created the array so we need to reintroduce those commas - for ($i = 0; $i -lt $arg.length; $i++) + for ($j = 0; $j -lt $args[$i].length - 1; $j++) { - $toCmd += $arg[$i] - if ($i -ne ($arg.length - 1)) - { - $toCmd += ', ' - } + $newArgs += ($args[$i][$j] + ',') } + $newArgs += $args[$i][-1] } else { - $toCmd += $arg + $newArgs += $args[$i] } } # Corrected arguments are ready to pass to batch file -cmd /c $toCmd \ No newline at end of file +& $mixBatPath $newArgs \ No newline at end of file diff --git a/lib/eex/lib/eex.ex b/lib/eex/lib/eex.ex index fd0095db04b..d7591873211 100644 --- a/lib/eex/lib/eex.ex +++ b/lib/eex/lib/eex.ex @@ -1,120 +1,167 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + defmodule EEx.SyntaxError do - defexception [:message] + defexception [:file, :line, :column, :snippet, message: "syntax error"] + + @impl true + def message(exception) do + %{file: file, line: line, column: column, message: message, snippet: snippet} = exception + + Exception.format_file_line_column(file && Path.relative_to_cwd(file), line, column, " ") <> + message <> (snippet || "") + end end defmodule EEx do @moduledoc ~S""" - EEx stands for Embedded Elixir. It allows you to embed - Elixir code inside a string in a robust way: + EEx stands for Embedded Elixir. - iex> EEx.eval_string "foo <%= bar %>", [bar: "baz"] - "foo baz" + Embedded Elixir allows you to embed Elixir code inside a string + in a robust way. - ## API + iex> EEx.eval_string("foo <%= bar %>", bar: "baz") + "foo baz" - This module provides 3 main APIs for you to use: + This module provides three main APIs for you to use: - 1. Evaluate a string (`eval_string`) or a file (`eval_file`) + 1. Evaluate a string (`eval_string/3`) or a file (`eval_file/3`) directly. This is the simplest API to use but also the - slowest, since the code is evaluated and not compiled before. + slowest, since the code is evaluated at runtime and not precompiled. - 2. Define a function from a string (`function_from_string`) - or a file (`function_from_file`). This allows you to embed + 2. Define a function from a string (`function_from_string/5`) + or a file (`function_from_file/5`). This allows you to embed the template as a function inside a module which will then be compiled. This is the preferred API if you have access to the template at compilation time. - 3. Compile a string (`compile_string`) or a file (`compile_file`) + 3. Compile a string (`compile_string/2`) or a file (`compile_file/2`) into Elixir syntax tree. This is the API used by both functions above and is available to you if you want to provide your own ways of handling the compiled template. + The APIs above support several options, documented below. You may + also pass an engine which customizes how the EEx code is compiled. + ## Options - All functions in this module accepts EEx-related options. - They are: + All functions in this module, unless otherwise noted, accept EEx-related + options. They are: - * `:line` - the line to be used as the template start. Defaults to 1. - * `:file` - the file to be used in the template. Defaults to the given - file the template is read from or to "nofile" when compiling - from a string. - * `:engine` - the EEx engine to be used for compilation. + * `:file` - the file to be used in the template. Defaults to the given + file the template is read from or to `"nofile"` when compiling from a string. - ## Engine + * `:line` - the line to be used as the template start. Defaults to `1`. - EEx has the concept of engines which allows you to modify or - transform the code extracted from the given string or file. + * `:indentation` - (since v1.11.0) an integer added to the column after every + new line. Defaults to `0`. - By default, `EEx` uses the `EEx.SmartEngine` that provides some - conveniences on top of the simple `EEx.Engine`. + * `:engine` - the EEx engine to be used for compilation. Defaults to `EEx.SmartEngine`. + + * `:trim` - if `true`, trims whitespace left and right of quotation as + long as at least one newline is present. All subsequent newlines and + spaces are removed but one newline is retained. Defaults to `false`. + + * `:parser_options` - (since: 1.13.0) allow customizing the parsed code + that is generated. See `Code.string_to_quoted/2` for available options. + Note that the options `:file`, `:line` and `:column` are ignored if + passed in. Defaults to `Code.get_compiler_option(:parser_options)` + (which defaults to `[]` if not set). + + ## Tags + + EEx supports multiple tags, declared below: - ### Tags + <% Elixir expression: executes code but discards output %> + <%= Elixir expression: executes code and prints result %> + <%% EEx quotation: returns the contents inside the tag as is %> + <%!-- Comments: they are discarded from source --%> - `EEx.SmartEngine` supports the following tags: + EEx supports additional tags, that may be used by some engines, + but they do not have a meaning by default: - <% Elixir expression - inline with output %> - <%= Elixir expression - replace with result %> - <%% EEx quotation - returns the contents inside %> - <%# Comments - they are discarded from source %> + <%| ... %> + <%/ ... %> - All expressions that output something to the template - **must** use the equals sign (`=`). Since everything in - Elixir is a macro, there are no exceptions for this rule. - For example, while some template languages would special- - case `if` clauses, they are treated the same in EEx and - also require `=` in order to have their result printed: + ## Engine - <%= if true do %> - It is obviously true - <% else %> - This will never appear - <% end %> + EEx has the concept of engines which allows you to modify or + transform the code extracted from the given string or file. - Notice that different engines may have different rules - for each tag. Other tags may be added in future versions. + By default, `EEx` uses the `EEx.SmartEngine` that provides some + conveniences on top of the simple `EEx.Engine`. - ### Macros + ### `EEx.SmartEngine` - `EEx.SmartEngine` also adds some macros to your template. - An example is the `@` macro which allows easy data access - in a template: + The smart engine uses EEx default rules and adds the `@` construct + for reading template assigns: - iex> EEx.eval_string "<%= @foo %>", assigns: [foo: 1] + iex> EEx.eval_string("<%= @foo %>", assigns: [foo: 1]) "1" - In other words, `<%= @foo %>` is simply translated to: + In other words, `<%= @foo %>` translates to: - <%= Dict.get assigns, :foo %> + <%= {:ok, v} = Access.fetch(assigns, :foo); v %> - The assigns extension is useful when the number of variables + The `assigns` extension is useful when the number of variables required by the template is not specified at compilation time. """ + @type line :: non_neg_integer + @type column :: non_neg_integer + @type marker :: [?=] | [?/] | [?|] | [] + @type metadata :: %{column: column, line: line} + @type token :: + {:comment, charlist, metadata} + | {:text, charlist, metadata} + | {:expr | :start_expr | :middle_expr | :end_expr, marker, charlist, metadata} + | {:eof, metadata} + + @type tokenize_opt :: + {:file, binary()} + | {:line, line} + | {:column, column} + | {:indentation, non_neg_integer} + | {:trim, boolean()} + + @type compile_opt :: + tokenize_opt + | {:engine, module()} + | {:parser_options, Code.parser_opts()} + | {atom(), term()} + @doc """ - Generates a function definition from the string. + Generates a function definition from the given string. - The kind (`:def` or `:defp`) must be given, the - function name, its arguments and the compilation options. + The first argument is the kind of the generated function (`:def` or `:defp`). + The `name` argument is the name that the generated function will have. + `template` is the string containing the EEx template. `args` is a list of arguments + that the generated function will accept. They will be available inside the EEx + template. + + The supported `options` are described [in the module docs](#module-options). + Additional options are passed to the underlying engine. ## Examples iex> defmodule Sample do ...> require EEx - ...> EEx.function_from_string :def, :sample, "<%= a + b %>", [:a, :b] + ...> EEx.function_from_string(:def, :sample, "<%= a + b %>", [:a, :b]) ...> end iex> Sample.sample(1, 2) "3" """ - defmacro function_from_string(kind, name, source, args \\ [], options \\ []) do - quote bind_quoted: binding do - info = Keyword.merge [file: __ENV__.file, line: __ENV__.line], options - args = Enum.map args, fn arg -> {arg, [line: info[:line]], nil} end - compiled = EEx.compile_string(source, info) + defmacro function_from_string(kind, name, template, args \\ [], options \\ []) do + quote bind_quoted: binding() do + info = Keyword.merge([file: __ENV__.file, line: __ENV__.line], options) + args = Enum.map(args, fn arg -> {arg, [line: info[:line]], nil} end) + compiled = EEx.compile_string(template, info) case kind do - :def -> def(unquote(name)(unquote_splicing(args)), do: unquote(compiled)) - :defp -> defp(unquote(name)(unquote_splicing(args)), do: unquote(compiled)) + :def -> def unquote(name)(unquote_splicing(args)), do: unquote(compiled) + :defp -> defp unquote(name)(unquote_splicing(args)), do: unquote(compiled) end end end @@ -122,12 +169,17 @@ defmodule EEx do @doc """ Generates a function definition from the file contents. - The kind (`:def` or `:defp`) must be given, the - function name, its arguments and the compilation options. + The first argument is the kind of the generated function (`:def` or `:defp`). + The `name` argument is the name that the generated function will have. + `file` is the path to the EEx template file. `args` is a list of arguments + that the generated function will accept. They will be available inside the EEx + template. This function is useful in case you have templates but you want to precompile inside a module for speed. + The supported `options` are described [in the module docs](#module-options). + ## Examples # sample.eex @@ -136,80 +188,183 @@ defmodule EEx do # sample.ex defmodule Sample do require EEx - EEx.function_from_file :def, :sample, "sample.eex", [:a, :b] + EEx.function_from_file(:def, :sample, "sample.eex", [:a, :b]) end # iex - Sample.sample(1, 2) #=> "3" + Sample.sample(1, 2) + #=> "3" """ defmacro function_from_file(kind, name, file, args \\ [], options \\ []) do - quote bind_quoted: binding do - info = Keyword.merge options, [file: file, line: 1] - args = Enum.map args, fn arg -> {arg, [line: 1], nil} end + quote bind_quoted: binding() do + info = Keyword.merge([file: IO.chardata_to_string(file), line: 1], options) + args = Enum.map(args, fn arg -> {arg, [line: 1], nil} end) compiled = EEx.compile_file(file, info) @external_resource file @file file case kind do - :def -> def(unquote(name)(unquote_splicing(args)), do: unquote(compiled)) - :defp -> defp(unquote(name)(unquote_splicing(args)), do: unquote(compiled)) + :def -> def unquote(name)(unquote_splicing(args)), do: unquote(compiled) + :defp -> defp unquote(name)(unquote_splicing(args)), do: unquote(compiled) end end end @doc """ - Get a string `source` and generate a quoted expression + Gets a string `source` and generates a quoted expression that can be evaluated by Elixir or compiled to a function. + + This is useful if you want to compile a EEx template into code and inject + that code somewhere or evaluate it at runtime. + + The generated quoted code will use variables defined in the template that + will be taken from the context where the code is evaluated. If you + have a template such as `<%= a + b %>`, then the returned quoted code + will use the `a` and `b` variables in the context where it's evaluated. See + examples below. + + The supported `options` are described [in the module docs](#module-options). + + ## Examples + + iex> quoted = EEx.compile_string("<%= a + b %>") + iex> {result, _bindings} = Code.eval_quoted(quoted, a: 1, b: 2) + iex> result + "3" + """ - def compile_string(source, options \\ []) do - EEx.Compiler.compile(source, options) + @spec compile_string(String.t(), [compile_opt]) :: Macro.t() + def compile_string(source, options \\ []) when is_binary(source) and is_list(options) do + tokenize_opts = Keyword.take(options, [:file, :line, :column, :indentation, :trim]) + + case tokenize(source, tokenize_opts) do + {:ok, tokens} -> + EEx.Compiler.compile(tokens, source, options) + + {:error, message, %{column: column, line: line}} -> + file = options[:file] || "nofile" + raise EEx.SyntaxError, file: file, line: line, column: column, message: message + end end @doc """ - Get a `filename` and generate a quoted expression + Gets a `filename` and generates a quoted expression that can be evaluated by Elixir or compiled to a function. + + This is useful if you want to compile a EEx template into code and inject + that code somewhere or evaluate it at runtime. + + The generated quoted code will use variables defined in the template that + will be taken from the context where the code is evaluated. If you + have a template such as `<%= a + b %>`, then the returned quoted code + will use the `a` and `b` variables in the context where it's evaluated. See + examples below. + + The supported `options` are described [in the module docs](#module-options). + + ## Examples + + # sample.eex + <%= a + b %> + + # In code: + quoted = EEx.compile_file("sample.eex") + {result, _bindings} = Code.eval_quoted(quoted, a: 1, b: 2) + result + #=> "3" + """ - def compile_file(filename, options \\ []) do - options = Keyword.merge options, [file: filename, line: 1] + @spec compile_file(Path.t(), [compile_opt]) :: Macro.t() + def compile_file(filename, options \\ []) when is_list(options) do + filename = IO.chardata_to_string(filename) + options = Keyword.merge([file: filename, line: 1], options) compile_string(File.read!(filename), options) end @doc """ - Get a string `source` and evaluate the values using the `bindings`. + Gets a string `source` and evaluate the values using the `bindings`. + + The supported `options` are described [in the module docs](#module-options). ## Examples - iex> EEx.eval_string "foo <%= bar %>", [bar: "baz"] + iex> EEx.eval_string("foo <%= bar %>", bar: "baz") "foo baz" """ - def eval_string(source, bindings \\ [], options \\ []) do + @spec eval_string(String.t(), keyword, [compile_opt]) :: term() + def eval_string(source, bindings \\ [], options \\ []) + when is_binary(source) and is_list(bindings) and is_list(options) do compiled = compile_string(source, options) do_eval(compiled, bindings, options) end @doc """ - Get a `filename` and evaluate the values using the `bindings`. + Gets a `filename` and evaluate the values using the `bindings`. + + The supported `options` are described [in the module docs](#module-options). ## Examples - # sample.ex + # sample.eex foo <%= bar %> - # iex - EEx.eval_file "sample.ex", [bar: "baz"] #=> "foo baz" + # IEx + EEx.eval_file("sample.eex", bar: "baz") + #=> "foo baz" """ - def eval_file(filename, bindings \\ [], options \\ []) do - options = Keyword.put options, :file, filename + @spec eval_file(Path.t(), keyword, [compile_opt]) :: String.t() + def eval_file(filename, bindings \\ [], options \\ []) + when is_list(bindings) and is_list(options) do + filename = IO.chardata_to_string(filename) + options = Keyword.put_new(options, :file, filename) compiled = compile_file(filename, options) do_eval(compiled, bindings, options) end + @doc """ + Tokenize the given contents according to the given options. + + ## Options + + * `:line` - An integer to start as line. Default is 1. + * `:column` - An integer to start as column. Default is 1. + * `:indentation` - An integer that indicates the indentation. Default is 0. + * `:trim` - Tells the tokenizer to either trim the content or not. Default is false. + * `:file` - Can be either a file or a string "nofile". + + ## Examples + + iex> EEx.tokenize(~c"foo", line: 1, column: 1) + {:ok, [{:text, ~c"foo", %{column: 1, line: 1}}, {:eof, %{column: 4, line: 1}}]} + + ## Result + + It returns `{:ok, [token]}` where a token is one of: + + * `{:text, content, %{column: column, line: line}}` + * `{:expr, marker, content, %{column: column, line: line}}` + * `{:start_expr, marker, content, %{column: column, line: line}}` + * `{:middle_expr, marker, content, %{column: column, line: line}}` + * `{:end_expr, marker, content, %{column: column, line: line}}` + * `{:eof, %{column: column, line: line}}` + + Or `{:error, message, %{column: column, line: line}}` in case of errors. + Note new tokens may be added in the future. + """ + @doc since: "1.14.0" + @spec tokenize([char()] | String.t(), [tokenize_opt]) :: + {:ok, [token()]} | {:error, String.t(), metadata()} + def tokenize(contents, opts \\ []) do + EEx.Compiler.tokenize(contents, opts) + end + ### Helpers defp do_eval(compiled, bindings, options) do + options = Keyword.take(options, [:file, :line, :module, :prune_binding]) {result, _} = Code.eval_quoted(compiled, bindings, options) result end diff --git a/lib/eex/lib/eex/compiler.ex b/lib/eex/lib/eex/compiler.ex index c34ef566c60..56d0a413971 100644 --- a/lib/eex/lib/eex/compiler.ex +++ b/lib/eex/lib/eex/compiler.ex @@ -1,100 +1,485 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + defmodule EEx.Compiler do @moduledoc false # When changing this setting, don't forget to update the docs for EEx @default_engine EEx.SmartEngine + @h_spaces [?\s, ?\t] + @all_spaces [?\s, ?\t, ?\n, ?\r] + + @doc """ + Tokenize EEx contents. + """ + def tokenize(contents, opts) when is_binary(contents) do + tokenize(String.to_charlist(contents), contents, opts) + end + + def tokenize(contents, opts) when is_list(contents) do + tokenize(contents, List.to_string(contents), opts) + end + + def tokenize(contents, source, opts) when is_list(contents) do + file = opts[:file] || "nofile" + line = opts[:line] || 1 + trim = opts[:trim] || false + indentation = opts[:indentation] || 0 + column = indentation + (opts[:column] || 1) + + state = %{trim: trim, indentation: indentation, file: file, source: source} + + {contents, line, column} = + (trim && trim_init(contents, line, column, state)) || {contents, line, column} + + tokenize(contents, line, column, state, [{line, column}], []) + end + + defp tokenize(~c"<%%" ++ t, line, column, state, buffer, acc) do + tokenize(t, line, column + 3, state, [?%, ?< | buffer], acc) + end + + defp tokenize(~c"<%!--" ++ t, line, column, state, buffer, acc) do + case comment(t, line, column + 5, state, []) do + {:error, message} -> + meta = %{line: line, column: column} + {:error, message <> code_snippet(state.source, state.indentation, meta), meta} + + {:ok, new_line, new_column, rest, comments} -> + token = {:comment, Enum.reverse(comments), %{line: line, column: column}} + trim_and_tokenize(rest, new_line, new_column, state, buffer, acc, &[token | &1]) + end + end + + # TODO: Remove me on Elixir v2.0 + defp tokenize(~c"<%#" ++ t, line, column, state, buffer, acc) do + IO.warn("<%# is deprecated, use <%!-- or add a space between <% and # instead", + line: line, + column: column, + file: state.file + ) + + case expr(t, line, column + 3, state, []) do + {:error, message} -> + {:error, message, %{line: line, column: column}} + + {:ok, _, new_line, new_column, rest} -> + trim_and_tokenize(rest, new_line, new_column, state, buffer, acc, & &1) + end + end + + defp tokenize(~c"<%" ++ t, line, column, state, buffer, acc) do + {marker, t} = retrieve_marker(t) + marker_length = length(marker) + + case expr(t, line, column + 2 + marker_length, state, []) do + {:error, message} -> + meta = %{line: line, column: column} + {:error, message <> code_snippet(state.source, state.indentation, meta), meta} + + {:ok, expr, new_line, new_column, rest} -> + {key, expr} = + case :elixir_tokenizer.tokenize(expr, 1, file: "eex", check_terminators: false) do + {:ok, _line, _column, _warnings, rev_tokens, []} -> + # We ignore warnings because the code will be tokenized + # again later with the right line+column info + token_key(rev_tokens, expr) + + {:error, _, _, _, _} -> + {:expr, expr} + end + + marker = + if key in [:middle_expr, :end_expr] and marker != ~c"" do + message = + "unexpected beginning of EEx tag \"<%#{marker}\" on \"<%#{marker}#{expr}%>\", " <> + "please remove \"#{marker}\"" + + IO.warn(message, file: state.file, line: line, column: column) + ~c"" + else + marker + end + + token = {key, marker, expr, %{line: line, column: column}} + trim_and_tokenize(rest, new_line, new_column, state, buffer, acc, &[token | &1]) + end + end + + defp tokenize([?\n | t], line, _column, state, buffer, acc) do + tokenize(t, line + 1, state.indentation + 1, state, [?\n | buffer], acc) + end + + defp tokenize([h | t], line, column, state, buffer, acc) do + tokenize(t, line, column + 1, state, [h | buffer], acc) + end + + defp tokenize([], line, column, _state, buffer, acc) do + eof = {:eof, %{line: line, column: column}} + {:ok, Enum.reverse([eof | tokenize_text(buffer, acc)])} + end + + defp trim_and_tokenize(rest, line, column, state, buffer, acc, fun) do + {rest, line, column, buffer} = trim_if_needed(rest, line, column, state, buffer) + + acc = tokenize_text(buffer, acc) + tokenize(rest, line, column, state, [{line, column}], fun.(acc)) + end + + # Retrieve marker for <% + + defp retrieve_marker([marker | t]) when marker in [?=, ?/, ?|] do + {[marker], t} + end + + defp retrieve_marker(t) do + {~c"", t} + end + + # Tokenize a multi-line comment until we find --%> + + defp comment([?-, ?-, ?%, ?> | t], line, column, _state, buffer) do + {:ok, line, column + 4, t, buffer} + end + + defp comment([?\n | t], line, _column, state, buffer) do + comment(t, line + 1, state.indentation + 1, state, [?\n | buffer]) + end + + defp comment([head | t], line, column, state, buffer) do + comment(t, line, column + 1, state, [head | buffer]) + end + + defp comment([], _line, _column, _state, _buffer) do + {:error, "expected closing '--%>' for EEx expression"} + end + + # Tokenize an expression until we find %> + + defp expr([?%, ?> | t], line, column, _state, buffer) do + {:ok, Enum.reverse(buffer), line, column + 2, t} + end + + defp expr([?\n | t], line, _column, state, buffer) do + expr(t, line + 1, state.indentation + 1, state, [?\n | buffer]) + end + + defp expr([h | t], line, column, state, buffer) do + expr(t, line, column + 1, state, [h | buffer]) + end + + defp expr([], _line, _column, _state, _buffer) do + {:error, "expected closing '%>' for EEx expression"} + end + + # Receives tokens and check if it is a start, middle or an end token. + defp token_key(rev_tokens, expr) do + case {Enum.reverse(rev_tokens), drop_eol(rev_tokens)} do + {[{:end, _} | _], [{:do, _} | _]} -> + {:middle_expr, expr} + + {_, [{:do, _} | _]} -> + {:start_expr, maybe_append_space(expr)} + + {_, [{:block_identifier, _, _} | _]} -> + {:middle_expr, maybe_append_space(expr)} + + {[{:end, _} | _], [{:stab_op, _, _} | _]} -> + {:middle_expr, expr} + + {_, [{:stab_op, _, _} | reverse_tokens]} -> + fn_index = Enum.find_index(reverse_tokens, &match?({:fn, _}, &1)) || :infinity + end_index = Enum.find_index(reverse_tokens, &match?({:end, _}, &1)) || :infinity + + if end_index > fn_index do + {:start_expr, expr} + else + {:middle_expr, expr} + end + + {tokens, _} -> + case Enum.drop_while(tokens, &closing_bracket?/1) do + [{:end, _} | _] -> {:end_expr, expr} + _ -> {:expr, expr} + end + end + end + + defp drop_eol([{:eol, _} | rest]), do: drop_eol(rest) + defp drop_eol(rest), do: rest + + defp maybe_append_space([?\s]), do: [?\s] + defp maybe_append_space([h]), do: [h, ?\s] + defp maybe_append_space([h | t]), do: [h | maybe_append_space(t)] + + defp closing_bracket?({closing, _}) when closing in ~w"( [ {"a, do: true + defp closing_bracket?(_), do: false + + # Tokenize the buffered text by appending + # it to the given accumulator. + + defp tokenize_text([{_line, _column}], acc) do + acc + end + + defp tokenize_text(buffer, acc) do + [{line, column} | buffer] = Enum.reverse(buffer) + [{:text, buffer, %{line: line, column: column}} | acc] + end + + ## Trim + + defp trim_if_needed(rest, line, column, state, buffer) do + if state.trim do + buffer = trim_left(buffer, 0) + {rest, line, column} = trim_right(rest, line, column, 0, state) + {rest, line, column, buffer} + else + {rest, line, column, buffer} + end + end + + defp trim_init([h | t], line, column, state) when h in @h_spaces, + do: trim_init(t, line, column + 1, state) + + defp trim_init([?\r, ?\n | t], line, _column, state), + do: trim_init(t, line + 1, state.indentation + 1, state) + + defp trim_init([?\n | t], line, _column, state), + do: trim_init(t, line + 1, state.indentation + 1, state) + + defp trim_init([?<, ?% | _] = rest, line, column, _state), + do: {rest, line, column} + + defp trim_init(_, _, _, _), do: false + + defp trim_left(buffer, count) do + case trim_whitespace(buffer, 0) do + {[?\n, ?\r | rest], _} -> trim_left(rest, count + 1) + {[?\n | rest], _} -> trim_left(rest, count + 1) + _ when count > 0 -> [?\n | buffer] + _ -> buffer + end + end + + defp trim_right(rest, line, column, last_column, state) do + case trim_whitespace(rest, column) do + {[?\r, ?\n | rest], column} -> + trim_right(rest, line + 1, state.indentation + 1, column + 1, state) + + {[?\n | rest], column} -> + trim_right(rest, line + 1, state.indentation + 1, column, state) + + {[], column} -> + {[], line, column} + + _ when last_column > 0 -> + {[?\n | rest], line - 1, last_column} + + _ -> + {rest, line, column} + end + end + + defp trim_whitespace([h | t], column) when h in @h_spaces, do: trim_whitespace(t, column + 1) + defp trim_whitespace(list, column), do: {list, column} @doc """ This is the compilation entry point. It glues the tokenizer and the engine together by handling the tokens and invoking the engine every time a full expression or text is received. """ - def compile(source, opts) do - file = opts[:file] || "nofile" - line = opts[:line] || 1 - tokens = EEx.Tokenizer.tokenize(source, line) - state = %{engine: opts[:engine] || @default_engine, - file: file, line: line, quoted: [], start_line: nil} - generate_buffer(tokens, "", [], state) + @spec compile([EEx.token()], String.t(), keyword) :: Macro.t() + def compile(tokens, source, opts) do + file = opts[:file] || "nofile" + line = opts[:line] || 1 + indentation = opts[:indentation] || 0 + parser_options = opts[:parser_options] || Code.get_compiler_option(:parser_options) + engine = opts[:engine] || @default_engine + + state = %{ + engine: engine, + file: file, + source: source, + line: line, + quoted: [], + parser_options: [indentation: indentation] ++ parser_options, + indentation: indentation + } + + init = state.engine.init(opts) + + if function_exported?(state.engine, :handle_text, 2) and + not function_exported?(state.engine, :handle_text, 3) do + IO.warn( + "#{inspect(state.engine)}.handle_text/2 is deprecated, implement handle_text/3 instead" + ) + end + + generate_buffer(tokens, init, [], state) end - # Generates the buffers by handling each expression from the tokenizer + # Ignore tokens related to comment. + defp generate_buffer([{:comment, _chars, _meta} | rest], buffer, scope, state) do + generate_buffer(rest, buffer, scope, state) + end + + # Generates the buffers by handling each expression from the tokenizer. + # It returns Macro.t/0 or it raises. - defp generate_buffer([{:text, chars}|t], buffer, scope, state) do - buffer = state.engine.handle_text(buffer, IO.chardata_to_string(chars)) - generate_buffer(t, buffer, scope, state) + defp generate_buffer([{:text, chars, meta} | rest], buffer, scope, state) do + buffer = + if function_exported?(state.engine, :handle_text, 3) do + meta = [line: meta.line, column: meta.column] + state.engine.handle_text(buffer, meta, IO.chardata_to_string(chars)) + else + # TODO: Remove this on Elixir v2.0. The deprecation is on init. + state.engine.handle_text(buffer, IO.chardata_to_string(chars)) + end + + generate_buffer(rest, buffer, scope, state) end - defp generate_buffer([{:expr, line, mark, chars}|t], buffer, scope, state) do - expr = Code.string_to_quoted!(chars, [line: line, file: state.file]) - buffer = state.engine.handle_expr(buffer, mark, expr) - generate_buffer(t, buffer, scope, state) + defp generate_buffer([{:expr, mark, chars, meta} | rest], buffer, scope, state) do + options = + [file: state.file, line: meta.line, column: column(meta.column, mark)] ++ + state.parser_options + + expr = Code.string_to_quoted!(chars, options) + buffer = state.engine.handle_expr(buffer, IO.chardata_to_string(mark), expr) + generate_buffer(rest, buffer, scope, state) end - defp generate_buffer([{:start_expr, start_line, mark, chars}|t], buffer, scope, state) do - {contents, line, t} = look_ahead_text(t, start_line, chars) - {contents, t} = generate_buffer(t, "", [contents|scope], - %{state | quoted: [], line: line, start_line: start_line}) - buffer = state.engine.handle_expr(buffer, mark, contents) - generate_buffer(t, buffer, scope, state) + defp generate_buffer( + [{:start_expr, mark, chars, meta} | rest], + buffer, + scope, + state + ) do + {rest, line, contents} = look_ahead_middle(rest, meta.line, chars) || {rest, meta.line, chars} + start_line = meta.line + start_column = column(meta.column, mark) + + {contents, rest} = + generate_buffer( + rest, + state.engine.handle_begin(buffer), + [{contents, start_line, start_column} | scope], + %{state | quoted: [], line: line} + ) + + if mark == ~c"" and not match?({:=, _, [_, _]}, contents) do + message = + "the contents of this expression won't be output unless the EEx block starts with \"<%=\"" + + IO.warn(message, file: state.file, line: meta.line, column: meta.column) + end + + buffer = state.engine.handle_expr(buffer, IO.chardata_to_string(mark), contents) + generate_buffer(rest, buffer, scope, state) + end + + defp generate_buffer( + [{:middle_expr, ~c"", chars, meta} | rest], + buffer, + [{current, current_line, current_column} | scope], + state + ) do + {wrapped, state} = wrap_expr(current, meta.line, buffer, chars, state) + state = %{state | line: meta.line} + + generate_buffer( + rest, + state.engine.handle_begin(buffer), + [{wrapped, current_line, current_column} | scope], + state + ) end - defp generate_buffer([{:middle_expr, line, _, chars}|t], buffer, [current|scope], state) do - {wrapped, state} = wrap_expr(current, line, buffer, chars, state) - generate_buffer(t, "", [wrapped|scope], %{state | line: line}) + defp generate_buffer([{:middle_expr, _, chars, meta} | _tokens], _buffer, [], state) do + message = "unexpected middle of expression <%#{chars}%>" + syntax_error!(message, meta, state) end - defp generate_buffer([{:end_expr, line, _, chars}|t], buffer, [current|_], state) do - {wrapped, state} = wrap_expr(current, line, buffer, chars, state) - tuples = Code.string_to_quoted!(wrapped, [line: state.start_line, file: state.file]) + defp generate_buffer( + [{:end_expr, ~c"", chars, meta} | rest], + buffer, + [{current, line, column} | _], + state + ) do + {wrapped, state} = wrap_expr(current, meta.line, buffer, chars, state) + options = [file: state.file, line: line, column: column] ++ state.parser_options + tuples = Code.string_to_quoted!(wrapped, options) buffer = insert_quoted(tuples, state.quoted) - {buffer, t} + {buffer, rest} end - defp generate_buffer([{:end_expr, line, _, chars}|_], _buffer, [], _state) do - raise EEx.SyntaxError, message: "unexpected token: #{inspect chars} at line #{inspect line}" + defp generate_buffer([{:end_expr, _, chars, meta} | _], _buffer, [], state) do + message = "unexpected end of expression <%#{chars}%>" + syntax_error!(message, meta, state) end - defp generate_buffer([], buffer, [], state) do + defp generate_buffer([{:eof, _meta}], buffer, [], state) do state.engine.handle_body(buffer) end - defp generate_buffer([], _buffer, _scope, _state) do - raise EEx.SyntaxError, message: "unexpected end of string. expecting a closing <% end %>." + defp generate_buffer([{:eof, _meta}], _buffer, [{content, line, column} | _scope], state) do + message = "expected a closing '<% end %>' for block expression in EEx" + expr_meta = non_whitespace_meta(content, line, column, state) + syntax_error!(message, expr_meta, state) end + defp non_whitespace_meta([space | rest], line, column, state) when space in @h_spaces, + do: non_whitespace_meta(rest, line, column + 1, state) + + defp non_whitespace_meta([?\n | rest], line, _column, state), + do: non_whitespace_meta(rest, line + 1, state.indentation + 1, state) + + defp non_whitespace_meta(_, line, column, _), + do: %{line: line, column: column} + # Creates a placeholder and wrap it inside the expression block defp wrap_expr(current, line, buffer, chars, state) do new_lines = List.duplicate(?\n, line - state.line) key = length(state.quoted) - placeholder = '__EEX__(' ++ Integer.to_char_list(key) ++ ');' - {current ++ placeholder ++ new_lines ++ chars, - %{state | quoted: [{key, buffer}|state.quoted]}} + placeholder = ~c"__EEX__(" ++ Integer.to_charlist(key) ++ ~c");" + count = current ++ placeholder ++ new_lines ++ chars + new_state = %{state | quoted: [{key, state.engine.handle_end(buffer)} | state.quoted]} + + {count, new_state} end - # Look text ahead on expressions + # Look middle expressions that immediately follow a start_expr - defp look_ahead_text([{:text, text}, {:middle_expr, line, _, chars}|t]=list, start, contents) do + defp look_ahead_middle([{:comment, _comment, _meta} | rest], start, contents), + do: look_ahead_middle(rest, start, contents) + + defp look_ahead_middle([{:text, text, _meta} | rest], start, contents) do if only_spaces?(text) do - {contents ++ text ++ chars, line, t} + look_ahead_middle(rest, start, contents ++ text) else - {contents, start, list} + nil end end - defp look_ahead_text(t, start, contents) do - {contents, start, t} + defp look_ahead_middle([{:middle_expr, _, chars, meta} | rest], _start, contents) do + {rest, meta.line, contents ++ chars} + end + + defp look_ahead_middle(_tokens, _start, _contents) do + nil end defp only_spaces?(chars) do - Enum.all?(chars, &(&1 in [?\s, ?\t, ?\r, ?\n])) + Enum.all?(chars, &(&1 in @all_spaces)) end # Changes placeholder to real expression defp insert_quoted({:__EEX__, _, [key]}, quoted) do - {^key, value} = List.keyfind quoted, key, 0 + {^key, value} = List.keyfind(quoted, key, 0) value end @@ -107,10 +492,49 @@ defmodule EEx.Compiler do end defp insert_quoted(list, quoted) when is_list(list) do - Enum.map list, &insert_quoted(&1, quoted) + Enum.map(list, &insert_quoted(&1, quoted)) end defp insert_quoted(other, _quoted) do other end + + defp column(column, mark) do + # length(~c"<%") == 2 + column + 2 + length(mark) + end + + defp syntax_error!(message, meta, state) do + raise EEx.SyntaxError, + message: message, + snippet: code_snippet(state.source, state.indentation, meta), + file: state.file, + line: meta.line, + column: meta.column + end + + defp code_snippet(source, indentation, meta) do + line_start = max(meta.line - 3, 1) + line_end = meta.line + digits = line_end |> Integer.to_string() |> byte_size() + number_padding = String.duplicate(" ", digits) + indentation = String.duplicate(" ", indentation) + + source + |> String.split(["\r\n", "\n"]) + |> Enum.slice((line_start - 1)..(line_end - 1)) + |> Enum.map_reduce(line_start, fn + expr, line_number when line_number == line_end -> + arrow = String.duplicate(" ", meta.column - 1) <> "^" + {"#{line_number} | #{indentation}#{expr}\n #{number_padding}| #{arrow}", line_number + 1} + + expr, line_number -> + line_number_padding = String.pad_leading("#{line_number}", digits) + {"#{line_number_padding} | #{indentation}#{expr}", line_number + 1} + end) + |> case do + {[], _} -> "" + {snippet, _} -> Enum.join(["\n #{number_padding}|" | snippet], "\n") + end + end end diff --git a/lib/eex/lib/eex/engine.ex b/lib/eex/lib/eex/engine.ex index 257f2f46d18..12234f048c8 100644 --- a/lib/eex/lib/eex/engine.ex +++ b/lib/eex/lib/eex/engine.ex @@ -1,112 +1,229 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + defmodule EEx.Engine do @moduledoc ~S""" Basic EEx engine that ships with Elixir. - An engine needs to implement three functions: + An engine needs to implement all callbacks below. + + This module also ships with a default engine implementation + you can delegate to. See `EEx.SmartEngine` as an example. + """ - * `handle_body(quoted)` - receives the final built quoted - expression, should do final post-processing and return a - quoted expression. + @type state :: term - * `handle_text(buffer, text)` - it receives the buffer, - the text and must return a new quoted expression. + @doc """ + Called at the beginning of every template. - * `handle_expr(buffer, marker, expr)` - it receives the buffer, - the marker, the expr and must return a new quoted expression. + It receives the options during compilation, including the + ones managed by EEx, such as `:line` and `:file`, as well + as custom engine options. - The marker is what follows exactly after `<%`. For example, - `<% foo %>` has an empty marker, but `<%= foo %>` has `"="` - as marker. The allowed markers so far are: `""` and `"="`. + It must return the initial state. + """ + @callback init(opts :: keyword) :: state - Read `handle_expr/3` below for more information about the markers - implemented by default by this engine. + @doc """ + Called at the end of every template. - `EEx.Engine` can be used directly if one desires to use the - default implementations for the functions above. + It must return Elixir's quoted expressions for the template. """ + @callback handle_body(state) :: Macro.t() - use Behaviour + @doc """ + Called for the text/static parts of a template. - defcallback handle_body(Macro.t) :: Macro.t - defcallback handle_text(Macro.t, binary) :: Macro.t - defcallback handle_expr(Macro.t, binary, Macro.t) :: Macro.t + It must return the updated state. + """ + @callback handle_text(state, [line: pos_integer, column: pos_integer], text :: String.t()) :: + state + + @doc """ + Called for the dynamic/code parts of a template. + + The marker is what follows exactly after `<%`. For example, + `<% foo %>` has an empty marker, but `<%= foo %>` has `"="` + as marker. The allowed markers so far are: + + * `""` + * `"="` + * `"/"` + * `"|"` + + Markers `"/"` and `"|"` are only for use in custom EEx engines + and are not implemented by default. Using them without an + appropriate implementation raises `EEx.SyntaxError`. + + It must return the updated state. + """ + @callback handle_expr(state, marker :: String.t(), expr :: Macro.t()) :: state + + @doc """ + Invoked at the beginning of every nesting. + + It must return a new state that is used only inside the nesting. + Once the nesting terminates, the current `state` is resumed. + """ + @callback handle_begin(state) :: state + + @doc """ + Invokes at the end of a nesting. + + It must return Elixir's quoted expressions for the nesting. + """ + @callback handle_end(state) :: Macro.t() @doc false + @deprecated "Use explicit delegation to EEx.Engine instead" defmacro __using__(_) do quote do @behaviour EEx.Engine - def handle_body(body) do - EEx.Engine.handle_body(body) + def init(opts) do + EEx.Engine.init(opts) + end + + def handle_body(state) do + EEx.Engine.handle_body(state) + end + + def handle_begin(state) do + EEx.Engine.handle_begin(state) + end + + def handle_end(state) do + EEx.Engine.handle_end(state) end - def handle_text(buffer, text) do - EEx.Engine.handle_text(buffer, text) + def handle_text(state, text) do + EEx.Engine.handle_text(state, [], text) end - def handle_expr(buffer, mark, expr) do - EEx.Engine.handle_expr(buffer, mark, expr) + def handle_expr(state, marker, expr) do + EEx.Engine.handle_expr(state, marker, expr) end - defoverridable [handle_body: 1, handle_expr: 3, handle_text: 2] + defoverridable EEx.Engine end end @doc """ Handles assigns in quoted expressions. + A warning will be printed on missing assigns. + Future versions will raise. + This can be added to any custom engine by invoking - `handle_assign/3` with `Macro.prewalk/1`: + `handle_assign/1` with `Macro.prewalk/2`: - def handle_expr(buffer, token, expr) do + def handle_expr(state, token, expr) do expr = Macro.prewalk(expr, &EEx.Engine.handle_assign/1) - EEx.Engine.handle_expr(buffer, token, expr) + super(state, token, expr) end """ - def handle_assign({:@, line, [{name, _, atom}]}) when is_atom(name) and is_atom(atom) do - quote line: line, do: Dict.get(var!(assigns), unquote(name)) + @spec handle_assign(Macro.t()) :: Macro.t() + def handle_assign({:@, meta, [{name, _, atom}]}) when is_atom(name) and is_atom(atom) do + line = meta[:line] || 0 + quote(line: line, do: EEx.Engine.fetch_assign!(var!(assigns), unquote(name))) end def handle_assign(arg) do arg end - @doc """ - The default implementation implementation simply returns the - given expression. - """ - def handle_body(quoted) do - quoted + @doc false + # TODO: Raise on v2.0 + @spec fetch_assign!(Access.t(), Access.key()) :: term | nil + def fetch_assign!(assigns, key) do + case Access.fetch(assigns, key) do + {:ok, val} -> + val + + :error -> + keys = Enum.map(assigns, &elem(&1, 0)) + + IO.warn( + "assign @#{key} not available in EEx template. " <> + "Please ensure all assigns are given as options. " <> + "Available assigns: #{inspect(keys)}" + ) + + nil + end end - @doc """ - The default implementation simply concatenates text to the buffer. - """ - def handle_text(buffer, text) do - quote do: unquote(buffer) <> unquote(text) + @doc "Default implementation for `c:init/1`." + def init(_opts) do + %{ + binary: [], + dynamic: [], + vars_count: 0 + } end - @doc """ - Implements expressions according to the markers. + @doc "Default implementation for `c:handle_begin/1`." + def handle_begin(state) do + check_state!(state) + %{state | binary: [], dynamic: []} + end - <% Elixir expression - inline with output %> - <%= Elixir expression - replace with result %> + @doc "Default implementation for `c:handle_end/1`." + def handle_end(quoted) do + handle_body(quoted) + end - All other markers are not implemented by this engine. - """ - def handle_expr(buffer, "=", expr) do - quote do - tmp = unquote(buffer) - tmp <> to_string(unquote(expr)) - end + @doc "Default implementation for `c:handle_body/1`." + def handle_body(state) do + check_state!(state) + %{binary: binary, dynamic: dynamic} = state + binary = {:<<>>, [], Enum.reverse(binary)} + dynamic = [binary | dynamic] + {:__block__, [], Enum.reverse(dynamic)} end - def handle_expr(buffer, "", expr) do - quote do - tmp = unquote(buffer) - unquote(expr) - tmp - end + @doc "Default implementation for `c:handle_text/3`." + def handle_text(state, _meta, text) do + check_state!(state) + %{binary: binary} = state + %{state | binary: [text | binary]} + end + + @doc "Default implementation for `c:handle_expr/3`." + def handle_expr(state, "=", ast) do + check_state!(state) + %{binary: binary, dynamic: dynamic, vars_count: vars_count} = state + var = Macro.var(:"arg#{vars_count}", __MODULE__) + + ast = + quote do + unquote(var) = String.Chars.to_string(unquote(ast)) + end + + segment = + quote do + unquote(var) :: binary + end + + %{state | dynamic: [ast | dynamic], binary: [segment | binary], vars_count: vars_count + 1} + end + + def handle_expr(state, "", ast) do + %{dynamic: dynamic} = state + %{state | dynamic: [ast | dynamic]} + end + + def handle_expr(_state, marker, _ast) when marker in ["/", "|"] do + raise EEx.SyntaxError, + "unsupported EEx syntax <%#{marker} %> (the syntax is valid but not supported by the current EEx engine)" + end + + defp check_state!(%{binary: _, dynamic: _, vars_count: _}), do: :ok + + defp check_state!(state) do + raise "unexpected EEx.Engine state: #{inspect(state)}. " <> + "This typically means a bug or an outdated EEx.Engine or tool" end end diff --git a/lib/eex/lib/eex/smart_engine.ex b/lib/eex/lib/eex/smart_engine.ex index c59db81596f..4604adeb89f 100644 --- a/lib/eex/lib/eex/smart_engine.ex +++ b/lib/eex/lib/eex/smart_engine.ex @@ -1,62 +1,6 @@ -defmodule EEx.TransformerEngine do - @moduledoc false - - @doc false - defmacro __using__(_) do - quote do - @behaviour EEx.Engine - - def handle_body(body) do - EEx.Engine.handle_body(body) - end - - def handle_text(buffer, text) do - EEx.Engine.handle_text(buffer, text) - end - - def handle_expr(buffer, mark, expr) do - EEx.Engine.handle_expr(buffer, mark, transform(expr)) - end - - defp transform({a, b, c}) do - {transform(a), b, transform(c)} - end - - defp transform({a, b}) do - {transform(a), transform(b)} - end - - defp transform(list) when is_list(list) do - for i <- list, do: transform(i) - end - - defp transform(other) do - other - end - - defoverridable [transform: 1, handle_body: 1, handle_expr: 3, handle_text: 2] - end - end -end - -defmodule EEx.AssignsEngine do - @moduledoc false - - @doc false - defmacro __using__(_) do - quote unquote: false do - defp transform({:@, line, [{name, _, atom}]}) when is_atom(name) and is_atom(atom) do - quote do: Dict.get(var!(assigns), unquote(name)) - end - - defp transform(arg) do - super(arg) - end - - defoverridable [transform: 1] - end - end -end +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec defmodule EEx.SmartEngine do @moduledoc """ @@ -71,9 +15,9 @@ defmodule EEx.SmartEngine do "1" In the example above, we can access the value `foo` under - the binding `assigns` using `@foo`. This is useful when - a template, after compiled, may receive different assigns - and the developer don't want to recompile it for each + the binding `assigns` using `@foo`. This is useful because + a template, after being compiled, can receive different + assigns and would not require recompilation for each variable set. Assigns can also be used when compiled to a function: @@ -84,18 +28,35 @@ defmodule EEx.SmartEngine do # sample.ex defmodule Sample do require EEx - EEx.function_from_file :def, :sample, "sample.eex", [:assigns] + EEx.function_from_file(:def, :sample, "sample.eex", [:assigns]) end # iex - Sample.sample(a: 1, b: 2) #=> "3" + Sample.sample(a: 1, b: 2) + #=> "3" """ - use EEx.Engine + @behaviour EEx.Engine + + @impl true + defdelegate init(opts), to: EEx.Engine + + @impl true + defdelegate handle_body(state), to: EEx.Engine + + @impl true + defdelegate handle_begin(state), to: EEx.Engine + + @impl true + defdelegate handle_end(state), to: EEx.Engine + + @impl true + defdelegate handle_text(state, meta, text), to: EEx.Engine - def handle_expr(buffer, mark, expr) do + @impl true + def handle_expr(state, marker, expr) do expr = Macro.prewalk(expr, &EEx.Engine.handle_assign/1) - super(buffer, mark, expr) + EEx.Engine.handle_expr(state, marker, expr) end end diff --git a/lib/eex/lib/eex/tokenizer.ex b/lib/eex/lib/eex/tokenizer.ex deleted file mode 100644 index 44d5b681eed..00000000000 --- a/lib/eex/lib/eex/tokenizer.ex +++ /dev/null @@ -1,161 +0,0 @@ -defmodule EEx.Tokenizer do - @moduledoc false - - @doc """ - Tokenizes the given char list or binary. - It returns 4 different types of tokens as result: - - * `{:text, contents}` - * `{:expr, line, marker, contents}` - * `{:start_expr, line, marker, contents}` - * `{:middle_expr, line, marker, contents}` - * `{:end_expr, line, marker, contents}` - - """ - def tokenize(bin, line) when is_binary(bin) do - tokenize(String.to_char_list(bin), line) - end - - def tokenize(list, line) do - Enum.reverse(tokenize(list, line, [], [])) - end - - defp tokenize('<%%' ++ t, line, buffer, acc) do - {buffer, new_line, rest} = tokenize_expr t, line, [?%, ?<|buffer] - tokenize rest, new_line, [?>, ?%|buffer], acc - end - - defp tokenize('<%#' ++ t, line, buffer, acc) do - {_, new_line, rest} = tokenize_expr t, line, [] - tokenize rest, new_line, buffer, acc - end - - defp tokenize('<%' ++ t, line, buffer, acc) do - {marker, t} = retrieve_marker(t) - {expr, new_line, rest} = tokenize_expr t, line, [] - - token = token_name(expr) - acc = tokenize_text(buffer, acc) - final = {token, line, marker, Enum.reverse(expr)} - tokenize rest, new_line, [], [final | acc] - end - - defp tokenize('\n' ++ t, line, buffer, acc) do - tokenize t, line + 1, [?\n|buffer], acc - end - - defp tokenize([h|t], line, buffer, acc) do - tokenize t, line, [h|buffer], acc - end - - defp tokenize([], _line, buffer, acc) do - tokenize_text(buffer, acc) - end - - # Retrieve marker for <% - - defp retrieve_marker('=' ++ t) do - {"=", t} - end - - defp retrieve_marker(t) do - {"", t} - end - - # Tokenize an expression until we find %> - - defp tokenize_expr([?%, ?>|t], line, buffer) do - {buffer, line, t} - end - - defp tokenize_expr('\n' ++ t, line, buffer) do - tokenize_expr t, line + 1, [?\n|buffer] - end - - defp tokenize_expr([h|t], line, buffer) do - tokenize_expr t, line, [h|buffer] - end - - defp tokenize_expr([], _line, _buffer) do - raise EEx.SyntaxError, message: "missing token: %>" - end - - # Receive an expression content and check - # if it is a start, middle or an end token. - # - # Start tokens finish with `do` and `fn ->` - # Middle tokens are marked with `->` or keywords - # End tokens contain only the end word - - defp token_name([h|t]) when h in [?\s, ?\t] do - token_name(t) - end - - defp token_name('od' ++ [h|_]) when h in [?\s, ?\t, ?)] do - :start_expr - end - - defp token_name('>-' ++ rest) do - rest = Enum.reverse(rest) - - # Tokenize the remaining passing check_terminators as - # false, which relax the tokenizer to not error on - # unmatched pairs. Then, we check if there is a "fn" - # token and, if so, it is not followed by an "end" - # token. If this is the case, we are on a start expr. - case :elixir_tokenizer.tokenize(rest, 1, file: "eex", check_terminators: false) do - {:ok, _line, tokens} -> - tokens = Enum.reverse(tokens) - fn_index = fn_index(tokens) - - if fn_index && end_index(tokens) > fn_index do - :start_expr - else - :middle_expr - end - _error -> - :middle_expr - end - end - - defp token_name('esle' ++ t), do: check_spaces(t, :middle_expr) - defp token_name('retfa' ++ t), do: check_spaces(t, :middle_expr) - defp token_name('hctac' ++ t), do: check_spaces(t, :middle_expr) - defp token_name('eucser' ++ t), do: check_spaces(t, :middle_expr) - defp token_name('dne' ++ t), do: check_spaces(t, :end_expr) - - defp token_name(_) do - :expr - end - - defp fn_index(tokens) do - Enum.find_index tokens, fn - {:fn_paren, _} -> true - {:fn, _} -> true - _ -> false - end - end - - defp end_index(tokens) do - Enum.find_index(tokens, &match?({:end, _}, &1)) || :infinity - end - - defp check_spaces(string, token) do - if Enum.all?(string, &(&1 in [?\s, ?\t])) do - token - else - :expr - end - end - - # Tokenize the buffered text by appending - # it to the given accumulator. - - defp tokenize_text([], acc) do - acc - end - - defp tokenize_text(buffer, acc) do - [{:text, Enum.reverse(buffer)} | acc] - end -end diff --git a/lib/eex/mix.exs b/lib/eex/mix.exs index 0a2877473c5..acf9eb901fd 100644 --- a/lib/eex/mix.exs +++ b/lib/eex/mix.exs @@ -1,9 +1,15 @@ -defmodule EEx.Mixfile do +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +defmodule EEx.MixProject do use Mix.Project def project do - [app: :eex, - version: System.version, - build_per_environment: false] + [ + app: :eex, + version: System.version(), + build_per_environment: false + ] end end diff --git a/lib/eex/test/eex/smart_engine_test.exs b/lib/eex/test/eex/smart_engine_test.exs index 1003da8da15..a4c3ee1cc13 100644 --- a/lib/eex/test/eex/smart_engine_test.exs +++ b/lib/eex/test/eex/smart_engine_test.exs @@ -1,26 +1,54 @@ -Code.require_file "../test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) defmodule EEx.SmartEngineTest do use ExUnit.Case, async: true test "evaluates simple string" do - assert_eval "foo bar", "foo bar" + assert_eval("foo bar", "foo bar") end test "evaluates with assigns as keywords" do - assert_eval "1", "<%= @foo %>", assigns: [foo: 1] + assert_eval("1", "<%= @foo %>", assigns: [foo: 1]) end test "evaluates with assigns as a map" do - assert_eval "1", "<%= @foo %>", assigns: %{foo: 1} + assert_eval("1", "<%= @foo %>", assigns: %{foo: 1}) + end + + test "error with missing assigns" do + stderr = + ExUnit.CaptureIO.capture_io(:stderr, fn -> + assert_eval("", "<%= @foo %>", assigns: %{}) + end) + + assert stderr =~ "assign @foo not available in EEx template" end test "evaluates with loops" do - assert_eval "1\n2\n3\n", "<%= for x <- [1, 2, 3] do %><%= x %>\n<% end %>" + assert_eval("1\n2\n3\n", "<%= for x <- [1, 2, 3] do %><%= x %>\n<% end %>") + end + + test "preserves line numbers in assignments" do + result = EEx.compile_string("foo\n<%= @hello %>", engine: EEx.SmartEngine) + + Macro.prewalk(result, fn + {_left, meta, [_, :hello]} -> + assert Keyword.get(meta, :line) == 2 + send(self(), :found) + + node -> + node + end) + + assert_received :found end defp assert_eval(expected, actual, binding \\ []) do - result = EEx.eval_string(actual, binding, file: __ENV__.file) + result = EEx.eval_string(actual, binding, file: __ENV__.file, engine: EEx.SmartEngine) assert result == expected end end diff --git a/lib/eex/test/eex/tokenizer_test.exs b/lib/eex/test/eex/tokenizer_test.exs index d58268b0903..434821dc984 100644 --- a/lib/eex/test/eex/tokenizer_test.exs +++ b/lib/eex/test/eex/tokenizer_test.exs @@ -1,105 +1,370 @@ -Code.require_file "../test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) defmodule EEx.TokenizerTest do use ExUnit.Case, async: true - require EEx.Tokenizer, as: T - test "simple chars lists" do - assert T.tokenize('foo', 1) == [ {:text, 'foo'} ] + @opts [indentation: 0, trim: false] + + test "simple charlists" do + assert EEx.tokenize(~c"foo", @opts) == + {:ok, [{:text, ~c"foo", %{column: 1, line: 1}}, {:eof, %{column: 4, line: 1}}]} end test "simple strings" do - assert T.tokenize("foo", 1) == [ {:text, 'foo'} ] + assert EEx.tokenize("foo", @opts) == + {:ok, [{:text, ~c"foo", %{column: 1, line: 1}}, {:eof, %{column: 4, line: 1}}]} end test "strings with embedded code" do - assert T.tokenize('foo <% bar %>', 1) == [ {:text, 'foo '}, {:expr, 1, "", ' bar '} ] + assert EEx.tokenize(~c"foo <% bar %>", @opts) == + {:ok, + [ + {:text, ~c"foo ", %{column: 1, line: 1}}, + {:expr, ~c"", ~c" bar ", %{column: 5, line: 1}}, + {:eof, %{column: 14, line: 1}} + ]} end test "strings with embedded equals code" do - assert T.tokenize('foo <%= bar %>', 1) == [ {:text, 'foo '}, {:expr, 1, "=", ' bar '} ] + assert EEx.tokenize(~c"foo <%= bar %>", @opts) == + {:ok, + [ + {:text, ~c"foo ", %{column: 1, line: 1}}, + {:expr, ~c"=", ~c" bar ", %{column: 5, line: 1}}, + {:eof, %{column: 15, line: 1}} + ]} + end + + test "strings with embedded slash code" do + assert EEx.tokenize(~c"foo <%/ bar %>", @opts) == + {:ok, + [ + {:text, ~c"foo ", %{column: 1, line: 1}}, + {:expr, ~c"/", ~c" bar ", %{column: 5, line: 1}}, + {:eof, %{column: 15, line: 1}} + ]} + end + + test "strings with embedded pipe code" do + assert EEx.tokenize(~c"foo <%| bar %>", @opts) == + {:ok, + [ + {:text, ~c"foo ", %{column: 1, line: 1}}, + {:expr, ~c"|", ~c" bar ", %{column: 5, line: 1}}, + {:eof, %{column: 15, line: 1}} + ]} end test "strings with more than one line" do - assert T.tokenize('foo\n<%= bar %>', 1) == [ {:text, 'foo\n'}, {:expr, 2, "=", ' bar '} ] + assert EEx.tokenize(~c"foo\n<%= bar %>", @opts) == + {:ok, + [ + {:text, ~c"foo\n", %{column: 1, line: 1}}, + {:expr, ~c"=", ~c" bar ", %{column: 1, line: 2}}, + {:eof, %{column: 11, line: 2}} + ]} end test "strings with more than one line and expression with more than one line" do - string = ''' -foo <%= bar + string = ~c""" + foo <%= bar + + baz %> + <% foo %> + """ -baz %> -<% foo %> -''' + exprs = [ + {:text, ~c"foo ", %{column: 1, line: 1}}, + {:expr, ~c"=", ~c" bar\n\nbaz ", %{column: 5, line: 1}}, + {:text, ~c"\n", %{column: 7, line: 3}}, + {:expr, ~c"", ~c" foo ", %{column: 1, line: 4}}, + {:text, ~c"\n", %{column: 10, line: 4}}, + {:eof, %{column: 1, line: 5}} + ] - assert T.tokenize(string, 1) == [ - {:text, 'foo '}, - {:expr, 1, "=", ' bar\n\nbaz '}, - {:text, '\n'}, - {:expr, 4, "", ' foo '}, - {:text, '\n'} - ] + assert EEx.tokenize(string, @opts) == {:ok, exprs} end test "quotation" do - assert T.tokenize('foo <%% true %>', 1) == [ - {:text, 'foo <% true %>'} + assert EEx.tokenize(~c"foo <%% true %>", @opts) == + {:ok, + [ + {:text, ~c"foo <% true %>", %{column: 1, line: 1}}, + {:eof, %{column: 16, line: 1}} + ]} + end + + test "quotation with do-end" do + assert EEx.tokenize(~c"foo <%% true do %>bar<%% end %>", @opts) == + {:ok, + [ + {:text, ~c"foo <% true do %>bar<% end %>", %{column: 1, line: 1}}, + {:eof, %{column: 32, line: 1}} + ]} + end + + test "quotation with interpolation" do + exprs = [ + {:text, ~c"a <% b ", %{column: 1, line: 1}}, + {:expr, ~c"=", ~c" c ", %{column: 9, line: 1}}, + {:text, ~c" ", %{column: 17, line: 1}}, + {:expr, ~c"=", ~c" d ", %{column: 18, line: 1}}, + {:text, ~c" e %> f", %{column: 26, line: 1}}, + {:eof, %{column: 33, line: 1}} ] + + assert EEx.tokenize(~c"a <%% b <%= c %> <%= d %> e %> f", @opts) == {:ok, exprs} + end + + test "improperly formatted quotation with interpolation" do + exprs = [ + {:text, ~c"<%% a <%= b %> c %>", %{column: 1, line: 1}}, + {:eof, %{column: 22, line: 1}} + ] + + assert EEx.tokenize(~c"<%%% a <%%= b %> c %>", @opts) == {:ok, exprs} + end + + test "EEx comments" do + ExUnit.CaptureIO.capture_io(:stderr, fn -> + exprs = [ + {:text, ~c"foo ", %{column: 1, line: 1}}, + {:eof, %{column: 16, line: 1}} + ] + + assert EEx.tokenize(~c"foo <%# true %>", @opts) == {:ok, exprs} + + exprs = [ + {:text, ~c"foo ", %{column: 1, line: 1}}, + {:eof, %{column: 8, line: 2}} + ] + + assert EEx.tokenize(~c"foo <%#\ntrue %>", @opts) == {:ok, exprs} + end) end - test "quotation with do/end" do - assert T.tokenize('foo <%% true do %>bar<%% end %>', 1) == [ - {:text, 'foo <% true do %>bar<% end %>'} + test "EEx multi-line comments" do + exprs = [ + {:text, ~c"foo ", %{column: 1, line: 1}}, + {:comment, ~c" true ", %{column: 5, line: 1}}, + {:text, ~c" bar", %{column: 20, line: 1}}, + {:eof, %{column: 24, line: 1}} + ] + + assert EEx.tokenize(~c"foo <%!-- true --%> bar", @opts) == {:ok, exprs} + + exprs = [ + {:text, ~c"foo ", %{column: 1, line: 1}}, + {:comment, ~c" \ntrue\n ", %{column: 5, line: 1}}, + {:text, ~c" bar", %{column: 6, line: 3}}, + {:eof, %{column: 10, line: 3}} + ] + + assert EEx.tokenize(~c"foo <%!-- \ntrue\n --%> bar", @opts) == {:ok, exprs} + + exprs = [ + {:text, ~c"foo ", %{column: 1, line: 1}}, + {:comment, ~c" <%= true %> ", %{column: 5, line: 1}}, + {:text, ~c" bar", %{column: 27, line: 1}}, + {:eof, %{column: 31, line: 1}} ] + + assert EEx.tokenize(~c"foo <%!-- <%= true %> --%> bar", @opts) == {:ok, exprs} end - test "comments" do - assert T.tokenize('foo <%# true %>', 1) == [ - {:text, 'foo '} + test "Elixir comments" do + exprs = [ + {:text, ~c"foo ", %{column: 1, line: 1}}, + {:expr, [], ~c" true # this is a boolean ", %{column: 5, line: 1}}, + {:eof, %{column: 35, line: 1}} ] + + assert EEx.tokenize(~c"foo <% true # this is a boolean %>", @opts) == {:ok, exprs} end - test "comments with do/end" do - assert T.tokenize('foo <%# true do %>bar<%# end %>', 1) == [ - {:text, 'foo bar'} + test "Elixir comments with do-end" do + exprs = [ + {:start_expr, [], ~c" if true do # startif ", %{column: 1, line: 1}}, + {:text, ~c"text", %{column: 27, line: 1}}, + {:end_expr, [], ~c" end # closeif ", %{column: 31, line: 1}}, + {:eof, %{column: 50, line: 1}} ] + + assert EEx.tokenize(~c"<% if true do # startif %>text<% end # closeif %>", @opts) == + {:ok, exprs} end test "strings with embedded do end" do - assert T.tokenize('foo <% if true do %>bar<% end %>', 1) == [ - {:text, 'foo '}, - {:start_expr, 1, "", ' if true do '}, - {:text, 'bar'}, - {:end_expr, 1, "", ' end '} + exprs = [ + {:text, ~c"foo ", %{column: 1, line: 1}}, + {:start_expr, ~c"", ~c" if true do ", %{column: 5, line: 1}}, + {:text, ~c"bar", %{column: 21, line: 1}}, + {:end_expr, ~c"", ~c" end ", %{column: 24, line: 1}}, + {:eof, %{column: 33, line: 1}} ] + + assert EEx.tokenize(~c"foo <% if true do %>bar<% end %>", @opts) == {:ok, exprs} end test "strings with embedded -> end" do - assert T.tokenize('foo <% cond do %><% false -> %>bar<% true -> %>baz<% end %>', 1) == [ - {:text, 'foo '}, - {:start_expr, 1, "", ' cond do '}, - {:middle_expr, 1, "", ' false -> '}, - {:text, 'bar'}, - {:middle_expr, 1, "", ' true -> '}, - {:text, 'baz'}, - {:end_expr, 1, "", ' end '} + exprs = [ + {:text, ~c"foo ", %{column: 1, line: 1}}, + {:start_expr, ~c"", ~c" cond do ", %{column: 5, line: 1}}, + {:middle_expr, ~c"", ~c" false -> ", %{column: 18, line: 1}}, + {:text, ~c"bar", %{column: 32, line: 1}}, + {:middle_expr, ~c"", ~c" true -> ", %{column: 35, line: 1}}, + {:text, ~c"baz", %{column: 48, line: 1}}, + {:end_expr, ~c"", ~c" end ", %{column: 51, line: 1}}, + {:eof, %{column: 60, line: 1}} ] + + assert EEx.tokenize(~c"foo <% cond do %><% false -> %>bar<% true -> %>baz<% end %>", @opts) == + {:ok, exprs} + end + + test "strings with fn-end with newline" do + exprs = [ + {:start_expr, ~c"=", ~c" a fn ->\n", %{column: 1, line: 1}}, + {:text, ~c"foo", %{column: 3, line: 2}}, + {:end_expr, [], ~c" end ", %{column: 6, line: 2}}, + {:eof, %{column: 15, line: 2}} + ] + + assert EEx.tokenize(~c"<%= a fn ->\n%>foo<% end %>", @opts) == + {:ok, exprs} + end + + test "strings with multiple fn-end" do + exprs = [ + {:start_expr, ~c"=", ~c" a fn -> ", %{column: 1, line: 1}}, + {:text, ~c"foo", %{column: 15, line: 1}}, + {:middle_expr, ~c"", ~c" end, fn -> ", %{column: 18, line: 1}}, + {:text, ~c"bar", %{column: 34, line: 1}}, + {:end_expr, ~c"", ~c" end ", %{column: 37, line: 1}}, + {:eof, %{column: 46, line: 1}} + ] + + assert EEx.tokenize(~c"<%= a fn -> %>foo<% end, fn -> %>bar<% end %>", @opts) == + {:ok, exprs} + end + + test "strings with fn-end followed by do block" do + exprs = [ + {:start_expr, ~c"=", ~c" a fn -> ", %{column: 1, line: 1}}, + {:text, ~c"foo", %{column: 15, line: 1}}, + {:middle_expr, ~c"", ~c" end do ", %{column: 18, line: 1}}, + {:text, ~c"bar", %{column: 30, line: 1}}, + {:end_expr, ~c"", ~c" end ", %{column: 33, line: 1}}, + {:eof, %{column: 42, line: 1}} + ] + + assert EEx.tokenize(~c"<%= a fn -> %>foo<% end do %>bar<% end %>", @opts) == {:ok, exprs} end test "strings with embedded keywords blocks" do - assert T.tokenize('foo <% if true do %>bar<% else %>baz<% end %>', 1) == [ - {:text, 'foo '}, - {:start_expr, 1, "", ' if true do '}, - {:text, 'bar'}, - {:middle_expr, 1, "", ' else '}, - {:text, 'baz'}, - {:end_expr, 1, "", ' end '} + exprs = [ + {:text, ~c"foo ", %{column: 1, line: 1}}, + {:start_expr, ~c"", ~c" if true do ", %{column: 5, line: 1}}, + {:text, ~c"bar", %{column: 21, line: 1}}, + {:middle_expr, ~c"", ~c" else ", %{column: 24, line: 1}}, + {:text, ~c"baz", %{column: 34, line: 1}}, + {:end_expr, ~c"", ~c" end ", %{column: 37, line: 1}}, + {:eof, %{column: 46, line: 1}} ] + + assert EEx.tokenize(~c"foo <% if true do %>bar<% else %>baz<% end %>", @opts) == + {:ok, exprs} end - test "raise syntax error when there is start mark and no end mark" do - assert_raise EEx.SyntaxError, "missing token: %>", fn -> - T.tokenize('foo <% :bar', 1) + test "trim mode" do + template = ~c"\t<%= if true do %> \n TRUE \n <% else %>\n FALSE \n <% end %> \n\n " + + exprs = [ + {:start_expr, ~c"=", ~c" if true do ", %{column: 2, line: 1}}, + {:text, ~c"\n TRUE \n", %{column: 20, line: 1}}, + {:middle_expr, ~c"", ~c" else ", %{column: 3, line: 3}}, + {:text, ~c"\n FALSE \n", %{column: 13, line: 3}}, + {:end_expr, ~c"", ~c" end ", %{column: 3, line: 5}}, + {:eof, %{column: 3, line: 7}} + ] + + assert EEx.tokenize(template, [trim: true] ++ @opts) == {:ok, exprs} + end + + test "trim mode with multi-line comment" do + exprs = [ + {:comment, ~c" comment ", %{column: 3, line: 1}}, + {:text, ~c"\n123", %{column: 23, line: 1}}, + {:eof, %{column: 4, line: 2}} + ] + + assert EEx.tokenize(~c" <%!-- comment --%> \n123", [trim: true] ++ @opts) == {:ok, exprs} + end + + test "trim mode with CRLF" do + exprs = [ + {:text, ~c"0\n", %{column: 1, line: 1}}, + {:expr, ~c"=", ~c" 12 ", %{column: 3, line: 2}}, + {:text, ~c"\n34", %{column: 15, line: 2}}, + {:eof, %{column: 3, line: 3}} + ] + + assert EEx.tokenize(~c"0\r\n <%= 12 %> \r\n34", [trim: true] ++ @opts) == {:ok, exprs} + end + + test "trim mode set to false" do + exprs = [ + {:text, ~c" ", %{column: 1, line: 1}}, + {:expr, ~c"=", ~c" 12 ", %{column: 2, line: 1}}, + {:text, ~c" \n", %{column: 11, line: 1}}, + {:eof, %{column: 1, line: 2}} + ] + + assert EEx.tokenize(~c" <%= 12 %> \n", [trim: false] ++ @opts) == {:ok, exprs} + end + + test "trim mode no false positives" do + assert_not_trimmed = fn x -> + assert EEx.tokenize(x, [trim: false] ++ @opts) == EEx.tokenize(x, @opts) end + + assert_not_trimmed.(~c"foo <%= \"bar\" %> ") + assert_not_trimmed.(~c"\n <%= \"foo\" %>bar") + assert_not_trimmed.(~c" <%% hello %> ") + assert_not_trimmed.(~c" <%= 01 %><%= 23 %>\n") + end + + test "returns error when there is start mark and no end mark" do + message = """ + expected closing '%>' for EEx expression + | + 1 | foo <% :bar + | ^\ + """ + + assert EEx.tokenize(~c"foo <% :bar", @opts) == + {:error, message, %{column: 5, line: 1}} + + message = """ + expected closing '--%>' for EEx expression + | + 1 | <%!-- foo + | ^\ + """ + + assert EEx.tokenize(~c"<%!-- foo", @opts) == {:error, message, %{column: 1, line: 1}} + end + + test "marks invalid expressions as regular expressions" do + assert EEx.tokenize(~c"<% 1 $ 2 %>", @opts) == + {:ok, + [ + {:expr, [], ~c" 1 $ 2 ", %{column: 1, line: 1}}, + {:eof, %{column: 12, line: 1}} + ]} end end diff --git a/lib/eex/test/eex_test.exs b/lib/eex/test/eex_test.exs index 5cd9318b33c..5214b3afc64 100644 --- a/lib/eex/test/eex_test.exs +++ b/lib/eex/test/eex_test.exs @@ -1,39 +1,40 @@ -Code.require_file "test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("test_helper.exs", __DIR__) require EEx defmodule EExTest.Compiled do def before_compile do - fill_in_stacktrace - {__ENV__.line, hd(tl(System.stacktrace))} + {__ENV__.line, hd(tl(get_stacktrace()))} end - EEx.function_from_string :def, :string_sample, "<%= a + b %>", [:a, :b] + EEx.function_from_string(:def, :string_sample, "<%= a + b %>", [:a, :b]) filename = Path.join(__DIR__, "fixtures/eex_template_with_bindings.eex") - EEx.function_from_file :defp, :private_file_sample, filename, [:bar] + EEx.function_from_file(:defp, :private_file_sample, filename, [:bar]) filename = Path.join(__DIR__, "fixtures/eex_template_with_bindings.eex") - EEx.function_from_file :def, :public_file_sample, filename, [:bar] + EEx.function_from_file(:def, :public_file_sample, filename, [:bar]) def file_sample(arg), do: private_file_sample(arg) def after_compile do - fill_in_stacktrace - {__ENV__.line, hd(tl(System.stacktrace))} + {__ENV__.line, hd(tl(get_stacktrace()))} end @file "unknown" def unknown do - fill_in_stacktrace - {__ENV__.line, hd(tl(System.stacktrace))} + {__ENV__.line, hd(tl(get_stacktrace()))} end - defp fill_in_stacktrace do + defp get_stacktrace do try do - :erlang.error "failed" - catch - :error, _ -> System.stacktrace + :erlang.error("failed") + rescue + _ -> __STACKTRACE__ end end end @@ -50,331 +51,911 @@ defmodule EExTest do use ExUnit.Case, async: true doctest EEx - doctest EEx.AssignsEngine + doctest EEx.Engine + doctest EEx.SmartEngine - test "evaluates simple string" do - assert_eval "foo bar", "foo bar" - end + describe "evaluates" do + test "simple string" do + assert_eval("foo bar", "foo bar") + end - test "evaluates with embedded" do - assert_eval "foo bar", "foo <%= :bar %>" - end + test "Unicode" do + template = """ + • <%= "•" %> • + <%= "Jößé Vâlìm" %> Jößé Vâlìm + """ - test "evaluates with embedded and the binding" do - assert EEx.eval_string("foo <%= bar %>", [bar: 1]) == "foo 1" - end + assert_eval(" • • •\n Jößé Vâlìm Jößé Vâlìm\n", template) + end - test "evaluates with embedded do end" do - assert_eval "foo bar", "foo <%= if true do %>bar<% end %>" - end + test "no spaces" do + string = """ + <%=cond do%> + <%false ->%> + this + <%true ->%> + that + <%end%> + """ + + expected = "\n that\n\n" + assert_eval(expected, string, []) + end - test "evaluates with embedded do end and eval the expression" do - assert_eval "foo ", "foo <%= if false do %>bar<% end %>" - end + test "trim mode" do + string = "<%= 123 %> \n \n <%= 789 %>" + expected = "123\n789" + assert_eval(expected, string, [], trim: true) - test "evaluates with embedded do end and nested print expression" do - assert_eval "foo bar", "foo <%= if true do %><%= :bar %><% end %>" - end + string = "<%= 123 %> \n456\n <%= 789 %>" + expected = "123\n456\n789" + assert_eval(expected, string, [], trim: true) - test "evaluates with embedded do end and nested expressions" do - assert_eval "foo bar baz", "foo <%= if true do %>bar <% Process.put(:eex_text, 1) %><%= :baz %><% end %>" - assert Process.get(:eex_text) == 1 - end + string = "<%= 123 %> \n\n456\n\n <%= 789 %>" + expected = "123\n456\n789" + assert_eval(expected, string, [], trim: true) - test "evaluates with embedded middle expression" do - assert_eval "foo bar", "foo <%= if true do %>bar<% else %>baz<% end %>" - end + string = "<%= 123 %> \n \n456\n \n <%= 789 %>" + expected = "123\n456\n789" + assert_eval(expected, string, [], trim: true) - test "evaluates with embedded middle expression and eval the expression" do - assert_eval "foo baz", "foo <%= if false do %>bar<% else %>baz<% end %>" - end + string = "\n <%= 123 %> \n <%= 456 %> \n <%= 789 %> \n" + expected = "123\n456\n789" + assert_eval(expected, string, [], trim: true) - test "evaluates with nested start expression" do - assert_eval "foo bar", "foo <%= if true do %><%= if true do %>bar<% end %><% end %>" - end + string = "\r\n <%= 123 %> \r\n <%= 456 %> \r\n <%= 789 %> \r\n" + expected = "123\n456\n789" + assert_eval(expected, string, [], trim: true) + end - test "evaluates with nested middle expression" do - assert_eval "foo baz", "foo <%= if true do %><%= if false do %>bar<% else %>baz<% end %><% end %>" - end + test "trim mode with middle expression" do + string = """ + <%= cond do %> + <% false -> %> + this + <% true -> %> + that + <% end %> + """ + + expected = "\n that\n" + assert_eval(expected, string, [], trim: true) + end - test "evaluates with defined variable" do - assert_eval "foo 1", "foo <% bar = 1 %><%= bar %>" - end + test "trim mode with multiple lines" do + string = """ + <%= "First line" %> + <%= "Second line" %> + <%= "Third line" %> + <%= "Fourth line" %> + """ - test "evaluates with require code" do - assert_eval "foo 1,2,3", "foo <% require Enum, as: E %><%= E.join [1, 2, 3], \",\" %>" - end + expected = "First line\nSecond line\nThird line\nFourth line" + assert_eval(expected, string, [], trim: true) + end - test "evaluates with end of token" do - assert_eval "foo bar %>", "foo bar %>" - end + test "trim mode with no spaces" do + string = """ + <%=if true do%> + this + <%else%> + that + <%end%> + """ + + expected = "\n this\n" + assert_eval(expected, string, [], trim: true) + + string = """ + <%=cond do%> + <%false ->%> + this + <%true ->%> + that + <%end%> + """ + + expected = "\n that\n" + assert_eval(expected, string, [], trim: true) + end - test "raises a syntax error when the token is invalid" do - assert_raise EEx.SyntaxError, "missing token: %>", fn -> - EEx.compile_string "foo <%= bar" + test "embedded code" do + assert_eval("foo bar", "foo <%= :bar %>") end - end - test "raises a syntax error when end expression is found without a start expression" do - assert_raise EEx.SyntaxError, "unexpected token: ' end ' at line 1", fn -> - EEx.compile_string "foo <% end %>" + test "embedded code with binding" do + assert EEx.eval_string("foo <%= bar %>", bar: 1) == "foo 1" end - end - test "raises a syntax error when start expression is found without an end expression" do - assert_raise EEx.SyntaxError, "unexpected end of string. expecting a closing <% end %>.", fn -> - EEx.compile_string "foo <% if true do %>" + test "embedded code with do end when true" do + assert_eval("foo bar", "foo <%= if true do %>bar<% end %>") end - end - test "raises a syntax error when nested end expression is found without an start expression" do - assert_raise EEx.SyntaxError, "unexpected token: ' end ' at line 1", fn -> - EEx.compile_string "foo <% if true do %><% end %><% end %>" + test "embedded code with do end when false" do + assert_eval("foo ", "foo <%= if false do %>bar<% end %>") + end + + test "embedded code with do preceded by bracket" do + assert_eval("foo bar", "foo <%= if {true}do %>bar<% end %>") + assert_eval("foo bar", "foo <%= if (true)do %>bar<% end %>") + assert_eval("foo bar", "foo <%= if [true]do %>bar<% end %>") end - end - test "respects line numbers" do - expected = """ -foo -2 -""" + test "embedded code with do end and expression" do + assert_eval("foo bar", "foo <%= if true do %><%= :bar %><% end %>") + end - string = """ -foo -<%= __ENV__.line %> -""" + test "embedded code with do end and multiple expressions" do + assert_eval( + "foo bar baz", + "foo <%= if true do %>bar <% Process.put(:eex_text, 1) %><%= :baz %><% end %>" + ) - assert_eval expected, string - end + assert Process.get(:eex_text) == 1 + end - test "respects line numbers inside nested expressions" do - expected = """ -foo + test "embedded code with middle expression" do + assert_eval("foo bar", "foo <%= if true do %>bar<% else %>baz<% end %>") + end -3 + test "embedded code with evaluated middle expression" do + assert_eval("foo baz", "foo <%= if false do %>bar<% else %>baz<% end %>") + end -5 -""" + test "embedded code with multi-line comments in do end" do + assert_eval("foo bar", "foo <%= case true do %><%!-- comment --%><% true -> %>bar<% end %>") - string = """ -foo -<%= if true do %> -<%= __ENV__.line %> -<% end %> -<%= __ENV__.line %> -""" + assert_eval( + "foo\n\nbar\n", + "foo\n<%= case true do %>\n<%!-- comment --%>\n<% true -> %>\nbar\n<% end %>" + ) + end - assert_eval expected, string - end + test "embedded code with nested do end" do + assert_eval("foo bar", "foo <%= if true do %><%= if true do %>bar<% end %><% end %>") + end - test "respects line numbers inside start expression" do - expected = """ -foo + test "embedded code with nested do end with middle expression" do + assert_eval( + "foo baz", + "foo <%= if true do %><%= if false do %>bar<% else %>baz<% end %><% end %>" + ) + end -true + test "embedded code with end followed by bracket" do + assert_eval( + " 101 102 103 ", + "<%= Enum.map([1, 2, 3], fn x -> %> <%= 100 + x %> <% end) %>" + ) + + assert_eval( + " 101 102 103 ", + "<%= Enum.map([1, 2, 3], fn x ->\n%> <%= 100 + x %> <% end) %>" + ) + + assert_eval( + " 101 102 103 ", + "<%= apply Enum, :map, [[1, 2, 3], fn x -> %> <%= 100 + x %> <% end] %>" + ) + + assert_eval( + " 101 102 103 ", + "<%= #{__MODULE__}.tuple_map {[1, 2, 3], fn x -> %> <%= 100 + x %> <% end} %>" + ) + + assert_eval( + " 101 102 103 ", + "<%= apply(Enum, :map, [[1, 2, 3], fn x -> %> <%= 100 + x %> <% end]) %>" + ) + + assert_eval( + " 101 102 103 ", + "<%= Enum.map([1, 2, 3], (fn x -> %> <%= 100 + x %> <% end) ) %>" + ) + end -5 -""" + test "embedded code with variable definition" do + assert_eval("foo 1", "foo <% bar = 1 %><%= bar %>") + end - string = """ -foo -<%= if __ENV__.line == 2 do %> -<%= true %> -<% end %> -<%= __ENV__.line %> -""" + test "embedded code with require" do + assert_eval("foo 1,2,3", "foo <% require Enum, as: E %><%= E.join [1, 2, 3], \",\" %>") + end - assert_eval expected, string + test "with end of token" do + assert_eval("foo bar %>", "foo bar %>") + end end - test "respects line numbers inside middle expression with ->" do - expected = """ -foo + describe "raises syntax errors" do + test "with relative file information" do + message = """ + foobar.eex:1:5: expected closing '%>' for EEx expression + | + 1 | foo <%= bar + | ^\ + """ + + assert_raise EEx.SyntaxError, message, fn -> + EEx.compile_string("foo <%= bar", file: Path.join(File.cwd!(), "foobar.eex")) + end + end + + test "when <%!-- is not closed" do + message = """ + my_file.eex:1:5: expected closing '--%>' for EEx expression + | + 1 | foo <%!-- bar + | ^\ + """ + + assert_raise EEx.SyntaxError, message, fn -> + EEx.compile_string("foo <%!-- bar", file: "my_file.eex") + end + end -true + test "when the token is invalid" do + message = """ + nofile:1:5: expected closing '%>' for EEx expression + | + 1 | foo <%= bar + | ^\ + """ + + assert_raise EEx.SyntaxError, message, fn -> + EEx.compile_string("foo <%= bar") + end + end -7 -""" + test "when middle expression is found without a start expression" do + message = """ + nofile:5:1: unexpected middle of expression <% else %> + | + 2 | <%= "content" %> + 3 | <%= if true %> + 4 | <%= "foo" %> + 5 | <% else %> + | ^\ + """ + + assert_raise EEx.SyntaxError, message, fn -> + EEx.compile_string( + ~s(

Hi!

\n<%= "content" %>\n<%= if true %>\n <%= "foo" %>\n<% else %>\n bar<% end %>) + ) + end + end - string = """ -foo -<%= cond do %> -<% false -> %> false -<% __ENV__.line == 4 -> %> -<%= true %> -<% end %> -<%= __ENV__.line %> -""" + test "proper format line number of code snippet" do + message = """ + nofile:11:1: unexpected middle of expression <% else %> + | + 8 | <%= "content" %> + 9 | <%= if true %> + 10 | <%= "foo" %> + 11 | <% else %> + | ^\ + """ + + assert_raise EEx.SyntaxError, message, fn -> + EEx.compile_string( + ~s(\n\n\n\n\n\n

Hi!

\n<%= "content" %>\n<%= if true %>\n <%= "foo" %>\n<% else %>\n bar<% end %>) + ) + end + end - assert_eval expected, string - end + test "when there is only middle expression" do + message = """ + nofile:1:1: unexpected middle of expression <% else %> + | + 1 | <% else %> + | ^\ + """ + + assert_raise EEx.SyntaxError, message, fn -> + EEx.compile_string(~s(<% else %>)) + end + end - test "respects line number inside middle expressions with keywords" do - expected = """ -foo + test "when it is missing a `do` in case expr" do + message = """ + nofile:3:3: unexpected middle of expression <% :something -> %> + | + 1 | content + 2 | <%= case @var %> + 3 | <% :something -> %> + | ^\ + """ + + assert_raise EEx.SyntaxError, message, fn -> + EEx.compile_string("content\n<%= case @var %>\n <% :something -> %>\n bar<% end %>") + end + end -5 + test "when it is a `do` in cond expr" do + message = """ + nofile:3:3: unexpected middle of expression <% true -> %> + | + 1 | content + 2 | <%= cond %> + 3 | <% true -> %> + | ^\ + """ + + assert_raise EEx.SyntaxError, message, fn -> + EEx.compile_string("content\n<%= cond %>\n <% true -> %>\n bar<% end %>") + end + end -7 -""" + test "when end expression is found without a start expression" do + message = """ + nofile:1:5: unexpected end of expression <% end %> + | + 1 | foo <% end %> + | ^\ + """ + + assert_raise EEx.SyntaxError, message, fn -> + EEx.compile_string("foo <% end %>") + end + end - string = """ -foo -<%= if false do %> -<%= __ENV__.line %> -<% else %> -<%= __ENV__.line %> -<% end %> -<%= __ENV__.line %> -""" + test "when start expression is found without an end expression" do + message = """ + nofile:2:5: expected a closing '<% end %>' for block expression in EEx + | + 1 | foo + 2 | <%= if true do %> + | ^\ + """ + + assert_raise EEx.SyntaxError, message, fn -> + EEx.compile_string("foo\n<%= if true do %>\nfoo\n") + end + + message = """ + nofile:3:3: expected a closing '<% end %>' for block expression in EEx + | + 1 | foo + 2 | <%= + 3 | if true do %> + | ^\ + """ + + assert_raise EEx.SyntaxError, message, fn -> + EEx.compile_string("foo\n<%=\n if true do %>\nfoo\n", indentation: 0) + end + + message = """ + nofile:3:6: expected a closing '<% end %>' for block expression in EEx + | + 1 | foo + 2 | <%= + 3 | if true do %> + | ^\ + """ + + assert_raise EEx.SyntaxError, message, fn -> + EEx.compile_string("foo\n<%=\n if true do %>\nfoo\n", indentation: 3) + end + end - assert_eval expected, string - end + test "when start expression with middle expression is found without an end expression" do + message = """ + nofile:2:5: expected a closing '<% end %>' for block expression in EEx + | + 1 | foo + 2 | <%= if true do %> + | ^\ + """ + + assert_raise EEx.SyntaxError, message, fn -> + EEx.compile_string("foo\n<%= if true do %>\nfoo\n<% else %>\n") + end + end + + test "when multiple start expressions is found without an end expression" do + message = """ + nofile:5:5: expected a closing '<% end %>' for block expression in EEx + | + 2 | <%= if true do %> + 3 | <%= @something %> + 4 |\s + 5 | <%= if @var do %> + | ^\ + """ + + assert_raise EEx.SyntaxError, message, fn -> + EEx.compile_string( + "foo\n<%= if true do %>\n <%= @something %>\n\n<%= if @var do %>\nfoo\n" + ) + end + end + + test "when nested end expression is found without a start expression" do + message = """ + nofile:1:31: unexpected end of expression <% end %> + | + 1 | foo <%= if true do %><% end %><% end %> + | ^\ + """ + + assert_raise EEx.SyntaxError, message, fn -> + EEx.compile_string("foo <%= if true do %><% end %><% end %>") + end + end + + test "when trying to use marker '|' without implementation" do + msg = + ~r/unsupported EEx syntax <%| %> \(the syntax is valid but not supported by the current EEx engine\)/ - test "properly handle functions" do - expected = """ + assert_raise EEx.SyntaxError, msg, fn -> + EEx.compile_string("<%| true %>") + end + end -Number 1 + test "when trying to use marker '/' without implementation" do + msg = + ~r/unsupported EEx syntax <%\/ %> \(the syntax is valid but not supported by the current EEx engine\)/ -Number 2 + assert_raise EEx.SyntaxError, msg, fn -> + EEx.compile_string("<%/ true %>") + end + end -Number 3 + test "from Elixir parser" do + line = __ENV__.line + 6 + + message = + assert_raise TokenMissingError, fn -> + EEx.compile_string( + """ +
  • + Some: + <%= true && @some[ %> +
  • + """, + file: __ENV__.file, + line: line, + indentation: 12 + ) + end + + assert message |> Exception.message() |> strip_ansi() =~ """ + │ + 514 │ true && @some[\s + │ │ └ missing closing delimiter (expected "]") + │ └ unclosed delimiter + """ + end -""" + test "from Elixir parser with line breaks" do + line = __ENV__.line + 6 + + message = + assert_raise TokenMissingError, fn -> + EEx.compile_string( + """ +
  • + Some: + <%= true && + @some[ %> +
  • + """, + file: __ENV__.file, + line: line, + indentation: 12 + ) + end + + assert message |> Exception.message() |> strip_ansi() =~ """ + │ + #{line + 3} │ @some[\s + │ │ └ missing closing delimiter (expected "]") + │ └ unclosed delimiter + """ + end - string = """ -<%= Enum.map [1, 2, 3], fn x -> %> -Number <%= x %> -<% end %> -""" + test "honor line numbers" do + assert_raise EEx.SyntaxError, + "nofile:100:6: expected closing '%>' for EEx expression", + fn -> + EEx.compile_string("foo\n bar <%= baz", line: 99) + end + end - assert_eval expected, string + test "honor file names" do + message = """ + my_file.eex:1:5: expected closing '%>' for EEx expression + | + 1 | foo <%= bar + | ^\ + """ + + assert_raise EEx.SyntaxError, message, fn -> + EEx.compile_string("foo <%= bar", file: "my_file.eex") + end + end end - test "do not consider already finished functions" do - expected = """ -foo + describe "warnings" do + test "when middle expression has a modifier" do + assert ExUnit.CaptureIO.capture_io(:stderr, fn -> + EEx.compile_string("foo <%= if true do %>true<%= else %>false<% end %>") + end) =~ ~s[unexpected beginning of EEx tag \"<%=\" on \"<%= else %>\"] + end -true + test "when end expression has a modifier" do + assert ExUnit.CaptureIO.capture_io(:stderr, fn -> + EEx.compile_string("foo <%= if true do %>true<% else %>false<%= end %>") + end) =~ + ~s[unexpected beginning of EEx tag \"<%=\" on \"<%= end %>\"] + end -""" + test "unused \"do\" block without \"<%=\" modifier" do + assert ExUnit.CaptureIO.capture_io(:stderr, fn -> + EEx.compile_string("<% if true do %>I'm invisible!<% end %>") + end) =~ "the contents of this expression won't be output" - string = """ -foo -<%= cond do %> -<% false -> %> false -<% fn -> 1 end -> %> -<%= true %> -<% end %> -""" + # These are fine though + EEx.compile_string("<% foo = fn -> %>Hello<% end %>") + EEx.compile_string("<% foo = if true do %>Hello<% end %>") + end - assert_eval expected, string - end + test "from tokenizer" do + warning = + ExUnit.CaptureIO.capture_io(:stderr, fn -> + EEx.compile_string(~s'<%= :"foo" %>', file: "tokenizer.ex") + end) - test "evaluates nested do expressions" do - string = """ - <% y = ["a", "b", "c"] %> - <%= cond do %> - <% "a" in y -> %> - Good - <% true -> %> - <% if true do %>true<% else %>false<% end %> - Bad - <% end %> - """ - - assert_eval "\n\n Good\n \n", string + assert warning =~ "found quoted atom \"foo\" but the quotes are not required" + assert warning =~ "tokenizer.ex:1:5" + end end - test "for comprehensions" do - string = """ - <%= for _name <- packages || [] do %> - <% end %> - <%= all || :done %> - """ - assert_eval "\ndone\n", string, packages: nil, all: nil - end + describe "environment" do + test "respects line numbers" do + expected = """ + foo + 2 + """ - test "unicode" do - template = """ - • <%= "•" %> • - <%= "Jößé Vâlìm" %> Jößé Vâlìm - """ - result = EEx.eval_string(template) - assert result == " • • •\n Jößé Vâlìm Jößé Vâlìm\n" - end + string = """ + foo + <%= __ENV__.line %> + """ - test "evaluates the source from a given file" do - filename = Path.join(__DIR__, "fixtures/eex_template.eex") - result = EEx.eval_file(filename) - assert result == "foo bar.\n" - end + assert_eval(expected, string) + end - test "evaluates the source from a given file with bindings" do - filename = Path.join(__DIR__, "fixtures/eex_template_with_bindings.eex") - result = EEx.eval_file(filename, [bar: 1]) - assert result == "foo 1\n" - end + test "respects line numbers inside nested expressions" do + expected = """ + foo + + 3 + + 5 + """ + + string = """ + foo + <%= if true do %> + <%= __ENV__.line %> + <% end %> + <%= __ENV__.line %> + """ + + assert_eval(expected, string) + end + + test "respects line numbers inside start expression" do + expected = """ + foo + + true + + 5 + """ + + string = """ + foo + <%= if __ENV__.line == 2 do %> + <%= true %> + <% end %> + <%= __ENV__.line %> + """ + + assert_eval(expected, string) + end + + test "respects line numbers inside middle expression with ->" do + expected = """ + foo + + true + + 7 + """ + + string = """ + foo + <%= cond do %> + <% false -> %> false + <% __ENV__.line == 4 -> %> + <%= true %> + <% end %> + <%= __ENV__.line %> + """ + + assert_eval(expected, string) + end + + test "respects line number inside middle expressions with keywords" do + expected = """ + foo + + 5 + + 7 + """ - test "raises an Exception when there's an error with the given file" do - assert_raise File.Error, "could not read file non-existent.eex: no such file or directory", fn -> - filename = "non-existent.eex" - EEx.compile_file(filename) + string = """ + foo + <%= if false do %> + <%= __ENV__.line %> + <% else %> + <%= __ENV__.line %> + <% end %> + <%= __ENV__.line %> + """ + + assert_eval(expected, string) + end + + test "respects files" do + assert_eval("sample.ex", "<%= __ENV__.file %>", [], file: "sample.ex") end end - test "sets external resource attribute" do - assert EExTest.Compiled.__info__(:attributes)[:external_resource] == - [Path.join(__DIR__, "fixtures/eex_template_with_bindings.eex")] + describe "clauses" do + test "inside functions" do + expected = """ + + Number 1 + + Number 2 + + Number 3 + + """ + + string = """ + <%= Enum.map [1, 2, 3], fn x -> %> + Number <%= x %> + <% end %> + """ + + assert_eval(expected, string) + end + + test "inside multiple functions" do + expected = """ + + A 1 + + B 2 + + A 3 + + """ + + string = """ + <%= #{__MODULE__}.switching_map [1, 2, 3], fn x -> %> + A <%= x %> + <% end, fn x -> %> + B <%= x %> + <% end %> + """ + + assert_eval(expected, string) + end + + test "inside callback and do block" do + expected = """ + + + A 1 + + B 2 + + A 3 + + """ + + string = """ + <% require #{__MODULE__} %> + <%= #{__MODULE__}.switching_macro [1, 2, 3], fn x -> %> + A <%= x %> + <% end do %> + B <%= x %> + <% end %> + """ + + assert_eval(expected, string) + end + + test "inside cond" do + expected = """ + foo + + true + + """ + + string = """ + foo + <%= cond do %> + <% false -> %> false + <% fn -> 1 end -> %> + <%= true %> + <% end %> + """ + + assert_eval(expected, string) + end + + test "inside cond with do end" do + string = """ + <% y = ["a", "b", "c"] %> + <%= cond do %> + <% "a" in y -> %> + Good + <% true -> %> + <%= if true do %>true<% else %>false<% end %> + Bad + <% end %> + """ + + assert_eval("\n\n Good\n \n", string) + end + + test "line and column meta" do + indentation = 12 + + ast = + EEx.compile_string( + """ + <%= f() %> <% f() %> + <%= f fn -> %> + <%= f() %> + <% end %> + """, + indentation: indentation + ) + + {_, calls} = + Macro.prewalk(ast, [], fn + {:f, meta, _args} = expr, acc -> {expr, [meta | acc]} + other, acc -> {other, acc} + end) + + assert Enum.reverse(calls) == [ + [line: 1, column: indentation + 5], + [line: 1, column: indentation + 15], + [line: 2, column: indentation + 7], + [line: 3, column: indentation + 9] + ] + end end - test "defined from string" do - assert EExTest.Compiled.string_sample(1, 2) == "3" + describe "buffers" do + test "inside comprehensions" do + string = """ + <%= for _name <- packages || [] do %> + <% end %> + <%= all || :done %> + """ + + assert_eval("\ndone\n", string, packages: nil, all: nil) + end end - test "defined from file" do - assert EExTest.Compiled.file_sample(1) == "foo 1\n" - assert EExTest.Compiled.public_file_sample(1) == "foo 1\n" + describe "from file" do + test "evaluates the source" do + filename = Path.join(__DIR__, "fixtures/eex_template.eex") + result = EEx.eval_file(filename) + assert_normalized_newline_equal("foo bar.\n", result) + end + + test "evaluates the source with bindings" do + filename = Path.join(__DIR__, "fixtures/eex_template_with_bindings.eex") + result = EEx.eval_file(filename, bar: 1) + assert_normalized_newline_equal("foo 1\n", result) + end + + test "raises an Exception when file is missing" do + msg = "could not read file \"non-existent.eex\": no such file or directory" + + assert_raise File.Error, msg, fn -> + filename = "non-existent.eex" + EEx.compile_file(filename) + end + end + + test "sets external resource attribute" do + assert EExTest.Compiled.__info__(:attributes)[:external_resource] == + [Path.join(__DIR__, "fixtures/eex_template_with_bindings.eex")] + end + + test "supports t:Path.t() paths" do + filename = to_charlist(Path.join(__DIR__, "fixtures/eex_template_with_bindings.eex")) + result = EEx.eval_file(filename, bar: 1) + assert_normalized_newline_equal("foo 1\n", result) + end + + test "supports overriding file and line through options" do + filename = Path.join(__DIR__, "fixtures/eex_template_with_syntax_error.eex") + + assert_raise EEx.SyntaxError, + "my_file.eex:10:5: expected closing '%>' for EEx expression", + fn -> + EEx.eval_file(filename, _bindings = [], file: "my_file.eex", line: 10) + end + end end - test "defined from file do not affect backtrace" do - assert EExTest.Compiled.before_compile == - {8, - {EExTest.Compiled, - :before_compile, - 0, - [file: to_char_list(Path.relative_to_cwd(__ENV__.file)), line: 7] - } - } - - assert EExTest.Compiled.after_compile == - {23, - {EExTest.Compiled, - :after_compile, - 0, - [file: to_char_list(Path.relative_to_cwd(__ENV__.file)), line: 22] - } - } - - assert EExTest.Compiled.unknown == - {29, - {EExTest.Compiled, - :unknown, - 0, - [file: 'unknown', line: 28] - } - } + describe "precompiled" do + test "from string" do + assert EExTest.Compiled.string_sample(1, 2) == "3" + end + + test "from file" do + assert_normalized_newline_equal("foo 1\n", EExTest.Compiled.file_sample(1)) + assert_normalized_newline_equal("foo 1\n", EExTest.Compiled.public_file_sample(1)) + end + + test "from file does not affect backtrace" do + file = to_charlist(Path.relative_to_cwd(__ENV__.file)) + + assert EExTest.Compiled.before_compile() == + {11, {EExTest.Compiled, :before_compile, 0, [file: file, line: 11]}} + + assert EExTest.Compiled.after_compile() == + {25, {EExTest.Compiled, :after_compile, 0, [file: file, line: 25]}} + + assert EExTest.Compiled.unknown() == + {30, {EExTest.Compiled, :unknown, 0, [file: ~c"unknown", line: 30]}} + end end defmodule TestEngine do @behaviour EEx.Engine + def init(_opts) do + "INIT" + end + def handle_body(body) do - {:wrapped, body} + "BODY(#{body})" + end + + def handle_begin(_) do + "BEGIN" + end + + def handle_end(buffer) do + buffer <> ":END" end - def handle_text(buffer, text) do - EEx.Engine.handle_text(buffer, text) + def handle_text(buffer, meta, text) do + buffer <> ":TEXT-#{meta[:line]}-#{meta[:column]}(#{String.trim(text)})" + end + + def handle_expr(buffer, "/", expr) do + buffer <> ":DIV(#{Macro.to_string(expr)})" + end + + def handle_expr(buffer, "=", expr) do + buffer <> ":EQUAL(#{Macro.to_string(expr)})" end def handle_expr(buffer, mark, expr) do @@ -382,12 +963,80 @@ foo end end - test "calls handle_body" do - assert {:wrapped, "foo"} = EEx.eval_string("foo", [], engine: TestEngine) + describe "custom engines" do + test "text" do + assert_eval("BODY(INIT:TEXT-1-1(foo))", "foo", [], engine: TestEngine) + end + + test "custom marker" do + assert_eval("BODY(INIT:TEXT-1-1(foo):DIV(:bar))", "foo <%/ :bar %>", [], engine: TestEngine) + end + + test "begin/end" do + assert_eval( + ~s[BODY(INIT:TEXT-1-1(foo):EQUAL(if do\n "BEGIN:TEXT-1-17(this):END"\nelse\n "BEGIN:TEXT-1-31(that):END"\nend))], + "foo <%= if do %>this<% else %>that<% end %>", + [], + engine: TestEngine + ) + end + + test "not implemented custom marker" do + msg = + ~r/unsupported EEx syntax <%| %> \(the syntax is valid but not supported by the current EEx engine\)/ + + assert_raise EEx.SyntaxError, msg, fn -> + assert_eval({:wrapped, "foo baz"}, "foo <%| :bar %>", [], engine: TestEngine) + end + end + end + + describe "parser options" do + test "customizes parsed code" do + atoms_encoder = fn "not_jose", _ -> {:ok, :jose} end + + assert_eval("valid", "<%= not_jose %>", [jose: "valid"], + parser_options: [static_atoms_encoder: atoms_encoder] + ) + end + end + + @strip_ansi [IO.ANSI.green(), IO.ANSI.red(), IO.ANSI.reset()] + + defp strip_ansi(doc) do + String.replace(doc, @strip_ansi, "") end - defp assert_eval(expected, actual, binding \\ []) do - result = EEx.eval_string(actual, binding, file: __ENV__.file, engine: EEx.Engine) + defp assert_eval(expected, actual, binding \\ [], opts \\ []) do + opts = Keyword.merge([file: __ENV__.file, engine: opts[:engine] || EEx.Engine], opts) + result = EEx.eval_string(actual, binding, opts) assert result == expected end + + defp assert_normalized_newline_equal(expected, actual) do + assert String.replace(expected, "\r\n", "\n") == String.replace(actual, "\r\n", "\n") + end + + def tuple_map({list, callback}) do + Enum.map(list, callback) + end + + def switching_map(list, a, b) do + list + |> Enum.with_index() + |> Enum.map(fn + {element, index} when rem(index, 2) == 0 -> a.(element) + {element, index} when rem(index, 2) == 1 -> b.(element) + end) + end + + defmacro switching_macro(list, a, do: block) do + quote do + b = fn var!(x) -> + unquote(block) + end + + unquote(__MODULE__).switching_map(unquote(list), unquote(a), b) + end + end end diff --git a/lib/eex/test/fixtures/eex_template_with_syntax_error.eex b/lib/eex/test/fixtures/eex_template_with_syntax_error.eex new file mode 100644 index 00000000000..e8aa61680f4 --- /dev/null +++ b/lib/eex/test/fixtures/eex_template_with_syntax_error.eex @@ -0,0 +1 @@ +foo <%= bar diff --git a/lib/eex/test/test_helper.exs b/lib/eex/test/test_helper.exs index bf1bd1990e9..4d157609935 100644 --- a/lib/eex/test/test_helper.exs +++ b/lib/eex/test/test_helper.exs @@ -1 +1,15 @@ -ExUnit.start [trace: "--trace" in System.argv] \ No newline at end of file +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +{line_exclude, line_include} = + if line = System.get_env("LINE"), do: {[:test], [line: line]}, else: {[], []} + +Code.require_file("../../elixir/scripts/cover_record.exs", __DIR__) +CoverageRecorder.maybe_record("eex") + +ExUnit.start( + trace: !!System.get_env("TRACE"), + include: line_include, + exclude: line_exclude +) diff --git a/lib/elixir/Emakefile b/lib/elixir/Emakefile new file mode 100644 index 00000000000..2d3d2e5e3e1 --- /dev/null +++ b/lib/elixir/Emakefile @@ -0,0 +1,20 @@ +%% SPDX-License-Identifier: Apache-2.0 +%% SPDX-FileCopyrightText: 2021 The Elixir Team +%% SPDX-FileCopyrightText: 2012 Plataformatec + +{'src/*', [ + warn_unused_vars, + warn_export_all, + warn_shadow_vars, + warn_unused_import, + warn_unused_function, + warn_bif_clash, + warn_unused_record, + warn_deprecated_function, + warn_obsolete_guard, + warn_exported_vars, + %% Enable this when we require Erlang/OTP 27+ + %% warnings_as_errors, + debug_info, + {outdir, "ebin/"} +]}. diff --git a/lib/elixir/include/elixir.hrl b/lib/elixir/include/elixir.hrl deleted file mode 100644 index 9d536c9184a..00000000000 --- a/lib/elixir/include/elixir.hrl +++ /dev/null @@ -1,68 +0,0 @@ --define(m(M, K), maps:get(K, M)). --define(line(Opts), elixir_utils:get_line(Opts)). - --record(elixir_scope, { - context=nil, %% can be match, guards or nil - extra=nil, %% extra information about the context, like fn_match and map_key - noname=false, %% when true, don't add new names (used by try) - super=false, %% when true, it means super was invoked - caller=false, %% when true, it means caller was invoked - return=true, %% when true, the return value is used - module=nil, %% the current module - function=nil, %% the current function - vars=[], %% a dict of defined variables and their alias - backup_vars=nil, %% a copy of vars to be used on ^var - match_vars=nil, %% a set of all variables defined in a particular match - export_vars=nil, %% a dict of all variables defined in a particular clause - extra_guards=nil, %% extra guards from args expansion - counter=[], %% a dict counting the variables defined - file=(<<"nofile">>) %% the current scope filename -}). - --record(elixir_quote, { - line=false, - keep=false, - context=nil, - vars_hygiene=true, - aliases_hygiene=true, - imports_hygiene=true, - unquote=true, - unquoted=false, - escape=false -}). - --record(elixir_tokenizer, { - file, - terminators=[], - check_terminators=true, - existing_atoms_only=false -}). - -%% Used in tokenization and interpolation - -%% Numbers --define(is_hex(S), ?is_digit(S) orelse (S >= $A andalso S =< $F) orelse (S >= $a andalso S =< $f)). --define(is_bin(S), S >= $0 andalso S =< $1). --define(is_octal(S), S >= $0 andalso S =< $7). --define(is_leading_octal(S), S >= $0 andalso S =< $3). - -%% Digits and letters --define(is_digit(S), S >= $0 andalso S =< $9). --define(is_upcase(S), S >= $A andalso S =< $Z). --define(is_downcase(S), S >= $a andalso S =< $z). - -%% Atoms --define(is_atom_start(S), ?is_quote(S) orelse ?is_upcase(S) orelse ?is_downcase(S) orelse (S == $_)). --define(is_atom(S), ?is_identifier(S) orelse (S == $@)). - --define(is_identifier(S), ?is_digit(S) orelse ?is_upcase(S) orelse ?is_downcase(S) orelse (S == $_)). --define(is_sigil(S), (S == $/) orelse (S == $<) orelse (S == $") orelse (S == $') orelse - (S == $[) orelse (S == $() orelse (S == ${) orelse (S == $|)). - -%% Quotes --define(is_quote(S), S == $" orelse S == $'). - -%% Spaces --define(is_horizontal_space(S), (S == $\s) orelse (S == $\t)). --define(is_vertical_space(S), (S == $\r) orelse (S == $\n)). --define(is_space(S), ?is_horizontal_space(S) orelse ?is_vertical_space(S)). diff --git a/lib/elixir/lib/access.ex b/lib/elixir/lib/access.ex index acf88b370b6..bacf3bc4e87 100644 --- a/lib/elixir/lib/access.ex +++ b/lib/elixir/lib/access.ex @@ -1,19 +1,23 @@ -defprotocol Access do +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +defmodule Access do @moduledoc """ - The Access protocol is used by `foo[bar]` and also - empowers the nested update functions in Kernel. + Key-based access to data structures. - For instance, `foo[bar]` translates `Access.get(foo, bar)`. - `Kernel.get_in/2`, `Kernel.put_in/3`, `Kernel.update_in/3` and - `Kernel.get_and_update_in/3` are also all powered by the Access - protocol. + The `Access` module defines a behaviour for dynamically accessing + keys of any type in a data structure via the `data[key]` syntax. - This protocol is implemented by default for keywords, maps - and dictionary like types: + `Access` supports keyword lists (`Keyword`) and maps (`Map`) out + of the box. Keywords supports only atoms keys, keys for maps can + be of any type. Both return `nil` if the key does not exist: iex> keywords = [a: 1, b: 2] iex> keywords[:a] 1 + iex> keywords[:c] + nil iex> map = %{a: 1, b: 2} iex> map[:a] @@ -23,127 +27,1173 @@ defprotocol Access do iex> star_ratings[1.5] "★☆" - The key comparison must be implemented using the `===` operator. + This syntax is very convenient as it can be nested arbitrarily: + + iex> keywords = [a: 1, b: 2] + iex> keywords[:c][:unknown] + nil + + This works because accessing anything on a `nil` value, returns + `nil` itself: + + iex> nil[:a] + nil + + ## Maps and structs + + While the access syntax is allowed in maps via `map[key]`, + if your map is made of predefined atom keys, you should prefer + to access those atom keys with `map.key` instead of `map[key]`, + as `map.key` will raise if the key is missing (which is not + supposed to happen if the keys are predefined) or if `map` is + `nil`. + + Similarly, since structs are maps and structs have predefined + keys, they only allow the `struct.key` syntax and they do not + allow the `struct[key]` access syntax. + + In other words, the `map[key]` syntax is loose, returning `nil` + for missing keys, while the `map.key` syntax is strict, raising + for both nil values and missing keys. + + To bridge this gap, Elixir provides the `get_in/1` and `get_in/2` + functions, which are capable of traversing nested data structures, + even in the presence of `nil`s: + + iex> users = %{"john" => %{age: 27}, "meg" => %{age: 23}} + iex> get_in(users["john"].age) + 27 + iex> get_in(users["unknown"].age) + nil + + Notice how, even if no user was found, `get_in/1` returned `nil`. + Outside of `get_in/1`, trying to access the field `.age` on `nil` + would raise. + + The `get_in/2` function takes one step further by allowing + different accessors to be mixed in. For example, given a user + map with the `:name` and `:languages` keys, here is how to + access the name of all programming languages: + + iex> languages = [ + ...> %{name: "elixir", type: :functional}, + ...> %{name: "c", type: :procedural} + ...> ] + iex> user = %{name: "john", languages: languages} + iex> get_in(user, [:languages, Access.all(), :name]) + ["elixir", "c"] + + This module provides convenience functions for traversing other + structures, like tuples and lists. As we will see next, they can + even be used to update nested data structures. + + If you want to learn more about the dual nature of maps in Elixir, + as they can be either for structured data or as a key-value store, + see the `Map` module. + + ## Updating nested data structures + + The access syntax can also be used with the `Kernel.put_in/2`, + `Kernel.update_in/2`, `Kernel.get_and_update_in/2`, and `Kernel.pop_in/1` + macros to further manipulate values in nested data structures: + + iex> users = %{"john" => %{age: 27}, "meg" => %{age: 23}} + iex> put_in(users["john"].age, 28) + %{"john" => %{age: 28}, "meg" => %{age: 23}} + + As shown in the previous section, you can also use the + `Kernel.put_in/3`, `Kernel.update_in/3`, `Kernel.pop_in/2`, and + `Kernel.get_and_update_in/3` functions to provide nested + custom accessors. For instance, given a user map with the + `:name` and `:languages` keys, here is how to deeply traverse + the map and convert all language names to uppercase: + + iex> languages = [ + ...> %{name: "elixir", type: :functional}, + ...> %{name: "c", type: :procedural} + ...> ] + iex> user = %{name: "john", languages: languages} + iex> update_in(user, [:languages, Access.all(), :name], &String.upcase/1) + %{ + name: "john", + languages: [ + %{name: "ELIXIR", type: :functional}, + %{name: "C", type: :procedural} + ] + } + + See the functions `key/1`, `key!/1`, `elem/1`, and `all/0` for + some of the available accessors. """ + @type container :: keyword | struct | map + @type nil_container :: nil + @type t :: container | nil_container | any + @type key :: any + @type value :: any + + @type get_fun(data) :: + (:get, data, (term -> term) -> new_data :: container) + + @type get_and_update_fun(data, current_value) :: + (:get_and_update, data, (term -> term) -> + {current_value, new_data :: container} | :pop) + + @type access_fun(data, current_value) :: + get_fun(data) | get_and_update_fun(data, current_value) + @doc """ - Accesses the given key in the container. + Invoked in order to access the value stored under `key` in the given term `term`. + + This function should return `{:ok, value}` where `value` is the value under + `key` if the key exists in the term, or `:error` if the key does not exist in + the term. + + Many of the functions defined in the `Access` module internally call this + function. This function is also used when the square-brackets access syntax + (`structure[key]`) is used: the `fetch/2` callback implemented by the module + that defines the `structure` struct is invoked and if it returns `{:ok, + value}` then `value` is returned, or if it returns `:error` then `nil` is + returned. + + See the `Map.fetch/2` and `Keyword.fetch/2` implementations for examples of + how to implement this callback. + """ + @callback fetch(term :: t, key) :: {:ok, value} | :error + + @doc """ + Invoked in order to access the value under `key` and update it at the same time. + + The implementation of this callback should invoke `fun` with the value under + `key` in the passed structure `data`, or with `nil` if `key` is not present in it. + This function must return either `{current_value, new_value}` or `:pop`. + + If the passed function returns `{current_value, new_value}`, + the return value of this callback should be `{current_value, new_data}`, where: + + * `current_value` is the retrieved value (which can be operated on before being returned) + + * `new_value` is the new value to be stored under `key` + + * `new_data` is `data` after updating the value of `key` with `new_value`. + + If the passed function returns `:pop`, the return value of this callback + must be `{value, new_data}` where `value` is the value under `key` + (or `nil` if not present) and `new_data` is `data` without `key`. + + See the implementations of `Map.get_and_update/3` or `Keyword.get_and_update/3` + for more examples. """ - @spec get(t, term) :: t - def get(container, key) + @callback get_and_update(data, key, (value | nil -> {current_value, new_value :: value} | :pop)) :: + {current_value, new_data :: data} + when current_value: value, data: container @doc """ - Gets a value and updates the given `key` in one pass. + Invoked to "pop" the value under `key` out of the given data structure. - The function must receive the value for the given `key` - (or `nil` if the key doesn't exist in `container`) and - the function must return a tuple containing the `get` - value and the new value to be stored in the `container`. + When `key` exists in the given structure `data`, the implementation should + return a `{value, new_data}` tuple where `value` is the value that was under + `key` and `new_data` is `term` without `key`. + + When `key` is not present in the given structure, a tuple `{value, data}` + should be returned, where `value` is implementation-defined. + + See the implementations for `Map.pop/3` or `Keyword.pop/3` for more examples. """ - @spec get_and_update(t, term, (term -> {get, term})) :: {get, t} when get: var - def get_and_update(container, key, fun) -end + @callback pop(data, key) :: {value, data} when data: container + + defmacrop raise_undefined_behaviour(exception, module, top) do + quote do + exception = + case __STACKTRACE__ do + [unquote(top) | _] -> + reason = + """ + #{inspect(unquote(module))} does not implement the Access behaviour -defimpl Access, for: List do - def get(dict, key) when is_atom(key) do - case :lists.keyfind(key, 1, dict) do - {^key, value} -> value - false -> nil + You can use the "struct.field" syntax to access struct fields. \ + You can also use Access.key!/1 to access struct fields dynamically \ + inside get_in/put_in/update_in\ + """ + + %{unquote(exception) | reason: reason} + + _ -> + unquote(exception) + end + + reraise exception, __STACKTRACE__ end end - def get(_dict, key) do - raise ArgumentError, - "the access protocol for lists expect the key to be an atom, got: #{inspect key}" + @doc """ + Fetches the value for the given key in a container (a map, keyword + list, or struct that implements the `Access` behaviour). + + Returns `{:ok, value}` where `value` is the value under `key` if there is such + a key, or `:error` if `key` is not found. + + ## Examples + + iex> Access.fetch(%{name: "meg", age: 26}, :name) + {:ok, "meg"} + + iex> Access.fetch([ordered: true, on_timeout: :exit], :timeout) + :error + + """ + @spec fetch(container, term) :: {:ok, term} | :error + @spec fetch(nil_container, any) :: :error + def fetch(container, key) + + def fetch(%module{} = container, key) do + module.fetch(container, key) + rescue + exception in UndefinedFunctionError -> + raise_undefined_behaviour(exception, module, {^module, :fetch, [^container, ^key], _}) end - def get_and_update(dict, key, fun) when is_atom(key) do - get_and_update(dict, [], key, fun) + def fetch(map, key) when is_map(map) do + case map do + %{^key => value} -> {:ok, value} + _ -> :error + end end - defp get_and_update([{key, value}|t], acc, key, fun) do - {get, update} = fun.(value) - {get, :lists.reverse(acc, [{key, update}|t])} + def fetch(list, key) when is_list(list) and is_atom(key) do + case :lists.keyfind(key, 1, list) do + {_, value} -> {:ok, value} + false -> :error + end end - defp get_and_update([h|t], acc, key, fun) do - get_and_update(t, [h|acc], key, fun) + def fetch(list, key) when is_list(list) do + raise ArgumentError, + "the Access calls for keywords expect the key to be an atom, got: " <> inspect(key) end - defp get_and_update([], acc, key, fun) do - {get, update} = fun.(nil) - {get, [{key, update}|:lists.reverse(acc)]} + def fetch(nil, _key) do + :error end -end -defimpl Access, for: Map do - def get(map, key) do - case :maps.find(key, map) do + @doc """ + Same as `fetch/2` but returns the value directly, + or raises a `KeyError` exception if `key` is not found. + + ## Examples + + iex> Access.fetch!(%{name: "meg", age: 26}, :name) + "meg" + + """ + @doc since: "1.10.0" + @spec fetch!(container, term) :: term + def fetch!(container, key) do + case fetch(container, key) do {:ok, value} -> value - :error -> nil + :error -> raise(KeyError, key: key, term: container) end end - def get_and_update(map, key, fun) do - value = - case :maps.find(key, map) do - {:ok, value} -> value - :error -> nil - end + @doc """ + Gets the value for the given key in a container (a map, keyword + list, or struct that implements the `Access` behaviour). - {get, update} = fun.(value) - {get, :maps.put(key, update, map)} - end + Returns the value under `key` if there is such a key, or `default` if `key` is + not found. + + ## Examples - def get!(%{} = map, key) do - case :maps.find(key, map) do + iex> Access.get(%{name: "john"}, :name, "default name") + "john" + iex> Access.get(%{name: "john"}, :age, 25) + 25 + + iex> Access.get([ordered: true], :timeout) + nil + + """ + @spec get(container, term, term) :: term + @spec get(nil_container, any, default) :: default when default: var + def get(container, key, default \\ nil) + + # Reimplementing the same logic as Access.fetch/2 here is done for performance, since + # this is called a lot and calling fetch/2 means introducing some overhead (like + # building the "{:ok, _}" tuple and deconstructing it back right away). + + def get(%module{} = container, key, default) do + try do + module.fetch(container, key) + rescue + exception in UndefinedFunctionError -> + raise_undefined_behaviour(exception, module, {^module, :fetch, [^container, ^key], _}) + else {:ok, value} -> value - :error -> raise KeyError, key: key, term: map + :error -> default end end - def get!(other, key) do - raise ArgumentError, - "could not get key #{inspect key}. Expected map/struct, got: #{inspect other}" + def get(map, key, default) when is_map(map) do + case map do + %{^key => value} -> value + _ -> default + end end - def get_and_update!(%{} = map, key, fun) do - case :maps.find(key, map) do - {:ok, value} -> - {get, update} = fun.(value) - {get, :maps.put(key, update, map)} - :error -> - raise KeyError, key: key, term: map + def get(list, key, default) when is_list(list) and is_atom(key) do + case :lists.keyfind(key, 1, list) do + {_, value} -> value + false -> default end end - def get_and_update!(other, key, _fun) do + def get(list, key, _default) when is_list(list) and is_integer(key) do + raise ArgumentError, """ + the Access module does not support accessing lists by index, got: #{inspect(key)} + + Accessing a list by index is typically discouraged in Elixir, \ + instead we prefer to use the Enum module to manipulate lists \ + as a whole. If you really must access a list element by index, \ + you can use Enum.at/2 or the functions in the List module\ + """ + end + + def get(list, key, _default) when is_list(list) do + raise ArgumentError, """ + the Access module supports only keyword lists (with atom keys), got: #{inspect(key)} + + If you want to search lists of tuples, use List.keyfind/3\ + """ + end + + def get(nil, _key, default) do + default + end + + @doc """ + Gets and updates the given key in a `container` (a map, a keyword list, + a struct that implements the `Access` behaviour). + + The `fun` argument receives the value of `key` (or `nil` if `key` is not + present in `container`) and must return a two-element tuple `{current_value, new_value}`: + the "get" value `current_value` (the retrieved value, which can be operated on before + being returned) and the new value to be stored under `key` (`new_value`). + `fun` may also return `:pop`, which means the current value + should be removed from the container and returned. + + The returned value is a two-element tuple with the "get" value returned by + `fun` and a new container with the updated value under `key`. + + ## Examples + + iex> Access.get_and_update([a: 1], :a, fn current_value -> + ...> {current_value, current_value + 1} + ...> end) + {1, [a: 2]} + + """ + @spec get_and_update(data, key, (value | nil -> {current_value, new_value :: value} | :pop)) :: + {current_value, new_data :: data} + when current_value: var, data: container + def get_and_update(container, key, fun) + + def get_and_update(%module{} = container, key, fun) do + module.get_and_update(container, key, fun) + rescue + exception in UndefinedFunctionError -> + raise_undefined_behaviour( + exception, + module, + {^module, :get_and_update, [^container, ^key, ^fun], _} + ) + end + + def get_and_update(map, key, fun) when is_map(map) do + Map.get_and_update(map, key, fun) + end + + def get_and_update(list, key, fun) when is_list(list) and is_atom(key) do + Keyword.get_and_update(list, key, fun) + end + + def get_and_update(list, key, _fun) when is_list(list) and is_integer(key) do + raise ArgumentError, """ + the Access module does not support accessing lists by index, got: #{inspect(key)} + + Accessing a list by index is typically discouraged in Elixir, \ + instead we prefer to use the Enum module to manipulate lists \ + as a whole. If you really must modify a list element by index, \ + you can use Access.at/1 or the functions in the List module\ + """ + end + + def get_and_update(list, key, _fun) when is_list(list) do raise ArgumentError, - "could not update key #{inspect key}. Expected map/struct, got: #{inspect other}" + "the Access module supports only keyword lists (with atom keys), got: " <> inspect(key) end -end -defimpl Access, for: Atom do - def get(nil, _) do - nil + def get_and_update(nil, key, _fun) do + raise ArgumentError, "could not put/update key #{inspect(key)} on a nil value" + end + + @doc """ + Removes the entry with a given key from a container (a map, keyword + list, or struct that implements the `Access` behaviour). + + Returns a tuple containing the value associated with the key and the + updated container. `nil` is returned for the value if the key isn't + in the container. + + ## Examples + + With a map: + + iex> Access.pop(%{name: "Elixir", creator: "Valim"}, :name) + {"Elixir", %{creator: "Valim"}} + + A keyword list: + + iex> Access.pop([name: "Elixir", creator: "Valim"], :name) + {"Elixir", [creator: "Valim"]} + + An unknown key: + + iex> Access.pop(%{name: "Elixir", creator: "Valim"}, :year) + {nil, %{creator: "Valim", name: "Elixir"}} + + """ + @spec pop(data, key) :: {value, data} when data: container + def pop(%module{} = container, key) do + module.pop(container, key) + rescue + exception in UndefinedFunctionError -> + raise_undefined_behaviour(exception, module, {^module, :pop, [^container, ^key], _}) end - def get(atom, _) do - undefined(atom) + def pop(map, key) when is_map(map) do + Map.pop(map, key) end - def get_and_update(nil, _, fun) do - fun.(nil) + def pop(list, key) when is_list(list) do + Keyword.pop(list, key) end - def get_and_update(atom, _key, _fun) do - undefined(atom) + def pop(nil, key) do + raise ArgumentError, "could not pop key #{inspect(key)} on a nil value" end - defp undefined(atom) do - raise Protocol.UndefinedError, - protocol: @protocol, - value: atom, - description: "only the nil atom is supported" + ## Accessors + + @doc """ + Returns a function that accesses the given key in a map/struct. + + The returned function is typically passed as an accessor to `Kernel.get_in/2`, + `Kernel.get_and_update_in/3`, and friends. + + The returned function uses the default value if the key does not exist. + This can be used to specify defaults and safely traverse missing keys: + + iex> get_in(%{}, [Access.key(:user, %{}), Access.key(:name, "meg")]) + "meg" + + Such is also useful when using update functions, allowing us to introduce + values as we traverse the data structure for updates: + + iex> put_in(%{}, [Access.key(:user, %{}), Access.key(:name)], "Mary") + %{user: %{name: "Mary"}} + + ## Examples + + iex> map = %{user: %{name: "john"}} + iex> get_in(map, [Access.key(:unknown, %{}), Access.key(:name, "john")]) + "john" + iex> get_and_update_in(map, [Access.key(:user), Access.key(:name)], fn prev -> + ...> {prev, String.upcase(prev)} + ...> end) + {"john", %{user: %{name: "JOHN"}}} + iex> pop_in(map, [Access.key(:user), Access.key(:name)]) + {"john", %{user: %{}}} + + An error is raised if the accessed structure is not a map or a struct: + + iex> get_in([], [Access.key(:foo)]) + ** (BadMapError) expected a map, got: + ... + """ + @spec key(key, term) :: access_fun(data :: struct | map, current_value :: term) + def key(key, default \\ nil) do + fn + :get, data, next -> + next.(Map.get(data, key, default)) + + :get_and_update, data, next -> + value = Map.get(data, key, default) + + case next.(value) do + {get, update} -> {get, Map.put(data, key, update)} + :pop -> {value, Map.delete(data, key)} + end + end + end + + @doc """ + Returns a function that accesses the given key in a map/struct. + + The returned function is typically passed as an accessor to `Kernel.get_in/2`, + `Kernel.get_and_update_in/3`, and friends. + + Similar to `key/2`, but the returned function raises if the key does not exist. + + ## Examples + + iex> map = %{user: %{name: "john"}} + iex> get_in(map, [Access.key!(:user), Access.key!(:name)]) + "john" + iex> get_and_update_in(map, [Access.key!(:user), Access.key!(:name)], fn prev -> + ...> {prev, String.upcase(prev)} + ...> end) + {"john", %{user: %{name: "JOHN"}}} + iex> pop_in(map, [Access.key!(:user), Access.key!(:name)]) + {"john", %{user: %{}}} + iex> get_in(map, [Access.key!(:user), Access.key!(:unknown)]) + ** (KeyError) key :unknown not found in: + ... + + The examples above could be partially written as: + + iex> map = %{user: %{name: "john"}} + iex> map.user.name + "john" + iex> get_and_update_in(map.user.name, fn prev -> + ...> {prev, String.upcase(prev)} + ...> end) + {"john", %{user: %{name: "JOHN"}}} + + However, it is not possible to remove fields using the dot notation, + as it is implied those fields must also be present. In any case, + `Access.key!/1` is useful when the key is not known in advance + and must be accessed dynamically. + + An error is raised if the accessed structure is not a map/struct: + + iex> get_in([], [Access.key!(:foo)]) + ** (RuntimeError) Access.key!/1 expected a map/struct, got: [] + + """ + @spec key!(key) :: access_fun(data :: struct | map, current_value :: term) + def key!(key) do + fn + :get, %{} = data, next -> + next.(Map.fetch!(data, key)) + + :get_and_update, %{} = data, next -> + value = Map.fetch!(data, key) + + case next.(value) do + {get, update} -> {get, Map.put(data, key, update)} + :pop -> {value, Map.delete(data, key)} + end + + _op, data, _next -> + raise "Access.key!/1 expected a map/struct, got: #{inspect(data)}" + end + end + + @doc ~S""" + Returns a function that accesses the element at the given index in a tuple. + + The returned function is typically passed as an accessor to `Kernel.get_in/2`, + `Kernel.get_and_update_in/3`, and friends. + + The returned function raises if `index` is out of bounds. + + Note that popping elements out of tuples is not possible and raises an + error. + + ## Examples + + iex> map = %{user: {"john", 27}} + iex> get_in(map, [:user, Access.elem(0)]) + "john" + iex> get_and_update_in(map, [:user, Access.elem(0)], fn prev -> + ...> {prev, String.upcase(prev)} + ...> end) + {"john", %{user: {"JOHN", 27}}} + iex> pop_in(map, [:user, Access.elem(0)]) + ** (RuntimeError) cannot pop data from a tuple + + An error is raised if the accessed structure is not a tuple: + + iex> get_in(%{}, [Access.elem(0)]) + ** (RuntimeError) Access.elem/1 expected a tuple, got: %{} + + """ + @spec elem(non_neg_integer) :: access_fun(data :: tuple, current_value :: term) + def elem(index) when is_integer(index) and index >= 0 do + pos = index + 1 + + fn + :get, data, next when is_tuple(data) -> + next.(:erlang.element(pos, data)) + + :get_and_update, data, next when is_tuple(data) -> + value = :erlang.element(pos, data) + + case next.(value) do + {get, update} -> {get, :erlang.setelement(pos, data, update)} + :pop -> raise "cannot pop data from a tuple" + end + + _op, data, _next -> + raise "Access.elem/1 expected a tuple, got: #{inspect(data)}" + end + end + + @doc ~S""" + Returns a function that accesses all the elements in a list. + + The returned function is typically passed as an accessor to `Kernel.get_in/2`, + `Kernel.get_and_update_in/3`, and friends. + + ## Examples + + iex> list = [%{name: "john"}, %{name: "mary"}] + iex> get_in(list, [Access.all(), :name]) + ["john", "mary"] + iex> get_and_update_in(list, [Access.all(), :name], fn prev -> + ...> {prev, String.upcase(prev)} + ...> end) + {["john", "mary"], [%{name: "JOHN"}, %{name: "MARY"}]} + iex> pop_in(list, [Access.all(), :name]) + {["john", "mary"], [%{}, %{}]} + + Here is an example that traverses the list dropping even + numbers and multiplying odd numbers by 2: + + iex> require Integer + iex> get_and_update_in([1, 2, 3, 4, 5], [Access.all()], fn num -> + ...> if Integer.is_even(num), do: :pop, else: {num, num * 2} + ...> end) + {[1, 2, 3, 4, 5], [2, 6, 10]} + + An error is raised if the accessed structure is not a list: + + iex> get_in(%{}, [Access.all()]) + ** (RuntimeError) Access.all/0 expected a list, got: %{} + + """ + @spec all() :: access_fun(data :: list, current_value :: list) + def all() do + &all/3 + end + + defp all(:get, data, next) when is_list(data) do + Enum.map(data, next) + end + + defp all(:get_and_update, data, next) when is_list(data) do + all(data, next, _gets = [], _updates = []) + end + + defp all(_op, data, _next) do + raise "Access.all/0 expected a list, got: #{inspect(data)}" + end + + defp all([head | rest], next, gets, updates) do + case next.(head) do + {get, update} -> all(rest, next, [get | gets], [update | updates]) + :pop -> all(rest, next, [head | gets], updates) + end + end + + defp all([], _next, gets, updates) do + {:lists.reverse(gets), :lists.reverse(updates)} + end + + @doc ~S""" + Returns a function that accesses the element at `index` (zero based) of a list. + + Keep in mind that index lookups in lists take linear time: the larger the list, + the longer it will take to access its index. Therefore index-based operations + are generally avoided in favor of other functions in the `Enum` module. + + The returned function is typically passed as an accessor to `Kernel.get_in/2`, + `Kernel.get_and_update_in/3`, and friends. + + ## Examples + + iex> list = [%{name: "john"}, %{name: "mary"}] + iex> get_in(list, [Access.at(1), :name]) + "mary" + iex> get_in(list, [Access.at(-1), :name]) + "mary" + iex> get_and_update_in(list, [Access.at(0), :name], fn prev -> + ...> {prev, String.upcase(prev)} + ...> end) + {"john", [%{name: "JOHN"}, %{name: "mary"}]} + iex> get_and_update_in(list, [Access.at(-1), :name], fn prev -> + ...> {prev, String.upcase(prev)} + ...> end) + {"mary", [%{name: "john"}, %{name: "MARY"}]} + + `at/1` can also be used to pop elements out of a list or + a key inside of a list: + + iex> list = [%{name: "john"}, %{name: "mary"}] + iex> pop_in(list, [Access.at(0)]) + {%{name: "john"}, [%{name: "mary"}]} + iex> pop_in(list, [Access.at(0), :name]) + {"john", [%{}, %{name: "mary"}]} + + When the index is out of bounds, `nil` is returned and the update function is never called: + + iex> list = [%{name: "john"}, %{name: "mary"}] + iex> get_in(list, [Access.at(10), :name]) + nil + iex> get_and_update_in(list, [Access.at(10), :name], fn prev -> + ...> {prev, String.upcase(prev)} + ...> end) + {nil, [%{name: "john"}, %{name: "mary"}]} + + An error is raised if the accessed structure is not a list: + + iex> get_in(%{}, [Access.at(1)]) + ** (RuntimeError) Access.at/1 expected a list, got: %{} + + """ + @spec at(integer) :: access_fun(data :: list, current_value :: term) + def at(index) when is_integer(index) do + fn op, data, next -> at(op, data, index, next) end + end + + defp at(:get, data, index, next) when is_list(data) do + data |> Enum.at(index) |> next.() + end + + defp at(:get_and_update, data, index, next) when is_list(data) do + get_and_update_at(data, index, next, [], fn -> nil end) + end + + defp at(_op, data, _index, _next) do + raise "Access.at/1 expected a list, got: #{inspect(data)}" + end + + defp get_and_update_at([head | rest], 0, next, updates, _default_fun) do + case next.(head) do + {get, update} -> {get, :lists.reverse([update | updates], rest)} + :pop -> {head, :lists.reverse(updates, rest)} + end + end + + defp get_and_update_at([_ | _] = list, index, next, updates, default_fun) when index < 0 do + list_length = length(list) + + if list_length + index >= 0 do + get_and_update_at(list, list_length + index, next, updates, default_fun) + else + {default_fun.(), list} + end + end + + defp get_and_update_at([head | rest], index, next, updates, default_fun) when index > 0 do + get_and_update_at(rest, index - 1, next, [head | updates], default_fun) + end + + defp get_and_update_at([], _index, _next, updates, default_fun) do + {default_fun.(), :lists.reverse(updates)} + end + + @doc ~S""" + Same as `at/1` except that it raises `Enum.OutOfBoundsError` + if the given index is out of bounds. + + ## Examples + + iex> get_in([:a, :b, :c], [Access.at!(2)]) + :c + iex> get_in([:a, :b, :c], [Access.at!(3)]) + ** (Enum.OutOfBoundsError) out of bounds error at position 3 when traversing enumerable [:a, :b, :c] + + """ + @doc since: "1.11.0" + @spec at!(integer) :: access_fun(data :: list, current_value :: term) + def at!(index) when is_integer(index) do + fn op, data, next -> at!(op, data, index, next) end + end + + defp at!(:get, data, index, next) when is_list(data) do + case Enum.fetch(data, index) do + {:ok, value} -> next.(value) + :error -> raise Enum.OutOfBoundsError, index: index, enumerable: data + end + end + + defp at!(:get_and_update, data, index, next) when is_list(data) do + get_and_update_at(data, index, next, [], fn -> + raise Enum.OutOfBoundsError, index: index, enumerable: data + end) + end + + defp at!(_op, data, _index, _next) do + raise "Access.at!/1 expected a list, got: #{inspect(data)}" + end + + @doc ~S""" + Returns a function that accesses all elements of a list that match the provided predicate. + + The returned function is typically passed as an accessor to `Kernel.get_in/2`, + `Kernel.get_and_update_in/3`, and friends. + + ## Examples + + iex> list = [%{name: "john", salary: 10}, %{name: "francine", salary: 30}] + iex> get_in(list, [Access.filter(&(&1.salary > 20)), :name]) + ["francine"] + iex> get_and_update_in(list, [Access.filter(&(&1.salary <= 20)), :name], fn prev -> + ...> {prev, String.upcase(prev)} + ...> end) + {["john"], [%{name: "JOHN", salary: 10}, %{name: "francine", salary: 30}]} + + `filter/1` can also be used to pop elements out of a list or + a key inside of a list: + + iex> list = [%{name: "john", salary: 10}, %{name: "francine", salary: 30}] + iex> pop_in(list, [Access.filter(&(&1.salary >= 20))]) + {[%{name: "francine", salary: 30}], [%{name: "john", salary: 10}]} + iex> pop_in(list, [Access.filter(&(&1.salary >= 20)), :name]) + {["francine"], [%{name: "john", salary: 10}, %{salary: 30}]} + + When no match is found, an empty list is returned and the update function is never called + + iex> list = [%{name: "john", salary: 10}, %{name: "francine", salary: 30}] + iex> get_in(list, [Access.filter(&(&1.salary >= 50)), :name]) + [] + iex> get_and_update_in(list, [Access.filter(&(&1.salary >= 50)), :name], fn prev -> + ...> {prev, String.upcase(prev)} + ...> end) + {[], [%{name: "john", salary: 10}, %{name: "francine", salary: 30}]} + + An error is raised if the predicate is not a function or is of the incorrect arity: + + iex> get_in([], [Access.filter(5)]) + ** (FunctionClauseError) no function clause matching in Access.filter/1 + + An error is raised if the accessed structure is not a list: + + iex> get_in(%{}, [Access.filter(fn a -> a == 10 end)]) + ** (RuntimeError) Access.filter/1 expected a list, got: %{} + + """ + @doc since: "1.6.0" + @spec filter((term -> boolean)) :: access_fun(data :: list, current_value :: list) + def filter(func) when is_function(func) do + fn op, data, next -> filter(op, data, func, next) end + end + + defp filter(:get, data, func, next) when is_list(data) do + for elem <- data, func.(elem), do: next.(elem) + end + + defp filter(:get_and_update, data, func, next) when is_list(data) do + get_and_update_filter(data, func, next, [], []) + end + + defp filter(_op, data, _func, _next) do + raise "Access.filter/1 expected a list, got: #{inspect(data)}" + end + + defp get_and_update_filter([head | rest], func, next, updates, gets) do + if func.(head) do + case next.(head) do + {get, update} -> + get_and_update_filter(rest, func, next, [update | updates], [get | gets]) + + :pop -> + get_and_update_filter(rest, func, next, updates, [head | gets]) + end + else + get_and_update_filter(rest, func, next, [head | updates], gets) + end + end + + defp get_and_update_filter([], _func, _next, updates, gets) do + {:lists.reverse(gets), :lists.reverse(updates)} + end + + @doc ~S""" + Returns a function that accesses all items of a list that are within the provided range. + + The range will be normalized following the same rules from `Enum.slice/2`. + + The returned function is typically passed as an accessor to `Kernel.get_in/2`, + `Kernel.get_and_update_in/3`, and friends. + + ## Examples + + iex> list = [%{name: "john", salary: 10}, %{name: "francine", salary: 30}, %{name: "vitor", salary: 25}] + iex> get_in(list, [Access.slice(1..2), :name]) + ["francine", "vitor"] + iex> get_and_update_in(list, [Access.slice(1..3//2), :name], fn prev -> + ...> {prev, String.upcase(prev)} + ...> end) + {["francine"], [%{name: "john", salary: 10}, %{name: "FRANCINE", salary: 30}, %{name: "vitor", salary: 25}]} + + `slice/1` can also be used to pop elements out of a list or + a key inside of a list: + + iex> list = [%{name: "john", salary: 10}, %{name: "francine", salary: 30}, %{name: "vitor", salary: 25}] + iex> pop_in(list, [Access.slice(-2..-1)]) + {[%{name: "francine", salary: 30}, %{name: "vitor", salary: 25}], [%{name: "john", salary: 10}]} + iex> pop_in(list, [Access.slice(-2..-1), :name]) + {["francine", "vitor"], [%{name: "john", salary: 10}, %{salary: 30}, %{salary: 25}]} + + When no match is found, an empty list is returned and the update function is never called + + iex> list = [%{name: "john", salary: 10}, %{name: "francine", salary: 30}, %{name: "vitor", salary: 25}] + iex> get_in(list, [Access.slice(5..10//2), :name]) + [] + iex> get_and_update_in(list, [Access.slice(5..10//2), :name], fn prev -> + ...> {prev, String.upcase(prev)} + ...> end) + {[], [%{name: "john", salary: 10}, %{name: "francine", salary: 30}, %{name: "vitor", salary: 25}]} + + An error is raised if the accessed structure is not a list: + + iex> get_in(%{}, [Access.slice(2..10//3)]) + ** (ArgumentError) Access.slice/1 expected a list, got: %{} + + An error is raised if the step of the range is negative: + + iex> get_in([], [Access.slice(2..10//-1)]) + ** (ArgumentError) Access.slice/1 does not accept ranges with negative steps, got: 2..10//-1 + + """ + @doc since: "1.14" + @spec slice(Range.t()) :: access_fun(data :: list, current_value :: list) + def slice(%Range{} = range) do + if range.step > 0 do + fn op, data, next -> slice(op, data, range, next) end + else + raise ArgumentError, + "Access.slice/1 does not accept ranges with negative steps, got: #{inspect(range)}" + end + end + + defp slice(:get, data, %Range{} = range, next) when is_list(data) do + data + |> Enum.slice(range) + |> Enum.map(next) + end + + defp slice(:get_and_update, data, range, next) when is_list(data) do + range = normalize_range(range, data) + + if range.first > range.last do + {[], data} + else + get_and_update_slice(data, range, next, [], [], 0) + end + end + + defp slice(_op, data, _range, _next) do + raise ArgumentError, "Access.slice/1 expected a list, got: #{inspect(data)}" + end + + @doc """ + Returns a function that accesses all values in a map or a keyword list. + + The returned function is typically passed as an accessor to `Kernel.get_in/2`, + `Kernel.get_and_update_in/3`, and friends. + + ## Examples + + iex> users = %{"john" => %{age: 27}, "meg" => %{age: 23}} + iex> get_in(users, [Access.values(), :age]) |> Enum.sort() + [23, 27] + iex> update_in(users, [Access.values(), :age], fn age -> age + 1 end) + %{"john" => %{age: 28}, "meg" => %{age: 24}} + iex> put_in(users, [Access.values(), :planet], "Earth") + %{"john" => %{age: 27, planet: "Earth"}, "meg" => %{age: 23, planet: "Earth"}} + + Values in keyword lists can be accessed as well: + + iex> users = [john: %{age: 27}, meg: %{age: 23}] + iex> get_and_update_in(users, [Access.values(), :age], fn age -> {age, age + 1} end) + {[27, 23], [john: %{age: 28}, meg: %{age: 24}]} + + By returning `:pop` from an accessor function, you can remove the accessed key and value + from the map or keyword list: + + iex> require Integer + iex> numbers = [one: 1, two: 2, three: 3, four: 4] + iex> get_and_update_in(numbers, [Access.values()], fn num -> + ...> if Integer.is_even(num), do: :pop, else: {num, to_string(num)} + ...> end) + {[1, 2, 3, 4], [one: "1", three: "3"]} + + An error is raised if the accessed structure is not a map nor a keyword list: + + iex> get_in([1, 2, 3], [Access.values()]) + ** (RuntimeError) Access.values/0 expected a map or a keyword list, got: [1, 2, 3] + """ + @doc since: "1.19.0" + @spec values() :: Access.access_fun(data :: map() | keyword(), current_value :: list()) + def values do + &values/3 + end + + defp values(:get, data = %{}, next) do + Enum.map(data, fn {_key, value} -> next.(value) end) + end + + defp values(:get_and_update, data = %{}, next) do + {reverse_gets, updated_data} = + Enum.reduce(data, {[], %{}}, fn {key, value}, {gets, data_acc} -> + case next.(value) do + {get, update} -> {[get | gets], Map.put(data_acc, key, update)} + :pop -> {[value | gets], data_acc} + end + end) + + {Enum.reverse(reverse_gets), updated_data} + end + + defp values(op, data = [], next) do + values_keyword(op, data, next) + end + + defp values(op, data = [{key, _value} | _tail], next) when is_atom(key) do + values_keyword(op, data, next) + end + + defp values(_op, data, _next) do + raise "Access.values/0 expected a map or a keyword list, got: #{inspect(data)}" + end + + defp values_keyword(:get, data, next) do + Enum.map(data, fn {key, value} when is_atom(key) -> next.(value) end) + end + + defp values_keyword(:get_and_update, data, next) do + {reverse_gets, reverse_updated_data} = + Enum.reduce(data, {[], []}, fn {key, value}, {gets, data_acc} when is_atom(key) -> + case next.(value) do + {get, update} -> {[get | gets], [{key, update} | data_acc]} + :pop -> {[value | gets], data_acc} + end + end) + + {Enum.reverse(reverse_gets), Enum.reverse(reverse_updated_data)} + end + + defp normalize_range(%Range{first: first, last: last, step: step}, list) + when first < 0 or last < 0 do + count = length(list) + first = if first >= 0, do: first, else: Kernel.max(first + count, 0) + last = if last >= 0, do: last, else: last + count + Range.new(first, last, step) + end + + defp normalize_range(range, _list), do: range + + defp get_and_update_slice([head | rest], range, next, updates, gets, index) do + if index in range do + case next.(head) do + :pop -> + get_and_update_slice(rest, range, next, updates, [head | gets], index + 1) + + {get, update} -> + get_and_update_slice( + rest, + range, + next, + [update | updates], + [get | gets], + index + 1 + ) + end + else + get_and_update_slice(rest, range, next, [head | updates], gets, index + 1) + end + end + + defp get_and_update_slice([], _range, _next, updates, gets, _index) do + {:lists.reverse(gets), :lists.reverse(updates)} + end + + @doc ~S""" + Returns a function that accesses the first element of a list that matches the provided predicate. + + The returned function is typically passed as an accessor to `Kernel.get_in/2`, + `Kernel.get_and_update_in/3`, and friends. + + ## Examples + + iex> list = [%{name: "john", salary: 10}, %{name: "francine", salary: 30}] + iex> get_in(list, [Access.find(&(&1.salary > 20)), :name]) + "francine" + iex> get_and_update_in(list, [Access.find(&(&1.salary <= 40)), :name], fn prev -> + ...> {prev, String.upcase(prev)} + ...> end) + {"john", [%{name: "JOHN", salary: 10}, %{name: "francine", salary: 30}]} + + `find/1` can also be used to pop the first found element out of a list or + a key inside of a list: + + iex> list = [%{name: "john", salary: 10}, %{name: "francine", salary: 30}] + iex> pop_in(list, [Access.find(&(&1.salary <= 40))]) + {%{name: "john", salary: 10}, [%{name: "francine", salary: 30}]} + + When no match is found, nil is returned and the update function is never called + + iex> list = [%{name: "john", salary: 10}, %{name: "francine", salary: 30}] + iex> get_in(list, [Access.find(&(&1.salary >= 50)), :name]) + nil + iex> get_and_update_in(list, [Access.find(&(&1.salary >= 50)), :name], fn prev -> + ...> {prev, String.upcase(prev)} + ...> end) + {nil, [%{name: "john", salary: 10}, %{name: "francine", salary: 30}]} + + An error is raised if the predicate is not a function or is of the incorrect arity: + + iex> get_in([], [Access.find(5)]) + ** (FunctionClauseError) no function clause matching in Access.find/1 + + An error is raised if the accessed structure is not a list: + + iex> get_in(%{}, [Access.find(fn a -> a == 10 end)]) + ** (RuntimeError) Access.find/1 expected a list, got: %{} + """ + @doc since: "1.17.0" + @spec find((term -> as_boolean(term))) :: access_fun(data :: list, current_value :: term) + def find(predicate) when is_function(predicate, 1) do + fn op, data, next -> find(op, data, predicate, next) end + end + + defp find(:get, data, predicate, next) when is_list(data) do + data |> Enum.find(predicate) |> next.() + end + + defp find(:get_and_update, data, predicate, next) when is_list(data) do + get_and_update_find(data, [], predicate, next) + end + + defp find(_op, data, _predicate, _next) do + raise "Access.find/1 expected a list, got: #{inspect(data)}" + end + + defp get_and_update_find([], updates, _predicate, _next) do + {nil, :lists.reverse(updates)} + end + + defp get_and_update_find([head | rest], updates, predicate, next) do + if predicate.(head) do + case next.(head) do + {get, update} -> {get, :lists.reverse([update | updates], rest)} + :pop -> {head, :lists.reverse(updates, rest)} + end + else + get_and_update_find(rest, [head | updates], predicate, next) + end end end diff --git a/lib/elixir/lib/agent.ex b/lib/elixir/lib/agent.ex index 628f0675f79..df0cbd3e534 100644 --- a/lib/elixir/lib/agent.ex +++ b/lib/elixir/lib/agent.ex @@ -1,3 +1,7 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + defmodule Agent do @moduledoc """ Agents are a simple abstraction around state. @@ -6,41 +10,61 @@ defmodule Agent do must be accessed from different processes or by the same process at different points in time. - The Agent module provides a basic server implementation that + The `Agent` module provides a basic server implementation that allows state to be retrieved and updated via a simple API. ## Examples - For example, in the Mix tool that ships with Elixir, we need - to keep a set of all tasks executed by a given project. Since - this set is shared, we can implement it with an Agent: + For example, the following agent implements a counter: + + defmodule Counter do + use Agent - defmodule Mix.TasksServer do - def start_link do - Agent.start_link(fn -> HashSet.new end, name: __MODULE__) + def start_link(initial_value) do + Agent.start_link(fn -> initial_value end, name: __MODULE__) end - @doc "Checks if the task has already executed" - def executed?(task, project) do - item = {task, project} - Agent.get(__MODULE__, fn set -> - item in set - end) + def value do + Agent.get(__MODULE__, & &1) end - @doc "Marks a task as executed" - def put_task(task, project) do - item = {task, project} - Agent.update(__MODULE__, &Set.put(&1, item)) + def increment do + Agent.update(__MODULE__, &(&1 + 1)) end end - Note that agents still provide a segregation between the - client and server APIs, as seen in GenServers. In particular, - all code inside the function passed to the agent is executed - by the agent. This distinction is important because you may - want to avoid expensive operations inside the agent, as it will - effectively block the agent until the request is fulfilled. + Usage would be: + + Counter.start_link(0) + #=> {:ok, #PID<0.123.0>} + + Counter.value() + #=> 0 + + Counter.increment() + #=> :ok + + Counter.increment() + #=> :ok + + Counter.value() + #=> 2 + + Thanks to the agent server process, the counter can be safely incremented + concurrently. + + > #### `use Agent` {: .info} + > + > When you `use Agent`, the `Agent` module will define a + > `child_spec/1` function, so your module can be used + > as a child in a supervision tree. + + Agents provide a segregation between the client and server APIs (similar to + `GenServer`s). In particular, the functions passed as arguments to the calls to + `Agent` functions are invoked inside the agent (the server). This distinction + is important because you may want to avoid expensive operations inside the + agent, as they will effectively block the agent until the request is + fulfilled. Consider these two examples: @@ -51,51 +75,112 @@ defmodule Agent do # Compute in the agent/client def get_something(agent) do - Agent.get(agent, &(&1)) |> do_something_expensive() + Agent.get(agent, & &1) |> do_something_expensive() end - The first one blocks the agent while the second one copies - all the state to the client and executes the operation in the client. - The trade-off here is exactly if the data is small enough to be - sent to the client cheaply or large enough to require processing on - the server (or at least some initial processing). + The first function blocks the agent. The second function copies all the state + to the client and then executes the operation in the client. One aspect to + consider is whether the data is large enough to require processing in the server, + at least initially, or small enough to be sent to the client cheaply. Another + factor is whether the data needs to be processed atomically: getting the + state and calling `do_something_expensive(state)` outside of the agent means + that the agent's state can be updated in the meantime. This is specially + important in case of updates as computing the new state in the client rather + than in the server can lead to race conditions if multiple clients are trying + to update the same state to different values. + + ## How to supervise + + An `Agent` is most commonly started under a supervision tree. + When we invoke `use Agent`, it automatically defines a `child_spec/1` + function that allows us to start the agent directly under a supervisor. + To start an agent under a supervisor with an initial counter of 0, + one may do: + + children = [ + {Counter, 0} + ] + + Supervisor.start_link(children, strategy: :one_for_all) + + While one could also simply pass the `Counter` as a child to the supervisor, + such as: + + children = [ + Counter # Same as {Counter, []} + ] + + Supervisor.start_link(children, strategy: :one_for_all) + + The definition above wouldn't work for this particular example, + as it would attempt to start the counter with an initial value + of an empty list. However, this may be a viable option in your + own agents. A common approach is to use a keyword list, as that + would allow setting the initial value and giving a name to the + counter process, for example: + + def start_link(opts) do + {initial_value, opts} = Keyword.pop(opts, :initial_value, 0) + Agent.start_link(fn -> initial_value end, opts) + end + + and then you can use `Counter`, `{Counter, name: :my_counter}` or + even `{Counter, initial_value: 0, name: :my_counter}` as a child + specification. + + `use Agent` also accepts a list of options which configures the + child specification and therefore how it runs under a supervisor. + The generated `child_spec/1` can be customized with the following options: - ## Name Registration + * `:id` - the child specification identifier, defaults to the current module + * `:restart` - when the child should be restarted, defaults to `:permanent` + * `:shutdown` - how to shut down the child, either immediately or by giving it time to shut down - An Agent is bound to the same name registration rules as GenServers. - Read more about it in the `GenServer` docs. + For example: + + use Agent, restart: :transient, shutdown: 10_000 + + See the "Child specification" section in the `Supervisor` module for more + detailed information. The `@doc` annotation immediately preceding + `use Agent` will be attached to the generated `child_spec/1` function. + + ## Name registration + + An agent is bound to the same name registration rules as GenServers. + Read more about it in the `GenServer` documentation. ## A word on distributed agents It is important to consider the limitations of distributed agents. Agents - work by sending anonymous functions between the caller and the agent. - In a distributed setup with multiple nodes, agents only work if the caller - (client) and the agent have the same version of a given module. - - This setup may exhibit issues when doing "rolling upgrades". By rolling - upgrades we mean the following situation: you wish to deploy a new version of - your software by *shutting down* some of your nodes and replacing them with - nodes running a new version of the software. In this setup, part of your - environment will have one version of a given module and the other part - another version (the newer one) of the same module; this may cause agents to - crash. That said, if you plan to run in distributed environments, agents - should likely be avoided. - - Note, however, that agents work fine if you want to perform hot code - swapping, as it keeps both the old and new versions of a given module. - We detail how to do hot code swapping with agents in the next section. + provide two APIs, one that works with anonymous functions and another + that expects an explicit module, function, and arguments. + + In a distributed setup with multiple nodes, the API that accepts anonymous + functions only works if the caller (client) and the agent have the same + version of the caller module. + + Keep in mind this issue also shows up when performing "rolling upgrades" + with agents. By rolling upgrades we mean the following situation: you wish + to deploy a new version of your software by *shutting down* some of your + nodes and replacing them with nodes running a new version of the software. + In this setup, part of your environment will have one version of a given + module and the other part another version (the newer one) of the same module. + + The best solution is to simply use the explicit module, function, and arguments + APIs when working with distributed agents. ## Hot code swapping An agent can have its code hot swapped live by simply passing a module, - function and args tuple to the update instruction. For example, imagine + function, and arguments tuple to the update instruction. For example, imagine you have an agent named `:sample` and you want to convert its inner state - from some dict structure to a map. It can be done with the following + from a keyword list to a map. It can be done with the following instruction: {:update, :sample, {:advanced, {Enum, :into, [%{}]}}} - The agent's state will be added to the given list as the first argument. + The agent's state will be added to the given list of arguments (`[%{}]`) as + the first argument. """ @typedoc "Return values of `start*` functions" @@ -111,13 +196,50 @@ defmodule Agent do @type state :: term @doc """ - Starts an agent linked to the current process. + Returns a specification to start an agent under a supervisor. + + See the "Child specification" section in the `Supervisor` module for more detailed information. + """ + @doc since: "1.5.0" + def child_spec(arg) do + %{ + id: Agent, + start: {Agent, :start_link, [arg]} + } + end + + @doc false + defmacro __using__(opts) do + quote location: :keep, bind_quoted: [opts: opts] do + if not Module.has_attribute?(__MODULE__, :doc) do + @doc """ + Returns a specification to start this module under a supervisor. + + See `Supervisor`. + """ + end + + def child_spec(arg) do + default = %{ + id: __MODULE__, + start: {__MODULE__, :start_link, [arg]} + } + + Supervisor.child_spec(default, unquote(Macro.escape(opts))) + end + + defoverridable child_spec: 1 + end + end + + @doc """ + Starts an agent linked to the current process with the given function. This is often used to start the agent as part of a supervision tree. - Once the agent is spawned, the given function is invoked and its return - value is used as the agent state. Note that `start_link` does not return - until the given function has returned. + Once the agent is spawned, the given function `fun` is invoked in the server + process, and should return the initial agent state. Note that `start_link/2` + does not return until the given function has returned. ## Options @@ -129,7 +251,7 @@ defmodule Agent do and the start function will return `{:error, :timeout}`. If the `:debug` option is present, the corresponding function in the - [`:sys` module](http://www.erlang.org/doc/man/sys.html) will be invoked. + [`:sys` module](`:sys`) will be invoked. If the `:spawn_opt` option is present, its value will be passed as options to the underlying process as in `Process.spawn/4`. @@ -137,36 +259,86 @@ defmodule Agent do ## Return values If the server is successfully created and initialized, the function returns - `{:ok, pid}`, where `pid` is the pid of the server. If there already exists - an agent with the specified name, the function returns - `{:error, {:already_started, pid}}` with the pid of that process. + `{:ok, pid}`, where `pid` is the PID of the server. If an agent with the + specified name already exists, the function returns + `{:error, {:already_started, pid}}` with the PID of that process. + + If the given function callback fails, the function returns `{:error, reason}`. + + ## Examples + + iex> {:ok, pid} = Agent.start_link(fn -> 42 end) + iex> Agent.get(pid, fn state -> state end) + 42 + + iex> {:error, {exception, _stacktrace}} = Agent.start(fn -> raise "oops" end) + iex> exception + %RuntimeError{message: "oops"} - If the given function callback fails with `reason`, the function returns - `{:error, reason}`. """ - @spec start_link((() -> term), GenServer.options) :: on_start + @spec start_link((-> term), GenServer.options()) :: on_start def start_link(fun, options \\ []) when is_function(fun, 0) do GenServer.start_link(Agent.Server, fun, options) end + @doc """ + Starts an agent linked to the current process. + + Same as `start_link/2` but a module, function, and arguments are expected + instead of an anonymous function; `fun` in `module` will be called with the + given arguments `args` to initialize the state. + """ + @spec start_link(module, atom, [term], GenServer.options()) :: on_start + def start_link(module, fun, args, options \\ []) do + GenServer.start_link(Agent.Server, {module, fun, args}, options) + end + @doc """ Starts an agent process without links (outside of a supervision tree). See `start_link/2` for more information. + + ## Examples + + iex> {:ok, pid} = Agent.start(fn -> 42 end) + iex> Agent.get(pid, fn state -> state end) + 42 + """ - @spec start((() -> term), GenServer.options) :: on_start + @spec start((-> term), GenServer.options()) :: on_start def start(fun, options \\ []) when is_function(fun, 0) do GenServer.start(Agent.Server, fun, options) end @doc """ - Gets the agent value and executes the given function. + Starts an agent without links with the given module, function, and arguments. + + See `start_link/4` for more information. + """ + @spec start(module, atom, [term], GenServer.options()) :: on_start + def start(module, fun, args, options \\ []) do + GenServer.start(Agent.Server, {module, fun, args}, options) + end + + @doc """ + Gets an agent value via the given anonymous function. The function `fun` is sent to the `agent` which invokes the function passing the agent state. The result of the function invocation is - returned. + returned from this function. + + `timeout` is an integer greater than zero which specifies how many + milliseconds are allowed before the agent executes the function and returns + the result value, or the atom `:infinity` to wait indefinitely. If no result + is received within the specified time, the function call fails and the caller + exits. + + ## Examples + + iex> {:ok, pid} = Agent.start_link(fn -> 42 end) + iex> Agent.get(pid, fn state -> state end) + 42 - A timeout can also be specified (it has a default value of 5000). """ @spec get(agent, (state -> a), timeout) :: a when a: var def get(agent, fun, timeout \\ 5000) when is_function(fun, 1) do @@ -174,14 +346,40 @@ defmodule Agent do end @doc """ - Gets and updates the agent state in one operation. + Gets an agent value via the given function. + + Same as `get/3` but a module, function, and arguments are expected + instead of an anonymous function. The state is added as first + argument to the given list of arguments. + """ + @spec get(agent, module, atom, [term], timeout) :: term + def get(agent, module, fun, args, timeout \\ 5000) do + GenServer.call(agent, {:get, {module, fun, args}}, timeout) + end + + @doc """ + Gets and updates the agent state in one operation via the given anonymous + function. The function `fun` is sent to the `agent` which invokes the function passing the agent state. The function must return a tuple with two - elements, the first being the value to return (i.e. the `get` value) - and the second one is the new state. + elements, the first being the value to return (that is, the "get" value) + and the second one being the new state of the agent. + + `timeout` is an integer greater than zero which specifies how many + milliseconds are allowed before the agent executes the function and returns + the result value, or the atom `:infinity` to wait indefinitely. If no result + is received within the specified time, the function call fails and the caller + exits. + + ## Examples + + iex> {:ok, pid} = Agent.start_link(fn -> 42 end) + iex> Agent.get_and_update(pid, fn state -> {state, state + 1} end) + 42 + iex> Agent.get(pid, fn state -> state end) + 43 - A timeout can also be specified (it has a default value of 5000). """ @spec get_and_update(agent, (state -> {a, state}), timeout) :: a when a: var def get_and_update(agent, fun, timeout \\ 5000) when is_function(fun, 1) do @@ -189,40 +387,132 @@ defmodule Agent do end @doc """ - Updates the agent state. + Gets and updates the agent state in one operation via the given function. + + Same as `get_and_update/3` but a module, function, and arguments are expected + instead of an anonymous function. The state is added as first + argument to the given list of arguments. + """ + @spec get_and_update(agent, module, atom, [term], timeout) :: term + def get_and_update(agent, module, fun, args, timeout \\ 5000) do + GenServer.call(agent, {:get_and_update, {module, fun, args}}, timeout) + end + + @doc """ + Updates the agent state via the given anonymous function. The function `fun` is sent to the `agent` which invokes the function - passing the agent state. The function must return the new state. + passing the agent state. The return value of `fun` becomes the new + state of the agent. - A timeout can also be specified (it has a default value of 5000). This function always returns `:ok`. + + `timeout` is an integer greater than zero which specifies how many + milliseconds are allowed before the agent executes the function and returns + the result value, or the atom `:infinity` to wait indefinitely. If no result + is received within the specified time, the function call fails and the caller + exits. + + ## Examples + + iex> {:ok, pid} = Agent.start_link(fn -> 42 end) + iex> Agent.update(pid, fn state -> state + 1 end) + :ok + iex> Agent.get(pid, fn state -> state end) + 43 + """ - @spec update(agent, (state -> state)) :: :ok + @spec update(agent, (state -> state), timeout) :: :ok def update(agent, fun, timeout \\ 5000) when is_function(fun, 1) do GenServer.call(agent, {:update, fun}, timeout) end @doc """ - Performs a cast (fire and forget) operation on the agent state. + Updates the agent state via the given function. + + Same as `update/3` but a module, function, and arguments are expected + instead of an anonymous function. The state is added as first + argument to the given list of arguments. + + ## Examples + + iex> {:ok, pid} = Agent.start_link(fn -> 42 end) + iex> Agent.update(pid, Kernel, :+, [12]) + :ok + iex> Agent.get(pid, fn state -> state end) + 54 + + """ + @spec update(agent, module, atom, [term], timeout) :: :ok + def update(agent, module, fun, args, timeout \\ 5000) do + GenServer.call(agent, {:update, {module, fun, args}}, timeout) + end + + @doc """ + Performs a cast (*fire and forget*) operation on the agent state. The function `fun` is sent to the `agent` which invokes the function - passing the agent state. The function must return the new state. + passing the agent state. The return value of `fun` becomes the new + state of the agent. + + Note that `cast` returns `:ok` immediately, regardless of whether `agent` (or + the node it should live on) exists. + + ## Examples + + iex> {:ok, pid} = Agent.start_link(fn -> 42 end) + iex> Agent.cast(pid, fn state -> state + 1 end) + :ok + iex> Agent.get(pid, fn state -> state end) + 43 - Note that `cast` returns `:ok` immediately, regardless of whether the - destination node or agent exists. """ @spec cast(agent, (state -> state)) :: :ok def cast(agent, fun) when is_function(fun, 1) do - GenServer.cast(agent, fun) + GenServer.cast(agent, {:cast, fun}) + end + + @doc """ + Performs a cast (*fire and forget*) operation on the agent state. + + Same as `cast/2` but a module, function, and arguments are expected + instead of an anonymous function. The state is added as first + argument to the given list of arguments. + + ## Examples + + iex> {:ok, pid} = Agent.start_link(fn -> 42 end) + iex> Agent.cast(pid, Kernel, :+, [12]) + :ok + iex> Agent.get(pid, fn state -> state end) + 54 + + """ + @spec cast(agent, module, atom, [term]) :: :ok + def cast(agent, module, fun, args) do + GenServer.cast(agent, {:cast, {module, fun, args}}) end @doc """ - Stops the agent. + Synchronously stops the agent with the given `reason`. + + It returns `:ok` if the agent terminates with the given + reason. If the agent terminates with another reason, the call will + exit. + + This function keeps OTP semantics regarding error reporting. + If the reason is any other than `:normal`, `:shutdown` or + `{:shutdown, _}`, an error report will be logged. + + ## Examples + + iex> {:ok, pid} = Agent.start_link(fn -> 42 end) + iex> Agent.stop(pid) + :ok - Returns `:ok` if the agent is stopped within the given `timeout`. """ - @spec stop(agent, timeout) :: :ok - def stop(agent, timeout \\ 5000) do - GenServer.call(agent, :stop, timeout) + @spec stop(agent, reason :: term, timeout) :: :ok + def stop(agent, reason \\ :normal, timeout \\ :infinity) do + GenServer.stop(agent, reason, timeout) end end diff --git a/lib/elixir/lib/agent/server.ex b/lib/elixir/lib/agent/server.ex index 0b06e4cddbe..cba85b49ac9 100644 --- a/lib/elixir/lib/agent/server.ex +++ b/lib/elixir/lib/agent/server.ex @@ -1,53 +1,55 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + defmodule Agent.Server do @moduledoc false use GenServer def init(fun) do - {:ok, fun.()} + _ = initial_call(fun) + {:ok, run(fun, [])} end def handle_call({:get, fun}, _from, state) do - {:reply, fun.(state), state} + {:reply, run(fun, [state]), state} end def handle_call({:get_and_update, fun}, _from, state) do - {reply, state} = fun.(state) - {:reply, reply, state} + case run(fun, [state]) do + {reply, state} -> {:reply, reply, state} + other -> {:stop, {:bad_return_value, other}, state} + end end def handle_call({:update, fun}, _from, state) do - {:reply, :ok, fun.(state)} + {:reply, :ok, run(fun, [state])} end - def handle_call(:stop, _from, state) do - {:stop, :normal, :ok, state} + def handle_cast({:cast, fun}, state) do + {:noreply, run(fun, [state])} end - def handle_call(msg, from, state) do - super(msg, from, state) + def code_change(_old, state, fun) do + {:ok, run(fun, [state])} end - def handle_cast(fun, state) when is_function(fun, 1) do - {:noreply, fun.(state)} + defp initial_call(mfa) do + _ = Process.put(:"$initial_call", get_initial_call(mfa)) + :ok end - def handle_cast(msg, state) do - super(msg, state) + defp get_initial_call(fun) when is_function(fun, 0) do + {:module, module} = Function.info(fun, :module) + {:name, name} = Function.info(fun, :name) + {module, name, 0} end - def code_change(_old, state, { m, f, a }) do - {:ok, apply(m, f, [state|a])} + defp get_initial_call({mod, fun, args}) do + {mod, fun, length(args)} end - def terminate(_reason, _state) do - # There is a race condition if the agent is - # restarted too fast and it is registered. - try do - self |> :erlang.process_info(:registered_name) |> elem(1) |> Process.unregister - rescue - _ -> :ok - end - :ok - end + defp run({m, f, a}, extra), do: apply(m, f, extra ++ a) + defp run(fun, extra), do: apply(fun, extra) end diff --git a/lib/elixir/lib/application.ex b/lib/elixir/lib/application.ex index 691704e0258..de8fe127fa6 100644 --- a/lib/elixir/lib/application.ex +++ b/lib/elixir/lib/application.ex @@ -1,213 +1,943 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + defmodule Application do @moduledoc """ A module for working with applications and defining application callbacks. - In Elixir (actually, in Erlang/OTP), an application is a component - implementing some specific functionality, that can be started and stopped - as a unit, and which can be re-used in other systems as well. - - Applications are defined with an application file named `APP.app` where - `APP` is the APP name, usually in `underscore_case` convention. The - application file must reside in the same `ebin` directory as the - application's modules bytecode. + Applications are the idiomatic way to package software in Erlang/OTP. To get + the idea, they are similar to the "library" concept common in other + programming languages, but with some additional characteristics. - In Elixir, Mix is responsible for compiling your source code and - generating your application `.app` file. Furthermore, Mix is also - responsible for configuring, starting and stopping your application - and its dependencies. For this reason, this documentation will focus - on the remaining aspects of your application: the application environment, - and the application callback module. + An application is a component implementing some specific functionality, with a + standardized directory structure, configuration, and life cycle. Applications + are *loaded*, *started*, and *stopped*. Each application also has its own + environment, which provides a unified API for configuring each application. - You can learn more about Mix compilation of `.app` files by typing - `mix help compile.app`. + Developers typically interact with the application environment and its + callback module. Therefore those will be the topics we will cover first + before jumping into details about the application resource file and life cycle. - ## Application environment + ## The application environment - Once an application is started, OTP provides an application environment - that can be used to configure applications. + Each application has its own environment. The environment is a keyword list + that maps atoms to terms. Note that this environment is unrelated to the + operating system environment. - Assuming you are inside a Mix project, you can edit your application - function in the `mix.exs` file to the following: + By default, the environment of an application is an empty list. In a Mix + project's `mix.exs` file, you can set the `:env` key in `application/0`: def application do - [env: [hello: :world]] + [env: [db_host: "localhost"]] + end + + Now, in your application, you can read this environment by using functions + such as `fetch_env!/2` and friends: + + defmodule MyApp.DBClient do + def start_link() do + SomeLib.DBClient.start_link(host: db_host()) + end + + defp db_host do + Application.fetch_env!(:my_app, :db_host) + end end - In the application function, we can define the default environment values - for our application. By starting your application with `iex -S mix`, you - can access the default value: + In Mix projects, the environment of the application and its dependencies can + be overridden via the `config/config.exs` and `config/runtime.exs` files. The + former is loaded at build-time, before your code compiles, and the latter at + runtime, just before your app starts. For example, someone using your application + can override its `:db_host` environment variable as follows: + + import Config + config :my_app, :db_host, "db.local" + + See the "Configuration" section in the `Mix` module for more information. + You can also change the application environment dynamically by using functions + such as `put_env/3` and `delete_env/2`. + + > #### Application environment in libraries {: .info} + > + > If you are writing a library to be used by other developers, + > it is generally recommended to avoid the application environment, as the + > application environment is effectively a global storage. For more information, + > read about this [anti-pattern](design-anti-patterns.md#using-application-configuration-for-libraries). + + > #### Reading the environment of other applications {: .warning} + > + > Each application is responsible for its own environment. Do not + > use the functions in this module for directly accessing or modifying + > the environment of other applications. Whenever you change the application + > environment, Elixir's build tool will only recompile the files that + > belong to that application. So if you read the application environment + > of another application, there is a chance you will be depending on + > outdated configuration, as your file won't be recompiled as it changes. + + ## Compile-time environment + + In the previous example, we read the application environment at runtime: + + defmodule MyApp.DBClient do + def start_link() do + SomeLib.DBClient.start_link(host: db_host()) + end - Application.get_env(:APP_NAME, :hello) - #=> {:ok, :hello} + defp db_host do + Application.fetch_env!(:my_app, :db_host) + end + end - It is also possible to put and delete values from the application value, - including new values that are not defined in the environment file (although - those should be avoided). + In other words, the environment key `:db_host` for application `:my_app` + will only be read when `MyApp.DBClient` effectively starts. While reading + the application environment at runtime is the preferred approach, in some + rare occasions you may want to use the application environment to configure + the compilation of a certain project. However, if you try to access + `Application.fetch_env!/2` outside of a function: - In the future, we plan to support configuration files which allows - developers to configure the environment of their dependencies. + defmodule MyApp.DBClient do + @db_host Application.fetch_env!(:my_app, :db_host) - Keep in mind that each application is responsible for its environment. - Do not use the functions in this module for directly access or modify - the environment of other application (as it may lead to inconsistent - data in the application environment). + def start_link() do + SomeLib.DBClient.start_link(host: @db_host) + end + end - ## Application module callback + You might see warnings and errors: - Often times, an application defines a supervision tree that must be started - and stopped when the application starts and stops. For such, we need to - define an application module callback. The first step is to define the - module callback in the application definition in the `mix.exs` file: + warning: Application.fetch_env!/2 is discouraged in the module body, + use Application.compile_env/3 instead + iex:3: MyApp.DBClient + + ** (ArgumentError) could not fetch application environment :db_host + for application :my_app because the application was not loaded nor + configured + + This happens because, when defining modules, the application environment + is not yet available. Luckily, the warning tells us how to solve this + issue, by using `Application.compile_env/3` instead: + + defmodule MyApp.DBClient do + @db_host Application.compile_env(:my_app, :db_host, "db.local") + + def start_link() do + SomeLib.DBClient.start_link(host: @db_host) + end + end + + The difference here is that `compile_env` expects the default value to be + given as an argument, instead of using the `def application` function of + your `mix.exs`. Furthermore, by using `compile_env/3`, tools like Mix will + store the values used during compilation and compare the compilation values + with the runtime values whenever your system starts, raising an error in + case they differ. + + In any case, compile-time environments should be avoided. Whenever possible, + reading the application environment at runtime should be the first choice. + + ## The application callback module + + Applications can be loaded, started, and stopped. Generally, build tools + like Mix take care of starting an application and all of its dependencies + for you, but you can also do it manually by calling: + + {:ok, _} = Application.ensure_all_started(:some_app) + + When an application starts, developers may configure a callback module + that executes custom code. Developers use this callback to start the + application supervision tree. + + The first step to do so is to add a `:mod` key to the `application/0` + definition in your `mix.exs` file. It expects a tuple, with the application + callback module and start argument (commonly an empty list): def application do [mod: {MyApp, []}] end - Our application now requires the `MyApp` module to provide an application - callback. This can be done by invoking `use Application` in that module - and defining a `start/2` callback, for example: + The `MyApp` module given to `:mod` needs to implement the `Application` behaviour. + This can be done by putting `use Application` in that module and implementing the + `c:start/2` callback, for example: defmodule MyApp do use Application def start(_type, _args) do - MyApp.Supervisor.start_link() + children = [] + Supervisor.start_link(children, strategy: :one_for_one) end end - `start/2` most commonly returns `{:ok, pid}` or `{:ok, pid, state}` where - `pid` identifies the supervision tree and the state is the application state. - `args` is second element of the tuple given to the `:mod` option. + > #### `use Application` {: .info} + > + > When you `use Application`, the `Application` module will + > set `@behaviour Application` and define an overridable + > definition for the `c:stop/1` function, which is required + > by Erlang/OTP. + + The `c:start/2` callback has to spawn and link a supervisor and return `{:ok, + pid}` or `{:ok, pid, state}`, where `pid` is the PID of the supervisor, and + `state` is an optional application state. `args` is the second element of the + tuple given to the `:mod` option. + + The `type` argument passed to `c:start/2` is usually `:normal` unless in a + distributed setup where application takeovers and failovers are configured. + Distributed applications are beyond the scope of this documentation. + + When an application is shutting down, its `c:stop/1` callback is called after + the supervision tree has been stopped by the runtime. This callback allows the + application to do any final cleanup. The argument is the state returned by + `c:start/2`, if it did, or `[]` otherwise. The return value of `c:stop/1` is + ignored. + + By using `Application`, modules get a default implementation of `c:stop/1` + that ignores its argument and returns `:ok`, but it can be overridden. + + Application callback modules may also implement the optional callback + `c:prep_stop/1`. If present, `c:prep_stop/1` is invoked before the supervision + tree is terminated. Its argument is the state returned by `c:start/2`, if it did, + or `[]` otherwise, and its return value is passed to `c:stop/1`. + + ## The application resource file + + In the sections above, we have configured an application in the + `application/0` section of the `mix.exs` file. Ultimately, Mix will use + this configuration to create an [*application resource + file*](https://www.erlang.org/doc/man/app), which is a file called + `APP_NAME.app`. For example, the application resource file of the OTP + application `ex_unit` is called `ex_unit.app`. + + You can learn more about the generation of application resource files in + the documentation of `Mix.Tasks.Compile.App`, available as well by running + `mix help compile.app`. + + ## The application life cycle + + ### Loading applications + + Applications are *loaded*, which means that the runtime finds and processes + their resource files: + + Application.load(:ex_unit) + #=> :ok + + When an application is loaded, the environment specified in its resource file + is merged with any overrides from config files. + + Loading an application *does not* load its modules. + + In practice, you rarely load applications by hand because that is part of the + start process, explained next. + + ### Starting applications + + Applications are also *started*: + + Application.start(:ex_unit) + #=> :ok + + Once your application is compiled, running your system is a matter of starting + your current application and its dependencies. Differently from other languages, + Elixir does not have a `main` procedure that is responsible for starting your + system. Instead, you start one or more applications, each with their own + initialization and termination logic. + + When an application is started, the `Application.load/1` is automatically + invoked if it hasn't been done yet. Then, it checks if the dependencies listed + in the `applications` key of the resource file are already started. Having at + least one dependency not started is an error condition. Functions like + `ensure_all_started/1` take care of starting an application and all of its + dependencies for you. + + If the application does not have a callback module configured, starting is + done at this point. Otherwise, its `c:start/2` callback is invoked. The PID of + the top-level supervisor returned by this function is stored by the runtime + for later use, and the returned application state is saved too, if any. + + ### Stopping applications + + Started applications are, finally, *stopped*: + + Application.stop(:ex_unit) + #=> :ok + + Stopping an application without a callback module defined, is in practice a + no-op, except for some system tracing. + + Stopping an application with a callback module has three steps: + + 1. If present, invoke the optional callback `c:prep_stop/1`. + 2. Terminate the top-level supervisor. + 3. Invoke the required callback `c:stop/1`. + + The arguments passed to the callbacks are related to the state optionally + returned by `c:start/2`, and are documented in the section about the callback + module above. + + It is important to highlight that step 2 is a blocking one. Termination of a + supervisor triggers a recursive chain of children terminations, therefore + orderly shutting down all descendant processes. The `c:stop/1` callback is + invoked only after termination of the whole supervision tree. + + Shutting down a live system cleanly can be done by calling `System.stop/1`. It + will shut down every application in the reverse order they were started. + + By default, a SIGTERM from the operating system will automatically translate to + `System.stop/0`. You can also have more explicit control over operating system + signals via the `:os.set_signal/2` function. + + ## Tooling + + The Mix build tool automates most of the application management tasks. For example, + `mix test` automatically starts your application dependencies and your application + itself before your test runs. `mix run --no-halt` boots your current project and + can be used to start a long running system. See `mix help run`. + + Developers can also use `mix release` to build **releases**. Releases are able to + package all of your source code as well as the Erlang VM into a single directory. + Releases also give you explicit control over how each application is started and in + which order. They also provide a more streamlined mechanism for starting and + stopping systems, debugging, logging, as well as system monitoring. + + Finally, Elixir provides tools such as escripts and archives, which are + different mechanisms for packaging your application. Those are typically used + when tools must be shared between developers and not as deployment options. + See `mix help archive.build` and `mix help escript.build` for more detail. + + ## Further information + + For further details on applications please check the documentation of the + [`:application` Erlang module](`:application`), and the + [Applications](https://www.erlang.org/doc/design_principles/applications.html) + section of the [OTP Design Principles User's + Guide](https://www.erlang.org/doc/design_principles/users_guide.html). + """ + + @doc """ + Called when an application is started. + + This function is called when an application is started using + `Application.start/2` (and functions on top of that, such as + `Application.ensure_started/2`). This function should start the top-level + process of the application (which should be the top supervisor of the + application's supervision tree if the application follows the OTP design + principles around supervision). + + `start_type` defines how the application is started: + + * `:normal` - used if the startup is a normal startup or if the application + is distributed and is started on the current node because of a failover + from another node and the application specification key `:start_phases` + is `:undefined`. + * `{:takeover, node}` - used if the application is distributed and is + started on the current node because of a failover on the node `node`. + * `{:failover, node}` - used if the application is distributed and is + started on the current node because of a failover on node `node`, and the + application specification key `:start_phases` is not `:undefined`. + + `start_args` are the arguments passed to the application in the `:mod` + specification key (for example, `mod: {MyApp, [:my_args]}`). + + This function should either return `{:ok, pid}` or `{:ok, pid, state}` if + startup is successful. `pid` should be the PID of the top supervisor. `state` + can be an arbitrary term, and if omitted will default to `[]`; if the + application is later stopped, `state` is passed to the `stop/1` callback (see + the documentation for the `c:stop/1` callback for more information). + + `use Application` provides no default implementation for the `start/2` + callback. + """ + @callback start(start_type, start_args :: term) :: + {:ok, pid} + | {:ok, pid, state} + | {:error, reason :: term} + + @doc """ + Called before stopping the application. + + This function is called before the top-level supervisor is terminated. It + receives the state returned by `c:start/2`, if it did, or `[]` otherwise. + The return value is later passed to `c:stop/1`. + """ + @callback prep_stop(state) :: state + + @doc """ + Called after an application has been stopped. + + This function is called after an application has been stopped, i.e., after its + supervision tree has been stopped. It should do the opposite of what the + `c:start/2` callback did, and should perform any necessary cleanup. The return + value of this callback is ignored. + + `state` is the state returned by `c:start/2`, if it did, or `[]` otherwise. + If the optional callback `c:prep_stop/1` is present, `state` is its return + value instead. + + `use Application` defines a default implementation of this function which does + nothing and just returns `:ok`. + """ + @callback stop(state) :: term - The `type` passed into `start/2` is usually `:normal` unless in a distributed - setup where applications takeover and failovers are configured. This particular - aspect of applications can be read with more detail in the OTP documentation: + @doc """ + Starts an application in synchronous phases. - * http://www.erlang.org/doc/man/application.html - * http://www.erlang.org/doc/design_principles/applications.html + This function is called after `start/2` finishes but before + `Application.start/2` returns. It will be called once for every start phase + defined in the application's (and any included applications') specification, + in the order they are listed in. + """ + @callback start_phase(phase :: term, start_type, phase_args :: term) :: + :ok | {:error, reason :: term} + + @doc """ + Callback invoked after code upgrade, if the application environment + has changed. - A developer may also implement the `stop/1` callback (automatically defined - by `use Application`) which does any application cleanup. It receives the - application state and can return any value. Notice that shutting down the - supervisor is automatically handled by the VM; + `changed` is a keyword list of keys and their changed values in the + application environment. `new` is a keyword list with all new keys + and their values. `removed` is a list with all removed keys. """ + @callback config_change(changed, new, removed) :: :ok + when changed: keyword, new: keyword, removed: [atom] + + @optional_callbacks start_phase: 3, prep_stop: 1, config_change: 3 @doc false defmacro __using__(_) do quote location: :keep do - @behaviour :application + @behaviour Application @doc false def stop(_state) do :ok end - defoverridable [stop: 1] + defoverridable Application end end + @application_keys [ + :description, + :id, + :vsn, + :modules, + :maxP, + :maxT, + :registered, + :included_applications, + :optional_applications, + :applications, + :mod, + :start_phases + ] + + application_key_specs = Enum.reduce(@application_keys, &{:|, [], [&1, &2]}) + @type app :: atom @type key :: atom + @type application_key :: unquote(application_key_specs) @type value :: term - @type start_type :: :permanent | :transient | :temporary + @type state :: term + @type start_type :: :normal | {:takeover, node} | {:failover, node} + + @typedoc """ + Specifies the type of the application: + + * `:permanent` - if `app` terminates, all other applications and the entire + node are also terminated. + + * `:transient` - if `app` terminates with `:normal` reason, it is reported + but no other applications are terminated. If a transient application + terminates abnormally, all other applications and the entire node are + also terminated. + + * `:temporary` - if `app` terminates, it is reported but no other + applications are terminated (the default). + + Note that it is always possible to stop an application explicitly by calling + `stop/1`. Regardless of the type of the application, no other applications will + be affected. + + Note also that the `:transient` type is of little practical use, since when a + supervision tree terminates, the reason is set to `:shutdown`, not `:normal`. + """ + @type restart_type :: :permanent | :transient | :temporary + + @doc """ + Returns the spec for `app`. + + The following keys are returned: + + * #{Enum.map_join(@application_keys, "\n * ", &"`#{inspect(&1)}`")} + + For a description of all fields, see [Erlang's application + specification](https://www.erlang.org/doc/man/app). + + Note the environment is not returned as it can be accessed via + `fetch_env/2`. Returns `nil` if the application is not loaded. + """ + @spec spec(app) :: [{application_key, value}] | nil + def spec(app) when is_atom(app) do + case :application.get_all_key(app) do + {:ok, info} -> :lists.keydelete(:env, 1, info) + :undefined -> nil + end + end + + @doc """ + Returns the value for `key` in `app`'s specification. + + See `spec/1` for the supported keys. If the given + specification parameter does not exist, this function + will raise. Returns `nil` if the application is not loaded. + """ + @spec spec(app, application_key) :: value | nil + def spec(app, key) when is_atom(app) and key in @application_keys do + case :application.get_key(app, key) do + {:ok, value} -> value + :undefined -> nil + end + end + + @doc """ + Gets the application for the given module. + + The application is located by analyzing the spec + of all loaded applications. Returns `nil` if + the module is not listed in any application spec. + """ + @spec get_application(module) :: app | nil + def get_application(module) when is_atom(module) do + case :application.get_application(module) do + {:ok, app} -> app + :undefined -> nil + end + end @doc """ Returns all key-value pairs for `app`. """ - @spec get_all_env(app) :: [{key,value}] - def get_all_env(app) do + @spec get_all_env(app) :: [{key, value}] + def get_all_env(app) when is_atom(app) do :application.get_all_env(app) end @doc """ - Returns the value for `key` in `app`'s environment. + Reads the application environment at compilation time. + + Similar to `get_env/3`, except it must be used to read values + at compile time. This allows Elixir to track when configuration + values change between compile time and runtime. + + The first argument is the application name. The second argument + `key_or_path` is either an atom key or a path to traverse in + search of the configuration, starting with an atom key. + + For example, imagine the following configuration: - If the specified application is not loaded, or the configuration parameter - does not exist, the function returns the `default` value. + config :my_app, :key, [foo: [bar: :baz]] + + We can access it during compile time as: + + Application.compile_env(:my_app, :key) + #=> [foo: [bar: :baz]] + + Application.compile_env(:my_app, [:key, :foo]) + #=> [bar: :baz] + + Application.compile_env(:my_app, [:key, :foo, :bar]) + #=> :baz + + A default value can also be given as third argument. If + any of the keys in the path along the way is missing, the + default value is used: + + Application.compile_env(:my_app, [:unknown, :foo, :bar], :default) + #=> :default + + Application.compile_env(:my_app, [:key, :unknown, :bar], :default) + #=> :default + + Application.compile_env(:my_app, [:key, :foo, :unknown], :default) + #=> :default + + Giving a path is useful to let Elixir know that only certain paths + in a large configuration are compile time dependent. """ - @spec get_env(app, key, value) :: value - def get_env(app, key, default \\ nil) do - case :application.get_env(app, key) do + @doc since: "1.10.0" + @spec compile_env(app, key | list, value) :: value + defmacro compile_env(app, key_or_path, default \\ nil) do + if __CALLER__.function do + raise "Application.compile_env/3 cannot be called inside functions, only in the module body" + end + + key_or_path = Macro.expand_literals(key_or_path, %{__CALLER__ | function: {:__info__, 1}}) + + quote do + Application.compile_env(__ENV__, unquote(app), unquote(key_or_path), unquote(default)) + end + end + + @doc """ + Reads the application environment at compilation time from a macro. + + Typically, developers will use `compile_env/3`. This function must + only be invoked from macros which aim to read the compilation environment + dynamically. + + It expects a `Macro.Env` as first argument, where the `Macro.Env` is + typically the `__CALLER__` in a macro. It raises if `Macro.Env` comes + from a function. + """ + @doc since: "1.14.0" + @spec compile_env(Macro.Env.t(), app, key | list, value) :: value + def compile_env(%Macro.Env{} = env, app, key_or_path, default) do + case fetch_compile_env(app, key_or_path, env) do {:ok, value} -> value - :undefined -> default + :error -> default + end + end + + @doc """ + Reads the application environment at compilation time or raises. + + This is the same as `compile_env/3` but it raises an + `ArgumentError` if the configuration is not available. + """ + @doc since: "1.10.0" + @spec compile_env!(app, key | list) :: value + defmacro compile_env!(app, key_or_path) do + if __CALLER__.function do + raise "Application.compile_env!/2 cannot be called inside functions, only in the module body" end + + key_or_path = Macro.expand_literals(key_or_path, %{__CALLER__ | function: {:__info__, 1}}) + + quote do + Application.compile_env!(__ENV__, unquote(app), unquote(key_or_path)) + end + end + + @doc """ + Reads the application environment at compilation time from a macro + or raises. + + Typically, developers will use `compile_env!/2`. This function must + only be invoked from macros which aim to read the compilation environment + dynamically. + + It expects a `Macro.Env` as first argument, where the `Macro.Env` is + typically the `__CALLER__` in a macro. It raises if `Macro.Env` comes + from a function. + """ + @doc since: "1.14.0" + @spec compile_env!(Macro.Env.t(), app, key | list) :: value + def compile_env!(%Macro.Env{} = env, app, key_or_path) do + case fetch_compile_env(app, key_or_path, env) do + {:ok, value} -> + value + + :error -> + raise ArgumentError, + "could not fetch application environment #{inspect(key_or_path)} for application " <> + "#{inspect(app)} #{fetch_env_failed_reason(app, key_or_path)}" + end + end + + defp fetch_compile_env(app, key, env) when is_atom(key) do + fetch_compile_env(app, key, [], env) + end + + defp fetch_compile_env(app, [key | paths], env) when is_atom(key), + do: fetch_compile_env(app, key, paths, env) + + defp fetch_compile_env(app, key, path, env) do + return = traverse_env(fetch_env(app, key), path) + + for tracer <- env.tracers do + tracer.trace({:compile_env, app, [key | path], return}, env) + end + + return + end + + defp traverse_env(return, []), do: return + defp traverse_env(:error, _paths), do: :error + defp traverse_env({:ok, value}, [key | keys]), do: traverse_env(Access.fetch(value, key), keys) + + @doc """ + Returns the value for `key` in `app`'s environment. + + If the configuration parameter does not exist, the function returns the + `default` value. + + > #### Warning {: .warning} + > + > You must use this function to read only your own application + > environment. Do not read the environment of other applications. + + ## Examples + + `get_env/3` is commonly used to read the configuration of your OTP applications. + Since Mix configurations are commonly used to configure applications, we will use + this as a point of illustration. + + Consider a new application `:my_app`. `:my_app` contains a database engine which + supports a pool of databases. The database engine needs to know the configuration for + each of those databases, and that configuration is supplied by key-value pairs in + environment of `:my_app`. + + config :my_app, Databases.RepoOne, + # A database configuration + ip: "localhost", + port: 5433 + + config :my_app, Databases.RepoTwo, + # Another database configuration (for the same OTP app) + ip: "localhost", + port: 20_717 + + config :my_app, my_app_databases: [Databases.RepoOne, Databases.RepoTwo] + + Our database engine used by `:my_app` needs to know what databases exist, and + what the database configurations are. The database engine can make a call to + `Application.get_env(:my_app, :my_app_databases, [])` to retrieve the list of + databases (specified by module names). + + The engine can then traverse each repository in the list and call + `Application.get_env(:my_app, Databases.RepoOne)` and so forth to retrieve the + configuration of each one. In this case, each configuration will be a keyword + list, so you can use the functions in the `Keyword` module or even the `Access` + module to traverse it, for example: + + config = Application.get_env(:my_app, Databases.RepoOne) + config[:ip] + + """ + @spec get_env(app, key, value) :: value + def get_env(app, key, default \\ nil) when is_atom(app) do + maybe_warn_on_app_env_key(app, key) + :application.get_env(app, key, default) end @doc """ Returns the value for `key` in `app`'s environment in a tuple. - If the specified application is not loaded, or the configuration parameter - does not exist, the function returns `:error`. + If the configuration parameter does not exist, the function returns `:error`. + + > #### Warning {: .warning} + > + > You must use this function to read only your own application + > environment. Do not read the environment of other applications. + + > #### Application environment in info + > + > If you are writing a library to be used by other developers, + > it is generally recommended to avoid the application environment, as the + > application environment is effectively a global storage. For more information, + > read our [library guidelines](library-guidelines.md). """ @spec fetch_env(app, key) :: {:ok, value} | :error - def fetch_env(app, key) do + def fetch_env(app, key) when is_atom(app) do + maybe_warn_on_app_env_key(app, key) + case :application.get_env(app, key) do {:ok, value} -> {:ok, value} :undefined -> :error end end + @doc """ + Returns the value for `key` in `app`'s environment. + + If the configuration parameter does not exist, raises `ArgumentError`. + + > #### Warning {: .warning} + > + > You must use this function to read only your own application + > environment. Do not read the environment of other applications. + + > #### Application environment in info + > + > If you are writing a library to be used by other developers, + > it is generally recommended to avoid the application environment, as the + > application environment is effectively a global storage. For more information, + > read our [library guidelines](library-guidelines.md). + """ + @spec fetch_env!(app, key) :: value + def fetch_env!(app, key) when is_atom(app) do + case fetch_env(app, key) do + {:ok, value} -> + value + + :error -> + raise ArgumentError, + "could not fetch application environment #{inspect(key)} for application " <> + "#{inspect(app)} #{fetch_env_failed_reason(app, key)}" + end + end + + defp fetch_env_failed_reason(app, key) do + vsn = :application.get_key(app, :vsn) + + case vsn do + {:ok, _} -> + "because configuration at #{inspect(key)} was not set" + + :undefined -> + "because the application was not loaded nor configured" + end + end + @doc """ Puts the `value` in `key` for the given `app`. + > #### Compile environment {: .warning} + > + > Do not use this function to change environment variables read + > via `Application.compile_env/2`. The compile environment must + > be exclusively set before compilation, in your config files. + ## Options - * `:timeout` - the timeout for the change (defaults to 5000ms) + * `:timeout` - the timeout for the change (defaults to `5_000` milliseconds) * `:persistent` - persists the given value on application load and reloads If `put_env/4` is called before the application is loaded, the application environment values specified in the `.app` file will override the ones previously set. - The persistent option can be set to true when there is a need to guarantee + The `:persistent` option can be set to `true` when there is a need to guarantee parameters set with this function will not be overridden by the ones defined in the application resource file on load. This means persistent values will stick after the application is loaded and also on application reload. """ - @spec put_env(app, key, value, [timeout: timeout, persistent: boolean]) :: :ok - def put_env(app, key, value, opts \\ []) do + @spec put_env(app, key, value, timeout: timeout, persistent: boolean) :: :ok + def put_env(app, key, value, opts \\ []) when is_atom(app) and is_list(opts) do + maybe_warn_on_app_env_key(app, key) :application.set_env(app, key, value, opts) end + @doc """ + Puts the environment for multiple applications at the same time. + + The given config should not: + + * have the same application listed more than once + * have the same key inside the same application listed more than once + + If those conditions are not met, this function will raise. + + This function receives the same options as `put_env/4`. Returns `:ok`. + + ## Examples + + Application.put_all_env( + my_app: [ + key: :value, + another_key: :another_value + ], + another_app: [ + key: :value + ] + ) + + """ + @doc since: "1.9.0" + @spec put_all_env([{app, [{key, value}]}], timeout: timeout, persistent: boolean) :: :ok + def put_all_env(config, opts \\ []) when is_list(config) and is_list(opts) do + :application.set_env(config, opts) + end + @doc """ Deletes the `key` from the given `app` environment. - See `put_env/4` for a description of the options. + It receives the same options as `put_env/4`. Returns `:ok`. """ - @spec delete_env(app, key, [timeout: timeout, persistent: boolean]) :: :ok - def delete_env(app, key, opts \\ []) do + @spec delete_env(app, key, timeout: timeout, persistent: boolean) :: :ok + def delete_env(app, key, opts \\ []) when is_atom(app) and is_list(opts) do + maybe_warn_on_app_env_key(app, key) :application.unset_env(app, key, opts) end + defp maybe_warn_on_app_env_key(_app, key) when is_atom(key), + do: :ok + + # TODO: Remove this deprecation warning on 2.0+ and allow list lookups as in compile_env. + defp maybe_warn_on_app_env_key(app, key) do + message = fn -> + "passing non-atom as application env key is deprecated, got: #{inspect(key)}" + end + + IO.warn_once({Application, :key, app, key}, message, _stacktrace_drop_levels = 2) + end + @doc """ - Ensures the given `app` is started. + Ensures the given `app` is started with `t:restart_type/0`. Same as `start/2` but returns `:ok` if the application was already - started. This is useful in scripts and in test setup, where test - applications need to be explicitly started: + started. + """ + @spec ensure_started(app, restart_type()) :: :ok | {:error, term} + def ensure_started(app, type \\ :temporary) when is_atom(app) and is_atom(type) do + :application.ensure_started(app, type) + end - :ok = Application.ensure_started(:my_test_dep) + @doc """ + Ensures the given `app` is loaded. + Same as `load/1` but returns `:ok` if the application was already + loaded. """ - @spec ensure_started(app, start_type) :: :ok | {:error, term} - def ensure_started(app, type \\ :temporary) when is_atom(app) do - :application.ensure_started(app, type) + @doc since: "1.10.0" + @spec ensure_loaded(app) :: :ok | {:error, term} + def ensure_loaded(app) when is_atom(app) do + case :application.load(app) do + :ok -> :ok + {:error, {:already_loaded, ^app}} -> :ok + {:error, _} = error -> error + end end @doc """ - Ensures the given `app` and its applications are started. + Ensures the given `app` or `apps` and their child applications are started. + + The second argument is either the `t:restart_type/0` (for consistency with + `start/2`) or a keyword list. + + ## Options + + * `:type` - if the application should be started `:temporary` (default), + `:permanent`, or `:transient`. See `t:restart_type/0` for more information. + + * `:mode` - (since v1.15.0) if the applications should be started serially + (`:serial`, default) or concurrently (`:concurrent`). - Same as `start/2` but also starts the applications listed under - `:applications` in the `.app` file in case they were not previously - started. """ - @spec ensure_all_started(app, start_type) :: {:ok, [app]} | {:error, term} - def ensure_all_started(app, type \\ :temporary) when is_atom(app) do - :application.ensure_all_started(app, type) + @spec ensure_all_started(app | [app], type: restart_type(), mode: :serial | :concurrent) :: + {:ok, [app]} | {:error, term} + @spec ensure_all_started(app | [app], restart_type()) :: + {:ok, [app]} | {:error, term} + def ensure_all_started(app_or_apps, type_or_opts \\ []) + + def ensure_all_started(app_or_apps, type) when is_atom(type) do + ensure_all_started(app_or_apps, type: type) + end + + def ensure_all_started(app, opts) when is_atom(app) and is_list(opts) do + ensure_all_started([app], opts) + end + + @compile {:no_warn_undefined, {:application, :ensure_all_started, 3}} + + def ensure_all_started(apps, opts) when is_list(apps) and is_list(opts) do + opts = Keyword.validate!(opts, type: :temporary, mode: :serial) + :application.ensure_all_started(apps, opts[:type], opts[:mode]) end @doc """ - Starts the given `app`. + Starts the given `app` with `t:restart_type/0`. If the `app` is not loaded, the application will first be loaded using `load/1`. Any included application, defined in the `:included_applications` key of the @@ -217,31 +947,11 @@ defmodule Application do started before this application is. If not, `{:error, {:not_started, app}}` is returned, where `app` is the name of the missing application. - In case you want to automatically load **and start** all of `app`'s dependencies, + In case you want to automatically load **and start** all of `app`'s dependencies, see `ensure_all_started/2`. - - The `type` argument specifies the type of the application: - - * `:permanent` - if `app` terminates, all other applications and the entire - node are also terminated. - - * `:transient` - if `app` terminates with `:normal` reason, it is reported - but no other applications are terminated. If a transient application - terminates abnormally, all other applications and the entire node are - also terminated. - - * `:temporary` - if `app` terminates, it is reported but no other - applications are terminated (the default). - - Note that it is always possible to stop an application explicitly by calling - `stop/1`. Regardless of the type of the application, no other applications will - be affected. - - Note also that the `:transient` type is of little practical use, since when a - supervision tree terminates, the reason is set to `:shutdown`, not `:normal`. """ - @spec start(app, start_type) :: :ok | {:error, term} - def start(app, type \\ :temporary) when is_atom(app) do + @spec start(app, restart_type()) :: :ok | {:error, term} + def start(app, type \\ :temporary) when is_atom(app) and is_atom(type) do :application.start(app, type) end @@ -251,7 +961,7 @@ defmodule Application do When stopped, the application is still loaded. """ @spec stop(app) :: :ok | {:error, term} - def stop(app) do + def stop(app) when is_atom(app) do :application.stop(app) end @@ -302,35 +1012,74 @@ defmodule Application do #=> "bar-123" For more information on code paths, check the `Code` module in - Elixir and also Erlang's `:code` module. + Elixir and also Erlang's [`:code` module](`:code`). """ - @spec app_dir(app) :: String.t + @spec app_dir(app) :: String.t() def app_dir(app) when is_atom(app) do case :code.lib_dir(app) do lib when is_list(lib) -> IO.chardata_to_string(lib) - {:error, :bad_name} -> raise ArgumentError, "unknown application: #{inspect app}" + {:error, :bad_name} -> raise ArgumentError, "unknown application: #{inspect(app)}" end end @doc """ Returns the given path inside `app_dir/1`. + + If `path` is a string, then it will be used as the path inside `app_dir/1`. If + `path` is a list of strings, it will be joined (see `Path.join/1`) and the result + will be used as the path inside `app_dir/1`. + + ## Examples + + File.mkdir_p!("foo/ebin") + Code.prepend_path("foo/ebin") + + Application.app_dir(:foo, "my_path") + #=> "foo/my_path" + + Application.app_dir(:foo, ["my", "nested", "path"]) + #=> "foo/my/nested/path" + """ - @spec app_dir(app, String.t) :: String.t - def app_dir(app, path) when is_binary(path) do + @spec app_dir(app, String.t() | [String.t()]) :: String.t() + def app_dir(app, path) + + def app_dir(app, path) when is_atom(app) and is_binary(path) do Path.join(app_dir(app), path) end + def app_dir(app, path) when is_atom(app) and is_list(path) do + Path.join([app_dir(app) | path]) + end + + @doc """ + Returns a list with information about the applications which are currently running. + """ + @spec started_applications(timeout) :: [{app, description :: charlist(), vsn :: charlist()}] + def started_applications(timeout \\ 5000) + when timeout == :infinity or (is_integer(timeout) and timeout >= 0) do + :application.which_applications(timeout) + end + + @doc """ + Returns a list with information about the applications which have been loaded. + """ + @spec loaded_applications :: [{app, description :: charlist(), vsn :: charlist()}] + def loaded_applications do + :application.loaded_applications() + end + @doc """ Formats the error reason returned by `start/2`, - `ensure_started/2, `stop/1`, `load/1` and `unload/1`, + `ensure_started/2`, `stop/1`, `load/1` and `unload/1`, returns a string. """ - @spec format_error(any) :: String.t + @spec format_error(any) :: String.t() def format_error(reason) do try do - impl_format_error(reason) + do_format_error(reason) catch - # A user could create an error that looks like a builtin one + # A user could create an error that looks like a built-in one # causing an error. :error, _ -> inspect(reason) @@ -338,68 +1087,67 @@ defmodule Application do end # exit(:normal) call is special cased, undo the special case. - defp impl_format_error({{:EXIT, :normal}, {mod, :start, args}}) do + defp do_format_error({{:EXIT, :normal}, {mod, :start, args}}) do Exception.format_exit({:normal, {mod, :start, args}}) end # {:error, reason} return value - defp impl_format_error({reason, {mod, :start, args}}) do - Exception.format_mfa(mod, :start, args) <> " returned an error: " <> - Exception.format_exit(reason) + defp do_format_error({reason, {mod, :start, args}}) do + Exception.format_mfa(mod, :start, args) <> + " returned an error: " <> Exception.format_exit(reason) end # error or exit(reason) call, use exit reason as reason. - defp impl_format_error({:bad_return, {{mod, :start, args}, {:EXIT, reason}}}) do + defp do_format_error({:bad_return, {{mod, :start, args}, {:EXIT, reason}}}) do Exception.format_exit({reason, {mod, :start, args}}) end # bad return value - defp impl_format_error({:bad_return, {{mod, :start, args}, return}}) do - Exception.format_mfa(mod, :start, args) <> - " returned a bad value: " <> inspect(return) + defp do_format_error({:bad_return, {{mod, :start, args}, return}}) do + Exception.format_mfa(mod, :start, args) <> " returned a bad value: " <> inspect(return) end - defp impl_format_error({:already_started, app}) when is_atom(app) do + defp do_format_error({:already_started, app}) when is_atom(app) do "already started application #{app}" end - defp impl_format_error({:not_started, app}) when is_atom(app) do + defp do_format_error({:not_started, app}) when is_atom(app) do "not started application #{app}" end - defp impl_format_error({:bad_application, app}) do + defp do_format_error({:bad_application, app}) do "bad application: #{inspect(app)}" end - defp impl_format_error({:already_loaded, app}) when is_atom(app) do + defp do_format_error({:already_loaded, app}) when is_atom(app) do "already loaded application #{app}" end - defp impl_format_error({:not_loaded, app}) when is_atom(app) do + defp do_format_error({:not_loaded, app}) when is_atom(app) do "not loaded application #{app}" end - defp impl_format_error({:invalid_restart_type, restart}) do + defp do_format_error({:invalid_restart_type, restart}) do "invalid application restart type: #{inspect(restart)}" end - defp impl_format_error({:invalid_name, name}) do + defp do_format_error({:invalid_name, name}) do "invalid application name: #{inspect(name)}" end - defp impl_format_error({:invalid_options, opts}) do + defp do_format_error({:invalid_options, opts}) do "invalid application options: #{inspect(opts)}" end - defp impl_format_error({:badstartspec, spec}) do + defp do_format_error({:badstartspec, spec}) do "bad application start specs: #{inspect(spec)}" end - defp impl_format_error({'no such file or directory', file}) do + defp do_format_error({~c"no such file or directory", file}) do "could not find application file: #{file}" end - defp impl_format_error(reason) do + defp do_format_error(reason) do Exception.format_exit(reason) end end diff --git a/lib/elixir/lib/atom.ex b/lib/elixir/lib/atom.ex index 36e62f9f416..1e11f6b9799 100644 --- a/lib/elixir/lib/atom.ex +++ b/lib/elixir/lib/atom.ex @@ -1,25 +1,86 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + defmodule Atom do - @doc """ - Convenience functions for working with atoms. + @moduledoc """ + Atoms are constants whose values are their own name. + + They are often useful to enumerate over distinct values, such as: + + iex> :apple + :apple + iex> :orange + :orange + iex> :watermelon + :watermelon + + Atoms are equal if their names are equal. + + iex> :apple == :apple + true + iex> :apple == :orange + false + + Often they are used to express the state of an operation, by using + values such as `:ok` and `:error`. + + The booleans `true` and `false` are also atoms: + + iex> true == :true + true + iex> is_atom(false) + true + iex> is_boolean(:false) + true + + Elixir allows you to skip the leading `:` for the atoms `false`, `true`, + and `nil`. + + Atoms must be composed of Unicode characters such as letters, numbers, + underscore, and `@`. If the keyword has a character that does not + belong to the category above, such as spaces, you can wrap it in + quotes: + + iex> :"this is an atom with spaces" + :"this is an atom with spaces" + """ @doc """ - Converts an atom to string. + Converts an atom to a string. Inlined by the compiler. + + ## Examples + + iex> Atom.to_string(:foo) + "foo" + """ - @spec to_string(atom) :: String.t + @spec to_string(atom) :: String.t() def to_string(atom) do - :erlang.atom_to_binary(atom, :utf8) + :erlang.atom_to_binary(atom) end @doc """ - Converts an atom to a char list. + Converts an atom to a charlist. Inlined by the compiler. + + ## Examples + + iex> Atom.to_charlist(:"An atom") + ~c"An atom" + """ - @spec to_char_list(atom) :: char_list - def to_char_list(atom) do + @spec to_charlist(atom) :: charlist + def to_charlist(atom) do :erlang.atom_to_list(atom) end + + @doc false + @deprecated "Use Atom.to_charlist/1 instead" + @spec to_char_list(atom) :: charlist + def to_char_list(atom), do: Atom.to_charlist(atom) end diff --git a/lib/elixir/lib/base.ex b/lib/elixir/lib/base.ex index ff01c991e73..d2e3da7ca77 100644 --- a/lib/elixir/lib/base.ex +++ b/lib/elixir/lib/base.ex @@ -1,151 +1,173 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + defmodule Base do import Bitwise @moduledoc """ This module provides data encoding and decoding functions - according to [RFC 4648](http://tools.ietf.org/html/rfc4648). + according to [RFC 4648](https://tools.ietf.org/html/rfc4648). This document defines the commonly used base 16, base 32, and base 64 encoding schemes. ## Base 16 alphabet - | Value | Encoding | Value | Encoding | Value | Encoding | Value | Encoding | - |------:|---------:|------:|---------:|------:|---------:|------:|---------:| - | 0| 0| 4| 4| 8| 8| 12| C| - | 1| 1| 5| 5| 9| 9| 13| D| - | 2| 2| 6| 6| 10| A| 14| E| - | 3| 3| 7| 7| 11| B| 15| F| + | Value | Encoding | Value | Encoding | Value | Encoding | Value | Encoding | + |------:|:---------|------:|:---------|------:|:---------|------:|:---------| + | 0 | 0 | 4 | 4 | 8 | 8 | 12 | C | + | 1 | 1 | 5 | 5 | 9 | 9 | 13 | D | + | 2 | 2 | 6 | 6 | 10 | A | 14 | E | + | 3 | 3 | 7 | 7 | 11 | B | 15 | F | ## Base 32 alphabet - | Value | Encoding | Value | Encoding | Value | Encoding | Value | Encoding | - |------:|---------:|------:|---------:|------:|---------:|------:|---------:| - | 0| A| 9| J| 18| S| 27| 3| - | 1| B| 10| K| 19| T| 28| 4| - | 2| C| 11| L| 20| U| 29| 5| - | 3| D| 12| M| 21| V| 30| 6| - | 4| E| 13| N| 22| W| 31| 7| - | 5| F| 14| O| 23| X| | | - | 6| G| 15| P| 24| Y| (pad)| =| - | 7| H| 16| Q| 25| Z| | | - | 8| I| 17| R| 26| 2| | | + | Value | Encoding | Value | Encoding | Value | Encoding | Value | Encoding | + |------:|:---------|------:|:---------|------:|:---------|------:|:---------| + | 0 | A | 9 | J | 18 | S | 27 | 3 | + | 1 | B | 10 | K | 19 | T | 28 | 4 | + | 2 | C | 11 | L | 20 | U | 29 | 5 | + | 3 | D | 12 | M | 21 | V | 30 | 6 | + | 4 | E | 13 | N | 22 | W | 31 | 7 | + | 5 | F | 14 | O | 23 | X | | | + | 6 | G | 15 | P | 24 | Y | (pad) | = | + | 7 | H | 16 | Q | 25 | Z | | | + | 8 | I | 17 | R | 26 | 2 | | | ## Base 32 (extended hex) alphabet - | Value | Encoding | Value | Encoding | Value | Encoding | Value | Encoding | - |------:|---------:|------:|---------:|------:|---------:|------:|---------:| - | 0| 0| 9| 9| 18| I| 27| R| - | 1| 1| 10| A| 19| J| 28| S| - | 2| 2| 11| B| 20| K| 29| T| - | 3| 3| 12| C| 21| L| 30| U| - | 4| 4| 13| D| 22| M| 31| V| - | 5| 5| 14| E| 23| N| | | - | 6| 6| 15| F| 24| O| (pad)| =| - | 7| 7| 16| G| 25| P| | | - | 8| 8| 17| H| 26| Q| | | + | Value | Encoding | Value | Encoding | Value | Encoding | Value | Encoding | + |------:|:---------|------:|:---------|------:|:---------|------:|:---------| + | 0 | 0 | 9 | 9 | 18 | I | 27 | R | + | 1 | 1 | 10 | A | 19 | J | 28 | S | + | 2 | 2 | 11 | B | 20 | K | 29 | T | + | 3 | 3 | 12 | C | 21 | L | 30 | U | + | 4 | 4 | 13 | D | 22 | M | 31 | V | + | 5 | 5 | 14 | E | 23 | N | | | + | 6 | 6 | 15 | F | 24 | O | (pad) | = | + | 7 | 7 | 16 | G | 25 | P | | | + | 8 | 8 | 17 | H | 26 | Q | | | ## Base 64 alphabet - | Value | Encoding | Value | Encoding | Value | Encoding | Value | Encoding | - |------:|---------:|------:|---------:|------:|---------:|------:|---------:| - | 0| A| 17| R| 34| i| 51| z| - | 1| B| 18| S| 35| j| 52| 0| - | 2| C| 19| T| 36| k| 53| 1| - | 3| D| 20| U| 37| l| 54| 2| - | 4| E| 21| V| 38| m| 55| 3| - | 5| F| 22| W| 39| n| 56| 4| - | 6| G| 23| X| 40| o| 57| 5| - | 7| H| 24| Y| 41| p| 58| 6| - | 8| I| 25| Z| 42| q| 59| 7| - | 9| J| 26| a| 43| r| 60| 8| - | 10| K| 27| b| 44| s| 61| 9| - | 11| L| 28| c| 45| t| 62| +| - | 12| M| 29| d| 46| u| 63| /| - | 13| N| 30| e| 47| v| | | - | 14| O| 31| f| 48| w| (pad)| =| - | 15| P| 32| g| 49| x| | | - | 16| Q| 33| h| 50| y| | | + | Value | Encoding | Value | Encoding | Value | Encoding | Value | Encoding | + |------:|:----------|------:|:---------|------:|:---------|------:|:---------| + | 0 | A | 17 | R | 34 | i | 51 | z | + | 1 | B | 18 | S | 35 | j | 52 | 0 | + | 2 | C | 19 | T | 36 | k | 53 | 1 | + | 3 | D | 20 | U | 37 | l | 54 | 2 | + | 4 | E | 21 | V | 38 | m | 55 | 3 | + | 5 | F | 22 | W | 39 | n | 56 | 4 | + | 6 | G | 23 | X | 40 | o | 57 | 5 | + | 7 | H | 24 | Y | 41 | p | 58 | 6 | + | 8 | I | 25 | Z | 42 | q | 59 | 7 | + | 9 | J | 26 | a | 43 | r | 60 | 8 | + | 10 | K | 27 | b | 44 | s | 61 | 9 | + | 11 | L | 28 | c | 45 | t | 62 | + | + | 12 | M | 29 | d | 46 | u | 63 | / | + | 13 | N | 30 | e | 47 | v | | | + | 14 | O | 31 | f | 48 | w | (pad) | = | + | 15 | P | 32 | g | 49 | x | | | + | 16 | Q | 33 | h | 50 | y | | | ## Base 64 (URL and filename safe) alphabet - | Value | Encoding | Value | Encoding | Value | Encoding | Value | Encoding | - |------:|---------:|------:|---------:|------:|---------:|------:|---------:| - | 0| A| 17| R| 34| i| 51| z| - | 1| B| 18| S| 35| j| 52| 0| - | 2| C| 19| T| 36| k| 53| 1| - | 3| D| 20| U| 37| l| 54| 2| - | 4| E| 21| V| 38| m| 55| 3| - | 5| F| 22| W| 39| n| 56| 4| - | 6| G| 23| X| 40| o| 57| 5| - | 7| H| 24| Y| 41| p| 58| 6| - | 8| I| 25| Z| 42| q| 59| 7| - | 9| J| 26| a| 43| r| 60| 8| - | 10| K| 27| b| 44| s| 61| 9| - | 11| L| 28| c| 45| t| 62| -| - | 12| M| 29| d| 46| u| 63| _| - | 13| N| 30| e| 47| v| | | - | 14| O| 31| f| 48| w| (pad)| =| - | 15| P| 32| g| 49| x| | | - | 16| Q| 33| h| 50| y| | | + | Value | Encoding | Value | Encoding | Value | Encoding | Value | Encoding | + |------:|:---------|------:|:---------|------:|:---------|------:|:---------| + | 0 | A | 17 | R | 34 | i | 51 | z | + | 1 | B | 18 | S | 35 | j | 52 | 0 | + | 2 | C | 19 | T | 36 | k | 53 | 1 | + | 3 | D | 20 | U | 37 | l | 54 | 2 | + | 4 | E | 21 | V | 38 | m | 55 | 3 | + | 5 | F | 22 | W | 39 | n | 56 | 4 | + | 6 | G | 23 | X | 40 | o | 57 | 5 | + | 7 | H | 24 | Y | 41 | p | 58 | 6 | + | 8 | I | 25 | Z | 42 | q | 59 | 7 | + | 9 | J | 26 | a | 43 | r | 60 | 8 | + | 10 | K | 27 | b | 44 | s | 61 | 9 | + | 11 | L | 28 | c | 45 | t | 62 | - | + | 12 | M | 29 | d | 46 | u | 63 | _ | + | 13 | N | 30 | e | 47 | v | | | + | 14 | O | 31 | f | 48 | w | (pad) | = | + | 15 | P | 32 | g | 49 | x | | | + | 16 | Q | 33 | h | 50 | y | | | """ - b16_alphabet = Enum.with_index '0123456789ABCDEF' - b64_alphabet = Enum.with_index 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/' - b64url_alphabet = Enum.with_index 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_' - b32_alphabet = Enum.with_index 'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567' - b32hex_alphabet = Enum.with_index '0123456789ABCDEFGHIJKLMNOPQRSTUV' - - Enum.each [ {:enc16, :dec16, b16_alphabet}, - {:enc64, :dec64, b64_alphabet}, - {:enc32, :dec32, b32_alphabet}, - {:enc64url, :dec64url, b64url_alphabet}, - {:enc32hex, :dec32hex, b32hex_alphabet} ], fn({enc, dec, alphabet}) -> - for {encoding, value} <- alphabet do - defp unquote(enc)(unquote(value)), do: unquote(encoding) - defp unquote(dec)(unquote(encoding)), do: unquote(value) - end - defp unquote(dec)(c) do - raise ArgumentError, "non-alphabet digit found: #{<>}" - end - end - - defp encode_case(:upper, func), - do: func - defp encode_case(:lower, func), - do: &to_lower(func.(&1)) - - defp decode_case(:upper, func), - do: func - defp decode_case(:lower, func), - do: &func.(from_lower(&1)) - defp decode_case(:mixed, func), - do: &func.(from_mixed(&1)) - - defp to_lower(char) when char in ?A..?Z, - do: char + (?a - ?A) - defp to_lower(char), - do: char - - defp from_lower(char) when char in ?a..?z, - do: char - (?a - ?A) - defp from_lower(char) when not char in ?A..?Z, - do: char - defp from_lower(char), - do: raise(ArgumentError, "non-alphabet digit found: #{<>}") - - defp from_mixed(char) when char in ?a..?z, - do: char - (?a - ?A) - defp from_mixed(char), - do: char + @type encode_case :: :upper | :lower + @type decode_case :: :upper | :lower | :mixed + + b16_alphabet = ~c"0123456789ABCDEF" + b64_alphabet = ~c"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" + b64url_alphabet = ~c"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" + b32_alphabet = ~c"ABCDEFGHIJKLMNOPQRSTUVWXYZ234567" + b32hex_alphabet = ~c"0123456789ABCDEFGHIJKLMNOPQRSTUV" + + to_lower_enc = &Enum.map(&1, fn c -> if c in ?A..?Z, do: c - ?A + ?a, else: c end) + + to_mixed_dec = + &Enum.flat_map(&1, fn {encoding, value} = pair -> + if encoding in ?A..?Z do + [pair, {encoding - ?A + ?a, value}] + else + [pair] + end + end) + + to_lower_dec = + &Enum.map(&1, fn {encoding, value} = pair -> + if encoding in ?A..?Z do + {encoding - ?A + ?a, value} + else + pair + end + end) + + to_encode_list = fn alphabet -> + for e1 <- alphabet, e2 <- alphabet, do: bsl(e1, 8) + e2 + end + + to_decode_list = fn alphabet -> + alphabet = Enum.sort(alphabet) + map = Map.new(alphabet) + {min, _} = List.first(alphabet) + {max, _} = List.last(alphabet) + {min, Enum.map(min..max, &map[&1])} + end + + defp bad_character!(byte) do + raise ArgumentError, + "non-alphabet character found: #{inspect(<>, binaries: :as_strings)} (byte #{byte})" + end + + defp maybe_pad(acc, false, _count), do: acc + defp maybe_pad(acc, true, 6), do: acc <> "======" + defp maybe_pad(acc, true, 4), do: acc <> "====" + defp maybe_pad(acc, true, 3), do: acc <> "===" + defp maybe_pad(acc, true, 2), do: acc <> "==" + defp maybe_pad(acc, true, 1), do: acc <> "=" + + defp remove_ignored(string, nil), do: string + + defp remove_ignored(string, :whitespace) do + for <>, char not in ~c"\s\t\r\n", into: <<>>, do: <> + end @doc """ Encodes a binary string into a base 16 encoded string. - Accepts an atom `:upper` (default) for encoding to upper case characters or - `:lower` for lower case characters. + ## Options + + The accepted options are: + + * `:case` - specifies the character case to use when encoding + + The values for `:case` can be: + + * `:upper` - uses upper case characters (default) + * `:lower` - uses lower case characters ## Examples @@ -156,20 +178,80 @@ defmodule Base do "666f6f626172" """ - @spec encode16(binary) :: binary - @spec encode16(binary, Keyword.t) :: binary + @spec encode16(binary, case: encode_case) :: binary def encode16(data, opts \\ []) when is_binary(data) do - case = Keyword.get(opts, :case, :upper) - do_encode16(data, encode_case(case, &enc16/1)) + case Keyword.get(opts, :case, :upper) do + :upper -> encode16upper(data, "") + :lower -> encode16lower(data, "") + end end + for {base, alphabet} <- [upper: b16_alphabet, lower: to_lower_enc.(b16_alphabet)] do + name = :"encode16#{base}" + encoded = to_encode_list.(alphabet) + + @compile {:inline, [{name, 1}]} + defp unquote(name)(byte) do + elem({unquote_splicing(encoded)}, byte) + end + + defp unquote(name)(<>, acc) do + unquote(name)( + rest, + << + acc::binary, + unquote(name)(c1)::16, + unquote(name)(c2)::16, + unquote(name)(c3)::16, + unquote(name)(c4)::16, + unquote(name)(c5)::16, + unquote(name)(c6)::16, + unquote(name)(c7)::16, + unquote(name)(c8)::16 + >> + ) + end + + defp unquote(name)(<>, acc) do + unquote(name)( + rest, + << + acc::binary, + unquote(name)(c1)::16, + unquote(name)(c2)::16, + unquote(name)(c3)::16, + unquote(name)(c4)::16 + >> + ) + end + + defp unquote(name)(<>, acc) do + unquote(name)(rest, <>) + end + + defp unquote(name)(<>, acc) do + unquote(name)(rest, <>) + end + + defp unquote(name)(<<>>, acc) do + acc + end + end @doc """ Decodes a base 16 encoded string into a binary string. - Accepts an atom `:upper` (default) for decoding from upper case characters or - `:lower` for lower case characters. `:mixed` can be given for mixed case - characters. + ## Options + + The accepted options are: + + * `:case` - specifies the character case to accept when decoding + + The values for `:case` can be: + + * `:upper` - only allows upper case characters (default) + * `:lower` - only allows lower case characters + * `:mixed` - allows mixed case characters ## Examples @@ -183,11 +265,9 @@ defmodule Base do {:ok, "foobar"} """ - @spec decode16(binary) :: {:ok, binary} | :error - @spec decode16(binary, Keyword.t) :: {:ok, binary} | :error - def decode16(string, opts \\ []) when is_binary(string) do - case = Keyword.get(opts, :case, :upper) - {:ok, do_decode16(string, decode_case(case, &dec16/1))} + @spec decode16(binary, case: decode_case) :: {:ok, binary} | :error + def decode16(string, opts \\ []) do + {:ok, decode16!(string, opts)} rescue ArgumentError -> :error end @@ -195,9 +275,17 @@ defmodule Base do @doc """ Decodes a base 16 encoded string into a binary string. - Accepts an atom `:upper` (default) for decoding from upper case characters or - `:lower` for lower case characters. `:mixed` can be given for mixed case - characters. + ## Options + + The accepted options are: + + * `:case` - specifies the character case to accept when decoding + + The values for `:case` can be: + + * `:upper` - only allows upper case characters (default) + * `:lower` - only allows lower case characters + * `:mixed` - allows mixed case characters An `ArgumentError` exception is raised if the padding is incorrect or a non-alphabet character is present in the string. @@ -214,39 +302,294 @@ defmodule Base do "foobar" """ - @spec decode16!(binary) :: binary - @spec decode16!(binary, Keyword.t) :: binary - def decode16!(string, opts \\ []) when is_binary(string) do - case = Keyword.get(opts, :case, :upper) - do_decode16(string, decode_case(case, &dec16/1)) + @spec decode16!(binary, case: decode_case) :: binary + def decode16!(string, opts \\ []) + + def decode16!(string, opts) when is_binary(string) and rem(byte_size(string), 2) == 0 do + case Keyword.get(opts, :case, :upper) do + :upper -> decode16upper!(string, "") + :lower -> decode16lower!(string, "") + :mixed -> decode16mixed!(string, "") + end + end + + def decode16!(string, _opts) when is_binary(string) do + raise ArgumentError, + "string given to decode has wrong length. An even number of bytes was expected, got: #{byte_size(string)}. " <> + "Double check your string for unwanted characters or pad it accordingly" + end + + @doc """ + Checks if a string is a valid base 16 encoded string. + + > #### When to use this {: .tip} + > + > Use this function when you just need to *validate* that a string is + > valid base 16 data, without actually producing a decoded output string. + > This function is both more performant and memory efficient than using + > `decode16/2`, checking that the result is `{:ok, ...}`, and then + > discarding the decoded binary. + + ## Options + + Accepts the same options as `decode16/2`. + + ## Examples + + iex> Base.valid16?("666F6F626172") + true + + iex> Base.valid16?("666f6f626172", case: :lower) + true + + iex> Base.valid16?("666f6F626172", case: :mixed) + true + + iex> Base.valid16?("ff", case: :upper) + false + + """ + @doc since: "1.19.0" + @spec valid16?(binary, case: decode_case) :: boolean + def valid16?(string, opts \\ []) + + def valid16?(string, opts) when is_binary(string) and rem(byte_size(string), 2) == 0 do + case Keyword.get(opts, :case, :upper) do + :upper -> validate16upper?(string) + :lower -> validate16lower?(string) + :mixed -> validate16mixed?(string) + end + end + + def valid16?(string, _opts) when is_binary(string) do + false + end + + upper = Enum.with_index(b16_alphabet) + + for {base, alphabet} <- [upper: upper, lower: to_lower_dec.(upper), mixed: to_mixed_dec.(upper)] do + decode_name = :"decode16#{base}!" + validate_name = :"validate16#{base}?" + valid_char_name = :"valid_char16#{base}?" + + {min, decoded} = to_decode_list.(alphabet) + + defp unquote(validate_name)(<<>>), do: true + + defp unquote(validate_name)(<>) do + unquote(valid_char_name)(c1) and + unquote(valid_char_name)(c2) and + unquote(validate_name)(rest) + end + + defp unquote(validate_name)(<<_char, _rest::binary>>), do: false + + @compile {:inline, [{valid_char_name, 1}]} + defp unquote(valid_char_name)(char) + when elem({unquote_splicing(decoded)}, char - unquote(min)) != nil, + do: true + + defp unquote(valid_char_name)(_char), do: false + + defp unquote(decode_name)(char) do + index = char - unquote(min) + + cond do + index not in 0..unquote(length(decoded) - 1) -> bad_character!(char) + new_char = elem({unquote_splicing(decoded)}, index) -> new_char + true -> bad_character!(char) + end + end + + defp unquote(decode_name)(<>, acc) do + unquote(decode_name)( + rest, + << + acc::binary, + unquote(decode_name)(c1)::4, + unquote(decode_name)(c2)::4, + unquote(decode_name)(c3)::4, + unquote(decode_name)(c4)::4, + unquote(decode_name)(c5)::4, + unquote(decode_name)(c6)::4, + unquote(decode_name)(c7)::4, + unquote(decode_name)(c8)::4 + >> + ) + end + + defp unquote(decode_name)(<>, acc) do + unquote(decode_name)( + rest, + << + acc::binary, + unquote(decode_name)(c1)::4, + unquote(decode_name)(c2)::4, + unquote(decode_name)(c3)::4, + unquote(decode_name)(c4)::4 + >> + ) + end + + defp unquote(decode_name)(<>, acc) do + unquote(decode_name)( + rest, + <> + ) + end + + defp unquote(decode_name)(<<>>, acc) do + acc + end end @doc """ Encodes a binary string into a base 64 encoded string. + Accepts `padding: false` option which will omit padding from + the output string. + ## Examples iex> Base.encode64("foobar") "Zm9vYmFy" + iex> Base.encode64("foob") + "Zm9vYg==" + + iex> Base.encode64("foob", padding: false) + "Zm9vYg" + + """ + @spec encode64(binary, padding: boolean) :: binary + def encode64(data, opts \\ []) when is_binary(data) do + pad? = Keyword.get(opts, :padding, true) + encode64base(data, "", pad?) + end + + @doc """ + Encodes a binary string into a base 64 encoded string with URL and filename + safe alphabet. + + Accepts `padding: false` option which will omit padding from + the output string. + + ## Examples + + iex> Base.url_encode64(<<255, 127, 254, 252>>) + "_3_-_A==" + + iex> Base.url_encode64(<<255, 127, 254, 252>>, padding: false) + "_3_-_A" + """ - @spec encode64(binary) :: binary - def encode64(data) when is_binary(data) do - do_encode64(data, &enc64/1) + @spec url_encode64(binary, padding: boolean) :: binary + def url_encode64(data, opts \\ []) when is_binary(data) do + pad? = Keyword.get(opts, :padding, true) + encode64url(data, "", pad?) + end + + for {base, alphabet} <- [base: b64_alphabet, url: b64url_alphabet] do + name = :"encode64#{base}" + encoded = to_encode_list.(alphabet) + + @compile {:inline, [{name, 1}]} + defp unquote(name)(byte) do + elem({unquote_splicing(encoded)}, byte) + end + + defp unquote(name)(<>, acc, pad?) do + unquote(name)( + rest, + << + acc::binary, + unquote(name)(c1)::16, + unquote(name)(c2)::16, + unquote(name)(c3)::16, + unquote(name)(c4)::16 + >>, + pad? + ) + end + + defp unquote(name)(<>, acc, pad?) do + << + acc::binary, + unquote(name)(c1)::16, + unquote(name)(c2)::16, + unquote(name)(c3)::16, + c4 |> bsl(2) |> unquote(name)() |> band(0x00FF)::8 + >> + |> maybe_pad(pad?, 1) + end + + defp unquote(name)(<>, acc, pad?) do + << + acc::binary, + unquote(name)(c1)::16, + unquote(name)(c2)::16, + c3 |> bsl(4) |> unquote(name)()::16 + >> + |> maybe_pad(pad?, 2) + end + + defp unquote(name)(<>, acc, _pad?) do + << + acc::binary, + unquote(name)(c1)::16, + unquote(name)(c2)::16 + >> + end + + defp unquote(name)(<>, acc, pad?) do + << + acc::binary, + unquote(name)(c1)::16, + c2 |> bsl(2) |> unquote(name)() |> band(0x00FF)::8 + >> + |> maybe_pad(pad?, 1) + end + + defp unquote(name)(<>, acc, pad?) do + << + acc::binary, + c1 |> bsl(4) |> unquote(name)()::16 + >> + |> maybe_pad(pad?, 2) + end + + defp unquote(name)(<<>>, acc, _pad?) do + acc + end end @doc """ Decodes a base 64 encoded string into a binary string. + Accepts `ignore: :whitespace` option which will ignore all the + whitespace characters in the input string. + + Accepts `padding: false` option which will ignore padding from + the input string. + ## Examples iex> Base.decode64("Zm9vYmFy") {:ok, "foobar"} + iex> Base.decode64("Zm9vYmFy\\n", ignore: :whitespace) + {:ok, "foobar"} + + iex> Base.decode64("Zm9vYg==") + {:ok, "foob"} + + iex> Base.decode64("Zm9vYg", padding: false) + {:ok, "foob"} + """ - @spec decode64(binary) :: {:ok, binary} | :error - def decode64(string) when is_binary(string) do - {:ok, do_decode64(string, &dec64/1)} + @spec decode64(binary, ignore: :whitespace, padding: boolean) :: {:ok, binary} | :error + def decode64(string, opts \\ []) when is_binary(string) do + {:ok, decode64!(string, opts)} rescue ArgumentError -> :error end @@ -254,7 +597,11 @@ defmodule Base do @doc """ Decodes a base 64 encoded string into a binary string. - The following alphabet is used both for encoding and decoding: + Accepts `ignore: :whitespace` option which will ignore all the + whitespace characters in the input string. + + Accepts `padding: false` option which will ignore padding from + the input string. An `ArgumentError` exception is raised if the padding is incorrect or a non-alphabet character is present in the string. @@ -264,40 +611,81 @@ defmodule Base do iex> Base.decode64!("Zm9vYmFy") "foobar" + iex> Base.decode64!("Zm9vYmFy\\n", ignore: :whitespace) + "foobar" + + iex> Base.decode64!("Zm9vYg==") + "foob" + + iex> Base.decode64!("Zm9vYg", padding: false) + "foob" + """ - @spec decode64!(binary) :: binary - def decode64!(string) when is_binary(string) do - do_decode64(string, &dec64/1) + @spec decode64!(binary, ignore: :whitespace, padding: boolean) :: binary + def decode64!(string, opts \\ []) when is_binary(string) do + pad? = Keyword.get(opts, :padding, true) + string |> remove_ignored(opts[:ignore]) |> decode64base!(pad?) end @doc """ - Encodes a binary string into a base 64 encoded string with URL and filename - safe alphabet. + Validates a base 64 encoded string. + + > #### When to use this {: .tip} + > + > Use this function when you just need to *validate* that a string is + > valid base 64 data, without actually producing a decoded output string. + > This function is both more performant and memory efficient than using + > `decode64/2`, checking that the result is `{:ok, ...}`, and then + > discarding the decoded binary. + + ## Options + + Accepts the same options as `decode64/2`. ## Examples - iex> Base.url_encode64(<<255,127,254,252>>) - "_3_-_A==" + iex> Base.valid64?("Zm9vYmFy") + true + + iex> Base.valid64?("Zm9vYmFy\\n", ignore: :whitespace) + true + + iex> Base.valid64?("Zm9vYg==") + true """ - @spec url_encode64(binary) :: binary - def url_encode64(data) when is_binary(data) do - do_encode64(data, &enc64url/1) + @doc since: "1.19.0" + @spec valid64?(binary, ignore: :whitespace, padding: boolean) :: boolean + def valid64?(string, opts \\ []) when is_binary(string) do + pad? = Keyword.get(opts, :padding, true) + string |> remove_ignored(opts[:ignore]) |> validate64base?(pad?) end @doc """ Decodes a base 64 encoded string with URL and filename safe alphabet into a binary string. + Accepts `ignore: :whitespace` option which will ignore all the + whitespace characters in the input string. + + Accepts `padding: false` option which will ignore padding from + the input string. + ## Examples iex> Base.url_decode64("_3_-_A==") - {:ok, <<255,127,254,252>>} + {:ok, <<255, 127, 254, 252>>} + + iex> Base.url_decode64("_3_-_A==\\n", ignore: :whitespace) + {:ok, <<255, 127, 254, 252>>} + + iex> Base.url_decode64("_3_-_A", padding: false) + {:ok, <<255, 127, 254, 252>>} """ - @spec url_decode64(binary) :: {:ok, binary} | :error - def url_decode64(string) when is_binary(string) do - {:ok, do_decode64(string, &dec64url/1)} + @spec url_decode64(binary, ignore: :whitespace, padding: boolean) :: {:ok, binary} | :error + def url_decode64(string, opts \\ []) when is_binary(string) do + {:ok, url_decode64!(string, opts)} rescue ArgumentError -> :error end @@ -306,25 +694,320 @@ defmodule Base do Decodes a base 64 encoded string with URL and filename safe alphabet into a binary string. + Accepts `ignore: :whitespace` option which will ignore all the + whitespace characters in the input string. + + Accepts `padding: false` option which will ignore padding from + the input string. + An `ArgumentError` exception is raised if the padding is incorrect or a non-alphabet character is present in the string. ## Examples iex> Base.url_decode64!("_3_-_A==") - <<255,127,254,252>> + <<255, 127, 254, 252>> + + iex> Base.url_decode64!("_3_-_A==\\n", ignore: :whitespace) + <<255, 127, 254, 252>> + + iex> Base.url_decode64!("_3_-_A", padding: false) + <<255, 127, 254, 252>> + + """ + @spec url_decode64!(binary, ignore: :whitespace, padding: boolean) :: binary + def url_decode64!(string, opts \\ []) when is_binary(string) do + pad? = Keyword.get(opts, :padding, true) + string |> remove_ignored(opts[:ignore]) |> decode64url!(pad?) + end + + @doc """ + Validates a base 64 encoded string with URL and filename safe alphabet. + + > #### When to use this {: .tip} + > + > Use this function when you just need to *validate* that a string is + > valid (URL-safe) base 64 data, without actually producing a decoded + > output string. This function is both more performant and memory efficient + > than using `url_decode64/2`, checking that the result is `{:ok, ...}`, + > and then discarding the decoded binary. + + ## Options + + Accepts the same options as `url_decode64/2`. + + ## Examples + + iex> Base.url_valid64?("_3_-_A==") + true + + iex> Base.url_valid64?("_3_-_A==\\n", ignore: :whitespace) + true + + iex> Base.url_valid64?("_3_-_A", padding: false) + true """ - @spec url_decode64!(binary) :: binary - def url_decode64!(string) when is_binary(string) do - do_decode64(string, &dec64url/1) + @doc since: "1.19.0" + @spec url_valid64?(binary, ignore: :whitespace, padding: boolean) :: boolean + def url_valid64?(string, opts \\ []) when is_binary(string) do + pad? = Keyword.get(opts, :padding, true) + string |> remove_ignored(opts[:ignore]) |> validate64url?(pad?) + end + + for {base, alphabet} <- [base: b64_alphabet, url: b64url_alphabet] do + decode_name = :"decode64#{base}!" + + validate_name = :"validate64#{base}?" + validate_main_name = :"validate_main64#{validate_name}?" + valid_char_name = :"valid_char64#{base}?" + {min, decoded} = alphabet |> Enum.with_index() |> to_decode_list.() + + defp unquote(validate_main_name)(<<>>), do: true + + defp unquote(validate_main_name)( + <> + ) do + unquote(valid_char_name)(c1) and + unquote(valid_char_name)(c2) and + unquote(valid_char_name)(c3) and + unquote(valid_char_name)(c4) and + unquote(valid_char_name)(c5) and + unquote(valid_char_name)(c6) and + unquote(valid_char_name)(c7) and + unquote(valid_char_name)(c8) and + unquote(validate_main_name)(rest) + end + + defp unquote(validate_name)(<<>>, _pad?), do: true + + defp unquote(validate_name)(string, pad?) do + segs = div(byte_size(string) + 7, 8) - 1 + <> = string + main_valid? = unquote(validate_main_name)(main) + + case rest do + _ when not main_valid? -> + false + + <> -> + unquote(valid_char_name)(c1) and + unquote(valid_char_name)(c2) + + <> -> + unquote(valid_char_name)(c1) and + unquote(valid_char_name)(c2) and + unquote(valid_char_name)(c3) + + <> -> + unquote(valid_char_name)(c1) and + unquote(valid_char_name)(c2) and + unquote(valid_char_name)(c3) and + unquote(valid_char_name)(c4) + + <> -> + unquote(valid_char_name)(c1) and + unquote(valid_char_name)(c2) and + unquote(valid_char_name)(c3) and + unquote(valid_char_name)(c4) and + unquote(valid_char_name)(c5) and + unquote(valid_char_name)(c6) + + <> -> + unquote(valid_char_name)(c1) and + unquote(valid_char_name)(c2) and + unquote(valid_char_name)(c3) and + unquote(valid_char_name)(c4) and + unquote(valid_char_name)(c5) and + unquote(valid_char_name)(c6) and + unquote(valid_char_name)(c7) + + <> -> + unquote(valid_char_name)(c1) and + unquote(valid_char_name)(c2) and + unquote(valid_char_name)(c3) and + unquote(valid_char_name)(c4) and + unquote(valid_char_name)(c5) and + unquote(valid_char_name)(c6) and + unquote(valid_char_name)(c7) and + unquote(valid_char_name)(c8) + + <> when not pad? -> + unquote(valid_char_name)(c1) and + unquote(valid_char_name)(c2) + + <> when not pad? -> + unquote(valid_char_name)(c1) and + unquote(valid_char_name)(c2) and + unquote(valid_char_name)(c3) + + <> when not pad? -> + unquote(valid_char_name)(c1) and + unquote(valid_char_name)(c2) and + unquote(valid_char_name)(c3) and + unquote(valid_char_name)(c4) and + unquote(valid_char_name)(c5) and + unquote(valid_char_name)(c6) + + <> when not pad? -> + unquote(valid_char_name)(c1) and + unquote(valid_char_name)(c2) and + unquote(valid_char_name)(c3) and + unquote(valid_char_name)(c4) and + unquote(valid_char_name)(c5) and + unquote(valid_char_name)(c6) and + unquote(valid_char_name)(c7) + + _ -> + false + end + end + + @compile {:inline, [{valid_char_name, 1}]} + defp unquote(valid_char_name)(char) + when elem({unquote_splicing(decoded)}, char - unquote(min)) != nil, + do: true + + defp unquote(valid_char_name)(_char), do: false + + defp unquote(decode_name)(char) do + index = char - unquote(min) + + cond do + index not in 0..unquote(length(decoded) - 1) -> bad_character!(char) + new_char = elem({unquote_splicing(decoded)}, index) -> new_char + true -> bad_character!(char) + end + end + + defp unquote(decode_name)(<<>>, _pad?), do: <<>> + + defp unquote(decode_name)(string, pad?) do + segs = div(byte_size(string) + 7, 8) - 1 + <> = string + + main = + for <>, into: <<>> do + << + unquote(decode_name)(c1)::6, + unquote(decode_name)(c2)::6, + unquote(decode_name)(c3)::6, + unquote(decode_name)(c4)::6, + unquote(decode_name)(c5)::6, + unquote(decode_name)(c6)::6, + unquote(decode_name)(c7)::6, + unquote(decode_name)(c8)::6 + >> + end + + case rest do + <> -> + <> + + <> -> + <> + + <> -> + << + main::bits, + unquote(decode_name)(c1)::6, + unquote(decode_name)(c2)::6, + unquote(decode_name)(c3)::6, + unquote(decode_name)(c4)::6 + >> + + <> -> + << + main::bits, + unquote(decode_name)(c1)::6, + unquote(decode_name)(c2)::6, + unquote(decode_name)(c3)::6, + unquote(decode_name)(c4)::6, + unquote(decode_name)(c5)::6, + bsr(unquote(decode_name)(c6), 4)::2 + >> + + <> -> + << + main::bits, + unquote(decode_name)(c1)::6, + unquote(decode_name)(c2)::6, + unquote(decode_name)(c3)::6, + unquote(decode_name)(c4)::6, + unquote(decode_name)(c5)::6, + unquote(decode_name)(c6)::6, + bsr(unquote(decode_name)(c7), 2)::4 + >> + + <> -> + << + main::bits, + unquote(decode_name)(c1)::6, + unquote(decode_name)(c2)::6, + unquote(decode_name)(c3)::6, + unquote(decode_name)(c4)::6, + unquote(decode_name)(c5)::6, + unquote(decode_name)(c6)::6, + unquote(decode_name)(c7)::6, + unquote(decode_name)(c8)::6 + >> + + <> when not pad? -> + <> + + <> when not pad? -> + <> + + <> when not pad? -> + << + main::bits, + unquote(decode_name)(c1)::6, + unquote(decode_name)(c2)::6, + unquote(decode_name)(c3)::6, + unquote(decode_name)(c4)::6, + unquote(decode_name)(c5)::6, + bsr(unquote(decode_name)(c6), 4)::2 + >> + + <> when not pad? -> + << + main::bits, + unquote(decode_name)(c1)::6, + unquote(decode_name)(c2)::6, + unquote(decode_name)(c3)::6, + unquote(decode_name)(c4)::6, + unquote(decode_name)(c5)::6, + unquote(decode_name)(c6)::6, + bsr(unquote(decode_name)(c7), 2)::4 + >> + + _ -> + raise ArgumentError, "incorrect padding" + end + end end @doc """ Encodes a binary string into a base 32 encoded string. - Accepts an atom `:upper` (default) for encoding to upper case characters or - `:lower` for lower case characters. + ## Options + + The accepted options are: + + * `:case` - specifies the character case to use when encoding + * `:padding` - specifies whether to apply padding + + The values for `:case` can be: + + * `:upper` - uses upper case characters (default) + * `:lower` - uses lower case characters + + The values for `:padding` can be: + + * `true` - pad the output string to the nearest multiple of 8 (default) + * `false` - omit padding from the output string ## Examples @@ -334,20 +1017,151 @@ defmodule Base do iex> Base.encode32("foobar", case: :lower) "mzxw6ytboi======" + iex> Base.encode32("foobar", padding: false) + "MZXW6YTBOI" + """ - @spec encode32(binary) :: binary - @spec encode32(binary, Keyword.t) :: binary + @spec encode32(binary, case: encode_case, padding: boolean) :: binary def encode32(data, opts \\ []) when is_binary(data) do - case = Keyword.get(opts, :case, :upper) - do_encode32(data, encode_case(case, &enc32/1)) + pad? = Keyword.get(opts, :padding, true) + + case Keyword.get(opts, :case, :upper) do + :upper -> encode32upper(data, "", pad?) + :lower -> encode32lower(data, "", pad?) + end + end + + @doc """ + Encodes a binary string into a base 32 encoded string with an + extended hexadecimal alphabet. + + ## Options + + The accepted options are: + + * `:case` - specifies the character case to use when encoding + * `:padding` - specifies whether to apply padding + + The values for `:case` can be: + + * `:upper` - uses upper case characters (default) + * `:lower` - uses lower case characters + + The values for `:padding` can be: + + * `true` - pad the output string to the nearest multiple of 8 (default) + * `false` - omit padding from the output string + + ## Examples + + iex> Base.hex_encode32("foobar") + "CPNMUOJ1E8======" + + iex> Base.hex_encode32("foobar", case: :lower) + "cpnmuoj1e8======" + + iex> Base.hex_encode32("foobar", padding: false) + "CPNMUOJ1E8" + + """ + @spec hex_encode32(binary, case: encode_case, padding: boolean) :: binary + def hex_encode32(data, opts \\ []) when is_binary(data) do + pad? = Keyword.get(opts, :padding, true) + + case Keyword.get(opts, :case, :upper) do + :upper -> encode32hexupper(data, "", pad?) + :lower -> encode32hexlower(data, "", pad?) + end + end + + for {base, alphabet} <- [ + upper: b32_alphabet, + lower: to_lower_enc.(b32_alphabet), + hexupper: b32hex_alphabet, + hexlower: to_lower_enc.(b32hex_alphabet) + ] do + name = :"encode32#{base}" + encoded = to_encode_list.(alphabet) + + @compile {:inline, [{name, 1}]} + defp unquote(name)(byte) do + elem({unquote_splicing(encoded)}, byte) + end + + defp unquote(name)(<>, acc, pad?) do + unquote(name)( + rest, + << + acc::binary, + unquote(name)(c1)::16, + unquote(name)(c2)::16, + unquote(name)(c3)::16, + unquote(name)(c4)::16 + >>, + pad? + ) + end + + defp unquote(name)(<>, acc, pad?) do + << + acc::binary, + unquote(name)(c1)::16, + unquote(name)(c2)::16, + unquote(name)(c3)::16, + c4 |> bsl(3) |> unquote(name)() |> band(0x00FF)::8 + >> + |> maybe_pad(pad?, 1) + end + + defp unquote(name)(<>, acc, pad?) do + << + acc::binary, + unquote(name)(c1)::16, + unquote(name)(c2)::16, + c3 |> bsl(1) |> unquote(name)() |> band(0x00FF)::8 + >> + |> maybe_pad(pad?, 3) + end + + defp unquote(name)(<>, acc, pad?) do + << + acc::binary, + unquote(name)(c1)::16, + c2 |> bsl(4) |> unquote(name)()::16 + >> + |> maybe_pad(pad?, 4) + end + + defp unquote(name)(<>, acc, pad?) do + < bsl(2) |> unquote(name)()::16>> + |> maybe_pad(pad?, 6) + end + + defp unquote(name)(<<>>, acc, _pad?) do + acc + end end @doc """ Decodes a base 32 encoded string into a binary string. - Accepts an atom `:upper` (default) for decoding from upper case characters or - `:lower` for lower case characters. `:mixed` can be given for mixed case - characters. + ## Options + + The accepted options are: + + * `:case` - specifies the character case to accept when decoding + * `:padding` - specifies whether to require padding + + The values for `:case` can be: + + * `:upper` - only allows upper case characters (default) + * `:lower` - only allows lower case characters + * `:mixed` - allows mixed case characters + + The values for `:padding` can be: + + * `true` - requires the input string to be padded to the nearest multiple of 8 (default) + * `false` - ignores padding from the input string ## Examples @@ -360,12 +1174,13 @@ defmodule Base do iex> Base.decode32("mzXW6ytBOi======", case: :mixed) {:ok, "foobar"} + iex> Base.decode32("MZXW6YTBOI", padding: false) + {:ok, "foobar"} + """ - @spec decode32(binary) :: {:ok, binary} | :error - @spec decode32(binary, Keyword.t) :: {:ok, binary} | :error + @spec decode32(binary, case: decode_case, padding: boolean) :: {:ok, binary} | :error def decode32(string, opts \\ []) do - case = Keyword.get(opts, :case, :upper) - {:ok, do_decode32(string, decode_case(case, &dec32/1))} + {:ok, decode32!(string, opts)} rescue ArgumentError -> :error end @@ -373,13 +1188,27 @@ defmodule Base do @doc """ Decodes a base 32 encoded string into a binary string. - Accepts an atom `:upper` (default) for decoding from upper case characters or - `:lower` for lower case characters. `:mixed` can be given for mixed case - characters. - An `ArgumentError` exception is raised if the padding is incorrect or a non-alphabet character is present in the string. + ## Options + + The accepted options are: + + * `:case` - specifies the character case to accept when decoding + * `:padding` - specifies whether to require padding + + The values for `:case` can be: + + * `:upper` - only allows upper case characters (default) + * `:lower` - only allows lower case characters + * `:mixed` - allows mixed case characters + + The values for `:padding` can be: + + * `true` - requires the input string to be padded to the nearest multiple of 8 (default) + * `false` - ignores padding from the input string + ## Examples iex> Base.decode32!("MZXW6YTBOI======") @@ -391,44 +1220,81 @@ defmodule Base do iex> Base.decode32!("mzXW6ytBOi======", case: :mixed) "foobar" + iex> Base.decode32!("MZXW6YTBOI", padding: false) + "foobar" + """ - @spec decode32!(binary) :: binary - @spec decode32!(binary, Keyword.t) :: binary - def decode32!(string, opts \\ []) do - case = Keyword.get(opts, :case, :upper) - do_decode32(string, decode_case(case, &dec32/1)) + @spec decode32!(binary, case: decode_case, padding: boolean) :: binary + def decode32!(string, opts \\ []) when is_binary(string) do + pad? = Keyword.get(opts, :padding, true) + + case Keyword.get(opts, :case, :upper) do + :upper -> decode32upper!(string, pad?) + :lower -> decode32lower!(string, pad?) + :mixed -> decode32mixed!(string, pad?) + end end @doc """ - Encodes a binary string into a base 32 encoded string with an - extended hexadecimal alphabet. + Checks if a base 32 encoded string is valid. - Accepts an atom `:upper` (default) for encoding to upper case characters or - `:lower` for lower case characters. + > #### When to use this {: .tip} + > + > Use this function when you just need to *validate* that a string is + > valid base 32 data, without actually producing a decoded output string. + > This function is both more performant and memory efficient than using + > `decode32/2`, checking that the result is `{:ok, ...}`, and then + > discarding the decoded binary. + + ## Options + + Accepts the same options as `decode32/2`. ## Examples - iex> Base.hex_encode32("foobar") - "CPNMUOJ1E8======" + iex> Base.valid32?("MZXW6YTBOI======") + true - iex> Base.hex_encode32("foobar", case: :lower) - "cpnmuoj1e8======" + iex> Base.valid32?("mzxw6ytboi======", case: :lower) + true + + iex> Base.valid32?("zzz") + false """ - @spec hex_encode32(binary) :: binary - @spec hex_encode32(binary, Keyword.t) :: binary - def hex_encode32(data, opts \\ []) when is_binary(data) do - case = Keyword.get(opts, :case, :upper) - do_encode32(data, encode_case(case, &enc32hex/1)) + @doc since: "1.19.0" + @spec valid32?(binary, case: decode_case, padding: boolean) :: boolean() + def valid32?(string, opts \\ []) when is_binary(string) do + pad? = Keyword.get(opts, :padding, true) + + case Keyword.get(opts, :case, :upper) do + :upper -> validate32upper?(string, pad?) + :lower -> validate32lower?(string, pad?) + :mixed -> validate32mixed?(string, pad?) + end end @doc """ Decodes a base 32 encoded string with extended hexadecimal alphabet into a binary string. - Accepts an atom `:upper` (default) for decoding from upper case characters or - `:lower` for lower case characters. `:mixed` can be given for mixed case - characters. + ## Options + + The accepted options are: + + * `:case` - specifies the character case to accept when decoding + * `:padding` - specifies whether to require padding + + The values for `:case` can be: + + * `:upper` - only allows upper case characters (default) + * `:lower` - only allows lower case characters + * `:mixed` - allows mixed case characters + + The values for `:padding` can be: + + * `true` - requires the input string to be padded to the nearest multiple of 8 (default) + * `false` - ignores padding from the input string ## Examples @@ -441,12 +1307,13 @@ defmodule Base do iex> Base.hex_decode32("cpnMuOJ1E8======", case: :mixed) {:ok, "foobar"} + iex> Base.hex_decode32("CPNMUOJ1E8", padding: false) + {:ok, "foobar"} + """ - @spec hex_decode32(binary) :: {:ok, binary} | :error - @spec hex_decode32(binary, Keyword.t) :: {:ok, binary} | :error - def hex_decode32(string, opts \\ []) when is_binary(string) do - case = Keyword.get(opts, :case, :upper) - {:ok, do_decode32(string, decode_case(case, &dec32hex/1))} + @spec hex_decode32(binary, case: decode_case, padding: boolean) :: {:ok, binary} | :error + def hex_decode32(string, opts \\ []) do + {:ok, hex_decode32!(string, opts)} rescue ArgumentError -> :error end @@ -455,13 +1322,27 @@ defmodule Base do Decodes a base 32 encoded string with extended hexadecimal alphabet into a binary string. - Accepts an atom `:upper` (default) for decoding from upper case characters or - `:lower` for lower case characters. `:mixed` can be given for mixed case - characters. - An `ArgumentError` exception is raised if the padding is incorrect or a non-alphabet character is present in the string. + ## Options + + The accepted options are: + + * `:case` - specifies the character case to accept when decoding + * `:padding` - specifies whether to require padding + + The values for `:case` can be: + + * `:upper` - only allows upper case characters (default) + * `:lower` - only allows lower case characters + * `:mixed` - allows mixed case characters + + The values for `:padding` can be: + + * `true` - requires the input string to be padded to the nearest multiple of 8 (default) + * `false` - ignores padding from the input string + ## Examples iex> Base.hex_decode32!("CPNMUOJ1E8======") @@ -473,120 +1354,293 @@ defmodule Base do iex> Base.hex_decode32!("cpnMuOJ1E8======", case: :mixed) "foobar" + iex> Base.hex_decode32!("CPNMUOJ1E8", padding: false) + "foobar" + """ - @spec hex_decode32!(binary) :: binary - @spec hex_decode32!(binary, Keyword.t) :: binary + @spec hex_decode32!(binary, case: decode_case, padding: boolean) :: binary def hex_decode32!(string, opts \\ []) when is_binary(string) do - case = Keyword.get(opts, :case, :upper) - do_decode32(string, decode_case(case, &dec32hex/1)) - end - - defp do_encode16(<<>>, _), do: <<>> - defp do_encode16(data, enc) do - for <>, into: <<>>, do: <> - end - - defp do_decode16(<<>>, _), do: <<>> - defp do_decode16(string, dec) when rem(byte_size(string), 2) == 0 do - for <>, into: <<>> do - <> - end - end - defp do_decode16(_, _) do - raise ArgumentError, "odd-length string" - end - - defp do_encode64(<<>>, _), do: <<>> - defp do_encode64(data, enc) do - split = 3 * div(byte_size(data), 3) - <> = data - main = for <>, into: <<>>, do: <> - case rest do - <> -> - <> - <> -> - <> - <<>> -> - main - end - end - - defp do_decode64(<<>>, _), do: <<>> - defp do_decode64(string, dec) when rem(byte_size(string), 4) == 0 do - split = byte_size(string) - 4 - <> = string - main = for <>, into: <<>>, do: <> - case rest do - <> -> - <> - <> -> - <> - <> -> - <> - <<>> -> - main - end - end - defp do_decode64(_, _) do - raise ArgumentError, "incorrect padding" - end - - defp do_encode32(<<>>, _), do: <<>> - defp do_encode32(data, enc) do - split = 5 * div(byte_size(data), 5) - <> = data - main = for <>, into: <<>>, do: <> - case rest do - <> -> - <> - <> -> - <> - <> -> - <> - <> -> - <> - <<>> -> - main - end - end - - defp do_decode32(<<>>, _), do: <<>> - defp do_decode32(string, dec) when rem(byte_size(string), 8) == 0 do - split = byte_size(string) - 8 - <> = string - main = for <>, into: <<>>, do: <> - case rest do - <> -> - <> - <> -> - <> - <> -> - <> - <> -> - <> - <> -> - <> - <<>> -> - main - end - end - defp do_decode32(_, _) do - raise ArgumentError, "incorrect padding" + pad? = Keyword.get(opts, :padding, true) + + case Keyword.get(opts, :case, :upper) do + :upper -> decode32hexupper!(string, pad?) + :lower -> decode32hexlower!(string, pad?) + :mixed -> decode32hexmixed!(string, pad?) + end + end + + @doc """ + Checks if a base 32 encoded string with extended hexadecimal alphabet is valid. + + > #### When to use this {: .tip} + > + > Use this function when you just need to *validate* that a string is + > valid (extended hexadecimal) base 32 data, without actually producing + > a decoded output string. This function is both more performant and + > memory efficient than using `hex_decode32/2`, checking that the result + > is `{:ok, ...}`, and then discarding the decoded binary. + + ## Options + + Accepts the same options as `hex_decode32/2`. + + ## Examples + + iex> Base.hex_valid32?("CPNMUOJ1E8======") + true + + iex> Base.hex_valid32?("cpnmuoj1e8======", case: :lower) + true + + iex> Base.hex_valid32?("zzz", padding: false) + false + + """ + @doc since: "1.19.0" + @spec hex_valid32?(binary, case: decode_case, padding: boolean) :: boolean + def hex_valid32?(string, opts \\ []) when is_binary(string) do + pad? = Keyword.get(opts, :padding, true) + + case Keyword.get(opts, :case, :upper) do + :upper -> validate32hexupper?(string, pad?) + :lower -> validate32hexlower?(string, pad?) + :mixed -> validate32hexmixed?(string, pad?) + end end + upper = Enum.with_index(b32_alphabet) + hexupper = Enum.with_index(b32hex_alphabet) + + for {base, alphabet} <- [ + upper: upper, + lower: to_lower_dec.(upper), + mixed: to_mixed_dec.(upper), + hexupper: hexupper, + hexlower: to_lower_dec.(hexupper), + hexmixed: to_mixed_dec.(hexupper) + ] do + decode_name = :"decode32#{base}!" + validate_name = :"validate32#{base}?" + validate_main_name = :"validate_main32#{validate_name}?" + valid_char_name = :"valid_char32#{base}?" + {min, decoded} = to_decode_list.(alphabet) + + defp unquote(validate_main_name)(<<>>), do: true + + defp unquote(validate_main_name)( + <> + ) do + unquote(valid_char_name)(c1) and + unquote(valid_char_name)(c2) and + unquote(valid_char_name)(c3) and + unquote(valid_char_name)(c4) and + unquote(valid_char_name)(c5) and + unquote(valid_char_name)(c6) and + unquote(valid_char_name)(c7) and + unquote(valid_char_name)(c8) and + unquote(validate_main_name)(rest) + end + + defp unquote(validate_name)(<<>>, _pad?), do: true + + defp unquote(validate_name)(string, pad?) do + segs = div(byte_size(string) + 7, 8) - 1 + <> = string + main_valid? = unquote(validate_main_name)(main) + + case rest do + _ when not main_valid? -> + false + + <> -> + unquote(valid_char_name)(c1) and + unquote(valid_char_name)(c2) + + <> -> + unquote(valid_char_name)(c1) and + unquote(valid_char_name)(c2) and + unquote(valid_char_name)(c3) and + unquote(valid_char_name)(c4) + + <> -> + unquote(valid_char_name)(c1) and + unquote(valid_char_name)(c2) and + unquote(valid_char_name)(c3) and + unquote(valid_char_name)(c4) and + unquote(valid_char_name)(c5) + + <> -> + unquote(valid_char_name)(c1) and + unquote(valid_char_name)(c2) and + unquote(valid_char_name)(c3) and + unquote(valid_char_name)(c4) and + unquote(valid_char_name)(c5) and + unquote(valid_char_name)(c6) and + unquote(valid_char_name)(c7) + + <> -> + unquote(valid_char_name)(c1) and + unquote(valid_char_name)(c2) and + unquote(valid_char_name)(c3) and + unquote(valid_char_name)(c4) and + unquote(valid_char_name)(c5) and + unquote(valid_char_name)(c6) and + unquote(valid_char_name)(c7) and + unquote(valid_char_name)(c8) + + <> when not pad? -> + unquote(valid_char_name)(c1) and + unquote(valid_char_name)(c2) + + <> when not pad? -> + unquote(valid_char_name)(c1) and + unquote(valid_char_name)(c2) and + unquote(valid_char_name)(c3) and + unquote(valid_char_name)(c4) + + <> when not pad? -> + unquote(valid_char_name)(c1) and + unquote(valid_char_name)(c2) and + unquote(valid_char_name)(c3) and + unquote(valid_char_name)(c4) and + unquote(valid_char_name)(c5) + + <> when not pad? -> + unquote(valid_char_name)(c1) and + unquote(valid_char_name)(c2) and + unquote(valid_char_name)(c3) and + unquote(valid_char_name)(c4) and + unquote(valid_char_name)(c5) and + unquote(valid_char_name)(c6) and + unquote(valid_char_name)(c7) + + _ -> + false + end + end + + @compile {:inline, [{valid_char_name, 1}]} + defp unquote(valid_char_name)(char) + when elem({unquote_splicing(decoded)}, char - unquote(min)) != nil, + do: true + + defp unquote(valid_char_name)(_char), do: false + + defp unquote(decode_name)(char) do + index = char - unquote(min) + + cond do + index not in 0..unquote(length(decoded) - 1) -> bad_character!(char) + new_char = elem({unquote_splicing(decoded)}, index) -> new_char + true -> bad_character!(char) + end + end + + defp unquote(decode_name)(<<>>, _), do: <<>> + + defp unquote(decode_name)(string, pad?) do + segs = div(byte_size(string) + 7, 8) - 1 + <> = string + + main = + for <>, into: <<>> do + << + unquote(decode_name)(c1)::5, + unquote(decode_name)(c2)::5, + unquote(decode_name)(c3)::5, + unquote(decode_name)(c4)::5, + unquote(decode_name)(c5)::5, + unquote(decode_name)(c6)::5, + unquote(decode_name)(c7)::5, + unquote(decode_name)(c8)::5 + >> + end + + case rest do + <> -> + <> + + <> -> + << + main::bits, + unquote(decode_name)(c1)::5, + unquote(decode_name)(c2)::5, + unquote(decode_name)(c3)::5, + bsr(unquote(decode_name)(c4), 4)::1 + >> + + <> -> + << + main::bits, + unquote(decode_name)(c1)::5, + unquote(decode_name)(c2)::5, + unquote(decode_name)(c3)::5, + unquote(decode_name)(c4)::5, + bsr(unquote(decode_name)(c5), 1)::4 + >> + + <> -> + << + main::bits, + unquote(decode_name)(c1)::5, + unquote(decode_name)(c2)::5, + unquote(decode_name)(c3)::5, + unquote(decode_name)(c4)::5, + unquote(decode_name)(c5)::5, + unquote(decode_name)(c6)::5, + bsr(unquote(decode_name)(c7), 3)::2 + >> + + <> -> + << + main::bits, + unquote(decode_name)(c1)::5, + unquote(decode_name)(c2)::5, + unquote(decode_name)(c3)::5, + unquote(decode_name)(c4)::5, + unquote(decode_name)(c5)::5, + unquote(decode_name)(c6)::5, + unquote(decode_name)(c7)::5, + unquote(decode_name)(c8)::5 + >> + + <> when not pad? -> + <> + + <> when not pad? -> + << + main::bits, + unquote(decode_name)(c1)::5, + unquote(decode_name)(c2)::5, + unquote(decode_name)(c3)::5, + bsr(unquote(decode_name)(c4), 4)::1 + >> + + <> when not pad? -> + << + main::bits, + unquote(decode_name)(c1)::5, + unquote(decode_name)(c2)::5, + unquote(decode_name)(c3)::5, + unquote(decode_name)(c4)::5, + bsr(unquote(decode_name)(c5), 1)::4 + >> + + <> when not pad? -> + << + main::bits, + unquote(decode_name)(c1)::5, + unquote(decode_name)(c2)::5, + unquote(decode_name)(c3)::5, + unquote(decode_name)(c4)::5, + unquote(decode_name)(c5)::5, + unquote(decode_name)(c6)::5, + bsr(unquote(decode_name)(c7), 3)::2 + >> + + _ -> + raise ArgumentError, "incorrect padding" + end + end + end end diff --git a/lib/elixir/lib/behaviour.ex b/lib/elixir/lib/behaviour.ex index 8f882a00294..4a43fccc068 100644 --- a/lib/elixir/lib/behaviour.ex +++ b/lib/elixir/lib/behaviour.ex @@ -1,63 +1,41 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + defmodule Behaviour do @moduledoc """ - Utilities for defining behaviour interfaces. - - Behaviours can be referenced by other modules - to ensure they implement required callbacks. - - For example, you can specify the `URI.Parser` - behaviour as follows: - - defmodule URI.Parser do - use Behaviour - - @doc "Parses the given URL" - defcallback parse(uri_info :: URI.t) :: URI.t - - @doc "Defines a default port" - defcallback default_port() :: integer - end - - And then a module may use it as: - - defmodule URI.HTTP do - @behaviour URI.Parser - def default_port(), do: 80 - def parse(info), do: info - end - - If the behaviour changes or `URI.HTTP` does - not implement one of the callbacks, a warning - will be raised. - - ## Implementation + Mechanism for handling behaviours. - Since Erlang R15, behaviours must be defined via - `@callback` attributes. `defcallback` is a simple - mechanism that defines the `@callback` attribute - according to the given type specification. `defcallback` allows - documentation to be created for the callback and defines - a custom function signature. + This module is deprecated. Instead of `defcallback/1` and + `defmacrocallback/1`, the `@callback` and `@macrocallback` + module attributes can be used respectively. See the + documentation for `Module` for more information on these + attributes. - The callbacks and their documentation can be retrieved - via the `__behaviour__` callback function. + Instead of `MyModule.__behaviour__(:callbacks)`, + `MyModule.behaviour_info(:callbacks)` can be used. `behaviour_info/1` + is documented in `Module`. """ + @moduledoc deprecated: "Use @callback and @macrocallback attributes instead" + @doc """ - Define a function callback according to the given type specification. + Defines a function callback according to the given type specification. """ + @deprecated "Use the @callback module attribute instead" defmacro defcallback(spec) do - do_defcallback(split_spec(spec, quote(do: term)), __CALLER__) + do_defcallback(:def, split_spec(spec, quote(do: term))) end @doc """ - Define a macro callback according to the given type specification. + Defines a macro callback according to the given type specification. """ + @deprecated "Use the @macrocallback module attribute instead" defmacro defmacrocallback(spec) do - do_defmacrocallback(split_spec(spec, quote(do: Macro.t)), __CALLER__) + do_defcallback(:defmacro, split_spec(spec, quote(do: Macro.t()))) end - defp split_spec({:when, _, [{:::, _, [spec, return]}, guard]}, _default) do + defp split_spec({:when, _, [{:"::", _, [spec, return]}, guard]}, _default) do {spec, return, guard} end @@ -65,7 +43,7 @@ defmodule Behaviour do {spec, default, guard} end - defp split_spec({:::, _, [spec, return]}, _default) do + defp split_spec({:"::", _, [spec, return]}, _default) do {spec, return, []} end @@ -73,40 +51,38 @@ defmodule Behaviour do {spec, default, []} end - defp do_defcallback({spec, return, guards}, caller) do + defp do_defcallback(kind, {spec, return, guards}) do case Macro.decompose_call(spec) do {name, args} -> - do_callback(:def, name, args, name, length(args), args, return, guards, caller) - _ -> - raise ArgumentError, "invalid syntax in defcallback #{Macro.to_string(spec)}" - end - end + do_callback(kind, name, args, return, guards) - defp do_defmacrocallback({spec, return, guards}, caller) do - case Macro.decompose_call(spec) do - {name, args} -> - do_callback(:defmacro, :"MACRO-#{name}", [quote(do: env :: Macro.Env.t)|args], - name, length(args), args, return, guards, caller) _ -> - raise ArgumentError, "invalid syntax in defmacrocallback #{Macro.to_string(spec)}" + raise ArgumentError, "invalid syntax in #{kind}callback #{Macro.to_string(spec)}" end end - defp do_callback(kind, name, args, docs_name, docs_arity, _docs_args, return, guards, caller) do - Enum.each args, fn - {:::, _, [left, right]} -> + defp do_callback(kind, name, args, return, guards) do + fun = fn + {:"::", _, [left, right]} -> ensure_not_default(left) ensure_not_default(right) left + other -> ensure_not_default(other) other end - quote do - @callback unquote(name)(unquote_splicing(args)) :: unquote(return) when unquote(guards) - Behaviour.store_docs(__MODULE__, unquote(caller.line), unquote(kind), - unquote(docs_name), unquote(docs_arity)) + :lists.foreach(fun, args) + + spec = + quote do + unquote(name)(unquote_splicing(args)) :: unquote(return) when unquote(guards) + end + + case kind do + :def -> quote(do: @callback(unquote(spec))) + :defmacro -> quote(do: @macrocallback(unquote(spec))) end end @@ -116,37 +92,37 @@ defmodule Behaviour do defp ensure_not_default(_), do: :ok - @doc false - def store_docs(module, line, kind, name, arity) do - doc = Module.get_attribute module, :doc - Module.delete_attribute module, :doc - Module.put_attribute module, :behaviour_docs, {{name, arity}, line, kind, doc} - end - @doc false defmacro __using__(_) do quote do - Module.register_attribute(__MODULE__, :behaviour_docs, accumulate: true) - @before_compile unquote(__MODULE__) - import unquote(__MODULE__) - end - end + warning = + "the Behaviour module is deprecated. Instead of using this module, " <> + "use the @callback and @macrocallback module attributes. See the " <> + "documentation for Module for more information on these attributes" - @doc false - defmacro __before_compile__(env) do - docs = if Code.compiler_options[:docs] do - Enum.reverse Module.get_attribute(env.module, :behaviour_docs) - end + IO.warn(warning) - quote do @doc false def __behaviour__(:callbacks) do __MODULE__.behaviour_info(:callbacks) end def __behaviour__(:docs) do - unquote(Macro.escape(docs)) + {:docs_v1, _, :elixir, _, _, _, docs} = Code.fetch_docs(__MODULE__) + + for {{kind, name, arity}, line, _, doc, _} <- docs, kind in [:callback, :macrocallback] do + case kind do + :callback -> {{name, arity}, line, :def, __behaviour__doc_value(doc)} + :macrocallback -> {{name, arity}, line, :defmacro, __behaviour__doc_value(doc)} + end + end end + + defp __behaviour__doc_value(%{"en" => doc}), do: doc + defp __behaviour__doc_value(:hidden), do: false + defp __behaviour__doc_value(_), do: nil + + import unquote(__MODULE__) end end end diff --git a/lib/elixir/lib/bitwise.ex b/lib/elixir/lib/bitwise.ex index 9d74a608983..e00ecb76278 100644 --- a/lib/elixir/lib/bitwise.ex +++ b/lib/elixir/lib/bitwise.ex @@ -1,41 +1,44 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + defmodule Bitwise do @moduledoc """ - This module provides macros and operators for bitwise operators. - These macros can be used in guards. + A set of functions that perform calculations on bits. - The easiest way to use is to simply import them into - your module: + All bitwise functions work only on integers, otherwise an + `ArithmeticError` is raised. The functions `band/2`, + `bor/2`, `bsl/2`, and `bsr/2` also have operators, + respectively: `&&&/2`, `|||/2`, `<<>>/2`. - iex> use Bitwise - iex> bnot 1 - -2 - iex> 1 &&& 1 - 1 + ## Guards - You can select to include only or skip operators by passing options: + All bitwise functions can be used in guards: - iex> use Bitwise, only_operators: true - iex> 1 &&& 1 - 1 + iex> odd? = fn + ...> int when Bitwise.band(int, 1) == 1 -> true + ...> _ -> false + ...> end + iex> odd?.(1) + true + All functions in this module are inlined by the compiler. """ - @doc """ - Allow a developer to use this module in their programs with - the following options: + @doc false + @deprecated "import Bitwise instead" + defmacro __using__(options) do + except = + cond do + Keyword.get(options, :only_operators) -> + [bnot: 1, band: 2, bor: 2, bxor: 2, bsl: 2, bsr: 2] - * `:only_operators` - include only operators - * `:skip_operators` - skip operators + Keyword.get(options, :skip_operators) -> + ["~~~": 1, &&&: 2, |||: 2, "^^^": 2, <<<: 2, >>>: 2] - """ - defmacro __using__(options) do - except = cond do - Keyword.get(options, :only_operators) -> - [bnot: 1, band: 2, bor: 2, bxor: 2, bsl: 2, bsr: 2] - Keyword.get(options, :skip_operators) -> - [~~~: 1, &&&: 2, |||: 2, ^^^: 2, <<<: 2, >>>: 2] - true -> [] - end + true -> + [] + end quote do import Bitwise, except: unquote(except) @@ -43,86 +46,229 @@ defmodule Bitwise do end @doc """ - Bitwise not. + Calculates the bitwise NOT of the argument. + + Allowed in guard tests. Inlined by the compiler. + + ## Examples + + iex> bnot(2) + -3 + + iex> bnot(2) &&& 3 + 1 + """ - defmacro bnot(expr) do - quote do: :erlang.bnot(unquote(expr)) + @doc guard: true + @spec bnot(integer) :: integer + def bnot(expr) do + :erlang.bnot(expr) end - @doc """ - Bitwise not as operator. - """ - defmacro ~~~expr do - quote do: :erlang.bnot(unquote(expr)) + @doc false + def unquote(:"~~~")(expr) do + :erlang.bnot(expr) end @doc """ - Bitwise and. + Calculates the bitwise AND of its arguments. + + Allowed in guard tests. Inlined by the compiler. + + ## Examples + + iex> band(9, 3) + 1 + """ - defmacro band(left, right) do - quote do: :erlang.band(unquote(left), unquote(right)) + @doc guard: true + @spec band(integer, integer) :: integer + def band(left, right) do + :erlang.band(left, right) end @doc """ - Bitwise and as operator. + Bitwise AND operator. + + Calculates the bitwise AND of its arguments. + + Allowed in guard tests. Inlined by the compiler. + + ## Examples + + iex> 9 &&& 3 + 1 + """ - defmacro left &&& right do - quote do: :erlang.band(unquote(left), unquote(right)) + @doc guard: true + @spec integer &&& integer :: integer + def left &&& right do + :erlang.band(left, right) end @doc """ - Bitwise or. + Calculates the bitwise OR of its arguments. + + Allowed in guard tests. Inlined by the compiler. + + ## Examples + + iex> bor(9, 3) + 11 + """ - defmacro bor(left, right) do - quote do: :erlang.bor(unquote(left), unquote(right)) + @doc guard: true + @spec bor(integer, integer) :: integer + def bor(left, right) do + :erlang.bor(left, right) end @doc """ - Bitwise or as operator. + Bitwise OR operator. + + Calculates the bitwise OR of its arguments. + + Allowed in guard tests. Inlined by the compiler. + + ## Examples + + iex> 9 ||| 3 + 11 + """ - defmacro left ||| right do - quote do: :erlang.bor(unquote(left), unquote(right)) + @doc guard: true + @spec integer ||| integer :: integer + def left ||| right do + :erlang.bor(left, right) end @doc """ - Bitwise xor. + Calculates the bitwise XOR of its arguments. + + Allowed in guard tests. Inlined by the compiler. + + ## Examples + + iex> bxor(9, 3) + 10 + """ - defmacro bxor(left, right) do - quote do: :erlang.bxor(unquote(left), unquote(right)) + @doc guard: true + @spec bxor(integer, integer) :: integer + def bxor(left, right) do + :erlang.bxor(left, right) end - @doc """ - Bitwise xor as operator. - """ - defmacro left ^^^ right do - quote do: :erlang.bxor(unquote(left), unquote(right)) + @doc false + def unquote(:"^^^")(left, right) do + :erlang.bxor(left, right) end @doc """ - Arithmetic bitshift left. + Calculates the result of an arithmetic left bitshift. + + Allowed in guard tests. Inlined by the compiler. + + ## Examples + + iex> bsl(1, 2) + 4 + + iex> bsl(1, -2) + 0 + + iex> bsl(-1, 2) + -4 + + iex> bsl(-1, -2) + -1 + """ - defmacro bsl(left, right) do - quote do: :erlang.bsl(unquote(left), unquote(right)) + @doc guard: true + @spec bsl(integer, integer) :: integer + def bsl(left, right) do + :erlang.bsl(left, right) end @doc """ - Arithmetic bitshift left as operator. + Arithmetic left bitshift operator. + + Calculates the result of an arithmetic left bitshift. + + Allowed in guard tests. Inlined by the compiler. + + ## Examples + + iex> 1 <<< 2 + 4 + + iex> 1 <<< -2 + 0 + + iex> -1 <<< 2 + -4 + + iex> -1 <<< -2 + -1 + """ - defmacro left <<< right do - quote do: :erlang.bsl(unquote(left), unquote(right)) + @doc guard: true + @spec integer <<< integer :: integer + def left <<< right do + :erlang.bsl(left, right) end @doc """ - Arithmetic bitshift right. + Calculates the result of an arithmetic right bitshift. + + Allowed in guard tests. Inlined by the compiler. + + ## Examples + + iex> bsr(1, 2) + 0 + + iex> bsr(1, -2) + 4 + + iex> bsr(-1, 2) + -1 + + iex> bsr(-1, -2) + -4 + """ - defmacro bsr(left, right) do - quote do: :erlang.bsr(unquote(left), unquote(right)) + @doc guard: true + @spec bsr(integer, integer) :: integer + def bsr(left, right) do + :erlang.bsr(left, right) end @doc """ - Arithmetic bitshift right as operator. + Arithmetic right bitshift operator. + + Calculates the result of an arithmetic right bitshift. + + Allowed in guard tests. Inlined by the compiler. + + ## Examples + + iex> 1 >>> 2 + 0 + + iex> 1 >>> -2 + 4 + + iex> -1 >>> 2 + -1 + + iex> -1 >>> -2 + -4 + """ - defmacro left >>> right do - quote do: :erlang.bsr(unquote(left), unquote(right)) + @doc guard: true + @spec integer >>> integer :: integer + def left >>> right do + :erlang.bsr(left, right) end end diff --git a/lib/elixir/lib/calendar.ex b/lib/elixir/lib/calendar.ex new file mode 100644 index 00000000000..89160389b8d --- /dev/null +++ b/lib/elixir/lib/calendar.ex @@ -0,0 +1,1054 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +defmodule Calendar do + @moduledoc """ + This module defines the responsibilities for working with + calendars, dates, times and datetimes in Elixir. + + It defines types and the minimal implementation + for a calendar behaviour in Elixir. The goal of the calendar + features in Elixir is to provide a base for interoperability + rather than a full-featured datetime API. + + For the actual date, time and datetime structs, see `Date`, + `Time`, `NaiveDateTime`, and `DateTime`. + + Types for year, month, day, and more are *overspecified*. + For example, the `t:month/0` type is specified as an integer + instead of `1..12`. This is because different calendars may + have a different number of days per month. + """ + + @type year :: integer + @type month :: pos_integer + @type day :: pos_integer + @type week :: pos_integer + @type day_of_week :: non_neg_integer + @type era :: non_neg_integer + + @typedoc """ + A tuple representing the `day` and the `era`. + """ + @type day_of_era :: {day :: non_neg_integer(), era} + + @type hour :: non_neg_integer + @type minute :: non_neg_integer + @type second :: non_neg_integer + + @typedoc """ + The internal time format is used when converting between calendars. + + It represents time as a fraction of a day (starting from midnight). + `parts_in_day` specifies how much of the day is already passed, + while `parts_per_day` signifies how many parts are there in a day. + """ + @type day_fraction :: {parts_in_day :: non_neg_integer, parts_per_day :: pos_integer} + + @typedoc """ + The internal date format that is used when converting between calendars. + + This is the number of days including the fractional part that has passed of + the last day since `0000-01-01+00:00T00:00.000000` in ISO 8601 notation (also + known as *midnight 1 January BC 1* of the proleptic Gregorian calendar). + """ + @type iso_days :: {days :: integer, day_fraction} + + @typedoc """ + Microseconds with stored precision. + + `value` always represents the total value in microseconds. + + The `precision` represents the number of digits that must be used when + representing the microseconds to external format. If the precision is `0`, + it means microseconds must be skipped. If the precision is `6`, it means + that `value` represents exactly the number of microseconds to be used. + + ## Examples + + * `{0, 0}` means no microseconds. + * `{1, 6}` means 1µs. + * `{1000, 6}` means 1000µs (which is 1ms but measured at the microsecond precision). + * `{1000, 3}` means 1ms (which is measured at the millisecond precision). + + """ + @type microsecond :: {value :: non_neg_integer, precision :: non_neg_integer} + + @typedoc "A calendar implementation." + @type calendar :: module + + @typedoc "The time zone ID according to the IANA tz database (for example, `Europe/Zurich`)." + @type time_zone :: String.t() + + @typedoc "The time zone abbreviation (for example, `CET` or `CEST` or `BST`)." + @type zone_abbr :: String.t() + + @typedoc """ + The time zone UTC offset in ISO seconds for standard time. + + See also `t:std_offset/0`. + """ + @type utc_offset :: integer + + @typedoc """ + The time zone standard offset in ISO seconds (typically not zero in summer times). + + It must be added to `t:utc_offset/0` to get the total offset from UTC used for "wall time". + """ + @type std_offset :: integer + + @typedoc "Any map or struct that contains the date fields." + @type date :: %{optional(any) => any, calendar: calendar, year: year, month: month, day: day} + + @typedoc "Any map or struct that contains the time fields." + @type time :: %{ + optional(any) => any, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond + } + + @typedoc "Any map or struct that contains the naive datetime fields." + @type naive_datetime :: %{ + optional(any) => any, + calendar: calendar, + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond + } + + @typedoc "Any map or struct that contains the datetime fields." + @type datetime :: %{ + optional(any) => any, + calendar: calendar, + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond, + time_zone: time_zone, + zone_abbr: zone_abbr, + utc_offset: utc_offset, + std_offset: std_offset + } + + @typedoc """ + Specifies the time zone database for calendar operations. + + Many functions in the `DateTime` module require a time zone database. + By default, this module uses the default time zone database returned by + `Calendar.get_time_zone_database/0`, which defaults to + `Calendar.UTCOnlyTimeZoneDatabase`. This database only handles `Etc/UTC` + datetimes and returns `{:error, :utc_only_time_zone_database}` + for any other time zone. + + Other time zone databases (including ones provided by packages) + can be configured as default either via configuration: + + config :elixir, :time_zone_database, CustomTimeZoneDatabase + + or by calling `Calendar.put_time_zone_database/1`. + + See `Calendar.TimeZoneDatabase` for more information on custom + time zone databases. + """ + @type time_zone_database :: module() + + @typedoc """ + Options for formatting dates and times with `strftime/3`. + """ + @type strftime_opts :: [ + preferred_datetime: String.t(), + preferred_date: String.t(), + preferred_time: String.t(), + am_pm_names: (:am | :pm -> String.t()) | (:am | :pm, map() -> String.t()), + month_names: (pos_integer() -> String.t()) | (pos_integer(), map() -> String.t()), + abbreviated_month_names: + (pos_integer() -> String.t()) | (pos_integer(), map() -> String.t()), + day_of_week_names: (pos_integer() -> String.t()) | (pos_integer(), map() -> String.t()), + abbreviated_day_of_week_names: + (pos_integer() -> String.t()) | (pos_integer(), map() -> String.t()) + ] + + @doc """ + Returns how many days there are in the given month of the given year. + """ + @callback days_in_month(year, month) :: day + + @doc """ + Returns how many months there are in the given year. + """ + @callback months_in_year(year) :: month + + @doc """ + Returns `true` if the given year is a leap year. + + A leap year is a year of a longer length than normal. The exact meaning + is up to the calendar. A calendar must return `false` if it does not support + the concept of leap years. + """ + @callback leap_year?(year) :: boolean + + @doc """ + Calculates the day of the week from the given `year`, `month`, and `day`. + + `starting_on` represents the starting day of the week. All + calendars must support at least the `:default` value. They may + also support other values representing their days of the week. + + The value of `day_of_week` is an ordinal number meaning that a + value of `1` is defined to mean "first day of the week". It is + specifically not defined to mean `1` is `Monday`. + + It is a requirement that `first_day_of_week` is less than `last_day_of_week` + and that `day_of_week` must be within that range. Therefore it can be said + that `day_of_week in first_day_of_week..last_day_of_week//1` must be + `true` for all values of `day_of_week`. + """ + @callback day_of_week(year, month, day, starting_on :: :default | atom) :: + {day_of_week(), first_day_of_week :: non_neg_integer(), + last_day_of_week :: non_neg_integer()} + + @doc """ + Calculates the day of the year from the given `year`, `month`, and `day`. + """ + @callback day_of_year(year, month, day) :: non_neg_integer() + + @doc """ + Calculates the quarter of the year from the given `year`, `month`, and `day`. + """ + @callback quarter_of_year(year, month, day) :: non_neg_integer() + + @doc """ + Calculates the year and era from the given `year`. + """ + @callback year_of_era(year, month, day) :: {year, era} + + @doc """ + Calculates the day and era from the given `year`, `month`, and `day`. + """ + @callback day_of_era(year, month, day) :: day_of_era() + + @doc """ + Converts the date into a string according to the calendar. + """ + @callback date_to_string(year, month, day) :: String.t() + + @doc """ + Converts the naive datetime (without time zone) into a string according to the calendar. + """ + @callback naive_datetime_to_string(year, month, day, hour, minute, second, microsecond) :: + String.t() + + @doc """ + Converts the datetime (with time zone) into a string according to the calendar. + """ + @callback datetime_to_string( + year, + month, + day, + hour, + minute, + second, + microsecond, + time_zone, + zone_abbr, + utc_offset, + std_offset + ) :: String.t() + + @doc """ + Converts the time into a string according to the calendar. + """ + @callback time_to_string(hour, minute, second, microsecond) :: String.t() + + @doc """ + Converts the datetime (without time zone) into the `t:iso_days/0` format. + """ + @callback naive_datetime_to_iso_days(year, month, day, hour, minute, second, microsecond) :: + iso_days + + @doc """ + Converts `t:iso_days/0` to the calendar's datetime format. + """ + @callback naive_datetime_from_iso_days(iso_days) :: + {year, month, day, hour, minute, second, microsecond} + + @doc """ + Converts the given time to the `t:day_fraction/0` format. + """ + @callback time_to_day_fraction(hour, minute, second, microsecond) :: day_fraction + + @doc """ + Converts `t:day_fraction/0` to the calendar's time format. + """ + @callback time_from_day_fraction(day_fraction) :: {hour, minute, second, microsecond} + + @doc """ + Define the rollover moment for the calendar. + + This is the moment, in your calendar, when the current day ends + and the next day starts. + + The result of this function is used to check if two calendars roll over at + the same time of day. If they do not, we can only convert datetimes and times + between them. If they do, this means that we can also convert dates as well + as naive datetimes between them. + + This day fraction should be in its most simplified form possible, to make comparisons fast. + + ## Examples + + * If in your calendar a new day starts at midnight, return `{0, 1}`. + * If in your calendar a new day starts at sunrise, return `{1, 4}`. + * If in your calendar a new day starts at noon, return `{1, 2}`. + * If in your calendar a new day starts at sunset, return `{3, 4}`. + + """ + @callback day_rollover_relative_to_midnight_utc() :: day_fraction + + @doc """ + Should return `true` if the given date describes a proper date in the calendar. + """ + @callback valid_date?(year, month, day) :: boolean + + @doc """ + Should return `true` if the given time describes a proper time in the calendar. + """ + @callback valid_time?(hour, minute, second, microsecond) :: boolean + + @doc """ + Parses the string representation for a time returned by `c:time_to_string/4` + into a time tuple. + """ + @doc since: "1.10.0" + @callback parse_time(String.t()) :: + {:ok, {hour, minute, second, microsecond}} + | {:error, atom} + + @doc """ + Parses the string representation for a date returned by `c:date_to_string/3` + into a date tuple. + """ + @doc since: "1.10.0" + @callback parse_date(String.t()) :: + {:ok, {year, month, day}} + | {:error, atom} + + @doc """ + Parses the string representation for a naive datetime returned by + `c:naive_datetime_to_string/7` into a naive datetime tuple. + + The given string may contain a timezone offset but it is ignored. + """ + @doc since: "1.10.0" + @callback parse_naive_datetime(String.t()) :: + {:ok, {year, month, day, hour, minute, second, microsecond}} + | {:error, atom} + + @doc """ + Parses the string representation for a datetime returned by + `c:datetime_to_string/11` into a datetime tuple. + + The returned datetime must be in UTC. The original `utc_offset` + it was written in must be returned in the result. + """ + @doc since: "1.10.0" + @callback parse_utc_datetime(String.t()) :: + {:ok, {year, month, day, hour, minute, second, microsecond}, utc_offset} + | {:error, atom} + + @doc """ + Converts the given `t:iso_days/0` to the first moment of the day. + """ + @doc since: "1.15.0" + @callback iso_days_to_beginning_of_day(iso_days) :: iso_days + + @doc """ + Converts the given `t:iso_days/0` to the last moment of the day. + """ + @doc since: "1.15.0" + @callback iso_days_to_end_of_day(iso_days) :: iso_days + + @doc """ + Shifts date by given duration according to its calendar. + """ + @doc since: "1.17.0" + @callback shift_date(year, month, day, Duration.t()) :: {year, month, day} + + @doc """ + Shifts naive datetime by given duration according to its calendar. + """ + @doc since: "1.17.0" + @callback shift_naive_datetime( + year, + month, + day, + hour, + minute, + second, + microsecond, + Duration.t() + ) :: {year, month, day, hour, minute, second, microsecond} + + @doc """ + Shifts time by given duration according to its calendar. + """ + @doc since: "1.17.0" + @callback shift_time(hour, minute, second, microsecond, Duration.t()) :: + {hour, minute, second, microsecond} + + # General Helpers + + @doc """ + Returns `true` if two calendars have the same moment of starting a new day, + `false` otherwise. + + If two calendars are not compatible, we can only convert datetimes and times + between them. If they are compatible, this means that we can also convert + dates as well as naive datetimes between them. + """ + @doc since: "1.5.0" + @spec compatible_calendars?(Calendar.calendar(), Calendar.calendar()) :: boolean + def compatible_calendars?(calendar, calendar), do: true + + def compatible_calendars?(calendar1, calendar2) do + calendar1.day_rollover_relative_to_midnight_utc() == + calendar2.day_rollover_relative_to_midnight_utc() + end + + @doc """ + Returns a microsecond tuple truncated to a given precision (`:microsecond`, + `:millisecond`, or `:second`). + """ + @doc since: "1.6.0" + @spec truncate(Calendar.microsecond(), :microsecond | :millisecond | :second) :: + Calendar.microsecond() + def truncate(microsecond_tuple, :microsecond), do: microsecond_tuple + + def truncate({microsecond, precision}, :millisecond) do + output_precision = min(precision, 3) + {div(microsecond, 1000) * 1000, output_precision} + end + + def truncate(_, :second), do: {0, 0} + + @doc """ + Sets the current time zone database. + """ + @doc since: "1.8.0" + @spec put_time_zone_database(time_zone_database()) :: :ok + def put_time_zone_database(database) when is_atom(database) do + Application.put_env(:elixir, :time_zone_database, database) + end + + @doc """ + Gets the current time zone database. + """ + @doc since: "1.8.0" + @spec get_time_zone_database() :: time_zone_database() + def get_time_zone_database() do + Application.fetch_env!(:elixir, :time_zone_database) + end + + @doc """ + Formats the given date, time, or datetime into a string. + + The datetime can be any of the `Calendar` types (`Time`, `Date`, + `NaiveDateTime`, and `DateTime`) or any map, as long as they + contain all of the relevant fields necessary for formatting. + For example, if you use `%Y` to format the year, the datetime + must have the `:year` field. Therefore, if you pass a `Time`, + or a map without the `:year` field to a format that expects `%Y`, + an error will be raised. + + Examples of common usage: + + iex> Calendar.strftime(~U[2019-08-26 13:52:06.0Z], "%y-%m-%d %I:%M:%S %p") + "19-08-26 01:52:06 PM" + + iex> Calendar.strftime(~U[2019-08-26 13:52:06.0Z], "%a, %B %d %Y") + "Mon, August 26 2019" + + ## User Options + + * `:preferred_datetime` - a string for the preferred format to show datetimes, + it can't contain the `%c` format and defaults to `"%Y-%m-%d %H:%M:%S"` + if the option is not received + + * `:preferred_date` - a string for the preferred format to show dates, + it can't contain the `%x` format and defaults to `"%Y-%m-%d"` + if the option is not received + + * `:preferred_time` - a string for the preferred format to show times, + it can't contain the `%X` format and defaults to `"%H:%M:%S"` + if the option is not received + + * `:am_pm_names` - a function that receives either `:am` or `:pm` + (and also the datetime if the function is arity/2) and returns + the name of the period of the day, if the option is not received it defaults + to a function that returns `"am"` and `"pm"`, respectively + + * `:month_names` - a function that receives a number (and also the + datetime if the function is arity/2) and returns the name of + the corresponding month, if the option is not received it defaults to a + function that returns the month names in English + + * `:abbreviated_month_names` - a function that receives a number (and also + the datetime if the function is arity/2) and returns the + abbreviated name of the corresponding month, if the option is not received it + defaults to a function that returns the abbreviated month names in English + + * `:day_of_week_names` - a function that receives a number and (and also the + datetime if the function is arity/2) returns the name of + the corresponding day of week, if the option is not received it defaults to a + function that returns the day of week names in English + + * `:abbreviated_day_of_week_names` - a function that receives a number (and also + the datetime if the function is arity/2) and returns the abbreviated name of + the corresponding day of week, if the option is not received it defaults to a + function that returns the abbreviated day of week names in English + + ## Formatting syntax + + The formatting syntax for the `string_format` argument is a sequence of characters in + the following format: + + % + + where: + + * `%`: indicates the start of a formatted section + * ``: set the padding (see below) + * ``: a number indicating the minimum size of the formatted section + * ``: the format itself (see below) + + ### Accepted padding options + + * `-`: no padding, removes all padding from the format + * `_`: pad with spaces + * `0`: pad with zeroes + + ### Accepted string formats + + The accepted formats for `string_format` are: + + Format | Description | Examples (in ISO) + :----- | :-----------------------------------------------------------------------| :------------------------ + a | Abbreviated name of day | Mon + A | Full name of day | Monday + b | Abbreviated month name | Jan + B | Full month name | January + c | Preferred date+time representation | 2018-10-17 12:34:56 + d | Day of the month | 01, 31 + f | Microseconds (uses its precision for width and padding) | 000000, 999999, 0123 + H | Hour using a 24-hour clock | 00, 23 + I | Hour using a 12-hour clock | 01, 12 + j | Day of the year | 001, 366 + m | Month | 01, 12 + M | Minute | 00, 59 + p | "AM" or "PM" (noon is "PM", midnight as "AM") | AM, PM + P | "am" or "pm" (noon is "pm", midnight as "am") | am, pm + q | Quarter | 1, 2, 3, 4 + s | Number of seconds since the Epoch, 1970-01-01 00:00:00+0000 (UTC) | 1565888877 + S | Second | 00, 59, 60 + u | Day of the week | 1 (Monday), 7 (Sunday) + x | Preferred date (without time) representation | 2018-10-17 + X | Preferred time (without date) representation | 12:34:56 + y | Year as 2-digits | 01, 01, 86, 18 + Y | Year | -0001, 0001, 1986 + z | +hhmm/-hhmm time zone offset from UTC (empty string if naive) | +0300, -0530 + Z | Time zone abbreviation (empty string if naive) | CET, BRST + % | Literal "%" character | % + + Any other character will be interpreted as an invalid format and raise an error. + + ### `%f` Microseconds + + `%f` does not support width and padding modifiers. It will be formatted by truncating + the microseconds to the precision of the `microseconds` field of the struct, with a + minimum precision of 1. + + ## Examples + + Without user options: + + iex> Calendar.strftime(~U[2019-08-26 13:52:06.0Z], "%y-%m-%d %I:%M:%S %p") + "19-08-26 01:52:06 PM" + + iex> Calendar.strftime(~U[2019-08-26 13:52:06.0Z], "%a, %B %d %Y") + "Mon, August 26 2019" + + iex> Calendar.strftime(~U[2020-04-02 13:52:06.0Z], "%B %-d, %Y") + "April 2, 2020" + + iex> Calendar.strftime(~U[2019-08-26 13:52:06.0Z], "%c") + "2019-08-26 13:52:06" + + With user options: + + iex> Calendar.strftime(~U[2019-08-26 13:52:06.0Z], "%c", preferred_datetime: "%H:%M:%S %d-%m-%y") + "13:52:06 26-08-19" + + iex> Calendar.strftime( + ...> ~U[2019-08-26 13:52:06.0Z], + ...> "%A", + ...> day_of_week_names: fn day_of_week -> + ...> {"segunda-feira", "terça-feira", "quarta-feira", "quinta-feira", + ...> "sexta-feira", "sábado", "domingo"} + ...> |> elem(day_of_week - 1) + ...> end + ...>) + "segunda-feira" + + iex> Calendar.strftime( + ...> ~U[2019-08-26 13:52:06.0Z], + ...> "%B", + ...> month_names: fn month -> + ...> {"січень", "лютий", "березень", "квітень", "травень", "червень", + ...> "липень", "серпень", "вересень", "жовтень", "листопад", "грудень"} + ...> |> elem(month - 1) + ...> end + ...>) + "серпень" + + Microsecond formatting: + + iex> Calendar.strftime(~U[2019-08-26 13:52:06Z], "%y-%m-%d %H:%M:%S.%f") + "19-08-26 13:52:06.0" + + iex> Calendar.strftime(~U[2019-08-26 13:52:06.048Z], "%y-%m-%d %H:%M:%S.%f") + "19-08-26 13:52:06.048" + + iex> Calendar.strftime(~U[2019-08-26 13:52:06.048531Z], "%y-%m-%d %H:%M:%S.%f") + "19-08-26 13:52:06.048531" + + """ + @doc since: "1.11.0" + @spec strftime(map(), String.t(), strftime_opts()) :: String.t() + def strftime(date_or_time_or_datetime, string_format, user_options \\ []) + when is_map(date_or_time_or_datetime) and is_binary(string_format) do + parse( + string_format, + date_or_time_or_datetime, + options(user_options), + [] + ) + |> IO.iodata_to_binary() + end + + defp parse("", _datetime, _format_options, acc), + do: Enum.reverse(acc) + + defp parse("%" <> rest, datetime, format_options, acc), + do: parse_modifiers(rest, nil, nil, {datetime, format_options, acc}) + + defp parse(<>, datetime, format_options, acc), + do: parse(rest, datetime, format_options, [char | acc]) + + defp parse_modifiers("-" <> rest, width, nil, parser_data) do + parse_modifiers(rest, width, "", parser_data) + end + + defp parse_modifiers("0" <> rest, nil, nil, parser_data) do + parse_modifiers(rest, nil, ?0, parser_data) + end + + defp parse_modifiers("_" <> rest, width, nil, parser_data) do + parse_modifiers(rest, width, ?\s, parser_data) + end + + defp parse_modifiers(<>, width, pad, parser_data) when digit in ?0..?9 do + new_width = (width || 0) * 10 + (digit - ?0) + + parse_modifiers(rest, new_width, pad, parser_data) + end + + # set default padding if none was specified + defp parse_modifiers(<> = rest, width, nil, parser_data) do + parse_modifiers(rest, width, default_pad(format), parser_data) + end + + # set default width if none was specified + defp parse_modifiers(<> = rest, nil, pad, parser_data) do + parse_modifiers(rest, default_width(format), pad, parser_data) + end + + defp parse_modifiers(rest, width, pad, {datetime, format_options, acc}) do + format_modifiers(rest, width, pad, datetime, format_options, acc) + end + + defp am_pm(hour, format_options, datetime) when hour > 11 do + apply_format(:pm, format_options.am_pm_names, datetime) + end + + defp am_pm(hour, format_options, datetime) when hour <= 11 do + apply_format(:am, format_options.am_pm_names, datetime) + end + + defp default_pad(format) when format in ~c"aAbBpPZ", do: ?\s + defp default_pad(_format), do: ?0 + + defp default_width(format) when format in ~c"dHImMSy", do: 2 + defp default_width(?j), do: 3 + defp default_width(format) when format in ~c"Yz", do: 4 + defp default_width(_format), do: 0 + + # Literally just % + defp format_modifiers("%" <> rest, width, pad, datetime, format_options, acc) do + parse(rest, datetime, format_options, [pad_leading("%", width, pad) | acc]) + end + + # Abbreviated name of day + defp format_modifiers("a" <> rest, width, pad, datetime, format_options, acc) do + result = + datetime + |> Date.day_of_week() + |> apply_format(format_options.abbreviated_day_of_week_names, datetime) + |> pad_leading(width, pad) + + parse(rest, datetime, format_options, [result | acc]) + end + + # Full name of day + defp format_modifiers("A" <> rest, width, pad, datetime, format_options, acc) do + result = + datetime + |> Date.day_of_week() + |> apply_format(format_options.day_of_week_names, datetime) + |> pad_leading(width, pad) + + parse(rest, datetime, format_options, [result | acc]) + end + + # Abbreviated month name + defp format_modifiers("b" <> rest, width, pad, datetime, format_options, acc) do + result = + datetime.month + |> apply_format(format_options.abbreviated_month_names, datetime) + |> pad_leading(width, pad) + + parse(rest, datetime, format_options, [result | acc]) + end + + # Full month name + defp format_modifiers("B" <> rest, width, pad, datetime, format_options, acc) do + result = + datetime.month + |> apply_format(format_options.month_names, datetime) + |> pad_leading(width, pad) + + parse(rest, datetime, format_options, [result | acc]) + end + + # Preferred date+time representation + defp format_modifiers( + "c" <> _rest, + _width, + _pad, + _datetime, + %{preferred_datetime_invoked: true}, + _acc + ) do + raise ArgumentError, + "tried to format preferred_datetime within another preferred_datetime format" + end + + defp format_modifiers("c" <> rest, width, pad, datetime, format_options, acc) do + result = + format_options.preferred_datetime + |> parse(datetime, %{format_options | preferred_datetime_invoked: true}, []) + |> pad_preferred(width, pad) + + parse(rest, datetime, format_options, [result | acc]) + end + + # Day of the month + defp format_modifiers("d" <> rest, width, pad, datetime, format_options, acc) do + result = datetime.day |> Integer.to_string() |> pad_leading(width, pad) + parse(rest, datetime, format_options, [result | acc]) + end + + # Microseconds + defp format_modifiers("f" <> rest, _width, _pad, datetime, format_options, acc) do + {microsecond, precision} = datetime.microsecond + + result = + microsecond + |> Integer.to_string() + |> String.pad_leading(6, "0") + |> binary_part(0, max(precision, 1)) + + parse(rest, datetime, format_options, [result | acc]) + end + + # Hour using a 24-hour clock + defp format_modifiers("H" <> rest, width, pad, datetime, format_options, acc) do + result = datetime.hour |> Integer.to_string() |> pad_leading(width, pad) + parse(rest, datetime, format_options, [result | acc]) + end + + # Hour using a 12-hour clock + defp format_modifiers("I" <> rest, width, pad, datetime, format_options, acc) do + result = (rem(datetime.hour + 23, 12) + 1) |> Integer.to_string() |> pad_leading(width, pad) + parse(rest, datetime, format_options, [result | acc]) + end + + # Day of the year + defp format_modifiers("j" <> rest, width, pad, datetime, format_options, acc) do + result = datetime |> Date.day_of_year() |> Integer.to_string() |> pad_leading(width, pad) + parse(rest, datetime, format_options, [result | acc]) + end + + # Month + defp format_modifiers("m" <> rest, width, pad, datetime, format_options, acc) do + result = datetime.month |> Integer.to_string() |> pad_leading(width, pad) + parse(rest, datetime, format_options, [result | acc]) + end + + # Minute + defp format_modifiers("M" <> rest, width, pad, datetime, format_options, acc) do + result = datetime.minute |> Integer.to_string() |> pad_leading(width, pad) + parse(rest, datetime, format_options, [result | acc]) + end + + # "AM" or "PM" (noon is "PM", midnight as "AM") + defp format_modifiers("p" <> rest, width, pad, datetime, format_options, acc) do + result = + datetime.hour + |> am_pm(format_options, datetime) + |> String.upcase() + |> pad_leading(width, pad) + + parse(rest, datetime, format_options, [result | acc]) + end + + # "am" or "pm" (noon is "pm", midnight as "am") + defp format_modifiers("P" <> rest, width, pad, datetime, format_options, acc) do + result = + datetime.hour + |> am_pm(format_options, datetime) + |> String.downcase() + |> pad_leading(width, pad) + + parse(rest, datetime, format_options, [result | acc]) + end + + # Quarter + defp format_modifiers("q" <> rest, width, pad, datetime, format_options, acc) do + result = datetime |> Date.quarter_of_year() |> Integer.to_string() |> pad_leading(width, pad) + parse(rest, datetime, format_options, [result | acc]) + end + + # Second + defp format_modifiers("S" <> rest, width, pad, datetime, format_options, acc) do + result = datetime.second |> Integer.to_string() |> pad_leading(width, pad) + parse(rest, datetime, format_options, [result | acc]) + end + + # Day of the week + defp format_modifiers("u" <> rest, width, pad, datetime, format_options, acc) do + result = datetime |> Date.day_of_week() |> Integer.to_string() |> pad_leading(width, pad) + parse(rest, datetime, format_options, [result | acc]) + end + + # Preferred date (without time) representation + defp format_modifiers( + "x" <> _rest, + _width, + _pad, + _datetime, + %{preferred_date_invoked: true}, + _acc + ) do + raise ArgumentError, + "tried to format preferred_date within another preferred_date format" + end + + defp format_modifiers("x" <> rest, width, pad, datetime, format_options, acc) do + result = + format_options.preferred_date + |> parse(datetime, %{format_options | preferred_date_invoked: true}, []) + |> pad_preferred(width, pad) + + parse(rest, datetime, format_options, [result | acc]) + end + + # Preferred time (without date) representation + defp format_modifiers( + "X" <> _rest, + _width, + _pad, + _datetime, + %{preferred_time_invoked: true}, + _acc + ) do + raise ArgumentError, + "tried to format preferred_time within another preferred_time format" + end + + defp format_modifiers("X" <> rest, width, pad, datetime, format_options, acc) do + result = + format_options.preferred_time + |> parse(datetime, %{format_options | preferred_time_invoked: true}, []) + |> pad_preferred(width, pad) + + parse(rest, datetime, format_options, [result | acc]) + end + + # Year as 2-digits + defp format_modifiers("y" <> rest, width, pad, datetime, format_options, acc) do + result = datetime.year |> rem(100) |> Integer.to_string() |> pad_leading(width, pad) + parse(rest, datetime, format_options, [result | acc]) + end + + # Year + defp format_modifiers("Y" <> rest, width, pad, datetime, format_options, acc) do + {sign, year} = + if datetime.year < 0 do + {?-, -datetime.year} + else + {[], datetime.year} + end + + result = [sign | year |> Integer.to_string() |> pad_leading(width, pad)] + parse(rest, datetime, format_options, [result | acc]) + end + + # Epoch time for DateTime with time zones + defp format_modifiers( + "s" <> rest, + _width, + _pad, + datetime = %{utc_offset: _utc_offset, std_offset: _std_offset}, + format_options, + acc + ) do + result = + datetime + |> DateTime.shift_zone!("Etc/UTC") + |> NaiveDateTime.diff(~N[1970-01-01 00:00:00]) + |> Integer.to_string() + + parse(rest, datetime, format_options, [result | acc]) + end + + # Epoch time + defp format_modifiers("s" <> rest, _width, _pad, datetime, format_options, acc) do + result = + datetime + |> NaiveDateTime.diff(~N[1970-01-01 00:00:00]) + |> Integer.to_string() + + parse(rest, datetime, format_options, [result | acc]) + end + + # +hhmm/-hhmm time zone offset from UTC (empty string if naive) + defp format_modifiers( + "z" <> rest, + width, + pad, + datetime = %{utc_offset: utc_offset, std_offset: std_offset}, + format_options, + acc + ) do + absolute_offset = abs(utc_offset + std_offset) + + offset_number = + Integer.to_string(div(absolute_offset, 3600) * 100 + rem(div(absolute_offset, 60), 60)) + + sign = if utc_offset + std_offset >= 0, do: "+", else: "-" + result = "#{sign}#{pad_leading(offset_number, width, pad)}" + parse(rest, datetime, format_options, [result | acc]) + end + + defp format_modifiers("z" <> rest, _width, _pad, datetime, format_options, acc) do + parse(rest, datetime, format_options, ["" | acc]) + end + + # Time zone abbreviation (empty string if naive) + defp format_modifiers("Z" <> rest, width, pad, datetime, format_options, acc) do + result = datetime |> Map.get(:zone_abbr, "") |> pad_leading(width, pad) + parse(rest, datetime, format_options, [result | acc]) + end + + defp format_modifiers(rest, _width, _pad, _datetime, _format_options, _acc) do + {next, _rest} = String.next_grapheme(rest) || {"", ""} + raise ArgumentError, "invalid strftime format: %#{next}" + end + + defp pad_preferred(result, width, pad) when length(result) < width do + pad_preferred([pad | result], width, pad) + end + + defp pad_preferred(result, _width, _pad), do: result + + defp pad_leading(string, count, padding) do + to_pad = count - byte_size(string) + if to_pad > 0, do: do_pad_leading(to_pad, padding, string), else: string + end + + defp do_pad_leading(0, _, acc), do: acc + + defp do_pad_leading(count, padding, acc), + do: do_pad_leading(count - 1, padding, [padding | acc]) + + defp apply_format(term, formatter, _datetime) when is_function(formatter, 1) do + formatter.(term) + end + + defp apply_format(term, formatter, datetime) when is_function(formatter, 2) do + formatter.(term, datetime) + end + + defp apply_format(_term, formatter, _datetime) do + raise ArgumentError, "formatter functions must be of arity 1 or 2, got: #{inspect(formatter)}" + end + + defp options(user_options) do + default_options = %{ + preferred_date: "%Y-%m-%d", + preferred_time: "%H:%M:%S", + preferred_datetime: "%Y-%m-%d %H:%M:%S", + am_pm_names: fn + :am -> "am" + :pm -> "pm" + end, + month_names: fn month -> + {"January", "February", "March", "April", "May", "June", "July", "August", "September", + "October", "November", "December"} + |> elem(month - 1) + end, + day_of_week_names: fn day_of_week -> + {"Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"} + |> elem(day_of_week - 1) + end, + abbreviated_month_names: fn month -> + {"Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"} + |> elem(month - 1) + end, + abbreviated_day_of_week_names: fn day_of_week -> + {"Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"} |> elem(day_of_week - 1) + end, + preferred_datetime_invoked: false, + preferred_date_invoked: false, + preferred_time_invoked: false + } + + Enum.reduce(user_options, default_options, fn {key, value}, acc -> + if Map.has_key?(acc, key) do + %{acc | key => value} + else + raise ArgumentError, "unknown option #{inspect(key)} given to Calendar.strftime/3" + end + end) + end +end diff --git a/lib/elixir/lib/calendar/date.ex b/lib/elixir/lib/calendar/date.ex new file mode 100644 index 00000000000..35c4b75d3f4 --- /dev/null +++ b/lib/elixir/lib/calendar/date.ex @@ -0,0 +1,1190 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +defmodule Date do + @moduledoc """ + A Date struct and functions. + + The Date struct contains the fields year, month, day and calendar. + New dates can be built with the `new/3` function or using the + `~D` (see `sigil_D/2`) sigil: + + iex> ~D[2000-01-01] + ~D[2000-01-01] + + Both `new/3` and sigil return a struct where the date fields can + be accessed directly: + + iex> date = ~D[2000-01-01] + iex> date.year + 2000 + iex> date.month + 1 + + The functions on this module work with the `Date` struct as well + as any struct that contains the same fields as the `Date` struct, + such as `NaiveDateTime` and `DateTime`. Such functions expect + `t:Calendar.date/0` in their typespecs (instead of `t:t/0`). + + Developers should avoid creating the Date structs directly + and instead rely on the functions provided by this module as well + as the ones in third-party calendar libraries. + + ## Comparing dates + + Comparisons in Elixir using `==/2`, `>/2`, ` Enum.min([~D[2017-03-31], ~D[2017-04-01]], Date) + ~D[2017-03-31] + + ## Using epochs + + The `add/2`, `diff/2` and `shift/2` functions can be used for computing dates + or retrieving the number of days between instants. For example, if there + is an interest in computing the number of days from the Unix epoch + (1970-01-01): + + iex> Date.diff(~D[2010-04-17], ~D[1970-01-01]) + 14716 + + iex> Date.add(~D[1970-01-01], 14_716) + ~D[2010-04-17] + + iex> Date.shift(~D[1970-01-01], year: 40, month: 3, week: 2, day: 2) + ~D[2010-04-17] + + Those functions are optimized to deal with common epochs, such + as the Unix Epoch above or the Gregorian Epoch (0000-01-01). + """ + + @enforce_keys [:year, :month, :day] + defstruct [:year, :month, :day, calendar: Calendar.ISO] + + @type t :: %__MODULE__{ + year: Calendar.year(), + month: Calendar.month(), + day: Calendar.day(), + calendar: Calendar.calendar() + } + + @doc """ + Returns a range of dates. + + A range of dates represents a discrete number of dates where + the first and last values are dates with matching calendars. + + Ranges of dates can be increasing (`first <= last`) and are + always inclusive. For a decreasing range, use `range/3` with + a step of -1 as first argument. + + ## Examples + + iex> Date.range(~D[1999-01-01], ~D[2000-01-01]) + Date.range(~D[1999-01-01], ~D[2000-01-01]) + + A range of dates implements the `Enumerable` protocol, which means + functions in the `Enum` module can be used to work with + ranges: + + iex> range = Date.range(~D[2001-01-01], ~D[2002-01-01]) + iex> range + Date.range(~D[2001-01-01], ~D[2002-01-01]) + iex> Enum.count(range) + 366 + iex> ~D[2001-02-01] in range + true + iex> Enum.take(range, 3) + [~D[2001-01-01], ~D[2001-01-02], ~D[2001-01-03]] + + """ + @doc since: "1.5.0" + @spec range(Calendar.date(), Calendar.date()) :: Date.Range.t() + def range(%{calendar: calendar} = first, %{calendar: calendar} = last) do + {first_days, _} = to_iso_days(first) + {last_days, _} = to_iso_days(last) + + step = + if first_days <= last_days do + 1 + else + IO.warn( + "a negative range was inferred for Date.range/2, call Date.range/3 instead with -1 as third argument" + ) + + -1 + end + + range(first, first_days, last, last_days, calendar, step) + end + + def range(%{calendar: _, year: _, month: _, day: _}, %{calendar: _, year: _, month: _, day: _}) do + raise ArgumentError, "both dates must have matching calendars" + end + + @doc """ + Returns a range of dates with a step. + + ## Examples + + iex> range = Date.range(~D[2001-01-01], ~D[2002-01-01], 2) + iex> range + Date.range(~D[2001-01-01], ~D[2002-01-01], 2) + iex> Enum.count(range) + 183 + iex> ~D[2001-01-03] in range + true + iex> Enum.take(range, 3) + [~D[2001-01-01], ~D[2001-01-03], ~D[2001-01-05]] + + """ + @doc since: "1.12.0" + @spec range(Calendar.date(), Calendar.date(), step :: pos_integer | neg_integer) :: + Date.Range.t() + def range(%{calendar: calendar} = first, %{calendar: calendar} = last, step) + when is_integer(step) and step != 0 do + {first_days, _} = to_iso_days(first) + {last_days, _} = to_iso_days(last) + range(first, first_days, last, last_days, calendar, step) + end + + def range( + %{calendar: _, year: _, month: _, day: _} = first, + %{calendar: _, year: _, month: _, day: _} = last, + step + ) do + raise ArgumentError, + "both dates must have matching calendar and the step must be a " <> + "non-zero integer, got: #{inspect(first)}, #{inspect(last)}, #{step}" + end + + defp range(first, first_days, last, last_days, calendar, step) do + %Date.Range{ + first: %Date{calendar: calendar, year: first.year, month: first.month, day: first.day}, + last: %Date{calendar: calendar, year: last.year, month: last.month, day: last.day}, + first_in_iso_days: first_days, + last_in_iso_days: last_days, + step: step + } + end + + @doc """ + Returns the current date in UTC. + + ## Examples + + iex> date = Date.utc_today() + iex> date.year >= 2016 + true + + """ + @doc since: "1.4.0" + @spec utc_today(Calendar.calendar()) :: t + def utc_today(calendar \\ Calendar.ISO) + + def utc_today(Calendar.ISO) do + {:ok, {year, month, day}, _, _} = Calendar.ISO.from_unix(System.os_time(), :native) + %Date{year: year, month: month, day: day} + end + + def utc_today(calendar) do + %{year: year, month: month, day: day} = DateTime.utc_now(calendar) + %Date{year: year, month: month, day: day, calendar: calendar} + end + + @doc """ + Returns `true` if the year in the given `date` is a leap year. + + ## Examples + + iex> Date.leap_year?(~D[2000-01-01]) + true + iex> Date.leap_year?(~D[2001-01-01]) + false + iex> Date.leap_year?(~D[2004-01-01]) + true + iex> Date.leap_year?(~D[1900-01-01]) + false + iex> Date.leap_year?(~N[2004-01-01 01:23:45]) + true + + """ + @doc since: "1.4.0" + @spec leap_year?(Calendar.date()) :: boolean() + def leap_year?(date) + + def leap_year?(%{calendar: calendar, year: year}) do + calendar.leap_year?(year) + end + + @doc """ + Returns the number of days in the given `date` month. + + ## Examples + + iex> Date.days_in_month(~D[1900-01-13]) + 31 + iex> Date.days_in_month(~D[1900-02-09]) + 28 + iex> Date.days_in_month(~N[2000-02-20 01:23:45]) + 29 + + """ + @doc since: "1.4.0" + @spec days_in_month(Calendar.date()) :: Calendar.day() + def days_in_month(date) + + def days_in_month(%{calendar: calendar, year: year, month: month}) do + calendar.days_in_month(year, month) + end + + @doc """ + Returns the number of months in the given `date` year. + + ## Example + + iex> Date.months_in_year(~D[1900-01-13]) + 12 + + """ + @doc since: "1.7.0" + @spec months_in_year(Calendar.date()) :: Calendar.month() + def months_in_year(date) + + def months_in_year(%{calendar: calendar, year: year}) do + calendar.months_in_year(year) + end + + @doc """ + Builds a new ISO date. + + Expects all values to be integers. Returns `{:ok, date}` if each + entry fits its appropriate range, returns `{:error, reason}` otherwise. + + ## Examples + + iex> Date.new(2000, 1, 1) + {:ok, ~D[2000-01-01]} + iex> Date.new(2000, 13, 1) + {:error, :invalid_date} + iex> Date.new(2000, 2, 29) + {:ok, ~D[2000-02-29]} + + iex> Date.new(2000, 2, 30) + {:error, :invalid_date} + iex> Date.new(2001, 2, 29) + {:error, :invalid_date} + + """ + @spec new(Calendar.year(), Calendar.month(), Calendar.day(), Calendar.calendar()) :: + {:ok, t} | {:error, atom} + def new(year, month, day, calendar \\ Calendar.ISO) do + if calendar.valid_date?(year, month, day) do + {:ok, %Date{year: year, month: month, day: day, calendar: calendar}} + else + {:error, :invalid_date} + end + end + + @doc """ + Builds a new ISO date. + + Expects all values to be integers. Returns `date` if each + entry fits its appropriate range, raises if the date is invalid. + + ## Examples + + iex> Date.new!(2000, 1, 1) + ~D[2000-01-01] + iex> Date.new!(2000, 13, 1) + ** (ArgumentError) cannot build date, reason: :invalid_date + iex> Date.new!(2000, 2, 29) + ~D[2000-02-29] + """ + @doc since: "1.11.0" + @spec new!(Calendar.year(), Calendar.month(), Calendar.day(), Calendar.calendar()) :: t + def new!(year, month, day, calendar \\ Calendar.ISO) do + case new(year, month, day, calendar) do + {:ok, value} -> + value + + {:error, reason} -> + raise ArgumentError, "cannot build date, reason: #{inspect(reason)}" + end + end + + @doc """ + Converts the given date to a string according to its calendar. + + ## Examples + + iex> Date.to_string(~D[2000-02-28]) + "2000-02-28" + iex> Date.to_string(~N[2000-02-28 01:23:45]) + "2000-02-28" + iex> Date.to_string(~D[-0100-12-15]) + "-0100-12-15" + + """ + @spec to_string(Calendar.date()) :: String.t() + def to_string(date) + + def to_string(%{calendar: calendar, year: year, month: month, day: day}) do + calendar.date_to_string(year, month, day) + end + + @doc """ + Parses the extended "Dates" format described by + [ISO 8601:2019](https://en.wikipedia.org/wiki/ISO_8601). + + The year parsed by this function is limited to four digits. + + ## Examples + + iex> Date.from_iso8601("2015-01-23") + {:ok, ~D[2015-01-23]} + + iex> Date.from_iso8601("2015:01:23") + {:error, :invalid_format} + + iex> Date.from_iso8601("2015-01-32") + {:error, :invalid_date} + + """ + @spec from_iso8601(String.t(), Calendar.calendar()) :: {:ok, t} | {:error, atom} + def from_iso8601(string, calendar \\ Calendar.ISO) do + with {:ok, {year, month, day}} <- Calendar.ISO.parse_date(string) do + convert(%Date{year: year, month: month, day: day}, calendar) + end + end + + @doc """ + Parses the extended "Dates" format described by + [ISO 8601:2019](https://en.wikipedia.org/wiki/ISO_8601). + + Raises if the format is invalid. + + ## Examples + + iex> Date.from_iso8601!("2015-01-23") + ~D[2015-01-23] + iex> Date.from_iso8601!("2015:01:23") + ** (ArgumentError) cannot parse "2015:01:23" as date, reason: :invalid_format + + """ + @spec from_iso8601!(String.t(), Calendar.calendar()) :: t + def from_iso8601!(string, calendar \\ Calendar.ISO) do + case from_iso8601(string, calendar) do + {:ok, value} -> + value + + {:error, reason} -> + raise ArgumentError, "cannot parse #{inspect(string)} as date, reason: #{inspect(reason)}" + end + end + + @doc """ + Converts the given `date` to + [ISO 8601:2019](https://en.wikipedia.org/wiki/ISO_8601). + + By default, `Date.to_iso8601/2` returns dates formatted in the "extended" + format, for human readability. It also supports the "basic" format through passing the `:basic` option. + + Only supports converting dates which are in the ISO calendar, + or other calendars in which the days also start at midnight. + Attempting to convert dates from other calendars will raise an `ArgumentError`. + + ## Examples + + iex> Date.to_iso8601(~D[2000-02-28]) + "2000-02-28" + + iex> Date.to_iso8601(~D[2000-02-28], :basic) + "20000228" + + iex> Date.to_iso8601(~N[2000-02-28 00:00:00]) + "2000-02-28" + + """ + @spec to_iso8601(Calendar.date(), :extended | :basic) :: String.t() + def to_iso8601(date, format \\ :extended) + + def to_iso8601(%{calendar: Calendar.ISO} = date, format) when format in [:basic, :extended] do + %{year: year, month: month, day: day} = date + Calendar.ISO.date_to_string(year, month, day, format) + end + + def to_iso8601(%{calendar: _} = date, format) when format in [:basic, :extended] do + date + |> convert!(Calendar.ISO) + |> to_iso8601() + end + + @doc """ + Converts the given `date` to an Erlang date tuple. + + Only supports converting dates which are in the ISO calendar, + or other calendars in which the days also start at midnight. + Attempting to convert dates from other calendars will raise. + + ## Examples + + iex> Date.to_erl(~D[2000-01-01]) + {2000, 1, 1} + + iex> Date.to_erl(~N[2000-01-01 00:00:00]) + {2000, 1, 1} + + """ + @spec to_erl(Calendar.date()) :: :calendar.date() + def to_erl(date) do + %{year: year, month: month, day: day} = convert!(date, Calendar.ISO) + {year, month, day} + end + + @doc """ + Converts an Erlang date tuple to a `Date` struct. + + Only supports converting dates which are in the ISO calendar, + or other calendars in which the days also start at midnight. + Attempting to convert dates from other calendars will return an error tuple. + + ## Examples + + iex> Date.from_erl({2000, 1, 1}) + {:ok, ~D[2000-01-01]} + iex> Date.from_erl({2000, 13, 1}) + {:error, :invalid_date} + + """ + @spec from_erl(:calendar.date(), Calendar.calendar()) :: {:ok, t} | {:error, atom} + def from_erl(tuple, calendar \\ Calendar.ISO) + + def from_erl({year, month, day}, calendar) do + with {:ok, date} <- new(year, month, day, Calendar.ISO), do: convert(date, calendar) + end + + @doc """ + Converts an Erlang date tuple but raises for invalid dates. + + ## Examples + + iex> Date.from_erl!({2000, 1, 1}) + ~D[2000-01-01] + iex> Date.from_erl!({2000, 13, 1}) + ** (ArgumentError) cannot convert {2000, 13, 1} to date, reason: :invalid_date + + """ + @spec from_erl!(:calendar.date(), Calendar.calendar()) :: t + def from_erl!(tuple, calendar \\ Calendar.ISO) do + case from_erl(tuple, calendar) do + {:ok, value} -> + value + + {:error, reason} -> + raise ArgumentError, + "cannot convert #{inspect(tuple)} to date, reason: #{inspect(reason)}" + end + end + + @doc """ + Converts a number of gregorian days to a `Date` struct. + + ## Examples + + iex> Date.from_gregorian_days(1) + ~D[0000-01-02] + iex> Date.from_gregorian_days(730_485) + ~D[2000-01-01] + iex> Date.from_gregorian_days(-1) + ~D[-0001-12-31] + + """ + @doc since: "1.11.0" + @spec from_gregorian_days(integer(), Calendar.calendar()) :: t + def from_gregorian_days(days, calendar \\ Calendar.ISO) when is_integer(days) do + from_iso_days({days, 0}, calendar) + end + + @doc """ + Converts a `date` struct to a number of gregorian days. + + ## Examples + + iex> Date.to_gregorian_days(~D[0000-01-02]) + 1 + iex> Date.to_gregorian_days(~D[2000-01-01]) + 730_485 + iex> Date.to_gregorian_days(~N[2000-01-01 00:00:00]) + 730_485 + + """ + @doc since: "1.11.0" + @spec to_gregorian_days(Calendar.date()) :: integer() + def to_gregorian_days(date) do + {days, _} = to_iso_days(date) + days + end + + @doc """ + Compares two date structs. + + Returns `:gt` if first date is later than the second + and `:lt` for vice versa. If the two dates are equal + `:eq` is returned. + + ## Examples + + iex> Date.compare(~D[2016-04-16], ~D[2016-04-28]) + :lt + + This function can also be used to compare across more + complex calendar types by considering only the date fields: + + iex> Date.compare(~D[2016-04-16], ~N[2016-04-28 01:23:45]) + :lt + iex> Date.compare(~D[2016-04-16], ~N[2016-04-16 01:23:45]) + :eq + iex> Date.compare(~N[2016-04-16 12:34:56], ~N[2016-04-16 01:23:45]) + :eq + + """ + @doc since: "1.4.0" + @spec compare(Calendar.date(), Calendar.date()) :: :lt | :eq | :gt + def compare(%{calendar: calendar} = date1, %{calendar: calendar} = date2) do + %{year: year1, month: month1, day: day1} = date1 + %{year: year2, month: month2, day: day2} = date2 + + case {{year1, month1, day1}, {year2, month2, day2}} do + {first, second} when first > second -> :gt + {first, second} when first < second -> :lt + _ -> :eq + end + end + + def compare(%{calendar: calendar1} = date1, %{calendar: calendar2} = date2) do + if Calendar.compatible_calendars?(calendar1, calendar2) do + case {to_iso_days(date1), to_iso_days(date2)} do + {first, second} when first > second -> :gt + {first, second} when first < second -> :lt + _ -> :eq + end + else + raise ArgumentError, """ + cannot compare #{inspect(date1)} with #{inspect(date2)}. + + This comparison would be ambiguous as their calendars have incompatible day rollover moments. + Specify an exact time of day (using DateTime) to resolve this ambiguity + """ + end + end + + @doc """ + Returns `true` if the first date is strictly earlier than the second. + + ## Examples + + iex> Date.before?(~D[2021-01-01], ~D[2022-02-02]) + true + iex> Date.before?(~D[2021-01-01], ~D[2021-01-01]) + false + iex> Date.before?(~D[2022-02-02], ~D[2021-01-01]) + false + + """ + @doc since: "1.15.0" + @spec before?(Calendar.date(), Calendar.date()) :: boolean() + def before?(date1, date2) do + compare(date1, date2) == :lt + end + + @doc """ + Returns `true` if the first date is strictly later than the second. + + ## Examples + + iex> Date.after?(~D[2022-02-02], ~D[2021-01-01]) + true + iex> Date.after?(~D[2021-01-01], ~D[2021-01-01]) + false + iex> Date.after?(~D[2021-01-01], ~D[2022-02-02]) + false + + """ + @doc since: "1.15.0" + @spec after?(Calendar.date(), Calendar.date()) :: boolean() + def after?(date1, date2) do + compare(date1, date2) == :gt + end + + @doc """ + Converts the given `date` from its calendar to the given `calendar`. + + Returns `{:ok, date}` if the calendars are compatible, + or `{:error, :incompatible_calendars}` if they are not. + + See also `Calendar.compatible_calendars?/2`. + + ## Examples + + Imagine someone implements `Calendar.Holocene`, a calendar based on the + Gregorian calendar that adds exactly 10 000 years to the current Gregorian + year: + + iex> Date.convert(~D[2000-01-01], Calendar.Holocene) + {:ok, %Date{calendar: Calendar.Holocene, year: 12000, month: 1, day: 1}} + + """ + @doc since: "1.5.0" + @spec convert(Calendar.date(), Calendar.calendar()) :: + {:ok, t} | {:error, :incompatible_calendars} + def convert(%{calendar: calendar, year: year, month: month, day: day}, calendar) do + {:ok, %Date{calendar: calendar, year: year, month: month, day: day}} + end + + def convert(%{calendar: calendar} = date, target_calendar) do + if Calendar.compatible_calendars?(calendar, target_calendar) do + result_date = + date + |> to_iso_days() + |> from_iso_days(target_calendar) + + {:ok, result_date} + else + {:error, :incompatible_calendars} + end + end + + @doc """ + Similar to `Date.convert/2`, but raises an `ArgumentError` + if the conversion between the two calendars is not possible. + + ## Examples + + Imagine someone implements `Calendar.Holocene`, a calendar based on the + Gregorian calendar that adds exactly 10 000 years to the current Gregorian + year: + + iex> Date.convert!(~D[2000-01-01], Calendar.Holocene) + %Date{calendar: Calendar.Holocene, year: 12000, month: 1, day: 1} + + """ + @doc since: "1.5.0" + @spec convert!(Calendar.date(), Calendar.calendar()) :: t + def convert!(date, calendar) do + case convert(date, calendar) do + {:ok, value} -> + value + + {:error, reason} -> + raise ArgumentError, + "cannot convert #{inspect(date)} to target calendar #{inspect(calendar)}, " <> + "reason: #{inspect(reason)}" + end + end + + @doc """ + Adds the number of days to the given `date`. + + > #### Prefer `shift/2` {: .info} + > + > Prefer `shift/2` over `add/2`, as it offers a more ergonomic API. + > + > `add/2` always considers a day to be measured according to the + > `Calendar.ISO`. + + The days are counted as Gregorian days, independent of the underlying + calendar. The date is returned in the same calendar as it was given in. + + ## Examples + + iex> Date.add(~D[2000-01-03], -2) + ~D[2000-01-01] + iex> Date.add(~D[2000-01-01], 2) + ~D[2000-01-03] + iex> Date.add(~N[2000-01-01 09:00:00], 2) + ~D[2000-01-03] + iex> Date.add(~D[-0010-01-01], -2) + ~D[-0011-12-30] + + """ + @doc since: "1.5.0" + @spec add(Calendar.date(), integer()) :: t + def add(%{calendar: Calendar.ISO} = date, days) do + %{year: year, month: month, day: day} = date + {year, month, day} = Calendar.ISO.shift_days({year, month, day}, days) + %Date{calendar: Calendar.ISO, year: year, month: month, day: day} + end + + def add(%{calendar: calendar} = date, days) do + {base_days, fraction} = to_iso_days(date) + from_iso_days({base_days + days, fraction}, calendar) + end + + @doc """ + Calculates the difference between two dates, in a full number of days. + + It returns the number of Gregorian days between the dates. Only `Date` + structs that follow the same or compatible calendars can be compared + this way. If two calendars are not compatible, it will raise. + + ## Examples + + iex> Date.diff(~D[2000-01-03], ~D[2000-01-01]) + 2 + iex> Date.diff(~D[2000-01-01], ~D[2000-01-03]) + -2 + iex> Date.diff(~D[0000-01-02], ~D[-0001-12-30]) + 3 + iex> Date.diff(~D[2000-01-01], ~N[2000-01-03 09:00:00]) + -2 + + """ + @doc since: "1.5.0" + @spec diff(Calendar.date(), Calendar.date()) :: integer + def diff(%{calendar: Calendar.ISO} = date1, %{calendar: Calendar.ISO} = date2) do + %{year: year1, month: month1, day: day1} = date1 + %{year: year2, month: month2, day: day2} = date2 + + Calendar.ISO.date_to_iso_days(year1, month1, day1) - + Calendar.ISO.date_to_iso_days(year2, month2, day2) + end + + def diff(%{calendar: calendar1} = date1, %{calendar: calendar2} = date2) do + if Calendar.compatible_calendars?(calendar1, calendar2) do + {days1, _} = to_iso_days(date1) + {days2, _} = to_iso_days(date2) + days1 - days2 + else + raise ArgumentError, + "cannot calculate the difference between #{inspect(date1)} and #{inspect(date2)} because their calendars are not compatible and thus the result would be ambiguous" + end + end + + @doc """ + Shifts given `date` by `duration` according to its calendar. + + Allowed units are: `:year`, `:month`, `:week`, `:day`. + + When using the default ISO calendar, durations are collapsed and + applied in the order of months and then days: + + * when shifting by 1 year and 2 months the date is actually shifted by 14 months + * when shifting by 2 weeks and 3 days the date is shifted by 17 days + + When shifting by month, days are rounded down to the nearest valid date. + + Raises an `ArgumentError` when called with time scale units. + + ## Examples + + iex> Date.shift(~D[2016-01-03], month: 2) + ~D[2016-03-03] + iex> Date.shift(~D[2016-01-30], month: -1) + ~D[2015-12-30] + iex> Date.shift(~D[2016-01-31], year: 4, day: 1) + ~D[2020-02-01] + iex> Date.shift(~D[2016-01-03], Duration.new!(month: 2)) + ~D[2016-03-03] + + # leap years + iex> Date.shift(~D[2024-02-29], year: 1) + ~D[2025-02-28] + iex> Date.shift(~D[2024-02-29], year: 4) + ~D[2028-02-29] + + # rounding down + iex> Date.shift(~D[2015-01-31], month: 1) + ~D[2015-02-28] + + """ + @doc since: "1.17.0" + @spec shift(Calendar.date(), Duration.t() | [unit_pair]) :: t + when unit_pair: {:year, integer} | {:month, integer} | {:week, integer} | {:day, integer} + def shift(%{calendar: calendar} = date, duration) do + %{year: year, month: month, day: day} = date + {year, month, day} = calendar.shift_date(year, month, day, __duration__!(duration)) + %Date{calendar: calendar, year: year, month: month, day: day} + end + + @doc false + def __duration__!(%Duration{} = duration) do + duration + end + + # This part is inlined by the compiler on constant values + def __duration__!(unit_pairs) do + Enum.each(unit_pairs, &validate_duration_unit!/1) + struct!(Duration, unit_pairs) + end + + defp validate_duration_unit!({unit, _value}) + when unit in [:hour, :minute, :second, :microsecond] do + raise ArgumentError, "unsupported unit #{inspect(unit)}. Expected :year, :month, :week, :day" + end + + defp validate_duration_unit!({unit, _value}) when unit not in [:year, :month, :week, :day] do + raise ArgumentError, "unknown unit #{inspect(unit)}. Expected :year, :month, :week, :day" + end + + defp validate_duration_unit!({_unit, value}) when is_integer(value) do + :ok + end + + defp validate_duration_unit!({unit, value}) do + raise ArgumentError, + "unsupported value #{inspect(value)} for #{inspect(unit)}. Expected an integer" + end + + @doc false + def to_iso_days(%{calendar: Calendar.ISO, year: year, month: month, day: day}) do + {Calendar.ISO.date_to_iso_days(year, month, day), {0, 86_400_000_000}} + end + + def to_iso_days(%{calendar: calendar, year: year, month: month, day: day}) do + calendar.naive_datetime_to_iso_days(year, month, day, 0, 0, 0, {0, 0}) + end + + defp from_iso_days({days, _}, Calendar.ISO) do + {year, month, day} = Calendar.ISO.date_from_iso_days(days) + %Date{year: year, month: month, day: day, calendar: Calendar.ISO} + end + + defp from_iso_days(iso_days, target_calendar) do + {year, month, day, _, _, _, _} = target_calendar.naive_datetime_from_iso_days(iso_days) + %Date{year: year, month: month, day: day, calendar: target_calendar} + end + + @doc """ + Calculates the ordinal day of the week of a given `date`. + + Returns the day of the week as an integer. For the ISO 8601 + calendar (the default), it is an integer from 1 to 7, where + 1 is Monday and 7 is Sunday. + + An optional `starting_on` value may be supplied, which + configures the weekday the week starts on. The default value + for it is `:default`, which translates to `:monday` for the + built-in ISO 8601 calendar. Any other weekday may be used for + `starting_on`, in such cases, that weekday will be considered the first + day of the week, and therefore it will be assigned the ordinal number 1. + + The other calendars, the value returned is an ordinal day of week. + For example, `1` may mean "first day of the week" and `7` is + defined to mean "seventh day of the week". Custom calendars may + also accept their own variations of the `starting_on` parameter + with their own meaning. + + ## Examples + + # 2016-10-31 is a Monday and by default Monday is the first day of the week + iex> Date.day_of_week(~D[2016-10-31]) + 1 + iex> Date.day_of_week(~D[2016-11-01]) + 2 + iex> Date.day_of_week(~N[2016-11-01 01:23:45]) + 2 + iex> Date.day_of_week(~D[-0015-10-30]) + 3 + + # 2016-10-31 is a Monday but, as we start the week on Sunday, now it returns 2 + iex> Date.day_of_week(~D[2016-10-31], :sunday) + 2 + iex> Date.day_of_week(~D[2016-11-01], :sunday) + 3 + iex> Date.day_of_week(~N[2016-11-01 01:23:45], :sunday) + 3 + iex> Date.day_of_week(~D[-0015-10-30], :sunday) + 4 + + """ + @doc since: "1.4.0" + @spec day_of_week(Calendar.date(), starting_on :: :default | atom) :: Calendar.day_of_week() + def day_of_week(date, starting_on \\ :default) + + def day_of_week(%{calendar: calendar, year: year, month: month, day: day}, starting_on) do + {day_of_week, _first, _last} = calendar.day_of_week(year, month, day, starting_on) + day_of_week + end + + @doc """ + Calculates a date that is the first day of the week for the given `date`. + + If the day is already the first day of the week, it returns the + day itself. For the built-in ISO calendar, the week starts on Monday. + A weekday rather than `:default` can be given as `starting_on`. + + ## Examples + + iex> Date.beginning_of_week(~D[2020-07-11]) + ~D[2020-07-06] + iex> Date.beginning_of_week(~D[2020-07-06]) + ~D[2020-07-06] + iex> Date.beginning_of_week(~D[2020-07-11], :sunday) + ~D[2020-07-05] + iex> Date.beginning_of_week(~D[2020-07-11], :saturday) + ~D[2020-07-11] + iex> Date.beginning_of_week(~N[2020-07-11 01:23:45]) + ~D[2020-07-06] + + """ + @doc since: "1.11.0" + @spec beginning_of_week(Calendar.date(), starting_on :: :default | atom) :: Date.t() + def beginning_of_week(date, starting_on \\ :default) + + def beginning_of_week(%{calendar: Calendar.ISO} = date, starting_on) do + %{year: year, month: month, day: day} = date + iso_days = Calendar.ISO.date_to_iso_days(year, month, day) + + {year, month, day} = + case Calendar.ISO.iso_days_to_day_of_week(iso_days, starting_on) do + 1 -> + {year, month, day} + + day_of_week -> + Calendar.ISO.date_from_iso_days(iso_days - day_of_week + 1) + end + + %Date{calendar: Calendar.ISO, year: year, month: month, day: day} + end + + def beginning_of_week(%{calendar: calendar} = date, starting_on) do + %{year: year, month: month, day: day} = date + + case calendar.day_of_week(year, month, day, starting_on) do + {day_of_week, day_of_week, _} -> + %Date{calendar: calendar, year: year, month: month, day: day} + + {day_of_week, first_day_of_week, _} -> + add(date, -(day_of_week - first_day_of_week)) + end + end + + @doc """ + Calculates a date that is the last day of the week for the given `date`. + + If the day is already the last day of the week, it returns the + day itself. For the built-in ISO calendar, the week ends on Sunday. + A weekday rather than `:default` can be given as `starting_on`. + + ## Examples + + iex> Date.end_of_week(~D[2020-07-11]) + ~D[2020-07-12] + iex> Date.end_of_week(~D[2020-07-05]) + ~D[2020-07-05] + iex> Date.end_of_week(~D[2020-07-06], :sunday) + ~D[2020-07-11] + iex> Date.end_of_week(~D[2020-07-06], :saturday) + ~D[2020-07-10] + iex> Date.end_of_week(~N[2020-07-11 01:23:45]) + ~D[2020-07-12] + + """ + @doc since: "1.11.0" + @spec end_of_week(Calendar.date(), starting_on :: :default | atom) :: Date.t() + def end_of_week(date, starting_on \\ :default) + + def end_of_week(%{calendar: Calendar.ISO} = date, starting_on) do + %{year: year, month: month, day: day} = date + iso_days = Calendar.ISO.date_to_iso_days(year, month, day) + + {year, month, day} = + case Calendar.ISO.iso_days_to_day_of_week(iso_days, starting_on) do + 7 -> + {year, month, day} + + day_of_week -> + Calendar.ISO.date_from_iso_days(iso_days + 7 - day_of_week) + end + + %Date{calendar: Calendar.ISO, year: year, month: month, day: day} + end + + def end_of_week(%{calendar: calendar} = date, starting_on) do + %{year: year, month: month, day: day} = date + + case calendar.day_of_week(year, month, day, starting_on) do + {day_of_week, _, day_of_week} -> + %Date{calendar: calendar, year: year, month: month, day: day} + + {day_of_week, _, last_day_of_week} -> + add(date, last_day_of_week - day_of_week) + end + end + + @doc """ + Calculates the day of the year of a given `date`. + + Returns the day of the year as an integer. For the ISO 8601 + calendar (the default), it is an integer from 1 to 366. + + ## Examples + + iex> Date.day_of_year(~D[2016-01-01]) + 1 + iex> Date.day_of_year(~D[2016-11-01]) + 306 + iex> Date.day_of_year(~D[-0015-10-30]) + 303 + iex> Date.day_of_year(~D[2004-12-31]) + 366 + + """ + @doc since: "1.8.0" + @spec day_of_year(Calendar.date()) :: Calendar.day() + def day_of_year(date) + + def day_of_year(%{calendar: calendar, year: year, month: month, day: day}) do + calendar.day_of_year(year, month, day) + end + + @doc """ + Calculates the quarter of the year of a given `date`. + + Returns the day of the year as an integer. For the ISO 8601 + calendar (the default), it is an integer from 1 to 4. + + ## Examples + + iex> Date.quarter_of_year(~D[2016-10-31]) + 4 + iex> Date.quarter_of_year(~D[2016-01-01]) + 1 + iex> Date.quarter_of_year(~N[2016-04-01 01:23:45]) + 2 + iex> Date.quarter_of_year(~D[-0015-09-30]) + 3 + + """ + @doc since: "1.8.0" + @spec quarter_of_year(Calendar.date()) :: non_neg_integer() + def quarter_of_year(date) + + def quarter_of_year(%{calendar: calendar, year: year, month: month, day: day}) do + calendar.quarter_of_year(year, month, day) + end + + @doc """ + Calculates the year-of-era and era for a given + calendar year. + + Returns a tuple `{year, era}` representing the + year within the era and the era number. + + ## Examples + + iex> Date.year_of_era(~D[0001-01-01]) + {1, 1} + iex> Date.year_of_era(~D[0000-12-31]) + {1, 0} + iex> Date.year_of_era(~D[-0001-01-01]) + {2, 0} + + """ + @doc since: "1.8.0" + @spec year_of_era(Calendar.date()) :: {Calendar.year(), non_neg_integer()} + def year_of_era(date) + + def year_of_era(%{calendar: calendar, year: year, month: month, day: day}) do + calendar.year_of_era(year, month, day) + end + + @doc """ + Calculates the day-of-era and era for a given + calendar `date`. + + Returns a tuple `{day, era}` representing the + day within the era and the era number. + + ## Examples + + iex> Date.day_of_era(~D[0001-01-01]) + {1, 1} + + iex> Date.day_of_era(~D[0000-12-31]) + {1, 0} + + """ + @doc since: "1.8.0" + @spec day_of_era(Calendar.date()) :: {Calendar.day(), non_neg_integer()} + def day_of_era(date) + + def day_of_era(%{calendar: calendar, year: year, month: month, day: day}) do + calendar.day_of_era(year, month, day) + end + + @doc """ + Calculates a date that is the first day of the month for the given `date`. + + ## Examples + + iex> Date.beginning_of_month(~D[2000-01-31]) + ~D[2000-01-01] + iex> Date.beginning_of_month(~D[2000-01-01]) + ~D[2000-01-01] + iex> Date.beginning_of_month(~N[2000-01-31 01:23:45]) + ~D[2000-01-01] + + """ + @doc since: "1.11.0" + @spec beginning_of_month(Calendar.date()) :: t() + def beginning_of_month(date) + + def beginning_of_month(%{year: year, month: month, calendar: calendar}) do + %Date{year: year, month: month, day: 1, calendar: calendar} + end + + @doc """ + Calculates a date that is the last day of the month for the given `date`. + + ## Examples + + iex> Date.end_of_month(~D[2000-01-01]) + ~D[2000-01-31] + iex> Date.end_of_month(~D[2000-01-31]) + ~D[2000-01-31] + iex> Date.end_of_month(~N[2000-01-01 01:23:45]) + ~D[2000-01-31] + + """ + @doc since: "1.11.0" + @spec end_of_month(Calendar.date()) :: t() + def end_of_month(date) + + def end_of_month(%{year: year, month: month, calendar: calendar} = date) do + day = Date.days_in_month(date) + %Date{year: year, month: month, day: day, calendar: calendar} + end + + ## Helpers + + defimpl String.Chars do + def to_string(%{calendar: calendar, year: year, month: month, day: day}) do + calendar.date_to_string(year, month, day) + end + end + + defimpl Inspect do + def inspect(%{calendar: calendar, year: year, month: month, day: day}, _) + when calendar != Calendar.ISO or year in -9999..9999 do + "~D[" <> calendar.date_to_string(year, month, day) <> suffix(calendar) <> "]" + end + + def inspect(%{calendar: Calendar.ISO, year: year, month: month, day: day}, _) do + "Date.new!(#{Integer.to_string(year)}, #{Integer.to_string(month)}, #{Integer.to_string(day)})" + end + + def inspect(%{calendar: calendar, year: year, month: month, day: day}, _) do + "Date.new!(#{Integer.to_string(year)}, #{Integer.to_string(month)}, #{Integer.to_string(day)}, #{inspect(calendar)})" + end + + defp suffix(Calendar.ISO), do: "" + defp suffix(calendar), do: " " <> inspect(calendar) + end +end diff --git a/lib/elixir/lib/calendar/date_range.ex b/lib/elixir/lib/calendar/date_range.ex new file mode 100644 index 00000000000..5e8c652b781 --- /dev/null +++ b/lib/elixir/lib/calendar/date_range.ex @@ -0,0 +1,235 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +defmodule Date.Range do + @moduledoc """ + Returns an inclusive range between dates. + + Ranges must be created with the `Date.range/2` or `Date.range/3` function. + + The following fields are public: + + * `:first` - the initial date on the range + * `:last` - the last date on the range + * `:step` - (since v1.12.0) the step + + The remaining fields are private and should not be accessed. + """ + + @type t :: %__MODULE__{ + first: Date.t(), + last: Date.t(), + first_in_iso_days: days(), + last_in_iso_days: days(), + step: pos_integer | neg_integer + } + + @typep days() :: integer() + + @enforce_keys [:first, :last, :first_in_iso_days, :last_in_iso_days, :step] + defstruct [:first, :last, :first_in_iso_days, :last_in_iso_days, :step] + + defimpl Enumerable do + def member?( + %Date.Range{ + first: %{calendar: calendar}, + first_in_iso_days: first_days, + last_in_iso_days: last_days, + step: step + } = range, + %Date{calendar: calendar} = date + ) do + {days, _} = Date.to_iso_days(date) + + cond do + empty?(range) -> + {:ok, false} + + first_days <= last_days -> + {:ok, first_days <= days and days <= last_days and rem(days - first_days, step) == 0} + + true -> + {:ok, last_days <= days and days <= first_days and rem(days - first_days, step) == 0} + end + end + + def member?(%Date.Range{step: _}, _) do + {:ok, false} + end + + # TODO: Remove me on v2.0 + def member?( + %{__struct__: Date.Range, first_in_iso_days: first_days, last_in_iso_days: last_days} = + date_range, + date + ) do + step = if first_days <= last_days, do: 1, else: -1 + member?(Map.put(date_range, :step, step), date) + end + + def count(range) do + {:ok, size(range)} + end + + def slice( + %Date.Range{ + first_in_iso_days: first, + first: %{calendar: calendar}, + step: step + } = range + ) do + {:ok, size(range), &slice(first + &1 * step, step + &3 - 1, &2, calendar)} + end + + # TODO: Remove me on v2.0 + def slice( + %{__struct__: Date.Range, first_in_iso_days: first_days, last_in_iso_days: last_days} = + date_range + ) do + step = if first_days <= last_days, do: 1, else: -1 + slice(Map.put(date_range, :step, step)) + end + + defp slice(current, _step, 1, calendar) do + [date_from_iso_days(current, calendar)] + end + + defp slice(current, step, remaining, calendar) when remaining > 1 do + [ + date_from_iso_days(current, calendar) + | slice(current + step, step, remaining - 1, calendar) + ] + end + + def reduce( + %Date.Range{ + first_in_iso_days: first_days, + last_in_iso_days: last_days, + first: %{calendar: calendar}, + step: step + }, + acc, + fun + ) do + reduce(first_days, last_days, acc, fun, step, calendar) + end + + # TODO: Remove me on v2.0 + def reduce( + %{__struct__: Date.Range, first_in_iso_days: first_days, last_in_iso_days: last_days} = + date_range, + acc, + fun + ) do + step = if first_days <= last_days, do: 1, else: -1 + reduce(Map.put(date_range, :step, step), acc, fun) + end + + defp reduce(_first_days, _last_days, {:halt, acc}, _fun, _step, _calendar) do + {:halted, acc} + end + + defp reduce(first_days, last_days, {:suspend, acc}, fun, step, calendar) do + {:suspended, acc, &reduce(first_days, last_days, &1, fun, step, calendar)} + end + + defp reduce(first_days, last_days, {:cont, acc}, fun, step, calendar) + when step > 0 and first_days <= last_days + when step < 0 and first_days >= last_days do + reduce( + first_days + step, + last_days, + fun.(date_from_iso_days(first_days, calendar), acc), + fun, + step, + calendar + ) + end + + defp reduce(_, _, {:cont, acc}, _fun, _step, _calendar) do + {:done, acc} + end + + defp date_from_iso_days(days, Calendar.ISO) do + {year, month, day} = Calendar.ISO.date_from_iso_days(days) + %Date{year: year, month: month, day: day, calendar: Calendar.ISO} + end + + defp date_from_iso_days(days, calendar) do + {year, month, day, _, _, _, _} = + calendar.naive_datetime_from_iso_days({days, {0, 86_400_000_000}}) + + %Date{year: year, month: month, day: day, calendar: calendar} + end + + defp size(%Date.Range{first_in_iso_days: first_days, last_in_iso_days: last_days, step: step}) + when step > 0 and first_days > last_days, + do: 0 + + defp size(%Date.Range{first_in_iso_days: first_days, last_in_iso_days: last_days, step: step}) + when step < 0 and first_days < last_days, + do: 0 + + defp size(%Date.Range{ + first_in_iso_days: first_days, + last_in_iso_days: last_days, + step: step + }), + do: abs(div(last_days - first_days, step)) + 1 + + # TODO: Remove me on v2.0 + defp size( + %{__struct__: Date.Range, first_in_iso_days: first_days, last_in_iso_days: last_days} = + date_range + ) do + step = if first_days <= last_days, do: 1, else: -1 + size(Map.put(date_range, :step, step)) + end + + defp empty?(%Date.Range{ + first_in_iso_days: first_days, + last_in_iso_days: last_days, + step: step + }) + when step > 0 and first_days > last_days, + do: true + + defp empty?(%Date.Range{ + first_in_iso_days: first_days, + last_in_iso_days: last_days, + step: step + }) + when step < 0 and first_days < last_days, + do: true + + defp empty?(%Date.Range{step: _}), do: false + + # TODO: Remove me on v2.0 + defp empty?( + %{__struct__: Date.Range, first_in_iso_days: first_days, last_in_iso_days: last_days} = + date_range + ) do + step = if first_days <= last_days, do: 1, else: -1 + empty?(Map.put(date_range, :step, step)) + end + end + + defimpl Inspect do + import Kernel, except: [inspect: 2] + + def inspect(%Date.Range{first: first, last: last, step: 1}, _) do + "Date.range(" <> inspect(first) <> ", " <> inspect(last) <> ")" + end + + def inspect(%Date.Range{first: first, last: last, step: step}, _) do + "Date.range(" <> inspect(first) <> ", " <> inspect(last) <> ", #{step})" + end + + # TODO: Remove me on v2.0 + def inspect(%{__struct__: Date.Range, first: first, last: last} = date_range, opts) do + step = if first <= last, do: 1, else: -1 + inspect(Map.put(date_range, :step, step), opts) + end + end +end diff --git a/lib/elixir/lib/calendar/datetime.ex b/lib/elixir/lib/calendar/datetime.ex new file mode 100644 index 00000000000..6cf43e37766 --- /dev/null +++ b/lib/elixir/lib/calendar/datetime.ex @@ -0,0 +1,2166 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +defmodule DateTime do + @moduledoc """ + A datetime implementation with a time zone. + + This datetime can be seen as a snapshot of a date and time + at a given time zone. For such purposes, it also includes both + UTC and Standard offsets, as well as the zone abbreviation + field used exclusively for formatting purposes. Note future + datetimes are not necessarily guaranteed to exist, as time + zones may change any time in the future due to geopolitical + reasons. See the "Datetimes as snapshots" section for more + information. + + Remember, comparisons in Elixir using `==/2`, `>/2`, ` Enum.min([~U[2022-01-12 00:01:00.00Z], ~U[2021-01-12 00:01:00.00Z]], DateTime) + ~U[2021-01-12 00:01:00.00Z] + + Developers should avoid creating the `DateTime` struct directly + and instead rely on the functions provided by this module as + well as the ones in third-party calendar libraries. + + ## Time zone database + + Many functions in this module require a time zone database. + A time zone database is a record of the UTC offsets that its locales have + used at various times in the past, are using, and are expected to use in the + future. + Because those plans can change, it needs to be periodically updated. + + By default, `DateTime` uses the default time zone database returned by + `Calendar.get_time_zone_database/0`, which defaults to + `Calendar.UTCOnlyTimeZoneDatabase` which only handles "Etc/UTC" + datetimes and returns `{:error, :utc_only_time_zone_database}` + for any other time zone. + + Other time zone databases can also be configured. Here are some + available options and libraries: + + * [`time_zone_info`](https://github.com/hrzndhrn/time_zone_info) + * [`tz`](https://github.com/mathieuprog/tz) + * [`tzdata`](https://github.com/lau/tzdata) + * [`zoneinfo`](https://github.com/smartrent/zoneinfo) - + recommended for embedded devices + + To use one of them, first make sure it is added as a dependency in `mix.exs`. + It can then be configured either via configuration: + + config :elixir, :time_zone_database, Tz.TimeZoneDatabase + + or by calling `Calendar.put_time_zone_database/1`: + + Calendar.put_time_zone_database(Tz.TimeZoneDatabase) + + See the proper names in the library installation instructions. + + ## Datetimes as snapshots + + In the first section, we described datetimes as a "snapshot of + a date and time at a given time zone". To understand precisely + what we mean, let's see an example. + + Imagine someone in Poland who wants to schedule a meeting with someone + in Brazil in the next year. The meeting will happen at 2:30 AM + in the Polish time zone. At what time will the meeting happen in + Brazil? + + You can consult the time zone database today, one year before, + using the API in this module and it will give you an answer that + is valid right now. However, this answer may not be valid in the + future. Why? Because both Brazil and Poland may change their timezone + rules, ultimately affecting the result. For example, a country may + choose to enter or abandon "Daylight Saving Time", which is a + process where we adjust the clock one hour forward or one hour + back once per year. Whenever the rules change, the exact instant + that 2:30 AM in Polish time will be in Brazil may change. + + In other words, whenever working with future DateTimes, there is + no guarantee the results you get will always be correct, until + the event actually happens. Therefore, when you ask for a future + time, the answers you get are a snapshot that reflects the current + state of the time zone rules. For datetimes in the past, this is + not a problem, because time zone rules do not change for past + events. + + To make matters worse, it may be that 2:30 AM in Polish time + does not actually even exist or it is ambiguous. If a certain + time zone observes "Daylight Saving Time", they will move their + clock forward once a year. When this happens, there is a whole + hour that does not exist. Then, when they move the clock back, + there is a certain hour that will happen twice. So if you want to + schedule a meeting when this shift back happens, you would need to + explicitly say which occurrence of 2:30 AM you mean: the one in + "Summer Time", which occurs before the shift, or the one + in "Standard Time", which occurs after it. Applications that are + date and time sensitive need to take these scenarios into account + and correctly communicate them to users. + + The good news is: Elixir contains all of the building blocks + necessary to tackle those problems. The default timezone database + used by Elixir, `Calendar.UTCOnlyTimeZoneDatabase`, only works + with UTC, which does not observe those issues. Once you bring + a proper time zone database, the functions in this module will + query the database and return the relevant information. For + example, look at how `DateTime.new/4` returns different results + based on the scenarios described in this section. + + ## Converting between timezones + + Bearing in mind the cautions above, and assuming you've brought in a full + timezone database, here are some examples of common shifts between time + zones. + + # Local time to UTC + new_york = DateTime.from_naive!(~N[2023-06-26T09:30:00], "America/New_York") + #=> #DateTime<2023-06-26 09:30:00-04:00 EDT America/New_York> + + utc = DateTime.shift_zone!(new_york, "Etc/UTC") + #=> ~U[2023-06-26 13:30:00Z] + + # UTC to local time + DateTime.shift_zone!(utc, "Europe/Paris") + #=> #DateTime<2023-06-26 15:30:00+02:00 CEST Europe/Paris> + + """ + + @enforce_keys [:year, :month, :day, :hour, :minute, :second] ++ + [:time_zone, :zone_abbr, :utc_offset, :std_offset] + + defstruct [ + :year, + :month, + :day, + :hour, + :minute, + :second, + :time_zone, + :zone_abbr, + :utc_offset, + :std_offset, + microsecond: {0, 0}, + calendar: Calendar.ISO + ] + + @type t :: %__MODULE__{ + year: Calendar.year(), + month: Calendar.month(), + day: Calendar.day(), + calendar: Calendar.calendar(), + hour: Calendar.hour(), + minute: Calendar.minute(), + second: Calendar.second(), + microsecond: Calendar.microsecond(), + time_zone: Calendar.time_zone(), + zone_abbr: Calendar.zone_abbr(), + utc_offset: Calendar.utc_offset(), + std_offset: Calendar.std_offset() + } + + @unix_days :calendar.date_to_gregorian_days({1970, 1, 1}) + @seconds_per_day 24 * 60 * 60 + + @doc """ + Returns the current datetime in UTC. + + If you want the current time in Unix seconds, + use `System.os_time/1` instead. + + You can also pass a time unit to automatically + truncate the resulting datetime. This is available + since v1.15.0. + + The default unit if none gets passed is `:native`, + which results on a default resolution of microseconds. + + ## Examples + + iex> datetime = DateTime.utc_now() + iex> datetime.time_zone + "Etc/UTC" + + iex> datetime = DateTime.utc_now(:second) + iex> datetime.microsecond + {0, 0} + + """ + @spec utc_now(Calendar.calendar() | :native | :microsecond | :millisecond | :second) :: t + def utc_now(calendar_or_time_unit \\ Calendar.ISO) do + case calendar_or_time_unit do + unit when unit in [:microsecond, :millisecond, :second, :native] -> + utc_now(unit, Calendar.ISO) + + calendar -> + System.os_time() |> from_unix!(:native, calendar) + end + end + + @doc """ + Returns the current datetime in UTC, supporting + a specific calendar and precision. + + If you want the current time in Unix seconds, + use `System.os_time/1` instead. + + ## Examples + + iex> datetime = DateTime.utc_now(:microsecond, Calendar.ISO) + iex> datetime.time_zone + "Etc/UTC" + + iex> datetime = DateTime.utc_now(:second, Calendar.ISO) + iex> datetime.microsecond + {0, 0} + + """ + @doc since: "1.15.0" + @spec utc_now(:native | :microsecond | :millisecond | :second, Calendar.calendar()) :: t + def utc_now(time_unit, calendar) + when time_unit in [:native, :microsecond, :millisecond, :second] do + System.os_time(time_unit) |> from_unix!(time_unit, calendar) + end + + @doc """ + Builds a datetime from date and time structs. + + It expects a time zone to put the `DateTime` in. + If the time zone is not passed it will default to `"Etc/UTC"`, + which always succeeds. Otherwise, the `DateTime` is checked against the time zone database + given as `time_zone_database`. See the "Time zone database" + section in the module documentation. + + ## Examples + + iex> DateTime.new(~D[2016-05-24], ~T[13:26:08.003], "Etc/UTC") + {:ok, ~U[2016-05-24 13:26:08.003Z]} + + When the datetime is ambiguous - for instance during changing from summer + to winter time - the two possible valid datetimes are returned in a tuple. + The first datetime is also the one which comes first chronologically, while + the second one comes last. + + iex> {:ambiguous, first_dt, second_dt} = DateTime.new(~D[2018-10-28], ~T[02:30:00], "Europe/Copenhagen", FakeTimeZoneDatabase) + iex> first_dt + #DateTime<2018-10-28 02:30:00+02:00 CEST Europe/Copenhagen> + iex> second_dt + #DateTime<2018-10-28 02:30:00+01:00 CET Europe/Copenhagen> + + When there is a gap in wall time - for instance in spring when the clocks are + turned forward - the latest valid datetime just before the gap and the first + valid datetime just after the gap. + + iex> {:gap, just_before, just_after} = DateTime.new(~D[2019-03-31], ~T[02:30:00], "Europe/Copenhagen", FakeTimeZoneDatabase) + iex> just_before + #DateTime<2019-03-31 01:59:59.999999+01:00 CET Europe/Copenhagen> + iex> just_after + #DateTime<2019-03-31 03:00:00+02:00 CEST Europe/Copenhagen> + + Most of the time there is one, and just one, valid datetime for a certain + date and time in a certain time zone. + + iex> {:ok, datetime} = DateTime.new(~D[2018-07-28], ~T[12:30:00], "Europe/Copenhagen", FakeTimeZoneDatabase) + iex> datetime + #DateTime<2018-07-28 12:30:00+02:00 CEST Europe/Copenhagen> + + """ + @doc since: "1.11.0" + @spec new(Date.t(), Time.t(), Calendar.time_zone(), Calendar.time_zone_database()) :: + {:ok, t} + | {:ambiguous, first_datetime :: t, second_datetime :: t} + | {:gap, t, t} + | {:error, + :incompatible_calendars | :time_zone_not_found | :utc_only_time_zone_database} + def new( + date, + time, + time_zone \\ "Etc/UTC", + time_zone_database \\ Calendar.get_time_zone_database() + ) + + def new(%Date{calendar: calendar} = date, %Time{calendar: calendar} = time, "Etc/UTC", _db) do + %{year: year, month: month, day: day} = date + %{hour: hour, minute: minute, second: second, microsecond: microsecond} = time + + datetime = %DateTime{ + calendar: calendar, + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond, + std_offset: 0, + utc_offset: 0, + zone_abbr: "UTC", + time_zone: "Etc/UTC" + } + + {:ok, datetime} + end + + def new(date, time, time_zone, time_zone_database) do + with {:ok, naive_datetime} <- NaiveDateTime.new(date, time) do + from_naive(naive_datetime, time_zone, time_zone_database) + end + end + + @doc """ + Builds a datetime from date and time structs, raising on errors. + + It expects a time zone to put the `DateTime` in. + If the time zone is not passed it will default to `"Etc/UTC"`, + which always succeeds. Otherwise, the DateTime is checked against the time zone database + given as `time_zone_database`. See the "Time zone database" + section in the module documentation. + + ## Examples + + iex> DateTime.new!(~D[2016-05-24], ~T[13:26:08.003], "Etc/UTC") + ~U[2016-05-24 13:26:08.003Z] + + When the datetime is ambiguous - for instance during changing from summer + to winter time - an error will be raised. + + iex> DateTime.new!(~D[2018-10-28], ~T[02:30:00], "Europe/Copenhagen", FakeTimeZoneDatabase) + ** (ArgumentError) cannot build datetime with ~D[2018-10-28] and ~T[02:30:00] because such instant is ambiguous in time zone Europe/Copenhagen as there is an overlap between #DateTime<2018-10-28 02:30:00+02:00 CEST Europe/Copenhagen> and #DateTime<2018-10-28 02:30:00+01:00 CET Europe/Copenhagen> + + When there is a gap in wall time - for instance in spring when the clocks are + turned forward - an error will be raised. + + iex> DateTime.new!(~D[2019-03-31], ~T[02:30:00], "Europe/Copenhagen", FakeTimeZoneDatabase) + ** (ArgumentError) cannot build datetime with ~D[2019-03-31] and ~T[02:30:00] because such instant does not exist in time zone Europe/Copenhagen as there is a gap between #DateTime<2019-03-31 01:59:59.999999+01:00 CET Europe/Copenhagen> and #DateTime<2019-03-31 03:00:00+02:00 CEST Europe/Copenhagen> + + Most of the time there is one, and just one, valid datetime for a certain + date and time in a certain time zone. + + iex> datetime = DateTime.new!(~D[2018-07-28], ~T[12:30:00], "Europe/Copenhagen", FakeTimeZoneDatabase) + iex> datetime + #DateTime<2018-07-28 12:30:00+02:00 CEST Europe/Copenhagen> + + """ + @doc since: "1.11.0" + @spec new!(Date.t(), Time.t(), Calendar.time_zone(), Calendar.time_zone_database()) :: t + def new!( + date, + time, + time_zone \\ "Etc/UTC", + time_zone_database \\ Calendar.get_time_zone_database() + ) + + def new!(date, time, time_zone, time_zone_database) do + case new(date, time, time_zone, time_zone_database) do + {:ok, datetime} -> + datetime + + {:ambiguous, dt1, dt2} -> + raise ArgumentError, + "cannot build datetime with #{inspect(date)} and #{inspect(time)} because such " <> + "instant is ambiguous in time zone #{time_zone} as there is an overlap " <> + "between #{inspect(dt1)} and #{inspect(dt2)}" + + {:gap, dt1, dt2} -> + raise ArgumentError, + "cannot build datetime with #{inspect(date)} and #{inspect(time)} because such " <> + "instant does not exist in time zone #{time_zone} as there is a gap " <> + "between #{inspect(dt1)} and #{inspect(dt2)}" + + {:error, reason} -> + raise ArgumentError, + "cannot build datetime with #{inspect(date)} and #{inspect(time)}, reason: #{inspect(reason)}" + end + end + + @doc """ + Converts the given Unix time to `DateTime`. + + The integer can be given in different unit, according to `System.convert_time_unit/3`, + and it will be converted to microseconds internally, which is the maximum precision + supported by `DateTime`. In other words, any precision higher than microseconds will + lead to truncation. + + Unix times are always in UTC. Therefore the DateTime will be returned in UTC. + + ## Examples + + iex> {:ok, datetime} = DateTime.from_unix(1_464_096_368) + iex> datetime + ~U[2016-05-24 13:26:08Z] + + iex> {:ok, datetime} = DateTime.from_unix(1_432_560_368_868_569, :microsecond) + iex> datetime + ~U[2015-05-25 13:26:08.868569Z] + + iex> {:ok, datetime} = DateTime.from_unix(253_402_300_799) + iex> datetime + ~U[9999-12-31 23:59:59Z] + + iex> {:error, :invalid_unix_time} = DateTime.from_unix(253_402_300_800) + + The unit can also be an integer as in `t:System.time_unit/0`: + + iex> {:ok, datetime} = DateTime.from_unix(143_256_036_886_856, 1024) + iex> datetime + ~U[6403-03-17 07:05:22.320312Z] + + Negative Unix times are supported up to -377705116800 seconds: + + iex> {:ok, datetime} = DateTime.from_unix(-377_705_116_800) + iex> datetime + ~U[-9999-01-01 00:00:00Z] + + iex> {:error, :invalid_unix_time} = DateTime.from_unix(-377_705_116_801) + + """ + @spec from_unix(integer, :native | System.time_unit(), Calendar.calendar()) :: + {:ok, t} | {:error, atom} + def from_unix(integer, unit \\ :second, calendar \\ Calendar.ISO) when is_integer(integer) do + case Calendar.ISO.from_unix(integer, unit) do + {:ok, {year, month, day}, {hour, minute, second}, microsecond} -> + iso_datetime = %DateTime{ + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond, + std_offset: 0, + utc_offset: 0, + zone_abbr: "UTC", + time_zone: "Etc/UTC" + } + + convert(iso_datetime, calendar) + + {:error, _} = error -> + error + end + end + + @doc """ + Converts the given Unix time to `DateTime`. + + The integer can be given in different unit + according to `System.convert_time_unit/3` and it will + be converted to microseconds internally. + + Unix times are always in UTC and therefore the DateTime + will be returned in UTC. + + ## Examples + + # An easy way to get the Unix epoch is passing 0 to this function + iex> DateTime.from_unix!(0) + ~U[1970-01-01 00:00:00Z] + + iex> DateTime.from_unix!(1_464_096_368) + ~U[2016-05-24 13:26:08Z] + + iex> DateTime.from_unix!(1_432_560_368_868_569, :microsecond) + ~U[2015-05-25 13:26:08.868569Z] + + iex> DateTime.from_unix!(143_256_036_886_856, 1024) + ~U[6403-03-17 07:05:22.320312Z] + + """ + @spec from_unix!(integer, :native | System.time_unit(), Calendar.calendar()) :: t + def from_unix!(integer, unit \\ :second, calendar \\ Calendar.ISO) do + case from_unix(integer, unit, calendar) do + {:ok, datetime} -> + datetime + + {:error, :invalid_unix_time} -> + raise ArgumentError, "invalid Unix time #{integer}" + end + end + + @doc """ + Converts the given `NaiveDateTime` to `DateTime`. + + It expects a time zone to put the `NaiveDateTime` in. + If the time zone is "Etc/UTC", it always succeeds. Otherwise, + the NaiveDateTime is checked against the time zone database + given as `time_zone_database`. See the "Time zone database" + section in the module documentation. + + ## Examples + + iex> DateTime.from_naive(~N[2016-05-24 13:26:08.003], "Etc/UTC") + {:ok, ~U[2016-05-24 13:26:08.003Z]} + + When the datetime is ambiguous - for instance during changing from summer + to winter time - the two possible valid datetimes are returned in a tuple. + The first datetime is also the one which comes first chronologically, while + the second one comes last. + + iex> {:ambiguous, first_dt, second_dt} = DateTime.from_naive(~N[2018-10-28 02:30:00], "Europe/Copenhagen", FakeTimeZoneDatabase) + iex> first_dt + #DateTime<2018-10-28 02:30:00+02:00 CEST Europe/Copenhagen> + iex> second_dt + #DateTime<2018-10-28 02:30:00+01:00 CET Europe/Copenhagen> + + When there is a gap in wall time - for instance in spring when the clocks are + turned forward - the latest valid datetime just before the gap and the first + valid datetime just after the gap. + + iex> {:gap, just_before, just_after} = DateTime.from_naive(~N[2019-03-31 02:30:00], "Europe/Copenhagen", FakeTimeZoneDatabase) + iex> just_before + #DateTime<2019-03-31 01:59:59.999999+01:00 CET Europe/Copenhagen> + iex> just_after + #DateTime<2019-03-31 03:00:00+02:00 CEST Europe/Copenhagen> + + Most of the time there is one, and just one, valid datetime for a certain + date and time in a certain time zone. + + iex> {:ok, datetime} = DateTime.from_naive(~N[2018-07-28 12:30:00], "Europe/Copenhagen", FakeTimeZoneDatabase) + iex> datetime + #DateTime<2018-07-28 12:30:00+02:00 CEST Europe/Copenhagen> + + This function accepts any map or struct that contains at least the same fields as a `NaiveDateTime` + struct. The most common example of that is a `DateTime`. In this case the information about the time + zone of that `DateTime` is completely ignored. This is the same principle as passing a `DateTime` to + `Date.to_iso8601/2`. `Date.to_iso8601/2` extracts only the date-specific fields (calendar, year, + month and day) of the given structure and ignores all others. + + This way if you have a `DateTime` in one time zone, you can get the same wall time in another time zone. + For instance if you have 2018-08-24 10:00:00 in Copenhagen and want a `DateTime` for 2018-08-24 10:00:00 + in UTC you can do: + + iex> cph_datetime = DateTime.from_naive!(~N[2018-08-24 10:00:00], "Europe/Copenhagen", FakeTimeZoneDatabase) + iex> {:ok, utc_datetime} = DateTime.from_naive(cph_datetime, "Etc/UTC", FakeTimeZoneDatabase) + iex> utc_datetime + ~U[2018-08-24 10:00:00Z] + + If instead you want a `DateTime` for the same point time in a different time zone see the + `DateTime.shift_zone/3` function which would convert 2018-08-24 10:00:00 in Copenhagen + to 2018-08-24 08:00:00 in UTC. + """ + @doc since: "1.4.0" + @spec from_naive( + Calendar.naive_datetime(), + Calendar.time_zone(), + Calendar.time_zone_database() + ) :: + {:ok, t} + | {:ambiguous, first_datetime :: t, second_datetime :: t} + | {:gap, t, t} + | {:error, + :incompatible_calendars | :time_zone_not_found | :utc_only_time_zone_database} + + def from_naive( + naive_datetime, + time_zone, + time_zone_database \\ Calendar.get_time_zone_database() + ) + + def from_naive(naive_datetime, "Etc/UTC", _) do + utc_period = %{std_offset: 0, utc_offset: 0, zone_abbr: "UTC"} + {:ok, from_naive_with_period(naive_datetime, "Etc/UTC", utc_period)} + end + + def from_naive(%{calendar: Calendar.ISO} = naive_datetime, time_zone, time_zone_database) do + case time_zone_database.time_zone_periods_from_wall_datetime(naive_datetime, time_zone) do + {:ok, period} -> + {:ok, from_naive_with_period(naive_datetime, time_zone, period)} + + {:ambiguous, first_period, second_period} -> + first_datetime = from_naive_with_period(naive_datetime, time_zone, first_period) + second_datetime = from_naive_with_period(naive_datetime, time_zone, second_period) + {:ambiguous, first_datetime, second_datetime} + + {:gap, {first_period, first_period_until_wall}, {second_period, second_period_from_wall}} -> + # `until_wall` is not valid, but any time just before is. + # So by subtracting a second and adding .999999 seconds + # we get the last microsecond just before. + before_naive = + first_period_until_wall + |> Map.replace!(:microsecond, {999_999, 6}) + |> NaiveDateTime.add(-1) + + after_naive = second_period_from_wall + + latest_datetime_before = from_naive_with_period(before_naive, time_zone, first_period) + first_datetime_after = from_naive_with_period(after_naive, time_zone, second_period) + {:gap, latest_datetime_before, first_datetime_after} + + {:error, _} = error -> + error + end + end + + def from_naive(%{calendar: calendar} = naive_datetime, time_zone, time_zone_database) + when calendar != Calendar.ISO do + # For non-ISO calendars, convert to ISO, create ISO DateTime, and then + # convert to original calendar + iso_result = + with {:ok, in_iso} <- NaiveDateTime.convert(naive_datetime, Calendar.ISO) do + from_naive(in_iso, time_zone, time_zone_database) + end + + case iso_result do + {:ok, dt} -> + convert(dt, calendar) + + {:ambiguous, dt1, dt2} -> + with {:ok, dt1converted} <- convert(dt1, calendar), + {:ok, dt2converted} <- convert(dt2, calendar), + do: {:ambiguous, dt1converted, dt2converted} + + {:gap, dt1, dt2} -> + with {:ok, dt1converted} <- convert(dt1, calendar), + {:ok, dt2converted} <- convert(dt2, calendar), + do: {:gap, dt1converted, dt2converted} + + {:error, _} = error -> + error + end + end + + defp from_naive_with_period(naive_datetime, time_zone, period) do + %{std_offset: std_offset, utc_offset: utc_offset, zone_abbr: zone_abbr} = period + + %{ + calendar: calendar, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond, + year: year, + month: month, + day: day + } = naive_datetime + + %DateTime{ + calendar: calendar, + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond, + std_offset: std_offset, + utc_offset: utc_offset, + zone_abbr: zone_abbr, + time_zone: time_zone + } + end + + @doc """ + Converts the given `NaiveDateTime` to `DateTime`. + + It expects a time zone to put the NaiveDateTime in. + If the time zone is "Etc/UTC", it always succeeds. Otherwise, + the NaiveDateTime is checked against the time zone database + given as `time_zone_database`. See the "Time zone database" + section in the module documentation. + + ## Examples + + iex> DateTime.from_naive!(~N[2016-05-24 13:26:08.003], "Etc/UTC") + ~U[2016-05-24 13:26:08.003Z] + + iex> DateTime.from_naive!(~N[2018-05-24 13:26:08.003], "Europe/Copenhagen", FakeTimeZoneDatabase) + #DateTime<2018-05-24 13:26:08.003+02:00 CEST Europe/Copenhagen> + + """ + @doc since: "1.4.0" + @spec from_naive!( + NaiveDateTime.t(), + Calendar.time_zone(), + Calendar.time_zone_database() + ) :: t + def from_naive!( + naive_datetime, + time_zone, + time_zone_database \\ Calendar.get_time_zone_database() + ) do + case from_naive(naive_datetime, time_zone, time_zone_database) do + {:ok, datetime} -> + datetime + + {:ambiguous, dt1, dt2} -> + raise ArgumentError, + "cannot convert #{inspect(naive_datetime)} to datetime because such " <> + "instant is ambiguous in time zone #{time_zone} as there is an overlap " <> + "between #{inspect(dt1)} and #{inspect(dt2)}" + + {:gap, dt1, dt2} -> + raise ArgumentError, + "cannot convert #{inspect(naive_datetime)} to datetime because such " <> + "instant does not exist in time zone #{time_zone} as there is a gap " <> + "between #{inspect(dt1)} and #{inspect(dt2)}" + + {:error, reason} -> + raise ArgumentError, + "cannot convert #{inspect(naive_datetime)} to datetime, reason: #{inspect(reason)}" + end + end + + @doc """ + Changes the time zone of a `DateTime`. + + Returns a `DateTime` for the same point in time, but instead at + the time zone provided. It assumes that `DateTime` is valid and + exists in the given time zone and calendar. + + By default, it uses the default time zone database returned by + `Calendar.get_time_zone_database/0`, which defaults to + `Calendar.UTCOnlyTimeZoneDatabase` which only handles "Etc/UTC" datetimes. + Other time zone databases can be passed as argument or set globally. + See the "Time zone database" section in the module docs. + + ## Examples + + iex> {:ok, pacific_datetime} = DateTime.shift_zone(~U[2018-07-16 10:00:00Z], "America/Los_Angeles", FakeTimeZoneDatabase) + iex> pacific_datetime + #DateTime<2018-07-16 03:00:00-07:00 PDT America/Los_Angeles> + + iex> DateTime.shift_zone(~U[2018-07-16 10:00:00Z], "bad timezone", FakeTimeZoneDatabase) + {:error, :time_zone_not_found} + + """ + @doc since: "1.8.0" + @spec shift_zone(t, Calendar.time_zone(), Calendar.time_zone_database()) :: + {:ok, t} | {:error, :time_zone_not_found | :utc_only_time_zone_database} + def shift_zone(datetime, time_zone, time_zone_database \\ Calendar.get_time_zone_database()) + + def shift_zone(%{time_zone: time_zone} = datetime, time_zone, _) do + {:ok, datetime} + end + + def shift_zone(datetime, time_zone, time_zone_database) do + %{ + std_offset: std_offset, + utc_offset: utc_offset, + calendar: calendar, + microsecond: {_, precision} + } = datetime + + datetime + |> to_iso_days() + |> apply_tz_offset(utc_offset + std_offset) + |> shift_zone_for_iso_days_utc(calendar, precision, time_zone, time_zone_database) + end + + defp shift_zone_for_iso_days_utc(iso_days_utc, calendar, precision, time_zone, time_zone_db) do + case time_zone_db.time_zone_period_from_utc_iso_days(iso_days_utc, time_zone) do + {:ok, %{std_offset: std_offset, utc_offset: utc_offset, zone_abbr: zone_abbr}} -> + {year, month, day, hour, minute, second, {microsecond_without_precision, _}} = + iso_days_utc + |> apply_tz_offset(-(utc_offset + std_offset)) + |> calendar.naive_datetime_from_iso_days() + + datetime = %DateTime{ + calendar: calendar, + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: {microsecond_without_precision, precision}, + std_offset: std_offset, + utc_offset: utc_offset, + zone_abbr: zone_abbr, + time_zone: time_zone + } + + {:ok, datetime} + + {:error, _} = error -> + error + end + end + + @doc """ + Changes the time zone of a `DateTime` or raises on errors. + + See `shift_zone/3` for more information. + + ## Examples + + iex> DateTime.shift_zone!(~U[2018-07-16 10:00:00Z], "America/Los_Angeles", FakeTimeZoneDatabase) + #DateTime<2018-07-16 03:00:00-07:00 PDT America/Los_Angeles> + + iex> DateTime.shift_zone!(~U[2018-07-16 10:00:00Z], "bad timezone", FakeTimeZoneDatabase) + ** (ArgumentError) cannot shift ~U[2018-07-16 10:00:00Z] to "bad timezone" time zone, reason: :time_zone_not_found + + """ + @doc since: "1.10.0" + @spec shift_zone!(t, Calendar.time_zone(), Calendar.time_zone_database()) :: t + def shift_zone!(datetime, time_zone, time_zone_database \\ Calendar.get_time_zone_database()) do + case shift_zone(datetime, time_zone, time_zone_database) do + {:ok, datetime} -> + datetime + + {:error, reason} -> + raise ArgumentError, + "cannot shift #{inspect(datetime)} to #{inspect(time_zone)} time zone" <> + ", reason: #{inspect(reason)}" + end + end + + @doc """ + Returns the current datetime in the provided time zone. + + By default, it uses the default time_zone returned by + `Calendar.get_time_zone_database/0`, which defaults to + `Calendar.UTCOnlyTimeZoneDatabase` which only handles "Etc/UTC" datetimes. + Other time zone databases can be passed as argument or set globally. + See the "Time zone database" section in the module docs. + + ## Examples + + iex> {:ok, datetime} = DateTime.now("Etc/UTC") + iex> datetime.time_zone + "Etc/UTC" + + iex> DateTime.now("Europe/Copenhagen") + {:error, :utc_only_time_zone_database} + + iex> DateTime.now("bad timezone", FakeTimeZoneDatabase) + {:error, :time_zone_not_found} + + """ + @doc since: "1.8.0" + @spec now(Calendar.time_zone(), Calendar.time_zone_database()) :: + {:ok, t} | {:error, :time_zone_not_found | :utc_only_time_zone_database} + def now(time_zone, time_zone_database \\ Calendar.get_time_zone_database()) + + def now("Etc/UTC", _) do + {:ok, utc_now()} + end + + def now(time_zone, time_zone_database) do + shift_zone(utc_now(), time_zone, time_zone_database) + end + + @doc """ + Returns the current datetime in the provided time zone or raises on errors + + See `now/2` for more information. + + ## Examples + + iex> datetime = DateTime.now!("Etc/UTC") + iex> datetime.time_zone + "Etc/UTC" + + iex> DateTime.now!("Europe/Copenhagen") + ** (ArgumentError) cannot get current datetime in "Europe/Copenhagen" time zone, reason: :utc_only_time_zone_database + + iex> DateTime.now!("bad timezone", FakeTimeZoneDatabase) + ** (ArgumentError) cannot get current datetime in "bad timezone" time zone, reason: :time_zone_not_found + + """ + @doc since: "1.10.0" + @spec now!(Calendar.time_zone(), Calendar.time_zone_database()) :: t + def now!(time_zone, time_zone_database \\ Calendar.get_time_zone_database()) do + case now(time_zone, time_zone_database) do + {:ok, datetime} -> + datetime + + {:error, reason} -> + raise ArgumentError, + "cannot get current datetime in #{inspect(time_zone)} time zone, reason: " <> + inspect(reason) + end + end + + @doc """ + Converts the given `datetime` to Unix time. + + The `datetime` is expected to be using the ISO calendar + with a year greater than or equal to 0. + + It will return the integer with the given unit, according + to `System.convert_time_unit/3`. If the given unit is different + than microseconds, the returned value will be either truncated + or padded accordingly. + + ## Examples + + iex> 1_464_096_368 |> DateTime.from_unix!() |> DateTime.to_unix() + 1464096368 + + iex> dt = %DateTime{calendar: Calendar.ISO, day: 20, hour: 18, microsecond: {273806, 6}, + ...> minute: 58, month: 11, second: 19, time_zone: "America/Montevideo", + ...> utc_offset: -10800, std_offset: 3600, year: 2014, zone_abbr: "UYST"} + iex> DateTime.to_unix(dt) + 1416517099 + + iex> flamel = %DateTime{calendar: Calendar.ISO, day: 22, hour: 8, microsecond: {527771, 6}, + ...> minute: 2, month: 3, second: 25, std_offset: 0, time_zone: "Etc/UTC", + ...> utc_offset: 0, year: 1418, zone_abbr: "UTC"} + iex> DateTime.to_unix(flamel) + -17412508655 + + """ + @spec to_unix(Calendar.datetime(), :native | System.time_unit()) :: integer + def to_unix(datetime, unit \\ :second) + + def to_unix(%{utc_offset: utc_offset, std_offset: std_offset} = datetime, unit) do + {days, fraction} = to_iso_days(datetime) + unix_units = Calendar.ISO.iso_days_to_unit({days - @unix_days, fraction}, unit) + offset_units = System.convert_time_unit(utc_offset + std_offset, :second, unit) + unix_units - offset_units + end + + @doc """ + Converts the given `datetime` into a `NaiveDateTime`. + + Because `NaiveDateTime` does not hold time zone information, + any time zone related data will be lost during the conversion. + + ## Examples + + iex> dt = %DateTime{year: 2000, month: 2, day: 29, zone_abbr: "CET", + ...> hour: 23, minute: 0, second: 7, microsecond: {0, 1}, + ...> utc_offset: 3600, std_offset: 0, time_zone: "Europe/Warsaw"} + iex> DateTime.to_naive(dt) + ~N[2000-02-29 23:00:07.0] + + """ + @spec to_naive(Calendar.datetime()) :: NaiveDateTime.t() + def to_naive(datetime) + + def to_naive(%{ + calendar: calendar, + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond, + time_zone: _ + }) do + %NaiveDateTime{ + year: year, + month: month, + day: day, + calendar: calendar, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond + } + end + + @doc """ + Converts a `DateTime` into a `Date`. + + Because `Date` does not hold time nor time zone information, + data will be lost during the conversion. + + ## Examples + + iex> dt = %DateTime{year: 2000, month: 2, day: 29, zone_abbr: "CET", + ...> hour: 23, minute: 0, second: 7, microsecond: {0, 0}, + ...> utc_offset: 3600, std_offset: 0, time_zone: "Europe/Warsaw"} + iex> DateTime.to_date(dt) + ~D[2000-02-29] + + """ + @spec to_date(Calendar.datetime()) :: Date.t() + def to_date(datetime) + + def to_date(%{ + year: year, + month: month, + day: day, + calendar: calendar, + hour: _, + minute: _, + second: _, + microsecond: _, + time_zone: _ + }) do + %Date{year: year, month: month, day: day, calendar: calendar} + end + + @doc """ + Converts a `DateTime` into `Time`. + + Because `Time` does not hold date nor time zone information, + data will be lost during the conversion. + + ## Examples + + iex> dt = %DateTime{year: 2000, month: 2, day: 29, zone_abbr: "CET", + ...> hour: 23, minute: 0, second: 7, microsecond: {0, 1}, + ...> utc_offset: 3600, std_offset: 0, time_zone: "Europe/Warsaw"} + iex> DateTime.to_time(dt) + ~T[23:00:07.0] + + """ + @spec to_time(Calendar.datetime()) :: Time.t() + def to_time(datetime) + + def to_time(%{ + year: _, + month: _, + day: _, + calendar: calendar, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond, + time_zone: _ + }) do + %Time{ + hour: hour, + minute: minute, + second: second, + microsecond: microsecond, + calendar: calendar + } + end + + @doc """ + Converts the given datetime to + [ISO 8601:2019](https://en.wikipedia.org/wiki/ISO_8601) format. + + By default, `DateTime.to_iso8601/2` returns datetimes formatted in the "extended" + format, for human readability. It also supports the "basic" format through passing the `:basic` option. + + You can also optionally specify an offset for the formatted string. + If none is given, the one in the given `datetime` is used. + + Only supports converting datetimes which are in the ISO calendar. + If another calendar is given, it is automatically converted to ISO. + It raises if not possible. + + WARNING: the ISO 8601 datetime format does not contain the time zone nor + its abbreviation, which means information is lost when converting to such + format. + + ## Examples + + iex> dt = %DateTime{year: 2000, month: 2, day: 29, zone_abbr: "CET", + ...> hour: 23, minute: 0, second: 7, microsecond: {0, 0}, + ...> utc_offset: 3600, std_offset: 0, time_zone: "Europe/Warsaw"} + iex> DateTime.to_iso8601(dt) + "2000-02-29T23:00:07+01:00" + + iex> dt = %DateTime{year: 2000, month: 2, day: 29, zone_abbr: "UTC", + ...> hour: 23, minute: 0, second: 7, microsecond: {0, 0}, + ...> utc_offset: 0, std_offset: 0, time_zone: "Etc/UTC"} + iex> DateTime.to_iso8601(dt) + "2000-02-29T23:00:07Z" + + iex> dt = %DateTime{year: 2000, month: 2, day: 29, zone_abbr: "AMT", + ...> hour: 23, minute: 0, second: 7, microsecond: {0, 0}, + ...> utc_offset: -14400, std_offset: 0, time_zone: "America/Manaus"} + iex> DateTime.to_iso8601(dt, :extended) + "2000-02-29T23:00:07-04:00" + + iex> dt = %DateTime{year: 2000, month: 2, day: 29, zone_abbr: "AMT", + ...> hour: 23, minute: 0, second: 7, microsecond: {0, 0}, + ...> utc_offset: -14400, std_offset: 0, time_zone: "America/Manaus"} + iex> DateTime.to_iso8601(dt, :basic) + "20000229T230007-0400" + + iex> dt = %DateTime{year: 2000, month: 2, day: 29, zone_abbr: "AMT", + ...> hour: 23, minute: 0, second: 7, microsecond: {0, 0}, + ...> utc_offset: -14400, std_offset: 0, time_zone: "America/Manaus"} + iex> DateTime.to_iso8601(dt, :extended, 3600) + "2000-03-01T04:00:07+01:00" + + iex> dt = %DateTime{year: 2000, month: 2, day: 29, zone_abbr: "AMT", + ...> hour: 23, minute: 0, second: 7, microsecond: {0, 0}, + ...> utc_offset: -14400, std_offset: 0, time_zone: "America/Manaus"} + iex> DateTime.to_iso8601(dt, :extended, 0) + "2000-03-01T03:00:07+00:00" + + iex> dt = %DateTime{year: 2000, month: 3, day: 01, zone_abbr: "UTC", + ...> hour: 03, minute: 0, second: 7, microsecond: {0, 0}, + ...> utc_offset: 0, std_offset: 0, time_zone: "Etc/UTC"} + iex> DateTime.to_iso8601(dt, :extended, 0) + "2000-03-01T03:00:07Z" + + iex> {:ok, dt, offset} = DateTime.from_iso8601("2000-03-01T03:00:07Z") + iex> "2000-03-01T03:00:07Z" = DateTime.to_iso8601(dt, :extended, offset) + """ + @spec to_iso8601(Calendar.datetime(), :basic | :extended, nil | integer()) :: String.t() + def to_iso8601(datetime, format \\ :extended, offset \\ nil) + + def to_iso8601(%{calendar: Calendar.ISO} = datetime, format, offset) + when format in [:extended, :basic] do + datetime + |> to_iso8601_iodata(format, offset) + |> IO.iodata_to_binary() + end + + def to_iso8601(%{calendar: _} = datetime, format, offset) + when format in [:extended, :basic] do + datetime + |> convert!(Calendar.ISO) + |> to_iso8601(format, offset) + end + + defp to_iso8601_iodata(datetime, format, nil) do + %{ + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond, + time_zone: time_zone, + utc_offset: utc_offset, + std_offset: std_offset + } = datetime + + [ + datetime_to_iodata(year, month, day, hour, minute, second, microsecond, format), + Calendar.ISO.offset_to_iodata(utc_offset, std_offset, time_zone, format) + ] + end + + defp to_iso8601_iodata( + %{microsecond: {_, precision}, time_zone: "Etc/UTC"} = datetime, + format, + 0 + ) do + {year, month, day, hour, minute, second, {microsecond, _}} = shift_by_offset(datetime, 0) + + [ + datetime_to_iodata( + year, + month, + day, + hour, + minute, + second, + {microsecond, precision}, + format + ), + ?Z + ] + end + + defp to_iso8601_iodata(datetime, format, offset) do + {_, precision} = datetime.microsecond + {year, month, day, hour, minute, second, {microsecond, _}} = shift_by_offset(datetime, offset) + + [ + datetime_to_iodata( + year, + month, + day, + hour, + minute, + second, + {microsecond, precision}, + format + ), + Calendar.ISO.offset_to_iodata(offset, 0, nil, format) + ] + end + + defp shift_by_offset(%{calendar: calendar} = datetime, offset) do + total_offset = datetime.utc_offset + datetime.std_offset + + datetime + |> to_iso_days() + # Subtract total original offset in order to get UTC and add the new offset + |> Calendar.ISO.add_day_fraction_to_iso_days(offset - total_offset, 86400) + |> calendar.naive_datetime_from_iso_days() + end + + defp datetime_to_iodata(year, month, day, hour, minute, second, microsecond, format) do + [ + Calendar.ISO.date_to_iodata(year, month, day, format), + ?T, + Calendar.ISO.time_to_iodata(hour, minute, second, microsecond, format) + ] + end + + @doc """ + Parses the extended "Date and time of day" format described by + [ISO 8601:2019](https://en.wikipedia.org/wiki/ISO_8601). + + Since ISO 8601 does not include the proper time zone, the given + string will be converted to UTC and its offset in seconds will be + returned as part of this function. Therefore offset information + must be present in the string. + + As specified in the standard, the separator "T" may be omitted if + desired as there is no ambiguity within this function. + + Note leap seconds are not supported by the built-in Calendar.ISO. + + ## Examples + + iex> {:ok, datetime, 0} = DateTime.from_iso8601("2015-01-23T23:50:07Z") + iex> datetime + ~U[2015-01-23 23:50:07Z] + + iex> {:ok, datetime, 9000} = DateTime.from_iso8601("2015-01-23T23:50:07.123+02:30") + iex> datetime + ~U[2015-01-23 21:20:07.123Z] + + iex> {:ok, datetime, 9000} = DateTime.from_iso8601("2015-01-23T23:50:07,123+02:30") + iex> datetime + ~U[2015-01-23 21:20:07.123Z] + + iex> {:ok, datetime, 0} = DateTime.from_iso8601("-2015-01-23T23:50:07Z") + iex> datetime + ~U[-2015-01-23 23:50:07Z] + + iex> {:ok, datetime, 9000} = DateTime.from_iso8601("-2015-01-23T23:50:07,123+02:30") + iex> datetime + ~U[-2015-01-23 21:20:07.123Z] + + iex> {:ok, datetime, 9000} = DateTime.from_iso8601("20150123T235007.123+0230", :basic) + iex> datetime + ~U[2015-01-23 21:20:07.123Z] + + iex> DateTime.from_iso8601("2015-01-23P23:50:07") + {:error, :invalid_format} + iex> DateTime.from_iso8601("2015-01-23T23:50:07") + {:error, :missing_offset} + iex> DateTime.from_iso8601("2015-01-23 23:50:61") + {:error, :invalid_time} + iex> DateTime.from_iso8601("2015-01-32 23:50:07") + {:error, :invalid_date} + iex> DateTime.from_iso8601("2015-01-23T23:50:07.123-00:00") + {:error, :invalid_format} + + """ + @doc since: "1.4.0" + @spec from_iso8601(String.t(), Calendar.calendar() | :extended | :basic) :: + {:ok, t, Calendar.utc_offset()} | {:error, atom} + def from_iso8601(string, format_or_calendar \\ Calendar.ISO) + + def from_iso8601(string, format) when format in [:basic, :extended] do + from_iso8601(string, Calendar.ISO, format) + end + + def from_iso8601(string, calendar) when is_atom(calendar) do + from_iso8601(string, calendar, :extended) + end + + @doc """ + Converts from ISO8601 specifying both a calendar and a mode. + + See `from_iso8601/2` for more information. + + ## Examples + + iex> {:ok, datetime, 9000} = DateTime.from_iso8601("2015-01-23T23:50:07,123+02:30", Calendar.ISO, :extended) + iex> datetime + ~U[2015-01-23 21:20:07.123Z] + + iex> {:ok, datetime, 9000} = DateTime.from_iso8601("20150123T235007.123+0230", Calendar.ISO, :basic) + iex> datetime + ~U[2015-01-23 21:20:07.123Z] + + """ + @spec from_iso8601(String.t(), Calendar.calendar(), :extended | :basic) :: + {:ok, t, Calendar.utc_offset()} | {:error, atom} + def from_iso8601(string, calendar, format) do + with {:ok, {year, month, day, hour, minute, second, microsecond}, offset} <- + Calendar.ISO.parse_utc_datetime(string, format) do + datetime = %DateTime{ + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond, + std_offset: 0, + utc_offset: 0, + zone_abbr: "UTC", + time_zone: "Etc/UTC" + } + + with {:ok, converted} <- convert(datetime, calendar) do + {:ok, converted, offset} + end + end + end + + @doc """ + Converts a number of gregorian seconds to a `DateTime` struct. + + The returned `DateTime` will have `UTC` timezone, if you want other timezone, please use + `DateTime.shift_zone/3`. + + ## Examples + + iex> DateTime.from_gregorian_seconds(1) + ~U[0000-01-01 00:00:01Z] + iex> DateTime.from_gregorian_seconds(63_755_511_991, {5000, 3}) + ~U[2020-05-01 00:26:31.005Z] + iex> DateTime.from_gregorian_seconds(-1) + ~U[-0001-12-31 23:59:59Z] + + """ + @doc since: "1.11.0" + @spec from_gregorian_seconds(integer(), Calendar.microsecond(), Calendar.calendar()) :: t + def from_gregorian_seconds( + seconds, + {microsecond, precision} \\ {0, 0}, + calendar \\ Calendar.ISO + ) + when is_integer(seconds) do + iso_days = Calendar.ISO.gregorian_seconds_to_iso_days(seconds, microsecond) + + {year, month, day, hour, minute, second, {microsecond, _}} = + calendar.naive_datetime_from_iso_days(iso_days) + + %DateTime{ + calendar: calendar, + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: {microsecond, precision}, + std_offset: 0, + utc_offset: 0, + zone_abbr: "UTC", + time_zone: "Etc/UTC" + } + end + + @doc """ + Converts a `DateTime` struct to a number of gregorian seconds and microseconds. + + ## Examples + + iex> dt = %DateTime{year: 0000, month: 1, day: 1, zone_abbr: "UTC", + ...> hour: 0, minute: 0, second: 1, microsecond: {0, 0}, + ...> utc_offset: 0, std_offset: 0, time_zone: "Etc/UTC"} + iex> DateTime.to_gregorian_seconds(dt) + {1, 0} + + iex> dt = %DateTime{year: 2020, month: 5, day: 1, zone_abbr: "UTC", + ...> hour: 0, minute: 26, second: 31, microsecond: {5000, 0}, + ...> utc_offset: 0, std_offset: 0, time_zone: "Etc/UTC"} + iex> DateTime.to_gregorian_seconds(dt) + {63_755_511_991, 5000} + + iex> dt = %DateTime{year: 2020, month: 5, day: 1, zone_abbr: "CET", + ...> hour: 1, minute: 26, second: 31, microsecond: {5000, 0}, + ...> utc_offset: 3600, std_offset: 0, time_zone: "Europe/Warsaw"} + iex> DateTime.to_gregorian_seconds(dt) + {63_755_511_991, 5000} + + """ + @doc since: "1.11.0" + @spec to_gregorian_seconds(Calendar.datetime()) :: {integer(), non_neg_integer()} + def to_gregorian_seconds( + %{ + std_offset: std_offset, + utc_offset: utc_offset, + microsecond: {microsecond, _} + } = datetime + ) do + {days, day_fraction} = + datetime + |> to_iso_days() + |> apply_tz_offset(utc_offset + std_offset) + + seconds_in_day = seconds_from_day_fraction(day_fraction) + {days * @seconds_per_day + seconds_in_day, microsecond} + end + + @doc """ + Converts the given `datetime` to a string according to its calendar. + + Unfortunately, there is no standard that specifies rendering of a + datetime with its complete time zone information, so Elixir uses a + custom (but relatively common) representation which appends the time + zone abbreviation and full name to the datetime. + + ## Examples + + iex> dt = %DateTime{year: 2000, month: 2, day: 29, zone_abbr: "CET", + ...> hour: 23, minute: 0, second: 7, microsecond: {0, 0}, + ...> utc_offset: 3600, std_offset: 0, time_zone: "Europe/Warsaw"} + iex> DateTime.to_string(dt) + "2000-02-29 23:00:07+01:00 CET Europe/Warsaw" + + iex> dt = %DateTime{year: 2000, month: 2, day: 29, zone_abbr: "UTC", + ...> hour: 23, minute: 0, second: 7, microsecond: {0, 0}, + ...> utc_offset: 0, std_offset: 0, time_zone: "Etc/UTC"} + iex> DateTime.to_string(dt) + "2000-02-29 23:00:07Z" + + iex> dt = %DateTime{year: 2000, month: 2, day: 29, zone_abbr: "AMT", + ...> hour: 23, minute: 0, second: 7, microsecond: {0, 0}, + ...> utc_offset: -14400, std_offset: 0, time_zone: "America/Manaus"} + iex> DateTime.to_string(dt) + "2000-02-29 23:00:07-04:00 AMT America/Manaus" + + iex> dt = %DateTime{year: -100, month: 12, day: 19, zone_abbr: "CET", + ...> hour: 3, minute: 20, second: 31, microsecond: {0, 0}, + ...> utc_offset: 3600, std_offset: 0, time_zone: "Europe/Stockholm"} + iex> DateTime.to_string(dt) + "-0100-12-19 03:20:31+01:00 CET Europe/Stockholm" + + """ + @spec to_string(Calendar.datetime()) :: String.t() + def to_string(%{calendar: calendar} = datetime) do + %{ + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond, + time_zone: time_zone, + zone_abbr: zone_abbr, + utc_offset: utc_offset, + std_offset: std_offset + } = datetime + + calendar.datetime_to_string( + year, + month, + day, + hour, + minute, + second, + microsecond, + time_zone, + zone_abbr, + utc_offset, + std_offset + ) + end + + @doc """ + Compares two datetime structs. + + Returns `:gt` if the first datetime is later than the second + and `:lt` for vice versa. If the two datetimes are equal + `:eq` is returned. + + Note that both UTC and Standard offsets will be taken into + account when comparison is done. + + ## Examples + + iex> dt1 = %DateTime{year: 2000, month: 2, day: 29, zone_abbr: "AMT", + ...> hour: 23, minute: 0, second: 7, microsecond: {0, 0}, + ...> utc_offset: -14400, std_offset: 0, time_zone: "America/Manaus"} + iex> dt2 = %DateTime{year: 2000, month: 2, day: 29, zone_abbr: "CET", + ...> hour: 23, minute: 0, second: 7, microsecond: {0, 0}, + ...> utc_offset: 3600, std_offset: 0, time_zone: "Europe/Warsaw"} + iex> DateTime.compare(dt1, dt2) + :gt + + """ + @doc since: "1.4.0" + @spec compare(Calendar.datetime(), Calendar.datetime()) :: :lt | :eq | :gt + def compare( + %{utc_offset: utc_offset1, std_offset: std_offset1} = datetime1, + %{utc_offset: utc_offset2, std_offset: std_offset2} = datetime2 + ) do + {days1, {parts1, ppd1}} = + datetime1 + |> to_iso_days() + |> apply_tz_offset(utc_offset1 + std_offset1) + + {days2, {parts2, ppd2}} = + datetime2 + |> to_iso_days() + |> apply_tz_offset(utc_offset2 + std_offset2) + + # Ensure fraction tuples have same denominator. + first = {days1, parts1 * ppd2} + second = {days2, parts2 * ppd1} + + cond do + first > second -> :gt + first < second -> :lt + true -> :eq + end + end + + @doc """ + Returns `true` if the first datetime is strictly earlier than the second. + + ## Examples + + iex> DateTime.before?(~U[2021-01-01 11:00:00Z], ~U[2022-02-02 11:00:00Z]) + true + iex> DateTime.before?(~U[2021-01-01 11:00:00Z], ~U[2021-01-01 11:00:00Z]) + false + iex> DateTime.before?(~U[2022-02-02 11:00:00Z], ~U[2021-01-01 11:00:00Z]) + false + + """ + @doc since: "1.15.0" + @spec before?(Calendar.datetime(), Calendar.datetime()) :: boolean() + def before?(datetime1, datetime2) do + compare(datetime1, datetime2) == :lt + end + + @doc """ + Returns `true` if the first datetime is strictly later than the second. + + ## Examples + + iex> DateTime.after?(~U[2022-02-02 11:00:00Z], ~U[2021-01-01 11:00:00Z]) + true + iex> DateTime.after?(~U[2021-01-01 11:00:00Z], ~U[2021-01-01 11:00:00Z]) + false + iex> DateTime.after?(~U[2021-01-01 11:00:00Z], ~U[2022-02-02 11:00:00Z]) + false + + """ + @doc since: "1.15.0" + @spec after?(Calendar.datetime(), Calendar.datetime()) :: boolean() + def after?(datetime1, datetime2) do + compare(datetime1, datetime2) == :gt + end + + @doc """ + Subtracts `datetime2` from `datetime1`. + + The answer can be returned in any `:day`, `:hour`, `:minute`, or any `unit` + available from `t:System.time_unit/0`. The unit is measured according to + `Calendar.ISO` and defaults to `:second`. + + Fractional results are not supported and are truncated. + + ## Examples + + iex> DateTime.diff(~U[2024-01-15 10:00:10Z], ~U[2024-01-15 10:00:00Z]) + 10 + + This function also considers timezone offsets: + + iex> dt1 = %DateTime{year: 2000, month: 2, day: 29, zone_abbr: "AMT", + ...> hour: 23, minute: 0, second: 7, microsecond: {0, 0}, + ...> utc_offset: -14400, std_offset: 0, time_zone: "America/Manaus"} + iex> dt2 = %DateTime{year: 2000, month: 2, day: 29, zone_abbr: "CET", + ...> hour: 23, minute: 0, second: 7, microsecond: {0, 0}, + ...> utc_offset: 3600, std_offset: 0, time_zone: "Europe/Warsaw"} + iex> DateTime.diff(dt1, dt2) + 18000 + iex> DateTime.diff(dt2, dt1) + -18000 + iex> DateTime.diff(dt1, dt2, :hour) + 5 + iex> DateTime.diff(dt2, dt1, :hour) + -5 + + """ + @doc since: "1.5.0" + @spec diff( + Calendar.datetime(), + Calendar.datetime(), + :day | :hour | :minute | System.time_unit() + ) :: integer() + def diff(datetime1, datetime2, unit \\ :second) + + def diff(datetime1, datetime2, :day) do + diff(datetime1, datetime2, :second) |> div(86400) + end + + def diff(datetime1, datetime2, :hour) do + diff(datetime1, datetime2, :second) |> div(3600) + end + + def diff(datetime1, datetime2, :minute) do + diff(datetime1, datetime2, :second) |> div(60) + end + + def diff( + %{utc_offset: utc_offset1, std_offset: std_offset1} = datetime1, + %{utc_offset: utc_offset2, std_offset: std_offset2} = datetime2, + unit + ) do + if not is_integer(unit) and + unit not in ~w(second millisecond microsecond nanosecond)a do + raise ArgumentError, + "unsupported time unit. Expected :day, :hour, :minute, :second, :millisecond, :microsecond, :nanosecond, or a positive integer, got #{inspect(unit)}" + end + + naive_diff = + (datetime1 |> to_iso_days() |> Calendar.ISO.iso_days_to_unit(:microsecond)) - + (datetime2 |> to_iso_days() |> Calendar.ISO.iso_days_to_unit(:microsecond)) + + offset_diff = utc_offset2 + std_offset2 - (utc_offset1 + std_offset1) + + System.convert_time_unit(naive_diff, :microsecond, unit) + + System.convert_time_unit(offset_diff, :second, unit) + end + + @doc """ + Adds a specified amount of time to a `DateTime`. + + > #### Prefer `shift/2` {: .info} + > + > Prefer `shift/2` over `add/3`, as it offers a more ergonomic API. + > + > `add/3` provides a lower-level API which only supports fixed units + > such as `:hour` and `:second`, but not `:month` (as the exact length + > of a month depends on the current month). `add/3` always considers + > the unit to be computed according to the `Calendar.ISO`. + + Accepts an `amount_to_add` in any `unit`. `unit` can be `:day`, + `:hour`, `:minute`, `:second` or any subsecond precision from + `t:System.time_unit/0` for convenience but ultimately they are + all converted to microseconds. Negative values will move backwards + in time and the default precision is `:second`. + + This function relies on a contiguous representation of time, + ignoring timezone changes. For example, if you add one day when there + are summer time/daylight saving time changes, it will also change the + time forward or backward by one hour, so the elapsed time is precisely + 24 hours. Similarly, adding just a few seconds to a datetime just before + "spring forward" can cause wall time to increase by more than an hour. + + While this means this function is precise in terms of elapsed time, + its result may be confusing in certain use cases. For example, if a + user requests a meeting to happen every day at 15:00 and you use this + function to compute all future meetings by adding day after day, this + function may change the meeting time to 14:00 or 16:00 if there are + changes to the current timezone. + + In case you don't want these changes to happen automatically or you + want to surface time zone conflicts to the user, you can add to + the datetime as a naive datetime and then use `from_naive/2`: + + dt |> NaiveDateTime.add(1, :day) |> DateTime.from_naive(dt.time_zone) + + The above will surface time jumps and ambiguous datetimes, allowing you + to deal with them accordingly. + + ## Examples + + iex> dt = DateTime.from_naive!(~N[2018-11-15 10:00:00], "Europe/Copenhagen", FakeTimeZoneDatabase) + iex> dt |> DateTime.add(3600, :second, FakeTimeZoneDatabase) + #DateTime<2018-11-15 11:00:00+01:00 CET Europe/Copenhagen> + + iex> DateTime.add(~U[2018-11-15 10:00:00Z], 3600, :second) + ~U[2018-11-15 11:00:00Z] + + When adding 3 seconds just before "spring forward" we go from 1:59:59 to 3:00:02: + + iex> dt = DateTime.from_naive!(~N[2019-03-31 01:59:59.123], "Europe/Copenhagen", FakeTimeZoneDatabase) + iex> dt |> DateTime.add(3, :second, FakeTimeZoneDatabase) + #DateTime<2019-03-31 03:00:02.123+02:00 CEST Europe/Copenhagen> + + When adding 1 day during "spring forward", the hour also changes: + + iex> dt = DateTime.from_naive!(~N[2019-03-31 01:00:00], "Europe/Copenhagen", FakeTimeZoneDatabase) + iex> dt |> DateTime.add(1, :day, FakeTimeZoneDatabase) + #DateTime<2019-04-01 02:00:00+02:00 CEST Europe/Copenhagen> + + This operation merges the precision of the naive date time with the given unit: + + iex> result = DateTime.add(~U[2014-10-02 00:29:10Z], 21, :millisecond) + ~U[2014-10-02 00:29:10.021Z] + iex> result.microsecond + {21000, 3} + + """ + @doc since: "1.8.0" + @spec add( + Calendar.datetime(), + integer, + :day | :hour | :minute | System.time_unit(), + Calendar.time_zone_database() + ) :: + t() + def add( + datetime, + amount_to_add, + unit \\ :second, + time_zone_database \\ Calendar.get_time_zone_database() + ) + + def add(datetime, amount_to_add, :day, time_zone_database) when is_integer(amount_to_add) do + add(datetime, amount_to_add * 86400, :second, time_zone_database) + end + + def add(datetime, amount_to_add, :hour, time_zone_database) when is_integer(amount_to_add) do + add(datetime, amount_to_add * 3600, :second, time_zone_database) + end + + def add(datetime, amount_to_add, :minute, time_zone_database) when is_integer(amount_to_add) do + add(datetime, amount_to_add * 60, :second, time_zone_database) + end + + def add(%{calendar: calendar} = datetime, amount_to_add, unit, time_zone_database) + when is_integer(amount_to_add) do + %{ + microsecond: {_, precision}, + time_zone: time_zone, + utc_offset: utc_offset, + std_offset: std_offset + } = datetime + + if not is_integer(unit) and unit not in ~w(second millisecond microsecond nanosecond)a do + raise ArgumentError, + "unsupported time unit. Expected :day, :hour, :minute, :second, :millisecond, :microsecond, :nanosecond, or a positive integer, got #{inspect(unit)}" + end + + precision = max(Calendar.ISO.time_unit_to_precision(unit), precision) + + result = + datetime + |> to_iso_days() + |> Calendar.ISO.shift_time_unit(amount_to_add, unit) + |> apply_tz_offset(utc_offset + std_offset) + |> shift_zone_for_iso_days_utc(calendar, precision, time_zone, time_zone_database) + + case result do + {:ok, result_datetime} -> + result_datetime + + {:error, error} -> + raise ArgumentError, + "cannot add #{amount_to_add} #{unit} to #{inspect(datetime)} (with time zone " <> + "database #{inspect(time_zone_database)}), reason: #{inspect(error)}" + end + end + + @doc """ + Shifts given `datetime` by `duration` according to its calendar. + + Allowed units are: `:year`, `:month`, `:week`, `:day`, `:hour`, `:minute`, `:second`, `:microsecond`. + + This operation is equivalent to shifting the datetime wall clock + (in other words, the value as someone in that timezone would see + on their watch), then applying the time zone offset to convert it + to UTC, and finally computing the new timezone in case of shifts. + This ensures `shift/3` always returns a valid datetime. + + Consequently, time zones that observe "Daylight Saving Time" + or other changes, across summer/winter time will add/remove hours + from the resulting datetime: + + dt = DateTime.new!(~D[2019-03-31], ~T[01:00:00], "Europe/Copenhagen") + DateTime.shift(dt, hour: 1) + #=> #DateTime<2019-03-31 03:00:00+02:00 CEST Europe/Copenhagen> + + dt = DateTime.new!(~D[2018-11-04], ~T[00:00:00], "America/Los_Angeles") + DateTime.shift(dt, hour: 2) + #=> #DateTime<2018-11-04 01:00:00-08:00 PST America/Los_Angeles> + + Although the first example shows a difference of 2 hours when + comparing the wall clocks of the given datetime with the returned one, + due to the "spring forward" time jump, the actual elapsed time is + still exactly of 1 hour. + + In case you don't want these changes to happen automatically or you + want to surface time zone conflicts to the user, you can shift + the datetime as a naive datetime and then use `from_naive/2`: + + dt |> NaiveDateTime.shift(duration) |> DateTime.from_naive(dt.time_zone) + + The above will surface time jumps and ambiguous datetimes, allowing you + to deal with them accordingly. + + ## ISO calendar considerations + + When using the default ISO calendar, durations are collapsed and + applied in the order of months, then seconds and microseconds: + + * when shifting by 1 year and 2 months the date is actually shifted by 14 months + * weeks, days and smaller units are collapsed into seconds and microseconds + + When shifting by month, days are rounded down to the nearest valid date. + + ## Examples + + iex> DateTime.shift(~U[2016-01-01 00:00:00Z], month: 2) + ~U[2016-03-01 00:00:00Z] + iex> DateTime.shift(~U[2016-01-01 00:00:00Z], year: 1, week: 4) + ~U[2017-01-29 00:00:00Z] + iex> DateTime.shift(~U[2016-01-01 00:00:00Z], minute: -25) + ~U[2015-12-31 23:35:00Z] + iex> DateTime.shift(~U[2016-01-01 00:00:00Z], minute: 5, microsecond: {500, 4}) + ~U[2016-01-01 00:05:00.0005Z] + + # leap years + iex> DateTime.shift(~U[2024-02-29 00:00:00Z], year: 1) + ~U[2025-02-28 00:00:00Z] + iex> DateTime.shift(~U[2024-02-29 00:00:00Z], year: 4) + ~U[2028-02-29 00:00:00Z] + + # rounding down + iex> DateTime.shift(~U[2015-01-31 00:00:00Z], month: 1) + ~U[2015-02-28 00:00:00Z] + + """ + @doc since: "1.17.0" + @spec shift(Calendar.datetime(), Duration.duration(), Calendar.time_zone_database()) :: t + def shift(datetime, duration, time_zone_database \\ Calendar.get_time_zone_database()) + + def shift(%{calendar: calendar, time_zone: "Etc/UTC"} = datetime, duration, _time_zone_database) do + %{ + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond + } = datetime + + {year, month, day, hour, minute, second, microsecond} = + calendar.shift_naive_datetime( + year, + month, + day, + hour, + minute, + second, + microsecond, + __duration__!(duration) + ) + + %DateTime{ + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond, + time_zone: "Etc/UTC", + zone_abbr: "UTC", + std_offset: 0, + utc_offset: 0 + } + end + + def shift(%{calendar: calendar} = datetime, duration, time_zone_database) do + %{ + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond, + std_offset: std_offset, + utc_offset: utc_offset, + time_zone: time_zone + } = datetime + + {year, month, day, hour, minute, second, {_, precision} = microsecond} = + calendar.shift_naive_datetime( + year, + month, + day, + hour, + minute, + second, + microsecond, + __duration__!(duration) + ) + + result = + calendar.naive_datetime_to_iso_days(year, month, day, hour, minute, second, microsecond) + |> apply_tz_offset(utc_offset + std_offset) + |> shift_zone_for_iso_days_utc(calendar, precision, time_zone, time_zone_database) + + case result do + {:ok, result_datetime} -> + result_datetime + + {:error, error} -> + raise ArgumentError, + "cannot shift #{inspect(datetime)} to #{inspect(duration)} (with time zone " <> + "database #{inspect(time_zone_database)}), reason: #{inspect(error)}" + end + end + + @doc false + defdelegate __duration__!(params), to: Duration, as: :new! + + @doc """ + Returns the given datetime with the microsecond field truncated to the given + precision (`:microsecond`, `:millisecond` or `:second`). + + The given datetime is returned unchanged if it already has lower precision than + the given precision. + + ## Examples + + iex> dt1 = %DateTime{year: 2017, month: 11, day: 7, zone_abbr: "CET", + ...> hour: 11, minute: 45, second: 18, microsecond: {123456, 6}, + ...> utc_offset: 3600, std_offset: 0, time_zone: "Europe/Paris"} + iex> DateTime.truncate(dt1, :microsecond) + #DateTime<2017-11-07 11:45:18.123456+01:00 CET Europe/Paris> + + iex> dt2 = %DateTime{year: 2017, month: 11, day: 7, zone_abbr: "CET", + ...> hour: 11, minute: 45, second: 18, microsecond: {123456, 6}, + ...> utc_offset: 3600, std_offset: 0, time_zone: "Europe/Paris"} + iex> DateTime.truncate(dt2, :millisecond) + #DateTime<2017-11-07 11:45:18.123+01:00 CET Europe/Paris> + + iex> dt3 = %DateTime{year: 2017, month: 11, day: 7, zone_abbr: "CET", + ...> hour: 11, minute: 45, second: 18, microsecond: {123456, 6}, + ...> utc_offset: 3600, std_offset: 0, time_zone: "Europe/Paris"} + iex> DateTime.truncate(dt3, :second) + #DateTime<2017-11-07 11:45:18+01:00 CET Europe/Paris> + + """ + @doc since: "1.6.0" + @spec truncate(Calendar.datetime(), :microsecond | :millisecond | :second) :: t() + def truncate(%DateTime{microsecond: microsecond} = datetime, precision) do + %{datetime | microsecond: Calendar.truncate(microsecond, precision)} + end + + def truncate(%{} = datetime_map, precision) do + truncate(from_map(datetime_map), precision) + end + + @doc """ + Converts a given `datetime` from one calendar to another. + + If it is not possible to convert unambiguously between the calendars + (see `Calendar.compatible_calendars?/2`), an `{:error, :incompatible_calendars}` tuple + is returned. + + ## Examples + + Imagine someone implements `Calendar.Holocene`, a calendar based on the + Gregorian calendar that adds exactly 10 000 years to the current Gregorian + year: + + iex> dt1 = %DateTime{year: 2000, month: 2, day: 29, zone_abbr: "AMT", + ...> hour: 23, minute: 0, second: 7, microsecond: {0, 0}, + ...> utc_offset: -14400, std_offset: 0, time_zone: "America/Manaus"} + iex> DateTime.convert(dt1, Calendar.Holocene) + {:ok, %DateTime{calendar: Calendar.Holocene, day: 29, hour: 23, + microsecond: {0, 0}, minute: 0, month: 2, second: 7, std_offset: 0, + time_zone: "America/Manaus", utc_offset: -14400, year: 12000, + zone_abbr: "AMT"}} + + """ + @doc since: "1.5.0" + @spec convert(Calendar.datetime(), Calendar.calendar()) :: + {:ok, t} | {:error, :incompatible_calendars} + + def convert(%DateTime{calendar: calendar} = datetime, calendar) do + {:ok, datetime} + end + + def convert(%{calendar: calendar} = datetime, calendar) do + {:ok, from_map(datetime)} + end + + def convert(%{calendar: dt_calendar, microsecond: {_, precision}} = datetime, calendar) do + if Calendar.compatible_calendars?(dt_calendar, calendar) do + result_datetime = + datetime + |> to_iso_days() + |> from_iso_days(datetime, calendar, precision) + + {:ok, result_datetime} + else + {:error, :incompatible_calendars} + end + end + + @doc """ + Converts a given `datetime` from one calendar to another. + + If it is not possible to convert unambiguously between the calendars + (see `Calendar.compatible_calendars?/2`), an ArgumentError is raised. + + ## Examples + + Imagine someone implements `Calendar.Holocene`, a calendar based on the + Gregorian calendar that adds exactly 10 000 years to the current Gregorian + year: + + iex> dt1 = %DateTime{year: 2000, month: 2, day: 29, zone_abbr: "AMT", + ...> hour: 23, minute: 0, second: 7, microsecond: {0, 0}, + ...> utc_offset: -14400, std_offset: 0, time_zone: "America/Manaus"} + iex> DateTime.convert!(dt1, Calendar.Holocene) + %DateTime{calendar: Calendar.Holocene, day: 29, hour: 23, + microsecond: {0, 0}, minute: 0, month: 2, second: 7, std_offset: 0, + time_zone: "America/Manaus", utc_offset: -14400, year: 12000, + zone_abbr: "AMT"} + + """ + @doc since: "1.5.0" + @spec convert!(Calendar.datetime(), Calendar.calendar()) :: t + def convert!(datetime, calendar) do + case convert(datetime, calendar) do + {:ok, value} -> + value + + {:error, :incompatible_calendars} -> + raise ArgumentError, + "cannot convert #{inspect(datetime)} to target calendar #{inspect(calendar)}, " <> + "reason: #{inspect(datetime.calendar)} and #{inspect(calendar)} have different " <> + "day rollover moments, making this conversion ambiguous" + end + end + + # Keep it multiline for proper function clause errors. + defp to_iso_days(%{ + calendar: calendar, + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond + }) do + calendar.naive_datetime_to_iso_days(year, month, day, hour, minute, second, microsecond) + end + + defp from_iso_days(iso_days, datetime, calendar, precision) do + %{time_zone: time_zone, zone_abbr: zone_abbr, utc_offset: utc_offset, std_offset: std_offset} = + datetime + + {year, month, day, hour, minute, second, {microsecond, _}} = + calendar.naive_datetime_from_iso_days(iso_days) + + %DateTime{ + calendar: calendar, + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: {microsecond, precision}, + time_zone: time_zone, + zone_abbr: zone_abbr, + utc_offset: utc_offset, + std_offset: std_offset + } + end + + defp apply_tz_offset(iso_days, 0) do + iso_days + end + + defp apply_tz_offset(iso_days, offset) do + Calendar.ISO.add_day_fraction_to_iso_days(iso_days, -offset, 86400) + end + + defp from_map(%{} = datetime_map) do + %DateTime{ + year: datetime_map.year, + month: datetime_map.month, + day: datetime_map.day, + hour: datetime_map.hour, + minute: datetime_map.minute, + second: datetime_map.second, + microsecond: datetime_map.microsecond, + time_zone: datetime_map.time_zone, + zone_abbr: datetime_map.zone_abbr, + utc_offset: datetime_map.utc_offset, + std_offset: datetime_map.std_offset + } + end + + defp seconds_from_day_fraction({parts_in_day, @seconds_per_day}), + do: parts_in_day + + defp seconds_from_day_fraction({parts_in_day, parts_per_day}), + do: div(parts_in_day * @seconds_per_day, parts_per_day) + + defimpl String.Chars do + def to_string(datetime) do + %{ + calendar: calendar, + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond, + time_zone: time_zone, + zone_abbr: zone_abbr, + utc_offset: utc_offset, + std_offset: std_offset + } = datetime + + calendar.datetime_to_string( + year, + month, + day, + hour, + minute, + second, + microsecond, + time_zone, + zone_abbr, + utc_offset, + std_offset + ) + end + end + + defimpl Inspect do + def inspect(datetime, _) do + %{ + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond, + time_zone: time_zone, + zone_abbr: zone_abbr, + utc_offset: utc_offset, + std_offset: std_offset, + calendar: calendar + } = datetime + + formatted = + calendar.datetime_to_string( + year, + month, + day, + hour, + minute, + second, + microsecond, + time_zone, + zone_abbr, + utc_offset, + std_offset + ) + + case datetime do + %{utc_offset: 0, std_offset: 0, time_zone: "Etc/UTC", year: year} + when calendar != Calendar.ISO or year in -9999..9999 -> + "~U[" <> formatted <> suffix(calendar) <> "]" + + _ -> + "#DateTime<" <> formatted <> suffix(calendar) <> ">" + end + end + + defp suffix(Calendar.ISO), do: "" + defp suffix(calendar), do: " " <> inspect(calendar) + end +end diff --git a/lib/elixir/lib/calendar/duration.ex b/lib/elixir/lib/calendar/duration.ex new file mode 100644 index 00000000000..b96e8b8f030 --- /dev/null +++ b/lib/elixir/lib/calendar/duration.ex @@ -0,0 +1,586 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team + +defmodule Duration do + @moduledoc """ + Struct and functions for handling durations. + + A `Duration` struct represents a collection of time scale units, + allowing for manipulation and calculation of durations. + + Date and time scale units are represented as integers, allowing for + both positive and negative values. + + Microseconds are represented using a tuple `{microsecond, precision}`. + This ensures compatibility with other calendar types implementing time, + such as `Time`, `DateTime`, and `NaiveDateTime`. + + ## Shifting + + The most common use of durations in Elixir's standard library is to + "shift" the calendar types. + + iex> Date.shift(~D[2016-01-03], month: 2) + ~D[2016-03-03] + + In the example above, `Date.shift/2` automatically converts the units + into a `Duration` struct, although one can also be given directly: + + iex> Date.shift(~D[2016-01-03], Duration.new!(month: 2)) + ~D[2016-03-03] + + It is important to note that shifting is not an arithmetic operation. + For example, adding `date + 1 month + 1 month` does not yield the same + result as `date + 2 months`. Let's see an example: + + iex> ~D[2016-01-31] |> Date.shift(month: 1) |> Date.shift(month: 1) + ~D[2016-03-29] + + iex> ~D[2016-01-31] |> Date.shift(month: 2) + ~D[2016-03-31] + + As you can see above, the results differ, which explains why operations + with durations are called "shift" rather than "add". This happens because, + once we add one month to `2016-01-31`, we get `2016-02-29`. Then adding + one extra month gives us `2016-03-29` instead of `2016-03-31`. + + In particular, when applying durations to `Calendar.ISO` types: + + * larger units (such as years and months) are applied before + smaller ones (such as weeks, hours, days, and so on) + + * units are collapsed into months (`:year` and `:month`), + seconds (`:week`, `:day`, `:hour`, `:minute`, `:second`) + and microseconds (`:microsecond`) before they are applied + + * 1 year is equivalent to 12 months, 1 week is equivalent to 7 days. + Therefore, 4 weeks _are not_ equivalent to 1 month + + * in case of non-existing dates, the results are rounded down to the + nearest valid date + + As the `shift/2` functions are calendar aware, they are guaranteed to return + valid date/times, considering leap years as well as DST in applicable time zones. + + ## Intervals + + Durations in Elixir can be combined with stream operations to build intervals. + For example, to retrieve the next three Wednesdays starting from 17th April, 2024: + + iex> ~D[2024-04-17] |> Stream.iterate(&Date.shift(&1, week: 1)) |> Enum.take(3) + [~D[2024-04-17], ~D[2024-04-24], ~D[2024-05-01]] + + However, once again, it is important to remember that shifting a duration is not + arithmetic, so you may want to use the functions in this module depending on what + you to achieve. Compare the results of both examples below: + + # Adding one month after the other + iex> date = ~D[2016-01-31] + iex> duration = Duration.new!(month: 1) + iex> stream = Stream.iterate(date, fn prev_date -> Date.shift(prev_date, duration) end) + iex> Enum.take(stream, 3) + [~D[2016-01-31], ~D[2016-02-29], ~D[2016-03-29]] + + # Multiplying durations by an index + iex> date = ~D[2016-01-31] + iex> duration = Duration.new!(month: 1) + iex> stream = Stream.from_index(fn i -> Date.shift(date, Duration.multiply(duration, i)) end) + iex> Enum.take(stream, 3) + [~D[2016-01-31], ~D[2016-02-29], ~D[2016-03-31]] + + The second example consistently points to the last day of the month, + as it performs operations on the duration, rather than shifting date + after date. + + ## Comparing durations + + In order to accurately compare durations, you need to either compare + only certain fields or use a reference time instant. This is because + some fields are relative to others. For example, you may say that + 1 month is the same as 30 days, but if you add both of these durations + to `~D[2015-02-01]`, you would get different results, as that month + has only 28 days. + + Therefore, if you wish to compare durations, one option is to use + `Date.shift/2` (or `DateTime.shift/2` or similar), and then compare + the dates: + + iex> date = ~D[2015-02-01] + iex> Date.compare(Date.shift(date, month: 1), Date.shift(date, day: 30)) + :lt + + Or alternatively convert the durations to a fixed unit by using `to_timeout/1`, + which supports durations only up to weeks, raising if it has the month or year + fields set. + + iex> to_timeout(hour: 24) == to_timeout(day: 1) + true + """ + + @moduledoc since: "1.17.0" + + @derive {Inspect, optional: [:year, :month, :week, :day, :hour, :minute, :second, :microsecond]} + defstruct year: 0, + month: 0, + week: 0, + day: 0, + hour: 0, + minute: 0, + second: 0, + microsecond: {0, 0} + + @typedoc """ + The duration struct type. + """ + @type t :: %Duration{ + year: integer, + month: integer, + week: integer, + day: integer, + hour: integer, + minute: integer, + second: integer, + microsecond: Calendar.microsecond() + } + + @typedoc """ + The unit pair type specifies a pair of a valid duration unit key and value. + """ + @type unit_pair :: + {:year, integer} + | {:month, integer} + | {:week, integer} + | {:day, integer} + | {:hour, integer} + | {:minute, integer} + | {:second, integer} + | {:microsecond, Calendar.microsecond()} + + @typedoc """ + The duration type specifies a `%Duration{}` struct or a keyword list of valid duration unit pairs. + """ + @type duration :: t | [unit_pair] + + @typedoc """ + Options for `Duration.to_string/2`. + """ + @type to_string_opts :: [ + units: [ + year: String.t(), + month: String.t(), + week: String.t(), + day: String.t(), + hour: String.t(), + minute: String.t(), + second: String.t() + ], + separator: String.t() + ] + + @microseconds_per_second 1_000_000 + + @doc """ + Creates a new `Duration` struct from given `unit_pairs`. + + Raises an `ArgumentError` when called with invalid unit pairs. + + ## Examples + + iex> Duration.new!(year: 1, week: 3, hour: 4, second: 1) + %Duration{year: 1, week: 3, hour: 4, second: 1} + iex> Duration.new!(second: 1, microsecond: {1000, 6}) + %Duration{second: 1, microsecond: {1000, 6}} + iex> Duration.new!(month: 2) + %Duration{month: 2} + + """ + @spec new!(duration()) :: t + def new!(%Duration{} = duration) do + duration + end + + def new!(unit_pairs) do + Enum.each(unit_pairs, &validate_unit!/1) + struct!(Duration, unit_pairs) + end + + defp validate_unit!({:microsecond, {ms, precision}}) + when is_integer(ms) and precision in 0..6 do + :ok + end + + defp validate_unit!({:microsecond, microsecond}) do + raise ArgumentError, + "unsupported value #{inspect(microsecond)} for :microsecond. Expected a tuple {ms, precision} where precision is an integer from 0 to 6" + end + + defp validate_unit!({unit, _value}) + when unit not in [:year, :month, :week, :day, :hour, :minute, :second] do + raise ArgumentError, + "unknown unit #{inspect(unit)}. Expected :year, :month, :week, :day, :hour, :minute, :second, :microsecond" + end + + defp validate_unit!({_unit, value}) when is_integer(value) do + :ok + end + + defp validate_unit!({unit, value}) do + raise ArgumentError, + "unsupported value #{inspect(value)} for #{inspect(unit)}. Expected an integer" + end + + @doc """ + Adds units of given durations `d1` and `d2`. + + Respects the the highest microsecond precision of the two. + + ## Examples + + iex> Duration.add(Duration.new!(week: 2, day: 1), Duration.new!(day: 2)) + %Duration{week: 2, day: 3} + iex> Duration.add(Duration.new!(microsecond: {400, 3}), Duration.new!(microsecond: {600, 6})) + %Duration{microsecond: {1000, 6}} + + """ + @spec add(t, t) :: t + def add(%Duration{} = d1, %Duration{} = d2) do + {m1, p1} = d1.microsecond + {m2, p2} = d2.microsecond + + %Duration{ + year: d1.year + d2.year, + month: d1.month + d2.month, + week: d1.week + d2.week, + day: d1.day + d2.day, + hour: d1.hour + d2.hour, + minute: d1.minute + d2.minute, + second: d1.second + d2.second, + microsecond: {m1 + m2, max(p1, p2)} + } + end + + @doc """ + Subtracts units of given durations `d1` and `d2`. + + Respects the the highest microsecond precision of the two. + + ## Examples + + iex> Duration.subtract(Duration.new!(week: 2, day: 1), Duration.new!(day: 2)) + %Duration{week: 2, day: -1} + iex> Duration.subtract(Duration.new!(microsecond: {400, 6}), Duration.new!(microsecond: {600, 3})) + %Duration{microsecond: {-200, 6}} + + """ + @spec subtract(t, t) :: t + def subtract(%Duration{} = d1, %Duration{} = d2) do + {m1, p1} = d1.microsecond + {m2, p2} = d2.microsecond + + %Duration{ + year: d1.year - d2.year, + month: d1.month - d2.month, + week: d1.week - d2.week, + day: d1.day - d2.day, + hour: d1.hour - d2.hour, + minute: d1.minute - d2.minute, + second: d1.second - d2.second, + microsecond: {m1 - m2, max(p1, p2)} + } + end + + @doc """ + Multiplies `duration` units by given `integer`. + + ## Examples + + iex> Duration.multiply(Duration.new!(day: 1, minute: 15, second: -10), 3) + %Duration{day: 3, minute: 45, second: -30} + iex> Duration.multiply(Duration.new!(microsecond: {200, 4}), 3) + %Duration{microsecond: {600, 4}} + + """ + @spec multiply(t, integer) :: t + def multiply(%Duration{microsecond: {ms, p}} = duration, integer) when is_integer(integer) do + %Duration{ + year: duration.year * integer, + month: duration.month * integer, + week: duration.week * integer, + day: duration.day * integer, + hour: duration.hour * integer, + minute: duration.minute * integer, + second: duration.second * integer, + microsecond: {ms * integer, p} + } + end + + @doc """ + Negates `duration` units. + + ## Examples + + iex> Duration.negate(Duration.new!(day: 1, minute: 15, second: -10)) + %Duration{day: -1, minute: -15, second: 10} + iex> Duration.negate(Duration.new!(microsecond: {500000, 4})) + %Duration{microsecond: {-500000, 4}} + + """ + @spec negate(t) :: t + def negate(%Duration{microsecond: {ms, p}} = duration) do + %Duration{ + year: -duration.year, + month: -duration.month, + week: -duration.week, + day: -duration.day, + hour: -duration.hour, + minute: -duration.minute, + second: -duration.second, + microsecond: {-ms, p} + } + end + + @doc """ + Parses an [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) formatted duration string to a `Duration` struct. + + Duration strings, as well as individual units, may be prefixed with plus/minus signs so that: + + - `-PT6H3M` parses as `%Duration{hour: -6, minute: -3}` + - `-PT6H-3M` parses as `%Duration{hour: -6, minute: 3}` + - `+PT6H3M` parses as `%Duration{hour: 6, minute: 3}` + - `+PT6H-3M` parses as `%Duration{hour: 6, minute: -3}` + + Duration designators must be provided in order of magnitude: `P[n]Y[n]M[n]W[n]DT[n]H[n]M[n]S`. + + Only seconds may be specified with a decimal fraction, using either a comma or a full stop: `P1DT4,5S`. + + ## Examples + + iex> Duration.from_iso8601("P1Y2M3DT4H5M6S") + {:ok, %Duration{year: 1, month: 2, day: 3, hour: 4, minute: 5, second: 6}} + iex> Duration.from_iso8601("P3Y-2MT3H") + {:ok, %Duration{year: 3, month: -2, hour: 3}} + iex> Duration.from_iso8601("-PT10H-30M") + {:ok, %Duration{hour: -10, minute: 30}} + iex> Duration.from_iso8601("PT4.650S") + {:ok, %Duration{second: 4, microsecond: {650000, 3}}} + + """ + @spec from_iso8601(String.t()) :: {:ok, t} | {:error, atom} + def from_iso8601(string) when is_binary(string) do + case Calendar.ISO.parse_duration(string) do + {:ok, duration} -> + {:ok, new!(duration)} + + error -> + error + end + end + + @doc """ + Same as `from_iso8601/1` but raises an `ArgumentError`. + + ## Examples + + iex> Duration.from_iso8601!("P1Y2M3DT4H5M6S") + %Duration{year: 1, month: 2, day: 3, hour: 4, minute: 5, second: 6} + iex> Duration.from_iso8601!("P10D") + %Duration{day: 10} + + """ + @spec from_iso8601!(String.t()) :: t + def from_iso8601!(string) when is_binary(string) do + case from_iso8601(string) do + {:ok, duration} -> + duration + + {:error, reason} -> + raise ArgumentError, ~s/failed to parse duration "#{string}". reason: #{inspect(reason)}/ + end + end + + @doc """ + Converts the given `duration` to a human readable representation. + + ## Options + + * `:units` - the units to be used alongside each duration component. + The default units follow the ISO 80000-3 standard: + + [ + year: "a", + month: "mo", + week: "wk", + day: "d", + hour: "h", + minute: "min", + second: "s" + ] + + * `:separator` - a string used to separate the distinct components. Defaults to `" "`. + + ## Examples + + iex> Duration.to_string(Duration.new!(second: 30)) + "30s" + iex> Duration.to_string(Duration.new!(day: 40, hour: 12, minute: 42, second: 12)) + "40d 12h 42min 12s" + + By default, this function uses ISO 80000-3 units, which uses "a" for years. + But you can customize all units via the units option: + + iex> Duration.to_string(Duration.new!(year: 3)) + "3a" + iex> Duration.to_string(Duration.new!(year: 3), units: [year: "y"]) + "3y" + + You may also choose the separator: + + iex> Duration.to_string(Duration.new!(day: 40, hour: 12, minute: 42, second: 12), separator: ", ") + "40d, 12h, 42min, 12s" + + A duration without components is rendered as "0s": + + iex> Duration.to_string(Duration.new!([])) + "0s" + + Microseconds are rendered as part of seconds with the appropriate precision: + + iex> Duration.to_string(Duration.new!(second: 1, microsecond: {2_200, 3})) + "1.002s" + iex> Duration.to_string(Duration.new!(second: 1, microsecond: {-1_200_000, 4})) + "-0.2000s" + + """ + @doc since: "1.18.0" + @spec to_string(t, to_string_opts) :: String.t() + def to_string(%Duration{} = duration, opts \\ []) do + units = Keyword.get(opts, :units, []) + separator = Keyword.get(opts, :separator, " ") + + case to_string_year(duration, [], units) do + [] -> + "0" <> Keyword.get(units, :second, "s") + + [part] -> + IO.iodata_to_binary(part) + + parts -> + parts |> Enum.reduce(&[&1, separator | &2]) |> IO.iodata_to_binary() + end + end + + defp to_string_part(0, _units, _key, _default, acc), + do: acc + + defp to_string_part(x, units, key, default, acc), + do: [[Integer.to_string(x) | Keyword.get(units, key, default)] | acc] + + defp to_string_year(%{year: year} = duration, acc, units) do + to_string_month(duration, to_string_part(year, units, :year, "a", acc), units) + end + + defp to_string_month(%{month: month} = duration, acc, units) do + to_string_week(duration, to_string_part(month, units, :month, "mo", acc), units) + end + + defp to_string_week(%{week: week} = duration, acc, units) do + to_string_day(duration, to_string_part(week, units, :week, "wk", acc), units) + end + + defp to_string_day(%{day: day} = duration, acc, units) do + to_string_hour(duration, to_string_part(day, units, :day, "d", acc), units) + end + + defp to_string_hour(%{hour: hour} = duration, acc, units) do + to_string_minute(duration, to_string_part(hour, units, :hour, "h", acc), units) + end + + defp to_string_minute(%{minute: minute} = duration, acc, units) do + to_string_second(duration, to_string_part(minute, units, :minute, "min", acc), units) + end + + defp to_string_second(%{second: 0, microsecond: {0, _}}, acc, _units) do + acc + end + + defp to_string_second(%{second: s, microsecond: {ms, p}}, acc, units) do + [[second_component(s, ms, p) | Keyword.get(units, :second, "s")] | acc] + end + + @doc """ + Converts the given `duration` to an [ISO 8601-2:2019](https://en.wikipedia.org/wiki/ISO_8601) formatted string. + + This function implements the extension of ISO 8601:2019, allowing weeks to appear between months and days: `P3M3W3D`. + + ## Examples + + iex> Duration.to_iso8601(Duration.new!(year: 3)) + "P3Y" + iex> Duration.to_iso8601(Duration.new!(day: 40, hour: 12, minute: 42, second: 12)) + "P40DT12H42M12S" + iex> Duration.to_iso8601(Duration.new!(second: 30)) + "PT30S" + + iex> Duration.to_iso8601(Duration.new!([])) + "PT0S" + + iex> Duration.to_iso8601(Duration.new!(second: 1, microsecond: {2_200, 3})) + "PT1.002S" + iex> Duration.to_iso8601(Duration.new!(second: 1, microsecond: {-1_200_000, 4})) + "PT-0.2000S" + """ + + @spec to_iso8601(t) :: String.t() + def to_iso8601(%Duration{} = duration) do + case {to_iso8601_duration_date(duration), to_iso8601_duration_time(duration)} do + {[], []} -> "PT0S" + {date, time} -> IO.iodata_to_binary([?P, date, time]) + end + end + + defp to_iso8601_duration_date(%{year: 0, month: 0, week: 0, day: 0}) do + [] + end + + defp to_iso8601_duration_date(%{year: year, month: month, week: week, day: day}) do + [pair(year, ?Y), pair(month, ?M), pair(week, ?W), pair(day, ?D)] + end + + defp to_iso8601_duration_time(%{hour: 0, minute: 0, second: 0, microsecond: {0, _}}) do + [] + end + + defp to_iso8601_duration_time(%{hour: hour, minute: minute} = d) do + [?T, pair(hour, ?H), pair(minute, ?M), second_component(d)] + end + + defp second_component(%{second: 0, microsecond: {0, _}}) do + [] + end + + defp second_component(%{second: second, microsecond: {ms, p}}) do + [second_component(second, ms, p), ?S] + end + + defp second_component(second, _ms, 0) do + Integer.to_string(second) + end + + defp second_component(second, ms, p) do + total_ms = second * @microseconds_per_second + ms + second = total_ms |> div(@microseconds_per_second) |> abs() + ms = total_ms |> rem(@microseconds_per_second) |> abs() + sign = if total_ms < 0, do: ?-, else: [] + + [ + sign, + Integer.to_string(second), + ?., + Calendar.ISO.microseconds_to_iodata(ms, p) + ] + end + + @compile {:inline, pair: 2} + defp pair(0, _key), do: [] + defp pair(num, key), do: [Integer.to_string(num), key] +end diff --git a/lib/elixir/lib/calendar/iso.ex b/lib/elixir/lib/calendar/iso.ex new file mode 100644 index 00000000000..172da374c78 --- /dev/null +++ b/lib/elixir/lib/calendar/iso.ex @@ -0,0 +1,2173 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +defmodule Calendar.ISO do + @moduledoc """ + The default calendar implementation, a Gregorian calendar following ISO 8601. + + This calendar implements a proleptic Gregorian calendar and + is therefore compatible with the calendar used in most countries + today. The proleptic means the Gregorian rules for leap years are + applied for all time, consequently the dates give different results + before the year 1583 from when the Gregorian calendar was adopted. + + ## ISO 8601 compliance + + The ISO 8601 specification is feature-rich, but allows applications + to selectively implement most parts of it. The choices Elixir makes + are catalogued below. + + ### Features + + The standard library supports a minimal set of possible ISO 8601 features. + Specifically, the parser only supports calendar dates and does not support + ordinal and week formats. Additionally, it supports parsing ISO 8601 + formatted durations, including negative time units and fractional seconds. + + By default Elixir only parses extended-formatted date/times. You can opt-in + to parse basic-formatted date/times. + + `NaiveDateTime.to_iso8601/2` and `DateTime.to_iso8601/2` allow you to produce + either basic or extended formatted strings, and `Calendar.strftime/2` allows + you to format datetimes however else you desire. + + Elixir does not support reduced accuracy formats (for example, a date without + the day component) nor decimal precisions in the lowest component (such as + `10:01:25,5`). + + #### Examples + + Elixir expects the extended format by default when parsing: + + iex> Calendar.ISO.parse_naive_datetime("2015-01-23T23:50:07") + {:ok, {2015, 1, 23, 23, 50, 7, {0, 0}}} + iex> Calendar.ISO.parse_naive_datetime("20150123T235007") + {:error, :invalid_format} + + Parsing can be restricted to basic if desired: + + iex> Calendar.ISO.parse_naive_datetime("20150123T235007Z", :basic) + {:ok, {2015, 1, 23, 23, 50, 7, {0, 0}}} + iex> Calendar.ISO.parse_naive_datetime("20150123T235007Z", :extended) + {:error, :invalid_format} + + Only calendar dates are supported in parsing; ordinal and week dates are not. + + iex> Calendar.ISO.parse_date("2015-04-15") + {:ok, {2015, 4, 15}} + iex> Calendar.ISO.parse_date("2015-105") + {:error, :invalid_format} + iex> Calendar.ISO.parse_date("2015-W16") + {:error, :invalid_format} + iex> Calendar.ISO.parse_date("2015-W016-3") + {:error, :invalid_format} + + Years, months, days, hours, minutes, and seconds must be fully specified: + + iex> Calendar.ISO.parse_date("2015-04-15") + {:ok, {2015, 4, 15}} + iex> Calendar.ISO.parse_date("2015-04") + {:error, :invalid_format} + iex> Calendar.ISO.parse_date("2015") + {:error, :invalid_format} + + iex> Calendar.ISO.parse_time("23:50:07.0123456") + {:ok, {23, 50, 7, {12345, 6}}} + iex> Calendar.ISO.parse_time("23:50:07") + {:ok, {23, 50, 7, {0, 0}}} + iex> Calendar.ISO.parse_time("23:50") + {:error, :invalid_format} + iex> Calendar.ISO.parse_time("23") + {:error, :invalid_format} + + ### Extensions + + The parser and formatter adopt one ISO 8601 extension: extended year notation. + + This allows dates to be prefixed with a `+` or `-` sign, extending the range of + expressible years from the default (`0000..9999`) to `-9999..9999`. Elixir still + restricts years in this format to four digits. + + #### Examples + + iex> Calendar.ISO.parse_date("-2015-01-23") + {:ok, {-2015, 1, 23}} + iex> Calendar.ISO.parse_date("+2015-01-23") + {:ok, {2015, 1, 23}} + + iex> Calendar.ISO.parse_naive_datetime("-2015-01-23 23:50:07") + {:ok, {-2015, 1, 23, 23, 50, 7, {0, 0}}} + iex> Calendar.ISO.parse_naive_datetime("+2015-01-23 23:50:07") + {:ok, {2015, 1, 23, 23, 50, 7, {0, 0}}} + + iex> Calendar.ISO.parse_utc_datetime("-2015-01-23 23:50:07Z") + {:ok, {-2015, 1, 23, 23, 50, 7, {0, 0}}, 0} + iex> Calendar.ISO.parse_utc_datetime("+2015-01-23 23:50:07Z") + {:ok, {2015, 1, 23, 23, 50, 7, {0, 0}}, 0} + + ### Additions + + ISO 8601 does not allow a whitespace instead of `T` as a separator + between date and times, both when parsing and formatting. + This is a common enough representation, Elixir allows it during parsing. + + The formatting of dates in `NaiveDateTime.to_iso8601/1` and `DateTime.to_iso8601/1` + do produce specification-compliant string representations using the `T` separator. + + #### Examples + + iex> Calendar.ISO.parse_naive_datetime("2015-01-23 23:50:07.0123456") + {:ok, {2015, 1, 23, 23, 50, 7, {12345, 6}}} + iex> Calendar.ISO.parse_naive_datetime("2015-01-23T23:50:07.0123456") + {:ok, {2015, 1, 23, 23, 50, 7, {12345, 6}}} + + iex> Calendar.ISO.parse_utc_datetime("2015-01-23 23:50:07.0123456Z") + {:ok, {2015, 1, 23, 23, 50, 7, {12345, 6}}, 0} + iex> Calendar.ISO.parse_utc_datetime("2015-01-23T23:50:07.0123456Z") + {:ok, {2015, 1, 23, 23, 50, 7, {12345, 6}}, 0} + + """ + + @behaviour Calendar + + @unix_epoch 62_167_219_200 + unix_start = (315_537_897_600 + @unix_epoch) * -1_000_000 + unix_end = 315_569_519_999_999_999 - @unix_epoch * 1_000_000 + @unix_range_microseconds unix_start..unix_end + + defguardp is_format(term) when term in [:basic, :extended] + + @typedoc """ + "Before the Current Era" or "Before the Common Era" (BCE), for those years less than `1`. + """ + @type bce :: 0 + + @typedoc """ + The "Current Era" or the "Common Era" (CE) which starts in year `1`. + """ + @type ce :: 1 + + @typedoc """ + The calendar era. + + The ISO calendar has two eras: + * [CE](`t:ce/0`) - which starts in year `1` and is defined as era `1`. + * [BCE](`t:bce/0`) - for those years less than `1` and is defined as era `0`. + """ + @type era :: bce | ce + @type year :: -9999..9999 + @type month :: 1..12 + @type day :: 1..31 + @type hour :: 0..23 + @type minute :: 0..59 + @type second :: 0..59 + @type weekday :: :monday | :tuesday | :wednesday | :thursday | :friday | :saturday | :sunday + @type utc_offset :: integer + @type format :: :basic | :extended + + @typedoc """ + Microseconds with stored precision. + + The precision represents the number of digits that must be used when + representing the microseconds to external format. If the precision is 0, + it means microseconds must be skipped. + """ + @type microsecond :: {0..999_999, 0..6} + + @typedoc """ + Integer that represents the day of the week, where 1 is Monday and 7 is Sunday. + """ + @type day_of_week :: 1..7 + + @type day_of_year :: 1..366 + @type quarter_of_year :: 1..4 + @type year_of_era :: {1..10_000, era} + + @seconds_per_minute 60 + @seconds_per_hour 60 * 60 + # Note that this does *not* handle leap seconds. + @seconds_per_day 24 * 60 * 60 + @last_second_of_the_day @seconds_per_day - 1 + @microseconds_per_second 1_000_000 + @parts_per_day @seconds_per_day * @microseconds_per_second + + @datetime_seps [?\s, ?T] + @ext_date_sep ?- + @ext_time_sep ?: + + # The ISO epoch starts, in this implementation, + # with ~D[0000-01-01]. Era "1" starts + # on ~D[0001-01-01] which is 366 days later. + @iso_epoch 366 + + # Constants for date calculations using 400-year era cycles. + # The algorithm uses a March-based year where March 1 is day 0. + # Reference: Neri C, Schneider L. "Euclidean Affine Functions and + # their Application to Calendar Algorithms". Softw Pract Exper. 2022. + @days_per_year 365 + @years_per_era 400 + @days_per_era @years_per_era * @days_per_year + 97 + @days_per_4_years 4 * @days_per_year + @days_per_100_years 100 * @days_per_year + 24 + @march_1_offset 31 + 29 + @unix_epoch_days 719_528 + + # Month calculation constants: in a March-based year, each 5-month + # cycle has exactly 153 days (31+30+31+30+31 or 31+30+31+30+31). + @days_per_5_months 153 + @months_per_cycle 5 + + [match_basic_date, match_ext_date, guard_date, read_date] = + quote do + [ + <>, + <>, + y1 >= ?0 and y1 <= ?9 and y2 >= ?0 and y2 <= ?9 and y3 >= ?0 and y3 <= ?9 and y4 >= ?0 and + y4 <= ?9 and m1 >= ?0 and m1 <= ?9 and m2 >= ?0 and m2 <= ?9 and d1 >= ?0 and d1 <= ?9 and + d2 >= ?0 and d2 <= ?9, + { + (y1 - ?0) * 1000 + (y2 - ?0) * 100 + (y3 - ?0) * 10 + (y4 - ?0), + (m1 - ?0) * 10 + (m2 - ?0), + (d1 - ?0) * 10 + (d2 - ?0) + } + ] + end + + [match_basic_time, match_ext_time, guard_time, read_time] = + quote do + [ + <>, + <>, + h1 >= ?0 and h1 <= ?9 and h2 >= ?0 and h2 <= ?9 and i1 >= ?0 and i1 <= ?9 and i2 >= ?0 and + i2 <= ?9 and s1 >= ?0 and s1 <= ?9 and s2 >= ?0 and s2 <= ?9, + { + (h1 - ?0) * 10 + (h2 - ?0), + (i1 - ?0) * 10 + (i2 - ?0), + (s1 - ?0) * 10 + (s2 - ?0) + } + ] + end + + defguardp is_year(year) when is_integer(year) + defguardp is_year_BCE(year) when year <= 0 + defguardp is_year_CE(year) when year >= 1 + defguardp is_month(month) when month in 1..12 + defguardp is_day(day) when day in 1..31 + defguardp is_hour(hour) when hour in 0..23 + defguardp is_minute(minute) when minute in 0..59 + defguardp is_second(second) when second in 0..59 + + defguardp is_microsecond(microsecond, precision) + when microsecond in 0..999_999 and precision in 0..6 + + defguardp is_time_zone(term) when is_binary(term) + defguardp is_zone_abbr(term) when is_binary(term) + defguardp is_utc_offset(offset) when is_integer(offset) + defguardp is_std_offset(offset) when is_integer(offset) + + @doc """ + Converts a `t:System.time_unit/0` to precision. + + Integer-based time units always get maximum precision. + + ## Examples + + iex> Calendar.ISO.time_unit_to_precision(:nanosecond) + 6 + + iex> Calendar.ISO.time_unit_to_precision(:second) + 0 + + iex> Calendar.ISO.time_unit_to_precision(1) + 6 + + """ + @doc since: "1.15.0" + @spec time_unit_to_precision(System.time_unit()) :: 0..6 + def time_unit_to_precision(:nanosecond), do: 6 + def time_unit_to_precision(:microsecond), do: 6 + def time_unit_to_precision(:millisecond), do: 3 + def time_unit_to_precision(:second), do: 0 + def time_unit_to_precision(int) when is_integer(int), do: 6 + + @doc """ + Parses a time `string` in the `:extended` format. + + For more information on supported strings, see how this + module implements [ISO 8601](#module-iso-8601-compliance). + + ## Examples + + iex> Calendar.ISO.parse_time("23:50:07") + {:ok, {23, 50, 7, {0, 0}}} + + iex> Calendar.ISO.parse_time("23:50:07Z") + {:ok, {23, 50, 7, {0, 0}}} + iex> Calendar.ISO.parse_time("T23:50:07Z") + {:ok, {23, 50, 7, {0, 0}}} + + """ + @doc since: "1.10.0" + @impl true + @spec parse_time(String.t()) :: + {:ok, {hour, minute, second, microsecond}} + | {:error, atom} + def parse_time(string) when is_binary(string), + do: parse_time(string, :extended) + + @doc """ + Parses a time `string` according to a given `format`. + + The `format` can either be `:basic` or `:extended`. + + For more information on supported strings, see how this + module implements [ISO 8601](#module-iso-8601-compliance). + + ## Examples + + iex> Calendar.ISO.parse_time("235007", :basic) + {:ok, {23, 50, 7, {0, 0}}} + iex> Calendar.ISO.parse_time("235007", :extended) + {:error, :invalid_format} + + """ + @doc since: "1.12.0" + @spec parse_time(String.t(), format) :: + {:ok, {hour, minute, second, microsecond}} + | {:error, atom} + def parse_time(string, format) when is_binary(string) and is_format(format) do + case string do + "T" <> rest -> do_parse_time(rest, format) + _ -> do_parse_time(string, format) + end + end + + defp do_parse_time(<>, :basic) + when unquote(guard_time) do + {hour, minute, second} = unquote(read_time) + parse_formatted_time(hour, minute, second, rest) + end + + defp do_parse_time(<>, :extended) + when unquote(guard_time) do + {hour, minute, second} = unquote(read_time) + parse_formatted_time(hour, minute, second, rest) + end + + defp do_parse_time(_, _) do + {:error, :invalid_format} + end + + defp parse_formatted_time(hour, minute, second, rest) do + with {microsecond, rest} <- parse_microsecond(rest), + {_offset, ""} <- parse_offset(rest) do + if valid_time?(hour, minute, second, microsecond) do + {:ok, {hour, minute, second, microsecond}} + else + {:error, :invalid_time} + end + else + _ -> {:error, :invalid_format} + end + end + + @doc """ + Parses a date `string` in the `:extended` format. + + For more information on supported strings, see how this + module implements [ISO 8601](#module-iso-8601-compliance). + + ## Examples + + iex> Calendar.ISO.parse_date("2015-01-23") + {:ok, {2015, 1, 23}} + + iex> Calendar.ISO.parse_date("2015:01:23") + {:error, :invalid_format} + iex> Calendar.ISO.parse_date("2015-01-32") + {:error, :invalid_date} + + """ + @doc since: "1.10.0" + @impl true + @spec parse_date(String.t()) :: + {:ok, {year, month, day}} + | {:error, atom} + def parse_date(string) when is_binary(string), + do: parse_date(string, :extended) + + @doc """ + Parses a date `string` according to a given `format`. + + The `format` can either be `:basic` or `:extended`. + + For more information on supported strings, see how this + module implements [ISO 8601](#module-iso-8601-compliance). + + ## Examples + + iex> Calendar.ISO.parse_date("20150123", :basic) + {:ok, {2015, 1, 23}} + iex> Calendar.ISO.parse_date("20150123", :extended) + {:error, :invalid_format} + + """ + @doc since: "1.12.0" + @spec parse_date(String.t(), format) :: + {:ok, {year, month, day}} + | {:error, atom} + def parse_date(string, format) when is_binary(string) and is_format(format), + do: parse_date_guarded(string, format) + + defp parse_date_guarded("-" <> string, format), + do: do_parse_date(string, -1, format) + + defp parse_date_guarded("+" <> string, format), + do: do_parse_date(string, 1, format) + + defp parse_date_guarded(string, format), + do: do_parse_date(string, 1, format) + + defp do_parse_date(unquote(match_basic_date), multiplier, :basic) when unquote(guard_date) do + {year, month, day} = unquote(read_date) + parse_formatted_date(year, month, day, multiplier) + end + + defp do_parse_date(unquote(match_ext_date), multiplier, :extended) when unquote(guard_date) do + {year, month, day} = unquote(read_date) + parse_formatted_date(year, month, day, multiplier) + end + + defp do_parse_date(_, _, _) do + {:error, :invalid_format} + end + + defp parse_formatted_date(year, month, day, multiplier) do + year = multiplier * year + + if valid_date?(year, month, day) do + {:ok, {year, month, day}} + else + {:error, :invalid_date} + end + end + + @doc """ + Parses a naive datetime `string` in the `:extended` format. + + For more information on supported strings, see how this + module implements [ISO 8601](#module-iso-8601-compliance). + + ## Examples + + iex> Calendar.ISO.parse_naive_datetime("2015-01-23 23:50:07") + {:ok, {2015, 1, 23, 23, 50, 7, {0, 0}}} + iex> Calendar.ISO.parse_naive_datetime("2015-01-23 23:50:07Z") + {:ok, {2015, 1, 23, 23, 50, 7, {0, 0}}} + iex> Calendar.ISO.parse_naive_datetime("2015-01-23 23:50:07-02:30") + {:ok, {2015, 1, 23, 23, 50, 7, {0, 0}}} + + iex> Calendar.ISO.parse_naive_datetime("2015-01-23 23:50:07.0") + {:ok, {2015, 1, 23, 23, 50, 7, {0, 1}}} + iex> Calendar.ISO.parse_naive_datetime("2015-01-23 23:50:07,0123456") + {:ok, {2015, 1, 23, 23, 50, 7, {12345, 6}}} + + """ + @doc since: "1.10.0" + @impl true + @spec parse_naive_datetime(String.t()) :: + {:ok, {year, month, day, hour, minute, second, microsecond}} + | {:error, atom} + def parse_naive_datetime(string) when is_binary(string), + do: parse_naive_datetime(string, :extended) + + @doc """ + Parses a naive datetime `string` according to a given `format`. + + The `format` can either be `:basic` or `:extended`. + + For more information on supported strings, see how this + module implements [ISO 8601](#module-iso-8601-compliance). + + ## Examples + + iex> Calendar.ISO.parse_naive_datetime("20150123 235007", :basic) + {:ok, {2015, 1, 23, 23, 50, 7, {0, 0}}} + iex> Calendar.ISO.parse_naive_datetime("20150123 235007", :extended) + {:error, :invalid_format} + + """ + @doc since: "1.12.0" + @spec parse_naive_datetime(String.t(), format) :: + {:ok, {year, month, day, hour, minute, second, microsecond}} + | {:error, atom} + def parse_naive_datetime(string, format) when is_binary(string) and is_format(format), + do: parse_naive_datetime_guarded(string, format) + + defp parse_naive_datetime_guarded("-" <> string, format), + do: do_parse_naive_datetime(string, -1, format) + + defp parse_naive_datetime_guarded("+" <> string, format), + do: do_parse_naive_datetime(string, 1, format) + + defp parse_naive_datetime_guarded(string, format), + do: do_parse_naive_datetime(string, 1, format) + + defp do_parse_naive_datetime( + <>, + multiplier, + :basic + ) + when unquote(guard_date) and datetime_sep in @datetime_seps and unquote(guard_time) do + {year, month, day} = unquote(read_date) + {hour, minute, second} = unquote(read_time) + parse_formatted_naive_datetime(year, month, day, hour, minute, second, rest, multiplier) + end + + defp do_parse_naive_datetime( + <>, + multiplier, + :extended + ) + when unquote(guard_date) and datetime_sep in @datetime_seps and unquote(guard_time) do + {year, month, day} = unquote(read_date) + {hour, minute, second} = unquote(read_time) + parse_formatted_naive_datetime(year, month, day, hour, minute, second, rest, multiplier) + end + + defp do_parse_naive_datetime(_, _, _) do + {:error, :invalid_format} + end + + defp parse_formatted_naive_datetime(year, month, day, hour, minute, second, rest, multiplier) do + year = multiplier * year + + with {microsecond, rest} <- parse_microsecond(rest), + {_offset, ""} <- parse_offset(rest) do + cond do + not valid_date?(year, month, day) -> + {:error, :invalid_date} + + not valid_time?(hour, minute, second, microsecond) -> + {:error, :invalid_time} + + true -> + {:ok, {year, month, day, hour, minute, second, microsecond}} + end + else + _ -> {:error, :invalid_format} + end + end + + @doc """ + Parses a UTC datetime `string` in the `:extended` format. + + For more information on supported strings, see how this + module implements [ISO 8601](#module-iso-8601-compliance). + + ## Examples + + iex> Calendar.ISO.parse_utc_datetime("2015-01-23 23:50:07Z") + {:ok, {2015, 1, 23, 23, 50, 7, {0, 0}}, 0} + + iex> Calendar.ISO.parse_utc_datetime("2015-01-23 23:50:07+02:30") + {:ok, {2015, 1, 23, 21, 20, 7, {0, 0}}, 9000} + + iex> Calendar.ISO.parse_utc_datetime("2015-01-23 23:50:07") + {:error, :missing_offset} + + """ + @doc since: "1.10.0" + @impl true + @spec parse_utc_datetime(String.t()) :: + {:ok, {year, month, day, hour, minute, second, microsecond}, utc_offset} + | {:error, atom} + def parse_utc_datetime(string) when is_binary(string), + do: parse_utc_datetime(string, :extended) + + @doc """ + Parses a UTC datetime `string` according to a given `format`. + + The `format` can either be `:basic` or `:extended`. + + For more information on supported strings, see how this + module implements [ISO 8601](#module-iso-8601-compliance). + + ## Examples + + iex> Calendar.ISO.parse_utc_datetime("20150123 235007Z", :basic) + {:ok, {2015, 1, 23, 23, 50, 7, {0, 0}}, 0} + iex> Calendar.ISO.parse_utc_datetime("20150123 235007Z", :extended) + {:error, :invalid_format} + + """ + @doc since: "1.12.0" + @spec parse_utc_datetime(String.t(), format) :: + {:ok, {year, month, day, hour, minute, second, microsecond}, utc_offset} + | {:error, atom} + def parse_utc_datetime(string, format) when is_binary(string) and is_format(format), + do: parse_utc_datetime_guarded(string, format) + + defp parse_utc_datetime_guarded("-" <> string, format), + do: do_parse_utc_datetime(string, -1, format) + + defp parse_utc_datetime_guarded("+" <> string, format), + do: do_parse_utc_datetime(string, 1, format) + + defp parse_utc_datetime_guarded(string, format), + do: do_parse_utc_datetime(string, 1, format) + + defp do_parse_utc_datetime( + <>, + multiplier, + :basic + ) + when unquote(guard_date) and datetime_sep in @datetime_seps and unquote(guard_time) do + {year, month, day} = unquote(read_date) + {hour, minute, second} = unquote(read_time) + parse_formatted_utc_datetime(year, month, day, hour, minute, second, rest, multiplier) + end + + defp do_parse_utc_datetime( + <>, + multiplier, + :extended + ) + when unquote(guard_date) and datetime_sep in @datetime_seps and unquote(guard_time) do + {year, month, day} = unquote(read_date) + {hour, minute, second} = unquote(read_time) + parse_formatted_utc_datetime(year, month, day, hour, minute, second, rest, multiplier) + end + + defp do_parse_utc_datetime(_, _, _) do + {:error, :invalid_format} + end + + defp parse_formatted_utc_datetime(year, month, day, hour, minute, second, rest, multiplier) do + year = multiplier * year + + with {microsecond, rest} <- parse_microsecond(rest), + {offset, ""} <- parse_offset(rest) do + cond do + not valid_date?(year, month, day) -> + {:error, :invalid_date} + + not valid_time?(hour, minute, second, microsecond) -> + {:error, :invalid_time} + + offset == 0 -> + {:ok, {year, month, day, hour, minute, second, microsecond}, offset} + + is_nil(offset) -> + {:error, :missing_offset} + + true -> + day_fraction = time_to_day_fraction(hour, minute, second, {0, 0}) + + {{year, month, day}, {hour, minute, second, _}} = + case add_day_fraction_to_iso_days({0, day_fraction}, -offset, 86_400) do + {0, day_fraction} -> + {{year, month, day}, time_from_day_fraction(day_fraction)} + + {extra_days, day_fraction} -> + base_days = date_to_iso_days(year, month, day) + {date_from_iso_days(base_days + extra_days), time_from_day_fraction(day_fraction)} + end + + {:ok, {year, month, day, hour, minute, second, microsecond}, offset} + end + else + _ -> {:error, :invalid_format} + end + end + + @doc """ + Parses an ISO 8601 formatted duration string to a list of `Duration` compabitble unit pairs. + + See `Duration.from_iso8601/1`. + """ + @doc since: "1.17.0" + @spec parse_duration(String.t()) :: {:ok, [Duration.unit_pair()]} | {:error, atom} + def parse_duration("P" <> string) when byte_size(string) > 0 do + parse_duration_date(string, [], year: ?Y, month: ?M, week: ?W, day: ?D) + end + + def parse_duration("+P" <> string) when byte_size(string) > 0 do + parse_duration_date(string, [], year: ?Y, month: ?M, week: ?W, day: ?D) + end + + def parse_duration("-P" <> string) when byte_size(string) > 0 do + with {:ok, fields} <- parse_duration_date(string, [], year: ?Y, month: ?M, week: ?W, day: ?D) do + {:ok, + Enum.map(fields, fn + {:microsecond, {value, precision}} -> {:microsecond, {-value, precision}} + {unit, value} -> {unit, -value} + end)} + end + end + + def parse_duration(_) do + {:error, :invalid_duration} + end + + defp parse_duration_date("", acc, _allowed), do: {:ok, acc} + + defp parse_duration_date("T" <> string, acc, _allowed) when byte_size(string) > 0 do + parse_duration_time(string, acc, hour: ?H, minute: ?M, second: ?S) + end + + defp parse_duration_date(string, acc, allowed) do + with {integer, <>} <- Integer.parse(string), + {key, allowed} <- find_unit(allowed, next) do + parse_duration_date(rest, [{key, integer} | acc], allowed) + else + _ -> {:error, :invalid_date_component} + end + end + + defp parse_duration_time("", acc, _allowed), do: {:ok, acc} + + defp parse_duration_time(string, acc, allowed) do + case Integer.parse(string) do + {second, <> = rest} when delimiter in [?., ?,] -> + case parse_microsecond(rest) do + {{ms, precision}, "S"} -> + ms = + case string do + "-" <> _ -> + -ms + + _ -> + ms + end + + {:ok, [second: second, microsecond: {ms, precision}] ++ acc} + + _ -> + {:error, :invalid_time_component} + end + + {integer, <>} -> + case find_unit(allowed, next) do + {key, allowed} -> parse_duration_time(rest, [{key, integer} | acc], allowed) + false -> {:error, :invalid_time_component} + end + + _ -> + {:error, :invalid_time_component} + end + end + + defp find_unit([{key, unit} | rest], unit), do: {key, rest} + defp find_unit([_ | rest], unit), do: find_unit(rest, unit) + defp find_unit([], _unit), do: false + + @doc """ + Returns the `t:Calendar.iso_days/0` format of the specified date. + + ## Examples + + iex> Calendar.ISO.naive_datetime_to_iso_days(0, 1, 1, 0, 0, 0, {0, 6}) + {0, {0, 86400000000}} + iex> Calendar.ISO.naive_datetime_to_iso_days(2000, 1, 1, 12, 0, 0, {0, 6}) + {730485, {43200000000, 86400000000}} + iex> Calendar.ISO.naive_datetime_to_iso_days(2000, 1, 1, 13, 0, 0, {0, 6}) + {730485, {46800000000, 86400000000}} + iex> Calendar.ISO.naive_datetime_to_iso_days(-1, 1, 1, 0, 0, 0, {0, 6}) + {-365, {0, 86400000000}} + + """ + @doc since: "1.5.0" + @impl true + @spec naive_datetime_to_iso_days( + Calendar.year(), + Calendar.month(), + Calendar.day(), + Calendar.hour(), + Calendar.minute(), + Calendar.second(), + Calendar.microsecond() + ) :: Calendar.iso_days() + def naive_datetime_to_iso_days(year, month, day, hour, minute, second, microsecond) do + {date_to_iso_days(year, month, day), time_to_day_fraction(hour, minute, second, microsecond)} + end + + @doc """ + Converts the `t:Calendar.iso_days/0` format to the datetime format specified by this calendar. + + ## Examples + + iex> Calendar.ISO.naive_datetime_from_iso_days({0, {0, 86_400}}) + {0, 1, 1, 0, 0, 0, {0, 6}} + iex> Calendar.ISO.naive_datetime_from_iso_days({730_485, {0, 86_400}}) + {2000, 1, 1, 0, 0, 0, {0, 6}} + iex> Calendar.ISO.naive_datetime_from_iso_days({730_485, {43_200, 86_400}}) + {2000, 1, 1, 12, 0, 0, {0, 6}} + iex> Calendar.ISO.naive_datetime_from_iso_days({-365, {0, 86_400_000_000}}) + {-1, 1, 1, 0, 0, 0, {0, 6}} + + """ + @doc since: "1.5.0" + @spec naive_datetime_from_iso_days(Calendar.iso_days()) :: { + Calendar.year(), + Calendar.month(), + Calendar.day(), + Calendar.hour(), + Calendar.minute(), + Calendar.second(), + Calendar.microsecond() + } + @impl true + def naive_datetime_from_iso_days({days, day_fraction}) do + {year, month, day} = date_from_iso_days(days) + {hour, minute, second, microsecond} = time_from_day_fraction(day_fraction) + {year, month, day, hour, minute, second, microsecond} + end + + @doc """ + Returns the normalized day fraction of the specified time. + + ## Examples + + iex> Calendar.ISO.time_to_day_fraction(0, 0, 0, {0, 6}) + {0, 86400000000} + iex> Calendar.ISO.time_to_day_fraction(12, 34, 56, {123, 6}) + {45296000123, 86400000000} + + """ + @doc since: "1.5.0" + @impl true + @spec time_to_day_fraction( + Calendar.hour(), + Calendar.minute(), + Calendar.second(), + Calendar.microsecond() + ) :: Calendar.day_fraction() + def time_to_day_fraction(0, 0, 0, {0, _}) do + {0, @parts_per_day} + end + + def time_to_day_fraction(hour, minute, second, {microsecond, _}) do + combined_seconds = hour * @seconds_per_hour + minute * @seconds_per_minute + second + {combined_seconds * @microseconds_per_second + microsecond, @parts_per_day} + end + + @doc """ + Converts a day fraction to this Calendar's representation of time. + + ## Examples + + iex> Calendar.ISO.time_from_day_fraction({1, 2}) + {12, 0, 0, {0, 6}} + iex> Calendar.ISO.time_from_day_fraction({13, 24}) + {13, 0, 0, {0, 6}} + + """ + @doc since: "1.5.0" + @impl true + @spec time_from_day_fraction(Calendar.day_fraction()) :: + {hour(), minute(), second(), microsecond()} + def time_from_day_fraction({0, _}) do + {0, 0, 0, {0, 6}} + end + + def time_from_day_fraction({parts_in_day, parts_per_day}) do + total_microseconds = divide_by_parts_per_day(parts_in_day, parts_per_day) + + {hours, rest_microseconds1} = + div_rem(total_microseconds, @seconds_per_hour * @microseconds_per_second) + + {minutes, rest_microseconds2} = + div_rem(rest_microseconds1, @seconds_per_minute * @microseconds_per_second) + + {seconds, microseconds} = div_rem(rest_microseconds2, @microseconds_per_second) + {hours, minutes, seconds, {microseconds, 6}} + end + + defp divide_by_parts_per_day(parts_in_day, @parts_per_day), do: parts_in_day + + defp divide_by_parts_per_day(parts_in_day, parts_per_day), + do: div(parts_in_day * @parts_per_day, parts_per_day) + + # Converts year, month, day to count of days since 0000-01-01. + @doc false + def date_to_iso_days(0, 1, 1), do: 0 + def date_to_iso_days(1970, 1, 1), do: @unix_epoch_days + + def date_to_iso_days(year, month, day) do + ensure_day_in_month!(year, month, day) + + y = if month <= 2, do: year - 1, else: year + era = if y >= 0, do: div(y, @years_per_era), else: div(y - 399, @years_per_era) + year_of_era = y - era * @years_per_era + month_prime = if month > 2, do: month - 3, else: month + 9 + day_of_year = div(@days_per_5_months * month_prime + 2, @months_per_cycle) + day - 1 + + day_of_era = + @days_per_year * year_of_era + div(year_of_era, 4) - div(year_of_era, 100) + day_of_year + + era * @days_per_era + day_of_era + @march_1_offset + end + + # Converts count of days since 0000-01-01 to {year, month, day} tuple. + @doc false + def date_from_iso_days(days) do + z = days - @march_1_offset + era = if z >= 0, do: div(z, @days_per_era), else: div(z - @days_per_era + 1, @days_per_era) + day_of_era = z - era * @days_per_era + + year_of_era = + div( + day_of_era - div(day_of_era, @days_per_4_years) + div(day_of_era, @days_per_100_years) - + div(day_of_era, @days_per_era - 1), + @days_per_year + ) + + day_of_year = + day_of_era - + (@days_per_year * year_of_era + div(year_of_era, 4) - div(year_of_era, 100)) + + month_prime = div(@months_per_cycle * day_of_year + 2, @days_per_5_months) + day = day_of_year - div(@days_per_5_months * month_prime + 2, @months_per_cycle) + 1 + month = if month_prime < 10, do: month_prime + 3, else: month_prime - 9 + year = year_of_era + era * @years_per_era + year = if month <= 2, do: year + 1, else: year + + {year, month, day} + end + + defp div_rem(int1, int2) do + div = div(int1, int2) + rem = int1 - div * int2 + + if rem >= 0 do + {div, rem} + else + {div - 1, rem + int2} + end + end + + defp floor_div_positive_divisor(int1, int2) when int1 >= 0, do: div(int1, int2) + defp floor_div_positive_divisor(int1, int2), do: -div(-int1 - 1, int2) - 1 + + @doc """ + Returns how many days there are in the given year-month. + + ## Examples + + iex> Calendar.ISO.days_in_month(1900, 1) + 31 + iex> Calendar.ISO.days_in_month(1900, 2) + 28 + iex> Calendar.ISO.days_in_month(2000, 2) + 29 + iex> Calendar.ISO.days_in_month(2001, 2) + 28 + iex> Calendar.ISO.days_in_month(2004, 2) + 29 + iex> Calendar.ISO.days_in_month(2004, 4) + 30 + iex> Calendar.ISO.days_in_month(-1, 5) + 31 + + """ + @doc since: "1.4.0" + @spec days_in_month(year, month) :: 28..31 + @impl true + def days_in_month(year, month) when is_year(year) and is_month(month) do + days_in_month_guarded(year, month) + end + + defp days_in_month_guarded(year, 2) do + if leap_year?(year), do: 29, else: 28 + end + + defp days_in_month_guarded(_, month) when month in [4, 6, 9, 11], do: 30 + defp days_in_month_guarded(_, _), do: 31 + + @doc """ + Returns how many months there are in the given year. + + ## Example + + iex> Calendar.ISO.months_in_year(2004) + 12 + + """ + @doc since: "1.7.0" + @impl true + @spec months_in_year(year) :: 12 + def months_in_year(year) when is_year(year) do + 12 + end + + @doc """ + Returns if the given year is a leap year. + + ## Examples + + iex> Calendar.ISO.leap_year?(2000) + true + iex> Calendar.ISO.leap_year?(2001) + false + iex> Calendar.ISO.leap_year?(2004) + true + iex> Calendar.ISO.leap_year?(1900) + false + iex> Calendar.ISO.leap_year?(-4) + true + + """ + @doc since: "1.3.0" + @spec leap_year?(year) :: boolean() + @impl true + def leap_year?(year) when is_year(year) do + rem(year, 4) === 0 and (rem(year, 100) !== 0 or rem(year, 400) === 0) + end + + @doc false + @deprecated "Use Calendar.ISO.day_of_week/4 instead" + def day_of_week(year, month, day) do + day_of_week(year, month, day, :default) |> elem(0) + end + + @doc """ + Calculates the day of the week from the given `year`, `month`, and `day`. + + It is an integer from 1 to 7, where 1 is the given `starting_on` weekday. + For example, if `starting_on` is set to `:monday`, then 1 is Monday and + 7 is Sunday. + + `starting_on` can also be `:default`, which is equivalent to `:monday`. + + ## Examples + + iex> Calendar.ISO.day_of_week(2016, 10, 31, :monday) + {1, 1, 7} + iex> Calendar.ISO.day_of_week(2016, 11, 1, :monday) + {2, 1, 7} + iex> Calendar.ISO.day_of_week(2016, 11, 2, :monday) + {3, 1, 7} + iex> Calendar.ISO.day_of_week(2016, 11, 3, :monday) + {4, 1, 7} + iex> Calendar.ISO.day_of_week(2016, 11, 4, :monday) + {5, 1, 7} + iex> Calendar.ISO.day_of_week(2016, 11, 5, :monday) + {6, 1, 7} + iex> Calendar.ISO.day_of_week(2016, 11, 6, :monday) + {7, 1, 7} + iex> Calendar.ISO.day_of_week(-99, 1, 31, :monday) + {4, 1, 7} + + iex> Calendar.ISO.day_of_week(2016, 10, 31, :sunday) + {2, 1, 7} + iex> Calendar.ISO.day_of_week(2016, 11, 1, :sunday) + {3, 1, 7} + iex> Calendar.ISO.day_of_week(2016, 11, 2, :sunday) + {4, 1, 7} + iex> Calendar.ISO.day_of_week(2016, 11, 3, :sunday) + {5, 1, 7} + iex> Calendar.ISO.day_of_week(2016, 11, 4, :sunday) + {6, 1, 7} + iex> Calendar.ISO.day_of_week(2016, 11, 5, :sunday) + {7, 1, 7} + iex> Calendar.ISO.day_of_week(2016, 11, 6, :sunday) + {1, 1, 7} + iex> Calendar.ISO.day_of_week(-99, 1, 31, :sunday) + {5, 1, 7} + + iex> Calendar.ISO.day_of_week(2016, 10, 31, :saturday) + {3, 1, 7} + + """ + @doc since: "1.11.0" + @spec day_of_week(year, month, day, :default | weekday) :: {day_of_week(), 1, 7} + @impl true + def day_of_week(year, month, day, starting_on) do + iso_days = date_to_iso_days(year, month, day) + {iso_days_to_day_of_week(iso_days, starting_on), 1, 7} + end + + @doc false + def iso_days_to_day_of_week(iso_days, starting_on) do + Integer.mod(iso_days + day_of_week_offset(starting_on), 7) + 1 + end + + defp day_of_week_offset(:default), do: 5 + defp day_of_week_offset(:wednesday), do: 3 + defp day_of_week_offset(:thursday), do: 2 + defp day_of_week_offset(:friday), do: 1 + defp day_of_week_offset(:saturday), do: 0 + defp day_of_week_offset(:sunday), do: 6 + defp day_of_week_offset(:monday), do: 5 + defp day_of_week_offset(:tuesday), do: 4 + + @doc """ + Calculates the day of the year from the given `year`, `month`, and `day`. + + It is an integer from 1 to 366. + + ## Examples + + iex> Calendar.ISO.day_of_year(2016, 1, 31) + 31 + iex> Calendar.ISO.day_of_year(-99, 2, 1) + 32 + iex> Calendar.ISO.day_of_year(2018, 2, 28) + 59 + + """ + @doc since: "1.8.0" + @spec day_of_year(year, month, day) :: day_of_year() + @impl true + def day_of_year(year, month, day) do + ensure_day_in_month!(year, month, day) + days_before_month(month) + leap_day_offset(year, month) + day + end + + @doc """ + Calculates the quarter of the year from the given `year`, `month`, and `day`. + + It is an integer from 1 to 4. + + ## Examples + + iex> Calendar.ISO.quarter_of_year(2016, 1, 31) + 1 + iex> Calendar.ISO.quarter_of_year(2016, 4, 3) + 2 + iex> Calendar.ISO.quarter_of_year(-99, 9, 31) + 3 + iex> Calendar.ISO.quarter_of_year(2018, 12, 28) + 4 + + """ + @doc since: "1.8.0" + @spec quarter_of_year(year, month, day) :: quarter_of_year() + @impl true + def quarter_of_year(year, month, day) + when is_year(year) and is_month(month) and is_day(day) do + div(month - 1, 3) + 1 + end + + @doc """ + Calculates the year and era from the given `year`. + + The ISO calendar has two eras: the "current era" (CE) which + starts in year `1` and is defined as era `1`. And "before the current + era" (BCE) for those years less than `1`, defined as era `0`. + + ## Examples + + iex> Calendar.ISO.year_of_era(1) + {1, 1} + iex> Calendar.ISO.year_of_era(2018) + {2018, 1} + iex> Calendar.ISO.year_of_era(0) + {1, 0} + iex> Calendar.ISO.year_of_era(-1) + {2, 0} + + """ + @doc since: "1.8.0" + @spec year_of_era(year) :: {1..10_000, era} + def year_of_era(year) when is_year_CE(year), do: {year, 1} + def year_of_era(year) when is_year_BCE(year), do: {abs(year) + 1, 0} + + @doc """ + Calendar callback to compute the year and era from the + given `year`, `month` and `day`. + + In the ISO calendar, the new year coincides with the new era, + so the `month` and `day` arguments are discarded. If you only + have the year available, you can `year_of_era/1` instead. + + ## Examples + + iex> Calendar.ISO.year_of_era(1, 1, 1) + {1, 1} + iex> Calendar.ISO.year_of_era(2018, 12, 1) + {2018, 1} + iex> Calendar.ISO.year_of_era(0, 1, 1) + {1, 0} + iex> Calendar.ISO.year_of_era(-1, 12, 1) + {2, 0} + + """ + @doc since: "1.13.0" + @impl true + @spec year_of_era(year, month, day) :: {1..10_000, era} + def year_of_era(year, _month, _day), do: year_of_era(year) + + @doc """ + Calculates the day and era from the given `year`, `month`, and `day`. + + ## Examples + + iex> Calendar.ISO.day_of_era(0, 1, 1) + {366, 0} + iex> Calendar.ISO.day_of_era(1, 1, 1) + {1, 1} + iex> Calendar.ISO.day_of_era(0, 12, 31) + {1, 0} + iex> Calendar.ISO.day_of_era(0, 12, 30) + {2, 0} + iex> Calendar.ISO.day_of_era(-1, 12, 31) + {367, 0} + + """ + @doc since: "1.8.0" + @spec day_of_era(year, month, day) :: Calendar.day_of_era() + @impl true + def day_of_era(year, month, day) when is_year_CE(year) do + day = date_to_iso_days(year, month, day) - @iso_epoch + 1 + {day, 1} + end + + def day_of_era(year, month, day) when is_year_BCE(year) do + day = abs(date_to_iso_days(year, month, day) - @iso_epoch) + {day, 0} + end + + @doc """ + Converts the given time into a string. + + By default, returns times formatted in the "extended" format, + for human readability. It also supports the "basic" format + by passing the `:basic` option. + + ## Examples + + iex> Calendar.ISO.time_to_string(2, 2, 2, {2, 6}) + "02:02:02.000002" + iex> Calendar.ISO.time_to_string(2, 2, 2, {2, 2}) + "02:02:02.00" + iex> Calendar.ISO.time_to_string(2, 2, 2, {2, 0}) + "02:02:02" + + iex> Calendar.ISO.time_to_string(2, 2, 2, {2, 6}, :basic) + "020202.000002" + iex> Calendar.ISO.time_to_string(2, 2, 2, {2, 6}, :extended) + "02:02:02.000002" + + """ + @impl true + @doc since: "1.5.0" + @spec time_to_string( + Calendar.hour(), + Calendar.minute(), + Calendar.second(), + Calendar.microsecond(), + :basic | :extended + ) :: String.t() + def time_to_string( + hour, + minute, + second, + microsecond, + format \\ :extended + ) do + time_to_iodata(hour, minute, second, microsecond, format) + |> IO.iodata_to_binary() + end + + @doc """ + Converts the given time into a iodata. + + See `time_to_string/5` for more information. + + ## Examples + + iex> data = Calendar.ISO.time_to_iodata(2, 2, 2, {2, 6}) + iex> IO.iodata_to_binary(data) + "02:02:02.000002" + + """ + @doc since: "1.19.0" + @spec time_to_iodata( + Calendar.hour(), + Calendar.minute(), + Calendar.second(), + Calendar.microsecond(), + :basic | :extended + ) :: iodata + def time_to_iodata( + hour, + minute, + second, + {ms_value, ms_precision} = microsecond, + format \\ :extended + ) + when is_hour(hour) and is_minute(minute) and is_second(second) and + is_microsecond(ms_value, ms_precision) and format in [:basic, :extended] do + time_to_iodata_guarded(hour, minute, second, microsecond, format) + end + + defp time_to_iodata_guarded(hour, minute, second, {_, 0}, format) do + time_to_iodata_format(hour, minute, second, format) + end + + defp time_to_iodata_guarded(hour, minute, second, {microsecond, precision}, format) do + [ + time_to_iodata_format(hour, minute, second, format), + ?. + | microseconds_to_iodata(microsecond, precision) + ] + end + + @doc false + def microseconds_to_iodata(_microsecond, 0), do: [] + def microseconds_to_iodata(microsecond, 6), do: zero_pad(microsecond, 6) + + def microseconds_to_iodata(microsecond, precision) do + num = div(microsecond, scale_factor(precision)) + zero_pad(num, precision) + end + + defp scale_factor(1), do: 100_000 + defp scale_factor(2), do: 10_000 + defp scale_factor(3), do: 1_000 + defp scale_factor(4), do: 100 + defp scale_factor(5), do: 10 + defp scale_factor(6), do: 1 + + defp time_to_iodata_format(hour, minute, second, :extended) do + [zero_pad(hour, 2), ?:, zero_pad(minute, 2), ?: | zero_pad(second, 2)] + end + + defp time_to_iodata_format(hour, minute, second, :basic) do + [zero_pad(hour, 2), zero_pad(minute, 2) | zero_pad(second, 2)] + end + + @doc """ + Converts the given date into a string. + + By default, returns dates formatted in the "extended" format, + for human readability. It also supports the "basic" format + by passing the `:basic` option. + + ## Examples + + iex> Calendar.ISO.date_to_string(2015, 2, 28) + "2015-02-28" + iex> Calendar.ISO.date_to_string(2017, 8, 1) + "2017-08-01" + iex> Calendar.ISO.date_to_string(-99, 1, 31) + "-0099-01-31" + + iex> Calendar.ISO.date_to_string(2015, 2, 28, :basic) + "20150228" + iex> Calendar.ISO.date_to_string(-99, 1, 31, :basic) + "-00990131" + + """ + @doc since: "1.4.0" + @spec date_to_string(year, month, day, :basic | :extended) :: String.t() + @impl true + def date_to_string(year, month, day, format \\ :extended) do + date_to_iodata(year, month, day, format) + |> IO.iodata_to_binary() + end + + @doc """ + Converts the given date into a iodata. + + See `date_to_string/4` for more information. + + ## Examples + + iex> data = Calendar.ISO.date_to_iodata(2015, 2, 28) + iex> IO.iodata_to_binary(data) + "2015-02-28" + """ + @doc since: "1.19.0" + @spec date_to_iodata(year, month, day, :basic | :extended) :: iodata + def date_to_iodata(year, month, day, format \\ :extended) + when is_integer(year) and is_integer(month) and is_integer(day) and + format in [:basic, :extended] do + date_to_iodata_guarded(year, month, day, format) + end + + defp date_to_iodata_guarded(year, month, day, :extended) do + [zero_pad(year, 4), ?-, zero_pad(month, 2), ?- | zero_pad(day, 2)] + end + + defp date_to_iodata_guarded(year, month, day, :basic) do + [zero_pad(year, 4), zero_pad(month, 2) | zero_pad(day, 2)] + end + + @doc """ + Converts the datetime (without time zone) into a string. + + By default, returns datetimes formatted in the "extended" format, + for human readability. It also supports the "basic" format + by passing the `:basic` option. + + ## Examples + + iex> Calendar.ISO.naive_datetime_to_string(2015, 2, 28, 1, 2, 3, {4, 6}) + "2015-02-28 01:02:03.000004" + iex> Calendar.ISO.naive_datetime_to_string(2017, 8, 1, 1, 2, 3, {4, 5}) + "2017-08-01 01:02:03.00000" + + iex> Calendar.ISO.naive_datetime_to_string(2015, 2, 28, 1, 2, 3, {4, 6}, :basic) + "20150228 010203.000004" + + """ + @doc since: "1.4.0" + @impl true + @spec naive_datetime_to_string( + year, + month, + day, + Calendar.hour(), + Calendar.minute(), + Calendar.second(), + Calendar.microsecond(), + :basic | :extended + ) :: String.t() + def naive_datetime_to_string( + year, + month, + day, + hour, + minute, + second, + microsecond, + format \\ :extended + ) do + naive_datetime_to_iodata( + year, + month, + day, + hour, + minute, + second, + microsecond, + format + ) + |> IO.iodata_to_binary() + end + + @doc """ + Converts the given naive_datetime into a iodata. + + See `naive_datetime_to_iodata/8` for more information. + + ## Examples + + iex> data = Calendar.ISO.naive_datetime_to_iodata(2015, 2, 28, 1, 2, 3, {4, 6}, :basic) + iex> IO.iodata_to_binary(data) + "20150228 010203.000004" + + iex> data = Calendar.ISO.naive_datetime_to_iodata(2015, 2, 28, 1, 2, 3, {4, 6}, :extended) + iex> IO.iodata_to_binary(data) + "2015-02-28 01:02:03.000004" + + """ + @doc since: "1.19.0" + @spec naive_datetime_to_iodata( + year, + month, + day, + Calendar.hour(), + Calendar.minute(), + Calendar.second(), + Calendar.microsecond(), + :basic | :extended + ) :: iodata + def naive_datetime_to_iodata( + year, + month, + day, + hour, + minute, + second, + microsecond, + format \\ :extended + ) do + [ + date_to_iodata(year, month, day, format), + ?\s + | time_to_iodata(hour, minute, second, microsecond, format) + ] + end + + @doc """ + Converts the datetime (with time zone) into a string. + + By default, returns datetimes formatted in the "extended" format, + for human readability. It also supports the "basic" format + by passing the `:basic` option. + + ## Examples + + iex> time_zone = "Etc/UTC" + iex> Calendar.ISO.datetime_to_string(2017, 8, 1, 1, 2, 3, {4, 5}, time_zone, "UTC", 0, 0) + "2017-08-01 01:02:03.00000Z" + iex> Calendar.ISO.datetime_to_string(2017, 8, 1, 1, 2, 3, {4, 5}, time_zone, "UTC", 3600, 0) + "2017-08-01 01:02:03.00000+01:00" + iex> Calendar.ISO.datetime_to_string(2017, 8, 1, 1, 2, 3, {4, 5}, time_zone, "UTC", 3600, 3600) + "2017-08-01 01:02:03.00000+02:00" + + iex> time_zone = "Europe/Berlin" + iex> Calendar.ISO.datetime_to_string(2017, 8, 1, 1, 2, 3, {4, 5}, time_zone, "CET", 3600, 0) + "2017-08-01 01:02:03.00000+01:00 CET Europe/Berlin" + iex> Calendar.ISO.datetime_to_string(2017, 8, 1, 1, 2, 3, {4, 5}, time_zone, "CDT", 3600, 3600) + "2017-08-01 01:02:03.00000+02:00 CDT Europe/Berlin" + + iex> time_zone = "America/Los_Angeles" + iex> Calendar.ISO.datetime_to_string(2015, 2, 28, 1, 2, 3, {4, 5}, time_zone, "PST", -28800, 0) + "2015-02-28 01:02:03.00000-08:00 PST America/Los_Angeles" + iex> Calendar.ISO.datetime_to_string(2015, 2, 28, 1, 2, 3, {4, 5}, time_zone, "PDT", -28800, 3600) + "2015-02-28 01:02:03.00000-07:00 PDT America/Los_Angeles" + + iex> time_zone = "Europe/Berlin" + iex> Calendar.ISO.datetime_to_string(2017, 8, 1, 1, 2, 3, {4, 5}, time_zone, "CET", 3600, 0, :basic) + "20170801 010203.00000+0100 CET Europe/Berlin" + + """ + @doc since: "1.4.0" + @impl true + @spec datetime_to_string( + year, + month, + day, + Calendar.hour(), + Calendar.minute(), + Calendar.second(), + Calendar.microsecond(), + Calendar.time_zone(), + Calendar.zone_abbr(), + Calendar.utc_offset(), + Calendar.std_offset(), + :basic | :extended + ) :: String.t() + def datetime_to_string( + year, + month, + day, + hour, + minute, + second, + microsecond, + time_zone, + zone_abbr, + utc_offset, + std_offset, + format \\ :extended + ) do + datetime_to_iodata( + year, + month, + day, + hour, + minute, + second, + microsecond, + time_zone, + zone_abbr, + utc_offset, + std_offset, + format + ) + |> IO.iodata_to_binary() + end + + @doc """ + Converts the given datetime into a iodata. + + See `datetime_to_iodata/12` for more information. + + ## Examples + + iex> time_zone = "Etc/UTC" + iex> data = Calendar.ISO.datetime_to_iodata(2017, 8, 1, 1, 2, 3, {4, 5}, time_zone, "UTC", 0, 0) + iex> IO.iodata_to_binary(data) + "2017-08-01 01:02:03.00000Z" + + """ + @doc since: "1.19.0" + @spec datetime_to_iodata( + year, + month, + day, + Calendar.hour(), + Calendar.minute(), + Calendar.second(), + Calendar.microsecond(), + Calendar.time_zone(), + Calendar.zone_abbr(), + Calendar.utc_offset(), + Calendar.std_offset(), + :basic | :extended + ) :: iodata + def datetime_to_iodata( + year, + month, + day, + hour, + minute, + second, + microsecond, + time_zone, + zone_abbr, + utc_offset, + std_offset, + format \\ :extended + ) + when is_time_zone(time_zone) and is_zone_abbr(zone_abbr) and is_utc_offset(utc_offset) and + is_std_offset(std_offset) do + [ + date_to_iodata(year, month, day, format), + ?\s, + time_to_iodata(hour, minute, second, microsecond, format), + offset_to_iodata(utc_offset, std_offset, time_zone, format), + zone_to_iodata(utc_offset, std_offset, zone_abbr, time_zone) + ] + end + + @doc false + def offset_to_string(0, 0, "Etc/UTC", _format), do: "Z" + + def offset_to_string(utc, std, zone, format) do + offset_to_iodata(utc, std, zone, format) + |> IO.iodata_to_binary() + end + + @doc false + def offset_to_iodata(0, 0, "Etc/UTC", _format), do: ?Z + + def offset_to_iodata(utc, std, _zone, format) do + total = utc + std + second = abs(total) + minute = second |> rem(3600) |> div(60) + hour = div(second, 3600) + format_offset(total, hour, minute, format) + end + + defp format_offset(total, hour, minute, :extended) do + [sign(total), zero_pad(hour, 2), ?: | zero_pad(minute, 2)] + end + + defp format_offset(total, hour, minute, :basic) do + [sign(total), zero_pad(hour, 2) | zero_pad(minute, 2)] + end + + defp zone_to_iodata(_, _, _, "Etc/UTC"), do: [] + defp zone_to_iodata(_, _, abbr, zone), do: [?\s, abbr, ?\s | zone] + + @doc """ + Determines if the date given is valid according to the proleptic Gregorian calendar. + + ## Examples + + iex> Calendar.ISO.valid_date?(2015, 2, 28) + true + iex> Calendar.ISO.valid_date?(2015, 2, 30) + false + iex> Calendar.ISO.valid_date?(-1, 12, 31) + true + iex> Calendar.ISO.valid_date?(-1, 12, 32) + false + + """ + @doc since: "1.5.0" + @impl true + @spec valid_date?(year, month, day) :: boolean + def valid_date?(year, month, day) + when is_integer(year) and is_integer(month) and is_integer(day) do + is_month(month) and day in 1..days_in_month(year, month) + end + + @doc """ + Determines if the date given is valid according to the proleptic Gregorian calendar. + + Leap seconds are not supported by the built-in Calendar.ISO. + + ## Examples + + iex> Calendar.ISO.valid_time?(10, 50, 25, {3006, 6}) + true + iex> Calendar.ISO.valid_time?(23, 59, 60, {0, 0}) + false + iex> Calendar.ISO.valid_time?(24, 0, 0, {0, 0}) + false + + """ + @doc since: "1.5.0" + @impl true + @spec valid_time?(Calendar.hour(), Calendar.minute(), Calendar.second(), Calendar.microsecond()) :: + boolean + def valid_time?(hour, minute, second, {ms_value, ms_precision} = _microsecond) + when is_integer(hour) and is_integer(minute) and is_integer(second) and is_integer(ms_value) and + is_integer(ms_value) do + is_hour(hour) and is_minute(minute) and is_second(second) and + is_microsecond(ms_value, ms_precision) + end + + @doc """ + See `c:Calendar.day_rollover_relative_to_midnight_utc/0` for documentation. + """ + @doc since: "1.5.0" + @impl true + @spec day_rollover_relative_to_midnight_utc() :: {0, 1} + def day_rollover_relative_to_midnight_utc() do + {0, 1} + end + + defp sign(total) when total < 0, do: ?- + defp sign(_), do: ?+ + + defp zero_pad(val, count) when val >= 0 and count <= 6 do + num = Integer.to_string(val) + + case max(count - byte_size(num), 0) do + 0 -> num + 1 -> ["0" | num] + 2 -> ["00" | num] + 3 -> ["000" | num] + 4 -> ["0000" | num] + 5 -> ["00000" | num] + end + end + + defp zero_pad(val, count) do + [?- | zero_pad(-val, count)] + end + + @doc """ + Converts the `t:Calendar.iso_days/0` to the first moment of the day. + + ## Examples + + iex> Calendar.ISO.iso_days_to_beginning_of_day({0, {0, 86_400_000_000}}) + {0, {0, 86400000000}} + iex> Calendar.ISO.iso_days_to_beginning_of_day({730_485, {43_200_000_000, 86_400_000_000}}) + {730485, {0, 86400000000}} + iex> Calendar.ISO.iso_days_to_beginning_of_day({730_485, {46_800_000_000, 86_400_000_000}}) + {730485, {0, 86400000000}} + + """ + @doc since: "1.15.0" + @impl true + @spec iso_days_to_beginning_of_day(Calendar.iso_days()) :: Calendar.iso_days() + def iso_days_to_beginning_of_day({days, _day_fraction}) do + {days, {0, @parts_per_day}} + end + + @doc """ + Converts the `t:Calendar.iso_days/0` to the last moment of the day. + + ## Examples + + iex> Calendar.ISO.iso_days_to_end_of_day({0, {0, 86_400_000_000}}) + {0, {86399999999, 86400000000}} + iex> Calendar.ISO.iso_days_to_end_of_day({730_485, {43_200_000_000, 86_400_000_000}}) + {730485, {86399999999, 86400000000}} + iex> Calendar.ISO.iso_days_to_end_of_day({730_485, {46_800_000_000, 86_400_000_000}}) + {730485, {86399999999, 86400000000}} + + """ + @doc since: "1.15.0" + @impl true + @spec iso_days_to_end_of_day(Calendar.iso_days()) :: Calendar.iso_days() + def iso_days_to_end_of_day({days, _day_fraction}) do + {days, {@parts_per_day - 1, @parts_per_day}} + end + + @doc """ + Shifts Date by Duration according to its calendar. + + ## Examples + + iex> Calendar.ISO.shift_date(2016, 1, 3, Duration.new!(month: 2)) + {2016, 3, 3} + iex> Calendar.ISO.shift_date(2016, 2, 29, Duration.new!(month: 1)) + {2016, 3, 29} + iex> Calendar.ISO.shift_date(2016, 1, 31, Duration.new!(month: 1)) + {2016, 2, 29} + iex> Calendar.ISO.shift_date(2016, 1, 31, Duration.new!(year: 4, day: 1)) + {2020, 2, 1} + """ + @impl true + @spec shift_date(year, month, day, Duration.t()) :: {year, month, day} + def shift_date(year, month, day, duration) do + shift_options = shift_date_options(duration) + + Enum.reduce(shift_options, {year, month, day}, fn + {_, 0}, date -> + date + + {:month, value}, date -> + shift_months(date, value) + + {:day, value}, date -> + shift_days(date, value) + end) + end + + @doc """ + Shifts NaiveDateTime by Duration according to its calendar. + + ## Examples + + iex> Calendar.ISO.shift_naive_datetime(2016, 1, 3, 0, 0, 0, {0, 0}, Duration.new!(hour: 1)) + {2016, 1, 3, 1, 0, 0, {0, 0}} + iex> Calendar.ISO.shift_naive_datetime(2016, 1, 3, 0, 0, 0, {0, 0}, Duration.new!(hour: 30)) + {2016, 1, 4, 6, 0, 0, {0, 0}} + iex> Calendar.ISO.shift_naive_datetime(2016, 1, 3, 0, 0, 0, {0, 0}, Duration.new!(microsecond: {100, 6})) + {2016, 1, 3, 0, 0, 0, {100, 6}} + """ + @impl true + @spec shift_naive_datetime( + year, + month, + day, + hour, + minute, + second, + microsecond, + Duration.t() + ) :: {year, month, day, hour, minute, second, microsecond} + def shift_naive_datetime(year, month, day, hour, minute, second, microsecond, duration) do + shift_options = shift_datetime_options(duration) + + Enum.reduce(shift_options, {year, month, day, hour, minute, second, microsecond}, fn + {_, 0}, naive_datetime -> + naive_datetime + + {:month, value}, {year, month, day, hour, minute, second, microsecond} -> + {new_year, new_month, new_day} = shift_months({year, month, day}, value) + {new_year, new_month, new_day, hour, minute, second, microsecond} + + {time_unit, value}, naive_datetime -> + shift_time_unit(naive_datetime, value, time_unit) + end) + end + + @doc """ + Shifts Time by Duration units according to its calendar. + + ## Examples + + iex> Calendar.ISO.shift_time(13, 0, 0, {0, 0}, Duration.new!(hour: 2)) + {15, 0, 0, {0, 0}} + iex> Calendar.ISO.shift_time(13, 0, 0, {0, 0}, Duration.new!(microsecond: {100, 6})) + {13, 0, 0, {100, 6}} + """ + @impl true + @spec shift_time(hour, minute, second, microsecond, Duration.t()) :: + {hour, minute, second, microsecond} + def shift_time(hour, minute, second, microsecond, duration) do + shift_options = shift_time_options(duration) + + Enum.reduce(shift_options, {hour, minute, second, microsecond}, fn + {_, 0}, time -> + time + + {time_unit, value}, time -> + shift_time_unit(time, value, time_unit) + end) + end + + @doc false + def shift_days({year, month, day}, days) do + {year, month, day} = + date_to_iso_days(year, month, day) + |> Kernel.+(days) + |> date_from_iso_days() + + {year, month, day} + end + + defp shift_months({year, month, day}, months) do + months_in_year = 12 + total_months = year * months_in_year + month + months - 1 + + new_year = floor_div_positive_divisor(total_months, months_in_year) + + new_month = + case rem(total_months, months_in_year) + 1 do + new_month when new_month < 1 -> new_month + months_in_year + new_month -> new_month + end + + new_day = min(day, days_in_month(new_year, new_month)) + + {new_year, new_month, new_day} + end + + @doc false + def shift_time_unit({year, month, day, hour, minute, second, microsecond}, value, unit) + when unit in [:second, :millisecond, :microsecond, :nanosecond] or is_integer(unit) do + {value, precision} = shift_time_unit_values(value, microsecond) + + {year, month, day, hour, minute, second, {ms_value, _}} = + naive_datetime_to_iso_days(year, month, day, hour, minute, second, microsecond) + |> shift_time_unit(value, unit) + |> naive_datetime_from_iso_days() + + {year, month, day, hour, minute, second, {ms_value, precision}} + end + + def shift_time_unit({hour, minute, second, microsecond}, value, unit) + when unit in [:second, :millisecond, :microsecond, :nanosecond] or is_integer(unit) do + {value, precision} = shift_time_unit_values(value, microsecond) + + {_days, day_fraction} = + shift_time_unit({0, time_to_day_fraction(hour, minute, second, microsecond)}, value, unit) + + {hour, minute, second, {microsecond, _}} = time_from_day_fraction(day_fraction) + + {hour, minute, second, {microsecond, precision}} + end + + def shift_time_unit({_days, _day_fraction} = iso_days, value, unit) + when unit in [:second, :millisecond, :microsecond, :nanosecond] or is_integer(unit) do + ppd = System.convert_time_unit(86_400, :second, unit) + add_day_fraction_to_iso_days(iso_days, value, ppd) + end + + defp shift_time_unit_values({0, _}, {_, original_precision}) do + {0, original_precision} + end + + defp shift_time_unit_values({ms_value, ms_precision}, {_, _}) do + {ms_value, ms_precision} + end + + defp shift_time_unit_values(value, {_, original_precision}) do + {value, original_precision} + end + + defp shift_date_options(%Duration{ + year: year, + month: month, + week: week, + day: day, + hour: 0, + minute: 0, + second: 0, + microsecond: {0, _precision} + }) do + [ + month: year * 12 + month, + day: week * 7 + day + ] + end + + defp shift_date_options(_duration) do + raise ArgumentError, + "cannot shift date by time scale unit. Expected :year, :month, :week, :day" + end + + defp shift_datetime_options(%Duration{ + year: year, + month: month, + week: week, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond + }) do + [ + month: year * 12 + month, + second: week * 7 * 86_400 + day * 86_400 + hour * 3600 + minute * 60 + second, + microsecond: microsecond + ] + end + + defp shift_time_options(%Duration{ + year: 0, + month: 0, + week: 0, + day: 0, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond + }) do + [ + second: hour * 3600 + minute * 60 + second, + microsecond: microsecond + ] + end + + defp shift_time_options(_duration) do + raise ArgumentError, + "cannot shift time by date scale unit. Expected :hour, :minute, :second, :microsecond" + end + + ## Helpers + + @doc false + def from_unix(integer, unit) when is_integer(integer) do + total = System.convert_time_unit(integer, unit, :microsecond) + + if total in @unix_range_microseconds do + microseconds = Integer.mod(total, @microseconds_per_second) + seconds = @unix_epoch + floor_div_positive_divisor(total, @microseconds_per_second) + precision = precision_for_unit(unit) + {date, time} = iso_seconds_to_datetime(seconds) + {:ok, date, time, {microseconds, precision}} + else + {:error, :invalid_unix_time} + end + end + + defp precision_for_unit(unit) do + case System.convert_time_unit(1, :second, unit) do + 1 -> 0 + 10 -> 1 + 100 -> 2 + 1_000 -> 3 + 10_000 -> 4 + 100_000 -> 5 + _ -> 6 + end + end + + defp parse_microsecond("." <> rest) do + case parse_microsecond(rest, 0, []) do + {[], 0, _} -> + :error + + {microsecond, precision, rest} -> + scale = scale_factor(precision) + {{:erlang.list_to_integer(microsecond) * scale, precision}, rest} + end + end + + defp parse_microsecond("," <> rest) do + parse_microsecond("." <> rest) + end + + defp parse_microsecond(rest) do + {{0, 0}, rest} + end + + defp parse_microsecond(<>, 6, acc) when head in ?0..?9, + do: parse_microsecond(tail, 6, acc) + + defp parse_microsecond(<>, precision, acc) when head in ?0..?9, + do: parse_microsecond(tail, precision + 1, [head | acc]) + + defp parse_microsecond(rest, precision, acc) do + {:lists.reverse(acc), precision, rest} + end + + defp parse_offset(""), do: {nil, ""} + defp parse_offset("Z"), do: {0, ""} + defp parse_offset("-00:00"), do: :error + + defp parse_offset(<>), + do: parse_offset(1, h1, h2, m1, m2, rest) + + defp parse_offset(<>), + do: parse_offset(-1, h1, h2, m1, m2, rest) + + defp parse_offset(<>), + do: parse_offset(1, h1, h2, m1, m2, rest) + + defp parse_offset(<>), + do: parse_offset(-1, h1, h2, m1, m2, rest) + + defp parse_offset(<>), do: parse_offset(1, h1, h2, ?0, ?0, rest) + defp parse_offset(<>), do: parse_offset(-1, h1, h2, ?0, ?0, rest) + defp parse_offset(_), do: :error + + defp parse_offset(sign, h1, h2, m1, m2, rest) do + with true <- h1 in ?0..?2 and h2 in ?0..?9, + true <- m1 in ?0..?5 and m2 in ?0..?9, + hour = (h1 - ?0) * 10 + h2 - ?0, + min = (m1 - ?0) * 10 + m2 - ?0, + true <- hour < 24 do + {(hour * 60 + min) * 60 * sign, rest} + else + _ -> :error + end + end + + @doc false + def gregorian_seconds_to_iso_days(seconds, microsecond) do + {days, rest_seconds} = div_rem(seconds, @seconds_per_day) + microseconds_in_day = rest_seconds * @microseconds_per_second + microsecond + day_fraction = {microseconds_in_day, @parts_per_day} + {days, day_fraction} + end + + @doc false + def iso_days_to_unit({days, {parts, ppd}}, unit) do + day_microseconds = days * @parts_per_day + microseconds = divide_by_parts_per_day(parts, ppd) + System.convert_time_unit(day_microseconds + microseconds, :microsecond, unit) + end + + @doc false + def add_day_fraction_to_iso_days({days, {parts, ppd}}, add, ppd) do + normalize_iso_days(days, parts + add, ppd) + end + + def add_day_fraction_to_iso_days({days, {parts, ppd}}, add, add_ppd) do + parts = parts * add_ppd + add = add * ppd + gcd = Integer.gcd(ppd, add_ppd) + result_parts = div(parts + add, gcd) + result_ppd = div(ppd * add_ppd, gcd) + normalize_iso_days(days, result_parts, result_ppd) + end + + defp normalize_iso_days(days, parts, ppd) do + days_offset = div(parts, ppd) + parts = rem(parts, ppd) + + if parts < 0 do + {days + days_offset - 1, {parts + ppd, ppd}} + else + {days + days_offset, {parts, ppd}} + end + end + + defp leap_day_offset(_year, month) when month < 3, do: 0 + + defp leap_day_offset(year, _month) do + if leap_year?(year), do: 1, else: 0 + end + + defp days_before_month(1), do: 0 + defp days_before_month(2), do: 31 + defp days_before_month(3), do: 59 + defp days_before_month(4), do: 90 + defp days_before_month(5), do: 120 + defp days_before_month(6), do: 151 + defp days_before_month(7), do: 181 + defp days_before_month(8), do: 212 + defp days_before_month(9), do: 243 + defp days_before_month(10), do: 273 + defp days_before_month(11), do: 304 + defp days_before_month(12), do: 334 + + defp iso_seconds_to_datetime(seconds) do + {days, rest_seconds} = div_rem(seconds, @seconds_per_day) + + date = date_from_iso_days(days) + time = seconds_to_time(rest_seconds) + {date, time} + end + + defp seconds_to_time(seconds) when seconds in 0..@last_second_of_the_day do + {hour, rest_seconds} = div_rem(seconds, @seconds_per_hour) + {minute, second} = div_rem(rest_seconds, @seconds_per_minute) + + {hour, minute, second} + end + + defp ensure_day_in_month!(year, month, day) when is_integer(day) do + if day < 1 or day > days_in_month(year, month) do + raise ArgumentError, "invalid date: #{date_to_string(year, month, day)}" + end + end +end diff --git a/lib/elixir/lib/calendar/naive_datetime.ex b/lib/elixir/lib/calendar/naive_datetime.ex new file mode 100644 index 00000000000..732c8062270 --- /dev/null +++ b/lib/elixir/lib/calendar/naive_datetime.ex @@ -0,0 +1,1501 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +defmodule NaiveDateTime do + @moduledoc """ + A NaiveDateTime struct (without a time zone) and functions. + + The NaiveDateTime struct contains the fields year, month, day, hour, + minute, second, microsecond and calendar. New naive datetimes can be + built with the `new/2` and `new/8` functions or using the + `~N` (see `sigil_N/2`) sigil: + + iex> ~N[2000-01-01 23:00:07] + ~N[2000-01-01 23:00:07] + + The date and time fields in the struct can be accessed directly: + + iex> naive = ~N[2000-01-01 23:00:07] + iex> naive.year + 2000 + iex> naive.second + 7 + + We call them "naive" because this datetime representation does not + have a time zone. This means the datetime may not actually exist in + certain areas in the world even though it is valid. + + For example, when daylight saving changes are applied by a region, + the clock typically moves forward or backward by one hour. This means + certain datetimes never occur or may occur more than once. Since + `NaiveDateTime` is not validated against a time zone, such errors + would go unnoticed. + + Developers should avoid creating the NaiveDateTime structs directly + and instead, rely on the functions provided by this module as well + as the ones in third-party calendar libraries. + + ## Comparing naive date times + + Comparisons in Elixir using `==/2`, `>/2`, ` Enum.min([~N[2020-01-01 23:00:07], ~N[2000-01-01 23:00:07]], NaiveDateTime) + ~N[2000-01-01 23:00:07] + + ## Using epochs + + The `add/3` and `diff/3` functions can be used for computing date + times or retrieving the number of seconds between instants. + For example, if there is an interest in computing the number of + seconds from the Unix epoch (1970-01-01 00:00:00): + + iex> NaiveDateTime.diff(~N[2010-04-17 14:00:00], ~N[1970-01-01 00:00:00]) + 1271512800 + + iex> NaiveDateTime.add(~N[1970-01-01 00:00:00], 1_271_512_800) + ~N[2010-04-17 14:00:00] + + Those functions are optimized to deal with common epochs, such + as the Unix Epoch above or the Gregorian Epoch (0000-01-01 00:00:00). + """ + + @enforce_keys [:year, :month, :day, :hour, :minute, :second] + defstruct [ + :year, + :month, + :day, + :hour, + :minute, + :second, + microsecond: {0, 0}, + calendar: Calendar.ISO + ] + + @type t :: %__MODULE__{ + year: Calendar.year(), + month: Calendar.month(), + day: Calendar.day(), + calendar: Calendar.calendar(), + hour: Calendar.hour(), + minute: Calendar.minute(), + second: Calendar.second(), + microsecond: Calendar.microsecond() + } + + @seconds_per_day 24 * 60 * 60 + + @doc """ + Returns the current naive datetime in UTC. + + Prefer using `DateTime.utc_now/0` when possible as, opposite + to `NaiveDateTime`, it will keep the time zone information. + + You can also provide a time unit to automatically truncate + the naive datetime. This is available since v1.15.0. + + ## Examples + + iex> naive_datetime = NaiveDateTime.utc_now() + iex> naive_datetime.year >= 2016 + true + + iex> naive_datetime = NaiveDateTime.utc_now(:second) + iex> naive_datetime.microsecond + {0, 0} + + """ + @doc since: "1.4.0" + @spec utc_now(Calendar.calendar() | :native | :microsecond | :millisecond | :second) :: t + def utc_now(calendar_or_time_unit \\ Calendar.ISO) + + def utc_now(time_unit) when time_unit in [:microsecond, :millisecond, :second, :native] do + utc_now(time_unit, Calendar.ISO) + end + + def utc_now(calendar) do + utc_now(:native, calendar) + end + + @doc """ + Returns the current naive datetime in UTC, supporting a specific + calendar and precision. + + Prefer using `DateTime.utc_now/2` when possible as, opposite + to `NaiveDateTime`, it will keep the time zone information. + + ## Examples + + iex> naive_datetime = NaiveDateTime.utc_now(:second, Calendar.ISO) + iex> naive_datetime.year >= 2016 + true + + iex> naive_datetime = NaiveDateTime.utc_now(:second, Calendar.ISO) + iex> naive_datetime.microsecond + {0, 0} + + """ + @doc since: "1.15.0" + @spec utc_now(:native | :microsecond | :millisecond | :second, Calendar.calendar()) :: t + def utc_now(time_unit, calendar) + when time_unit in [:native, :microsecond, :millisecond, :second] do + {:ok, {year, month, day}, {hour, minute, second}, microsecond} = + Calendar.ISO.from_unix(System.os_time(time_unit), time_unit) + + %NaiveDateTime{ + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond, + calendar: Calendar.ISO + } + |> convert!(calendar) + end + + @doc """ + Returns the "local time" for the machine the Elixir program is running on. + + WARNING: This function can cause insidious bugs. It depends on the time zone + configuration at run time. This can changed and be set to a time zone that has + daylight saving jumps (spring forward or fall back). + + This function can be used to display what the time is right now for the time + zone configuration that the machine happens to have. An example would be a + desktop program displaying a clock to the user. For any other uses it is + probably a bad idea to use this function. + + For most cases, use `DateTime.now/2` or `DateTime.utc_now/1` instead. + + Does not include fractional seconds. + + ## Examples + + iex> naive_datetime = NaiveDateTime.local_now() + iex> naive_datetime.year >= 2019 + true + + """ + @doc since: "1.10.0" + @spec local_now(Calendar.calendar()) :: t + def local_now(calendar \\ Calendar.ISO) + + def local_now(Calendar.ISO) do + {{year, month, day}, {hour, minute, second}} = :erlang.localtime() + {:ok, ndt} = NaiveDateTime.new(year, month, day, hour, minute, second) + ndt + end + + def local_now(calendar) do + naive_datetime = local_now() + + case convert(naive_datetime, calendar) do + {:ok, value} -> + value + + {:error, :incompatible_calendars} -> + raise ArgumentError, + ~s(cannot get "local now" in target calendar #{inspect(calendar)}, ) <> + "reason: cannot convert from Calendar.ISO to #{inspect(calendar)}." + end + end + + @doc """ + Builds a new ISO naive datetime. + + Expects all values to be integers. Returns `{:ok, naive_datetime}` + if each entry fits its appropriate range, returns `{:error, reason}` + otherwise. + + ## Examples + + iex> NaiveDateTime.new(2000, 1, 1, 0, 0, 0) + {:ok, ~N[2000-01-01 00:00:00]} + iex> NaiveDateTime.new(2000, 13, 1, 0, 0, 0) + {:error, :invalid_date} + iex> NaiveDateTime.new(2000, 2, 29, 0, 0, 0) + {:ok, ~N[2000-02-29 00:00:00]} + iex> NaiveDateTime.new(2000, 2, 30, 0, 0, 0) + {:error, :invalid_date} + iex> NaiveDateTime.new(2001, 2, 29, 0, 0, 0) + {:error, :invalid_date} + + iex> NaiveDateTime.new(2000, 1, 1, 23, 59, 59, {0, 1}) + {:ok, ~N[2000-01-01 23:59:59.0]} + iex> NaiveDateTime.new(2000, 1, 1, 23, 59, 59, 999_999) + {:ok, ~N[2000-01-01 23:59:59.999999]} + iex> NaiveDateTime.new(2000, 1, 1, 24, 59, 59, 999_999) + {:error, :invalid_time} + iex> NaiveDateTime.new(2000, 1, 1, 23, 60, 59, 999_999) + {:error, :invalid_time} + iex> NaiveDateTime.new(2000, 1, 1, 23, 59, 60, 999_999) + {:error, :invalid_time} + iex> NaiveDateTime.new(2000, 1, 1, 23, 59, 59, 1_000_000) + {:error, :invalid_time} + + iex> NaiveDateTime.new(2000, 1, 1, 23, 59, 59, {0, 1}, Calendar.ISO) + {:ok, ~N[2000-01-01 23:59:59.0]} + + """ + @spec new( + Calendar.year(), + Calendar.month(), + Calendar.day(), + Calendar.hour(), + Calendar.minute(), + Calendar.second(), + Calendar.microsecond() | non_neg_integer(), + Calendar.calendar() + ) :: {:ok, t} | {:error, atom} + def new(year, month, day, hour, minute, second, microsecond \\ {0, 0}, calendar \\ Calendar.ISO) + + def new(year, month, day, hour, minute, second, microsecond, calendar) + when is_integer(microsecond) do + new(year, month, day, hour, minute, second, {microsecond, 6}, calendar) + end + + def new(year, month, day, hour, minute, second, microsecond, calendar) do + cond do + not calendar.valid_date?(year, month, day) -> + {:error, :invalid_date} + + not calendar.valid_time?(hour, minute, second, microsecond) -> + {:error, :invalid_time} + + true -> + naive_datetime = %NaiveDateTime{ + calendar: calendar, + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond + } + + {:ok, naive_datetime} + end + end + + @doc """ + Builds a new ISO naive datetime. + + Expects all values to be integers. Returns `naive_datetime` + if each entry fits its appropriate range, raises if + time or date is invalid. + + ## Examples + + iex> NaiveDateTime.new!(2000, 1, 1, 0, 0, 0) + ~N[2000-01-01 00:00:00] + iex> NaiveDateTime.new!(2000, 2, 29, 0, 0, 0) + ~N[2000-02-29 00:00:00] + iex> NaiveDateTime.new!(2000, 1, 1, 23, 59, 59, {0, 1}) + ~N[2000-01-01 23:59:59.0] + iex> NaiveDateTime.new!(2000, 1, 1, 23, 59, 59, 999_999) + ~N[2000-01-01 23:59:59.999999] + iex> NaiveDateTime.new!(2000, 1, 1, 23, 59, 59, {0, 1}, Calendar.ISO) + ~N[2000-01-01 23:59:59.0] + iex> NaiveDateTime.new!(2000, 1, 1, 24, 59, 59, 999_999) + ** (ArgumentError) cannot build naive datetime, reason: :invalid_time + + """ + @doc since: "1.11.0" + @spec new!( + Calendar.year(), + Calendar.month(), + Calendar.day(), + Calendar.hour(), + Calendar.minute(), + Calendar.second(), + Calendar.microsecond() | non_neg_integer(), + Calendar.calendar() + ) :: t + def new!( + year, + month, + day, + hour, + minute, + second, + microsecond \\ {0, 0}, + calendar \\ Calendar.ISO + ) + + def new!(year, month, day, hour, minute, second, microsecond, calendar) do + case new(year, month, day, hour, minute, second, microsecond, calendar) do + {:ok, naive_datetime} -> + naive_datetime + + {:error, reason} -> + raise ArgumentError, "cannot build naive datetime, reason: #{inspect(reason)}" + end + end + + @doc """ + Builds a naive datetime from date and time structs. + + ## Examples + + iex> NaiveDateTime.new(~D[2010-01-13], ~T[23:00:07.005]) + {:ok, ~N[2010-01-13 23:00:07.005]} + + """ + @spec new(Date.t(), Time.t()) :: {:ok, t} + def new(date, time) + + def new(%Date{calendar: calendar} = date, %Time{calendar: calendar} = time) do + %{year: year, month: month, day: day} = date + %{hour: hour, minute: minute, second: second, microsecond: microsecond} = time + + naive_datetime = %NaiveDateTime{ + calendar: calendar, + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond + } + + {:ok, naive_datetime} + end + + @doc """ + Builds a naive datetime from date and time structs. + + ## Examples + + iex> NaiveDateTime.new!(~D[2010-01-13], ~T[23:00:07.005]) + ~N[2010-01-13 23:00:07.005] + + """ + @doc since: "1.11.0" + @spec new!(Date.t(), Time.t()) :: t + def new!(date, time) + + def new!(%Date{calendar: calendar} = date, %Time{calendar: calendar} = time) do + {:ok, naive_datetime} = new(date, time) + naive_datetime + end + + @doc """ + Adds a specified amount of time to a `NaiveDateTime`. + + > #### Prefer `shift/2` {: .info} + > + > Prefer `shift/2` over `add/3`, as it offers a more ergonomic API. + > + > `add/3` provides a lower-level API which only supports fixed units + > such as `:hour` and `:second`, but not `:month` (as the exact length + > of a month depends on the current month). `add/3` always considers + > the unit to be computed according to the `Calendar.ISO`. + + Accepts an `amount_to_add` in any `unit`. `unit` can be `:day`, + `:hour`, `:minute`, `:second` or any subsecond precision from + `t:System.time_unit/0` for convenience but ultimately they are + all converted to microseconds. Negative values will move backwards + in time and the default precision is `:second`. + + ## Examples + + It uses seconds by default: + + # adds seconds by default + iex> NaiveDateTime.add(~N[2014-10-02 00:29:10], 2) + ~N[2014-10-02 00:29:12] + + # accepts negative offsets + iex> NaiveDateTime.add(~N[2014-10-02 00:29:10], -2) + ~N[2014-10-02 00:29:08] + + It can also work with subsecond precisions: + + iex> NaiveDateTime.add(~N[2014-10-02 00:29:10], 2_000, :millisecond) + ~N[2014-10-02 00:29:12.000] + + As well as days/hours/minutes: + + iex> NaiveDateTime.add(~N[2015-02-28 00:29:10], 2, :day) + ~N[2015-03-02 00:29:10] + iex> NaiveDateTime.add(~N[2015-02-28 00:29:10], 36, :hour) + ~N[2015-03-01 12:29:10] + iex> NaiveDateTime.add(~N[2015-02-28 00:29:10], 60, :minute) + ~N[2015-02-28 01:29:10] + + This operation merges the precision of the naive date time with the given unit: + + iex> result = NaiveDateTime.add(~N[2014-10-02 00:29:10], 21, :millisecond) + ~N[2014-10-02 00:29:10.021] + iex> result.microsecond + {21000, 3} + + Operations on top of gregorian seconds or the Unix epoch are optimized: + + # from Gregorian seconds + iex> NaiveDateTime.add(~N[0000-01-01 00:00:00], 63_579_428_950) + ~N[2014-10-02 00:29:10] + + Passing a `DateTime` automatically converts it to `NaiveDateTime`, + discarding the time zone information: + + iex> dt = %DateTime{year: 2000, month: 2, day: 29, zone_abbr: "CET", + ...> hour: 23, minute: 0, second: 7, microsecond: {0, 0}, + ...> utc_offset: 3600, std_offset: 0, time_zone: "Europe/Warsaw"} + iex> NaiveDateTime.add(dt, 21, :second) + ~N[2000-02-29 23:00:28] + + """ + @doc since: "1.4.0" + @spec add(Calendar.naive_datetime(), integer, :day | :hour | :minute | System.time_unit()) :: t + def add(naive_datetime, amount_to_add, unit \\ :second) + + def add(naive_datetime, amount_to_add, :day) when is_integer(amount_to_add) do + add(naive_datetime, amount_to_add * 86400, :second) + end + + def add(naive_datetime, amount_to_add, :hour) when is_integer(amount_to_add) do + add(naive_datetime, amount_to_add * 3600, :second) + end + + def add(naive_datetime, amount_to_add, :minute) when is_integer(amount_to_add) do + add(naive_datetime, amount_to_add * 60, :second) + end + + def add( + %{calendar: calendar, microsecond: {_, precision}} = naive_datetime, + amount_to_add, + unit + ) + when is_integer(amount_to_add) do + if not is_integer(unit) and unit not in ~w(second millisecond microsecond nanosecond)a do + raise ArgumentError, + "unsupported time unit. Expected :day, :hour, :minute, :second, :millisecond, :microsecond, :nanosecond, or a positive integer, got #{inspect(unit)}" + end + + precision = max(Calendar.ISO.time_unit_to_precision(unit), precision) + + naive_datetime + |> to_iso_days() + |> Calendar.ISO.shift_time_unit(amount_to_add, unit) + |> from_iso_days(calendar, precision) + end + + @doc """ + Subtracts `naive_datetime2` from `naive_datetime1`. + + The answer can be returned in any `:day`, `:hour`, `:minute`, or any `unit` + available from `t:System.time_unit/0`. The unit is measured according to + `Calendar.ISO` and defaults to `:second`. + + Fractional results are not supported and are truncated. + + ## Examples + + iex> NaiveDateTime.diff(~N[2014-10-02 00:29:12], ~N[2014-10-02 00:29:10]) + 2 + iex> NaiveDateTime.diff(~N[2014-10-02 00:29:12], ~N[2014-10-02 00:29:10], :microsecond) + 2_000_000 + + iex> NaiveDateTime.diff(~N[2014-10-02 00:29:10.042], ~N[2014-10-02 00:29:10.021]) + 0 + iex> NaiveDateTime.diff(~N[2014-10-02 00:29:10.042], ~N[2014-10-02 00:29:10.021], :millisecond) + 21 + + iex> NaiveDateTime.diff(~N[2014-10-02 00:29:10], ~N[2014-10-02 00:29:12]) + -2 + iex> NaiveDateTime.diff(~N[-0001-10-02 00:29:10], ~N[-0001-10-02 00:29:12]) + -2 + + It can also compute the difference in days, hours, or minutes: + + iex> NaiveDateTime.diff(~N[2014-10-10 00:29:10], ~N[2014-10-02 00:29:10], :day) + 8 + iex> NaiveDateTime.diff(~N[2014-10-02 12:29:10], ~N[2014-10-02 00:29:10], :hour) + 12 + iex> NaiveDateTime.diff(~N[2014-10-02 00:39:10], ~N[2014-10-02 00:29:10], :minute) + 10 + + But it also rounds incomplete days to zero: + + iex> NaiveDateTime.diff(~N[2014-10-10 00:29:09], ~N[2014-10-02 00:29:10], :day) + 7 + + """ + @doc since: "1.4.0" + @spec diff( + Calendar.naive_datetime(), + Calendar.naive_datetime(), + :day | :hour | :minute | System.time_unit() + ) :: integer + + def diff(naive_datetime1, naive_datetime2, unit \\ :second) + + def diff(naive_datetime1, naive_datetime2, :day) do + diff(naive_datetime1, naive_datetime2, :second) |> div(86400) + end + + def diff(naive_datetime1, naive_datetime2, :hour) do + diff(naive_datetime1, naive_datetime2, :second) |> div(3600) + end + + def diff(naive_datetime1, naive_datetime2, :minute) do + diff(naive_datetime1, naive_datetime2, :second) |> div(60) + end + + def diff( + %{calendar: calendar1} = naive_datetime1, + %{calendar: calendar2} = naive_datetime2, + unit + ) do + if not Calendar.compatible_calendars?(calendar1, calendar2) do + raise ArgumentError, + "cannot calculate the difference between #{inspect(naive_datetime1)} and " <> + "#{inspect(naive_datetime2)} because their calendars are not compatible " <> + "and thus the result would be ambiguous" + end + + if not is_integer(unit) and + unit not in ~w(second millisecond microsecond nanosecond)a do + raise ArgumentError, + "unsupported time unit. Expected :day, :hour, :minute, :second, :millisecond, :microsecond, :nanosecond, or a positive integer, got #{inspect(unit)}" + end + + units1 = naive_datetime1 |> to_iso_days() |> Calendar.ISO.iso_days_to_unit(unit) + units2 = naive_datetime2 |> to_iso_days() |> Calendar.ISO.iso_days_to_unit(unit) + units1 - units2 + end + + @doc """ + Shifts given `naive_datetime` by `duration` according to its calendar. + + Allowed units are: `:year`, `:month`, `:week`, `:day`, `:hour`, `:minute`, `:second`, `:microsecond`. + + When using the default ISO calendar, durations are collapsed and + applied in the order of months, then seconds and microseconds: + + * when shifting by 1 year and 2 months the date is actually shifted by 14 months + * weeks, days and smaller units are collapsed into seconds and microseconds + + When shifting by month, days are rounded down to the nearest valid date. + + ## Examples + + iex> NaiveDateTime.shift(~N[2016-01-31 00:00:00], month: 1) + ~N[2016-02-29 00:00:00] + iex> NaiveDateTime.shift(~N[2016-01-31 00:00:00], year: 4, day: 1) + ~N[2020-02-01 00:00:00] + iex> NaiveDateTime.shift(~N[2016-01-31 00:00:00], year: -2, day: 1) + ~N[2014-02-01 00:00:00] + iex> NaiveDateTime.shift(~N[2016-01-31 00:00:00], second: 45) + ~N[2016-01-31 00:00:45] + iex> NaiveDateTime.shift(~N[2016-01-31 00:00:00], microsecond: {100, 6}) + ~N[2016-01-31 00:00:00.000100] + + # leap years + iex> NaiveDateTime.shift(~N[2024-02-29 00:00:00], year: 1) + ~N[2025-02-28 00:00:00] + iex> NaiveDateTime.shift(~N[2024-02-29 00:00:00], year: 4) + ~N[2028-02-29 00:00:00] + + # rounding down + iex> NaiveDateTime.shift(~N[2015-01-31 00:00:00], month: 1) + ~N[2015-02-28 00:00:00] + + """ + @doc since: "1.17.0" + @spec shift(Calendar.naive_datetime(), Duration.duration()) :: t + def shift(%{calendar: calendar} = naive_datetime, duration) do + %{ + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond + } = naive_datetime + + {year, month, day, hour, minute, second, microsecond} = + calendar.shift_naive_datetime( + year, + month, + day, + hour, + minute, + second, + microsecond, + __duration__!(duration) + ) + + %NaiveDateTime{ + calendar: calendar, + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond + } + end + + @doc false + defdelegate __duration__!(params), to: Duration, as: :new! + + @doc """ + Returns the given naive datetime with the microsecond field truncated to the + given precision (`:microsecond`, `:millisecond` or `:second`). + + The given naive datetime is returned unchanged if it already has lower precision + than the given precision. + + ## Examples + + iex> NaiveDateTime.truncate(~N[2017-11-06 00:23:51.123456], :microsecond) + ~N[2017-11-06 00:23:51.123456] + + iex> NaiveDateTime.truncate(~N[2017-11-06 00:23:51.123456], :millisecond) + ~N[2017-11-06 00:23:51.123] + + iex> NaiveDateTime.truncate(~N[2017-11-06 00:23:51.123456], :second) + ~N[2017-11-06 00:23:51] + + """ + @doc since: "1.6.0" + @spec truncate(t(), :microsecond | :millisecond | :second) :: t() + def truncate(%NaiveDateTime{microsecond: microsecond} = naive_datetime, precision) do + %{naive_datetime | microsecond: Calendar.truncate(microsecond, precision)} + end + + def truncate( + %{ + calendar: calendar, + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond + }, + precision + ) do + %NaiveDateTime{ + calendar: calendar, + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: Calendar.truncate(microsecond, precision) + } + end + + @doc """ + Converts a `NaiveDateTime` into a `Date`. + + Because `Date` does not hold time information, + data will be lost during the conversion. + + ## Examples + + iex> NaiveDateTime.to_date(~N[2002-01-13 23:00:07]) + ~D[2002-01-13] + + """ + @spec to_date(Calendar.naive_datetime()) :: Date.t() + def to_date(%{ + year: year, + month: month, + day: day, + calendar: calendar, + hour: _, + minute: _, + second: _, + microsecond: _ + }) do + %Date{year: year, month: month, day: day, calendar: calendar} + end + + @doc """ + Converts a `NaiveDateTime` into `Time`. + + Because `Time` does not hold date information, + data will be lost during the conversion. + + ## Examples + + iex> NaiveDateTime.to_time(~N[2002-01-13 23:00:07]) + ~T[23:00:07] + + """ + @spec to_time(Calendar.naive_datetime()) :: Time.t() + def to_time(%{ + year: _, + month: _, + day: _, + calendar: calendar, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond + }) do + %Time{ + hour: hour, + minute: minute, + second: second, + microsecond: microsecond, + calendar: calendar + } + end + + @doc """ + Converts the given naive datetime to a string according to its calendar. + + For readability, this function follows the RFC3339 suggestion of removing + the "T" separator between the date and time components. + + ## Examples + + iex> NaiveDateTime.to_string(~N[2000-02-28 23:00:13]) + "2000-02-28 23:00:13" + iex> NaiveDateTime.to_string(~N[2000-02-28 23:00:13.001]) + "2000-02-28 23:00:13.001" + iex> NaiveDateTime.to_string(~N[-0100-12-15 03:20:31]) + "-0100-12-15 03:20:31" + + This function can also be used to convert a DateTime to a string without + the time zone information: + + iex> dt = %DateTime{year: 2000, month: 2, day: 29, zone_abbr: "CET", + ...> hour: 23, minute: 0, second: 7, microsecond: {0, 0}, + ...> utc_offset: 3600, std_offset: 0, time_zone: "Europe/Warsaw"} + iex> NaiveDateTime.to_string(dt) + "2000-02-29 23:00:07" + + """ + @spec to_string(Calendar.naive_datetime()) :: String.t() + def to_string(%{calendar: calendar} = naive_datetime) do + %{ + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond + } = naive_datetime + + calendar.naive_datetime_to_string(year, month, day, hour, minute, second, microsecond) + end + + @doc """ + Parses the extended "Date and time of day" format described by + [ISO 8601:2019](https://en.wikipedia.org/wiki/ISO_8601). + + Time zone offset may be included in the string but they will be + simply discarded as such information is not included in naive date + times. + + As specified in the standard, the separator "T" may be omitted if + desired as there is no ambiguity within this function. + + Note leap seconds are not supported by the built-in Calendar.ISO. + + ## Examples + + iex> NaiveDateTime.from_iso8601("2015-01-23 23:50:07") + {:ok, ~N[2015-01-23 23:50:07]} + iex> NaiveDateTime.from_iso8601("2015-01-23T23:50:07") + {:ok, ~N[2015-01-23 23:50:07]} + iex> NaiveDateTime.from_iso8601("2015-01-23T23:50:07Z") + {:ok, ~N[2015-01-23 23:50:07]} + + iex> NaiveDateTime.from_iso8601("2015-01-23 23:50:07.0") + {:ok, ~N[2015-01-23 23:50:07.0]} + iex> NaiveDateTime.from_iso8601("2015-01-23 23:50:07,0123456") + {:ok, ~N[2015-01-23 23:50:07.012345]} + iex> NaiveDateTime.from_iso8601("2015-01-23 23:50:07.0123456") + {:ok, ~N[2015-01-23 23:50:07.012345]} + iex> NaiveDateTime.from_iso8601("2015-01-23T23:50:07.123Z") + {:ok, ~N[2015-01-23 23:50:07.123]} + + iex> NaiveDateTime.from_iso8601("2015-01-23P23:50:07") + {:error, :invalid_format} + iex> NaiveDateTime.from_iso8601("2015:01:23 23-50-07") + {:error, :invalid_format} + iex> NaiveDateTime.from_iso8601("2015-01-23 23:50:07A") + {:error, :invalid_format} + iex> NaiveDateTime.from_iso8601("2015-01-23 23:50:61") + {:error, :invalid_time} + iex> NaiveDateTime.from_iso8601("2015-01-32 23:50:07") + {:error, :invalid_date} + + iex> NaiveDateTime.from_iso8601("2015-01-23T23:50:07.123+02:30") + {:ok, ~N[2015-01-23 23:50:07.123]} + iex> NaiveDateTime.from_iso8601("2015-01-23T23:50:07.123+00:00") + {:ok, ~N[2015-01-23 23:50:07.123]} + iex> NaiveDateTime.from_iso8601("2015-01-23T23:50:07.123-02:30") + {:ok, ~N[2015-01-23 23:50:07.123]} + iex> NaiveDateTime.from_iso8601("2015-01-23T23:50:07.123-00:00") + {:error, :invalid_format} + iex> NaiveDateTime.from_iso8601("2015-01-23T23:50:07.123-00:60") + {:error, :invalid_format} + iex> NaiveDateTime.from_iso8601("2015-01-23T23:50:07.123-24:00") + {:error, :invalid_format} + + """ + @spec from_iso8601(String.t(), Calendar.calendar()) :: {:ok, t} | {:error, atom} + def from_iso8601(string, calendar \\ Calendar.ISO) do + with {:ok, {year, month, day, hour, minute, second, microsecond}} <- + Calendar.ISO.parse_naive_datetime(string) do + convert( + %NaiveDateTime{ + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond + }, + calendar + ) + end + end + + @doc """ + Parses the extended "Date and time of day" format described by + [ISO 8601:2019](https://en.wikipedia.org/wiki/ISO_8601). + + Raises if the format is invalid. + + ## Examples + + iex> NaiveDateTime.from_iso8601!("2015-01-23T23:50:07.123Z") + ~N[2015-01-23 23:50:07.123] + iex> NaiveDateTime.from_iso8601!("2015-01-23T23:50:07,123Z") + ~N[2015-01-23 23:50:07.123] + iex> NaiveDateTime.from_iso8601!("2015-01-23P23:50:07") + ** (ArgumentError) cannot parse "2015-01-23P23:50:07" as naive datetime, reason: :invalid_format + + """ + @spec from_iso8601!(String.t(), Calendar.calendar()) :: t + def from_iso8601!(string, calendar \\ Calendar.ISO) do + case from_iso8601(string, calendar) do + {:ok, value} -> + value + + {:error, reason} -> + raise ArgumentError, + "cannot parse #{inspect(string)} as naive datetime, reason: #{inspect(reason)}" + end + end + + @doc """ + Converts the given naive datetime to + [ISO 8601:2019](https://en.wikipedia.org/wiki/ISO_8601). + + By default, `NaiveDateTime.to_iso8601/2` returns naive datetimes formatted in the "extended" + format, for human readability. It also supports the "basic" format through passing the `:basic` option. + + Only supports converting naive datetimes which are in the ISO calendar, + attempting to convert naive datetimes from other calendars will raise. + + ## Examples + + iex> NaiveDateTime.to_iso8601(~N[2000-02-28 23:00:13]) + "2000-02-28T23:00:13" + + iex> NaiveDateTime.to_iso8601(~N[2000-02-28 23:00:13.001]) + "2000-02-28T23:00:13.001" + + iex> NaiveDateTime.to_iso8601(~N[2000-02-28 23:00:13.001], :basic) + "20000228T230013.001" + + This function can also be used to convert a DateTime to ISO 8601 without + the time zone information: + + iex> dt = %DateTime{year: 2000, month: 2, day: 29, zone_abbr: "CET", + ...> hour: 23, minute: 0, second: 7, microsecond: {0, 0}, + ...> utc_offset: 3600, std_offset: 0, time_zone: "Europe/Warsaw"} + iex> NaiveDateTime.to_iso8601(dt) + "2000-02-29T23:00:07" + + """ + @spec to_iso8601(Calendar.naive_datetime(), :basic | :extended) :: String.t() + def to_iso8601(naive_datetime, format \\ :extended) + + def to_iso8601(%{calendar: Calendar.ISO} = naive_datetime, format) + when format in [:basic, :extended] do + naive_datetime + |> to_iso8601_iodata(format) + |> IO.iodata_to_binary() + end + + def to_iso8601(%{calendar: _} = naive_datetime, format) + when format in [:basic, :extended] do + naive_datetime + |> convert!(Calendar.ISO) + |> to_iso8601(format) + end + + defp to_iso8601_iodata(naive_datetime, format) do + %{ + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond + } = naive_datetime + + [ + Calendar.ISO.date_to_iodata(year, month, day, format), + ?T, + Calendar.ISO.time_to_iodata(hour, minute, second, microsecond, format) + ] + end + + @doc """ + Converts a `NaiveDateTime` struct to an Erlang datetime tuple. + + Only supports converting naive datetimes which are in the ISO calendar, + attempting to convert naive datetimes from other calendars will raise. + + WARNING: Loss of precision may occur, as Erlang time tuples only store + hour/minute/second. + + ## Examples + + iex> NaiveDateTime.to_erl(~N[2000-01-01 13:30:15]) + {{2000, 1, 1}, {13, 30, 15}} + + This function can also be used to convert a DateTime to an Erlang + datetime tuple without the time zone information: + + iex> dt = %DateTime{year: 2000, month: 2, day: 29, zone_abbr: "CET", + ...> hour: 23, minute: 0, second: 7, microsecond: {0, 0}, + ...> utc_offset: 3600, std_offset: 0, time_zone: "Europe/Warsaw"} + iex> NaiveDateTime.to_erl(dt) + {{2000, 2, 29}, {23, 00, 07}} + + """ + @spec to_erl(Calendar.naive_datetime()) :: :calendar.datetime() + def to_erl(%{calendar: _} = naive_datetime) do + %{year: year, month: month, day: day, hour: hour, minute: minute, second: second} = + convert!(naive_datetime, Calendar.ISO) + + {{year, month, day}, {hour, minute, second}} + end + + @doc """ + Converts an Erlang datetime tuple to a `NaiveDateTime` struct. + + Attempting to convert an invalid ISO calendar date will produce an error tuple. + + ## Examples + + iex> NaiveDateTime.from_erl({{2000, 1, 1}, {13, 30, 15}}) + {:ok, ~N[2000-01-01 13:30:15]} + iex> NaiveDateTime.from_erl({{2000, 1, 1}, {13, 30, 15}}, 5000) + {:ok, ~N[2000-01-01 13:30:15.005000]} + iex> NaiveDateTime.from_erl({{2000, 1, 1}, {13, 30, 15}}, {5000, 3}) + {:ok, ~N[2000-01-01 13:30:15.005]} + iex> NaiveDateTime.from_erl({{2000, 13, 1}, {13, 30, 15}}) + {:error, :invalid_date} + iex> NaiveDateTime.from_erl({{2000, 13, 1}, {13, 30, 15}}) + {:error, :invalid_date} + + """ + @spec from_erl( + :calendar.datetime(), + Calendar.microsecond() | non_neg_integer(), + Calendar.calendar() + ) :: + {:ok, t} | {:error, atom} + def from_erl(tuple, microsecond \\ {0, 0}, calendar \\ Calendar.ISO) + + def from_erl({{year, month, day}, {hour, minute, second}}, microsecond, calendar) do + with {:ok, iso_naive_dt} <- new(year, month, day, hour, minute, second, microsecond), + do: convert(iso_naive_dt, calendar) + end + + @doc """ + Converts an Erlang datetime tuple to a `NaiveDateTime` struct. + + Raises if the datetime is invalid. + Attempting to convert an invalid ISO calendar date will produce an error tuple. + + ## Examples + + iex> NaiveDateTime.from_erl!({{2000, 1, 1}, {13, 30, 15}}) + ~N[2000-01-01 13:30:15] + iex> NaiveDateTime.from_erl!({{2000, 1, 1}, {13, 30, 15}}, 5000) + ~N[2000-01-01 13:30:15.005000] + iex> NaiveDateTime.from_erl!({{2000, 1, 1}, {13, 30, 15}}, {5000, 3}) + ~N[2000-01-01 13:30:15.005] + iex> NaiveDateTime.from_erl!({{2000, 13, 1}, {13, 30, 15}}) + ** (ArgumentError) cannot convert {{2000, 13, 1}, {13, 30, 15}} to naive datetime, reason: :invalid_date + + """ + @spec from_erl!( + :calendar.datetime(), + Calendar.microsecond() | non_neg_integer(), + Calendar.calendar() + ) :: t + def from_erl!(tuple, microsecond \\ {0, 0}, calendar \\ Calendar.ISO) do + case from_erl(tuple, microsecond, calendar) do + {:ok, value} -> + value + + {:error, reason} -> + raise ArgumentError, + "cannot convert #{inspect(tuple)} to naive datetime, reason: #{inspect(reason)}" + end + end + + @doc """ + Converts a number of gregorian seconds to a `NaiveDateTime` struct. + + ## Examples + + iex> NaiveDateTime.from_gregorian_seconds(1) + ~N[0000-01-01 00:00:01] + iex> NaiveDateTime.from_gregorian_seconds(63_755_511_991, {5000, 3}) + ~N[2020-05-01 00:26:31.005] + iex> NaiveDateTime.from_gregorian_seconds(-1) + ~N[-0001-12-31 23:59:59] + + """ + @doc since: "1.11.0" + @spec from_gregorian_seconds(integer(), Calendar.microsecond(), Calendar.calendar()) :: t + def from_gregorian_seconds(seconds, microsecond_precision \\ {0, 0}, calendar \\ Calendar.ISO) + + def from_gregorian_seconds(seconds, {microsecond, precision}, Calendar.ISO) + when is_integer(seconds) do + {days, seconds} = div_rem(seconds, 24 * 60 * 60) + {hours, seconds} = div_rem(seconds, 60 * 60) + {minutes, seconds} = div_rem(seconds, 60) + {year, month, day} = Calendar.ISO.date_from_iso_days(days) + + %NaiveDateTime{ + calendar: Calendar.ISO, + year: year, + month: month, + day: day, + hour: hours, + minute: minutes, + second: seconds, + microsecond: {microsecond, precision} + } + end + + def from_gregorian_seconds(seconds, {microsecond, precision}, calendar) + when is_integer(seconds) do + iso_days = Calendar.ISO.gregorian_seconds_to_iso_days(seconds, microsecond) + + {year, month, day, hour, minute, second, {microsecond, _}} = + calendar.naive_datetime_from_iso_days(iso_days) + + %NaiveDateTime{ + calendar: calendar, + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: {microsecond, precision} + } + end + + defp div_rem(int1, int2) do + div = div(int1, int2) + rem = int1 - div * int2 + + if rem >= 0 do + {div, rem} + else + {div - 1, rem + int2} + end + end + + @doc """ + Converts a `NaiveDateTime` struct to a number of gregorian seconds and microseconds. + + ## Examples + + iex> NaiveDateTime.to_gregorian_seconds(~N[0000-01-01 00:00:01]) + {1, 0} + iex> NaiveDateTime.to_gregorian_seconds(~N[2020-05-01 00:26:31.005]) + {63_755_511_991, 5000} + + """ + @doc since: "1.11.0" + @spec to_gregorian_seconds(Calendar.naive_datetime()) :: {integer(), non_neg_integer()} + def to_gregorian_seconds(%{ + calendar: calendar, + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: {microsecond, precision} + }) do + {days, day_fraction} = + calendar.naive_datetime_to_iso_days( + year, + month, + day, + hour, + minute, + second, + {microsecond, precision} + ) + + seconds_in_day = seconds_from_day_fraction(day_fraction) + {days * @seconds_per_day + seconds_in_day, microsecond} + end + + @doc """ + Compares two `NaiveDateTime` structs. + + Returns `:gt` if first is later than the second + and `:lt` for vice versa. If the two NaiveDateTime + are equal `:eq` is returned. + + ## Examples + + iex> NaiveDateTime.compare(~N[2016-04-16 13:30:15], ~N[2016-04-28 16:19:25]) + :lt + iex> NaiveDateTime.compare(~N[2016-04-16 13:30:15.1], ~N[2016-04-16 13:30:15.01]) + :gt + + This function can also be used to compare a DateTime without + the time zone information: + + iex> dt = %DateTime{year: 2000, month: 2, day: 29, zone_abbr: "CET", + ...> hour: 23, minute: 0, second: 7, microsecond: {0, 0}, + ...> utc_offset: 3600, std_offset: 0, time_zone: "Europe/Warsaw"} + iex> NaiveDateTime.compare(dt, ~N[2000-02-29 23:00:07]) + :eq + iex> NaiveDateTime.compare(dt, ~N[2000-01-29 23:00:07]) + :gt + iex> NaiveDateTime.compare(dt, ~N[2000-03-29 23:00:07]) + :lt + + """ + @doc since: "1.4.0" + @spec compare(Calendar.naive_datetime(), Calendar.naive_datetime()) :: :lt | :eq | :gt + def compare(%{calendar: calendar1} = naive_datetime1, %{calendar: calendar2} = naive_datetime2) do + if Calendar.compatible_calendars?(calendar1, calendar2) do + case {to_iso_days(naive_datetime1), to_iso_days(naive_datetime2)} do + {first, second} when first > second -> :gt + {first, second} when first < second -> :lt + _ -> :eq + end + else + raise ArgumentError, """ + cannot compare #{inspect(naive_datetime1)} with #{inspect(naive_datetime2)}. + + This comparison would be ambiguous as their calendars have incompatible day rollover moments. + Specify an exact time of day (using `DateTime`s) to resolve this ambiguity + """ + end + end + + @doc """ + Returns `true` if the first `NaiveDateTime` is strictly earlier than the second. + + ## Examples + + iex> NaiveDateTime.before?(~N[2021-01-01 11:00:00], ~N[2022-02-02 11:00:00]) + true + iex> NaiveDateTime.before?(~N[2021-01-01 11:00:00], ~N[2021-01-01 11:00:00]) + false + iex> NaiveDateTime.before?(~N[2022-02-02 11:00:00], ~N[2021-01-01 11:00:00]) + false + + """ + @doc since: "1.15.0" + @spec before?(Calendar.naive_datetime(), Calendar.naive_datetime()) :: boolean() + def before?(naive_datetime1, naive_datetime2) do + compare(naive_datetime1, naive_datetime2) == :lt + end + + @doc """ + Returns `true` if the first `NaiveDateTime` is strictly later than the second. + + ## Examples + + iex> NaiveDateTime.after?(~N[2022-02-02 11:00:00], ~N[2021-01-01 11:00:00]) + true + iex> NaiveDateTime.after?(~N[2021-01-01 11:00:00], ~N[2021-01-01 11:00:00]) + false + iex> NaiveDateTime.after?(~N[2021-01-01 11:00:00], ~N[2022-02-02 11:00:00]) + false + + """ + @doc since: "1.15.0" + @spec after?(Calendar.naive_datetime(), Calendar.naive_datetime()) :: boolean() + def after?(naive_datetime1, naive_datetime2) do + compare(naive_datetime1, naive_datetime2) == :gt + end + + @doc """ + Converts the given `naive_datetime` from one calendar to another. + + If it is not possible to convert unambiguously between the calendars + (see `Calendar.compatible_calendars?/2`), an `{:error, :incompatible_calendars}` tuple + is returned. + + ## Examples + + Imagine someone implements `Calendar.Holocene`, a calendar based on the + Gregorian calendar that adds exactly 10 000 years to the current Gregorian + year: + + iex> NaiveDateTime.convert(~N[2000-01-01 13:30:15], Calendar.Holocene) + {:ok, %NaiveDateTime{calendar: Calendar.Holocene, year: 12000, month: 1, day: 1, + hour: 13, minute: 30, second: 15, microsecond: {0, 0}}} + + """ + @doc since: "1.5.0" + @spec convert(Calendar.naive_datetime(), Calendar.calendar()) :: + {:ok, t} | {:error, :incompatible_calendars} + + # Keep it multiline for proper function clause errors. + def convert(%NaiveDateTime{calendar: calendar} = ndt, calendar) do + {:ok, ndt} + end + + def convert( + %{ + calendar: calendar, + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond + }, + calendar + ) do + naive_datetime = %NaiveDateTime{ + calendar: calendar, + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond + } + + {:ok, naive_datetime} + end + + def convert(%{calendar: ndt_calendar, microsecond: {_, precision}} = naive_datetime, calendar) do + if Calendar.compatible_calendars?(ndt_calendar, calendar) do + result_naive_datetime = + naive_datetime + |> to_iso_days() + |> from_iso_days(calendar, precision) + + {:ok, result_naive_datetime} + else + {:error, :incompatible_calendars} + end + end + + @doc """ + Converts the given `naive_datetime` from one calendar to another. + + If it is not possible to convert unambiguously between the calendars + (see `Calendar.compatible_calendars?/2`), an ArgumentError is raised. + + ## Examples + + Imagine someone implements `Calendar.Holocene`, a calendar based on the + Gregorian calendar that adds exactly 10 000 years to the current Gregorian + year: + + iex> NaiveDateTime.convert!(~N[2000-01-01 13:30:15], Calendar.Holocene) + %NaiveDateTime{calendar: Calendar.Holocene, year: 12000, month: 1, day: 1, + hour: 13, minute: 30, second: 15, microsecond: {0, 0}} + + """ + @doc since: "1.5.0" + @spec convert!(Calendar.naive_datetime(), Calendar.calendar()) :: t + def convert!(naive_datetime, calendar) do + case convert(naive_datetime, calendar) do + {:ok, value} -> + value + + {:error, :incompatible_calendars} -> + raise ArgumentError, + "cannot convert #{inspect(naive_datetime)} to target calendar #{inspect(calendar)}, " <> + "reason: #{inspect(naive_datetime.calendar)} and #{inspect(calendar)} " <> + "have different day rollover moments, making this conversion ambiguous" + end + end + + @doc """ + Calculates a `NaiveDateTime` that is the first moment for the given `NaiveDateTime`. + + To calculate the beginning of day of a `DateTime`, call this function, then convert back to a `DateTime`: + + datetime + |> NaiveDateTime.beginning_of_day() + |> DateTime.from_naive(datetime.time_zone) + + Note that the beginning of the day may not exist or be ambiguous + in a given timezone, so you must handle those cases accordingly. + + ## Examples + + iex> NaiveDateTime.beginning_of_day(~N[2000-01-01 23:00:07.123456]) + ~N[2000-01-01 00:00:00.000000] + + """ + @doc since: "1.15.0" + @spec beginning_of_day(Calendar.naive_datetime()) :: t + def beginning_of_day(%{calendar: calendar, microsecond: {_, precision}} = naive_datetime) do + naive_datetime + |> to_iso_days() + |> calendar.iso_days_to_beginning_of_day() + |> from_iso_days(calendar, precision) + end + + @doc """ + Calculates a `NaiveDateTime` that is the last moment for the given `NaiveDateTime`. + + To calculate the end of day of a `DateTime`, call this function, then convert back to a `DateTime`: + + datetime + |> NaiveDateTime.end_of_day() + |> DateTime.from_naive(datetime.time_zone) + + Note that the end of the day may not exist or be ambiguous + in a given timezone, so you must handle those cases accordingly. + + ## Examples + + iex> NaiveDateTime.end_of_day(~N[2000-01-01 23:00:07.123456]) + ~N[2000-01-01 23:59:59.999999] + + """ + @doc since: "1.15.0" + @spec end_of_day(Calendar.naive_datetime()) :: t + def end_of_day(%{calendar: calendar, microsecond: {_, precision}} = naive_datetime) do + end_of_day = + naive_datetime + |> to_iso_days() + |> calendar.iso_days_to_end_of_day() + |> from_iso_days(calendar, precision) + + %{microsecond: {microsecond, precision}} = end_of_day + + multiplier = 10 ** (6 - precision) + + %{end_of_day | microsecond: {div(microsecond, multiplier) * multiplier, precision}} + end + + ## Helpers + + defp seconds_from_day_fraction({parts_in_day, @seconds_per_day}), + do: parts_in_day + + defp seconds_from_day_fraction({parts_in_day, parts_per_day}), + do: div(parts_in_day * @seconds_per_day, parts_per_day) + + # Keep it multiline for proper function clause errors. + defp to_iso_days(%{ + calendar: calendar, + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond + }) do + calendar.naive_datetime_to_iso_days(year, month, day, hour, minute, second, microsecond) + end + + defp from_iso_days(iso_days, calendar, precision) do + {year, month, day, hour, minute, second, {microsecond, _}} = + calendar.naive_datetime_from_iso_days(iso_days) + + %NaiveDateTime{ + calendar: calendar, + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: {microsecond, precision} + } + end + + defimpl String.Chars do + def to_string(naive_datetime) do + %{ + calendar: calendar, + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond + } = naive_datetime + + calendar.naive_datetime_to_string(year, month, day, hour, minute, second, microsecond) + end + end + + defimpl Inspect do + def inspect(naive_datetime, _) do + %{ + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond, + calendar: calendar + } = naive_datetime + + if calendar != Calendar.ISO or year in -9999..9999 do + formatted = + calendar.naive_datetime_to_string(year, month, day, hour, minute, second, microsecond) + + "~N[" <> formatted <> suffix(calendar) <> "]" + else + "NaiveDateTime.new!(#{Integer.to_string(year)}, #{Integer.to_string(month)}, #{Integer.to_string(day)}, " <> + "#{Integer.to_string(hour)}, #{Integer.to_string(minute)}, #{Integer.to_string(second)}, #{inspect(microsecond)})" + end + end + + defp suffix(Calendar.ISO), do: "" + defp suffix(calendar), do: " " <> inspect(calendar) + end +end diff --git a/lib/elixir/lib/calendar/time.ex b/lib/elixir/lib/calendar/time.ex new file mode 100644 index 00000000000..831a19aaed3 --- /dev/null +++ b/lib/elixir/lib/calendar/time.ex @@ -0,0 +1,1011 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +defmodule Time do + @moduledoc """ + A Time struct and functions. + + The Time struct contains the fields hour, minute, second and microseconds. + New times can be built with the `new/4` function or using the + `~T` (see `sigil_T/2`) sigil: + + iex> ~T[23:00:07.001] + ~T[23:00:07.001] + + Both `new/4` and sigil return a struct where the time fields can + be accessed directly: + + iex> time = ~T[23:00:07.001] + iex> time.hour + 23 + iex> time.microsecond + {1000, 3} + + The functions on this module work with the `Time` struct as well + as any struct that contains the same fields as the `Time` struct, + such as `NaiveDateTime` and `DateTime`. Such functions expect + `t:Calendar.time/0` in their typespecs (instead of `t:t/0`). + + Developers should avoid creating the Time structs directly + and instead rely on the functions provided by this module as well + as the ones in third-party calendar libraries. + + ## Comparing times + + Comparisons in Elixir using `==/2`, `>/2`, ` Enum.min([~T[23:00:07.001], ~T[10:00:07.001]], Time) + ~T[10:00:07.001] + """ + + @enforce_keys [:hour, :minute, :second] + defstruct [:hour, :minute, :second, microsecond: {0, 0}, calendar: Calendar.ISO] + + @type t :: %__MODULE__{ + hour: Calendar.hour(), + minute: Calendar.minute(), + second: Calendar.second(), + microsecond: Calendar.microsecond(), + calendar: Calendar.calendar() + } + + @seconds_per_day 24 * 60 * 60 + + @doc """ + Returns the current time in UTC. + + You can pass a time unit to automatically truncate the resulting time. + + The default unit if none gets passed is `:native` which results on a default resolution of microseconds. + + ## Examples + + iex> time = Time.utc_now() + iex> time.hour >= 0 + true + + iex> time = Time.utc_now(:second) + iex> time.microsecond + {0, 0} + + """ + @doc since: "1.4.0" + @spec utc_now(Calendar.calendar() | :native | :microsecond | :millisecond | :second) :: t + def utc_now(calendar_or_time_unit \\ Calendar.ISO) do + case calendar_or_time_unit do + unit when unit in [:native, :microsecond, :millisecond, :second] -> + utc_now(unit, Calendar.ISO) + + calendar -> + utc_now(:native, calendar) + end + end + + @doc """ + Returns the current time in UTC, supporting a precision and a specific calendar. + + ## Examples + + iex> time = Time.utc_now(:microsecond, Calendar.ISO) + iex> time.hour >= 0 + true + + iex> time = Time.utc_now(:second, Calendar.ISO) + iex> time.microsecond + {0, 0} + + """ + @doc since: "1.19.0" + @spec utc_now(:native | :microsecond | :millisecond | :second, Calendar.calendar()) :: t + def utc_now(time_unit, calendar) + when time_unit in [:native, :microsecond, :millisecond, :second] do + {:ok, _, time, microsecond} = Calendar.ISO.from_unix(System.os_time(time_unit), time_unit) + {hour, minute, second} = time + + iso_time = %Time{ + hour: hour, + minute: minute, + second: second, + microsecond: microsecond, + calendar: Calendar.ISO + } + + convert!(iso_time, calendar) + end + + @doc """ + Builds a new time. + + Expects all values to be integers. Returns `{:ok, time}` if each + entry fits its appropriate range, returns `{:error, reason}` otherwise. + + Microseconds can also be given with a precision, which must be an + integer between 0 and 6. + + The built-in calendar does not support leap seconds. + + ## Examples + + iex> Time.new(0, 0, 0, 0) + {:ok, ~T[00:00:00.000000]} + iex> Time.new(23, 59, 59, 999_999) + {:ok, ~T[23:59:59.999999]} + + iex> Time.new(24, 59, 59, 999_999) + {:error, :invalid_time} + iex> Time.new(23, 60, 59, 999_999) + {:error, :invalid_time} + iex> Time.new(23, 59, 60, 999_999) + {:error, :invalid_time} + iex> Time.new(23, 59, 59, 1_000_000) + {:error, :invalid_time} + + # Invalid precision + Time.new(23, 59, 59, {999_999, 10}) + {:error, :invalid_time} + + """ + @spec new( + Calendar.hour(), + Calendar.minute(), + Calendar.second(), + Calendar.microsecond() | non_neg_integer(), + Calendar.calendar() + ) :: {:ok, t} | {:error, atom} + def new(hour, minute, second, microsecond \\ {0, 0}, calendar \\ Calendar.ISO) + + def new(hour, minute, second, microsecond, calendar) when is_integer(microsecond) do + new(hour, minute, second, {microsecond, 6}, calendar) + end + + def new(hour, minute, second, {microsecond, precision}, calendar) + when is_integer(hour) and is_integer(minute) and is_integer(second) and + is_integer(microsecond) and is_integer(precision) do + case calendar.valid_time?(hour, minute, second, {microsecond, precision}) do + true -> + time = %Time{ + hour: hour, + minute: minute, + second: second, + microsecond: {microsecond, precision}, + calendar: calendar + } + + {:ok, time} + + false -> + {:error, :invalid_time} + end + end + + @doc """ + Builds a new time. + + Expects all values to be integers. Returns `time` if each + entry fits its appropriate range, raises if the time is invalid. + + Microseconds can also be given with a precision, which must be an + integer between 0 and 6. + + The built-in calendar does not support leap seconds. + + ## Examples + + iex> Time.new!(0, 0, 0, 0) + ~T[00:00:00.000000] + iex> Time.new!(23, 59, 59, 999_999) + ~T[23:59:59.999999] + iex> Time.new!(24, 59, 59, 999_999) + ** (ArgumentError) cannot build time, reason: :invalid_time + """ + @doc since: "1.11.0" + @spec new!( + Calendar.hour(), + Calendar.minute(), + Calendar.second(), + Calendar.microsecond() | non_neg_integer, + Calendar.calendar() + ) :: t + def new!(hour, minute, second, microsecond \\ {0, 0}, calendar \\ Calendar.ISO) do + case new(hour, minute, second, microsecond, calendar) do + {:ok, time} -> + time + + {:error, reason} -> + raise ArgumentError, "cannot build time, reason: #{inspect(reason)}" + end + end + + @doc """ + Converts the given `time` to a string. + + ## Examples + + iex> Time.to_string(~T[23:00:00]) + "23:00:00" + iex> Time.to_string(~T[23:00:00.001]) + "23:00:00.001" + iex> Time.to_string(~T[23:00:00.123456]) + "23:00:00.123456" + + iex> Time.to_string(~N[2015-01-01 23:00:00.001]) + "23:00:00.001" + iex> Time.to_string(~N[2015-01-01 23:00:00.123456]) + "23:00:00.123456" + + """ + @spec to_string(Calendar.time()) :: String.t() + def to_string(time) + + def to_string(%{ + hour: hour, + minute: minute, + second: second, + microsecond: microsecond, + calendar: calendar + }) do + calendar.time_to_string(hour, minute, second, microsecond) + end + + @doc """ + Parses the extended "Local time" format described by + [ISO 8601:2019](https://en.wikipedia.org/wiki/ISO_8601). + + Time zone offset may be included in the string but they will be + simply discarded as such information is not included in times. + + As specified in the standard, the separator "T" may be omitted if + desired as there is no ambiguity within this function. + + ## Examples + + iex> Time.from_iso8601("23:50:07") + {:ok, ~T[23:50:07]} + iex> Time.from_iso8601("23:50:07Z") + {:ok, ~T[23:50:07]} + iex> Time.from_iso8601("T23:50:07Z") + {:ok, ~T[23:50:07]} + + iex> Time.from_iso8601("23:50:07,0123456") + {:ok, ~T[23:50:07.012345]} + iex> Time.from_iso8601("23:50:07.0123456") + {:ok, ~T[23:50:07.012345]} + iex> Time.from_iso8601("23:50:07.123Z") + {:ok, ~T[23:50:07.123]} + + iex> Time.from_iso8601("2015:01:23 23-50-07") + {:error, :invalid_format} + iex> Time.from_iso8601("23:50:07A") + {:error, :invalid_format} + iex> Time.from_iso8601("23:50:07.") + {:error, :invalid_format} + iex> Time.from_iso8601("23:50:61") + {:error, :invalid_time} + + """ + @spec from_iso8601(String.t(), Calendar.calendar()) :: {:ok, t} | {:error, atom} + def from_iso8601(string, calendar \\ Calendar.ISO) do + with {:ok, {hour, minute, second, microsecond}} <- Calendar.ISO.parse_time(string) do + convert( + %Time{hour: hour, minute: minute, second: second, microsecond: microsecond}, + calendar + ) + end + end + + @doc """ + Parses the extended "Local time" format described by + [ISO 8601:2019](https://en.wikipedia.org/wiki/ISO_8601). + + Raises if the format is invalid. + + ## Examples + + iex> Time.from_iso8601!("23:50:07,123Z") + ~T[23:50:07.123] + iex> Time.from_iso8601!("23:50:07.123Z") + ~T[23:50:07.123] + iex> Time.from_iso8601!("2015:01:23 23-50-07") + ** (ArgumentError) cannot parse "2015:01:23 23-50-07" as time, reason: :invalid_format + + """ + @spec from_iso8601!(String.t(), Calendar.calendar()) :: t + def from_iso8601!(string, calendar \\ Calendar.ISO) do + case from_iso8601(string, calendar) do + {:ok, value} -> + value + + {:error, reason} -> + raise ArgumentError, "cannot parse #{inspect(string)} as time, reason: #{inspect(reason)}" + end + end + + @doc """ + Converts the given time to + [ISO 8601:2019](https://en.wikipedia.org/wiki/ISO_8601). + + By default, `Time.to_iso8601/2` returns times formatted in the "extended" + format, for human readability. It also supports the "basic" format through + passing the `:basic` option. + + ## Examples + + iex> Time.to_iso8601(~T[23:00:13]) + "23:00:13" + + iex> Time.to_iso8601(~T[23:00:13.001]) + "23:00:13.001" + + iex> Time.to_iso8601(~T[23:00:13.001], :basic) + "230013.001" + + iex> Time.to_iso8601(~N[2010-04-17 23:00:13]) + "23:00:13" + + """ + @spec to_iso8601(Calendar.time(), :extended | :basic) :: String.t() + def to_iso8601(time, format \\ :extended) + + def to_iso8601(%{calendar: Calendar.ISO} = time, format) when format in [:extended, :basic] do + %{ + hour: hour, + minute: minute, + second: second, + microsecond: microsecond + } = time + + Calendar.ISO.time_to_string(hour, minute, second, microsecond, format) + end + + def to_iso8601(%{calendar: _} = time, format) when format in [:extended, :basic] do + time + |> convert!(Calendar.ISO) + |> to_iso8601(format) + end + + @doc """ + Converts given `time` to an Erlang time tuple. + + WARNING: Loss of precision may occur, as Erlang time tuples + only contain hours/minutes/seconds. + + ## Examples + + iex> Time.to_erl(~T[23:30:15.999]) + {23, 30, 15} + + iex> Time.to_erl(~N[2010-04-17 23:30:15.999]) + {23, 30, 15} + + """ + @spec to_erl(Calendar.time()) :: :calendar.time() + def to_erl(time) do + %{hour: hour, minute: minute, second: second} = convert!(time, Calendar.ISO) + {hour, minute, second} + end + + @doc """ + Converts an Erlang time tuple to a `Time` struct. + + ## Examples + + iex> Time.from_erl({23, 30, 15}) + {:ok, ~T[23:30:15]} + iex> Time.from_erl({23, 30, 15}, 5000) + {:ok, ~T[23:30:15.005000]} + iex> Time.from_erl({23, 30, 15}, {5000, 3}) + {:ok, ~T[23:30:15.005]} + iex> Time.from_erl({24, 30, 15}) + {:error, :invalid_time} + + """ + @spec from_erl( + :calendar.time(), + Calendar.microsecond() | non_neg_integer(), + Calendar.calendar() + ) :: + {:ok, t} | {:error, atom} + def from_erl(tuple, microsecond \\ {0, 0}, calendar \\ Calendar.ISO) + + def from_erl({hour, minute, second}, microsecond, calendar) do + with {:ok, time} <- new(hour, minute, second, microsecond, Calendar.ISO), + do: convert(time, calendar) + end + + @doc """ + Converts an Erlang time tuple to a `Time` struct. + + ## Examples + + iex> Time.from_erl!({23, 30, 15}) + ~T[23:30:15] + iex> Time.from_erl!({23, 30, 15}, 5000) + ~T[23:30:15.005000] + iex> Time.from_erl!({23, 30, 15}, {5000, 3}) + ~T[23:30:15.005] + iex> Time.from_erl!({24, 30, 15}) + ** (ArgumentError) cannot convert {24, 30, 15} to time, reason: :invalid_time + + """ + @spec from_erl!(:calendar.time(), Calendar.microsecond(), Calendar.calendar()) :: t + def from_erl!(tuple, microsecond \\ {0, 0}, calendar \\ Calendar.ISO) do + case from_erl(tuple, microsecond, calendar) do + {:ok, value} -> + value + + {:error, reason} -> + raise ArgumentError, + "cannot convert #{inspect(tuple)} to time, reason: #{inspect(reason)}" + end + end + + @doc """ + Converts a number of seconds after midnight to a `Time` struct. + + ## Examples + + iex> Time.from_seconds_after_midnight(10_000) + ~T[02:46:40] + iex> Time.from_seconds_after_midnight(30_000, {5000, 3}) + ~T[08:20:00.005] + iex> Time.from_seconds_after_midnight(-1) + ~T[23:59:59] + iex> Time.from_seconds_after_midnight(100_000) + ~T[03:46:40] + + """ + @doc since: "1.11.0" + @spec from_seconds_after_midnight( + integer(), + Calendar.microsecond(), + Calendar.calendar() + ) :: t + def from_seconds_after_midnight(seconds, microsecond \\ {0, 0}, calendar \\ Calendar.ISO) + when is_integer(seconds) do + seconds_in_day = Integer.mod(seconds, @seconds_per_day) + + {hour, minute, second, {_, _}} = + calendar.time_from_day_fraction({seconds_in_day, @seconds_per_day}) + + %Time{ + calendar: calendar, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond + } + end + + @doc """ + Converts a `Time` struct to a number of seconds after midnight. + + The returned value is a two-element tuple with the number of seconds and microseconds. + + ## Examples + + iex> Time.to_seconds_after_midnight(~T[23:30:15]) + {84615, 0} + iex> Time.to_seconds_after_midnight(~N[2010-04-17 23:30:15.999]) + {84615, 999000} + + """ + @doc since: "1.11.0" + @spec to_seconds_after_midnight(Calendar.time()) :: {integer(), non_neg_integer()} + def to_seconds_after_midnight(%{microsecond: {microsecond, _precision}} = time) do + iso_days = {0, to_day_fraction(time)} + {Calendar.ISO.iso_days_to_unit(iso_days, :second), microsecond} + end + + @doc """ + Adds the `amount_to_add` of `unit`s to the given `time`. + + > #### Prefer `shift/2` {: .info} + > + > Prefer `shift/2` over `add/3`, as it offers a more ergonomic API. + > + > `add/3` always considers the unit to be computed according to + > the `Calendar.ISO`. + + Accepts an `amount_to_add` in any `unit`. `unit` can be + `:hour`, `:minute`, `:second` or any subsecond precision from + `t:System.time_unit/0` for convenience but ultimately they are + all converted to microseconds. Negative values will move backwards + in time and the default precision is `:second`. + + Note the result value represents the time of day, meaning that it is cyclic, + for instance, it will never go over 24 hours for the ISO calendar. + + ## Examples + + iex> Time.add(~T[10:00:00], 27000) + ~T[17:30:00] + iex> Time.add(~T[11:00:00.005], 2400) + ~T[11:40:00.005] + iex> Time.add(~T[00:00:00.000], 86_399_999, :millisecond) + ~T[23:59:59.999] + + Negative values are allowed: + + iex> Time.add(~T[23:00:00], -60) + ~T[22:59:00] + + Note that the time is cyclic: + + iex> Time.add(~T[17:10:05], 86400) + ~T[17:10:05] + + Hours and minutes are also supported: + + iex> Time.add(~T[17:10:05], 2, :hour) + ~T[19:10:05] + iex> Time.add(~T[17:10:05], 30, :minute) + ~T[17:40:05] + + This operation merges the precision of the time with the given unit: + + iex> result = Time.add(~T[00:29:10], 21, :millisecond) + ~T[00:29:10.021] + iex> result.microsecond + {21000, 3} + + """ + @doc since: "1.6.0" + @spec add(Calendar.time(), integer, :hour | :minute | System.time_unit()) :: t + def add(time, amount_to_add, unit \\ :second) + + def add(time, amount_to_add, :hour) when is_integer(amount_to_add) do + add(time, amount_to_add * 3600, :second) + end + + def add(time, amount_to_add, :minute) when is_integer(amount_to_add) do + add(time, amount_to_add * 60, :second) + end + + def add(%{calendar: calendar, microsecond: {_, precision}} = time, amount_to_add, unit) + when is_integer(amount_to_add) do + valid? = + if is_integer(unit), + do: unit > 0, + else: unit in ~w(second millisecond microsecond nanosecond)a + + if not valid? do + raise ArgumentError, + "unsupported time unit. Expected :hour, :minute, :second, :millisecond, :microsecond, :nanosecond, or a positive integer, got #{inspect(unit)}" + end + + %{hour: hour, minute: minute, second: second, microsecond: microsecond} = time + + precision = max(Calendar.ISO.time_unit_to_precision(unit), precision) + + {hour, minute, second, {microsecond, _precision}} = + Calendar.ISO.shift_time_unit( + {hour, minute, second, microsecond}, + amount_to_add, + unit + ) + + %Time{ + hour: hour, + minute: minute, + second: second, + microsecond: {microsecond, precision}, + calendar: calendar + } + end + + @doc """ + Shifts given `time` by `duration` according to its calendar. + + Available duration units are: `:hour`, `:minute`, `:second`, `:microsecond`. + + When using the default ISO calendar, durations are collapsed to seconds and + microseconds before they are applied. + + Raises an `ArgumentError` when called with date scale units. + + ## Examples + + iex> Time.shift(~T[01:00:15], hour: 12) + ~T[13:00:15] + iex> Time.shift(~T[01:35:00], hour: 6, minute: -15) + ~T[07:20:00] + iex> Time.shift(~T[01:15:00], second: 125) + ~T[01:17:05] + iex> Time.shift(~T[01:00:15], microsecond: {100, 6}) + ~T[01:00:15.000100] + iex> Time.shift(~T[01:15:00], Duration.new!(second: 65)) + ~T[01:16:05] + + """ + @doc since: "1.17.0" + @spec shift(Calendar.time(), Duration.t() | [unit_pair]) :: t + when unit_pair: + {:hour, integer} + | {:minute, integer} + | {:second, integer} + | {:microsecond, {integer, 0..6}} + def shift(%{calendar: calendar} = time, duration) do + %{hour: hour, minute: minute, second: second, microsecond: microsecond} = time + + {hour, minute, second, microsecond} = + calendar.shift_time(hour, minute, second, microsecond, __duration__!(duration)) + + %Time{ + calendar: calendar, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond + } + end + + @doc false + def __duration__!(%Duration{} = duration) do + duration + end + + # This part is inlined by the compiler on constant values + def __duration__!(unit_pairs) do + Enum.each(unit_pairs, &validate_duration_unit!/1) + struct!(Duration, unit_pairs) + end + + defp validate_duration_unit!({:microsecond, {ms, precision}}) + when is_integer(ms) and precision in 0..6 do + :ok + end + + defp validate_duration_unit!({:microsecond, microsecond}) do + raise ArgumentError, + "unsupported value #{inspect(microsecond)} for :microsecond. Expected a tuple {ms, precision} where precision is an integer from 0 to 6" + end + + defp validate_duration_unit!({unit, _value}) when unit in [:year, :month, :week, :day] do + raise ArgumentError, + "unsupported unit #{inspect(unit)}. Expected :hour, :minute, :second, :microsecond" + end + + defp validate_duration_unit!({unit, _value}) + when unit not in [:hour, :minute, :second, :microsecond] do + raise ArgumentError, + "unknown unit #{inspect(unit)}. Expected :hour, :minute, :second, :microsecond" + end + + defp validate_duration_unit!({_unit, value}) when is_integer(value) do + :ok + end + + defp validate_duration_unit!({unit, value}) do + raise ArgumentError, + "unsupported value #{inspect(value)} for #{inspect(unit)}. Expected an integer" + end + + @doc """ + Compares two time structs. + + Returns `:gt` if first time is later than the second + and `:lt` for vice versa. If the two times are equal + `:eq` is returned. + + ## Examples + + iex> Time.compare(~T[16:04:16], ~T[16:04:28]) + :lt + iex> Time.compare(~T[16:04:16], ~T[16:04:16]) + :eq + iex> Time.compare(~T[16:04:16.01], ~T[16:04:16.001]) + :gt + + This function can also be used to compare across more + complex calendar types by considering only the time fields: + + iex> Time.compare(~N[1900-01-01 16:04:16], ~N[2015-01-01 16:04:16]) + :eq + iex> Time.compare(~N[2015-01-01 16:04:16], ~N[2015-01-01 16:04:28]) + :lt + iex> Time.compare(~N[2015-01-01 16:04:16.01], ~N[2000-01-01 16:04:16.001]) + :gt + + """ + @doc since: "1.4.0" + @spec compare(Calendar.time(), Calendar.time()) :: :lt | :eq | :gt + def compare(%{calendar: calendar} = time1, %{calendar: calendar} = time2) do + %{hour: hour1, minute: minute1, second: second1, microsecond: {microsecond1, _}} = time1 + %{hour: hour2, minute: minute2, second: second2, microsecond: {microsecond2, _}} = time2 + + case {{hour1, minute1, second1, microsecond1}, {hour2, minute2, second2, microsecond2}} do + {first, second} when first > second -> :gt + {first, second} when first < second -> :lt + _ -> :eq + end + end + + def compare(time1, time2) do + {parts1, ppd1} = to_day_fraction(time1) + {parts2, ppd2} = to_day_fraction(time2) + + case {parts1 * ppd2, parts2 * ppd1} do + {first, second} when first > second -> :gt + {first, second} when first < second -> :lt + _ -> :eq + end + end + + @doc """ + Returns `true` if the first time is strictly earlier than the second. + + ## Examples + + iex> Time.before?(~T[16:04:16], ~T[16:04:28]) + true + iex> Time.before?(~T[16:04:16], ~T[16:04:16]) + false + iex> Time.before?(~T[16:04:16.01], ~T[16:04:16.001]) + false + + """ + @doc since: "1.15.0" + @spec before?(Calendar.time(), Calendar.time()) :: boolean() + def before?(time1, time2) do + compare(time1, time2) == :lt + end + + @doc """ + Returns `true` if the first time is strictly later than the second. + + ## Examples + + iex> Time.after?(~T[16:04:28], ~T[16:04:16]) + true + iex> Time.after?(~T[16:04:16], ~T[16:04:16]) + false + iex> Time.after?(~T[16:04:16.001], ~T[16:04:16.01]) + false + + """ + @doc since: "1.15.0" + @spec after?(Calendar.time(), Calendar.time()) :: boolean() + def after?(time1, time2) do + compare(time1, time2) == :gt + end + + @doc """ + Converts given `time` to a different calendar. + + Returns `{:ok, time}` if the conversion was successful, + or `{:error, reason}` if it was not, for some reason. + + ## Examples + + Imagine someone implements `Calendar.Holocene`, a calendar based on the + Gregorian calendar that adds exactly 10 000 years to the current Gregorian + year: + + iex> Time.convert(~T[13:30:15], Calendar.Holocene) + {:ok, %Time{calendar: Calendar.Holocene, hour: 13, minute: 30, second: 15, microsecond: {0, 0}}} + + """ + @doc since: "1.5.0" + @spec convert(Calendar.time(), Calendar.calendar()) :: {:ok, t} | {:error, atom} + + # Keep it multiline for proper function clause errors. + def convert( + %{ + calendar: calendar, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond + }, + calendar + ) do + time = %Time{ + calendar: calendar, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond + } + + {:ok, time} + end + + def convert(%{microsecond: {_, precision}} = time, calendar) do + {hour, minute, second, {microsecond, _}} = + time + |> to_day_fraction() + |> calendar.time_from_day_fraction() + + time = %Time{ + calendar: calendar, + hour: hour, + minute: minute, + second: second, + microsecond: {microsecond, precision} + } + + {:ok, time} + end + + @doc """ + Similar to `Time.convert/2`, but raises an `ArgumentError` + if the conversion between the two calendars is not possible. + + ## Examples + + Imagine someone implements `Calendar.Holocene`, a calendar based on the + Gregorian calendar that adds exactly 10 000 years to the current Gregorian + year: + + iex> Time.convert!(~T[13:30:15], Calendar.Holocene) + %Time{calendar: Calendar.Holocene, hour: 13, minute: 30, second: 15, microsecond: {0, 0}} + + """ + @doc since: "1.5.0" + @spec convert!(Calendar.time(), Calendar.calendar()) :: t + def convert!(time, calendar) do + {:ok, value} = convert(time, calendar) + value + end + + @doc """ + Returns the difference between two times, considering only the hour, minute, + second and microsecond. + + As with the `compare/2` function both `Time` structs and other structures + containing time can be used. If for instance a `NaiveDateTime` or `DateTime` + is passed, only the hour, minute, second, and microsecond is considered. Any + additional information about a date or time zone is ignored when calculating + the difference. + + The answer can be returned in any `:hour`, `:minute`, `:second` or any + subsecond `unit` available from `t:System.time_unit/0`. If the first time + value is earlier than the second, a negative number is returned. + + The unit is measured according to `Calendar.ISO` and defaults to `:second`. + Fractional results are not supported and are truncated. + + ## Examples + + iex> Time.diff(~T[00:29:12], ~T[00:29:10]) + 2 + + # When passing a `NaiveDateTime` the date part is ignored. + iex> Time.diff(~N[2017-01-01 00:29:12], ~T[00:29:10]) + 2 + + # Two `NaiveDateTime` structs could have big differences in the date + # but only the time part is considered. + iex> Time.diff(~N[2017-01-01 00:29:12], ~N[1900-02-03 00:29:10]) + 2 + + iex> Time.diff(~T[00:29:12], ~T[00:29:10], :microsecond) + 2_000_000 + iex> Time.diff(~T[00:29:10], ~T[00:29:12], :microsecond) + -2_000_000 + + iex> Time.diff(~T[02:29:10], ~T[00:29:10], :hour) + 2 + iex> Time.diff(~T[02:29:10], ~T[00:29:11], :hour) + 1 + + """ + @doc since: "1.5.0" + @spec diff(Calendar.time(), Calendar.time(), :hour | :minute | System.time_unit()) :: integer + def diff(time1, time2, unit \\ :second) + + def diff(time1, time2, :hour) do + diff(time1, time2, :second) |> div(3600) + end + + def diff(time1, time2, :minute) do + diff(time1, time2, :second) |> div(60) + end + + def diff( + %{ + calendar: Calendar.ISO, + hour: hour1, + minute: minute1, + second: second1, + microsecond: {microsecond1, _} + }, + %{ + calendar: Calendar.ISO, + hour: hour2, + minute: minute2, + second: second2, + microsecond: {microsecond2, _} + }, + unit + ) do + total = + (hour1 - hour2) * 3_600_000_000 + (minute1 - minute2) * 60_000_000 + + (second1 - second2) * 1_000_000 + (microsecond1 - microsecond2) + + System.convert_time_unit(total, :microsecond, unit) + end + + def diff(time1, time2, unit) do + fraction1 = to_day_fraction(time1) + fraction2 = to_day_fraction(time2) + + Calendar.ISO.iso_days_to_unit({0, fraction1}, unit) - + Calendar.ISO.iso_days_to_unit({0, fraction2}, unit) + end + + @doc """ + Returns the given time with the microsecond field truncated to the given + precision (`:microsecond`, `millisecond` or `:second`). + + The given time is returned unchanged if it already has lower precision than + the given precision. + + ## Examples + + iex> Time.truncate(~T[01:01:01.123456], :microsecond) + ~T[01:01:01.123456] + + iex> Time.truncate(~T[01:01:01.123456], :millisecond) + ~T[01:01:01.123] + + iex> Time.truncate(~T[01:01:01.123456], :second) + ~T[01:01:01] + + """ + @doc since: "1.6.0" + @spec truncate(t(), :microsecond | :millisecond | :second) :: t() + def truncate(%Time{microsecond: microsecond} = time, precision) do + %{time | microsecond: Calendar.truncate(microsecond, precision)} + end + + ## Helpers + + defp to_day_fraction(%{ + hour: hour, + minute: minute, + second: second, + microsecond: {_, _} = microsecond, + calendar: calendar + }) do + calendar.time_to_day_fraction(hour, minute, second, microsecond) + end + + defimpl String.Chars do + def to_string(time) do + %{ + hour: hour, + minute: minute, + second: second, + microsecond: microsecond, + calendar: calendar + } = time + + calendar.time_to_string(hour, minute, second, microsecond) + end + end + + defimpl Inspect do + def inspect(time, _) do + %{ + hour: hour, + minute: minute, + second: second, + microsecond: microsecond, + calendar: calendar + } = time + + "~T[" <> + calendar.time_to_string(hour, minute, second, microsecond) <> suffix(calendar) <> "]" + end + + defp suffix(Calendar.ISO), do: "" + defp suffix(calendar), do: " " <> inspect(calendar) + end +end diff --git a/lib/elixir/lib/calendar/time_zone_database.ex b/lib/elixir/lib/calendar/time_zone_database.ex new file mode 100644 index 00000000000..dcfac50f8da --- /dev/null +++ b/lib/elixir/lib/calendar/time_zone_database.ex @@ -0,0 +1,102 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +defmodule Calendar.TimeZoneDatabase do + @moduledoc """ + This module defines a behaviour for providing time zone data. + + IANA provides time zone data that includes data about different + UTC offsets and standard offsets for time zones. + """ + + @typedoc """ + A period where a certain combination of UTC offset, standard offset, and zone + abbreviation is in effect. + + For example, one period could be the summer of 2018 in the `Europe/London` timezone, + where summer time/daylight saving time is in effect and lasts from spring to autumn. + In autumn, the `std_offset` changes along with the `zone_abbr` so a different + period is needed during winter. + """ + @type time_zone_period :: %{ + optional(any) => any, + utc_offset: Calendar.utc_offset(), + std_offset: Calendar.std_offset(), + zone_abbr: Calendar.zone_abbr() + } + + @typedoc """ + Limit for when a certain time zone period begins or ends. + + A beginning is inclusive. An ending is exclusive. For example, if a period is from + `2015-03-29 01:00:00` and until `2015-10-25 01:00:00`, the period includes and + begins from the beginning of `2015-03-29 01:00:00` and lasts until just before + `2015-10-25 01:00:00`. + + A beginning or end for certain periods are infinite, such as the latest + period for time zones without DST or plans to change. However, for the purpose + of this behaviour, they are only used for gaps in wall time where the needed + period limits are at a certain time. + """ + @type time_zone_period_limit :: Calendar.naive_datetime() + + @doc """ + Time zone period for a point in time in UTC for a specific time zone. + + Takes a time zone name and a point in time for UTC and returns a + `time_zone_period` for that point in time. + """ + @doc since: "1.8.0" + @callback time_zone_period_from_utc_iso_days(Calendar.iso_days(), Calendar.time_zone()) :: + {:ok, time_zone_period} + | {:error, :time_zone_not_found | :utc_only_time_zone_database} + + @doc """ + Possible time zone periods for a certain time zone and wall clock date and time. + + When the provided naive datetime is ambiguous, return a tuple with `:ambiguous` + and the two possible periods. The periods in the tuple must be sorted with the + first element being the one that begins first. + + When the provided naive datetime is in a gap, such as during the "spring forward" when going + from winter time to summer time, return a tuple with `:gap` and two periods with limits + in a nested tuple. The first nested two-tuple is the period before the gap and a naive datetime + with a limit for when the period ends (wall time). The second nested two-tuple is the period + just after the gap and a datetime (wall time) for when the period begins just after the gap. + + If there is only a single possible period for the provided `datetime`, then return a tuple + with `:ok` and the `time_zone_period`. + """ + @doc since: "1.8.0" + @callback time_zone_periods_from_wall_datetime(Calendar.naive_datetime(), Calendar.time_zone()) :: + {:ok, time_zone_period} + | {:ambiguous, time_zone_period, time_zone_period} + | {:gap, {time_zone_period, time_zone_period_limit}, + {time_zone_period, time_zone_period_limit}} + | {:error, :time_zone_not_found | :utc_only_time_zone_database} +end + +defmodule Calendar.UTCOnlyTimeZoneDatabase do + @moduledoc """ + Built-in time zone database that works only in the `Etc/UTC` timezone. + + For all other time zones, it returns `{:error, :utc_only_time_zone_database}`. + """ + + @behaviour Calendar.TimeZoneDatabase + + @impl true + def time_zone_period_from_utc_iso_days(_, "Etc/UTC"), + do: {:ok, %{std_offset: 0, utc_offset: 0, zone_abbr: "UTC"}} + + def time_zone_period_from_utc_iso_days(_, _), + do: {:error, :utc_only_time_zone_database} + + @impl true + def time_zone_periods_from_wall_datetime(_, "Etc/UTC"), + do: {:ok, %{std_offset: 0, utc_offset: 0, zone_abbr: "UTC"}} + + def time_zone_periods_from_wall_datetime(_, _), + do: {:error, :utc_only_time_zone_database} +end diff --git a/lib/elixir/lib/code.ex b/lib/elixir/lib/code.ex index 21ec4ac78d6..54c6a601c00 100644 --- a/lib/elixir/lib/code.ex +++ b/lib/elixir/lib/code.ex @@ -1,415 +1,1987 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + defmodule Code do - @moduledoc """ - Utilities for managing code compilation, code evaluation and code loading. + @moduledoc ~S""" + Utilities for managing code compilation, code evaluation, and code loading. + + This module complements Erlang's [`:code` module](`:code`) + to add behavior which is specific to Elixir. For functions to + manipulate Elixir's AST (rather than evaluating it), see the + `Macro` module. + + ## Working with files + + This module contains three functions for compiling and evaluating files. + Here is a summary of them and their behavior: + + * `require_file/2` - compiles a file and tracks its name. It does not + compile the file again if it has been previously required. + + * `compile_file/2` - compiles a file without tracking its name. Compiles the + file multiple times when invoked multiple times. + + * `eval_file/2` - evaluates the file contents without tracking its name. It + returns the result of the last expression in the file, instead of the modules + defined in it. Evaluated files do not trigger the compilation tracers described + in the next section. + + In a nutshell, the first must be used when you want to keep track of the files + handled by the system, to avoid the same file from being compiled multiple + times. This is common in scripts. + + `compile_file/2` must be used when you are interested in the modules defined in a + file, without tracking. `eval_file/2` should be used when you are interested in + the result of evaluating the file rather than the modules it defines. + + The functions above work with Elixir source. If you want to work + with modules compiled to bytecode, which have the `.beam` extension + and are typically found below the _build directory of a Mix project, + see the functions in Erlang's [`:code`](`:code`) module. + + ## Code loading on the Erlang VM + + Erlang has two modes to load code: interactive and embedded. + + By default, the Erlang VM runs in interactive mode, where modules + are loaded as needed. In embedded mode the opposite happens, as all + modules need to be loaded upfront or explicitly. + + You can use `ensure_loaded/1` (as well as `ensure_loaded?/1` and + `ensure_loaded!/1`) to check if a module is loaded before using it and + act. + + ## `ensure_compiled/1` and `ensure_compiled!/1` + + Elixir also includes `ensure_compiled/1` and `ensure_compiled!/1` + functions that are a superset of `ensure_loaded/1`. + + Since Elixir's compilation happens in parallel, in some situations + you may need to use a module that was not yet compiled, therefore + it can't even be loaded. + + When invoked, `ensure_compiled/1` and `ensure_compiled!/1` halt the + compilation of the caller until the module becomes available. Note that + the distinction between `ensure_compiled/1` and `ensure_compiled!/1` + is important: if you are using `ensure_compiled!/1`, you are + indicating to the compiler that you can only continue if said module + is available. + + If you are using `Code.ensure_compiled/1`, you are implying you may + continue without the module and therefore Elixir may return + `{:error, :unavailable}` for cases where the module is not yet available + (but may be available later on). + + For those reasons, developers must typically use `Code.ensure_compiled!/1`. + In particular, do not do this: + + case Code.ensure_compiled(module) do + {:module, _} -> module + {:error, _} -> raise ... + end + + Finally, note you only need `ensure_compiled!/1` to check for modules + being defined within the same project. It does not apply to modules from + dependencies as dependencies are always compiled upfront. + + In most cases, `ensure_loaded/1` is enough. `ensure_compiled!/1` + must be used in rare cases, usually involving macros that need to + invoke a module for callback information. The use of `ensure_compiled/1` + is even less likely. + + ## Compilation tracers + + Elixir supports compilation tracers, which allow modules to observe constructs + handled by the Elixir compiler when compiling files. A tracer is a module + that implements the `trace/2` function. The function receives the event name + as first argument and `Macro.Env` as second and it must return `:ok`. It is + very important for a tracer to do as little work as possible synchronously + and dispatch the bulk of the work to a separate process. **Slow tracers will + slow down compilation**. + + You can configure your list of tracers via `put_compiler_option/2`. The + following events are available to tracers: + + * `:start` - (since v1.11.0) invoked whenever the compiler starts to trace + a new lexical context. A lexical context is started when compiling a new + file or when defining a module within a function. Note evaluated code + does not start a new lexical context (because they don't track unused + aliases, imports, etc) but defining a module inside evaluated code will. + + Note this event may be emitted in parallel, where multiple files/modules + invoke `:start` and run at the same time. The value of the `lexical_tracker` + of the macro environment, albeit opaque, can be used to uniquely identify + the environment. + + * `:stop` - (since v1.11.0) invoked whenever the compiler stops tracing a + new lexical context, such as a new file. + + * `{:import, meta, module, opts}` - traced whenever `module` is imported. + `meta` is the import AST metadata and `opts` are the import options. + + * `{:imported_function, meta, module, name, arity}` and + `{:imported_macro, meta, module, name, arity}` - traced whenever an + imported function or macro is invoked. `meta` is the call AST metadata, + `module` is the module the import is from, followed by the `name` and `arity` + of the imported function/macro. A :remote_function/:remote_macro event + may still be emitted for the imported module/name/arity. + + * `{:imported_quoted, meta, module, name, [arity]}` - traced whenever an + imported function or macro is processed inside a `quote/2`. `meta` is the + call AST metadata, `module` is the module the import is from, followed by + the `name` and a list of `arities` of the imported function/macro. + + * `{:alias, meta, alias, as, opts}` - traced whenever `alias` is aliased + to `as`. `meta` is the alias AST metadata and `opts` are the alias options. + + * `{:alias_expansion, meta, as, alias}` traced whenever there is an alias + expansion for a previously defined `alias`, i.e. when the user writes `as` + which is expanded to `alias`. `meta` is the alias expansion AST metadata. + + * `{:alias_reference, meta, module}` - traced whenever there is an alias + in the code, i.e. whenever the user writes `MyModule.Foo.Bar` in the code, + regardless if it was expanded or not. + + * `{:require, meta, module, opts}` - traced whenever `module` is required. + `meta` is the require AST metadata and `opts` are the require options. + If the `meta` option contains the `:from_macro`, then module was called + from within a macro and therefore must be treated as a compile-time dependency. + + * `{:struct_expansion, meta, module, keys}` - traced whenever `module`'s struct + is expanded. `meta` is the struct AST metadata and `keys` are the keys being + used by expansion + + * `{:remote_function, meta, module, name, arity}` and + `{:remote_macro, meta, module, name, arity}` - traced whenever a remote + function or macro is referenced. `meta` is the call AST metadata, `module` + is the invoked module, followed by the `name` and `arity`. + + * `{:local_function, meta, name, arity}` and + `{:local_macro, meta, name, arity}` - traced whenever a local + function or macro is referenced. `meta` is the call AST metadata, followed by + the `name` and `arity`. + + * `{:compile_env, app, path, return}` - traced whenever `Application.compile_env/3` + or `Application.compile_env!/2` are called. `app` is an atom, `path` is a list + of keys to traverse in the application environment and `return` is either + `{:ok, value}` or `:error`. + + * `:defmodule` - (since v1.16.2) traced as soon as the definition of a module + starts. This is invoked early on in the module life cycle, `Module.open?/1` + still returns `false` for such traces + + * `{:on_module, bytecode, _ignore}` - (since v1.13.0) traced whenever a module + is defined. This is equivalent to the `@after_compile` callback and invoked + after any `@after_compile` in the given module. The third element is currently + `:none` but it may provide more metadata in the future. It is best to ignore + it at the moment. Note that `Module` functions expecting not yet compiled modules + (such as `Module.definitions_in/1`) are still available at the time this event + is emitted. + + The `:tracers` compiler option can be combined with the `:parser_options` + compiler option to enrich the metadata of the traced events above. + + New events may be added at any time in the future, therefore it is advised + for the `trace/2` function to have a "catch-all" clause. + + Below is an example tracer that prints all remote function invocations: + + defmodule MyTracer do + def trace({:remote_function, _meta, module, name, arity}, env) do + IO.puts("#{env.file}:#{env.line} #{inspect(module)}.#{name}/#{arity}") + :ok + end + + def trace(_event, _env) do + :ok + end + end + """ + + @typedoc """ + A list with all variables and their values. + + The binding keys are usually atoms, but they may be a tuple for variables + defined in a different context. + """ + @type binding :: [{atom() | tuple(), any}] + + @typedoc """ + Diagnostics returned by the compiler and code evaluation. + + The file and position relate to where the diagnostic should be shown. + If there is a file and position, then the diagnostic is precise + and you can use the given file and position for generating snippets, + IDEs annotations, and so on. An optional span is available with + the line and column the diagnostic ends. + + Otherwise, a stacktrace may be given, which you can place your own + heuristics to provide better reporting. + + The source field points to the source file the compiler tracked + the error to. For example, a file `lib/foo.ex` may embed `.eex` + templates from `lib/foo/bar.eex`. A syntax error on the EEx template + will point to file `lib/foo/bar.eex` but the source is `lib/foo.ex`. + """ + @type diagnostic(severity) :: %{ + required(:source) => Path.t() | nil, + required(:file) => Path.t() | nil, + required(:severity) => severity, + required(:message) => String.t(), + required(:position) => position(), + required(:stacktrace) => Exception.stacktrace(), + required(:span) => {line :: pos_integer(), column :: pos_integer()} | nil, + optional(:details) => term(), + optional(any()) => any() + } + + @typedoc "The line. 0 indicates no line." + @type line() :: non_neg_integer() + + @typedoc """ + The position of the diagnostic. + + Can be either a line number or a `{line, column}`. + Line and columns numbers are one-based. + A position of `0` represents unknown. + """ + @type position() :: line() | {line :: pos_integer(), column :: pos_integer()} - This module complements [Erlang's code module](http://www.erlang.org/doc/man/code.html) - to add behaviour which is specific to Elixir. + @typedoc """ + Options for code formatting functions. + """ + @type format_opt :: + {:file, binary()} + | {:line, pos_integer()} + | {:line_length, pos_integer()} + | {:locals_without_parens, keyword()} + | {:force_do_end_blocks, boolean()} + | {:migrate, boolean()} + | {:migrate_bitstring_modifiers, boolean()} + | {:migrate_call_parens_on_pipe, boolean()} + | {:migrate_charlists_as_sigils, boolean()} + | {:migrate_unless, boolean()} + | {atom(), term()} + + @typedoc """ + Options for `quoted_to_algebra/2`. + """ + @type quoted_to_algebra_opt :: + {:line, pos_integer() | nil} + | {:escape, boolean()} + | {:locals_without_parens, keyword()} + | {:comments, [term()]} + + @typedoc """ + Options for parsing functions that convert strings to quoted expressions. """ + @type parser_opts :: [ + file: binary(), + line: pos_integer(), + column: pos_integer(), + indentation: non_neg_integer(), + columns: boolean(), + unescape: boolean(), + existing_atoms_only: boolean(), + token_metadata: boolean(), + literal_encoder: (term(), Macro.metadata() -> term()), + static_atoms_encoder: (atom() -> term()), + emit_warnings: boolean() + ] + + @typedoc """ + Options for environment evaluation functions like eval_string/3 and eval_quoted/3. + """ + @type env_eval_opts :: [ + file: binary(), + line: pos_integer(), + module: module(), + prune_binding: boolean() + ] + + @boolean_compiler_options [ + :docs, + :debug_info, + :ignore_already_consolidated, + :ignore_module_conflict, + :relative_paths + ] + + @list_compiler_options [:tracers, :parser_options] + + @available_compiler_options @boolean_compiler_options ++ + @list_compiler_options ++ + [:on_undefined_variable, :infer_signatures, :no_warn_undefined] @doc """ - List all loaded files. + Lists all required files. + + ## Examples + + Code.require_file("../eex/test/eex_test.exs") + List.first(Code.required_files()) =~ "eex_test.exs" + #=> true + """ + @doc since: "1.7.0" + @spec required_files() :: [binary] + def required_files do + :elixir_code_server.call(:required) + end + + @deprecated "Use Code.required_files/0 instead" + @doc false def loaded_files do - :elixir_code_server.call :loaded + required_files() + end + + @doc false + @deprecated "Use Code.Fragment.cursor_context/2 instead" + def cursor_context(code, options \\ []) do + Code.Fragment.cursor_context(code, options) end @doc """ - Remove files from the loaded files list. + Removes files from the required files list. The modules defined in the file are not removed; calling this function only removes them from the list, allowing them to be required again. + + The list of files is managed per Erlang VM node. + + ## Examples + + # Require EEx test code + Code.require_file("../eex/test/eex_test.exs") + + # Now unrequire all files + Code.unrequire_files(Code.required_files()) + + # Note that modules are still available + function_exported?(EExTest.Compiled, :before_compile, 0) + #=> true + """ + @doc since: "1.7.0" + @spec unrequire_files([binary]) :: :ok + def unrequire_files(files) when is_list(files) do + :elixir_code_server.cast({:unrequire_files, files}) + end + + @deprecated "Use Code.unrequire_files/1 instead" + @doc false def unload_files(files) do - :elixir_code_server.cast {:unload_files, files} + unrequire_files(files) + end + + @doc """ + Appends a path to the Erlang VM code path list. + + This is the list of directories the Erlang VM uses for + finding module code. The list of files is managed per Erlang + VM node. + + The path is expanded with `Path.expand/1` before being appended. + It requires the path to exist. Returns a boolean indicating if + the path was successfully added. + + ## Examples + + Code.append_path(".") + #=> true + + Code.append_path("/does_not_exist") + #=> false + + ## Options + + * `:cache` - (since v1.15.0) when true, the code path is cached + the first time it is traversed in order to reduce file system + operations. + + """ + @spec append_path(Path.t(), cache: boolean()) :: true | false + def append_path(path, opts \\ []) do + apply(:code, :add_pathz, [to_charlist(Path.expand(path)) | cache(opts)]) == true + end + + @doc """ + Prepends a path to the Erlang VM code path list. + + This is the list of directories the Erlang VM uses for + finding module code. The list of files is managed per Erlang + VM node. + + The path is expanded with `Path.expand/1` before being prepended. + It requires the path to exist. Returns a boolean indicating if + the path was successfully added. + + ## Examples + + Code.prepend_path(".") + #=> true + + Code.prepend_path("/does_not_exist") + #=> false + + ## Options + + * `:cache` - (since v1.15.0) when true, the code path is cached + the first time it is traversed in order to reduce file system + operations. + + """ + @spec prepend_path(Path.t(), cache: boolean()) :: boolean() + def prepend_path(path, opts \\ []) do + apply(:code, :add_patha, [to_charlist(Path.expand(path)) | cache(opts)]) == true + end + + @doc """ + Prepends a list of `paths` to the Erlang VM code path list. + + This is the list of directories the Erlang VM uses for + finding module code. The list of files is managed per Erlang + VM node. + + All paths are expanded with `Path.expand/1` before being prepended. + Only existing paths are prepended. This function always returns `:ok`, + regardless of how many paths were prepended. Use `prepend_path/1` + if you need more control. + + ## Examples + + Code.prepend_paths([".", "/does_not_exist"]) + #=> :ok + + ## Options + + * `:cache` - when true, the code path is cached the first time + it is traversed in order to reduce file system operations. + + """ + @doc since: "1.15.0" + @spec prepend_paths([Path.t()], cache: boolean()) :: :ok + def prepend_paths(paths, opts \\ []) when is_list(paths) do + apply(:code, :add_pathsa, [Enum.map(paths, &to_charlist(Path.expand(&1))) | cache(opts)]) + end + + @doc """ + Appends a list of `paths` to the Erlang VM code path list. + + This is the list of directories the Erlang VM uses for + finding module code. The list of files is managed per Erlang + VM node. + + All paths are expanded with `Path.expand/1` before being appended. + Only existing paths are appended. This function always returns `:ok`, + regardless of how many paths were appended. Use `append_path/1` + if you need more control. + + ## Examples + + Code.append_paths([".", "/does_not_exist"]) + #=> :ok + + ## Options + + * `:cache` - when true, the code path is cached the first time + it is traversed in order to reduce file system operations. + + """ + @doc since: "1.15.0" + @spec append_paths([Path.t()], cache: boolean()) :: :ok + def append_paths(paths, opts \\ []) when is_list(paths) do + apply(:code, :add_pathsz, [Enum.map(paths, &to_charlist(Path.expand(&1))) | cache(opts)]) + end + + defp cache(opts) do + if function_exported?(:code, :add_path, 2) do + if opts[:cache], do: [:cache], else: [:nocache] + else + [] + end + end + + @doc """ + Deletes a path from the Erlang VM code path list. + + This is the list of directories the Erlang VM uses for finding + module code. The list of files is managed per Erlang VM node. + + The path is expanded with `Path.expand/1` before being deleted. If the + path does not exist, this function returns `false`. + + ## Examples + + Code.prepend_path(".") + Code.delete_path(".") + #=> true + + Code.delete_path("/does_not_exist") + #=> false + + """ + @spec delete_path(Path.t()) :: boolean + def delete_path(path) do + case :code.del_path(to_charlist(Path.expand(path))) do + result when is_boolean(result) -> + result + + {:error, :bad_name} -> + raise ArgumentError, + "invalid argument #{inspect(path)}" + end + end + + @doc """ + Deletes a list of paths from the Erlang VM code path list. + + This is the list of directories the Erlang VM uses for finding + module code. The list of files is managed per Erlang VM node. + + The path is expanded with `Path.expand/1` before being deleted. If the + path does not exist, this function returns `false`. + """ + @doc since: "1.15.0" + @spec delete_paths([Path.t()]) :: :ok + def delete_paths(paths) when is_list(paths) do + for path <- paths do + _ = :code.del_path(to_charlist(Path.expand(path))) + end + + :ok + end + + @doc """ + Evaluates the contents given by `string`. + + The `binding` argument is a list of all variables and their values. + The `opts` argument is a keyword list of environment options. + + **Warning**: `string` can be any Elixir code and will be executed with + the same privileges as the Erlang VM: this means that such code could + compromise the machine (for example by executing system commands). + Don't use `eval_string/3` with untrusted input (such as strings coming + from the network). + + ## Options + + It accepts the same options as `env_for_eval/1`. Additionally, you may + also pass an environment as second argument, so the evaluation happens + within that environment. + + Returns a tuple of the form `{value, binding}`, where `value` is the value + returned from evaluating `string`. If an error occurs while evaluating + `string`, an exception will be raised. + + `binding` is a list with all variable names and their values after evaluating + `string`. The binding keys are usually atoms, but they may be a tuple for variables + defined in a different context. The names are in no particular order. + + ## Examples + + iex> {result, binding} = Code.eval_string("a + b", [a: 1, b: 2], file: __ENV__.file, line: __ENV__.line) + iex> result + 3 + iex> Enum.sort(binding) + [a: 1, b: 2] + + iex> {result, binding} = Code.eval_string("c = a + b", [a: 1, b: 2], __ENV__) + iex> result + 3 + iex> Enum.sort(binding) + [a: 1, b: 2, c: 3] + + iex> {result, binding} = Code.eval_string("a = a + b", [a: 1, b: 2]) + iex> result + 3 + iex> Enum.sort(binding) + [a: 3, b: 2] + + For convenience, you can pass `__ENV__/0` as the `opts` argument and + all imports, requires and aliases defined in the current environment + will be automatically carried over: + + iex> require Integer, warn: false + iex> {result, binding} = Code.eval_string("if Integer.is_odd(a), do: a + b", [a: 1, b: 2], __ENV__) + iex> result + 3 + iex> Enum.sort(binding) + [a: 1, b: 2] + + """ + @spec eval_string(List.Chars.t(), binding, Macro.Env.t() | env_eval_opts) :: {term, binding} + def eval_string(string, binding \\ [], opts \\ []) + + def eval_string(string, binding, %Macro.Env{} = env) do + validated_eval_string(string, binding, env) + end + + def eval_string(string, binding, opts) when is_list(opts) do + validated_eval_string(string, binding, opts) + end + + defp validated_eval_string(string, binding, opts_or_env) do + %{line: line, file: file} = env = env_for_eval(opts_or_env) + forms = :elixir.string_to_quoted!(to_charlist(string), line, 1, file, []) + {value, binding, _env} = eval_verify(:eval_forms, [forms, binding, env]) + {value, binding} + end + + defp eval_verify(fun, args) do + Module.ParallelChecker.verify(fn -> + apply(:elixir, fun, args) + end) + end + + @doc """ + Executes the given `fun` and capture all diagnostics. + + Diagnostics are warnings and errors emitted during code + evaluation or single-file compilation and by functions + such as `IO.warn/2`. + + If using `mix compile` or `Kernel.ParallelCompiler`, + note they already capture and return diagnostics. + + ## Options + + * `:log` - if the diagnostics should be logged as they happen. + Defaults to `false`. + + > #### Rescuing errors {: .info} + > + > `with_diagnostics/2` does not automatically handle exceptions. + > You may capture them by adding a `try/1` in `fun`: + > + > {result, all_errors_and_warnings} = + > Code.with_diagnostics(fn -> + > try do + > {:ok, Code.compile_quoted(quoted)} + > rescue + > err -> {:error, err} + > end + > end) + + """ + @doc since: "1.15.0" + @spec with_diagnostics([log: boolean()], (-> result)) :: + {result, [diagnostic(:warning | :error)]} + when result: term() + def with_diagnostics(opts \\ [], fun) do + value = :erlang.get(:elixir_code_diagnostics) + log = Keyword.get(opts, :log, false) + :erlang.put(:elixir_code_diagnostics, {[], log}) + + try do + result = fun.() + {diagnostics, _log?} = :erlang.get(:elixir_code_diagnostics) + {result, Enum.reverse(diagnostics)} + after + if value == :undefined do + :erlang.erase(:elixir_code_diagnostics) + else + :erlang.put(:elixir_code_diagnostics, value) + end + end + end + + @doc """ + Prints a diagnostic into the standard error. + + A diagnostic is either returned by `Kernel.ParallelCompiler` + or by `Code.with_diagnostics/2`. + + ## Options + + * `:snippet` - whether to read the code snippet in the diagnostic location. + As it may impact performance, it is not recommended to be used in runtime. + Defaults to `true`. + """ + @doc since: "1.15.0" + @spec print_diagnostic(diagnostic(:warning | :error), snippet: boolean()) :: :ok + def print_diagnostic(diagnostic, opts \\ []) do + read_snippet? = Keyword.get(opts, :snippet, true) + :elixir_errors.print_diagnostic(diagnostic, read_snippet?) + :ok + end + + @doc ~S""" + Formats the given code `string`. + + The formatter receives a string representing Elixir code and + returns iodata representing the formatted code according to + pre-defined rules. + + ## Options + + Regular options (do not change the AST): + + * `:file` - the file which contains the string, used for error + reporting + + * `:line` - the line the string starts, used for error reporting + + * `:line_length` - the line length to aim for when formatting + the document. Defaults to `98`. This value indicates when an expression + should be broken over multiple lines but it is not guaranteed + to do so. See the "Line length" section below for more information + + * `:locals_without_parens` - a keyword list of name and arity + pairs that should be kept without parens whenever possible. + The arity may be the atom `:*`, which implies all arities of + that name. The formatter already includes a list of functions + and this option augments this list. + + * `:force_do_end_blocks` (since v1.9.0) - when `true`, converts all + inline usages of `do: ...`, `else: ...` and friends into `do`-`end` + blocks. Defaults to `false`. Note that this option is convergent: + once you set it to `true`, **all keywords** will be converted. + If you set it to `false` later on, `do`-`end` blocks won't be + converted back to keywords. + + Migration options (change the AST), see the "Migration formatting" section below: + + * `:migrate` (since v1.18.0) - when `true`, sets all other migration options + to `true` by default. Defaults to `false`. + + * `:migrate_bitstring_modifiers` (since v1.18.0) - when `true`, + removes unnecessary parentheses in known bitstring + [modifiers](`<<>>/1`), for example `<>` + becomes `<>`, or adds parentheses for custom + modifiers, where `<>` becomes `<>`. + Defaults to the value of the `:migrate` option. This option changes the AST. + + * `:migrate_call_parens_on_pipe` (since v1.19.0) - when `true`, + formats calls on the right-hand side of the pipe operator to always include + parentheses, for example `foo |> bar` becomes `foo |> bar()` and + `foo |> mod.fun` becomes `foo |> mod.fun()`. + Parentheses are always added for qualified calls like `foo |> Bar.bar` even + when this option is `false`. + Defaults to the value of the `:migrate` option. This option changes the AST. + + * `:migrate_charlists_as_sigils` (since v1.18.0) - when `true`, + formats charlists as [`~c`](`Kernel.sigil_c/2`) sigils, for example + `'foo'` becomes `~c"foo"`. + Defaults to the value of the `:migrate` option. This option changes the AST. + + * `:migrate_unless` (since v1.18.0) - when `true`, + rewrites `unless` expressions using `if` with a negated condition, for example + `unless foo, do:` becomes `if !foo, do:`. + Defaults to the value of the `:migrate` option. This option changes the AST. + + ## Design principles + + The formatter was designed under three principles. + + First, the formatter never changes the semantics of the code by default. + This means the input AST and the output AST are almost always equivalent. + + The second principle is to provide as little configuration as possible. + This eases the formatter adoption by removing contention points while + making sure a single style is followed consistently by the community as + a whole. + + The formatter does not hard code names. The formatter will not behave + specially because a function is named `defmodule`, `def`, or the like. This + principle mirrors Elixir's goal of being an extensible language where + developers can extend the language with new constructs as if they were + part of the language. When it is absolutely necessary to change behavior + based on the name, this behavior should be configurable, such as the + `:locals_without_parens` option. + + ## Running the formatter + + The formatter attempts to fit the most it can on a single line and + introduces line breaks wherever possible when it cannot. + + In some cases, this may lead to undesired formatting. Therefore, **some + code generated by the formatter may not be aesthetically pleasing and + may require explicit intervention from the developer**. That's why we + do not recommend to run the formatter blindly in an existing codebase. + Instead you should format and sanity check each formatted file. + + For example, the formatter may break a long function definition over + multiple clauses: + + def my_function( + %User{name: name, age: age, ...}, + arg1, + arg2 + ) do + ... + end + + While the code above is completely valid, you may prefer to match on + the struct variables inside the function body in order to keep the + definition on a single line: + + def my_function(%User{} = user, arg1, arg2) do + %{name: name, age: age, ...} = user + ... + end + + In some situations, you can use the fact the formatter does not generate + elegant code as a hint for refactoring. Take this code: + + def board?(board_id, %User{} = user, available_permissions, required_permissions) do + Tracker.OrganizationMembers.user_in_organization?(user.id, board.organization_id) and + required_permissions == Enum.to_list(MapSet.intersection(MapSet.new(required_permissions), MapSet.new(available_permissions))) + end + + The code above has very long lines and running the formatter is not going + to address this issue. In fact, the formatter may make it more obvious that + you have complex expressions: + + def board?(board_id, %User{} = user, available_permissions, required_permissions) do + Tracker.OrganizationMembers.user_in_organization?(user.id, board.organization_id) and + required_permissions == + Enum.to_list( + MapSet.intersection( + MapSet.new(required_permissions), + MapSet.new(available_permissions) + ) + ) + end + + Take such cases as a suggestion that your code should be refactored: + + def board?(board_id, %User{} = user, available_permissions, required_permissions) do + Tracker.OrganizationMembers.user_in_organization?(user.id, board.organization_id) and + matching_permissions?(required_permissions, available_permissions) + end + + defp matching_permissions?(required_permissions, available_permissions) do + intersection = + required_permissions + |> MapSet.new() + |> MapSet.intersection(MapSet.new(available_permissions)) + |> Enum.to_list() + + required_permissions == intersection + end + + To sum it up: since the formatter cannot change the semantics of your + code, sometimes it is necessary to tweak or refactor the code to get + optimal formatting. To help better understand how to control the formatter, + we describe in the next sections the cases where the formatter keeps the + user encoding and how to control multiline expressions. + + ## Line length + + Another point about the formatter is that the `:line_length` configuration + indicates when an expression should be broken over multiple lines but it is + not guaranteed to do so. In many cases, it is not possible for the formatter + to break your code apart, which means it will go over the line length. + For example, if you have a long string: + + "this is a very long string that will go over the line length" + + The formatter doesn't know how to break it apart without changing the + code underlying syntax representation, so it is up to you to step in: + + "this is a very long string " <> + "that will go over the line length" + + The string concatenation makes the code fit on a single line and also + gives more options to the formatter. + + This may also appear in keywords such as do/end blocks and operators, + where the `do` keyword may go over the line length because there is no + opportunity for the formatter to introduce a line break in a readable way. + For example, if you do: + + case very_long_expression() do + end + + And only the `do` keyword is beyond the line length, Elixir **will not** + emit this: + + case very_long_expression() + do + end + + So it prefers to not touch the line at all and leave `do` above the + line limit. + + ## Keeping user's formatting + + The formatter respects the input format in some cases. Those are + listed below: + + * Insignificant digits in numbers are kept as is. The formatter, + however, always inserts underscores for decimal numbers with more + than 5 digits and converts hexadecimal digits to uppercase + + * Strings, charlists, atoms and sigils are kept as is. No character + is automatically escaped or unescaped. The choice of delimiter is + also respected from the input + + * Newlines inside blocks are kept as in the input except for: + 1) expressions that take multiple lines will always have an empty + line before and after and 2) empty lines are always squeezed + together into a single empty line + + * The choice between `:do` keyword and `do`-`end` blocks is left + to the user + + * Lists, tuples, bitstrings, maps, structs and function calls will be + broken into multiple lines if they are followed by a newline in the + opening bracket and preceded by a new line in the closing bracket + + * Newlines before certain operators (such as the pipeline operators) + and before other operators (such as comparison operators) + + The behaviors above are not guaranteed. We may remove or add new + rules in the future. The goal of documenting them is to provide better + understanding on what to expect from the formatter. + + ### Multi-line lists, maps, tuples, and the like + + You can force lists, tuples, bitstrings, maps, structs and function + calls to have one entry per line by adding a newline after the opening + bracket and a new line before the closing bracket lines. For example: + + [ + foo, + bar + ] + + If there are no newlines around the brackets, then the formatter will + try to fit everything on a single line, such that the snippet below + + [foo, + bar] + + will be formatted as + + [foo, bar] + + You can also force function calls and keywords to be rendered on multiple + lines by having each entry on its own line: + + defstruct name: nil, + age: 0 + + The code above will be kept with one keyword entry per line by the + formatter. To avoid that, just squash everything into a single line. + + ### Parens and no parens in function calls + + Elixir has two syntaxes for function calls. With parens and no parens. + By default, Elixir will add parens to all calls except for: + + 1. calls that have `do`-`end` blocks + 2. local calls without parens where the name and arity of the local + call is also listed under `:locals_without_parens` (except for + calls with arity 0, where the compiler always require parens) + + The choice of parens and no parens also affects indentation. When a + function call with parens doesn't fit on the same line, the formatter + introduces a newline around parens and indents the arguments with two + spaces: + + some_call( + arg1, + arg2, + arg3 + ) + + On the other hand, function calls without parens are always indented + by the function call length itself, like this: + + some_call arg1, + arg2, + arg3 + + If the last argument is a data structure, such as maps and lists, and + the beginning of the data structure fits on the same line as the function + call, then no indentation happens, this allows code like this: + + Enum.reduce(some_collection, initial_value, fn element, acc -> + # code + end) + + some_function_without_parens %{ + foo: :bar, + baz: :bat + } + + ## Code comments + + The formatter handles code comments and guarantees a space is always added + between the beginning of the comment (#) and the next character. + + The formatter also extracts all trailing comments to their previous line. + For example, the code below + + hello #world + + will be rewritten to + + # world + hello + + While the formatter attempts to preserve comments in most situations, + that's not always possible, because code comments are handled apart from + the code representation (AST). While the formatter can preserve code + comments between expressions and function arguments, the formatter + cannot currently preserve them around operators. For example, the following + code: + + foo() || + # also check for bar + bar() + + will move the code comments to before the operator usage: + + # also check for bar + foo() || + bar() + + In some situations, code comments can be seen as ambiguous by the formatter. + For example, the comment in the anonymous function below + + fn + arg1 -> + body1 + # comment + + arg2 -> + body2 + end + + and in this one + + fn + arg1 -> + body1 + + # comment + arg2 -> + body2 + end + + are considered equivalent (the nesting is discarded alongside most of + user formatting). In such cases, the code formatter will always format to + the latter. + + ## Newlines + + The formatter converts all newlines in code from `\r\n` to `\n`. + + ## Migration formatting + + As part of the Elixir release cycle, deprecations are being introduced, + emitting warnings which might require existing code to be changed. + In order to reduce the burden on developers when upgrading Elixir to the + next version, the formatter exposes some options, disabled by default, + in order to automate this process. + + These options should address most of the typical use cases, but given they + introduce changes to the AST, there is a non-zero risk for meta-programming + heavy projects that relied on a specific AST, or projects that are + re-defining functions from the `Kernel`. In such cases, migrations cannot + be applied blindly and some extra changes might be needed in order to + address the deprecation warnings. + """ + @doc since: "1.6.0" + @spec format_string!(binary, [format_opt]) :: iodata + def format_string!(string, opts \\ []) when is_binary(string) and is_list(opts) do + {line_length, opts} = Keyword.pop(opts, :line_length, 98) + + to_quoted_opts = + [ + unescape: false, + literal_encoder: &{:ok, {:__block__, &2, [&1]}}, + token_metadata: true, + emit_warnings: false + ] ++ opts + + {forms, comments} = string_to_quoted_with_comments!(string, to_quoted_opts) + to_algebra_opts = [comments: comments] ++ opts + doc = Code.Formatter.to_algebra(forms, to_algebra_opts) + Inspect.Algebra.format(doc, line_length) + end + + @doc """ + Formats a file. + + See `format_string!/2` for more information on code formatting and + available options. + """ + @doc since: "1.6.0" + @spec format_file!(binary, [format_opt]) :: iodata + def format_file!(file, opts \\ []) when is_binary(file) and is_list(opts) do + string = File.read!(file) + formatted = format_string!(string, [file: file, line: 1] ++ opts) + [formatted, ?\n] + end + + @doc """ + Evaluates the quoted contents. + + **Warning**: Calling this function inside a macro is considered bad + practice as it will attempt to evaluate runtime values at compile time. + Macro arguments are typically transformed by unquoting them into the + returned quoted expressions (instead of evaluated). + + See `eval_string/3` for a description of arguments and return types. + The options are described under `env_for_eval/1`. + + ## Examples + + iex> contents = quote(do: var!(a) + var!(b)) + iex> {result, binding} = Code.eval_quoted(contents, [a: 1, b: 2], file: __ENV__.file, line: __ENV__.line) + iex> result + 3 + iex> Enum.sort(binding) + [a: 1, b: 2] + + For convenience, you can pass `__ENV__/0` as the `opts` argument and + all options will be automatically extracted from the current environment: + + iex> contents = quote(do: var!(a) + var!(b)) + iex> {result, binding} = Code.eval_quoted(contents, [a: 1, b: 2], __ENV__) + iex> result + 3 + iex> Enum.sort(binding) + [a: 1, b: 2] + + """ + @spec eval_quoted(Macro.t(), binding, Macro.Env.t() | env_eval_opts) :: {term, binding} + def eval_quoted(quoted, binding \\ [], env_or_opts \\ []) do + {value, binding, _env} = + eval_verify(:eval_quoted, [quoted, binding, env_for_eval(env_or_opts)]) + + {value, binding} + end + + @doc """ + Returns an environment for evaluation. + + It accepts either a `Macro.Env`, that is then pruned and prepared, + or a list of options. It returns an environment that is ready for + evaluation. + + Most functions in this module will automatically prepare the given + environment for evaluation, so you don't need to explicitly call + this function, with the exception of `eval_quoted_with_env/3`, + which was designed precisely to be called in a loop, to implement + features such as interactive shells or anything else with multiple + evaluations. + + ## Options + + If an env is not given, the options can be: + + * `:file` - the file to be considered in the evaluation + + * `:line` - the line on which the script starts + + * `:module` - the module to run the environment on + + * `:prune_binding` - (since v1.14.2) prune binding to keep only + variables read or written by the evaluated code. Note that + variables used by modules are always pruned, even if later used + by the modules. You can submit to the `:on_module` tracer event + and access the variables used by the module from its environment. + """ + @doc since: "1.14.0" + @spec env_for_eval(Macro.Env.t() | env_eval_opts) :: Macro.Env.t() + def env_for_eval(env_or_opts), do: :elixir.env_for_eval(env_or_opts) + + @doc """ + Evaluates the given `quoted` contents with `binding` and `env`. + + This function is meant to be called in a loop, to implement features + such as interactive shells or anything else with multiple evaluations. + Therefore, the first time you call this function, you must compute + the initial environment with `env_for_eval/1`. The remaining calls + must pass the environment that was returned by this function. + + ## Options + + It accepts the same options as `env_for_eval/1`. + + """ + @doc since: "1.14.0" + @spec eval_quoted_with_env(Macro.t(), binding, Macro.Env.t(), env_eval_opts) :: + {term, binding, Macro.Env.t()} + def eval_quoted_with_env(quoted, binding, %Macro.Env{} = env, opts \\ []) + when is_list(binding) do + eval_verify(:eval_quoted, [quoted, binding, env, opts]) + end + + @doc ~S""" + Converts the given string to its quoted form. + + Returns `{:ok, quoted_form}` if it succeeds, + `{:error, {meta, message_info, token}}` otherwise. + + ## Options + + * `:file` - the filename to be reported in case of parsing errors. + Defaults to `"nofile"`. + + * `:line` - the starting line of the string being parsed. + Defaults to `1`. + + * `:column` - (since v1.11.0) the starting column of the string being parsed. + Defaults to `1`. + + * `:indentation` - (since v1.19.0) the indentation for the string being parsed. + This is useful when the code parsed is embedded within another document. + Defaults to `0`. + + * `:columns` - when `true`, attach a `:column` key to the quoted + metadata. Defaults to `false`. + + * `:unescape` (since v1.10.0) - when `false`, preserves escaped sequences. + For example, `"null byte\\t\\x00"` will be kept as is instead of being + converted to a bitstring literal. Note if you set this option to false, the + resulting AST is no longer valid, but it can be useful to analyze/transform + source code, typically in combination with `quoted_to_algebra/2`. + Defaults to `true`. + + * `:existing_atoms_only` - when `true`, raises an error + when non-existing atoms are found by the tokenizer. + Defaults to `false`. + + * `:token_metadata` (since v1.10.0) - when `true`, includes token-related + metadata in the expression AST, such as metadata for `do` and `end` + tokens, for closing tokens, end of expressions, as well as delimiters + for sigils. See `t:Macro.metadata/0`. Defaults to `false`. + + * `:literal_encoder` (since v1.10.0) - how to encode literals in the AST. + It must be a function that receives two arguments, the literal and its + metadata, and it must return `{:ok, ast :: Macro.t}` or + `{:error, reason :: binary}`. If you return anything than the literal + itself as the `term`, then the AST is no longer valid. This option + may still useful for textual analysis of the source code. + + * `:static_atoms_encoder` - the static atom encoder function, see + "The `:static_atoms_encoder` function" section below. Note this + option overrides the `:existing_atoms_only` behavior for static + atoms but `:existing_atoms_only` is still used for dynamic atoms, + such as atoms with interpolations. + + * `:emit_warnings` (since v1.16.0) - when `false`, does not emit + tokenizing/parsing related warnings. Defaults to `true`. + + ## `Macro.to_string/2` + + The opposite of converting a string to its quoted form is + `Macro.to_string/2`, which converts a quoted form to a string/binary + representation. + + ## The `:static_atoms_encoder` function + + When `static_atoms_encoder: &my_encoder/2` is passed as an argument, + `my_encoder/2` is called every time the tokenizer needs to create a + "static" atom. Static atoms are atoms in the AST that function as + aliases, remote calls, local calls, variable names, regular atoms + and keyword lists. + + The encoder function will receive the atom name (as a binary) and a + keyword list with the current file, line and column. It must return + `{:ok, token :: term} | {:error, reason :: binary}`. + + The encoder function is supposed to create an atom from the given + string. To produce a valid AST, it is required to return `{:ok, term}`, + where `term` is an atom. It is possible to return something other than an atom, + however, in that case the AST is no longer "valid" in that it cannot + be used to compile or evaluate Elixir code. A use case for this is + if you want to use the Elixir parser in a user-facing situation, but + you don't want to exhaust the atom table. + + The atom encoder is not called for *all* atoms that are present in + the AST. It won't be invoked for the following atoms: + + * operators (`:+`, `:-`, and so on) + + * syntax keywords (`fn`, `do`, `else`, and so on) + + * atoms containing interpolation (`:"#{1 + 1} is two"`), as these + atoms are constructed at runtime + + * atoms used to represent single-letter sigils like `:sigil_X` + (but multi-letter sigils like `:sigil_XYZ` are encoded). + + ## Examples + + iex> Code.string_to_quoted("1 + 3") + {:ok, {:+, [line: 1], [1, 3]}} + + iex> Code.string_to_quoted("1 \ 3") + {:error, {[line: 1, column: 4], "syntax error before: ", "\"3\""}} + + """ + @spec string_to_quoted(List.Chars.t(), parser_opts) :: + {:ok, Macro.t()} | {:error, {location :: keyword, binary | {binary, binary}, binary}} + def string_to_quoted(string, opts \\ []) when is_list(opts) do + file = Keyword.get(opts, :file, "nofile") + line = Keyword.get(opts, :line, 1) + column = Keyword.get(opts, :column, 1) + + case :elixir.string_to_tokens(to_charlist(string), line, column, file, opts) do + {:ok, tokens} -> + :elixir.tokens_to_quoted(tokens, file, opts) + + {:error, _error_msg} = error -> + error + end end @doc """ - Append a path to the Erlang VM code path. + Converts the given string to its quoted form. - The path is expanded with `Path.expand/1` before being appended. + It returns the AST if it succeeds, + raises an exception otherwise. The exception is a `TokenMissingError` + in case a token is missing (usually because the expression is incomplete), + `MismatchedDelimiterError` (in case of mismatched opening and closing delimiters) and + `SyntaxError` otherwise. + + Check `string_to_quoted/2` for options information. """ - def append_path(path) do - :code.add_pathz(to_char_list(Path.expand path)) + @spec string_to_quoted!(List.Chars.t(), parser_opts) :: Macro.t() + def string_to_quoted!(string, opts \\ []) when is_list(opts) do + file = Keyword.get(opts, :file, "nofile") + line = Keyword.get(opts, :line, 1) + column = Keyword.get(opts, :column, 1) + :elixir.string_to_quoted!(to_charlist(string), line, column, file, opts) end @doc """ - Prepend a path to the Erlang VM code path. + Converts the given string to its quoted form and a list of comments. + + This function is useful when performing textual changes to the source code, + while preserving information like comments and literals position. + + Returns `{:ok, quoted_form, comments}` if it succeeds, + `{:error, {line, error, token}}` otherwise. + + Comments are maps with the following fields: + + * `:line` - The line number of the source code + + * `:text` - The full text of the comment, including the leading `#` + + * `:previous_eol_count` - How many end of lines there are between the comment and the previous AST node or comment + + * `:next_eol_count` - How many end of lines there are between the comment and the next AST node or comment + + Check `string_to_quoted/2` for options information. + + ## Examples + + iex> Code.string_to_quoted_with_comments("\"" + ...> :foo + ...> + ...> # Hello, world! + ...> + ...> + ...> # Some more comments! + ...> "\"") + {:ok, :foo, [ + %{line: 3, column: 1, previous_eol_count: 2, next_eol_count: 3, text: "\# Hello, world!"}, + %{line: 6, column: 1, previous_eol_count: 3, next_eol_count: 1, text: "\# Some more comments!"}, + ]} + + iex> Code.string_to_quoted_with_comments(":foo # :bar") + {:ok, :foo, [ + %{line: 1, column: 6, previous_eol_count: 0, next_eol_count: 0, text: "\# :bar"} + ]} - The path is expanded with `Path.expand/1` before being prepended. """ - def prepend_path(path) do - :code.add_patha(to_char_list(Path.expand path)) + @doc since: "1.13.0" + @spec string_to_quoted_with_comments(List.Chars.t(), parser_opts) :: + {:ok, Macro.t(), list(map())} | {:error, {location :: keyword, term, term}} + def string_to_quoted_with_comments(string, opts \\ []) when is_list(opts) do + charlist = to_charlist(string) + file = Keyword.get(opts, :file, "nofile") + line = Keyword.get(opts, :line, 1) + column = Keyword.get(opts, :column, 1) + + Process.put(:code_formatter_comments, []) + opts = [preserve_comments: &preserve_comments/5] ++ opts + + with {:ok, tokens} <- :elixir.string_to_tokens(charlist, line, column, file, opts), + {:ok, forms} <- :elixir.tokens_to_quoted(tokens, file, opts) do + comments = Enum.reverse(Process.get(:code_formatter_comments)) + {:ok, forms, comments} + end + after + Process.delete(:code_formatter_comments) end @doc """ - Delete a path from the Erlang VM code path. + Converts the given string to its quoted form and a list of comments. + + Returns the AST and a list of comments if it succeeds, raises an exception + otherwise. The exception is a `TokenMissingError` in case a token is missing + (usually because the expression is incomplete), `SyntaxError` otherwise. - The path is expanded with `Path.expand/1` before being deleted. + Check `string_to_quoted/2` for options information. """ - def delete_path(path) do - :code.del_path(to_char_list(Path.expand path)) + @doc since: "1.13.0" + @spec string_to_quoted_with_comments!(List.Chars.t(), parser_opts) :: {Macro.t(), list(map())} + def string_to_quoted_with_comments!(string, opts \\ []) do + charlist = to_charlist(string) + + case string_to_quoted_with_comments(charlist, opts) do + {:ok, forms, comments} -> + {forms, comments} + + {:error, {location, error, token}} -> + file = Keyword.get(opts, :file, "nofile") + line = Keyword.get(opts, :line, 1) + column = Keyword.get(opts, :column, 1) + input = {charlist, line, column, Keyword.get(opts, :indentation, 0)} + :elixir_errors.parse_error(location, file, error, token, input) + end end - @doc """ - Evaluate the contents given by `string`. + defp preserve_comments(line, column, tokens, comment, rest) do + comments = Process.get(:code_formatter_comments) - The `binding` argument is a keyword list of variable bindings. - The `opts` argument is a keyword list of environment options. + comment = %{ + line: line, + column: column, + previous_eol_count: min(previous_eol_count(tokens), last_comment_distance(comments, line)), + next_eol_count: next_eol_count(rest, 0), + text: List.to_string(comment) + } - Those options can be: + Process.put(:code_formatter_comments, [comment | comments]) + end - * `:file` - the file to be considered in the evaluation - * `:line` - the line on which the script starts - * `:delegate_locals_to` - delegate local calls to the given module, - the default is to not delegate + defp next_eol_count([?\s | rest], count), do: next_eol_count(rest, count) + defp next_eol_count([?\t | rest], count), do: next_eol_count(rest, count) + defp next_eol_count([?\n | rest], count), do: next_eol_count(rest, count + 1) + defp next_eol_count([?\r, ?\n | rest], count), do: next_eol_count(rest, count + 1) + defp next_eol_count(_, count), do: count - Additionally, the following scope values can be configured: + defp last_comment_distance([%{line: last_line} | _], line), do: line - last_line + defp last_comment_distance([], _line), do: :infinity - * `:aliases` - a list of tuples with the alias and its target + defp previous_eol_count([{token, {_, _, count}} | _]) + when token in [:eol, :",", :";"] and count > 0 do + count + end - * `:requires` - a list of modules required + defp previous_eol_count([]), do: 1 + defp previous_eol_count(_), do: 0 - * `:functions` - a list of tuples where the first element is a module - and the second a list of imported function names and arity; the list - of function names and arity must be sorted + @doc ~S""" + Converts a quoted expression to an algebra document using Elixir's formatter rules. - * `:macros` - a list of tuples where the first element is a module - and the second a list of imported macro names and arity; the list - of function names and arity must be sorted + The algebra document can be converted into a string by calling: - Notice that setting any of the values above overrides Elixir's default - values. For example, setting `:requires` to `[]`, will no longer - automatically require the `Kernel` module; in the same way setting - `:macros` will no longer auto-import `Kernel` macros like `if`, `case`, - etc. + doc + |> Inspect.Algebra.format(:infinity) + |> IO.iodata_to_binary() - Returns a tuple of the form `{value, binding}`, - where `value` is the value returned from evaluating `string`. - If an error occurs while evaluating `string` an exception will be raised. + For a high-level function that does the same, see `Macro.to_string/1`. - `binding` is a keyword list with the value of all variable bindings - after evaluating `string`. The binding key is usually an atom, but it - may be a tuple for variables defined in a different context. + ## Formatting considerations - ## Examples + The Elixir AST does not contain metadata for literals like strings, lists, or + tuples with two elements, which means that the produced algebra document will + not respect all of the user preferences and comments may be misplaced. + To get better results, you can use the `:token_metadata`, `:unescape` and + `:literal_encoder` options to `string_to_quoted/2` to provide additional + information to the formatter: - iex> Code.eval_string("a + b", [a: 1, b: 2], file: __ENV__.file, line: __ENV__.line) - {3, [a: 1, b: 2]} + [ + literal_encoder: &{:ok, {:__block__, &2, [&1]}}, + token_metadata: true, + unescape: false + ] - iex> Code.eval_string("c = a + b", [a: 1, b: 2], __ENV__) - {3, [a: 1, b: 2, c: 3]} + This will produce an AST that contains information such as `do` blocks start + and end lines or sigil delimiters, and by wrapping literals in blocks they can + now hold metadata like line number, string delimiter and escaped sequences, or + integer formatting (such as `0x2a` instead of `47`). However, **note this AST is + not valid**. If you evaluate it, it won't have the same semantics as the regular + Elixir AST due to the `:unescape` and `:literal_encoder` options. However, + those options are useful if you're doing source code manipulation, where it's + important to preserve user choices and comments placing. - iex> Code.eval_string("a = a + b", [a: 1, b: 2]) - {3, [a: 3, b: 2]} + ## Options - For convenience, you can pass `__ENV__` as the `opts` argument and - all imports, requires and aliases defined in the current environment - will be automatically carried over: + This function accepts all options supported by `format_string!/2` for controlling + code formatting, plus these additional options: + + * `:comments` - the list of comments associated with the quoted expression. + Defaults to `[]`. It is recommended that both `:token_metadata` and + `:literal_encoder` options are given to `string_to_quoted_with_comments/2` + in order to get proper placement for comments - iex> Code.eval_string("a + b", [a: 1, b: 2], __ENV__) - {3, [a: 1, b: 2]} + * `:escape` - when `true`, escaped sequences like `\n` will be escaped into + `\\n`. If the `:unescape` option was set to `false` when using + `string_to_quoted/2`, setting this option to `false` will prevent it from + escaping the sequences twice. Defaults to `true`. + See `format_string!/2` for the full list of formatting options including + `:file`, `:line`, `:line_length`, `:locals_without_parens`, `:force_do_end_blocks`, + `:syntax_colors`, and all migration options like `:migrate_charlists_as_sigils`. """ - def eval_string(string, binding \\ [], opts \\ []) + @doc since: "1.13.0" + @spec quoted_to_algebra(Macro.t(), [format_opt() | quoted_to_algebra_opt()]) :: + Inspect.Algebra.t() + def quoted_to_algebra(quoted, opts \\ []) do + quoted + |> Code.Normalizer.normalize(opts) + |> Code.Formatter.to_algebra(opts) + end - def eval_string(string, binding, %Macro.Env{} = env) do - {value, binding, _env, _scope} = :elixir.eval to_char_list(string), binding, Map.to_list(env) - {value, binding} + @doc """ + Evaluates the given file. + + Accepts `relative_to` as an argument to tell where the file is located. + + While `require_file/2` and `compile_file/2` return the loaded modules and their + bytecode, `eval_file/2` simply evaluates the file contents and returns the + evaluation result and its binding (exactly the same return value as `eval_string/3`). + """ + @spec eval_file(binary, nil | binary) :: {term, binding} + def eval_file(file, relative_to \\ nil) when is_binary(file) do + {charlist, file} = find_file!(file, relative_to) + eval_string(charlist, [], file: file, line: 1) end - def eval_string(string, binding, opts) when is_list(opts) do - validate_eval_opts(opts) - {value, binding, _env, _scope} = :elixir.eval to_char_list(string), binding, opts - {value, binding} + @deprecated "Use Code.require_file/2 or Code.compile_file/2 instead" + @doc false + def load_file(file, relative_to \\ nil) when is_binary(file) do + {charlist, file} = find_file!(file, relative_to) + :elixir_code_server.call({:acquire, file}) + + loaded = + Module.ParallelChecker.verify(fn -> + :elixir_compiler.string(charlist, file, fn _, _ -> :ok end) + end) + + :elixir_code_server.cast({:required, file}) + loaded end @doc """ - Evaluate the quoted contents. + Requires the given `file`. - See `eval_string/3` for a description of arguments and return values. + Accepts `relative_to` as an argument to tell where the file is located. + If the file was already required, `require_file/2` doesn't do anything and + returns `nil`. - ## Examples + Note that if `require_file/2` is invoked by different processes concurrently, + the first process to invoke `require_file/2` acquires a lock and the remaining + ones will block until the file is available. This means that if `require_file/2` + is called more than once with a given file, that file will be compiled only once. + The first process to call `require_file/2` will get the list of loaded modules, + others will get `nil`. The list of required files is managed per Erlang VM node. - iex> contents = quote(do: var!(a) + var!(b)) - iex> Code.eval_quoted(contents, [a: 1, b: 2], file: __ENV__.file, line: __ENV__.line) - {3, [a: 1, b: 2]} + See `compile_file/2` if you would like to compile a file without tracking its + filenames. Finally, if you would like to get the result of evaluating a file rather + than the modules defined in it, see `eval_file/2`. - For convenience, you can pass `__ENV__` as the `opts` argument and - all options will be automatically extracted from the current environment: + ## Examples - iex> contents = quote(do: var!(a) + var!(b)) - iex> Code.eval_quoted(contents, [a: 1, b: 2], __ENV__) - {3, [a: 1, b: 2]} + If the file has not been required, it returns the list of modules: - """ - def eval_quoted(quoted, binding \\ [], opts \\ []) + modules = Code.require_file("eex_test.exs", "../eex/test") + List.first(modules) + #=> {EExTest.Compiled, <<70, 79, 82, 49, ...>>} - def eval_quoted(quoted, binding, %Macro.Env{} = env) do - {value, binding, _env, _scope} = :elixir.eval_quoted quoted, binding, Map.to_list(env) - {value, binding} - end + If the file has been required, it returns `nil`: - def eval_quoted(quoted, binding, opts) when is_list(opts) do - validate_eval_opts(opts) - {value, binding, _env, _scope} = :elixir.eval_quoted quoted, binding, opts - {value, binding} - end + Code.require_file("eex_test.exs", "../eex/test") + #=> nil - defp validate_eval_opts(opts) do - if f = opts[:functions], do: validate_imports(:functions, f) - if m = opts[:macros], do: validate_imports(:macros, m) - if a = opts[:aliases], do: validate_aliases(:aliases, a) - if r = opts[:requires], do: validate_requires(:requires, r) - end + """ + @spec require_file(binary, nil | binary) :: [{module, binary}] | nil + def require_file(file, relative_to \\ nil) when is_binary(file) do + {charlist, file} = find_file!(file, relative_to) + + case :elixir_code_server.call({:acquire, file}) do + :required -> + nil - defp validate_requires(kind, requires) do - valid = is_list(requires) and Enum.all?(requires, &is_atom(&1)) + :proceed -> + loaded = + Module.ParallelChecker.verify(fn -> + :elixir_compiler.string(charlist, file, fn _, _ -> :ok end) + end) - unless valid do - raise ArgumentError, "expected :#{kind} option given to eval in the format: [module]" + :elixir_code_server.cast({:required, file}) + loaded end end - defp validate_aliases(kind, aliases) do - valid = is_list(aliases) and Enum.all?(aliases, fn {k, v} -> - is_atom(k) and is_atom(v) - end) + @doc """ + Gets all compilation options from the code server. - unless valid do - raise ArgumentError, "expected :#{kind} option given to eval in the format: [{module, module}]" - end - end + To get individual options, see `get_compiler_option/1`. + For a description of all options, see `put_compiler_option/2`. - defp validate_imports(kind, imports) do - valid = is_list(imports) and Enum.all?(imports, fn {k, v} -> - is_atom(k) and is_list(v) and Enum.all?(v, fn {name, arity} -> - is_atom(name) and is_integer(arity) - end) - end) + ## Examples + + Code.compiler_options() + #=> %{debug_info: true, docs: true, ...} - unless valid do - raise ArgumentError, "expected :#{kind} option given to eval in the format: [{module, [{name, arity}]}]" + """ + @spec compiler_options :: map + def compiler_options do + for key <- @available_compiler_options, into: %{} do + {key, :elixir_config.get(key)} end end @doc """ - Convert the given string to its quoted form. - - Returns `{:ok, quoted_form}` - if it succeeds, `{:error, {line, error, token}}` otherwise. - - ## Options + Stores all given compilation options. - * `:file` - the filename to be used in stacktraces - and the file reported in the `__ENV__` variable + Changing the compilation options affect all processes + running in a given Erlang VM node. To store individual + options and for a description of all options, see + `put_compiler_option/2`. - * `:line` - the line reported in the `__ENV__` variable + Returns a map with previous values. - * `:existing_atoms_only` - when `true`, raises an error - when non-existing atoms are found by the tokenizer + ## Examples - ## Macro.to_string/2 + Code.compiler_options(infer_signatures: false) + #=> %{infer_signatures: [:elixir]} - The opposite of converting a string to its quoted form is - `Macro.to_string/2`, which converts a quoted form to a string/binary - representation. """ - def string_to_quoted(string, opts \\ []) when is_list(opts) do - file = Keyword.get opts, :file, "nofile" - line = Keyword.get opts, :line, 1 - :elixir.string_to_quoted(to_char_list(string), line, file, opts) + @spec compiler_options(Enumerable.t({atom, term})) :: %{optional(atom) => term} + def compiler_options(opts) do + for {key, value} <- opts, into: %{} do + previous = get_compiler_option(key) + put_compiler_option(key, value) + {key, previous} + end end @doc """ - Convert the given string to its quoted form. + Returns the value of a given compiler option. - It returns the ast if it succeeds, - raises an exception otherwise. The exception is a `TokenMissingError` - in case a token is missing (usually because the expression is incomplete), - `SyntaxError` otherwise. + For a description of all options, see `put_compiler_option/2`. + + ## Examples + + Code.get_compiler_option(:debug_info) + #=> true - Check `string_to_quoted/2` for options information. """ - def string_to_quoted!(string, opts \\ []) when is_list(opts) do - file = Keyword.get opts, :file, "nofile" - line = Keyword.get opts, :line, 1 - :elixir.string_to_quoted!(to_char_list(string), line, file, opts) + @doc since: "1.10.0" + @spec get_compiler_option(atom) :: term + def get_compiler_option(key) when key in @available_compiler_options do + :elixir_config.get(key) + end + + # TODO: Remove me in Elixir v2.0 + def get_compiler_option(:warnings_as_errors) do + IO.warn(":warnings_as_errors is deprecated as part of Code.get_compiler_option/1") + :ok end @doc """ - Evals the given file. + Returns a list with all available compiler options. - Accepts `relative_to` as an argument to tell where the file is located. + For a description of all options, see `put_compiler_option/2`. + + ## Examples + + Code.available_compiler_options() + #=> [:docs, :debug_info, ...] - While `load_file` loads a file and returns the loaded modules and their - byte code, `eval_file` simply evalutes the file contents and returns the - evaluation result and its bindings. """ - def eval_file(file, relative_to \\ nil) do - file = find_file(file, relative_to) - eval_string File.read!(file), [], [file: file, line: 1] + @spec available_compiler_options() :: [atom] + def available_compiler_options do + @available_compiler_options end @doc """ - Load the given file. + Stores a compilation option. - Accepts `relative_to` as an argument to tell where the file is located. - If the file was already required/loaded, loads it again. + Changing the compilation options affect all processes running in a + given Erlang VM node. + + Available options are: + + * `:docs` - when `true`, retains documentation in the compiled module. + Defaults to `true`. + + * `:debug_info` - when `true`, retains debug information in the compiled + module. This option can also be overridden per module using the `@compile` + directive. Defaults to `true`. + + This enables tooling to partially reconstruct the original source code, + for instance, to perform static analysis of code. Therefore, disabling + `:debug_info` is not recommended as it removes the ability of the + Elixir compiler and other tools to provide feedback. If you want to + remove the `:debug_info` while deploying, tools like `mix release` + already do such by default. + + Other environments, such as `mix test`, automatically disables this + via the `:test_elixirc_options` project configuration, as there is + typically no need to store debug chunks for test files. + + * `:ignore_already_consolidated` (since v1.10.0) - when `true`, does not warn + when a protocol has already been consolidated and a new implementation is added. + Defaults to `false`. + + * `:ignore_module_conflict` - when `true`, does not warn when a module has + already been defined. Defaults to `false`. + + * `:infer_signatures` (since v1.18.0) - a list of applications of which modules + should be using during type inference. When `false`, it disables module-local + signature inference used when type checking remote calls to the compiled + module. Type checking will be executed regardless of the value of this option. + Defaults to `true`, which is equivalent to setting it to `[:elixir]` only. + + When setting this option, we recommend running `mix clean` so the current module + may be compiled from scratch. `mix test` automatically disables this option via + the `:test_elixirc_options` project configuration, as there is typically no need + to infer signatures for test files. + + * `:relative_paths` - when `true`, uses relative paths in quoted nodes, + warnings, and errors generated by the compiler. Note disabling this option + won't affect runtime warnings and errors. Defaults to `true`. + + * `:no_warn_undefined` (since v1.10.0) - list of modules and `{Mod, fun, arity}` + tuples that will not emit warnings that the module or function does not exist + at compilation time. Pass atom `:all` to skip warning for all undefined + functions. This can be useful when doing dynamic compilation. Defaults to `[]`. + + * `:tracers` (since v1.10.0) - a list of tracers (modules) to be used during + compilation. See the module docs for more information. Defaults to `[]`. + + * `:parser_options` (since v1.10.0) - a keyword list of options to be given + to the parser when compiling files. It accepts the same options as + `string_to_quoted/2` (except by the options that change the AST itself). + This can be used in combination with the tracer to retrieve localized + information about events happening during compilation. Defaults to `[columns: true]`. + This option only affects code compilation functions, such as `compile_string/2` + and `compile_file/2` but not `string_to_quoted/2` and friends, as the + latter is used for other purposes beyond compilation. + + * `:on_undefined_variable` (since v1.15.0) - either `:raise` or `:warn`. + When `:raise` (the default), undefined variables will trigger a compilation + error. You may be set it to `:warn` if you want undefined variables to + emit a warning and expand as to a local call to the zero-arity function + of the same name (for example, `node` would be expanded as `node()`). + This `:warn` behavior only exists for compatibility reasons when working + with old dependencies, its usage is discouraged and it will be removed + in future releases. + + It always returns `:ok`. Raises an error for invalid options. - It returns a list of tuples `{ModuleName, <>}`, one tuple for - each module defined in the file. + ## Examples + + Code.put_compiler_option(:debug_info, true) + #=> :ok - Notice that if `load_file` is invoked by different processes concurrently, - the target file will be loaded concurrently many times. Check `require_file/2` - if you don't want a file to be loaded concurrently. """ - def load_file(file, relative_to \\ nil) when is_binary(file) do - file = find_file(file, relative_to) - :elixir_code_server.call {:acquire, file} - loaded = :elixir_compiler.file file - :elixir_code_server.cast {:loaded, file} - loaded + @doc since: "1.10.0" + @spec put_compiler_option(atom, term) :: :ok + def put_compiler_option(key, value) when key in @boolean_compiler_options do + if not is_boolean(value) do + raise "compiler option #{inspect(key)} should be a boolean, got: #{inspect(value)}" + end + + :elixir_config.put(key, value) + :ok end - @doc """ - Requires the given `file`. + def put_compiler_option(key, value) when key in @list_compiler_options do + if not is_list(value) do + raise "compiler option #{inspect(key)} should be a list, got: #{inspect(value)}" + end - Accepts `relative_to` as an argument to tell where the file is located. - The return value is the same as that of `load_file/2`. If the file was already - required/loaded, doesn't do anything and returns `nil`. + if key == :parser_options and not Keyword.keyword?(value) do + raise "compiler option #{inspect(key)} should be a keyword list, " <> + "got: #{inspect(value)}" + end - Notice that if `require_file` is invoked by different processes concurrently, - the first process to invoke `require_file` acquires a lock and the remaining - ones will block until the file is available. I.e. if `require_file` is called - N times with a given file, it will be loaded only once. The first process to - call `require_file` will get the list of loaded modules, others will get `nil`. + if key == :tracers and not Enum.all?(value, &is_atom/1) do + raise "compiler option #{inspect(key)} should be a list of modules, " <> + "got: #{inspect(value)}" + end - Check `load_file/2` if you want a file to be loaded multiple times. - """ - def require_file(file, relative_to \\ nil) when is_binary(file) do - file = find_file(file, relative_to) + :elixir_config.put(key, value) + :ok + end - case :elixir_code_server.call({:acquire, file}) do - :loaded -> - nil - {:queued, ref} -> - receive do {:elixir_code_server, ^ref, :loaded} -> nil end - :proceed -> - loaded = :elixir_compiler.file file - :elixir_code_server.cast {:loaded, file} - loaded - end + def put_compiler_option(:infer_signatures, value) do + value = + cond do + value == false -> + false + + value == true -> + [:elixir] + + is_list(value) and Enum.all?(value, &is_atom/1) -> + value + + true -> + raise "compiler option :infer_signatures should be a boolean or a list of applications, got: #{inspect(value)}" + end + + :elixir_config.put(:infer_signatures, value) + :ok end - @doc """ - Gets the compilation options from the code server. + def put_compiler_option(:no_warn_undefined, value) do + if value != :all and not is_list(value) do + raise "compiler option :no_warn_undefined should be a list or the atom :all, " <> + "got: #{inspect(value)}" + end - Check `compiler_options/1` for more information. - """ - def compiler_options do - :elixir_code_server.call :compiler_options + :elixir_config.put(:no_warn_undefined, value) + :ok end - @doc """ - Returns a list with the available compiler options. + # TODO: Remove me in Elixir v2.0 + def put_compiler_option(:warnings_as_errors, _value) do + IO.warn( + ":warnings_as_errors is deprecated as part of Code.put_compiler_option/2, " <> + "instead you must pass it as a --warnings-as-errors flag. " <> + "If you need to set it as a default in a mix task, you can also set it under aliases: " <> + "[compile: \"compile --warnings-as-errors\"]" + ) - See `Code.compiler_options/1` for more info. - """ - def available_compiler_options do - [:docs, :debug_info, :ignore_module_conflict, :warnings_as_errors] + :ok end - @doc """ - Sets compilation options. + # TODO: Remove me in Elixir v2.0 + def put_compiler_option(:on_undefined_variable, value) when value in [:raise, :warn] do + if value == :warn do + IO.warn_once( + {__MODULE__, :on_undefined_variable}, + fn -> + "setting :on_undefined_variable to :warn is deprecated. " <> + "The warning behaviour will be removed in future releases" + end, + 3 + ) + end - These options are global since they are stored by Elixir's Code Server. + :elixir_config.put(:on_undefined_variable, value) + :ok + end - Available options are: + def put_compiler_option(key, _value) do + raise "unknown compiler option: #{inspect(key)}" + end - * `:docs` - when `true`, retain documentation in the compiled module, - `true` by default + @doc """ + Purge compiler modules. - * `:debug_info` - when `true`, retain debug information in the compiled - module; this allows a developer to reconstruct the original source - code, `false` by default + The compiler utilizes temporary modules to compile code. For example, + `elixir_compiler_1`, `elixir_compiler_2`, and so on. In case the compiled code + stores references to anonymous functions or similar, the Elixir compiler + may be unable to reclaim those modules, keeping an unnecessary amount of + code in memory and eventually leading to modules such as `elixir_compiler_12345`. - * `:ignore_module_conflict` - when `true`, override modules that were - already defined without raising errors, `false` by default + This function purges all modules currently kept by the compiler, allowing + old compiler module names to be reused. If there are any processes running + any code from such modules, they will be terminated too. - * `:warnings_as_errors` - cause compilation to fail when warnings are - generated + This function is only meant to be called if you have a long running node + that is constantly evaluating code. + It returns `{:ok, number_of_modules_purged}`. """ - def compiler_options(opts) do - {opts, bad} = Keyword.split(opts, available_compiler_options) - if bad != [] do - bad = bad |> Keyword.keys |> Enum.join(", ") - raise ArgumentError, message: "unknown compiler options: #{bad}" - end - :elixir_code_server.cast {:compiler_options, opts} + @doc since: "1.7.0" + @spec purge_compiler_modules() :: {:ok, non_neg_integer()} + def purge_compiler_modules() do + :elixir_code_server.call(:purge_compiler_modules) end @doc """ Compiles the given string. Returns a list of tuples where the first element is the module name - and the second one is its byte code (as a binary). - - For compiling many files at once, check `Kernel.ParallelCompiler.files/2`. + and the second one is its bytecode (as a binary). A `file` can be + given as a second argument which will be used for reporting warnings + and errors. + + **Warning**: `string` can be any Elixir code and code can be executed with + the same privileges as the Erlang VM: this means that such code could + compromise the machine (for example by executing system commands). + Don't use `compile_string/2` with untrusted input (such as strings coming + from the network). """ + @spec compile_string(List.Chars.t(), binary) :: [{module, binary}] def compile_string(string, file \\ "nofile") when is_binary(file) do - :elixir_compiler.string to_char_list(string), file + Module.ParallelChecker.verify(fn -> + :elixir_compiler.string(to_charlist(string), file, fn _, _ -> :ok end) + end) end @doc """ Compiles the quoted expression. Returns a list of tuples where the first element is the module name and - the second one is its byte code (as a binary). + the second one is its bytecode (as a binary). A `file` can be + given as second argument which will be used for reporting warnings + and errors. """ + @spec compile_quoted(Macro.t(), binary) :: [{module, binary}] def compile_quoted(quoted, file \\ "nofile") when is_binary(file) do - :elixir_compiler.quoted quoted, file + Module.ParallelChecker.verify(fn -> + :elixir_compiler.quoted(quoted, file, fn _, _ -> :ok end) + end) end @doc """ - Ensures the given module is loaded. + Compiles the given file. - If the module is already loaded, this works as no-op. If the module - was not yet loaded, it tries to load it. + Accepts `relative_to` as an argument to tell where the file is located. - If it succeeds loading the module, it returns `{:module, module}`. - If not, returns `{:error, reason}` with the error reason. + Returns a list of tuples where the first element is the module name and + the second one is its bytecode (as a binary). Opposite to `require_file/2`, + it does not track the filename of the compiled file. - ## Code loading on the Erlang VM + If you would like to get the result of evaluating file rather than the + modules defined in it, see `eval_file/2`. - Erlang has two modes to load code: interactive and embedded. + For compiling many files concurrently, see `Kernel.ParallelCompiler.compile/2`. + """ + @doc since: "1.7.0" + @spec compile_file(binary, nil | binary) :: [{module, binary}] + def compile_file(file, relative_to \\ nil) when is_binary(file) do + Module.ParallelChecker.verify(fn -> + {charlist, file} = find_file!(file, relative_to) + :elixir_compiler.string(charlist, file, fn _, _ -> :ok end) + end) + end - By default, the Erlang VM runs in interactive mode, where modules - are loaded as needed. In embedded mode the opposite happens, as all - modules need to be loaded upfront or explicitly. + @doc """ + Ensures the given module is loaded. - Therefore, this function is used to check if a module is loaded - before using it and allows one to react accordingly. For example, the `URI` - module uses this function to check if a specific parser exists for a given - URI scheme. + If the module is already loaded, this works as no-op. If the module + was not yet loaded, it tries to load it. - ## `Code.ensure_compiled/1` + If it succeeds in loading the module, it returns `{:module, module}`. + If not, returns `{:error, reason}` with the error reason. - Elixir also contains an `ensure_compiled/1` function that is a - superset of `ensure_loaded/1`. + See the module documentation for more information on code loading. - Since Elixir's compilation happens in parallel, in some situations - you may need to use a module that was not yet compiled, therefore - it can't even be loaded. + ## Examples - `ensure_compiled/1` halts the current process until the - module we are depending on is available. + iex> Code.ensure_loaded(Atom) + {:module, Atom} + + iex> Code.ensure_loaded(DoesNotExist) + {:error, :nofile} - In most cases, `ensure_loaded/1` is enough. `ensure_compiled/1` - must be used in rare cases, usually involving macros that need to - invoke a module for callback information. """ + @spec ensure_loaded(module) :: + {:module, module} | {:error, :embedded | :badfile | :nofile | :on_load_failure} def ensure_loaded(module) when is_atom(module) do :code.ensure_loaded(module) end @@ -420,122 +1992,351 @@ defmodule Code do Similar to `ensure_loaded/1`, but returns `true` if the module is already loaded or was successfully loaded. Returns `false` otherwise. + + ## Examples + + iex> Code.ensure_loaded?(String) + true + """ - def ensure_loaded?(module) do + @spec ensure_loaded?(module) :: boolean + def ensure_loaded?(module) when is_atom(module) do match?({:module, ^module}, ensure_loaded(module)) end @doc """ - Ensures the given module is compiled and loaded. + Same as `ensure_loaded/1` but raises if the module cannot be loaded. + """ + @doc since: "1.12.0" + @spec ensure_loaded!(module) :: module + def ensure_loaded!(module) do + case ensure_loaded(module) do + {:module, module} -> + module + + {:error, reason} -> + raise ArgumentError, + "could not load module #{inspect(module)} due to reason #{inspect(reason)}" + end + end - If the module is already loaded, it works as no-op. If the module was - not loaded yet, it checks if it needs to be compiled first and then - tries to load it. + @doc """ + Ensures the given modules are loaded. + + Similar to `ensure_loaded/1`, but accepts a list of modules instead of a single + module, and loads all of them. + + If all modules load successfully, returns `:ok`. Otherwise, returns `{:error, errors}` + where `errors` is a list of tuples made of the module and the reason it failed to load. + + ## Examples + + iex> Code.ensure_all_loaded([Atom, String]) + :ok + + iex> Code.ensure_all_loaded([Atom, DoesNotExist]) + {:error, [{DoesNotExist, :nofile}]} + + """ + @doc since: "1.15.0" + @spec ensure_all_loaded([module]) :: :ok | {:error, [{module, reason}]} + when reason: :badfile | :nofile | :on_load_failure + def ensure_all_loaded(modules) when is_list(modules) do + :code.ensure_modules_loaded(modules) + end + + @doc """ + Same as `ensure_all_loaded/1` but raises if any of the modules cannot be loaded. + """ + @doc since: "1.15.0" + @spec ensure_all_loaded!([module]) :: :ok + def ensure_all_loaded!(modules) do + case ensure_all_loaded(modules) do + :ok -> + :ok + + {:error, errors} -> + formatted_errors = + errors + |> Enum.sort() + |> Enum.map_join("\n", fn {module, reason} -> + " * #{inspect(module)} due to reason #{inspect(reason)}" + end) + + raise ArgumentError, "could not load the following modules:\n\n" <> formatted_errors + end + end + + @doc """ + Similar to `ensure_compiled!/1` but indicates you can continue without said module. + + While `ensure_compiled!/1` indicates to the Elixir compiler you can + only continue when said module is available, this function indicates + you may continue compilation without said module. - If it succeeds loading the module, it returns `{:module, module}`. + If it succeeds in loading the module, it returns `{:module, module}`. If not, returns `{:error, reason}` with the error reason. + If the module being checked is currently in a compiler deadlock, + this function returns `{:error, :unavailable}`. Unavailable doesn't + necessarily mean the module doesn't exist, just that it is not currently + available, but it (or may not) become available in the future. + + Therefore, if you can only continue if the module is available, use + `ensure_compiled!/1` instead. In particular, do not do this: + + case Code.ensure_compiled(module) do + {:module, _} -> module + {:error, _} -> raise ... + end - Check `ensure_loaded/1` for more information on module loading - and when to use `ensure_loaded/1` or `ensure_compiled/1`. + See the module documentation for more information on code loading. """ + @spec ensure_compiled(module) :: + {:module, module} + | {:error, :embedded | :badfile | :nofile | :on_load_failure | :unavailable} def ensure_compiled(module) when is_atom(module) do + ensure_compiled(module, :soft) + end + + @doc """ + Ensures the given module is compiled and loaded. + + If the module is already loaded, it works as no-op. If the module was + not compiled yet, `ensure_compiled!/1` halts the compilation of the caller + until the module given to `ensure_compiled!/1` becomes available or + all files for the current project have been compiled. If compilation + finishes and the module is not available or is in a deadlock, an error + is raised. + + Given this function halts compilation, use it carefully. In particular, + avoid using it to guess which modules are in the system. Overuse of this + function can also lead to deadlocks, where two modules check at the same time + if the other is compiled. This returns a specific unavailable error code, + where we cannot successfully verify a module is available or not. + + See the module documentation for more information on code loading. + """ + @doc since: "1.12.0" + @spec ensure_compiled!(module) :: module + def ensure_compiled!(module) do + case ensure_compiled(module, :hard) do + {:module, module} -> + module + + {:error, reason} -> + raise ArgumentError, + "could not load module #{inspect(module)} due to reason #{inspect(reason)}" + end + end + + defp ensure_compiled(module, mode) do case :code.ensure_loaded(module) do {:error, :nofile} = error -> - case :erlang.get(:elixir_ensure_compiled) do - :undefined -> error - _ -> - try do - module.__info__(:module) - {:module, module} - rescue - UndefinedFunctionError -> error - end + if can_await_module_compilation?() do + case Kernel.ErrorHandler.ensure_compiled(module, :module, mode, nil) do + :found -> {:module, module} + :deadlock -> {:error, :unavailable} + :not_found -> {:error, :nofile} + end + else + error end - other -> other + + other -> + other end end @doc """ - Ensures the given module is compiled and loaded. + Returns `true` if the module is loaded. + + This function doesn't attempt to load the module. For such behavior, + `ensure_loaded?/1` can be used. + + ## Examples + + iex> Code.loaded?(String) + true + + iex> Code.loaded?(NotYetLoaded) + false - Similar to `ensure_compiled/1`, but returns `true` if the module - is already loaded or was successfully loaded and compiled. - Returns `false` otherwise. """ - def ensure_compiled?(module) do - match?({:module, ^module}, ensure_compiled(module)) + @doc since: "1.15.0" + @spec loaded?(module) :: boolean + def loaded?(module) do + :erlang.module_loaded(module) end @doc """ - Returns the docs for the given module. + Returns `true` if the current process can await for module compilation. + + When compiling Elixir code via `Kernel.ParallelCompiler`, which is + used by Mix and `elixirc`, calling a module that has not yet been + compiled will block the caller until the module becomes available. + Executing Elixir scripts, such as passing a filename to `elixir`, + does not await. + """ + @doc since: "1.11.0" + @spec can_await_module_compilation? :: boolean + def can_await_module_compilation? do + :erlang.process_info(self(), :error_handler) == {:error_handler, Kernel.ErrorHandler} + end + + @doc false + @deprecated "Use Code.ensure_compiled/1 instead (see the proper disclaimers in its docs)" + def ensure_compiled?(module) when is_atom(module) do + match?({:module, ^module}, ensure_compiled(module)) + end + + @doc ~S""" + Returns the docs for the given module or path to `.beam` file. When given a module name, it finds its BEAM code and reads the docs from it. - When given a path to a .beam file, it will load the docs directly from that + When given a path to a `.beam` file, it will load the docs directly from that file. - The return value depends on the `kind` value: + It returns the term stored in the documentation chunk in the format defined by + [EEP 48](https://www.erlang.org/eeps/eep-0048.html) or `{:error, reason}` if + the chunk is not available. - * `:docs` - list of all docstrings attached to functions and macros - using the `@doc` attribute + ## Examples - * `:moduledoc` - tuple `{, }` where `line` is the line on - which module definition starts and `doc` is the string - attached to the module using the `@moduledoc` attribute + # Module documentation of an existing module + iex> {:docs_v1, _, :elixir, _, %{"en" => module_doc}, _, _} = Code.fetch_docs(Atom) + iex> module_doc |> String.split("\n") |> Enum.at(0) + "Atoms are constants whose values are their own name." - * `:all` - a keyword list with both `:docs` and `:moduledoc` + # A module that doesn't exist + iex> Code.fetch_docs(ModuleNotGood) + {:error, :module_not_found} """ - def get_docs(module, kind) when is_atom(module) do - case :code.get_object_code(module) do - {_module, bin, _beam_path} -> - do_get_docs(bin, kind) + @doc since: "1.7.0" + @spec fetch_docs(module | String.t()) :: + {:docs_v1, annotation, beam_language, format, module_doc :: doc_content, metadata, + docs :: [doc_element]} + | {:error, + :module_not_found + | :chunk_not_found + | {:invalid_chunk, binary} + | :invalid_beam} + when annotation: :erl_anno.anno(), + beam_language: :elixir | :erlang | atom(), + doc_content: %{optional(binary) => binary} | :none | :hidden, + doc_element: + {{kind :: atom, function_name :: atom, arity}, annotation, signature, doc_content, + metadata}, + format: binary, + signature: [binary], + metadata: map + def fetch_docs(module_or_path) + + def fetch_docs(module) when is_atom(module) do + case get_beam_and_path(module) do + {bin, beam_path} -> + case fetch_docs_from_beam(bin) do + {:error, :chunk_not_found} -> + app_root = Path.expand(Path.join(["..", ".."]), beam_path) + path = Path.join([app_root, "doc", "chunks", "#{module}.chunk"]) + fetch_docs_from_chunk(path) + + other -> + other + end + + :error -> + case :code.is_loaded(module) do + {:file, :preloaded} -> + # The ERTS directory is not necessarily included in releases + # unless it is listed as an extra application. + case :code.lib_dir(:erts) do + path when is_list(path) -> + path = Path.join([path, "doc", "chunks", "#{module}.chunk"]) + fetch_docs_from_chunk(path) + + {:error, _} -> + {:error, :chunk_not_found} + end - :error -> nil + _ -> + {:error, :module_not_found} + end end end - def get_docs(binpath, kind) when is_binary(binpath) do - do_get_docs(String.to_char_list(binpath), kind) + def fetch_docs(path) when is_binary(path) do + fetch_docs_from_beam(String.to_charlist(path)) + end + + defp get_beam_and_path(module) do + with {^module, beam, filename} <- :code.get_object_code(module), + info_pairs when is_list(info_pairs) <- :beam_lib.info(beam), + {:ok, ^module} <- Keyword.fetch(info_pairs, :module) do + {beam, filename} + else + _ -> :error + end end - @docs_chunk 'ExDc' + @docs_chunk [?D, ?o, ?c, ?s] - defp do_get_docs(bin_or_path, kind) do + defp fetch_docs_from_beam(bin_or_path) do case :beam_lib.chunks(bin_or_path, [@docs_chunk]) do {:ok, {_module, [{@docs_chunk, bin}]}} -> - lookup_docs(:erlang.binary_to_term(bin), kind) + load_docs_chunk(bin) + + {:error, :beam_lib, {:missing_chunk, _, @docs_chunk}} -> + {:error, :chunk_not_found} - {:error, :beam_lib, {:missing_chunk, _, @docs_chunk}} -> nil + {:error, :beam_lib, {:file_error, _, :enoent}} -> + {:error, :module_not_found} + + {:error, :beam_lib, _} -> + {:error, :invalid_beam} end end - defp lookup_docs({:elixir_docs_v1, docs}, kind), - do: do_lookup_docs(docs, kind) + defp fetch_docs_from_chunk(path) do + case File.read(path) do + {:ok, bin} -> + load_docs_chunk(bin) + + {:error, _} -> + {:error, :chunk_not_found} + end + end - # unsupported chunk version - defp lookup_docs(_, _), do: nil + defp load_docs_chunk(bin) do + :erlang.binary_to_term(bin) + rescue + _ -> + {:error, {:invalid_chunk, bin}} + end - defp do_lookup_docs(docs, :all), do: docs - defp do_lookup_docs(docs, kind) when kind in [:docs, :moduledoc], - do: Keyword.get(docs, kind) + @doc false + @deprecated "Code.get_docs/2 always returns nil as its outdated documentation is no longer stored on BEAM files. Use Code.fetch_docs/1 instead" + def get_docs(_module, _kind) do + nil + end ## Helpers # Finds the file given the relative_to path. # # If the file is found, returns its path in binary, fails otherwise. - defp find_file(file, relative_to) do - file = if relative_to do - Path.expand(file, relative_to) - else - Path.expand(file) - end - - if File.regular?(file) do - file - else - raise Code.LoadError, file: file + defp find_file!(file, relative_to) do + file = + if relative_to do + Path.expand(file, relative_to) + else + Path.expand(file) + end + + case File.read(file) do + {:ok, bin} -> {String.to_charlist(bin), file} + {:error, reason} -> raise Code.LoadError, file: file, reason: reason end end end diff --git a/lib/elixir/lib/code/formatter.ex b/lib/elixir/lib/code/formatter.ex new file mode 100644 index 00000000000..70ea70ae0e8 --- /dev/null +++ b/lib/elixir/lib/code/formatter.ex @@ -0,0 +1,2605 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +defmodule Code.Formatter do + @moduledoc false + import Inspect.Algebra, except: [format: 2, surround: 3, surround: 4] + + @double_quote "\"" + @double_heredoc "\"\"\"" + @single_quote "'" + @single_heredoc "'''" + @sigil_c_double "~c\"" + @sigil_c_single "~c'" + @sigil_c_heredoc "~c\"\"\"" + @newlines 2 + @min_line 0 + @max_line 9_999_999 + @empty empty() + @ampersand_prec Code.Identifier.unary_op(:&) |> elem(1) + + # Operators that are composed of multiple binary operators + @multi_binary_operators [:..//] + + # Operators that do not have space between operands + @no_space_binary_operators [:.., :"//"] + + # Operators that do not have newline between operands (as well as => and keywords) + @no_newline_binary_operators [:\\, :in] + + # Left associative operators that start on the next line in case of breaks (always pipes) + @pipeline_operators [:|>, :~>>, :<<~, :~>, :<~, :<~>, :"<|>"] + + # Right associative operators that start on the next line in case of breaks + @right_new_line_before_binary_operators [:|, :when] + + # Operators that are logical cannot be mixed without parens + @required_parens_logical_binary_operands [:|||, :||, :or, :&&&, :&&, :and] + + # Operators with next break fits + @next_break_fits_operators [:<-, :==, :!=, :=~, :===, :!==, :<, :>, :<=, :>=, :=, :"::"] + + # Operators that always require parens even + # when they are their own parents as they are not semantically associative + @required_parens_even_when_parent [:--, :---] + + # Operators that always require parens on operands + # when they are the parent of another operator with a difference precedence + # Most operators are listed, except comparison, arithmetic, and low precedence + @required_parens_on_binary_operands [ + :<<<, + :>>>, + :|>, + :<~, + :~>, + :<<~, + :~>>, + :<~>, + :"<|>", + :in, + :"^^^", + :"//", + :++, + :--, + :+++, + :---, + :<>, + :.. + ] + + @locals_without_parens [ + # Special forms + alias: 1, + alias: 2, + case: 2, + cond: 1, + for: :*, + import: 1, + import: 2, + quote: 1, + quote: 2, + receive: 1, + require: 1, + require: 2, + try: 1, + with: :*, + + # Kernel + def: 1, + def: 2, + defp: 1, + defp: 2, + defguard: 1, + defguardp: 1, + defmacro: 1, + defmacro: 2, + defmacrop: 1, + defmacrop: 2, + defmodule: 2, + defdelegate: 2, + defexception: 1, + defoverridable: 1, + defstruct: 1, + destructure: 2, + raise: 1, + raise: 2, + reraise: 2, + reraise: 3, + if: 2, + unless: 2, + use: 1, + use: 2, + + # Stdlib, + defrecord: 2, + defrecord: 3, + defrecordp: 2, + defrecordp: 3, + + # Testing + assert: 1, + assert: 2, + assert_in_delta: 3, + assert_in_delta: 4, + assert_raise: 2, + assert_raise: 3, + assert_receive: 1, + assert_receive: 2, + assert_receive: 3, + assert_received: 1, + assert_received: 2, + doctest: 1, + doctest: 2, + refute: 1, + refute: 2, + refute_in_delta: 3, + refute_in_delta: 4, + refute_receive: 1, + refute_receive: 2, + refute_receive: 3, + refute_received: 1, + refute_received: 2, + setup: 1, + setup: 2, + setup_all: 1, + setup_all: 2, + test: 1, + test: 2, + + # Mix config + config: 2, + config: 3, + import_config: 1 + ] + + @do_end_keywords [:rescue, :catch, :else, :after] + + @doc """ + Converts the quoted expression into an algebra document. + """ + @spec to_algebra(Macro.t(), keyword()) :: Inspect.Algebra.t() + def to_algebra(quoted, opts \\ []) do + comments = Keyword.get(opts, :comments, []) + + state = + comments + |> Enum.map(&format_comment/1) + |> gather_comments() + |> state(opts) + + {doc, _} = block_to_algebra(quoted, @min_line, @max_line, state) + doc + end + + @doc """ + Lists all default locals without parens. + """ + def locals_without_parens do + @locals_without_parens + end + + @doc """ + Checks if a function is a local without parens. + """ + def local_without_parens?(fun, arity, locals_without_parens) do + arity > 0 and + Enum.any?(locals_without_parens, fn {key, val} -> + key == fun and (val == :* or val == arity) + end) + end + + defp state(comments, opts) do + force_do_end_blocks = Keyword.get(opts, :force_do_end_blocks, false) + locals_without_parens = Keyword.get(opts, :locals_without_parens, []) + file = Keyword.get(opts, :file, nil) + sigils = Keyword.get(opts, :sigils, []) + migrate = Keyword.get(opts, :migrate, false) + migrate_bitstring_modifiers = Keyword.get(opts, :migrate_bitstring_modifiers, migrate) + migrate_call_parens_on_pipe = Keyword.get(opts, :migrate_call_parens_on_pipe, migrate) + migrate_charlists_as_sigils = Keyword.get(opts, :migrate_charlists_as_sigils, migrate) + migrate_unless = Keyword.get(opts, :migrate_unless, migrate) + syntax_colors = Keyword.get(opts, :syntax_colors, []) + + sigils = + Map.new(sigils, fn {key, value} -> + with true <- is_atom(key) and is_function(value, 2), + name = Atom.to_charlist(key), + true <- Enum.all?(name, &(&1 in ?A..?Z)) do + {name, value} + else + _ -> + raise ArgumentError, + ":sigils must be a keyword list with uppercased atoms as keys and an " <> + "anonymous function expecting two arguments as value, got: #{inspect(sigils)}" + end + end) + + %{ + force_do_end_blocks: force_do_end_blocks, + locals_without_parens: locals_without_parens ++ locals_without_parens(), + operand_nesting: 2, + skip_eol: false, + comments: comments, + sigils: sigils, + file: file, + migrate_bitstring_modifiers: migrate_bitstring_modifiers, + migrate_call_parens_on_pipe: migrate_call_parens_on_pipe, + migrate_charlists_as_sigils: migrate_charlists_as_sigils, + migrate_unless: migrate_unless, + inspect_opts: %Inspect.Opts{syntax_colors: syntax_colors} + } + end + + defp format_comment(%{text: text} = comment) do + %{comment | text: format_comment_text(text)} + end + + defp format_comment_text("#"), do: "#" + defp format_comment_text("#!" <> rest), do: "#!" <> rest + defp format_comment_text("##" <> rest), do: "#" <> format_comment_text("#" <> rest) + defp format_comment_text("# " <> rest), do: "# " <> rest + defp format_comment_text("#" <> rest), do: "# " <> rest + + # If there is a no new line before, we can't gather all followup comments. + defp gather_comments([%{previous_eol_count: 0} = comment | comments]) do + comment = %{comment | previous_eol_count: @newlines} + [comment | gather_comments(comments)] + end + + defp gather_comments([comment | comments]) do + %{line: line, next_eol_count: next_eol_count, text: doc} = comment + + {next_eol_count, comments, doc} = + gather_followup_comments(line + 1, next_eol_count, comments, doc) + + comment = %{comment | next_eol_count: next_eol_count, text: doc} + [comment | gather_comments(comments)] + end + + defp gather_comments([]) do + [] + end + + defp gather_followup_comments(line, _, [%{line: line} = comment | comments], doc) + when comment.previous_eol_count != 0 do + %{next_eol_count: next_eol_count, text: text} = comment + gather_followup_comments(line + 1, next_eol_count, comments, line(doc, text)) + end + + defp gather_followup_comments(_line, next_eol_count, comments, doc) do + {next_eol_count, comments, doc} + end + + # Special AST nodes from compiler feedback + + defp quoted_to_algebra({{:special, :clause_args}, _meta, [args]}, _context, state) do + {doc, state} = clause_args_to_algebra(args, state) + {group(doc), state} + end + + defp quoted_to_algebra({{:special, :bitstring_segment}, _meta, [arg, last]}, _context, state) do + bitstring_segment_to_algebra({arg, -1}, state, last) + end + + defp quoted_to_algebra({var, _meta, var_context}, _context, state) when is_atom(var_context) do + {var |> Atom.to_string() |> string() |> color_doc(:variable, state.inspect_opts), state} + end + + defp quoted_to_algebra({:<<>>, meta, entries}, _context, state) do + cond do + entries == [] -> + {"<<>>", state} + + not interpolated?(entries) -> + bitstring_to_algebra(meta, entries, state) + + meta[:delimiter] == ~s["""] -> + {doc, state} = + entries + |> prepend_heredoc_line() + |> interpolation_to_algebra(~s["""], state, @double_heredoc, @double_heredoc) + + {force_unfit(doc), state} + + true -> + interpolation_to_algebra(entries, @double_quote, state, @double_quote, @double_quote) + end + end + + # TODO: Remove this clause on Elixir v2.0 once single-quoted charlists are removed + defp quoted_to_algebra( + {{:., _, [List, :to_charlist]}, meta, [entries]} = quoted, + context, + state + ) do + cond do + not list_interpolated?(entries) -> + remote_to_algebra(quoted, context, state) + + meta[:delimiter] == ~s['''] -> + {opener, quotes} = get_charlist_quotes(:heredoc, state) + + {doc, state} = + entries + |> prepend_heredoc_line() + |> list_interpolation_to_algebra(quotes, state, opener, quotes) + + {force_unfit(doc), state} + + true -> + {opener, quotes} = get_charlist_quotes({:regular, entries}, state) + list_interpolation_to_algebra(entries, quotes, state, opener, quotes) + end + end + + defp quoted_to_algebra( + {{:., _, [:erlang, :binary_to_atom]}, _, [{:<<>>, _, entries}, :utf8]} = quoted, + context, + state + ) do + if interpolated?(entries) do + interpolation_to_algebra(entries, @double_quote, state, ":\"", @double_quote) + else + remote_to_algebra(quoted, context, state) + end + end + + # foo[bar] + defp quoted_to_algebra({{:., _, [Access, :get]}, meta, [target, arg]}, _context, state) do + {target_doc, state} = remote_target_to_algebra(target, state) + + {access_doc, state} = + if keyword?(arg) do + list_to_algebra(meta, arg, state) + else + list_to_algebra(meta, [arg], state) + end + + {concat(target_doc, access_doc), state} + end + + # %Foo{} + # %name{foo: 1} + # %name{bar | foo: 1} + defp quoted_to_algebra({:%, _, [name, {:%{}, meta, args}]}, _context, state) do + {name_doc, state} = quoted_to_algebra(name, :parens_arg, state) + map_to_algebra(meta, name_doc, args, state) + end + + # %{foo: 1} + # %{foo => bar} + # %{name | foo => bar} + defp quoted_to_algebra({:%{}, meta, args}, _context, state) do + map_to_algebra(meta, @empty, args, state) + end + + # {} + # {1, 2} + defp quoted_to_algebra({:{}, meta, args}, _context, state) do + tuple_to_algebra(meta, args, :flex_break, state) + end + + defp quoted_to_algebra({:__block__, meta, [{left, right}]}, _context, state) do + tuple_to_algebra(meta, [left, right], :flex_break, state) + end + + # (left -> right) + defp quoted_to_algebra({:__block__, _, [[{:->, _, _} | _] = clauses]}, _context, state) do + paren_fun_to_algebra(clauses, @max_line, @min_line, state) + end + + defp quoted_to_algebra({:__block__, meta, [list]}, _context, state) when is_list(list) do + case meta[:delimiter] do + ~s['''] -> + {opener, quotes} = get_charlist_quotes(:heredoc, state) + string = list |> List.to_string() |> escape_heredoc(quotes) + {opener |> concat(string) |> concat(quotes) |> force_unfit(), state} + + ~s['] -> + string = list |> List.to_string() + {opener, quotes} = get_charlist_quotes({:regular, [string]}, state) + string = escape_string(string, quotes) + {opener |> concat(string) |> concat(quotes), state} + + _other -> + list_to_algebra(meta, list, state) + end + end + + defp quoted_to_algebra({:__block__, meta, [string]}, _context, state) when is_binary(string) do + if meta[:delimiter] == ~s["""] do + string = escape_heredoc(string, ~s["""]) + + {@double_heredoc + |> concat(string) + |> concat(@double_heredoc) + |> color_doc(:string, state.inspect_opts) + |> force_unfit(), state} + else + string = escape_string(string, @double_quote) + + {@double_quote + |> concat(string) + |> concat(@double_quote) + |> color_doc(:string, state.inspect_opts), state} + end + end + + defp quoted_to_algebra({:__block__, meta, [atom]}, _context, state) when is_atom(atom) do + {atom_to_algebra(atom, meta, state.inspect_opts), state} + end + + defp quoted_to_algebra({:__block__, meta, [integer]}, _context, state) + when is_integer(integer) do + {Keyword.fetch!(meta, :token) |> integer_to_algebra(state.inspect_opts), state} + end + + defp quoted_to_algebra({:__block__, meta, [float]}, _context, state) when is_float(float) do + {Keyword.fetch!(meta, :token) |> float_to_algebra(state.inspect_opts), state} + end + + # (unquote_splicing(...)) + defp quoted_to_algebra( + {:__block__, _meta, [{:unquote_splicing, meta, [_] = args}]}, + context, + state + ) do + {doc, state} = local_to_algebra(:unquote_splicing, meta, args, context, state) + {wrap_in_parens(doc), state} + end + + defp quoted_to_algebra({:__block__, _meta, [arg]}, context, state) do + quoted_to_algebra(arg, context, state) + end + + defp quoted_to_algebra({:__block__, _meta, []}, _context, state) do + {color_doc("nil", nil, state.inspect_opts), state} + end + + defp quoted_to_algebra({:__block__, meta, args} = block, _context, state) when is_list(args) do + {block, state} = block_to_algebra(block, line(meta), closing_line(meta), state) + {surround("(", block, ")"), state} + end + + defp quoted_to_algebra({:__aliases__, _meta, [head | tail]}, context, state) do + {doc, state} = + if is_atom(head) do + {Atom.to_string(head), state} + else + quoted_to_algebra_with_parens_if_operator(head, context, state) + end + + {Enum.reduce(tail, doc, &concat(&2, "." <> Atom.to_string(&1))) + |> color_doc(:atom, state.inspect_opts), state} + end + + # &1 + # &local(&1) + # &local/1 + # &Mod.remote/1 + # & &1 + # & &1 + &2 + defp quoted_to_algebra({:&, _, [arg]}, context, state) do + capture_to_algebra(arg, context, state) + end + + defp quoted_to_algebra({:@, meta, [arg]}, context, state) do + module_attribute_to_algebra(meta, arg, context, state) + end + + # not(left in right) + # left not in right + defp quoted_to_algebra({:not, meta, [{:in, _, [left, right]}]}, context, state) do + binary_op_to_algebra(:in, "not in", meta, left, right, context, state) + end + + # disable migrate_call_parens_on_pipe within defmacro + defp quoted_to_algebra( + {atom, _, [{:|>, _, _}, _]} = ast, + context, + %{migrate_call_parens_on_pipe: true} = state + ) + when atom in [:defmacro, :defmacrop] do + quoted_to_algebra(ast, context, %{state | migrate_call_parens_on_pipe: false}) + end + + defp quoted_to_algebra( + {atom, _, [{:unless, _, _}, _]} = ast, + context, + %{migrate_unless: true} = state + ) + when atom in [:defmacro, :defmacrop] do + quoted_to_algebra(ast, context, %{state | migrate_unless: false}) + end + + # rewrite unless as if! + defp quoted_to_algebra( + {:unless, meta, [condition, block]}, + context, + %{migrate_unless: true} = state + ) do + quoted_to_algebra({:if, meta, [negate_condition(condition), block]}, context, state) + end + + # a |> b() |> unless(...) => a |> b() |> Kernel.!() |> unless(...) + defp quoted_to_algebra( + {:|>, meta1, [{:|>, _, _} = condition, {:unless, meta2, [block]}]}, + context, + %{migrate_unless: true} = state + ) do + negated_condition = {:|>, [], [condition, {{:., [], [Kernel, :!]}, [closing: []], []}]} + + quoted_to_algebra( + {:|>, meta1, [negated_condition, {:if, meta2, [block]}]}, + context, + state + ) + end + + # condition |> unless(...) => negated(condition) |> unless(...) + defp quoted_to_algebra( + {:|>, meta1, [condition, {:unless, meta2, [block]}]}, + context, + %{migrate_unless: true} = state + ) do + quoted_to_algebra( + {:|>, meta1, [negate_condition(condition), {:if, meta2, [block]}]}, + context, + state + ) + end + + # .. + defp quoted_to_algebra({:.., _meta, []}, context, state) do + if context in [:no_parens_arg, :no_parens_one_arg] do + {"(..)", state} + else + {"..", state} + end + end + + # ... + defp quoted_to_algebra({:..., _meta, []}, _context, state) do + {"...", state} + end + + # 1..2//3 + defp quoted_to_algebra({:..//, meta, [left, middle, right]}, context, state) do + quoted_to_algebra({:"//", meta, [{:.., meta, [left, middle]}, right]}, context, state) + end + + defp quoted_to_algebra({:fn, meta, [_ | _] = clauses}, _context, state) do + anon_fun_to_algebra(clauses, line(meta), closing_line(meta), state, eol?(meta, state)) + end + + defp quoted_to_algebra({fun, meta, args}, context, state) when is_atom(fun) and is_list(args) do + with :error <- maybe_sigil_to_algebra(fun, meta, args, state), + :error <- maybe_unary_op_to_algebra(fun, meta, args, context, state), + :error <- maybe_binary_op_to_algebra(fun, meta, args, context, state), + do: local_to_algebra(fun, meta, args, context, state) + end + + defp quoted_to_algebra({_, _, args} = quoted, context, state) when is_list(args) do + remote_to_algebra(quoted, context, state) + end + + # [keyword: :list] (inner part) + # %{:foo => :bar} (inner part) + defp quoted_to_algebra(list, context, state) when is_list(list) do + many_args_to_algebra(list, state, "ed_to_algebra(&1, context, &2)) + end + + # keyword: :list + # key => value + defp quoted_to_algebra({left_arg, right_arg}, context, state) do + {left, op, right, state} = + if keyword_key?(left_arg) do + {left, state} = + case left_arg do + {:__block__, _, [atom]} when is_atom(atom) -> + formatted = Macro.inspect_atom(:key, atom, escape: &escape_atom/2) + + {formatted + |> string() + |> color_doc(:atom, state.inspect_opts), state} + + {{:., _, [:erlang, :binary_to_atom]}, _, [{:<<>>, _, entries}, :utf8]} -> + interpolation_to_algebra(entries, @double_quote, state, "\"", "\":") + end + + {right, state} = quoted_to_algebra(right_arg, context, state) + {left, "", right, state} + else + {left, state} = quoted_to_algebra(left_arg, context, state) + {right, state} = quoted_to_algebra(right_arg, context, state) + left = wrap_in_parens_if_binary_operator(left, left_arg) + {left, " =>", right, state} + end + + doc = + concat( + group(left), + with_next_break_fits(next_break_fits?(right_arg, state), right, fn right -> + nest(glue(op, right), 2, :break) + end) + ) + + {doc, state} + end + + # #PID's and #Ref's may appear on regular AST + # Other foreign structures, such as maps and structs, + # may appear from Macro.to_string, so we stick a limit, + # although they won't be formatted accordingly. + defp quoted_to_algebra(unknown, _context, state) do + {inspect(unknown, printable_limit: :infinity), state} + end + + ## Blocks + + defp block_to_algebra([{:->, _, _} | _] = paren_fun, min_line, max_line, state) do + paren_fun_to_algebra(paren_fun, min_line, max_line, state) + end + + defp block_to_algebra({:__block__, _, []}, min_line, max_line, state) do + block_args_to_algebra([], min_line, max_line, state) + end + + defp block_to_algebra({:__block__, _, [_, _ | _] = args}, min_line, max_line, state) do + block_args_to_algebra(args, min_line, max_line, state) + end + + defp block_to_algebra(block, min_line, max_line, state) do + block_args_to_algebra([block], min_line, max_line, state) + end + + defp block_args_to_algebra(args, min_line, max_line, state) do + quoted_to_algebra = fn {kind, meta, _} = arg, _args, state -> + newlines = meta[:end_of_expression][:newlines] || 1 + {doc, state} = quoted_to_algebra(arg, :block, state) + {{doc, block_next_line(kind), newlines}, state} + end + + {args_docs, _comments?, state} = + quoted_to_algebra_with_comments(args, [], min_line, max_line, state, quoted_to_algebra) + + case args_docs do + [] -> {@empty, state} + [line] -> {line, state} + lines -> {lines |> Enum.reduce(&line(&2, &1)) |> force_unfit(), state} + end + end + + defp block_next_line(:@), do: @empty + defp block_next_line(_), do: break("") + + ## Operators + + defp maybe_unary_op_to_algebra(fun, meta, args, context, state) do + with [arg] <- args, + {_, _} <- Code.Identifier.unary_op(fun) do + unary_op_to_algebra(fun, meta, arg, context, state) + else + _ -> :error + end + end + + defp unary_op_to_algebra(op, _meta, arg, context, state) do + {doc, state} = quoted_to_algebra(arg, force_many_args_or_operand(context, :operand), state) + + # not and ! are nestable, all others are not. + doc = + case arg do + {^op, _, [_]} when op in [:!, :not] -> doc + _ -> wrap_in_parens_if_operator(doc, arg) + end + + # not requires a space unless the doc was wrapped in parens. + op_string = + if op == :not do + "not " + else + Atom.to_string(op) + end + + {color_doc(op_string, :operator, state.inspect_opts) |> concat(doc), state} + end + + defp maybe_binary_op_to_algebra(fun, meta, args, context, state) do + with [left, right] <- args, + {_, _} <- augmented_binary_op(fun) do + binary_op_to_algebra(fun, Atom.to_string(fun), meta, left, right, context, state) + else + _ -> :error + end + end + + # There are five kinds of operators. + # + # 1. no space binary operators, for example, 1..2 + # 2. no newline binary operators, for example, left in right + # 3. strict newlines before a left precedent operator, for example, foo |> bar |> baz + # 4. strict newlines before a right precedent operator, for example, foo when bar when baz + # 5. flex newlines after the operator, for example, foo ++ bar ++ baz + # + # Cases 1, 2 and 5 are handled fairly easily by relying on the + # operator precedence and making sure nesting is applied only once. + # + # Cases 3 and 4 are the complex ones, as it requires passing the + # strict or flex mode around. + defp binary_op_to_algebra(op, op_string, meta, left_arg, right_arg, context, state) do + %{operand_nesting: nesting} = state + binary_op_to_algebra(op, op_string, meta, left_arg, right_arg, context, state, nesting) + end + + defp binary_op_to_algebra(op, op_string, meta, left_arg, right_arg, context, state, _nesting) + when op in @right_new_line_before_binary_operators do + op_info = augmented_binary_op(op) + op_string = op_string <> " " + left_context = left_op_context(context) + right_context = right_op_context(context) + + min_line = + case left_arg do + {_, left_meta, _} -> line(left_meta) + _ -> line(meta) + end + + {operands, max_line} = + unwrap_right(right_arg, op, meta, right_context, [{{:root, left_context}, left_arg}]) + + fun = fn + {{:root, context}, arg}, _args, state -> + {doc, state} = binary_operand_to_algebra(arg, context, state, op, op_info, :left, 2) + {{doc, @empty, 1}, state} + + {{kind, context}, arg}, _args, state -> + {doc, state} = binary_operand_to_algebra(arg, context, state, op, op_info, kind, 0) + doc = doc |> nest_by_length(op_string) |> force_keyword(arg) + {{concat(op_string, doc), @empty, 1}, state} + end + + {doc, state} = + operand_to_algebra_with_comments(operands, meta, min_line, max_line, context, state, fun) + + if keyword?(right_arg) and context in [:parens_arg, :no_parens_arg] do + {wrap_in_parens(doc), state} + else + {doc, state} + end + end + + defp binary_op_to_algebra(op, _, meta, left_arg, right_arg, context, state, _nesting) + when op in @pipeline_operators do + op_info = augmented_binary_op(op) + left_context = left_op_context(context) + right_context = right_op_context(context) + max_line = line(meta) + + {pipes, min_line} = + unwrap_pipes(left_arg, meta, left_context, [{{op, right_context}, right_arg}]) + + fun = fn + {{:root, context}, arg}, _args, state -> + {doc, state} = binary_operand_to_algebra(arg, context, state, op, op_info, :left, 2) + {{doc, @empty, 1}, state} + + {{op, context}, arg}, _args, state -> + op_info = augmented_binary_op(op) + op_string = Atom.to_string(op) <> " " + {doc, state} = binary_operand_to_algebra(arg, context, state, op, op_info, :right, 0) + {{concat(op_string, doc), @empty, 1}, state} + end + + operand_to_algebra_with_comments(pipes, meta, min_line, max_line, context, state, fun) + end + + defp binary_op_to_algebra(op, op_string, meta, left_arg, right_arg, context, state, nesting) do + op_info = augmented_binary_op(op) + left_context = left_op_context(context) + right_context = right_op_context(context) + + {left, state} = + binary_operand_to_algebra(left_arg, left_context, state, op, op_info, :left, 2) + + {right, state} = + binary_operand_to_algebra(right_arg, right_context, state, op, op_info, :right, 0) + + {op_string, right} = + cond do + op in @no_space_binary_operators -> + {op_string, group(right)} + + op in @no_newline_binary_operators -> + {" " <> op_string <> " ", group(right)} + + true -> + eol? = eol?(meta, state) + + next_break_fits? = + op in @next_break_fits_operators and next_break_fits?(right_arg, state) and not eol? + + {" " <> op_string, + with_next_break_fits(next_break_fits?, right, fn right -> + right = nest(concat(break(), right), nesting, :break) + if eol?, do: force_unfit(right), else: right + end)} + end + + op_doc = color_doc(op_string, :operator, state.inspect_opts) + doc = concat(concat(group(left), op_doc), group(right)) + {doc, state} + end + + # TODO: We can remove this workaround once we remove + # ?rearrange_uop from the parser on v2.0. + # (! left) in right + # (not left) in right + defp binary_operand_to_algebra( + {:__block__, _, [{op, meta, [arg]}]}, + context, + state, + :in, + _parent_info, + :left, + _nesting + ) + when op in [:not, :!] do + {doc, state} = unary_op_to_algebra(op, meta, arg, context, state) + {wrap_in_parens(doc), state} + end + + # |> var + # |> var() + defp binary_operand_to_algebra( + {var, meta, var_context}, + context, + %{migrate_call_parens_on_pipe: true} = state, + :|>, + _parent_info, + :right, + _nesting + ) + when is_atom(var) and is_atom(var_context) do + operand = {var, meta, []} + quoted_to_algebra(operand, context, state) + end + + # |> var.fun + # |> var.fun() + defp binary_operand_to_algebra( + {{:., _, [_, fun]} = call, meta, []}, + context, + %{migrate_call_parens_on_pipe: true} = state, + :|>, + _parent_info, + :right, + _nesting + ) + when is_atom(fun) do + meta = Keyword.put_new_lazy(meta, :closing, fn -> [line: meta[:line]] end) + quoted_to_algebra({call, meta, []}, context, state) + end + + defp binary_operand_to_algebra(operand, context, state, parent_op, parent_info, side, nesting) do + {parent_assoc, parent_prec} = parent_info + + with {op, meta, [left, right]} <- operand, + op_info = augmented_binary_op(op), + {_assoc, prec} <- op_info do + op_string = Atom.to_string(op) + + cond do + # If we have the same operator and it is in the correct side, + # we don't add parens unless it is explicitly required. + parent_assoc == side and op == parent_op and op not in @required_parens_even_when_parent -> + binary_op_to_algebra(op, op_string, meta, left, right, context, state, nesting) + + # If the operator requires parens (most of them do) or we are mixing logical operators + # or the precedence is inverted or it is in the wrong side, then we *need* parenthesis. + (parent_op in @required_parens_on_binary_operands and op not in @no_space_binary_operators) or + (op in @required_parens_logical_binary_operands and + parent_op in @required_parens_logical_binary_operands) or parent_prec > prec or + (parent_prec == prec and parent_assoc != side) -> + {operand, state} = + binary_op_to_algebra(op, op_string, meta, left, right, context, state, 2) + + {wrap_in_parens(operand), state} + + # Otherwise, we rely on precedence but also nest. + true -> + binary_op_to_algebra(op, op_string, meta, left, right, context, state, 2) + end + else + {:&, _, [arg]} + when not is_integer(arg) and side == :left + when not is_integer(arg) and parent_assoc == :left and parent_prec > @ampersand_prec -> + {doc, state} = quoted_to_algebra(operand, context, state) + {wrap_in_parens(doc), state} + + _ -> + quoted_to_algebra(operand, context, state) + end + end + + defp unwrap_pipes({op, meta, [left, right]}, _meta, context, acc) + when op in @pipeline_operators do + left_context = left_op_context(context) + right_context = right_op_context(context) + unwrap_pipes(left, meta, left_context, [{{op, right_context}, right} | acc]) + end + + defp unwrap_pipes(left, meta, context, acc) do + min_line = + case left do + {_, meta, _} -> line(meta) + _ -> line(meta) + end + + {[{{:root, context}, left} | acc], min_line} + end + + defp unwrap_right({op, meta, [left, right]}, op, _meta, context, acc) do + left_context = left_op_context(context) + right_context = right_op_context(context) + unwrap_right(right, op, meta, right_context, [{{:left, left_context}, left} | acc]) + end + + defp unwrap_right(right, _op, meta, context, acc) do + acc = [{{:right, context}, right} | acc] + {Enum.reverse(acc), line(meta)} + end + + defp operand_to_algebra_with_comments(operands, meta, min_line, max_line, context, state, fun) do + # If we are in a no_parens_one_arg expression, we actually cannot + # extract comments from the first operand, because it would rewrite: + # + # @spec function(x) :: + # # Comment + # any + # when x: any + # + # to: + # + # @spec # Comment + # function(x) :: + # any + # when x: any + # + # Instead we get: + # + # @spec function(x) :: + # any + # # Comment + # when x: any + # + # Which may look counter-intuitive but it actually makes sense, + # as the closest possible location for the comment is the when + # operator. + {operands, acc, state} = + if context == :no_parens_one_arg do + [operand | operands] = operands + {doc_triplet, state} = fun.(operand, :unused, state) + {operands, [doc_triplet], state} + else + {operands, [], state} + end + + {docs, comments?, state} = + quoted_to_algebra_with_comments(operands, acc, min_line, max_line, state, fun) + + if comments? or eol?(meta, state) do + {docs |> Enum.reduce(&line(&2, &1)) |> force_unfit(), state} + else + {docs |> Enum.reduce(&glue(&2, &1)), state} + end + end + + ## Module attributes + + # @Foo + # @Foo.Bar + defp module_attribute_to_algebra(_meta, {:__aliases__, _, [_, _ | _]} = quoted, _context, state) do + {doc, state} = quoted_to_algebra(quoted, :parens_arg, state) + {concat(concat("@(", doc), ")"), state} + end + + # @foo bar + # @foo(bar) + defp module_attribute_to_algebra(meta, {name, call_meta, [_] = args} = expr, context, state) + when is_atom(name) and name not in [:__block__, :__aliases__] do + if Macro.classify_atom(name) == :identifier do + {{call_doc, state}, wrap_in_parens?} = + call_args_to_algebra(args, call_meta, context, :skip_unless_many_args, false, state) + + doc = + "@#{name}" + |> string() + |> concat(call_doc) + + doc = if wrap_in_parens?, do: wrap_in_parens(doc), else: doc + {doc, state} + else + unary_op_to_algebra(:@, meta, expr, context, state) + end + end + + # @foo + # @(foo.bar()) + defp module_attribute_to_algebra(meta, quoted, context, state) do + unary_op_to_algebra(:@, meta, quoted, context, state) + end + + ## Capture operator + + defp capture_to_algebra(integer, _context, state) when is_integer(integer) do + {"&" <> Integer.to_string(integer), state} + end + + defp capture_to_algebra(arg, context, state) do + {doc, state} = capture_target_to_algebra(arg, context, state) + + case format_to_string(doc) do + <<"&", _::binary>> -> {concat("& ", doc), state} + <> when int in ?0..?9 -> {concat("& ", doc), state} + _ -> {concat("&", doc), state} + end + end + + defp capture_target_to_algebra( + {:/, _, [{{:., _, [target, fun]}, _, []}, {:__block__, _, [arity]}]}, + _context, + state + ) + when is_atom(fun) and is_integer(arity) do + {target_doc, state} = remote_target_to_algebra(target, state) + fun = Macro.inspect_atom(:remote_call, fun, escape: &escape_atom/2) + {target_doc |> nest(1) |> concat(string(".#{fun}/#{arity}")), state} + end + + defp capture_target_to_algebra( + {:/, _, [{name, _, var_context}, {:__block__, _, [arity]}]}, + _context, + state + ) + when is_atom(name) and is_atom(var_context) and is_integer(arity) do + {string("#{name}/#{arity}"), state} + end + + defp capture_target_to_algebra(arg, context, state) do + {doc, state} = quoted_to_algebra(arg, context, state) + {wrap_in_parens_if_operator(doc, arg), state} + end + + ## Calls (local, remote and anonymous) + + # expression.{arguments} + defp remote_to_algebra({{:., _, [target, :{}]}, meta, args}, _context, state) do + {target_doc, state} = remote_target_to_algebra(target, state) + {call_doc, state} = tuple_to_algebra(meta, args, :break, state) + {concat(concat(target_doc, "."), call_doc), state} + end + + # expression.(arguments) + defp remote_to_algebra({{:., _, [target]}, meta, args}, context, state) do + {target_doc, state} = remote_target_to_algebra(target, state) + + {{call_doc, state}, wrap_in_parens?} = + call_args_to_algebra(args, meta, context, :skip_if_do_end, true, state) + + doc = concat(concat(target_doc, "."), call_doc) + doc = if wrap_in_parens?, do: wrap_in_parens(doc), else: doc + {doc, state} + end + + # Mod.function() + # var.function + # expression.function(arguments) + defp remote_to_algebra({{:., _, [target, fun]}, meta, args}, context, state) + when is_atom(fun) do + {target_doc, state} = remote_target_to_algebra(target, state) + + fun_doc = + Macro.inspect_atom(:remote_call, fun, escape: &escape_atom/2) + |> string() + |> color_doc(:call, state.inspect_opts) + + remote_doc = target_doc |> concat(".") |> concat(fun_doc) + + if args == [] and not remote_target_is_a_module?(target) and not meta?(meta, :closing) do + {remote_doc, state} + else + {{call_doc, state}, wrap_in_parens?} = + call_args_to_algebra(args, meta, context, :skip_if_do_end, true, state) + + doc = concat(remote_doc, call_doc) + doc = if wrap_in_parens?, do: wrap_in_parens(doc), else: doc + {doc, state} + end + end + + # call(call)(arguments) + defp remote_to_algebra({target, meta, args}, context, state) do + {target_doc, state} = quoted_to_algebra(target, :no_parens_arg, state) + + {{call_doc, state}, wrap_in_parens?} = + call_args_to_algebra(args, meta, context, :required, true, state) + + doc = concat(target_doc, call_doc) + doc = if wrap_in_parens?, do: wrap_in_parens(doc), else: doc + {doc, state} + end + + defp remote_target_is_a_module?(target) do + case target do + {:__MODULE__, _, context} when is_atom(context) -> true + {:__block__, _, [atom]} when is_atom(atom) -> true + {:__aliases__, _, _} -> true + _ -> false + end + end + + defp remote_target_to_algebra({:fn, _, [_ | _]} = quoted, state) do + # This change is not semantically required but for beautification. + {doc, state} = quoted_to_algebra(quoted, :no_parens_arg, state) + {wrap_in_parens(doc), state} + end + + defp remote_target_to_algebra(quoted, state) do + quoted_to_algebra_with_parens_if_operator(quoted, :no_parens_arg, state) + end + + # function(arguments) + defp local_to_algebra(fun, meta, args, context, state) when is_atom(fun) do + skip_parens = + cond do + meta?(meta, :closing) -> + :skip_if_only_do_end + + local_without_parens?(fun, length(args), state.locals_without_parens) -> + :skip_unless_many_args + + true -> + :skip_if_do_end + end + + {{call_doc, state}, wrap_in_parens?} = + call_args_to_algebra(args, meta, context, skip_parens, true, state) + + doc = + fun + |> Atom.to_string() + |> string() + |> color_doc(:call, state.inspect_opts) + |> concat(call_doc) + + doc = if wrap_in_parens?, do: wrap_in_parens(doc), else: doc + {doc, state} + end + + # parens may be one of: + # + # * :skip_unless_many_args - skips parens unless we are the argument context + # * :skip_if_only_do_end - skip parens if we are do-end and the only arg + # * :skip_if_do_end - skip parens if we are do-end + # * :required - never skip parens + # + defp call_args_to_algebra([], meta, _context, _parens, _list_to_keyword?, state) do + {args_doc, _join, state} = + args_to_algebra_with_comments([], meta, false, :none, :break, state, &{&1, &2}) + + {{surround("(", args_doc, ")"), state}, false} + end + + defp call_args_to_algebra(args, meta, context, parens, list_to_keyword?, state) do + {rest, last} = split_last(args) + + if blocks = do_end_blocks(meta, last, state) do + {call_doc, state} = + case rest do + [] when parens == :required -> + {"() do", state} + + [] -> + {" do", state} + + _ -> + no_parens? = parens not in [:required, :skip_if_only_do_end] + call_args_to_algebra_no_blocks(meta, rest, no_parens?, list_to_keyword?, " do", state) + end + + {blocks_doc, state} = do_end_blocks_to_algebra(blocks, state) + call_doc = call_doc |> concat(blocks_doc) |> line("end") |> force_unfit() + {{call_doc, state}, context in [:no_parens_arg, :no_parens_one_arg]} + else + no_parens? = + parens == :skip_unless_many_args and + context in [:block, :operand, :no_parens_one_arg, :parens_one_arg] + + res = + call_args_to_algebra_no_blocks(meta, args, no_parens?, list_to_keyword?, @empty, state) + + {res, false} + end + end + + defp call_args_to_algebra_no_blocks(meta, args, skip_parens?, list_to_keyword?, extra, state) do + {left, right} = split_last(args) + {keyword?, right} = last_arg_to_keyword(right, list_to_keyword?, skip_parens?, state.comments) + + context = + if left == [] and not keyword? do + if skip_parens?, do: :no_parens_one_arg, else: :parens_one_arg + else + if skip_parens?, do: :no_parens_arg, else: :parens_arg + end + + args = if keyword?, do: left ++ right, else: left ++ [right] + many_eol? = match?([_, _ | _], args) and eol?(meta, state) + no_generators? = no_generators?(args) + to_algebra_fun = "ed_to_algebra(&1, context, &2) + + {args_doc, next_break_fits?, state} = + if left != [] and keyword? and no_generators? do + join = if force_args?(left) or many_eol?, do: :line, else: :break + + {left_doc, _join, state} = + args_to_algebra_with_comments( + left, + Keyword.delete(meta, :closing), + skip_parens?, + :force_comma, + join, + state, + to_algebra_fun + ) + + join = if force_args?(right) or force_args?(args) or many_eol?, do: :line, else: :break + + {right_doc, _join, state} = + args_to_algebra_with_comments(right, meta, false, :none, join, state, to_algebra_fun) + + right_doc = apply(Inspect.Algebra, join, []) |> concat(right_doc) + + args_doc = + if skip_parens? do + left_doc + |> concat(group(right_doc, :optimistic)) + |> nest(:cursor, :break) + else + right_doc = + right_doc + |> nest(2, :break) + |> concat(break("")) + |> concat(")") + |> group(:optimistic) + + concat(nest(left_doc, 2, :break), right_doc) + end + + {args_doc, true, state} + else + join = if force_args?(args) or many_eol?, do: :line, else: :break + next_break_fits? = join == :break and next_break_fits?(right, state) + last_arg_mode = if next_break_fits?, do: :next_break_fits, else: :none + + {args_doc, _join, state} = + args_to_algebra_with_comments( + args, + meta, + skip_parens?, + last_arg_mode, + join, + state, + to_algebra_fun + ) + + # If we have a single argument, then we won't have an option to break + # before the "extra" part, so we ungroup it and build it later. + args_doc = ungroup_if_group(args_doc) + + args_doc = + if skip_parens? do + nest(args_doc, :cursor, :break) + else + nest(args_doc, 2, :break) |> concat(break("")) |> concat(")") + end + + {args_doc, next_break_fits?, state} + end + + doc = + cond do + left != [] and keyword? and skip_parens? and no_generators? -> + " " + |> concat(args_doc) + |> nest(2) + |> concat(extra) + + skip_parens? -> + " " + |> concat(args_doc) + |> concat(extra) + + true -> + "(" + |> concat(break("")) + |> nest(2, :break) + |> concat(args_doc) + |> concat(extra) + end + + if next_break_fits? do + {group(doc, :pessimistic), state} + else + {group(doc), state} + end + end + + defp no_generators?(args) do + not Enum.any?(args, &match?({:<-, _, [_, _]}, &1)) + end + + defp do_end_blocks(meta, [{{:__block__, _, [:do]}, _} | rest] = blocks, state) do + if meta?(meta, :do) or can_force_do_end_blocks?(rest, state) do + blocks + |> Enum.map(fn {{:__block__, meta, [key]}, value} -> {key, line(meta), value} end) + |> do_end_blocks_with_range(end_line(meta)) + end + end + + defp do_end_blocks(_, _, _), do: nil + + defp can_force_do_end_blocks?(rest, state) do + state.force_do_end_blocks and + Enum.all?(rest, fn {{:__block__, _, [key]}, _} -> key in @do_end_keywords end) + end + + defp do_end_blocks_with_range([{key1, line1, value1}, {_, line2, _} = h | t], end_line) do + [{key1, line1, line2, value1} | do_end_blocks_with_range([h | t], end_line)] + end + + defp do_end_blocks_with_range([{key, line, value}], end_line) do + [{key, line, end_line, value}] + end + + defp do_end_blocks_to_algebra([{:do, line, end_line, value} | blocks], state) do + {acc, state} = do_end_block_to_algebra(@empty, line, end_line, value, state) + + Enum.reduce(blocks, {acc, state}, fn {key, line, end_line, value}, {acc, state} -> + {doc, state} = do_end_block_to_algebra(Atom.to_string(key), line, end_line, value, state) + {line(acc, doc), state} + end) + end + + defp do_end_block_to_algebra(key_doc, line, end_line, value, state) do + case clauses_to_algebra(value, line, end_line, state) do + {@empty, state} -> {key_doc, state} + {value_doc, state} -> {key_doc |> line(value_doc) |> nest(2), state} + end + end + + ## Interpolation + + defp list_interpolated?(entries) do + Enum.all?(entries, fn + {{:., _, [Kernel, :to_string]}, _, [_]} -> true + entry when is_binary(entry) -> true + _ -> false + end) + end + + defp interpolated?(entries) do + Enum.all?(entries, fn + {:"::", _, [{{:., _, [Kernel, :to_string]}, _, [_]}, {:binary, _, _}]} -> true + entry when is_binary(entry) -> true + _ -> false + end) + end + + defp prepend_heredoc_line([entry | entries]) when is_binary(entry) do + ["\n" <> entry | entries] + end + + defp prepend_heredoc_line(entries) do + ["\n" | entries] + end + + defp list_interpolation_to_algebra([entry | entries], escape, state, acc, last) + when is_binary(entry) do + acc = concat(acc, escape_string(entry, escape)) + list_interpolation_to_algebra(entries, escape, state, acc, last) + end + + defp list_interpolation_to_algebra([entry | entries], escape, state, acc, last) do + {{:., _, [Kernel, :to_string]}, _meta, [quoted]} = entry + {doc, state} = interpolation_to_algebra(quoted, state) + list_interpolation_to_algebra(entries, escape, state, concat(acc, doc), last) + end + + defp list_interpolation_to_algebra([], _escape, state, acc, last) do + {concat(acc, last), state} + end + + defp interpolation_to_algebra([entry | entries], escape, state, acc, last) + when is_binary(entry) do + acc = concat(acc, escape_string(entry, escape)) + interpolation_to_algebra(entries, escape, state, acc, last) + end + + defp interpolation_to_algebra([entry | entries], escape, state, acc, last) do + {:"::", _, [{{:., _, [Kernel, :to_string]}, _meta, [quoted]}, {:binary, _, _}]} = entry + {doc, state} = interpolation_to_algebra(quoted, state) + interpolation_to_algebra(entries, escape, state, concat(acc, doc), last) + end + + defp interpolation_to_algebra([], _escape, state, acc, last) do + {concat(acc, last), state} + end + + defp interpolation_to_algebra(quoted, %{skip_eol: skip_eol} = state) do + {doc, state} = block_to_algebra(quoted, @max_line, @min_line, %{state | skip_eol: true}) + {no_limit(surround("\#{", doc, "}")), %{state | skip_eol: skip_eol}} + end + + ## Sigils + + defp maybe_sigil_to_algebra(fun, meta, args, state) do + with <<"sigil_", name::binary>> <- Atom.to_string(fun), + [{:<<>>, _, entries}, modifiers] when is_list(modifiers) <- args, + opening_delimiter when not is_nil(opening_delimiter) <- meta[:delimiter] do + doc = <> + + entries = + case Map.fetch(state.sigils, String.to_charlist(name)) do + {:ok, callback} -> + metadata = [ + file: state.file, + line: meta[:line], + sigil: String.to_atom(name), + modifiers: modifiers, + opening_delimiter: opening_delimiter + ] + + case callback.(hd(entries), metadata) do + iodata when is_binary(iodata) or is_list(iodata) -> + [IO.iodata_to_binary(iodata)] + + other -> + raise ArgumentError, + "expected sigil callback to return iodata, got: #{inspect(other)}" + end + + :error -> + entries + end + + if opening_delimiter in [@double_heredoc, @single_heredoc] do + closing_delimiter = concat(opening_delimiter, List.to_string(modifiers)) + + {doc, state} = + entries + |> prepend_heredoc_line() + |> interpolation_to_algebra(opening_delimiter, state, doc, closing_delimiter) + + {force_unfit(doc), state} + else + escape = closing_sigil_delimiter(opening_delimiter) + closing_delimiter = concat(escape, List.to_string(modifiers)) + interpolation_to_algebra(entries, escape, state, doc, closing_delimiter) + end + else + _ -> + :error + end + end + + defp closing_sigil_delimiter("("), do: ")" + defp closing_sigil_delimiter("["), do: "]" + defp closing_sigil_delimiter("{"), do: "}" + defp closing_sigil_delimiter("<"), do: ">" + defp closing_sigil_delimiter(other) when other in ["\"", "'", "|", "/"], do: other + + ## Bitstrings + + defp bitstring_to_algebra(meta, args, state) do + last = length(args) - 1 + join = if eol?(meta, state), do: :line, else: :flex_break + to_algebra_fun = &bitstring_segment_to_algebra(&1, &2, last) + + {args_doc, join, state} = + args + |> Enum.with_index() + |> args_to_algebra_with_comments(meta, false, :none, join, state, to_algebra_fun) + + if join == :flex_break do + {"<<" |> concat(args_doc) |> nest(2) |> concat(">>") |> group(), state} + else + {surround("<<", args_doc, ">>"), state} + end + end + + defp bitstring_segment_to_algebra({{:<-, meta, [left, right]}, i}, state, last) do + left = {{:special, :bitstring_segment}, meta, [left, last]} + {doc, state} = quoted_to_algebra({:<-, meta, [left, right]}, :parens_arg, state) + {bitstring_wrap_parens(doc, i, last), state} + end + + defp bitstring_segment_to_algebra({{:"::", _, [segment, spec]}, i}, state, last) do + {doc, state} = quoted_to_algebra(segment, :parens_arg, state) + + {spec, state} = + bitstring_spec_to_algebra(spec, state, state.migrate_bitstring_modifiers, :"::") + + spec = wrap_in_parens_if_inspected_atom(spec) + spec = if i == last, do: bitstring_wrap_parens(spec, i, last), else: spec + + doc = + doc + |> bitstring_wrap_parens(i, -1) + |> concat("::") + |> concat(spec) + + {doc, state} + end + + defp bitstring_segment_to_algebra({segment, i}, state, last) do + {doc, state} = quoted_to_algebra(segment, :parens_arg, state) + {bitstring_wrap_parens(doc, i, last), state} + end + + defp bitstring_spec_to_algebra({op, _, [left, right]}, state, normalize_modifiers, paren_op) + when op in [:-, :*] do + normalize_modifiers = normalize_modifiers && op != :* + {left, state} = bitstring_spec_to_algebra(left, state, normalize_modifiers, op) + {right, state} = bitstring_spec_element_to_algebra(right, state, normalize_modifiers) + doc = concat(concat(left, Atom.to_string(op)), right) + doc = if paren_op == :*, do: wrap_in_parens(doc), else: doc + {doc, state} + end + + defp bitstring_spec_to_algebra(spec, state, normalize_modifiers, _paren_op) do + bitstring_spec_element_to_algebra(spec, state, normalize_modifiers) + end + + defp bitstring_spec_element_to_algebra( + {atom, meta, empty_args}, + state, + _normalize_modifiers = true + ) + when is_atom(atom) and empty_args in [nil, []] do + empty_args = bitstring_spec_normalize_empty_args(atom) + quoted_to_algebra_with_parens_if_operator({atom, meta, empty_args}, :parens_arg, state) + end + + defp bitstring_spec_element_to_algebra(spec_element, state, _normalize_modifiers) do + quoted_to_algebra_with_parens_if_operator(spec_element, :parens_arg, state) + end + + defp bitstring_spec_normalize_empty_args(:_), do: nil + + defp bitstring_spec_normalize_empty_args(atom) do + case :elixir_bitstring.validate_spec(atom, nil) do + :none -> [] + _ -> nil + end + end + + defp bitstring_wrap_parens(doc, i, last) when i == 0 or i == last do + string = format_to_string(doc) + + if (i == 0 and String.starts_with?(string, ["~", "<<"])) or + (i == last and String.ends_with?(string, [">>"])) do + wrap_in_parens(doc) + else + doc + end + end + + defp bitstring_wrap_parens(doc, _, _), do: doc + + ## Literals + + defp list_to_algebra(meta, args, state) do + join = if eol?(meta, state), do: :line, else: :break + fun = "ed_to_algebra(&1, :parens_arg, &2) + + {args_doc, _join, state} = + args_to_algebra_with_comments(args, meta, false, :none, join, state, fun) + + left_bracket = color_doc("[", :list, state.inspect_opts) + right_bracket = color_doc("]", :list, state.inspect_opts) + + {surround(left_bracket, args_doc, right_bracket), state} + end + + defp map_to_algebra(meta, name_doc, [{:|, _, [left, right]}], state) do + join = if eol?(meta, state), do: :line, else: :break + fun = "ed_to_algebra(&1, :parens_arg, &2) + {left_doc, state} = fun.(left, state) + + {right_doc, _join, state} = + args_to_algebra_with_comments(right, meta, false, :none, join, state, fun) + + args_doc = + left_doc + |> wrap_in_parens_if_binary_operator(left) + |> glue(concat("| ", nest(right_doc, 2))) + + do_map_to_algebra(name_doc, args_doc, state) + end + + defp map_to_algebra(meta, name_doc, args, state) do + join = if eol?(meta, state), do: :line, else: :break + fun = "ed_to_algebra(&1, :parens_arg, &2) + + {args_doc, _join, state} = + args_to_algebra_with_comments(args, meta, false, :none, join, state, fun) + + do_map_to_algebra(name_doc, args_doc, state) + end + + defp do_map_to_algebra(name_doc, args_doc, state) do + name_doc = "%" |> concat(name_doc) |> concat("{") |> color_doc(:map, state.inspect_opts) + {surround(name_doc, args_doc, color_doc("}", :map, state.inspect_opts)), state} + end + + defp tuple_to_algebra(meta, args, join, state) do + join = if eol?(meta, state), do: :line, else: join + fun = "ed_to_algebra(&1, :parens_arg, &2) + + {args_doc, join, state} = + args_to_algebra_with_comments(args, meta, false, :none, join, state, fun) + + left_bracket = color_doc("{", :tuple, state.inspect_opts) + right_bracket = color_doc("}", :tuple, state.inspect_opts) + + if join == :flex_break do + {left_bracket |> concat(args_doc) |> nest(1) |> concat(right_bracket) |> group(), state} + else + {surround(left_bracket, args_doc, right_bracket), state} + end + end + + defp atom_to_algebra(atom, _, inspect_opts) when atom in [true, false] do + Atom.to_string(atom) |> color_doc(:boolean, inspect_opts) + end + + defp atom_to_algebra(nil, _, inspect_opts) do + Atom.to_string(nil) |> color_doc(nil, inspect_opts) + end + + defp atom_to_algebra(:\\, meta, inspect_opts) do + # Since we parse strings without unescaping, the atoms + # :\\ and :"\\" have the same representation, so we need + # to check the delimiter and handle them accordingly. + string = + case Keyword.get(meta, :delimiter) do + "\"" -> ":\"\\\\\"" + _ -> ":\\\\" + end + + string(string) |> color_doc(:atom, inspect_opts) + end + + defp atom_to_algebra(atom, _, inspect_opts) do + string = Atom.to_string(atom) + + iodata = + if Macro.classify_atom(atom) in [:unquoted, :identifier] do + [?:, string] + else + [?:, ?", String.replace(string, "\"", "\\\""), ?"] + end + + iodata |> IO.iodata_to_binary() |> string() |> color_doc(:atom, inspect_opts) + end + + defp integer_to_algebra(text, inspect_otps) do + case text do + <> -> + "0x" <> String.upcase(rest) + + <> = digits when base in [?b, ?o] -> + digits + + <> = char -> + char + + decimal -> + insert_underscores(decimal) + end + |> color_doc(:number, inspect_otps) + end + + defp float_to_algebra(text, inspect_otps) do + [int_part, decimal_part] = :binary.split(text, ".") + decimal_part = String.downcase(decimal_part) + + string = insert_underscores(int_part) <> "." <> decimal_part + color_doc(string, :number, inspect_otps) + end + + defp insert_underscores("-" <> digits) do + "-" <> insert_underscores(digits) + end + + defp insert_underscores(digits) do + byte_size = byte_size(digits) + + cond do + digits =~ "_" -> + digits + + byte_size >= 6 -> + offset = rem(byte_size, 3) + {prefix, rest} = String.split_at(digits, offset) + do_insert_underscores(prefix, rest) + + true -> + digits + end + end + + defp do_insert_underscores(acc, ""), do: acc + + defp do_insert_underscores("", <>), + do: do_insert_underscores(next, rest) + + defp do_insert_underscores(acc, <>), + do: do_insert_underscores(<>, rest) + + defp escape_heredoc(string, escape) do + string = String.replace(string, escape, "\\" <> escape) + heredoc_to_algebra(["" | String.split(string, "\n")]) + end + + defp escape_string(string, <<_, _, _>> = escape) do + string = String.replace(string, escape, "\\" <> escape) + heredoc_to_algebra(String.split(string, "\n")) + end + + defp escape_string(string, escape) when is_binary(escape) do + string + |> String.replace(escape, "\\" <> escape) + |> String.split("\n") + |> Enum.reverse() + |> Enum.map(&string/1) + |> Enum.reduce(&concat(&1, concat(nest(line(), :reset), &2))) + end + + defp heredoc_to_algebra([string]) do + string(string) + end + + defp heredoc_to_algebra(["" | rest]) do + rest + |> heredoc_line() + |> concat(heredoc_to_algebra(rest)) + end + + defp heredoc_to_algebra([string | rest]) do + string + |> string() + |> concat(heredoc_line(rest)) + |> concat(heredoc_to_algebra(rest)) + end + + defp heredoc_line(["", _ | _]), do: nest(line(), :reset) + defp heredoc_line(["\r", _ | _]), do: nest(line(), :reset) + defp heredoc_line(_), do: line() + + defp args_to_algebra_with_comments(args, meta, skip_parens?, last_arg_mode, join, state, fun) do + min_line = line(meta) + max_line = closing_line(meta) + + arg_to_algebra = fn arg, args, state -> + {doc, state} = fun.(arg, state) + + doc = + case args do + [_ | _] -> + concat_to_last_group(doc, ",") + + [] when last_arg_mode == :force_comma -> + concat_to_last_group(doc, ",") + + [] when last_arg_mode == :next_break_fits -> + doc |> ungroup_if_group() |> group(:optimistic) + + [] when last_arg_mode == :none -> + doc + end + + {{doc, @empty, 1}, state} + end + + # If skipping parens, we cannot extract the comments of the first + # argument as there is no place to move them to, so we handle it now. + {args, acc, state} = + case args do + [head | tail] when skip_parens? -> + {doc_triplet, state} = arg_to_algebra.(head, tail, state) + {tail, [doc_triplet], state} + + _ -> + {args, [], state} + end + + {args_docs, comments?, state} = + quoted_to_algebra_with_comments(args, acc, min_line, max_line, state, arg_to_algebra) + + cond do + args_docs == [] -> + {@empty, :empty, state} + + join == :line or comments? -> + {args_docs |> Enum.reduce(&line(&2, &1)) |> force_unfit(), :line, state} + + join == :break -> + {args_docs |> Enum.reduce(&glue(&2, &1)), :break, state} + + join == :flex_break -> + {args_docs |> Enum.reduce(&flex_glue(&2, &1)), :flex_break, state} + end + end + + ## Anonymous functions + + # fn -> block end + defp anon_fun_to_algebra( + [{:->, meta, [[], body]}] = clauses, + _min_line, + max_line, + state, + _multi_clauses_style + ) do + min_line = line(meta) + {body_doc, state} = block_to_algebra(body, min_line, max_line, state) + break_or_line = clause_break_or_line(clauses, state) + + doc = + "fn ->" + |> concat(break_or_line) + |> concat(body_doc) + |> nest(2) + |> concat(break_or_line) + |> concat("end") + |> maybe_force_clauses(clauses, state) + |> group() + + {doc, state} + end + + # fn x -> y end + # fn x -> + # y + # end + defp anon_fun_to_algebra( + [{:->, meta, [args, body]}] = clauses, + _min_line, + max_line, + state, + false = _multi_clauses_style + ) do + min_line = line(meta) + {args_doc, state} = clause_args_to_algebra(args, min_line, state) + {body_doc, state} = block_to_algebra(body, min_line, max_line, state) + + head = + args_doc + |> ungroup_if_group() + |> concat(" ->") + |> nest(:cursor) + |> group() + + break_or_line = clause_break_or_line(clauses, state) + + doc = + "fn " + |> concat(head) + |> concat(break_or_line) + |> concat(body_doc) + |> nest(2) + |> concat(break_or_line) + |> concat("end") + |> maybe_force_clauses(clauses, state) + |> group() + + {doc, state} + end + + # fn + # args1 -> + # block1 + # args2 -> + # block2 + # end + defp anon_fun_to_algebra(clauses, min_line, max_line, state, _multi_clauses_style) do + {clauses_doc, state} = clauses_to_algebra(clauses, min_line, max_line, state) + {"fn" |> line(clauses_doc) |> nest(2) |> line("end") |> force_unfit(), state} + end + + ## Type functions + + # (-> block) + defp paren_fun_to_algebra([{:->, meta, [[], body]}] = clauses, _min_line, max_line, state) do + min_line = line(meta) + {body_doc, state} = block_to_algebra(body, min_line, max_line, state) + + doc = + "(-> " + |> concat(nest(body_doc, :cursor)) + |> concat(")") + |> maybe_force_clauses(clauses, state) + |> group() + + {doc, state} + end + + # (x -> y) + # (x -> + # y) + defp paren_fun_to_algebra([{:->, meta, [args, body]}] = clauses, _min_line, max_line, state) do + min_line = line(meta) + {args_doc, state} = clause_args_to_algebra(args, min_line, state) + {body_doc, state} = block_to_algebra(body, min_line, max_line, state) + break_or_line = clause_break_or_line(clauses, state) + + doc = + args_doc + |> ungroup_if_group() + |> concat(" ->") + |> group() + |> concat(break_or_line |> concat(body_doc) |> nest(2)) + |> wrap_in_parens() + |> maybe_force_clauses(clauses, state) + |> group() + + {doc, state} + end + + # ( + # args1 -> + # block1 + # args2 -> + # block2 + # ) + defp paren_fun_to_algebra(clauses, min_line, max_line, state) do + {clauses_doc, state} = clauses_to_algebra(clauses, min_line, max_line, state) + {"(" |> line(clauses_doc) |> nest(2) |> line(")") |> force_unfit(), state} + end + + ## Clauses + + defp multi_line_clauses?(clauses, state) do + Enum.any?(clauses, fn {:->, meta, [_, block]} -> + eol?(meta, state) or multi_line_block?(block) + end) + end + + defp multi_line_block?({:__block__, _, [_, _ | _]}), do: true + defp multi_line_block?(_), do: false + + defp clause_break_or_line(clauses, state) do + if multi_line_clauses?(clauses, state), do: line(), else: break() + end + + defp maybe_force_clauses(doc, clauses, state) do + if multi_line_clauses?(clauses, state), do: force_unfit(doc), else: doc + end + + defp clauses_to_algebra([{:->, _, _} | _] = clauses, min_line, max_line, state) do + [clause | clauses] = add_max_line_to_last_clause(clauses, max_line) + {clause_doc, state} = clause_to_algebra(clause, min_line, state) + + {clauses_doc, state} = + Enum.reduce(clauses, {clause_doc, state}, fn clause, {doc_acc, state_acc} -> + {clause_doc, state_acc} = clause_to_algebra(clause, min_line, state_acc) + + doc_acc = + doc_acc + |> concat(maybe_empty_line()) + |> line(clause_doc) + + {doc_acc, state_acc} + end) + + {clauses_doc |> maybe_force_clauses([clause | clauses], state) |> group(), state} + end + + defp clauses_to_algebra(other, min_line, max_line, state) do + case block_to_algebra(other, min_line, max_line, state) do + {@empty, state} -> {@empty, state} + {doc, state} -> {group(doc), state} + end + end + + defp clause_to_algebra({:->, meta, [[], body]}, _min_line, state) do + {body_doc, state} = block_to_algebra(body, line(meta), closing_line(meta), state) + {"() ->" |> glue(body_doc) |> nest(2), state} + end + + defp clause_to_algebra({:->, meta, [args, body]}, min_line, state) do + %{operand_nesting: nesting} = state + + state = %{state | operand_nesting: nesting + 2} + {args_doc, state} = clause_args_to_algebra(args, min_line, state) + + state = %{state | operand_nesting: nesting} + {body_doc, state} = block_to_algebra(body, min_line, closing_line(meta), state) + + doc = + args_doc + |> ungroup_if_group() + |> concat(" ->") + |> group() + |> concat(break() |> concat(body_doc) |> nest(2)) + + {doc, state} + end + + defp add_max_line_to_last_clause([{op, meta, args}], max_line) do + [{op, [closing: [line: max_line]] ++ meta, args}] + end + + defp add_max_line_to_last_clause([clause | clauses], max_line) do + [clause | add_max_line_to_last_clause(clauses, max_line)] + end + + defp clause_args_to_algebra(args, min_line, state) do + arg_to_algebra = fn arg, _args, state -> + {doc, state} = clause_args_to_algebra(arg, state) + {{doc, @empty, 1}, state} + end + + {args_docs, comments?, state} = + quoted_to_algebra_with_comments([args], [], min_line, @min_line, state, arg_to_algebra) + + if comments? do + {Enum.reduce(args_docs, &line(&2, &1)), state} + else + {Enum.reduce(args_docs, &glue(&2, &1)), state} + end + end + + # fn a, b, c when d -> e end + defp clause_args_to_algebra([{:when, meta, args}], state) do + {args, right} = split_last(args) + + # If there are any keywords, wrap them in lists + args = + Enum.map(args, fn + [_ | _] = keyword -> {:__block__, [], [keyword]} + other -> other + end) + + left = {{:special, :clause_args}, meta, [args]} + binary_op_to_algebra(:when, "when", meta, left, right, :no_parens_arg, state) + end + + # fn () -> e end + defp clause_args_to_algebra([], state) do + {"()", state} + end + + # fn a, b, c -> e end + defp clause_args_to_algebra(args, state) do + many_args_to_algebra(args, state, "ed_to_algebra(&1, :no_parens_arg, &2)) + end + + ## Quoted helpers for comments + + defp quoted_to_algebra_with_comments(args, acc, min_line, max_line, state, fun) do + {pre_comments, state} = + get_and_update_in(state.comments, fn comments -> + Enum.split_while(comments, fn %{line: line} -> line <= min_line end) + end) + + {reverse_docs, comments?, state} = + if state.comments == [] do + each_quoted_to_algebra_without_comments(args, acc, state, fun) + else + each_quoted_to_algebra_with_comments(args, acc, max_line, state, false, fun) + end + + docs = merge_algebra_with_comments(Enum.reverse(reverse_docs), @empty) + {docs, comments?, update_in(state.comments, &(pre_comments ++ &1))} + end + + defp each_quoted_to_algebra_without_comments([], acc, state, _fun) do + {acc, false, state} + end + + defp each_quoted_to_algebra_without_comments([arg | args], acc, state, fun) do + {doc_triplet, state} = fun.(arg, args, state) + acc = [doc_triplet | acc] + each_quoted_to_algebra_without_comments(args, acc, state, fun) + end + + defp each_quoted_to_algebra_with_comments([], acc, max_line, state, comments?, _fun) do + {acc, comments, comments?} = extract_comments_before(max_line, acc, state.comments, comments?) + {acc, comments?, %{state | comments: comments}} + end + + defp each_quoted_to_algebra_with_comments([arg | args], acc, max_line, state, comments?, fun) do + case traverse_line(arg, {@max_line, @min_line}) do + {@max_line, @min_line} -> + {doc_triplet, state} = fun.(arg, args, state) + acc = [doc_triplet | acc] + each_quoted_to_algebra_with_comments(args, acc, max_line, state, comments?, fun) + + {doc_start, doc_end} -> + {acc, comments, comments?} = + extract_comments_before(doc_start, acc, state.comments, comments?) + + {doc_triplet, state} = fun.(arg, args, %{state | comments: comments}) + + {acc, comments, comments?} = + extract_comments_trailing(doc_start, doc_end, acc, state.comments, comments?) + + acc = [adjust_trailing_newlines(doc_triplet, doc_end, comments) | acc] + state = %{state | comments: comments} + each_quoted_to_algebra_with_comments(args, acc, max_line, state, comments?, fun) + end + end + + defp extract_comments_before(max, acc, [%{line: line} = comment | rest], _) when line < max do + %{previous_eol_count: previous, next_eol_count: next, text: doc} = comment + acc = [{doc, @empty, next} | add_previous_to_acc(acc, previous)] + extract_comments_before(max, acc, rest, true) + end + + defp extract_comments_before(_max, acc, rest, comments?) do + {acc, rest, comments?} + end + + defp add_previous_to_acc([{doc, next_line, newlines} | acc], previous) when newlines < previous, + do: [{doc, next_line, previous} | acc] + + defp add_previous_to_acc(acc, _previous), + do: acc + + defp extract_comments_trailing(min, max, acc, [%{line: line, text: doc_comment} | rest], _) + when line >= min and line <= max do + acc = [{doc_comment, @empty, 1} | acc] + extract_comments_trailing(min, max, acc, rest, true) + end + + defp extract_comments_trailing(_min, _max, acc, rest, comments?) do + {acc, rest, comments?} + end + + # If the document is immediately followed by comment which is followed by newlines, + # its newlines wouldn't have considered the comment, so we need to adjust it. + defp adjust_trailing_newlines({doc, next_line, newlines}, doc_end, [%{line: line} | _]) + when newlines > 1 and line == doc_end + 1 do + {doc, next_line, 1} + end + + defp adjust_trailing_newlines(doc_triplet, _, _), do: doc_triplet + + defp traverse_line({expr, meta, args}, {min, max}) do + # This is a hot path, so use :lists.keyfind/3 instead Keyword.fetch!/2 + acc = + case :lists.keyfind(:line, 1, meta) do + {:line, line} -> {min(line, min), max(line, max)} + false -> {min, max} + end + + traverse_line(args, traverse_line(expr, acc)) + end + + defp traverse_line({left, right}, acc) do + traverse_line(right, traverse_line(left, acc)) + end + + defp traverse_line(args, acc) when is_list(args) do + Enum.reduce(args, acc, &traverse_line/2) + end + + defp traverse_line(_, acc) do + acc + end + + # Below are the rules for line rendering in the formatter: + # + # 1. respect the user's choice + # 2. and add empty lines around expressions that take multiple lines + # (except for module attributes) + # 3. empty lines are collapsed as to not exceed more than one + # + defp merge_algebra_with_comments([{doc, next_line, newlines} | docs], left) do + right = if newlines >= @newlines, do: line(), else: next_line + + doc = + if left != @empty do + concat(left, doc) + else + doc + end + + doc = + if docs != [] and right != @empty do + concat(doc, concat(collapse_lines(2), right)) + else + doc + end + + [group(doc) | merge_algebra_with_comments(docs, right)] + end + + defp merge_algebra_with_comments([], _) do + [] + end + + ## Quoted helpers + + defp left_op_context(context), do: force_many_args_or_operand(context, :parens_arg) + defp right_op_context(context), do: force_many_args_or_operand(context, :operand) + + defp force_many_args_or_operand(:no_parens_one_arg, _choice), do: :no_parens_arg + defp force_many_args_or_operand(:parens_one_arg, _choice), do: :parens_arg + defp force_many_args_or_operand(:no_parens_arg, _choice), do: :no_parens_arg + defp force_many_args_or_operand(:parens_arg, _choice), do: :parens_arg + defp force_many_args_or_operand(:operand, choice), do: choice + defp force_many_args_or_operand(:block, choice), do: choice + + defp quoted_to_algebra_with_parens_if_operator(ast, context, state) do + {doc, state} = quoted_to_algebra(ast, context, state) + {wrap_in_parens_if_operator(doc, ast), state} + end + + defp wrap_in_parens_if_operator(doc, {:__block__, _, [expr]}) do + wrap_in_parens_if_operator(doc, expr) + end + + defp wrap_in_parens_if_operator(doc, quoted) do + if operator?(quoted) and not module_attribute_read?(quoted) and not integer_capture?(quoted) do + wrap_in_parens(doc) + else + doc + end + end + + defp wrap_in_parens_if_binary_operator(doc, quoted) do + if binary_operator?(quoted) do + wrap_in_parens(doc) + else + doc + end + end + + defp wrap_in_parens_if_inspected_atom(":" <> _ = doc) do + "(" <> doc <> ")" + end + + defp wrap_in_parens_if_inspected_atom(doc) do + doc + end + + defp wrap_in_parens(doc) do + concat(concat("(", nest(doc, :cursor)), ")") + end + + defp many_args_to_algebra([arg | args], state, fun) do + Enum.reduce(args, fun.(arg, state), fn arg, {doc_acc, state_acc} -> + {arg_doc, state_acc} = fun.(arg, state_acc) + {glue(concat(doc_acc, ","), arg_doc), state_acc} + end) + end + + defp module_attribute_read?({:@, _, [{var, _, var_context}]}) + when is_atom(var) and is_atom(var_context) do + Macro.classify_atom(var) == :identifier + end + + defp module_attribute_read?(_), do: false + + defp integer_capture?({:&, _, [integer]}) when is_integer(integer), do: true + defp integer_capture?(_), do: false + + defp operator?(quoted) do + unary_operator?(quoted) or binary_operator?(quoted) + end + + # We convert ..// into two operators for simplicity, + # so we need to augment the binary table. + defp augmented_binary_op(:"//"), do: {:right, 190} + defp augmented_binary_op(op), do: Code.Identifier.binary_op(op) + + defp binary_operator?(quoted) do + case quoted do + {op, _, [_, _, _]} when op in @multi_binary_operators -> true + {op, _, [_, _]} when is_atom(op) -> augmented_binary_op(op) != :error + _ -> false + end + end + + defp unary_operator?(quoted) do + case quoted do + {op, _, [_]} when is_atom(op) -> Code.Identifier.unary_op(op) != :error + _ -> false + end + end + + defp with_next_break_fits(condition, doc, fun) do + if condition do + doc + |> group(:optimistic) + |> fun.() + |> group(:pessimistic) + else + doc + |> group() + |> fun.() + |> group() + end + end + + defp next_break_fits?({:{}, meta, _args}, state) do + eol_or_comments?(meta, state) + end + + defp next_break_fits?({:__block__, meta, [{_, _}]}, state) do + eol_or_comments?(meta, state) + end + + defp next_break_fits?({:<<>>, meta, [_ | _] = entries}, state) do + meta[:delimiter] == ~s["""] or + (not interpolated?(entries) and eol_or_comments?(meta, state)) + end + + # TODO: Remove this clause on Elixir v2.0 once single-quoted charlists are removed + defp next_break_fits?({{:., _, [List, :to_charlist]}, meta, [[_ | _]]}, _state) do + meta[:delimiter] == ~s['''] + end + + defp next_break_fits?({{:., _, [_left, :{}]}, _, _}, _state) do + true + end + + defp next_break_fits?({:__block__, meta, [string]}, _state) when is_binary(string) do + meta[:delimiter] == ~s["""] + end + + defp next_break_fits?({:__block__, meta, [list]}, _state) when is_list(list) do + meta[:delimiter] != ~s['] + end + + defp next_break_fits?({form, _, [_ | _]}, _state) when form in [:fn, :%{}, :%] do + true + end + + defp next_break_fits?({fun, meta, args}, _state) when is_atom(fun) and is_list(args) do + meta[:delimiter] in [@double_heredoc, @single_heredoc] and + fun |> Atom.to_string() |> String.starts_with?("sigil_") + end + + defp next_break_fits?({{:__block__, _, [atom]}, expr}, state) when is_atom(atom) do + next_break_fits?(expr, state) + end + + defp next_break_fits?(_, _state) do + false + end + + defp eol_or_comments?(meta, %{comments: comments} = state) do + eol?(meta, state) or + ( + min_line = line(meta) + max_line = closing_line(meta) + Enum.any?(comments, fn %{line: line} -> line > min_line and line < max_line end) + ) + end + + # A literal list is a keyword or (... -> ...) + defp last_arg_to_keyword([_ | _] = arg, _list_to_keyword?, _skip_parens?, _comments) do + {keyword?(arg), arg} + end + + # This is a list of tuples, it can be converted to keywords. + defp last_arg_to_keyword( + {:__block__, meta, [[_ | _] = arg]} = block, + true, + skip_parens?, + comments + ) do + cond do + not keyword?(arg) -> + {false, block} + + skip_parens? -> + block_line = line(meta) + {{_, arg_meta, _}, _} = hd(arg) + first_line = line(arg_meta) + + case Enum.drop_while(comments, fn %{line: line} -> line <= block_line end) do + [%{line: line} | _] when line <= first_line -> + {false, block} + + _ -> + {true, arg} + end + + true -> + {true, arg} + end + end + + # Otherwise we don't have a keyword. + defp last_arg_to_keyword(arg, _list_to_keyword?, _skip_parens?, _comments) do + {false, arg} + end + + defp force_args?(args) do + match?([_ | _], args) and force_args?(args, %{}) + end + + defp force_args?([[arg | _] | args], lines) do + force_args?([arg | args], lines) + end + + defp force_args?([arg | args], lines) do + line = + case arg do + {{_, meta, _}, _} -> meta[:line] + {_, meta, _} -> meta[:line] + end + + cond do + # Line may be missing from non-formatter AST + is_nil(line) -> force_args?(args, lines) + Map.has_key?(lines, line) -> false + true -> force_args?(args, Map.put(lines, line, true)) + end + end + + defp force_args?([], lines), do: map_size(lines) >= 2 + + defp force_keyword(doc, arg) do + if force_args?(arg), do: force_unfit(doc), else: doc + end + + defp keyword?([{_, _} | list]), do: keyword?(list) + defp keyword?(rest), do: rest == [] + + defp keyword_key?({:__block__, meta, [atom]}) when is_atom(atom), + do: meta[:format] == :keyword + + defp keyword_key?({{:., _, [:erlang, :binary_to_atom]}, meta, [{:<<>>, _, _}, :utf8]}), + do: meta[:format] == :keyword + + defp keyword_key?(_), + do: false + + defp eol?(_meta, %{skip_eol: true}), do: false + defp eol?(meta, _state), do: Keyword.get(meta, :newlines, 0) > 0 + + defp meta?(meta, key) do + is_list(meta[key]) + end + + defp line(meta) do + meta[:line] || @max_line + end + + defp end_line(meta) do + meta[:end][:line] || @min_line + end + + defp closing_line(meta) do + meta[:closing][:line] || @min_line + end + + defp escape_atom(string, char) do + String.replace(string, <>, <>) + end + + ## Algebra helpers + + # Relying on the inner document is brittle and error prone. + # It would be best if we had a mechanism to apply this. + defp concat_to_last_group([left | right], concat) do + [left | concat_to_last_group(right, concat)] + end + + defp concat_to_last_group({:doc_group, group, mode}, concat) do + {:doc_group, concat(group, concat), mode} + end + + defp concat_to_last_group(other, concat) do + concat(other, concat) + end + + defp ungroup_if_group({:doc_group, group, _mode}), do: group + defp ungroup_if_group(other), do: other + + defp format_to_string(doc) do + doc |> Inspect.Algebra.format(:infinity) |> IO.iodata_to_binary() + end + + defp maybe_empty_line() do + nest(break(""), :reset) + end + + defp surround(left, doc, right) do + if doc == @empty do + concat(left, right) + else + group(glue(nest(glue(left, "", doc), 2, :break), "", right)) + end + end + + defp nest_by_length(doc, string) do + nest(doc, String.length(string)) + end + + defp split_last(list) do + {left, [right]} = Enum.split(list, -1) + {left, right} + end + + defp get_charlist_quotes(:heredoc, state) do + if state.migrate_charlists_as_sigils do + {@sigil_c_heredoc, @double_heredoc} + else + {@single_heredoc, @single_heredoc} + end + end + + defp get_charlist_quotes({:regular, chunks}, state) do + cond do + !state.migrate_charlists_as_sigils -> {@single_quote, @single_quote} + Enum.any?(chunks, &has_double_quote?/1) -> {@sigil_c_single, @single_quote} + true -> {@sigil_c_double, @double_quote} + end + end + + defp has_double_quote?(chunk) do + is_binary(chunk) and chunk =~ @double_quote + end + + # Migration rewrites + + @bool_operators [ + :>, + :>=, + :<, + :<=, + :in + ] + @guards [ + :is_atom, + :is_boolean, + :is_nil, + :is_number, + :is_integer, + :is_float, + :is_binary, + :is_map, + :is_struct, + :is_non_struct_map, + :is_exception, + :is_list, + :is_tuple, + :is_function, + :is_reference, + :is_pid, + :is_port + ] + + defp negate_condition(condition) do + case condition do + {neg, _, [condition]} when neg in [:!, :not] -> condition + {op, _, [_, _]} when op in @bool_operators -> {:not, [], [condition]} + {guard, _, [_ | _]} when guard in @guards -> {:not, [], [condition]} + {:==, meta, [left, right]} -> {:!=, meta, [left, right]} + {:===, meta, [left, right]} -> {:!==, meta, [left, right]} + {:!=, meta, [left, right]} -> {:==, meta, [left, right]} + {:!==, meta, [left, right]} -> {:===, meta, [left, right]} + _ -> {:!, [], [condition]} + end + end +end diff --git a/lib/elixir/lib/code/fragment.ex b/lib/elixir/lib/code/fragment.ex new file mode 100644 index 00000000000..c4dcd2b8e87 --- /dev/null +++ b/lib/elixir/lib/code/fragment.ex @@ -0,0 +1,1346 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team + +defmodule Code.Fragment do + @moduledoc """ + This module provides conveniences for analyzing fragments of + textual code and extract available information whenever possible. + + This module should be considered experimental. + """ + + @type position :: {line :: pos_integer(), column :: pos_integer()} + + @typedoc """ + Options for cursor context functions. + + Currently, these options are not used but reserved for future extensibility. + """ + @type cursor_opts :: [] + + @typedoc """ + Options for converting code fragments to quoted expressions. + """ + @type container_cursor_to_quoted_opts :: [ + file: String.t(), + line: pos_integer(), + column: pos_integer(), + columns: boolean(), + token_metadata: boolean(), + literal_encoder: (term(), Macro.metadata() -> term()), + trailing_fragment: String.t() + ] + + @doc ~S""" + Returns the list of lines in the given string, preserving their line endings. + + Only the line endings recognized by the Elixir compiler are + considered, namely `\r\n` and `\n`. If you would like the retrieve + lines without their line endings, use `String.split(string, ["\r\n", "\n"])`. + + ## Examples + + iex> Code.Fragment.lines("foo\r\nbar\r\nbaz") + ["foo\r\n", "bar\r\n", "baz"] + + iex> Code.Fragment.lines("foo\nbar\nbaz") + ["foo\n", "bar\n", "baz"] + + iex> Code.Fragment.lines("") + [""] + + """ + @doc since: "1.19.0" + def lines(string) do + lines(string, <<>>) + end + + defp lines(<>, acc), + do: [<> | lines(rest, <<>>)] + + defp lines(<>, acc), + do: lines(rest, <>) + + defp lines(<<>>, acc), + do: [acc] + + @doc """ + Receives a string and returns the cursor context. + + This function receives a string with an Elixir code fragment, + representing a cursor position, and based on the string, it + provides contextual information about the latest token. + The return of this function can then be used to provide tips, + suggestions, and autocompletion functionality. + + This function performs its analyses on tokens. This means + it does not understand how constructs are nested within each + other. See the "Limitations" section below. + + Consider adding a catch-all clause when handling the return + type of this function as new cursor information may be added + in future releases. + + ## Examples + + iex> Code.Fragment.cursor_context("") + :expr + + iex> Code.Fragment.cursor_context("hello_wor") + {:local_or_var, ~c"hello_wor"} + + ## Return values + + * `{:alias, charlist}` - the context is an alias, potentially + a nested one, such as `Hello.Wor` or `HelloWor` + + * `{:alias, inside_alias, charlist}` - the context is an alias, potentially + a nested one, where `inside_alias` is an expression `{:module_attribute, charlist}` + or `{:local_or_var, charlist}` and `charlist` is a static part + Examples are `__MODULE__.Submodule` or `@hello.Submodule` + + * `{:block_keyword_or_binary_operator, charlist}` - may be a block keyword (do, end, after, + catch, else, rescue) or a binary operator + + * `{:dot, inside_dot, charlist}` - the context is a dot + where `inside_dot` is either a `{:var, charlist}`, `{:alias, charlist}`, + `{:module_attribute, charlist}`, `{:unquoted_atom, charlist}` or a `dot` + itself. If a var is given, this may either be a remote call or a map + field access. Examples are `Hello.wor`, `:hello.wor`, `hello.wor`, + `Hello.nested.wor`, `hello.nested.wor`, and `@hello.world`. If `charlist` + is empty and `inside_dot` is an alias, then the autocompletion may either + be an alias or a remote call. + + * `{:dot_arity, inside_dot, charlist}` - the context is a dot arity + where `inside_dot` is either a `{:var, charlist}`, `{:alias, charlist}`, + `{:module_attribute, charlist}`, `{:unquoted_atom, charlist}` or a `dot` + itself. If a var is given, it must be a remote arity. Examples are + `Hello.world/`, `:hello.world/`, `hello.world/2`, and `@hello.world/2` + + * `{:dot_call, inside_dot, charlist}` - the context is a dot + call. This means parentheses or space have been added after the expression. + where `inside_dot` is either a `{:var, charlist}`, `{:alias, charlist}`, + `{:module_attribute, charlist}`, `{:unquoted_atom, charlist}` or a `dot` + itself. If a var is given, it must be a remote call. Examples are + `Hello.world(`, `:hello.world(`, `Hello.world `, `hello.world(`, `hello.world `, + and `@hello.world(` + + * `:expr` - may be any expression. Autocompletion may suggest an alias, + local or var + + * `{:local_or_var, charlist}` - the context is a variable or a local + (import or local) call, such as `hello_wor` + + * `{:local_arity, charlist}` - the context is a local (import or local) + arity, such as `hello_world/` + + * `{:local_call, charlist}` - the context is a local (import or local) + call, such as `hello_world(` and `hello_world ` + + * `{:anonymous_call, inside_caller}` - the context is an anonymous + call, such as `fun.(` and `@fun.(`. + + * `{:module_attribute, charlist}` - the context is a module attribute, + such as `@hello_wor` + + * `{:operator, charlist}` - the context is an operator, such as `+` or + `==`. Note textual operators, such as `when` do not appear as operators + but rather as `:local_or_var`. `@` is never an `:operator` and always a + `:module_attribute` + + * `{:operator_arity, charlist}` - the context is an operator arity, which + is an operator followed by /, such as `+/`, `not/` or `when/` + + * `{:operator_call, charlist}` - the context is an operator call, which is + an operator followed by space, such as `left + `, `not ` or `x when ` + + * `:none` - no context possible + + * `{:sigil, charlist}` - the context is a sigil. It may be either the beginning + of a sigil, such as `~` or `~s`, or an operator starting with `~`, such as + `~>` and `~>>` + + * `{:struct, inside_struct}` - the context is a struct, such as `%`, `%UR` or `%URI`. + `inside_struct` can either be a `charlist` in case of a static alias or an + expression `{:alias, inside_alias, charlist}`, `{:module_attribute, charlist}`, + `{:local_or_var, charlist}`, `{:dot, inside_dot, charlist}` + + * `{:unquoted_atom, charlist}` - the context is an unquoted atom. This + can be any atom or an atom representing a module + + We recommend looking at the test suite of this function for a complete list + of examples and their return values. + + ## Limitations + + The analysis is based on the current token, by analysing the last line of + the input. For example, this code: + + iex> Code.Fragment.cursor_context("%URI{") + :expr + + returns `:expr`, which suggests any variable, local function or alias + could be used. However, given we are inside a struct, the best suggestion + would be a struct field. In such cases, you can use + `container_cursor_to_quoted`, which will return the container of the AST + the cursor is currently within. You can then analyse this AST to provide + completion of field names. + + As a consequence of its token-based implementation, this function considers + only the last line of the input. This means it will show suggestions inside + strings, heredocs, etc, which is intentional as it helps with doctests, + references, and more. + """ + @doc since: "1.13.0" + @spec cursor_context(List.Chars.t(), cursor_opts()) :: + {:alias, charlist} + | {:alias, inside_alias, charlist} + | {:block_keyword_or_binary_operator, charlist} + | {:dot, inside_dot, charlist} + | {:dot_arity, inside_dot, charlist} + | {:dot_call, inside_dot, charlist} + | :expr + | {:local_or_var, charlist} + | {:local_arity, charlist} + | {:local_call, charlist} + | {:anonymous_call, inside_caller} + | {:module_attribute, charlist} + | {:operator, charlist} + | {:operator_arity, charlist} + | {:operator_call, charlist} + | :none + | {:sigil, charlist} + | {:struct, inside_struct} + | {:unquoted_atom, charlist} + when inside_dot: + {:alias, charlist} + | {:alias, inside_alias, charlist} + | {:dot, inside_dot, charlist} + | {:module_attribute, charlist} + | {:unquoted_atom, charlist} + | {:var, charlist} + | :expr, + inside_alias: + {:local_or_var, charlist} + | {:module_attribute, charlist}, + inside_struct: + charlist + | {:alias, inside_alias, charlist} + | {:local_or_var, charlist} + | {:module_attribute, charlist} + | {:dot, inside_dot, charlist}, + inside_caller: {:var, charlist} | {:module_attribute, charlist} + def cursor_context(fragment, opts \\ []) + + def cursor_context(fragment, opts) + when (is_binary(fragment) or is_list(fragment)) and is_list(opts) do + fragment + |> last_line() + |> :lists.reverse() + |> codepoint_cursor_context(opts) + |> elem(0) + end + + def cursor_context(other, opts) when is_list(opts) do + cursor_context(to_charlist(other), opts) + end + + @operators ~c"\\<>+-*/:=|&~^%!$" + @starting_punctuation ~c",([{;" + @closing_punctuation ~c")]}\"'" + @space ~c"\t\s" + @trailing_identifier ~c"?!" + @tilde_op_prefix ~c"<=~" + + @non_identifier @trailing_identifier ++ + @operators ++ @starting_punctuation ++ @closing_punctuation ++ @space ++ [?.] + + @textual_operators ~w(when not and or in)c + @keywords ~w(do end after else catch rescue fn true false nil)c + + defp codepoint_cursor_context(reverse, _opts) do + {stripped, spaces} = strip_spaces(reverse, 0) + + case stripped do + # It is empty + [] -> {:expr, 0} + # Structs + [?%, ?:, ?: | _] -> {{:struct, ~c""}, 1} + [?%, ?: | _] -> {{:unquoted_atom, ~c"%"}, 2} + [?% | _] -> {{:struct, ~c""}, 1} + # Token/AST only operators + [?>, ?= | rest] when rest == [] or hd(rest) != ?: -> {:expr, 0} + [?>, ?- | rest] when rest == [] or hd(rest) != ?: -> {:expr, 0} + # Two-digit containers + [?<, ?< | rest] when rest == [] or hd(rest) != ?< -> {:expr, 0} + # Ambiguity around : + [?: | rest] when rest == [] or hd(rest) != ?: -> unquoted_atom_or_expr(spaces) + # Dots + [?.] -> {:none, 0} + [?. | rest] when hd(rest) not in ~c".:" -> dot(rest, spaces + 1, ~c"") + # It is a local or remote call with parens + [?( | rest] -> call_to_cursor_context(strip_spaces(rest, spaces + 1)) + # A local arity definition + [?/ | rest] -> arity_to_cursor_context(strip_spaces(rest, spaces + 1)) + # Starting a new expression + [h | _] when h in @starting_punctuation -> {:expr, 0} + # It is keyword, binary operator, a local or remote call without parens + rest when spaces > 0 -> closing_or_call_to_cursor_context({rest, spaces}) + # It is an identifier + _ -> identifier_to_cursor_context(reverse, spaces, false) + end + end + + defp strip_spaces([h | rest], count) when h in @space, do: strip_spaces(rest, count + 1) + defp strip_spaces(rest, count), do: {rest, count} + + defp unquoted_atom_or_expr(0), do: {{:unquoted_atom, ~c""}, 1} + defp unquoted_atom_or_expr(_), do: {:expr, 0} + + defp arity_to_cursor_context({reverse, spaces}) do + case identifier_to_cursor_context(reverse, spaces, true) do + {{:local_or_var, acc}, count} -> {{:local_arity, acc}, count} + {{:dot, base, acc}, count} -> {{:dot_arity, base, acc}, count} + {{:operator, acc}, count} -> {{:operator_arity, acc}, count} + {{:sigil, _}, _} -> {:none, 0} + {_, _} -> {{:operator, ~c"/"}, 1} + end + end + + defp call_to_cursor_context({reverse, spaces}) do + with [?. | rest] <- reverse, + {rest, spaces} = strip_spaces(rest, spaces), + [h | _] when h not in @non_identifier <- rest do + case identifier_to_cursor_context(rest, spaces, true) do + {{:local_or_var, acc}, count} -> {{:anonymous_call, {:var, acc}}, count + 1} + {{:module_attribute, _} = attr, count} -> {{:anonymous_call, attr}, count + 1} + {_, _} -> {:none, 0} + end + else + _ -> + case identifier_to_cursor_context(reverse, spaces, true) do + {{:local_or_var, acc}, count} -> {{:local_call, acc}, count} + {{:dot, base, acc}, count} -> {{:dot_call, base, acc}, count} + {{:operator, acc}, count} -> {{:operator_call, acc}, count} + {_, _} -> {:none, 0} + end + end + end + + defp closing_or_call_to_cursor_context({reverse, spaces}) do + if closing?(reverse) do + {{:block_keyword_or_binary_operator, ~c""}, 0} + else + call_to_cursor_context({reverse, spaces}) + end + end + + defp identifier_to_cursor_context([?., ?., ?: | _], n, _), do: {{:unquoted_atom, ~c".."}, n + 3} + defp identifier_to_cursor_context([?., ?., ?. | _], n, _), do: {{:operator, ~c"..."}, n + 3} + defp identifier_to_cursor_context([?., ?: | _], n, _), do: {{:unquoted_atom, ~c"."}, n + 2} + defp identifier_to_cursor_context([?., ?. | _], n, _), do: {{:operator, ~c".."}, n + 2} + + defp identifier_to_cursor_context(reverse, count, call_op?) do + case identifier(reverse, count) do + :none -> + {:none, 0} + + :operator -> + operator(reverse, count, [], call_op?) + + {:struct, {:module_attribute, acc}, count} -> + {{:struct, {:module_attribute, acc}}, count + 1} + + {:module_attribute, acc, count} -> + {{:module_attribute, acc}, count} + + {:sigil, acc, count} -> + {{:sigil, acc}, count} + + {:unquoted_atom, acc, count} -> + {{:unquoted_atom, acc}, count} + + {:alias, rest, acc, count} -> + case strip_spaces(rest, count) do + {~c"." ++ rest, count} when rest == [] or hd(rest) != ?. -> + nested_alias(rest, count + 1, acc) + + {~c"%" ++ _, count} -> + {{:struct, acc}, count + 1} + + _ -> + {{:alias, acc}, count} + end + + {:identifier, _, acc, count} when call_op? and acc in @textual_operators -> + {{:operator, acc}, count} + + {:identifier, [?%], acc, count} -> + case identifier_to_cursor_context(acc |> Enum.reverse(), count, true) do + {{:local_or_var, _} = identifier, _} -> {{:struct, identifier}, count + 1} + _ -> {:none, 0} + end + + {:identifier, rest, acc, count} -> + case strip_spaces(rest, count) do + {~c"." ++ rest, count} when rest == [] or hd(rest) != ?. -> + dot(rest, count + 1, acc) + + {rest, rest_count} -> + response = + if rest_count > count and closing?(rest), + do: :block_keyword_or_binary_operator, + else: :local_or_var + + {{response, acc}, count} + end + + {:capture_arg, acc, count} -> + {{:capture_arg, acc}, count} + end + end + + # If it is a closing punctuation + defp closing?([h | _]) when h in @closing_punctuation, do: true + # Closing bitstring (but deal with operators) + defp closing?([?>, ?> | rest]), do: rest == [] or hd(rest) not in [?>, ?~] + # Keywords + defp closing?(rest) do + case split_non_identifier(rest, []) do + {~c"nil", _} -> true + {~c"true", _} -> true + {~c"false", _} -> true + {[digit | _], _} when digit in ?0..?9 -> true + {[upper | _], _} when upper in ?A..?Z -> true + {[_ | _], [?: | rest]} -> rest == [] or hd(rest) != ?: + {_, _} -> false + end + end + + defp split_non_identifier([h | t], acc) when h not in @non_identifier, + do: split_non_identifier(t, [h | acc]) + + defp split_non_identifier(rest, acc), do: {acc, rest} + + defp identifier([?? | rest], count), do: check_identifier(rest, count + 1, [??]) + defp identifier([?! | rest], count), do: check_identifier(rest, count + 1, [?!]) + defp identifier(rest, count), do: check_identifier(rest, count, []) + + defp check_identifier([h | t], count, acc) when h not in @non_identifier, + do: rest_identifier(t, count + 1, [h | acc]) + + defp check_identifier(_, _, _), do: :operator + + defp rest_identifier([h | rest], count, acc) when h not in @non_identifier do + rest_identifier(rest, count + 1, [h | acc]) + end + + defp rest_identifier(rest, count, [?@ | acc]) do + case tokenize_identifier(rest, count, acc) do + {:identifier, [?% | _rest], acc, count} -> {:struct, {:module_attribute, acc}, count} + {:identifier, _rest, acc, count} -> {:module_attribute, acc, count} + :none when acc == [] -> {:module_attribute, ~c"", count} + _ -> :none + end + end + + defp rest_identifier([?~ | rest], count, [letter]) + when (letter in ?A..?Z or letter in ?a..?z) and + (rest == [] or hd(rest) not in @tilde_op_prefix) do + {:sigil, [letter], count + 1} + end + + defp rest_identifier([?: | rest], count, acc) when rest == [] or hd(rest) != ?: do + case String.Tokenizer.tokenize(acc) do + {_, _, [], _, _, _} -> {:unquoted_atom, acc, count + 1} + _ -> :none + end + end + + defp rest_identifier([?? | _], _count, _acc) do + :none + end + + defp rest_identifier([?& | tail] = rest, count, acc) when tail == [] or hd(tail) != ?& do + if Enum.all?(acc, &(&1 in ?0..?9)) do + {:capture_arg, [?& | acc], count + 1} + else + tokenize_identifier(rest, count, acc) + end + end + + defp rest_identifier(rest, count, acc) do + tokenize_identifier(rest, count, acc) + end + + defp tokenize_identifier(rest, count, acc) do + case String.Tokenizer.tokenize(acc) do + # Not actually an atom cause rest is not a : + {:atom, _, _, _, _, _} -> + :none + + # Aliases must be ascii only + {:alias, _, _, _, false, _} -> + :none + + {kind, _, [], _, _, extra} -> + if :at in extra do + :none + else + {kind, rest, acc, count} + end + + _ -> + :none + end + end + + defp nested_alias(rest, count, acc) do + {rest, count} = strip_spaces(rest, count) + + case identifier_to_cursor_context(rest, count, true) do + {{:struct, prev}, count} when is_list(prev) -> + {{:struct, prev ++ ~c"." ++ acc}, count} + + {{:struct, {:alias, parent, prev}}, count} -> + {{:struct, {:alias, parent, prev ++ ~c"." ++ acc}}, count} + + {{:struct, prev}, count} -> + {{:struct, {:alias, prev, acc}}, count} + + {{:alias, prev}, count} -> + {{:alias, prev ++ ~c"." ++ acc}, count} + + {{:alias, parent, prev}, count} -> + {{:alias, parent, prev ++ ~c"." ++ acc}, count} + + {{:local_or_var, prev}, count} -> + {{:alias, {:local_or_var, prev}, acc}, count} + + {{:module_attribute, prev}, count} -> + {{:alias, {:module_attribute, prev}, acc}, count} + + _ -> + {:none, 0} + end + end + + defp dot(rest, count, acc) do + {rest, count} = strip_spaces(rest, count) + + case identifier_to_cursor_context(rest, count, true) do + {{:local_or_var, var}, count} -> + {{:dot, {:var, var}, acc}, count} + + {{:unquoted_atom, _} = prev, count} -> + {{:dot, prev, acc}, count} + + {{:alias, _} = prev, count} -> + {{:dot, prev, acc}, count} + + {{:alias, _, _} = prev, count} -> + {{:dot, prev, acc}, count} + + {{:struct, inner}, count} when is_list(inner) -> + {{:struct, {:dot, {:alias, inner}, acc}}, count} + + {{:struct, inner}, count} -> + {{:struct, {:dot, inner, acc}}, count} + + {{:dot, _, _} = prev, count} -> + {{:dot, prev, acc}, count} + + {{:module_attribute, _} = prev, count} -> + {{:dot, prev, acc}, count} + + {:expr, count} -> + {{:dot, :expr, acc}, count} + + {_, _} -> + {:none, 0} + end + end + + defp operator([h | rest], count, acc, call_op?) when h in @operators do + operator(rest, count + 1, [h | acc], call_op?) + end + + # If we are opening a sigil, ignore the operator. + defp operator([letter, ?~ | rest], _count, [op], _call_op?) + when op in ~c"<|/" and (letter in ?A..?Z or letter in ?a..?z) and + (rest == [] or hd(rest) not in @tilde_op_prefix) do + {:none, 0} + end + + defp operator(rest, count, ~c"~", call_op?) do + {rest, _} = strip_spaces(rest, count) + + if call_op? or match?([?. | rest] when rest == [] or hd(rest) != ?., rest) do + {:none, 0} + else + {{:sigil, ~c""}, count} + end + end + + defp operator([?) | rest], _, [], true) when hd(rest) != ?? do + {:expr, 0} + end + + defp operator(rest, count, acc, _call_op?) do + case :elixir_tokenizer.tokenize(acc, 1, 1, []) do + {:ok, _, _, _, [{:atom, _, _}], []} -> + {{:unquoted_atom, tl(acc)}, count} + + {:ok, _, _, _, [{_, _, op}], []} -> + {rest, dot_count} = strip_spaces(rest, count) + + cond do + Code.Identifier.unary_op(op) == :error and Code.Identifier.binary_op(op) == :error -> + {:none, 0} + + match?([?. | rest] when rest == [] or hd(rest) != ?., rest) -> + dot(tl(rest), dot_count + 1, acc) + + true -> + {{:operator, acc}, count} + end + + _ -> + {:none, 0} + end + end + + @doc """ + Receives a string and returns the surround context. + + This function receives a string with an Elixir code fragment + and a `position`. It returns a map containing the beginning + and ending of the identifier alongside its context, or `:none` + if there is nothing with a known context. This is useful to + provide mouse-over and highlight functionality in editors. + + The difference between `cursor_context/2` and `surround_context/3` + is that the former assumes the expression in the code fragment + is incomplete. For example, `do` in `cursor_context/2` may be + a keyword or a variable or a local call, while `surround_context/3` + assumes the expression in the code fragment is complete, therefore + `do` would always be a keyword. + + The `position` contains both the `line` and `column`, both starting + with the index of 1. The column must precede the surrounding expression. + For example, the expression `foo`, will return something for the columns + 1, 2, and 3, but not 4: + + foo + ^ column 1 + + foo + ^ column 2 + + foo + ^ column 3 + + foo + ^ column 4 + + The returned map contains the column the expression starts and the + first column after the expression ends. + + Similar to `cursor_context/2`, this function is also token-based + and may not be accurate under all circumstances. See the + "Return values" and "Limitations" section under `cursor_context/2` + for more information. + + ## Examples + + iex> Code.Fragment.surround_context("foo", {1, 1}) + %{begin: {1, 1}, context: {:local_or_var, ~c"foo"}, end: {1, 4}} + + ## Differences to `cursor_context/2` + + Because `surround_context/3` attempts to capture complex expressions, + it has some differences to `cursor_context/2`: + + * `dot_call`/`dot_arity` and `operator_call`/`operator_arity` + are collapsed into `dot` and `operator` contexts respectively + as there aren't any meaningful distinctions between them + + * On the other hand, this function still makes a distinction between + `local_call`/`local_arity` and `local_or_var`, since the latter can + be a local or variable + + * `@` when not followed by any identifier is returned as `{:operator, ~c"@"}` + (in contrast to `{:module_attribute, ~c""}` in `cursor_context/2` + + * This function never returns empty sigils `{:sigil, ~c""}` or empty structs + `{:struct, ~c""}` as context + + * This function returns keywords as `{:keyword, ~c"do"}` + + * This function never returns `:expr` + + We recommend looking at the test suite of this function for a complete list + of examples and their return values. + """ + @doc since: "1.13.0" + @spec surround_context(List.Chars.t(), position(), cursor_opts()) :: + %{begin: position, end: position, context: context} | :none + when context: + {:alias, charlist} + | {:alias, inside_alias, charlist} + | {:dot, inside_dot, charlist} + | {:local_or_var, charlist} + | {:local_arity, charlist} + | {:local_call, charlist} + | {:module_attribute, charlist} + | {:operator, charlist} + | {:sigil, charlist} + | {:struct, inside_struct} + | {:unquoted_atom, charlist} + | {:keyword, charlist} + | {:key, charlist} + | {:capture_arg, charlist}, + inside_dot: + {:alias, charlist} + | {:alias, inside_alias, charlist} + | {:dot, inside_dot, charlist} + | {:module_attribute, charlist} + | {:unquoted_atom, charlist} + | {:var, charlist} + | :expr, + inside_alias: + {:local_or_var, charlist} + | {:module_attribute, charlist}, + inside_struct: + charlist + | {:alias, inside_alias, charlist} + | {:local_or_var, charlist} + | {:module_attribute, charlist} + | {:dot, inside_dot, charlist} + def surround_context(fragment, position, options \\ []) + + def surround_context(string, {line, column}, opts) + when (is_binary(string) or is_list(string)) and is_list(opts) do + {charlist, lines_before_lengths, lines_current_and_after_lengths} = + surround_line(string, line, column) + + prepended_columns = Enum.sum(lines_before_lengths) + + charlist + |> position_surround_context(line, column + prepended_columns, opts) + |> to_multiline_range( + prepended_columns, + lines_before_lengths, + lines_current_and_after_lengths + ) + end + + def surround_context(other, {_, _} = position, opts) do + surround_context(to_charlist(other), position, opts) + end + + defp position_surround_context(charlist, line, column, opts) + when is_integer(line) and line >= 1 and is_integer(column) and column >= 1 do + {reversed_pre, post} = string_reverse_at(charlist, column - 1, []) + {reversed_pre, post} = adjust_position(reversed_pre, post) + + case take_identifier(post, []) do + {_, [], _} -> + maybe_operator(reversed_pre, post, line, opts) + + {:identifier, reversed_post, rest} -> + {keyword_key?, rest} = + case rest do + [?: | tail] when tail == [] or hd(tail) in @space -> + {true, rest} + + _ -> + {rest, _} = strip_spaces(rest, 0) + {false, rest} + end + + reversed = reversed_post ++ reversed_pre + + case codepoint_cursor_context(reversed, opts) do + {{:struct, acc}, offset} -> + build_surround({:struct, acc}, reversed, line, offset) + + {{:alias, acc}, offset} -> + build_surround({:alias, acc}, reversed, line, offset) + + {{:alias, parent, acc}, offset} -> + build_surround({:alias, parent, acc}, reversed, line, offset) + + {{:dot, _, [_ | _]} = dot, offset} -> + build_surround(dot, reversed, line, offset) + + {{:local_or_var, acc}, offset} when keyword_key? -> + build_surround({:key, acc}, reversed, line, offset) + + {{:local_or_var, acc}, offset} when hd(rest) == ?( -> + build_surround({:local_call, acc}, reversed, line, offset) + + {{:local_or_var, acc}, offset} when hd(rest) == ?/ -> + build_surround({:local_arity, acc}, reversed, line, offset) + + {{:local_or_var, acc}, offset} when acc in @textual_operators -> + build_surround({:operator, acc}, reversed, line, offset) + + {{:local_or_var, acc}, offset} when acc in @keywords -> + build_surround({:keyword, acc}, reversed, line, offset) + + {{:local_or_var, acc}, offset} -> + build_surround({:local_or_var, acc}, reversed, line, offset) + + {{:block_keyword_or_binary_operator, acc}, offset} when acc in @textual_operators -> + build_surround({:operator, acc}, reversed, line, offset) + + {{:block_keyword_or_binary_operator, acc}, offset} when acc in @keywords -> + build_surround({:keyword, acc}, reversed, line, offset) + + {{:module_attribute, ~c""}, offset} -> + build_surround({:operator, ~c"@"}, reversed, line, offset) + + {{:module_attribute, acc}, offset} -> + build_surround({:module_attribute, acc}, reversed, line, offset) + + {{:sigil, acc}, offset} -> + build_surround({:sigil, acc}, reversed, line, offset) + + {{:unquoted_atom, acc}, offset} -> + build_surround({:unquoted_atom, acc}, reversed, line, offset) + + {{:capture_arg, acc}, offset} -> + build_surround({:capture_arg, acc}, reversed, line, offset) + + _ -> + maybe_operator(reversed_pre, post, line, opts) + end + + {:alias, reversed_post, _rest} -> + reversed = reversed_post ++ reversed_pre + + case codepoint_cursor_context(reversed, opts) do + {{:alias, acc}, offset} -> + build_surround({:alias, acc}, reversed, line, offset) + + {{:alias, parent, acc}, offset} -> + build_surround({:alias, parent, acc}, reversed, line, offset) + + {{:struct, acc}, offset} -> + build_surround({:struct, acc}, reversed, line, offset) + + _ -> + :none + end + end + end + + defp maybe_operator(reversed_pre, post, line, opts) do + case take_operator(post, []) do + {[], _rest} -> + :none + + {reversed_post, rest} -> + reversed = reversed_post ++ reversed_pre + + case codepoint_cursor_context(reversed, opts) do + {{:operator, ~c"&"}, offset} when hd(rest) in ?0..?9 -> + arg = Enum.take_while(rest, &(&1 in ?0..?9)) + + build_surround( + {:capture_arg, ~c"&" ++ arg}, + :lists.reverse(arg, reversed), + line, + offset + length(arg) + ) + + {{:operator, acc}, offset} -> + build_surround({:operator, acc}, reversed, line, offset) + + {{:sigil, ~c""}, offset} when hd(rest) in ?A..?Z or hd(rest) in ?a..?z -> + build_surround({:sigil, [hd(rest)]}, [hd(rest) | reversed], line, offset + 1) + + {{:dot, _, [_ | _]} = dot, offset} -> + build_surround(dot, reversed, line, offset) + + _ -> + :none + end + end + end + + defp build_surround(context, reversed, line, offset) do + {post, reversed_pre} = enum_reverse_at(reversed, offset, []) + pre = :lists.reverse(reversed_pre) + pre_length = :string.length(pre) + 1 + + %{ + context: context, + begin: {line, pre_length}, + end: {line, pre_length + :string.length(post)} + } + end + + defp take_identifier([h | t], acc) when h in @trailing_identifier, + do: {:identifier, [h | acc], t} + + defp take_identifier([h | t], acc) when h not in @non_identifier, + do: take_identifier(t, [h | acc]) + + defp take_identifier(rest, acc) do + with {[?. | t], _} <- strip_spaces(rest, 0), + {[h | _], _} when h in ?A..?Z <- strip_spaces(t, 0) do + take_alias(rest, acc) + else + _ -> {:identifier, acc, rest} + end + end + + defp take_alias([h | t], acc) when h in ?A..?Z or h in ?a..?z or h in ?0..?9 or h == ?_, + do: take_alias(t, [h | acc]) + + defp take_alias(rest, acc) do + with {[?. | t], acc} <- move_spaces(rest, acc), + {[h | t], acc} when h in ?A..?Z <- move_spaces(t, [?. | acc]) do + take_alias(t, [h | acc]) + else + _ -> {:alias, acc, rest} + end + end + + defp take_operator([h | t], acc) when h in @operators, do: take_operator(t, [h | acc]) + defp take_operator([h | t], acc) when h == ?., do: take_operator(t, [h | acc]) + defp take_operator(rest, acc), do: {acc, rest} + + # Unquoted atom handling + defp adjust_position(reversed_pre, [?: | post]) + when hd(post) != ?: and (reversed_pre == [] or hd(reversed_pre) != ?:) do + {[?: | reversed_pre], post} + end + + defp adjust_position(reversed_pre, [?% | post]) do + adjust_position([?% | reversed_pre], post) + end + + # Dot/struct handling + defp adjust_position(reversed_pre, post) do + case move_spaces(post, reversed_pre) do + # If we are between spaces and a dot, move past the dot + {[?. | post], reversed_pre} when hd(post) != ?. and hd(reversed_pre) != ?. -> + {post, reversed_pre} = move_spaces(post, [?. | reversed_pre]) + {reversed_pre, post} + + _ -> + case strip_spaces(reversed_pre, 0) do + # If there is a dot to our left, make sure to move to the first character + {[?. | rest], _} when rest == [] or hd(rest) not in ~c".:" -> + {post, reversed_pre} = move_spaces(post, reversed_pre) + {reversed_pre, post} + + # If there is a % to our left, make sure to move to the first character + {[?% | _], _} -> + case move_spaces(post, reversed_pre) do + {[h | _] = post, reversed_pre} when h in ?A..?Z -> + {reversed_pre, post} + + _ -> + {reversed_pre, post} + end + + _ -> + {reversed_pre, post} + end + end + end + + defp move_spaces([h | t], acc) when h in @space, do: move_spaces(t, [h | acc]) + defp move_spaces(t, acc), do: {t, acc} + + defp string_reverse_at(charlist, 0, acc), do: {acc, charlist} + + defp string_reverse_at(charlist, n, acc) do + case :unicode_util.gc(charlist) do + [gc | cont] when is_integer(gc) -> string_reverse_at(cont, n - 1, [gc | acc]) + [gc | cont] when is_list(gc) -> string_reverse_at(cont, n - 1, :lists.reverse(gc, acc)) + [] -> {acc, []} + end + end + + defp enum_reverse_at([h | t], n, acc) when n > 0, do: enum_reverse_at(t, n - 1, [h | acc]) + defp enum_reverse_at(rest, _, acc), do: {acc, rest} + + defp last_line(binary) when is_binary(binary) do + [last_line | lines_reverse] = + binary + |> String.split(["\r\n", "\n"]) + |> Enum.reverse() + + prepend_cursor_lines(lines_reverse, String.to_charlist(last_line)) + end + + defp last_line(charlist) when is_list(charlist) do + [last_line | lines_reverse] = + charlist + |> :string.replace(~c"\r\n", ~c"\n", :all) + |> :string.join(~c"") + |> :string.split(~c"\n", :all) + |> Enum.reverse() + + prepend_cursor_lines(lines_reverse, last_line) + end + + defp prepend_cursor_lines(lines, last_line) do + with [line | lines] <- lines, + {trimmed_line, incomplete?} = ends_as_incomplete(to_charlist(line), [], true), + true <- incomplete? or starts_with_dot?(last_line) do + prepend_cursor_lines(lines, Enum.reverse(trimmed_line, last_line)) + else + _ -> last_line + end + end + + defp starts_with_dot?([?. | _]), do: true + defp starts_with_dot?([h | t]) when h in @space, do: starts_with_dot?(t) + defp starts_with_dot?(_), do: false + + defp ends_as_incomplete([?# | _], acc, incomplete?), + do: {acc, incomplete?} + + defp ends_as_incomplete([h | t], acc, _incomplete?) when h in [?(, ?.], + do: ends_as_incomplete(t, [h | acc], true) + + defp ends_as_incomplete([h | t], acc, incomplete?) when h in @space, + do: ends_as_incomplete(t, [h | acc], incomplete?) + + defp ends_as_incomplete([h | t], acc, _incomplete?), + do: ends_as_incomplete(t, [h | acc], false) + + defp ends_as_incomplete([], acc, incomplete?), + do: {acc, incomplete?} + + defp surround_line(binary, line, column) when is_binary(binary) do + binary + |> String.split(["\r\n", "\n"]) + |> Enum.map(&String.to_charlist/1) + |> surround_lines(line, column) + end + + defp surround_line(charlist, line, column) when is_list(charlist) do + charlist + |> :string.replace(~c"\r\n", ~c"\n", :all) + |> :string.join(~c"") + |> :string.split(~c"\n", :all) + |> surround_lines(line, column) + end + + defp surround_lines(lines, line, column) do + {lines_before_reverse, cursor_line, lines_after} = split_at(lines, line, []) + {trimmed_cursor_line, incomplete?} = ends_as_incomplete(to_charlist(cursor_line), [], true) + + reversed_cursor_line = + if column - 1 > length(trimmed_cursor_line) do + # Don't strip comments if cursor is inside a comment + Enum.reverse(cursor_line) + else + trimmed_cursor_line + end + + {cursor_line, after_lengths} = + append_surround_lines(lines_after, [], [reversed_cursor_line], incomplete?) + + {cursor_line, before_lengths} = prepend_surround_lines(lines_before_reverse, [], cursor_line) + {cursor_line, before_lengths, [length(reversed_cursor_line) | after_lengths]} + end + + defp split_at([line], _, acc), do: {acc, line, []} + defp split_at([line | lines], 1, acc), do: {acc, line, lines} + defp split_at([line | lines], count, acc), do: split_at(lines, count - 1, [line | acc]) + + defp prepend_surround_lines(lines, lengths, last_line) do + with [line | lines] <- lines, + {trimmed_line, incomplete?} = ends_as_incomplete(to_charlist(line), [], true), + true <- incomplete? or starts_with_dot?(last_line) do + lengths = [length(trimmed_line) | lengths] + prepend_surround_lines(lines, lengths, Enum.reverse(trimmed_line, last_line)) + else + _ -> {last_line, Enum.reverse(lengths)} + end + end + + defp append_surround_lines(lines, lengths, acc_lines, incomplete?) do + with [line | lines] <- lines, + line = to_charlist(line), + true <- incomplete? or starts_with_dot?(line) do + {trimmed_line, incomplete?} = ends_as_incomplete(line, [], true) + lengths = [length(trimmed_line) | lengths] + append_surround_lines(lines, lengths, [trimmed_line | acc_lines], incomplete?) + else + _ -> {Enum.reduce(acc_lines, [], &Enum.reverse/2), Enum.reverse(lengths)} + end + end + + defp to_multiline_range(:none, _, _, _), do: :none + + defp to_multiline_range( + %{begin: {begin_line, begin_column}, end: {end_line, end_column}} = context, + prepended, + lines_before_lengths, + lines_current_and_after_lengths + ) do + {begin_line, begin_column} = + Enum.reduce_while(lines_before_lengths, {begin_line, begin_column - prepended}, fn + line_length, {acc_line, acc_column} -> + if acc_column < 1 do + {:cont, {acc_line - 1, acc_column + line_length}} + else + {:halt, {acc_line, acc_column}} + end + end) + + {end_line, end_column} = + Enum.reduce_while(lines_current_and_after_lengths, {end_line, end_column - prepended}, fn + line_length, {acc_line, acc_column} -> + if acc_column > line_length + 1 do + {:cont, {acc_line + 1, acc_column - line_length}} + else + {:halt, {acc_line, acc_column}} + end + end) + + %{context | begin: {begin_line, begin_column}, end: {end_line, end_column}} + end + + @doc """ + Receives a string and returns a quoted expression + with the cursor AST position within its parent expression. + + This function receives a string with an Elixir code fragment, + representing a cursor position, and converts such string to + AST with the inclusion of special `__cursor__()` node representing + the cursor position within its container (i.e. its parent). + + For example, take this code, which would be given as input: + + max(some_value, + + This function will return the AST equivalent to: + + max(some_value, __cursor__()) + + In other words, this function is capable of closing any open + brackets and insert the cursor position. Other content at the + cursor position which is not a parent is discarded. + For example, if this is given as input: + + max(some_value, another_val + + It will return the same AST: + + max(some_value, __cursor__()) + + Similarly, if only this is given: + + max(some_va + + Then it returns: + + max(__cursor__()) + + Calls without parenthesis are also supported, as we assume the + brackets are implicit. + + Tuples, lists, maps, and binaries all retain the cursor position: + + max(some_value, [1, 2, + + Returns the following AST: + + max(some_value, [1, 2, __cursor__()]) + + Keyword lists (and do-end blocks) are also retained. The following: + + if(some_value, do: + if(some_value, do: :token + if(some_value, do: 1 + val + + all return: + + if(some_value, do: __cursor__()) + + For multi-line blocks, all previous lines are preserved. + + The AST returned by this function is not safe to evaluate but + it can be analyzed and expanded. + + ## Examples + + Function call: + + iex> Code.Fragment.container_cursor_to_quoted("max(some_value, ") + {:ok, {:max, [line: 1], [{:some_value, [line: 1], nil}, {:__cursor__, [line: 1], []}]}} + + Containers (for example, a list): + + iex> Code.Fragment.container_cursor_to_quoted("[some, value") + {:ok, [{:some, [line: 1], nil}, {:__cursor__, [line: 1], []}]} + + If an expression is complete, then the whole expression is discarded + and only the parent is returned: + + iex> Code.Fragment.container_cursor_to_quoted("if(is_atom(var)") + {:ok, {:if, [line: 1], [{:__cursor__, [line: 1], []}]}} + + this means complete expressions themselves return only the cursor: + + iex> Code.Fragment.container_cursor_to_quoted("if(is_atom(var))") + {:ok, {:__cursor__, [line: 1], []}} + + Operators are also included from Elixir v1.15: + + iex> Code.Fragment.container_cursor_to_quoted("foo +") + {:ok, {:+, [line: 1], [{:foo, [line: 1], nil}, {:__cursor__, [line: 1], []}]}} + + In order to parse the left-side of `->` properly, which appears both + in anonymous functions and do-end blocks, the trailing fragment option + must be given with the rest of the contents: + + iex> Code.Fragment.container_cursor_to_quoted("fn x", trailing_fragment: " -> :ok end") + {:ok, {:fn, [line: 1], [{:->, [line: 1], [[{:__cursor__, [line: 1], []}], :ok]}]}} + + ## Options + + * `:file` - the filename to be reported in case of parsing errors. + Defaults to `"nofile"`. + + * `:line` - the starting line of the string being parsed. + Defaults to `1`. + + * `:column` - the starting column of the string being parsed. + Defaults to `1`. + + * `:columns` - when `true`, attach a `:column` key to the quoted + metadata. Defaults to `false`. + + * `:token_metadata` - when `true`, includes token-related + metadata in the expression AST, such as metadata for `do` and `end` + tokens, for closing tokens, end of expressions, as well as delimiters + for sigils. See `t:Macro.metadata/0`. Defaults to `false`. + + * `:literal_encoder` - a function to encode literals in the AST. + See the documentation for `Code.string_to_quoted/2` for more information. + + * `:trailing_fragment` (since v1.18.0) - the rest of the contents after + the cursor. This is necessary to correctly complete anonymous functions + and the left-hand side of `->` + + """ + @doc since: "1.13.0" + @spec container_cursor_to_quoted(List.Chars.t(), container_cursor_to_quoted_opts()) :: + {:ok, Macro.t()} | {:error, {location :: keyword, binary | {binary, binary}, binary}} + def container_cursor_to_quoted(fragment, opts \\ []) do + {trailing_fragment, opts} = Keyword.pop(opts, :trailing_fragment) + opts = Keyword.take(opts, [:columns, :token_metadata, :literal_encoder]) + opts = [check_terminators: {:cursor, []}, emit_warnings: false] ++ opts + + file = Keyword.get(opts, :file, "nofile") + line = Keyword.get(opts, :line, 1) + column = Keyword.get(opts, :column, 1) + + case :elixir_tokenizer.tokenize(to_charlist(fragment), line, column, opts) do + {:ok, line, column, _warnings, rev_tokens, rev_terminators} + when trailing_fragment == nil -> + {rev_tokens, rev_terminators} = + with [close, open, {_, _, :__cursor__} = cursor | rev_tokens] <- rev_tokens, + {_, [_ | after_fn]} <- Enum.split_while(rev_terminators, &(elem(&1, 0) != :fn)), + true <- maybe_missing_stab?(rev_tokens, false), + [_ | rev_tokens] <- Enum.drop_while(rev_tokens, &(elem(&1, 0) != :fn)) do + {[close, open, cursor | rev_tokens], after_fn} + else + _ -> {rev_tokens, rev_terminators} + end + + tokens = reverse_tokens(line, column, rev_tokens, rev_terminators) + :elixir.tokens_to_quoted(tokens, file, opts) + + {:ok, line, column, _warnings, rev_tokens, rev_terminators} -> + tokens = + with {before_start, [_ | _] = after_start} <- + Enum.split_while(rev_terminators, &(elem(&1, 0) not in [:do, :fn])), + true <- maybe_missing_stab?(rev_tokens, true), + opts = + Keyword.put(opts, :check_terminators, {:cursor, before_start}), + {:error, {meta, _, ~c"end"}, _rest, _warnings, trailing_rev_tokens} <- + :elixir_tokenizer.tokenize(to_charlist(trailing_fragment), line, column, opts) do + trailing_tokens = + reverse_tokens(meta[:line], meta[:column], trailing_rev_tokens, after_start) + + # If the cursor has its own line, then we do not trim new lines trailing tokens. + # Otherwise we want to drop any newline so we drop the next tokens after eol. + trailing_tokens = + case rev_tokens do + [_close, _open, {_, _, :__cursor__}, {:eol, _} | _] -> trailing_tokens + _ -> Enum.drop_while(trailing_tokens, &match?({:eol, _}, &1)) + end + + Enum.reverse(rev_tokens, drop_tokens(trailing_tokens, 0)) + else + _ -> reverse_tokens(line, column, rev_tokens, rev_terminators) + end + + :elixir.tokens_to_quoted(tokens, file, opts) + + {:error, info, _rest, _warnings, _so_far} -> + {:error, :elixir.format_token_error(info)} + end + end + + defp reverse_tokens(line, column, tokens, terminators) do + {terminators, _} = + Enum.map_reduce(terminators, column, fn {start, _, _}, column -> + atom = :elixir_tokenizer.terminator(start) + + {{atom, {line, column, nil}}, column + length(Atom.to_charlist(atom))} + end) + + Enum.reverse(tokens, terminators) + end + + # Otherwise we drop all tokens, trying to build a minimal AST + # for cursor completion. + defp drop_tokens([{:"}", _} | _] = tokens, 0), do: tokens + defp drop_tokens([{:"]", _} | _] = tokens, 0), do: tokens + defp drop_tokens([{:")", _} | _] = tokens, 0), do: tokens + defp drop_tokens([{:">>", _} | _] = tokens, 0), do: tokens + defp drop_tokens([{:end, _} | _] = tokens, 0), do: tokens + defp drop_tokens([{:",", _} | _] = tokens, 0), do: tokens + defp drop_tokens([{:";", _} | _] = tokens, 0), do: tokens + defp drop_tokens([{:eol, _} | _] = tokens, 0), do: tokens + defp drop_tokens([{:stab_op, _, :->} | _] = tokens, 0), do: tokens + + defp drop_tokens([{:"}", _} | tokens], counter), do: drop_tokens(tokens, counter - 1) + defp drop_tokens([{:"]", _} | tokens], counter), do: drop_tokens(tokens, counter - 1) + defp drop_tokens([{:")", _} | tokens], counter), do: drop_tokens(tokens, counter - 1) + defp drop_tokens([{:">>", _} | tokens], counter), do: drop_tokens(tokens, counter - 1) + defp drop_tokens([{:end, _} | tokens], counter), do: drop_tokens(tokens, counter - 1) + + defp drop_tokens([{:"{", _} | tokens], counter), do: drop_tokens(tokens, counter + 1) + defp drop_tokens([{:"[", _} | tokens], counter), do: drop_tokens(tokens, counter + 1) + defp drop_tokens([{:"(", _} | tokens], counter), do: drop_tokens(tokens, counter + 1) + defp drop_tokens([{:"<<", _} | tokens], counter), do: drop_tokens(tokens, counter + 1) + defp drop_tokens([{:fn, _} | tokens], counter), do: drop_tokens(tokens, counter + 1) + defp drop_tokens([{:do, _} | tokens], counter), do: drop_tokens(tokens, counter + 1) + + defp drop_tokens([_ | tokens], counter), do: drop_tokens(tokens, counter) + defp drop_tokens([], _counter), do: [] + + defp maybe_missing_stab?([{:after, _} | _], _stab_choice?), do: true + defp maybe_missing_stab?([{:do, _} | _], _stab_choice?), do: true + defp maybe_missing_stab?([{:fn, _} | _], _stab_choice?), do: true + defp maybe_missing_stab?([{:else, _} | _], _stab_choice?), do: true + defp maybe_missing_stab?([{:catch, _} | _], _stab_choice?), do: true + defp maybe_missing_stab?([{:rescue, _} | _], _stab_choice?), do: true + defp maybe_missing_stab?([{:stab_op, _, :->} | _], stab_choice?), do: stab_choice? + defp maybe_missing_stab?([_ | tail], stab_choice?), do: maybe_missing_stab?(tail, stab_choice?) + defp maybe_missing_stab?([], _stab_choice?), do: false +end diff --git a/lib/elixir/lib/code/identifier.ex b/lib/elixir/lib/code/identifier.ex new file mode 100644 index 00000000000..b3adeb5692f --- /dev/null +++ b/lib/elixir/lib/code/identifier.ex @@ -0,0 +1,188 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +defmodule Code.Identifier do + @moduledoc false + + @doc """ + Checks if the given identifier is an unary op. + + ## Examples + + iex> Code.Identifier.unary_op(:+) + {:non_associative, 300} + + """ + @spec unary_op(atom) :: {:non_associative, precedence :: pos_integer} | :error + def unary_op(op) do + cond do + op in [:&, :...] -> {:non_associative, 90} + op in [:!, :^, :not, :+, :-, :"~~~"] -> {:non_associative, 300} + op in [:@] -> {:non_associative, 320} + true -> :error + end + end + + @doc """ + Checks if the given identifier is a binary op. + + ## Examples + + iex> Code.Identifier.binary_op(:+) + {:left, 210} + + """ + @spec binary_op(atom) :: {:left | :right, precedence :: pos_integer} | :error + def binary_op(op) do + cond do + op in [:<-, :\\] -> {:left, 40} + op in [:when] -> {:right, 50} + op in [:"::"] -> {:right, 60} + op in [:|] -> {:right, 70} + op in [:=] -> {:right, 100} + op in [:||, :|||, :or] -> {:left, 120} + op in [:&&, :&&&, :and] -> {:left, 130} + op in [:==, :!=, :=~, :===, :!==] -> {:left, 140} + op in [:<, :<=, :>=, :>] -> {:left, 150} + op in [:|>, :<<<, :>>>, :<~, :~>, :<<~, :~>>, :<~>, :"<|>"] -> {:left, 160} + op in [:in] -> {:left, 170} + op in [:"^^^"] -> {:left, 180} + op in [:++, :--, :.., :<>, :+++, :---] -> {:right, 200} + op in [:+, :-] -> {:left, 210} + op in [:*, :/] -> {:left, 220} + op in [:**] -> {:left, 230} + op in [:.] -> {:left, 310} + true -> :error + end + end + + @doc """ + Extracts the name and arity of the parent from the anonymous function identifier. + """ + # Example of this format: -NAME/ARITY-fun-COUNT- + def extract_anonymous_fun_parent(atom) when is_atom(atom) do + with "-" <> rest <- Atom.to_string(atom), + [trailing | reversed] = rest |> String.split("/") |> Enum.reverse(), + [arity, _inner, _count, ""] <- String.split(trailing, "-") do + {reversed |> Enum.reverse() |> Enum.join("/") |> String.to_atom(), arity} + else + _ -> :error + end + end + + @doc """ + Escapes the given identifier. + """ + @spec escape(binary(), char() | nil, :infinity | non_neg_integer, (char() -> iolist() | false)) :: + {escaped :: binary(), remaining :: binary()} + def escape(binary, char, limit \\ :infinity, fun \\ &escape_map/1) + when (is_binary(binary) and ((char in 0..0x10FFFF or is_nil(char)) and limit == :infinity)) or + (is_integer(limit) and limit >= 0) do + escape(binary, char, limit, <<>>, fun) + end + + defp escape(<<_, _::binary>> = binary, _char, 0, acc, _fun) do + {acc, binary} + end + + defp escape(<>, char, count, acc, fun) do + escape(t, char, decrement(count), <>, fun) + end + + defp escape(<>, char, count, acc, fun) do + escape(t, char, decrement(count), <>, fun) + end + + defp escape(<>, char, count, acc, fun) do + if value = fun.(h) do + value = IO.iodata_to_binary(value) + escape(t, char, decrement(count), <>, fun) + else + escape(t, char, decrement(count), escape_char(h, acc), fun) + end + end + + defp escape(<>, char, count, acc, fun) do + escape(t, char, decrement(count), <>, fun) + end + + defp escape(<<>>, _char, _count, acc, _fun) do + {acc, <<>>} + end + + defp escape_char(0, acc), do: <> + + defp escape_char(char, acc) + # Some characters that are confusing (zero-width / alternative spaces) are displayed + # using their unicode representation: + # https://en.wikipedia.org/wiki/Universal_Character_Set_characters#Special-purpose_characters + + # BOM + when char == 0xFEFF + # Mathematical invisibles + when char in 0x2061..0x2064 + # Bidirectional neutral + when char in [0x061C, 0x200E, 0x200F] + # Bidirectional general (source of vulnerabilities) + when char in 0x202A..0x202E + when char in 0x2066..0x2069 + # Interlinear annotations + when char in 0xFFF9..0xFFFC + # Zero-width joiners and non-joiners + when char in [0x200C, 0x200D, 0x034F] + # Non-break space / zero-width space + when char in [0x00A0, 0x200B, 0x2060] + # Line/paragraph separators + when char in [0x2028, 0x2029] + # Spaces + when char in 0x2000..0x200A + when char == 0x205F do + <> = <> + <> + end + + defp escape_char(char, acc) + when char in 0x20..0x7E + when char in 0xA0..0xD7FF + when char in 0xE000..0xFFFD + when char in 0x10000..0x10FFFF do + <> + end + + defp escape_char(char, acc) when char < 0x100 do + <> = <> + <> + end + + defp escape_char(char, acc) when char < 0x10000 do + <> = <> + <> + end + + defp escape_char(char, acc) when char < 0x1000000 do + <> = <> + + <> + end + + defp escape_map(?\a), do: "\\a" + defp escape_map(?\b), do: "\\b" + defp escape_map(?\d), do: "\\d" + defp escape_map(?\e), do: "\\e" + defp escape_map(?\f), do: "\\f" + defp escape_map(?\n), do: "\\n" + defp escape_map(?\r), do: "\\r" + defp escape_map(?\t), do: "\\t" + defp escape_map(?\v), do: "\\v" + defp escape_map(?\\), do: "\\\\" + defp escape_map(_), do: false + + @compile {:inline, to_hex: 1, decrement: 1} + defp to_hex(c) when c in 0..9, do: ?0 + c + defp to_hex(c) when c in 10..15, do: ?A + c - 10 + + defp decrement(:infinity), do: :infinity + defp decrement(counter), do: counter - 1 +end diff --git a/lib/elixir/lib/code/normalizer.ex b/lib/elixir/lib/code/normalizer.ex new file mode 100644 index 00000000000..a4aa9492ec1 --- /dev/null +++ b/lib/elixir/lib/code/normalizer.ex @@ -0,0 +1,612 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team + +defmodule Code.Normalizer do + @moduledoc false + + defguard is_literal(x) + when is_integer(x) or + is_float(x) or + is_binary(x) or + is_atom(x) + + @doc """ + Wraps literals in the quoted expression to conform to the AST format expected + by the formatter. + """ + @spec normalize(Macro.t(), keyword()) :: Macro.t() + def normalize(quoted, opts \\ []) do + line = Keyword.get(opts, :line, nil) + escape = Keyword.get(opts, :escape, true) + locals_without_parens = Keyword.get(opts, :locals_without_parens, []) + + state = %{ + escape: escape, + parent_meta: [line: line], + locals_without_parens: locals_without_parens ++ Code.Formatter.locals_without_parens() + } + + do_normalize(quoted, state) + end + + # Wrapped literals should receive the block meta + defp do_normalize({:__block__, meta, [literal]}, state) + when not is_tuple(literal) or tuple_size(literal) == 2 do + normalize_literal(literal, meta, state) + end + + # Only normalize the first argument of an alias if it's not an atom + defp do_normalize({:__aliases__, meta, [first | rest]}, state) when not is_atom(first) do + meta = patch_meta_line(meta, state.parent_meta) + first = do_normalize(first, %{state | parent_meta: meta}) + {:__aliases__, meta, [first | rest]} + end + + defp do_normalize({:__aliases__, _, _} = quoted, _state) do + quoted + end + + # Skip captured arguments like &1 + defp do_normalize({:&, meta, [term]}, state) when is_integer(term) do + meta = patch_meta_line(meta, state.parent_meta) + {:&, meta, [term]} + end + + # Ranges + defp do_normalize(left..right//step, state) do + left = do_normalize(left, state) + right = do_normalize(right, state) + meta = meta_line(state) + + if step == 1 do + {:.., meta, [left, right]} + else + step = do_normalize(step, state) + {:..//, meta, [left, right, step]} + end + end + + # Bit containers + defp do_normalize({:<<>>, _, args} = quoted, state) when is_list(args) do + normalize_bitstring(quoted, state) + end + + # Atoms with interpolations + defp do_normalize( + {{:., dot_meta, [:erlang, :binary_to_atom]}, call_meta, + [{:<<>>, _, parts} = string, :utf8]}, + state + ) + when is_list(parts) do + dot_meta = patch_meta_line(dot_meta, state.parent_meta) + call_meta = patch_meta_line(call_meta, dot_meta) + + utf8 = + if parts == [] or binary_interpolated?(parts) do + # a non-normalized :utf8 atom signals an atom interpolation + :utf8 + else + normalize_literal(:utf8, [], state) + end + + string = + if state.escape do + normalize_bitstring(string, state, true) + else + normalize_bitstring(string, state) + end + + {{:., dot_meta, [:erlang, :binary_to_atom]}, call_meta, [string, utf8]} + end + + # Charlists with interpolations + # TODO: Remove this clause on Elixir v2.0 once single-quoted charlists are removed + defp do_normalize({{:., dot_meta, [List, :to_charlist]}, call_meta, [parts]} = quoted, state) do + if list_interpolated?(parts) do + parts = + Enum.map(parts, fn + {{:., part_dot_meta, [Kernel, :to_string]}, part_call_meta, args} -> + args = normalize_args(args, state) + + {{:., part_dot_meta, [Kernel, :to_string]}, part_call_meta, args} + + part when is_binary(part) -> + if state.escape do + maybe_escape_literal(part, state) + else + part + end + end) + + {{:., dot_meta, [List, :to_charlist]}, call_meta, [parts]} + else + normalize_call(quoted, state) + end + end + + # Don't normalize the `Access` atom in access syntax + defp do_normalize({:., meta, [Access, :get]}, state) do + meta = patch_meta_line(meta, state.parent_meta) + {:., meta, [Access, :get]} + end + + # The right hand side is an atom in the AST but it's not an atom literal, so + # it should not be wrapped. However, it should be escaped if applicable. + defp do_normalize({:., meta, [left, right]}, state) when is_atom(right) do + meta = patch_meta_line(meta, state.parent_meta) + + left = do_normalize(left, %{state | parent_meta: meta}) + right = maybe_escape_literal(right, state) + + {:., meta, [left, right]} + end + + # left -> right + defp do_normalize({:->, meta, [left, right]}, state) do + meta = patch_meta_line(meta, state.parent_meta) + + left = normalize_args(left, %{state | parent_meta: meta}) + right = do_normalize(right, %{state | parent_meta: meta}) + {:->, meta, [left, right]} + end + + # Maps + defp do_normalize({:%{}, meta, args}, state) when is_list(args) do + meta = + if meta == [] do + line = state.parent_meta[:line] + [line: line, closing: [line: line]] + else + meta + end + + state = %{state | parent_meta: meta} + + args = + case args do + [{:|, pipe_meta, [left, right]}] -> + left = do_normalize(left, state) + right = normalize_map_args(right, state) + [{:|, pipe_meta, [left, right]}] + + args -> + normalize_map_args(args, state) + end + + {:%{}, meta, args} + end + + # Sigils + defp do_normalize({sigil, meta, [{:<<>>, _, args} = string, modifiers]} = quoted, state) + when is_atom(sigil) and is_list(args) and is_list(modifiers) do + with "sigil_" <> _ <- Atom.to_string(sigil), + true <- binary_interpolated?(args), + true <- List.ascii_printable?(modifiers) do + meta = + meta + |> patch_meta_line(state.parent_meta) + |> Keyword.put_new(:delimiter, "\"") + + {sigil, meta, [do_normalize(string, %{state | parent_meta: meta}), modifiers]} + else + _ -> + normalize_call(quoted, state) + end + end + + # Tuples + defp do_normalize({:{}, meta, args} = quoted, state) when is_list(args) do + {last_arg, args} = List.pop_at(args, -1) + + if args != [] and match?([_ | _], last_arg) and keyword?(last_arg) do + args = normalize_args(args, state) + kw_list = normalize_kw_args(last_arg, state, true) + {:{}, meta, args ++ kw_list} + else + normalize_call(quoted, state) + end + end + + # Module attributes + defp do_normalize({:@, meta, [{name, name_meta, [value]}]}, state) do + value = + cond do + keyword?(value) and value != [] -> + normalize_kw_args(value, state, true) + + is_list(value) -> + normalize_literal(value, meta, state) + + true -> + do_normalize(value, state) + end + + {:@, meta, [{name, name_meta, [value]}]} + end + + # Regular blocks + defp do_normalize({:__block__, meta, args}, state) when is_list(args) do + {:__block__, meta, normalize_args(args, state)} + end + + # Calls + defp do_normalize({_, _, args} = quoted, state) when is_list(args) do + normalize_call(quoted, state) + end + + # Vars + defp do_normalize({_, _, context} = quoted, _state) when is_atom(context) do + quoted + end + + # Literals + defp do_normalize(quoted, state) do + normalize_literal(quoted, [], state) + end + + # Numbers + defp normalize_literal(number, meta, state) when is_number(number) do + meta = + meta + |> Keyword.put_new(:token, inspect(number)) + |> patch_meta_line(state.parent_meta) + + {:__block__, meta, [number]} + end + + # Atom, Strings + defp normalize_literal(literal, meta, state) when is_atom(literal) or is_binary(literal) do + meta = patch_meta_line(meta, state.parent_meta) + literal = maybe_escape_literal(literal, state) + + if is_atom(literal) and Macro.classify_atom(literal) == :alias and + is_nil(meta[:delimiter]) do + segments = + case Atom.to_string(literal) do + "Elixir" -> + [:"Elixir"] + + "Elixir." <> segments -> + segments + |> String.split(".") + |> Enum.map(&String.to_atom/1) + end + + {:__aliases__, meta, segments} + else + {:__block__, meta, [literal]} + end + end + + # 2-tuples + defp normalize_literal({left, right}, meta, state) do + meta = patch_meta_line(meta, state.parent_meta) + state = %{state | parent_meta: meta} + + if match?([_ | _], right) and keyword?(right) do + {:__block__, meta, [{do_normalize(left, state), normalize_kw_args(right, state, true)}]} + else + {:__block__, meta, [{do_normalize(left, state), do_normalize(right, state)}]} + end + end + + # Lists + defp normalize_literal(list, meta, state) when is_list(list) do + if list != [] and List.ascii_printable?(list) do + # It's a charlist, we normalize it as a ~C sigil + string = + if state.escape do + {iolist, _} = Code.Identifier.escape(IO.chardata_to_string(list), nil) + IO.iodata_to_binary(iolist) + else + List.to_string(list) + end + + meta = patch_meta_line([delimiter: "\""], state.parent_meta) + + {:sigil_c, meta, [{:<<>>, [], [string]}, []]} + else + meta = + if line = state.parent_meta[:line] do + meta + |> Keyword.put_new(:closing, line: line) + |> patch_meta_line(state.parent_meta) + else + meta + end + + {:__block__, meta, [normalize_kw_args(list, state, false)]} + end + end + + # Probably an invalid value, wrap it and send it upstream + defp normalize_literal(quoted, meta, _state) do + {:__block__, meta, [quoted]} + end + + defp normalize_call({form, meta, args}, state) do + meta = patch_meta_line(meta, state.parent_meta) + arity = length(args) + + # Only normalize the form if it's a qualified call + form = + if is_atom(form) do + form + else + do_normalize(form, %{state | parent_meta: meta}) + end + + meta = + if is_nil(meta[:no_parens]) and is_nil(meta[:closing]) and is_nil(meta[:do]) and + not Code.Formatter.local_without_parens?(form, arity, state.locals_without_parens) do + [closing: [line: meta[:line]]] ++ meta + else + meta + end + + last = List.last(args) + + cond do + not allow_keyword?(form, arity) -> + args = normalize_args(args, %{state | parent_meta: meta}) + {form, meta, args} + + Keyword.has_key?(meta, :do) -> + # def foo do :ok end + # def foo, do: :ok + normalize_kw_blocks(form, meta, args, state) + + match?([{:do, _} | _], last) and Keyword.keyword?(last) -> + # Non normalized kw blocks + line = state.parent_meta[:line] || meta[:line] + meta = meta ++ [do: [line: line], end: [line: line]] + normalize_kw_blocks(form, meta, args, state) + + true -> + args = normalize_args(args, %{state | parent_meta: meta}) + {last_arg, leading_args} = List.pop_at(args, -1, []) + + last_args = + case last_arg do + {:__block__, _meta, [[{{:__block__, key_meta, _}, _} | _] = keyword]} -> + cond do + key_meta[:format] == :keyword -> + [keyword] + + block_keyword?(keyword) -> + [ + Enum.map(keyword, fn {{:__block__, meta, args}, value} -> + {{:__block__, [format: :keyword] ++ meta, args}, value} + end) + ] + + true -> + [last_arg] + end + + [] -> + [] + + _ -> + [last_arg] + end + + {form, meta, leading_args ++ last_args} + end + end + + defp block_keyword?([{{:__block__, _, [key]}, _val} | tail]) when is_atom(key), + do: block_keyword?(tail) + + defp block_keyword?([]), do: true + defp block_keyword?(_), do: false + + defp allow_keyword?(:when, 2), do: true + defp allow_keyword?(:{}, _), do: false + defp allow_keyword?(op, arity), do: not is_atom(op) or not Macro.operator?(op, arity) + + defp normalize_bitstring({:<<>>, meta, parts}, state, escape_interpolation \\ false) do + meta = patch_meta_line(meta, state.parent_meta) + + parts = + if binary_interpolated?(parts) do + normalize_interpolation_parts(parts, %{state | parent_meta: meta}, escape_interpolation) + else + state = %{state | parent_meta: meta} + + Enum.map(parts, fn part -> + with {:"::", meta, [left, _]} <- part, + true <- meta[:inferred_bitstring_spec] do + do_normalize(left, state) + else + _ -> do_normalize(part, state) + end + end) + end + + {:<<>>, meta, parts} + end + + defp normalize_interpolation_parts(parts, state, escape_interpolation) do + Enum.map(parts, fn + {:"::", interpolation_meta, + [ + {{:., dot_meta, [Kernel, :to_string]}, middle_meta, [middle]}, + {:binary, binary_meta, context} + ]} -> + middle = do_normalize(middle, %{state | parent_meta: dot_meta}) + + {:"::", interpolation_meta, + [ + {{:., dot_meta, [Kernel, :to_string]}, middle_meta, [middle]}, + {:binary, binary_meta, context} + ]} + + part -> + if escape_interpolation do + maybe_escape_literal(part, state) + else + part + end + end) + end + + defp normalize_map_args(args, state) do + Enum.map(normalize_kw_args(args, state, false), fn + {:__block__, _, [{_, _} = pair]} -> pair + pair -> pair + end) + end + + defp normalize_kw_blocks(form, meta, args, state) do + {kw_blocks, leading_args} = List.pop_at(args, -1) + + kw_blocks = + Enum.map(kw_blocks, fn {tag, block} -> + block = do_normalize(block, %{state | parent_meta: meta}) + + block = + case block do + {_, _, [[{:->, _, _} | _] = block]} -> block + block -> block + end + + # Only wrap the tag if it isn't already wrapped + tag = + case tag do + {:__block__, _, _} -> tag + _ -> {:__block__, [line: meta[:line]], [tag]} + end + + {tag, block} + end) + + leading_args = normalize_args(leading_args, %{state | parent_meta: meta}) + {form, meta, leading_args ++ [kw_blocks]} + end + + defp normalize_kw_args(elems, state, keyword?) + + defp normalize_kw_args( + [{{:__block__, key_meta, [key]}, value} = first | rest] = current, + state, + keyword? + ) + when is_atom(key) do + keyword? = keyword? or keyword?(current) + + first = + if key_meta[:format] == :keyword and not keyword? do + key_meta = Keyword.delete(key_meta, :format) + line = key_meta[:line] || meta_line(state) + {:__block__, [line: line], [{{:__block__, key_meta, [key]}, value}]} + else + first + end + + [first | normalize_kw_args(rest, state, keyword?)] + end + + defp normalize_kw_args([{left, right} | rest] = current, state, keyword?) do + keyword? = keyword? or keyword?(current) + + left = + if keyword? do + meta = [format: :keyword] ++ meta_line(state) + {:__block__, meta, [maybe_escape_literal(left, state)]} + else + do_normalize(left, state) + end + + right = do_normalize(right, state) + + pair = + with {:__block__, meta, _} <- left, + :keyword <- meta[:format] do + {left, right} + else + _ -> {:__block__, meta_line(state), [{left, right}]} + end + + [pair | normalize_kw_args(rest, state, keyword?)] + end + + defp normalize_kw_args([first | rest], state, keyword?) do + [do_normalize(first, state) | normalize_kw_args(rest, state, keyword?)] + end + + defp normalize_kw_args([], _state, _keyword?) do + [] + end + + defp normalize_args(args, state) do + Enum.map(args, &do_normalize(&1, state)) + end + + defp maybe_escape_literal(string, %{escape: true}) when is_binary(string) do + {string, _} = Code.Identifier.escape(string, nil) + IO.iodata_to_binary(string) + end + + defp maybe_escape_literal(atom, %{escape: true} = state) when is_atom(atom) do + atom + |> Atom.to_string() + |> maybe_escape_literal(state) + |> String.to_atom() + end + + defp maybe_escape_literal(term, _) do + term + end + + defp binary_interpolated?(parts) do + Enum.all?(parts, fn + {:"::", _, [{{:., _, [Kernel, :to_string]}, _, [_]}, {:binary, _, _}]} -> true + binary when is_binary(binary) -> true + _ -> false + end) + end + + defp list_interpolated?(parts) do + Enum.all?(parts, fn + {{:., _, [Kernel, :to_string]}, _, [_]} -> true + binary when is_binary(binary) -> true + _ -> false + end) + end + + defp patch_meta_line(meta, parent_meta) do + with nil <- meta[:line], + line when is_integer(line) <- parent_meta[:line] do + [line: line] ++ meta + else + _ -> meta + end + end + + defp meta_line(state) do + if line = state.parent_meta[:line] do + [line: line] + else + [] + end + end + + defp keyword?([{{:__block__, key_meta, [key]}, _} | rest]) when is_atom(key) do + if key_meta[:format] == :keyword do + keyword?(rest) + else + false + end + end + + defp keyword?([{key, _value} | rest]) when is_atom(key) do + case Atom.to_charlist(key) do + ~c"Elixir." ++ _ -> false + _ -> keyword?(rest) + end + end + + defp keyword?([]), do: true + defp keyword?(_other), do: false +end diff --git a/lib/elixir/lib/code/typespec.ex b/lib/elixir/lib/code/typespec.ex new file mode 100644 index 00000000000..f04cac439f7 --- /dev/null +++ b/lib/elixir/lib/code/typespec.ex @@ -0,0 +1,430 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +defmodule Code.Typespec do + @moduledoc false + + @doc """ + Converts a spec clause back to Elixir quoted expression. + """ + @spec spec_to_quoted(atom, tuple) :: {atom, keyword, [Macro.t()]} + def spec_to_quoted(name, spec) + + def spec_to_quoted(name, {:type, anno, :fun, [{:type, _, :product, args}, result]}) + when is_atom(name) do + meta = meta(anno) + body = {name, meta, Enum.map(args, &typespec_to_quoted/1)} + + vars = + for type_expr <- args ++ [result], + var <- collect_vars(type_expr), + uniq: true, + do: {var, {:var, meta, nil}} + + spec = {:"::", meta, [body, typespec_to_quoted(result)]} + + if vars == [] do + spec + else + {:when, meta, [spec, vars]} + end + end + + def spec_to_quoted(name, {:type, anno, :bounded_fun, [type, constrs]}) when is_atom(name) do + meta = meta(anno) + {:type, _, :fun, [{:type, _, :product, args}, result]} = type + + guards = + for {:type, _, :constraint, [{:atom, _, :is_subtype}, [{:var, _, var}, type]]} <- constrs do + {erl_to_ex_var(var), typespec_to_quoted(type)} + end + + ignore_vars = Keyword.keys(guards) + + vars = + for type_expr <- args ++ [result], + var <- collect_vars(type_expr), + var not in ignore_vars, + uniq: true, + do: {var, {:var, meta, nil}} + + args = for arg <- args, do: typespec_to_quoted(arg) + + when_args = [ + {:"::", meta, [{name, meta, args}, typespec_to_quoted(result)]}, + guards ++ vars + ] + + {:when, meta, when_args} + end + + @doc """ + Converts a type clause back to Elixir AST. + """ + def type_to_quoted(type) + + def type_to_quoted({{:record, record}, fields, args}) when is_atom(record) do + fields = for field <- fields, do: typespec_to_quoted(field) + args = for arg <- args, do: typespec_to_quoted(arg) + type = {:{}, [], [record | fields]} + quote(do: unquote(record)(unquote_splicing(args)) :: unquote(type)) + end + + def type_to_quoted({name, type, args}) when is_atom(name) do + args = for arg <- args, do: typespec_to_quoted(arg) + quote(do: unquote(name)(unquote_splicing(args)) :: unquote(typespec_to_quoted(type))) + end + + @doc """ + Returns all types available from the module's BEAM code. + + The result is returned as a list of tuples where the first + element is the type (`:typep`, `:type` and `:opaque`). + + The module must have a corresponding BEAM file which can be + located by the runtime system. The types will be in the Erlang + Abstract Format. + """ + @spec fetch_types(module | binary) :: {:ok, [tuple]} | :error + def fetch_types(module) when is_atom(module) or is_binary(module) do + case typespecs_abstract_code(module) do + {:ok, abstract_code} -> + exported_types = for {:attribute, _, :export_type, types} <- abstract_code, do: types + exported_types = List.flatten(exported_types) + + types = + for {:attribute, _, kind, {name, _, args} = type} <- abstract_code, + kind in [:opaque, :type] do + cond do + kind == :opaque -> {:opaque, type} + {name, length(args)} in exported_types -> {:type, type} + true -> {:typep, type} + end + end + + {:ok, types} + + _ -> + :error + end + end + + @doc """ + Returns all specs available from the module's BEAM code. + + The result is returned as a list of tuples where the first + element is spec name and arity and the second is the spec. + + The module must have a corresponding BEAM file which can be + located by the runtime system. The types will be in the Erlang + Abstract Format. + """ + @spec fetch_specs(module | binary) :: {:ok, [tuple]} | :error + def fetch_specs(module) when is_atom(module) or is_binary(module) do + case typespecs_abstract_code(module) do + {:ok, abstract_code} -> + {:ok, for({:attribute, _, :spec, value} <- abstract_code, do: value)} + + :error -> + :error + end + end + + @doc """ + Returns all callbacks available from the module's BEAM code. + + The result is returned as a list of tuples where the first + element is spec name and arity and the second is the spec. + + The module must have a corresponding BEAM file + which can be located by the runtime system. The types will be + in the Erlang Abstract Format. + """ + @spec fetch_callbacks(module | binary) :: {:ok, [tuple]} | :error + def fetch_callbacks(module) when is_atom(module) or is_binary(module) do + case typespecs_abstract_code(module) do + {:ok, abstract_code} -> + {:ok, for({:attribute, _, :callback, value} <- abstract_code, do: value)} + + :error -> + :error + end + end + + defp typespecs_abstract_code(module) do + with {module, binary} <- get_module_and_beam(module), + {:ok, {_, [debug_info: {:debug_info_v1, backend, data}]}} <- + :beam_lib.chunks(binary, [:debug_info]) do + case data do + {:elixir_v1, %{}, specs} -> + # Fast path to avoid translation to Erlang from Elixir. + {:ok, specs} + + _ -> + case backend.debug_info(:erlang_v1, module, data, []) do + {:ok, abstract_code} -> {:ok, abstract_code} + _ -> :error + end + end + else + _ -> :error + end + end + + defp get_module_and_beam(module) when is_atom(module) do + with {^module, beam, _filename} <- :code.get_object_code(module), + info_pairs when is_list(info_pairs) <- :beam_lib.info(beam), + {:ok, ^module} <- Keyword.fetch(info_pairs, :module) do + {module, beam} + else + _ -> :error + end + end + + defp get_module_and_beam(beam) when is_binary(beam) do + case :beam_lib.info(beam) do + [_ | _] = info -> {info[:module], beam} + _ -> :error + end + end + + ## To AST conversion + + defp collect_vars({:ann_type, _anno, args}) when is_list(args) do + [] + end + + defp collect_vars({:type, _anno, _kind, args}) when is_list(args) do + Enum.flat_map(args, &collect_vars/1) + end + + defp collect_vars({:remote_type, _anno, args}) when is_list(args) do + Enum.flat_map(args, &collect_vars/1) + end + + defp collect_vars({:typed_record_field, _anno, type}) do + collect_vars(type) + end + + defp collect_vars({:paren_type, _anno, [type]}) do + collect_vars(type) + end + + defp collect_vars({:var, _anno, var}) do + [erl_to_ex_var(var)] + end + + defp collect_vars(_) do + [] + end + + defp typespec_to_quoted({:user_type, anno, name, args}) do + args = for arg <- args, do: typespec_to_quoted(arg) + {name, meta(anno), args} + end + + defp typespec_to_quoted({:type, anno, :tuple, :any}) do + {:tuple, meta(anno), []} + end + + defp typespec_to_quoted({:type, anno, :tuple, args}) do + args = for arg <- args, do: typespec_to_quoted(arg) + {:{}, meta(anno), args} + end + + defp typespec_to_quoted({:type, _anno, :list, [{:type, _, :union, unions} = arg]}) do + case unpack_typespec_kw(unions, []) do + {:ok, ast} -> ast + :error -> [typespec_to_quoted(arg)] + end + end + + defp typespec_to_quoted({:type, anno, :list, []}) do + {:list, meta(anno), []} + end + + defp typespec_to_quoted({:type, _anno, :list, [arg]}) do + [typespec_to_quoted(arg)] + end + + defp typespec_to_quoted({:type, anno, :nonempty_list, []}) do + [{:..., meta(anno), nil}] + end + + defp typespec_to_quoted({:type, anno, :nonempty_list, [arg]}) do + [typespec_to_quoted(arg), {:..., meta(anno), nil}] + end + + defp typespec_to_quoted({:type, anno, :map, :any}) do + {:map, meta(anno), []} + end + + defp typespec_to_quoted({:type, anno, :map, fields}) do + fields = + Enum.map(fields, fn + {:type, _, :map_field_assoc, :any} -> + {{:optional, [], [{:any, [], []}]}, {:any, [], []}} + + {:type, _, :map_field_exact, [{:atom, _, k}, v]} -> + {k, typespec_to_quoted(v)} + + {:type, _, :map_field_exact, [k, v]} -> + {{:required, [], [typespec_to_quoted(k)]}, typespec_to_quoted(v)} + + {:type, _, :map_field_assoc, [k, v]} -> + {{:optional, [], [typespec_to_quoted(k)]}, typespec_to_quoted(v)} + end) + + case List.keytake(fields, :__struct__, 0) do + {{:__struct__, struct}, fields_pruned} when is_atom(struct) and struct != nil -> + map_pruned = {:%{}, meta(anno), fields_pruned} + {:%, meta(anno), [struct, map_pruned]} + + _ -> + {:%{}, meta(anno), fields} + end + end + + defp typespec_to_quoted({:type, anno, :binary, [arg1, arg2]}) do + line = meta(anno)[:line] + + case {typespec_to_quoted(arg1), typespec_to_quoted(arg2)} do + {arg1, 0} -> + quote(line: line, do: <<_::unquote(arg1)>>) + + {0, arg2} -> + quote(line: line, do: <<_::_*unquote(arg2)>>) + + {arg1, arg2} -> + quote(line: line, do: <<_::unquote(arg1), _::_*unquote(arg2)>>) + end + end + + defp typespec_to_quoted({:type, anno, :union, args}) do + args = for arg <- args, do: typespec_to_quoted(arg) + Enum.reduce(Enum.reverse(args), fn arg, expr -> {:|, meta(anno), [arg, expr]} end) + end + + defp typespec_to_quoted({:type, anno, :fun, [{:type, _, :product, args}, result]}) do + args = for arg <- args, do: typespec_to_quoted(arg) + [{:->, meta(anno), [args, typespec_to_quoted(result)]}] + end + + defp typespec_to_quoted({:type, anno, :fun, [args, result]}) do + [{:->, meta(anno), [[typespec_to_quoted(args)], typespec_to_quoted(result)]}] + end + + defp typespec_to_quoted({:type, anno, :range, [left, right]}) do + {:.., meta(anno), [typespec_to_quoted(left), typespec_to_quoted(right)]} + end + + defp typespec_to_quoted({:type, _anno, nil, []}) do + [] + end + + defp typespec_to_quoted({:type, anno, name, args}) do + args = for arg <- args, do: typespec_to_quoted(arg) + {name, meta(anno), args} + end + + defp typespec_to_quoted({:var, anno, var}) do + {erl_to_ex_var(var), meta(anno), nil} + end + + defp typespec_to_quoted({:op, anno, op, arg}) when op in [:+, :-] do + {op, meta(anno), [typespec_to_quoted(arg)]} + end + + defp typespec_to_quoted({:op, anno, :*, arg1, arg2}) do + {:*, meta(anno), [typespec_to_quoted(arg1), typespec_to_quoted(arg2)]} + end + + defp typespec_to_quoted({:remote_type, anno, [mod, name, args]}) do + remote_type(anno, mod, name, args) + end + + defp typespec_to_quoted({:ann_type, anno, [var, type]}) do + {:"::", meta(anno), [typespec_to_quoted(var), typespec_to_quoted(type)]} + end + + defp typespec_to_quoted( + {:typed_record_field, {:record_field, anno1, {:atom, anno2, name}}, type} + ) do + typespec_to_quoted({:ann_type, anno1, [{:var, anno2, name}, type]}) + end + + defp typespec_to_quoted({:type, _, :any}) do + quote(do: ...) + end + + defp typespec_to_quoted({:paren_type, _, [type]}) do + typespec_to_quoted(type) + end + + defp typespec_to_quoted({type, _anno, atom}) when is_atom(type) do + atom + end + + defp typespec_to_quoted(other), do: other + + ## Helpers + + defp remote_type(anno, {:atom, _, :elixir}, {:atom, _, :charlist}, []) do + typespec_to_quoted({:type, anno, :charlist, []}) + end + + defp remote_type(anno, {:atom, _, :elixir}, {:atom, _, :nonempty_charlist}, []) do + typespec_to_quoted({:type, anno, :nonempty_charlist, []}) + end + + defp remote_type(anno, {:atom, _, :elixir}, {:atom, _, :struct}, []) do + typespec_to_quoted({:type, anno, :struct, []}) + end + + defp remote_type(anno, {:atom, _, :elixir}, {:atom, _, :as_boolean}, [arg]) do + typespec_to_quoted({:type, anno, :as_boolean, [arg]}) + end + + defp remote_type(anno, {:atom, _, :elixir}, {:atom, _, :keyword}, args) do + typespec_to_quoted({:type, anno, :keyword, args}) + end + + defp remote_type(anno, mod, name, args) do + args = for arg <- args, do: typespec_to_quoted(arg) + dot = {:., meta(anno), [typespec_to_quoted(mod), typespec_to_quoted(name)]} + {dot, meta(anno), args} + end + + defp erl_to_ex_var(var) do + case Atom.to_string(var) do + <<"_", c::utf8, rest::binary>> -> + String.to_atom("_#{String.downcase(<>)}#{rest}") + + <> -> + String.to_atom("#{String.downcase(<>)}#{rest}") + end + end + + defp unpack_typespec_kw([{:type, _, :tuple, [{:atom, _, atom}, type]} | t], acc) do + unpack_typespec_kw(t, [{atom, typespec_to_quoted(type)} | acc]) + end + + defp unpack_typespec_kw([], acc) do + {:ok, Enum.reverse(acc)} + end + + defp unpack_typespec_kw(_, _acc) do + :error + end + + defp meta(anno) do + case :erl_anno.location(anno) do + {line, column} -> + [line: line, column: column] + + line when is_integer(line) -> + [line: line] + end + end +end diff --git a/lib/elixir/lib/collectable.ex b/lib/elixir/lib/collectable.ex index 426f290a4e7..3433fb3f204 100644 --- a/lib/elixir/lib/collectable.ex +++ b/lib/elixir/lib/collectable.ex @@ -1,3 +1,7 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + defprotocol Collectable do @moduledoc """ A protocol to traverse data structures. @@ -8,110 +12,184 @@ defprotocol Collectable do iex> Enum.into([a: 1, b: 2], %{}) %{a: 1, b: 2} - If a collection implements both `Enumerable` and `Collectable`, both - operations can be combined with `Enum.traverse/2`: - - iex> Enum.traverse(%{a: 1, b: 2}, fn {k, v} -> {k, v * 2} end) - %{a: 2, b: 4} - ## Why Collectable? The `Enumerable` protocol is useful to take values out of a collection. In order to support a wide range of values, the functions provided by the `Enumerable` protocol do not keep shape. For example, passing a - dictionary to `Enum.map/2` always returns a list. + map to `Enum.map/2` always returns a list. This design is intentional. `Enumerable` was designed to support infinite collections, resources and other structures with fixed shape. For example, - it doesn't make sense to insert values into a range, as it has a fixed - shape where just the range limits are stored. + it doesn't make sense to insert values into a `Range`, as it has a + fixed shape where only the range limits and step are stored. The `Collectable` module was designed to fill the gap left by the - `Enumerable` protocol. It provides two functions: `into/1` and `empty/1`. + `Enumerable` protocol. `Collectable.into/1` can be seen as the opposite of + `Enumerable.reduce/3`. If the functions in `Enumerable` are about taking values out, + then `Collectable.into/1` is about collecting those values into a structure. + + ## Examples + + To show how to manually use the `Collectable` protocol, let's play with a + simplified implementation for `MapSet`. + + iex> {initial_acc, collector_fun} = Collectable.into(MapSet.new()) + iex> updated_acc = Enum.reduce([1, 2, 3], initial_acc, fn elem, acc -> + ...> collector_fun.(acc, {:cont, elem}) + ...> end) + iex> collector_fun.(updated_acc, :done) + MapSet.new([1, 2, 3]) + + To show how the protocol can be implemented, we can again look at the + simplified implementation for `MapSet`. In this implementation "collecting" elements + simply means inserting them in the set through `MapSet.put/2`. + + defimpl Collectable, for: MapSet do + def into(map_set) do + collector_fun = fn + map_set_acc, {:cont, elem} -> + MapSet.put(map_set_acc, elem) + + map_set_acc, :done -> + map_set_acc - `into/1` can be seen as the opposite of `Enumerable.reduce/3`. If - `Enumerable` is about taking values out, `Collectable.into/1` is about - collecting those values into a structure. + _map_set_acc, :halt -> + :ok + end + + initial_acc = map_set + + {initial_acc, collector_fun} + end + end + + So now we can call `Enum.into/2`: + + iex> Enum.into([1, 2, 3], MapSet.new()) + MapSet.new([1, 2, 3]) - `empty/1` receives a collectable and returns an empty version of the - same collectable. By combining the enumerable functionality with `into/1` - and `empty/1`, one can, for example, implement a traversal mechanism. """ @type command :: {:cont, term} | :done | :halt @doc """ - Receives a collectable structure and returns an empty one. - """ - @spec empty(t) :: t - def empty(collectable) + Returns an initial accumulator and a "collector" function. - @doc """ - Returns a function that collects values alongside - the initial accumulation value. + Receives a `collectable` which can be used as the initial accumulator that will + be passed to the function. - The returned function receives a collectable and injects a given - value into it for every `{:cont, term}` instruction. + The collector function receives a term and a command and injects the term into + the collectable accumulator on every `{:cont, term}` command. - `:done` is passed when no further values will be injected, useful - for closing resources and normalizing values. A collectable must - be returned on `:done`. + `:done` is passed as a command when no further values will be injected. This + is useful when there's a need to close resources or normalizing values. A + collectable must be returned when the command is `:done`. - If injection is suddenly interrupted, `:halt` is passed and it can - return any value, as it won't be used. + If injection is suddenly interrupted, `:halt` is passed and the function + can return any value as it won't be used. + + For examples on how to use the `Collectable` protocol and `into/1` see the + module documentation. """ - @spec into(t) :: {term, (term, command -> t | term)} + @spec into(t) :: {initial_acc :: term, collector :: (term, command -> t | term)} def into(collectable) end defimpl Collectable, for: List do - def empty(_list) do - [] - end - - def into(original) do - {[], fn - list, {:cont, x} -> [x|list] - list, :done -> original ++ :lists.reverse(list) - _, :halt -> :ok - end} + def into(list) do + # TODO: Change the behavior so the into always comes last on Elixir v2.0 + if list != [] do + IO.warn( + "the Collectable protocol is deprecated for non-empty lists. The behavior of " <> + "Enum.into/2 and \"for\" comprehensions with an :into option is incorrect " <> + "when collecting into non-empty lists. If you're collecting into a non-empty keyword " <> + "list, consider using Keyword.merge/2 instead. If you're collecting into a non-empty " <> + "list, consider concatenating the two lists with the ++ operator." + ) + end + + fun = fn + list_acc, {:cont, elem} -> + [elem | list_acc] + + list_acc, :done -> + list ++ :lists.reverse(list_acc) + + _list_acc, :halt -> + :ok + end + + {[], fun} end end defimpl Collectable, for: BitString do - def empty(_bitstring) do - "" - end + def into(binary) when is_binary(binary) do + fun = fn + acc, {:cont, x} when is_binary(x) and is_list(acc) -> + [acc | x] - def into(original) do - {original, fn - bitstring, {:cont, x} -> <> - bitstring, :done -> bitstring - _, :halt -> :ok - end} - end -end + acc, {:cont, x} when is_bitstring(x) and is_bitstring(acc) -> + <> + + acc, {:cont, x} when is_bitstring(x) -> + <> + + acc, :done when is_bitstring(acc) -> + acc + + acc, :done -> + IO.iodata_to_binary(acc) + + __acc, :halt -> + :ok + + _acc, {:cont, other} -> + raise ArgumentError, + "collecting into a binary requires a bitstring, got: #{inspect(other)}" + end -defimpl Collectable, for: Function do - def empty(function) do - function + {[binary], fun} end - def into(function) do - {function, function} + def into(bitstring) do + fun = fn + acc, {:cont, x} when is_bitstring(x) -> + <> + + acc, :done -> + acc + + _acc, :halt -> + :ok + + _acc, {:cont, other} -> + raise ArgumentError, + "collecting into a bitstring requires a bitstring, got: #{inspect(other)}" + end + + {bitstring, fun} end end defimpl Collectable, for: Map do - def empty(_map) do - %{} - end + def into(map) do + fun = fn + map_acc, {:cont, {key, value}} -> + Map.put(map_acc, key, value) + + map_acc, :done -> + map_acc + + _map_acc, :halt -> + :ok + + _map_acc, {:cont, other} -> + raise ArgumentError, + "collecting into a map requires {key, value} tuples, got: #{inspect(other)}" + end - def into(original) do - {original, fn - map, {:cont, {k, v}} -> :maps.put(k, v, map) - map, :done -> map - _, :halt -> :ok - end} + {map, fun} end end diff --git a/lib/elixir/lib/config.ex b/lib/elixir/lib/config.ex new file mode 100644 index 00000000000..409512b84f2 --- /dev/null +++ b/lib/elixir/lib/config.ex @@ -0,0 +1,397 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +defmodule Config do + @moduledoc ~S""" + A simple keyword-based configuration API. + + ## Example + + This module is most commonly used to define application configuration, + typically in `config/config.exs`: + + import Config + + config :some_app, + key1: "value1", + key2: "value2" + + import_config "#{config_env()}.exs" + + `import Config` will import the functions `config/2`, `config/3` + `config_env/0`, `config_target/0`, and `import_config/1` + to help you manage your configuration. + + `config/2` and `config/3` are used to define key-value configuration + for a given application. Once Mix starts, it will automatically + evaluate the configuration file and persist the configuration above + into `:some_app`'s application environment, which can be accessed in + as follows: + + "value1" = Application.fetch_env!(:some_app, :key1) + + Finally, the line `import_config "#{config_env()}.exs"` will import + other config files based on the current configuration environment, + such as `config/dev.exs` and `config/test.exs`. + + `Config` also provides a low-level API for evaluating and reading + configuration, under the `Config.Reader` module. + + > #### Avoid application environment in libraries {: .info} + > + > If you are writing a library to be used by other developers, + > it is generally recommended to avoid the application environment, as the + > application environment is effectively a global storage. Also note that + > the `config/config.exs` of a library is not evaluated when the library is + > used as a dependency, as configuration is always meant to configure the + > current project. For more information, see ["Using application configuration for + > libraries"](design-anti-patterns.md#using-application-configuration-for-libraries). + + ## Migrating from `use Mix.Config` + + The `Config` module in Elixir was introduced in v1.9 as a replacement to + `use Mix.Config`, which was specific to Mix and has been deprecated. + + You can leverage `Config` instead of `use Mix.Config` in three steps. The first + step is to replace `use Mix.Config` at the top of your config files by + `import Config`. + + The second is to make sure your `import_config/1` calls do not have a + wildcard character. If so, you need to perform the wildcard lookup + manually. For example, if you did: + + import_config "../apps/*/config/config.exs" + + It has to be replaced by: + + for config <- "../apps/*/config/config.exs" |> Path.expand(__DIR__) |> Path.wildcard() do + import_config config + end + + The last step is to replace all `Mix.env()` calls in the config files with `config_env()`. + + Keep in mind you must also avoid using `Mix.env()` inside your project files. + To check the environment at _runtime_, you may add a configuration key: + + # config.exs + ... + config :my_app, env: config_env() + + Then, in other scripts and modules, you may get the environment with + `Application.fetch_env!/2`: + + # router.exs + ... + if Application.fetch_env!(:my_app, :env) == :prod do + ... + end + + The only places where you may access functions from the `Mix` module are + the `mix.exs` file and inside custom Mix tasks, which are always within + the `Mix.Tasks` namespace. + + ## `config/runtime.exs` + + For runtime configuration, you can use the `config/runtime.exs` file. + It is executed right before applications start in both Mix and releases + (assembled with `mix release`). + """ + + @type config_opts :: [ + imports: [Path.t()] | :disabled, + env: atom(), + target: atom() + ] + + @opts_key {__MODULE__, :opts} + @config_key {__MODULE__, :config} + @imports_key {__MODULE__, :imports} + + defp get_opts!(), do: Process.get(@opts_key) || raise_improper_use!() + defp put_opts(value), do: Process.put(@opts_key, value) + defp delete_opts(), do: Process.delete(@opts_key) + + defp get_config!(), do: Process.get(@config_key) || raise_improper_use!() + defp put_config(value), do: Process.put(@config_key, value) + defp delete_config(), do: Process.delete(@config_key) + + defp get_imports!(), do: Process.get(@imports_key) || raise_improper_use!() + defp put_imports(value), do: Process.put(@imports_key, value) + defp delete_imports(), do: Process.delete(@imports_key) + + defp raise_improper_use!() do + raise "could not set configuration via Config. " <> + "This usually means you are trying to execute a configuration file " <> + "directly, instead of reading it with Config.Reader" + end + + @doc """ + Configures the given `root_key`. + + Keyword lists are always deep-merged. + + ## Examples + + The given `opts` are merged into the existing configuration + for the given `root_key`. Conflicting keys are overridden by the + ones specified in `opts`, unless they are keywords, which are + deep merged recursively. For example, the application configuration + below + + config :logger, + level: :warn, + + config :logger, + level: :info, + truncate: 1024 + + will have a final configuration for `:logger` of: + + [level: :info, truncate: 1024] + + """ + @doc since: "1.9.0" + def config(root_key, opts) when is_atom(root_key) and is_list(opts) do + if not Keyword.keyword?(opts) do + raise ArgumentError, "config/2 expected a keyword list, got: #{inspect(opts)}" + end + + get_config!() + |> __merge__([{root_key, opts}]) + |> put_config() + end + + @doc """ + Configures the given `key` for the given `root_key`. + + Keyword lists are always deep merged. + + ## Examples + + The given `opts` are merged into the existing values for `key` + in the given `root_key`. Conflicting keys are overridden by the + ones specified in `opts`, unless they are keywords, which are + deep merged recursively. For example, the application configuration + below + + config :ecto, Repo, + log_level: :warn, + adapter: Ecto.Adapters.Postgres, + metadata: [read_only: true] + + config :ecto, Repo, + log_level: :info, + pool_size: 10, + metadata: [replica: true] + + will have a final value of the configuration for the `Repo` + key in the `:ecto` application of: + + Application.get_env(:ecto, Repo) + #=> [ + #=> log_level: :info, + #=> pool_size: 10, + #=> adapter: Ecto.Adapters.Postgres, + #=> metadata: [read_only: true, replica: true] + #=> ] + + """ + @doc since: "1.9.0" + def config(root_key, key, opts) when is_atom(root_key) and is_atom(key) do + get_config!() + |> __merge__([{root_key, [{key, opts}]}]) + |> put_config() + end + + @doc """ + Reads the configuration for the given root key. + + This function only reads the configuration from a previous + `config/2` or `config/3` call. If `root_key` points to an + application, it does not read its actual application environment. + Its main use case is to make it easier to access and share + configuration values across files. + + If the `root_key` was not configured, it returns `nil`. + + ## Examples + + # In config/config.exs + config :my_app, foo: :bar + + # In config/dev.exs + config :another_app, foo: read_config(:my_app)[:foo] || raise "missing parent configuration" + + """ + @doc since: "1.18.0" + def read_config(root_key) when is_atom(root_key) do + get_config!()[root_key] + end + + @doc """ + Returns the environment this configuration file is executed on. + + In Mix projects this function returns the environment this configuration + file is executed on. + In releases, returns the `MIX_ENV` specified when running `mix release`. + + This is most often used to execute conditional code: + + if config_env() == :prod do + config :my_app, :debug, false + end + + """ + @doc since: "1.11.0" + defmacro config_env() do + quote do + Config.__env__!() + end + end + + @doc false + @spec __env__!() :: atom() + def __env__!() do + elem(get_opts!(), 0) || raise "no :env key was given to this configuration file" + end + + @doc """ + Returns the target this configuration file is executed on. + + This is most often used to execute conditional code: + + if config_target() == :host do + config :my_app, :debug, false + end + + """ + @doc since: "1.11.0" + defmacro config_target() do + quote do + Config.__target__!() + end + end + + @doc false + @spec __target__!() :: atom() + def __target__!() do + elem(get_opts!(), 1) || raise "no :target key was given to this configuration file" + end + + @doc ~S""" + Imports configuration from the given file. + + In case the file doesn't exist, an error is raised. + + If file is a relative, it will be expanded relatively to the + directory the current configuration file is in. + + ## Examples + + This is often used to emulate configuration across environments: + + import_config "#{config_env()}.exs" + + Note, however, some configuration files, such as `config/runtime.exs` + does not support imports, as they are meant to be copied across + systems. + """ + @doc since: "1.9.0" + defmacro import_config(file) do + quote do + Config.__import__!(Path.expand(unquote(file), __DIR__)) + :ok + end + end + + @doc false + @spec __import__!(Path.t()) :: {term, Code.binding()} + def __import__!(file) when is_binary(file) do + import_config!(file, File.read!(file), true) + end + + @doc false + @spec __eval__!(Path.t(), binary(), config_opts) :: {keyword, [Path.t()] | :disabled} + def __eval__!(file, content, opts \\ []) when is_binary(file) and is_list(opts) do + env = Keyword.get(opts, :env) + target = Keyword.get(opts, :target) + imports = Keyword.get(opts, :imports, []) + + previous_opts = put_opts({env, target}) + previous_config = put_config([]) + previous_imports = put_imports(imports) + + try do + {eval_config, _} = import_config!(file, content, false) + + case get_config!() do + [] when is_list(eval_config) -> + {validate!(eval_config, file), get_imports!()} + + pdict_config -> + {pdict_config, get_imports!()} + end + after + if previous_opts, do: put_opts(previous_opts), else: delete_opts() + if previous_config, do: put_config(previous_config), else: delete_config() + if previous_imports, do: put_imports(previous_imports), else: delete_imports() + end + end + + defp import_config!(file, contents, raise_when_disabled?) do + current_imports = get_imports!() + + cond do + current_imports == :disabled -> + if raise_when_disabled? do + raise "import_config/1 is not enabled for this configuration file. " <> + "Some configuration files do not allow importing other files " <> + "as they are often copied to external systems" + end + + file in current_imports -> + raise ArgumentError, + "attempting to load configuration #{Path.relative_to_cwd(file)} recursively" + + true -> + put_imports([file | current_imports]) + :ok + end + + Code.eval_string(contents, [], file: file) + end + + @doc false + def __merge__(config1, config2) when is_list(config1) and is_list(config2) do + Keyword.merge(config1, config2, fn _, app1, app2 -> + Keyword.merge(app1, app2, &deep_merge/3) + end) + end + + defp deep_merge(_key, value1, value2) do + if Keyword.keyword?(value1) and Keyword.keyword?(value2) do + Keyword.merge(value1, value2, &deep_merge/3) + else + value2 + end + end + + defp validate!(config, file) do + Enum.all?(config, fn + {app, value} when is_atom(app) -> + if Keyword.keyword?(value) do + true + else + raise ArgumentError, + "expected config for app #{inspect(app)} in #{Path.relative_to_cwd(file)} " <> + "to return keyword list, got: #{inspect(value)}" + end + + _ -> + false + end) + + config + end +end diff --git a/lib/elixir/lib/config/provider.ex b/lib/elixir/lib/config/provider.ex new file mode 100644 index 00000000000..b039a2233c7 --- /dev/null +++ b/lib/elixir/lib/config/provider.ex @@ -0,0 +1,449 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +defmodule Config.Provider do + @moduledoc """ + Specifies a provider API that loads configuration during boot. + + Config providers are typically used during releases to load + external configuration while the system boots. This is done + by starting the VM with the minimum amount of applications + running, then invoking all of the providers, and then + restarting the system. This requires a mutable configuration + file on disk, as the results of the providers are written to + the file system. For more information on runtime configuration, + see `mix release`. + + ## Multiple config files + + One common use of config providers is to specify multiple + configuration files in a release. Elixir ships with one provider, + called `Config.Reader`, which is capable of handling Elixir's + built-in config files. + + For example, imagine you want to list some basic configuration + on Mix's built-in `config/runtime.exs` file, but you also want + to support additional configuration files. To do so, you can add + this inside the `def project` portion of your `mix.exs`: + + releases: [ + demo: [ + config_providers: [ + {Config.Reader, {:system, "RELEASE_ROOT", "/extra_config.exs"}} + ] + ] + ] + + You can place this `extra_config.exs` file in your release in + multiple ways: + + 1. If it is available on the host when assembling the release, + you can place it on "rel/overlays/extra_config.exs" and it + will be automatically copied to the release root + + 2. If it is available on the target during deployment, you can + simply copy it to the release root as a step in your deployment + + Now once the system boots, it will load both `config/runtime.exs` + and `extra_config.exs` early in the boot process. You can learn + more options on `Config.Reader`. + + ## Custom config provider + + You can also implement custom config providers, similar to how + `Config.Reader` works. For example, imagine you need to load + some configuration from a JSON file and load that into the system. + Said configuration provider would look like: + + defmodule JSONConfigProvider do + @behaviour Config.Provider + + # Let's pass the path to the JSON file as config + @impl true + def init(path) when is_binary(path), do: path + + @impl true + def load(config, path) do + # We need to start any app we may depend on. + {:ok, _} = Application.ensure_all_started(:jason) + + json = path |> File.read!() |> Jason.decode!() + + Config.Reader.merge( + config, + my_app: [ + some_value: json["my_app_some_value"], + another_value: json["my_app_another_value"], + ] + ) + end + end + + Then, when specifying your release, you can specify the provider in + the release configuration: + + releases: [ + demo: [ + config_providers: [ + {JSONConfigProvider, "/etc/config.json"} + ] + ] + ] + + """ + + @type config :: keyword + @type state :: term + + @typedoc """ + A path pointing to a configuration file. + + Since configuration files are often accessed on target machines, + it can be expressed either as: + + * a binary representing an absolute path + + * a `{:system, system_var, path}` tuple where the config is the + concatenation of the environment variable `system_var` with + the given `path` + + """ + @type config_path :: {:system, binary(), binary()} | binary() + + @typedoc """ + Options for `init/3`. + """ + @type init_opts :: [ + extra_config: config(), + prune_runtime_sys_config_after_boot: boolean(), + reboot_system_after_config: boolean(), + validate_compile_env: [{atom(), [atom()], term()}] + ] + + @doc """ + Invoked when initializing a config provider. + + A config provider is typically initialized on the machine + where the system is assembled and not on the target machine. + The `c:init/1` callback is useful to verify the arguments + given to the provider and prepare the state that will be + given to `c:load/2`. + + Furthermore, because the state returned by `c:init/1` can + be written to text-based config files, it should be + restricted only to simple data types, such as integers, + strings, atoms, tuples, maps, and lists. Entries such as + PIDs, references, and functions cannot be serialized. + """ + @callback init(term) :: state + + @doc """ + Loads configuration (typically during system boot). + + It receives the current `config` and the `state` returned by + `c:init/1`. Then, you typically read the extra configuration + from an external source and merge it into the received `config`. + Merging should be done with `Config.Reader.merge/2`, as it + performs deep merge. It should return the updated config. + + Note that `c:load/2` is typically invoked very early in the + boot process, therefore if you need to use an application + in the provider, it is your responsibility to start it. + """ + @callback load(config, state) :: config + + @doc false + defstruct [ + :providers, + :config_path, + extra_config: [], + prune_runtime_sys_config_after_boot: false, + reboot_system_after_config: false, + validate_compile_env: false + ] + + @reserved_apps [:kernel, :stdlib] + + @doc """ + Validates a `t:config_path/0`. + """ + @doc since: "1.9.0" + @spec validate_config_path!(config_path) :: :ok + def validate_config_path!({:system, name, path}) + when is_binary(name) and is_binary(path), + do: :ok + + def validate_config_path!(path) do + if is_binary(path) and Path.type(path) != :relative do + :ok + else + raise ArgumentError, """ + expected configuration path to be: + + * a binary representing an absolute path + * a tuple {:system, system_var, path} where the config is the \ + concatenation of the `system_var` with the given `path` + + Got: #{inspect(path)} + """ + end + end + + @doc """ + Resolves a `t:config_path/0` to an actual path. + """ + @doc since: "1.9.0" + @spec resolve_config_path!(config_path) :: binary + def resolve_config_path!(path) when is_binary(path), do: path + def resolve_config_path!({:system, name, path}), do: System.fetch_env!(name) <> path + + # Private keys + @init_key :config_provider_init + @booted_key :config_provider_booted + + # Public keys + @reboot_mode_key :config_provider_reboot_mode + + @doc false + @spec init([{module(), term()}], config_path(), init_opts()) :: config() + def init(providers, config_path, opts \\ []) when is_list(providers) and is_list(opts) do + validate_config_path!(config_path) + providers = for {provider, init} <- providers, do: {provider, provider.init(init)} + init = struct!(%Config.Provider{config_path: config_path, providers: providers}, opts) + [elixir: [{@init_key, init}]] + end + + @doc false + def boot(reboot_fun \\ &restart_and_sleep/0) do + # The config provider typically runs very early in the + # release process, so we need to make sure Elixir is started + # before we go around running Elixir code. + {:ok, _} = :application.ensure_all_started(:elixir) + + case Application.fetch_env(:elixir, @booted_key) do + {:ok, {:booted, path}} -> + path && File.rm(path) + + with {:ok, %Config.Provider{} = provider} <- Application.fetch_env(:elixir, @init_key) do + maybe_validate_compile_env(provider) + end + + :booted + + _ -> + case Application.fetch_env(:elixir, @init_key) do + {:ok, %Config.Provider{} = provider} -> + path = resolve_config_path!(provider.config_path) + reboot_config = [elixir: [{@booted_key, booted_value(provider, path)}]] + boot_providers(path, provider, reboot_config, reboot_fun) + + _ -> + :skip + end + end + end + + defp boot_providers(path, provider, reboot_config, reboot_fun) do + original_config = read_config!(path) + + config = + original_config + |> Config.__merge__(provider.extra_config) + |> run_providers(provider) + + if provider.reboot_system_after_config do + config + |> Config.__merge__(reboot_config) + |> write_config!(path) + + reboot_fun.() + else + for app <- @reserved_apps, config[app] != original_config[app] do + abort(""" + Cannot configure #{inspect(app)} because :reboot_system_after_config has been set \ + to false and #{inspect(app)} has already been loaded, meaning any further \ + configuration won't have an effect. + + The configuration for #{inspect(app)} before config providers was: + + #{inspect(original_config[app])} + + The configuration for #{inspect(app)} after config providers was: + + #{inspect(config[app])} + """) + end + + _ = Application.put_all_env(config, persistent: true) + maybe_validate_compile_env(provider) + :ok + end + end + + defp maybe_validate_compile_env(provider) do + with [_ | _] = compile_env <- provider.validate_compile_env, + {:error, message} <- validate_compile_env(compile_env) do + abort(message) + end + end + + @doc false + def valid_compile_env?(compile_env) do + Enum.all?(compile_env, fn {app, [key | path], compile_return} -> + try do + traverse_env(Application.fetch_env(app, key), path) == compile_return + rescue + _ -> false + end + end) + end + + @doc false + def validate_compile_env(compile_env, ensure_loaded? \\ true) + + def validate_compile_env([{app, [key | path], compile_return} | compile_env], ensure_loaded?) do + if ensure_app_loaded?(app, ensure_loaded?) do + try do + traverse_env(Application.fetch_env(app, key), path) + rescue + e -> + {:error, + """ + application #{inspect(app)} failed reading its compile environment #{path(key, path)}: + + #{Exception.format(:error, e, __STACKTRACE__)} + + Expected it to match the compile time value of #{return_to_text(compile_return)}. + + #{compile_env_tips(app)} + """} + else + ^compile_return -> + validate_compile_env(compile_env, ensure_loaded?) + + runtime_return -> + {:error, + """ + the application #{inspect(app)} has a different value set #{path(key, path)} \ + during runtime compared to compile time. Since this application environment entry was \ + marked as compile time, this difference can lead to different behavior than expected: + + * Compile time value #{return_to_text(compile_return)} + * Runtime value #{return_to_text(runtime_return)} + + #{compile_env_tips(app)} + """} + end + else + validate_compile_env(compile_env, ensure_loaded?) + end + end + + def validate_compile_env([], _ensure_loaded?) do + :ok + end + + defp ensure_app_loaded?(app, true), do: Application.ensure_loaded(app) == :ok + defp ensure_app_loaded?(app, false), do: Application.spec(app, :vsn) != nil + + defp path(key, []), do: "for key #{inspect(key)}" + defp path(key, path), do: "for path #{inspect(path)} inside key #{inspect(key)}" + + defp compile_env_tips(app), + do: """ + To fix this error, you might: + + * Make the runtime value match the compile time one + + * Recompile your project. If the misconfigured application is a dependency, \ + you may need to run "mix deps.clean #{app} --build" + + * Alternatively, you can disable this check. If you are using releases, you can \ + set :validate_compile_env to false in your release configuration. If you are \ + using Mix to start your system, you can pass the --no-validate-compile-env flag + """ + + defp return_to_text({:ok, value}), do: "was set to: #{inspect(value)}" + defp return_to_text(:error), do: "was not set" + + defp traverse_env(return, []), do: return + defp traverse_env(:error, _paths), do: :error + defp traverse_env({:ok, value}, [key | keys]), do: traverse_env(Access.fetch(value, key), keys) + + @compile {:no_warn_undefined, {:init, :restart, 1}} + defp restart_and_sleep() do + mode = Application.get_env(:elixir, @reboot_mode_key) + + if mode in [:embedded, :interactive] do + :init.restart(mode: mode) + else + :init.restart() + end + + Process.sleep(:infinity) + end + + defp booted_value(%{prune_runtime_sys_config_after_boot: true}, path), do: {:booted, path} + defp booted_value(%{prune_runtime_sys_config_after_boot: false}, _path), do: {:booted, nil} + + defp read_config!(path) do + case :file.consult(path) do + {:ok, [inner]} -> + inner + + {:error, reason} -> + bad_path_abort( + "Could not read runtime configuration due to reason: #{inspect(reason)}", + path + ) + end + end + + defp run_providers(config, %{providers: providers}) do + Enum.reduce(providers, config, fn {provider, state}, acc -> + try do + provider.load(acc, state) + catch + kind, error -> + IO.puts(:stderr, "ERROR! Config provider #{inspect(provider)} failed with:") + IO.puts(:stderr, Exception.format(kind, error, __STACKTRACE__)) + :erlang.raise(kind, error, __STACKTRACE__) + else + term when is_list(term) -> + term + + term -> + abort("Expected provider #{inspect(provider)} to return a list, got: #{inspect(term)}") + end + end) + end + + defp write_config!(config, path) do + contents = :io_lib.format("%% coding: utf-8~n~tw.~n", [config]) + + case File.write(path, IO.chardata_to_string(contents)) do + :ok -> + :ok + + {:error, reason} -> + bad_path_abort( + "Could not write runtime configuration due to reason: #{inspect(reason)}", + path + ) + end + end + + defp bad_path_abort(msg, path) do + abort( + msg <> + ". Please make sure #{inspect(path)} is writable and accessible " <> + "or choose a different path" + ) + end + + defp abort(msg) do + IO.puts("ERROR! " <> msg) + :erlang.raise(:error, "aborting boot", [{Config.Provider, :boot, 2, []}]) + end +end diff --git a/lib/elixir/lib/config/reader.ex b/lib/elixir/lib/config/reader.ex new file mode 100644 index 00000000000..3a259d3602e --- /dev/null +++ b/lib/elixir/lib/config/reader.ex @@ -0,0 +1,148 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +defmodule Config.Reader do + @moduledoc """ + API for reading config files defined with `Config`. + + ## As a provider + + `Config.Reader` can also be used as a `Config.Provider`. A config + provider is used during releases to customize how applications are + configured. When used as a provider, it expects a single argument: + the configuration path (as outlined in `t:Config.Provider.config_path/0`) + for the file to be read and loaded during the system boot. + + For example, if you expect the target system to have a config file + in an absolute path, you can add this inside the `def project` portion + of your `mix.exs`: + + releases: [ + demo: [ + config_providers: [ + {Config.Reader, "/etc/config.exs"} + ] + ] + ] + + Or if you want to read a custom path inside the release: + + config_providers: [{Config.Reader, {:system, "RELEASE_ROOT", "/config.exs"}}] + + You can also pass a keyword list of options to the reader, + where the `:path` is a required key: + + config_providers: [ + {Config.Reader, + path: "/etc/config.exs", + env: :prod, + imports: :disabled} + ] + + Remember Mix already loads `config/runtime.exs` by default. + For more examples and scenarios, see the `Config.Provider` module. + """ + + @behaviour Config.Provider + + @type config_opts :: [ + imports: [Path.t()] | :disabled, + env: atom(), + target: atom() + ] + + @impl true + def init(opts) when is_list(opts) do + {path, opts} = Keyword.pop!(opts, :path) + Config.Provider.validate_config_path!(path) + {path, opts} + end + + def init(path) do + init(path: path) + end + + @impl true + def load(config, {path, opts}) do + merge(config, path |> Config.Provider.resolve_config_path!() |> read!(opts)) + end + + @doc """ + Evaluates the configuration `contents` for the given `file`. + + Accepts the same options as `read!/2`. + """ + @doc since: "1.11.0" + @spec eval!(Path.t(), binary, config_opts) :: keyword + def eval!(file, contents, opts \\ []) + when is_binary(file) and is_binary(contents) and is_list(opts) do + Config.__eval__!(Path.expand(file), contents, opts) |> elem(0) + end + + @doc """ + Reads the configuration file. + + ## Options + + * `:imports` - a list of already imported paths or `:disabled` + to disable imports + + * `:env` - the environment the configuration file runs on. + See `Config.config_env/0` for sample usage + + * `:target` - the target the configuration file runs on. + See `Config.config_target/0` for sample usage + + """ + @doc since: "1.9.0" + @spec read!(Path.t(), config_opts) :: keyword + def read!(file, opts \\ []) when is_binary(file) and is_list(opts) do + file = Path.expand(file) + Config.__eval__!(file, File.read!(file), opts) |> elem(0) + end + + @doc """ + Reads the given configuration file and returns the configuration + with its imports. + + Accepts the same options as `read!/2`. Although note the `:imports` + option cannot be disabled in `read_imports!/2`. + """ + @doc since: "1.9.0" + @spec read_imports!(Path.t(), config_opts) :: {keyword, [Path.t()]} + def read_imports!(file, opts \\ []) when is_binary(file) and is_list(opts) do + if opts[:imports] == :disabled do + raise ArgumentError, ":imports must be a list of paths" + end + + file = Path.expand(file) + Config.__eval__!(file, File.read!(file), opts) + end + + @doc """ + Merges two configurations. + + The configurations are merged together with the values in + the second one having higher preference than the first in + case of conflicts. In case both values are set to keyword + lists, it deep merges them. + + ## Examples + + iex> Config.Reader.merge([app: [k: :v1]], [app: [k: :v2]]) + [app: [k: :v2]] + + iex> Config.Reader.merge([app: [k: [v1: 1, v2: 2]]], [app: [k: [v2: :a, v3: :b]]]) + [app: [k: [v1: 1, v2: :a, v3: :b]]] + + iex> Config.Reader.merge([app1: []], [app2: []]) + [app1: [], app2: []] + + """ + @doc since: "1.9.0" + @spec merge(keyword, keyword) :: keyword + def merge(config1, config2) when is_list(config1) and is_list(config2) do + Config.__merge__(config1, config2) + end +end diff --git a/lib/elixir/lib/dict.ex b/lib/elixir/lib/dict.ex index 3f3b5d3e444..7a0cd02d2e4 100644 --- a/lib/elixir/lib/dict.ex +++ b/lib/elixir/lib/dict.ex @@ -1,139 +1,39 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + defmodule Dict do @moduledoc ~S""" - This module specifies the Dict API expected to be - implemented by different dictionaries. It also provides - functions that redirect to the underlying Dict, allowing - a developer to work with different Dict implementations - using one API. - - To create a new dict, use the `new` functions defined - by each dict type: - - HashDict.new #=> creates an empty HashDict - - In the examples below, `dict_impl` means a specific - `Dict` implementation, for example `HashDict` or `Map`. - - ## Protocols - - Besides implementing the functions in this module, all - dictionaries are required to implement the `Access` - protocol: - - iex> dict = dict_impl.new - iex> dict = Dict.put(dict, :hello, :world) - iex> dict[:hello] - :world - - As well as the `Enumerable` and `Collectable` protocols. - - ## Match + Generic API for dictionaries. - Dictionaries are required to implement all operations - using the match (`===`) operator. - - ## Default implementation - - Default implementations for some functions in the `Dict` module - are provided via `use Dict`. - - For example: - - defmodule MyDict do - use Dict - - # implement required functions (see below) - # override default implementations if optimization - # is needed - end - - The client module must contain the following functions: - - * `delete/2` - * `fetch/2` - * `put/3` - * `reduce/3` - * `size/1` - - All functions, except `reduce/3`, are required by the Dict behaviour. - `reduce/3` must be implemtented as per the Enumerable protocol. - - Based on these functions, `Dict` generates default implementations - for the following functions: - - * `drop/2` - * `equal?/2` - * `fetch!/2` - * `get/2` - * `get/3` - * `has_key?/2` - * `keys/1` - * `merge/2` - * `merge/3` - * `pop/2` - * `pop/3` - * `put_new/3` - * `split/2` - * `take/2` - * `to_list/1` - * `update/4` - * `update!/3` - * `values/1` - - All of these functions are defined as overridable, so you can provide - your own implementation if needed. - - Note you can also test your custom module via `Dict`'s doctests: - - defmodule MyDict do - # ... - end - - defmodule MyTests do - use ExUnit.Case - doctest Dict - defp dict_impl, do: MyDict - end + If you need a general dictionary, use the `Map` module. + If you need to manipulate keyword lists, use `Keyword`. + To convert maps into keywords and vice-versa, use the + `new` function in the respective modules. """ - use Behaviour + @moduledoc deprecated: "Use Map or Keyword modules instead" @type key :: any @type value :: any @type t :: list | map - defcallback new :: t - defcallback delete(t, key) :: t - defcallback drop(t, Enum.t) :: t - defcallback equal?(t, t) :: boolean - defcallback get(t, key) :: value - defcallback get(t, key, value) :: value - defcallback fetch(t, key) :: {:ok, value} | :error - defcallback fetch!(t, key) :: value | no_return - defcallback has_key?(t, key) :: boolean - defcallback keys(t) :: [key] - defcallback merge(t, t) :: t - defcallback merge(t, t, (key, value, value -> value)) :: t - defcallback pop(t, key) :: {value, t} - defcallback pop(t, key, value) :: {value, t} - defcallback put(t, key, value) :: t - defcallback put_new(t, key, value) :: t - defcallback size(t) :: non_neg_integer() - defcallback split(t, Enum.t) :: {t, t} - defcallback take(t, Enum.t) :: t - defcallback to_list(t) :: list() - defcallback update(t, key, value, (value -> value)) :: t - defcallback update!(t, key, (value -> value)) :: t | no_return - defcallback values(t) :: list(value) + message = + "Use the Map module for working with maps or the Keyword module for working with keyword lists" defmacro __using__(_) do # Use this import to guarantee proper code expansion import Kernel, except: [size: 1] + if __CALLER__.module != HashDict do + IO.warn("use Dict is deprecated. " <> unquote(message), __CALLER__) + end + quote do - @behaviour Dict + message = "Use maps and the Map module instead" + @deprecated message def get(dict, key, default \\ nil) do case fetch(dict, key) do {:ok, value} -> value @@ -141,6 +41,22 @@ defmodule Dict do end end + @deprecated message + def get_lazy(dict, key, fun) when is_function(fun, 0) do + case fetch(dict, key) do + {:ok, value} -> value + :error -> fun.() + end + end + + @deprecated message + def get_and_update(dict, key, fun) do + current_value = get(dict, key) + {get, new_value} = fun.(current_value) + {get, put(dict, key, new_value)} + end + + @deprecated message def fetch!(dict, key) do case fetch(dict, key) do {:ok, value} -> value @@ -148,23 +64,35 @@ defmodule Dict do end end + @deprecated message def has_key?(dict, key) do - match? {:ok, _}, fetch(dict, key) + match?({:ok, _}, fetch(dict, key)) end + @deprecated message def put_new(dict, key, value) do case has_key?(dict, key) do - true -> dict + true -> dict false -> put(dict, key, value) end end + @deprecated message + def put_new_lazy(dict, key, fun) when is_function(fun, 0) do + case has_key?(dict, key) do + true -> dict + false -> put(dict, key, fun.()) + end + end + + @deprecated message def drop(dict, keys) do Enum.reduce(keys, dict, &delete(&2, &1)) end + @deprecated message def take(dict, keys) do - Enum.reduce(keys, new, fn key, acc -> + Enum.reduce(keys, new(), fn key, acc -> case fetch(dict, key) do {:ok, value} -> put(acc, key, value) :error -> acc @@ -172,41 +100,49 @@ defmodule Dict do end) end + @deprecated message def to_list(dict) do - reduce(dict, {:cont, []}, fn - kv, acc -> {:cont, [kv|acc]} - end) |> elem(1) |> :lists.reverse + reduce(dict, {:cont, []}, fn kv, acc -> {:cont, [kv | acc]} end) + |> elem(1) + |> :lists.reverse() end + @deprecated message def keys(dict) do - reduce(dict, {:cont, []}, fn - {k, _}, acc -> {:cont, [k|acc]} - end) |> elem(1) |> :lists.reverse + reduce(dict, {:cont, []}, fn {k, _}, acc -> {:cont, [k | acc]} end) + |> elem(1) + |> :lists.reverse() end + @deprecated message def values(dict) do - reduce(dict, {:cont, []}, fn - {_, v}, acc -> {:cont, [v|acc]} - end) |> elem(1) |> :lists.reverse + reduce(dict, {:cont, []}, fn {_, v}, acc -> {:cont, [v | acc]} end) + |> elem(1) + |> :lists.reverse() end + @deprecated message def equal?(dict1, dict2) do # Use this import to avoid conflicts in the user code import Kernel, except: [size: 1] case size(dict1) == size(dict2) do - false -> false - true -> - reduce(dict1, {:cont, true}, fn({k, v}, _acc) -> + false -> + false + + true -> + reduce(dict1, {:cont, true}, fn {k, v}, _acc -> case fetch(dict2, k) do {:ok, ^v} -> {:cont, true} _ -> {:halt, false} end - end) |> elem(1) + end) + |> elem(1) end end - def merge(dict1, dict2, fun \\ fn(_k, _v1, v2) -> v2 end) do + @deprecated message + def merge(dict1, dict2, fun \\ fn _k, _v1, v2 -> v2 end) do # Use this import to avoid conflicts in the user code import Kernel, except: [size: 1] @@ -218,444 +154,263 @@ defmodule Dict do reduce(dict2, {:cont, dict1}, fn {k, v2}, acc -> {:cont, update(acc, k, v2, &fun.(k, &1, v2))} end) - end |> elem(1) + end + |> elem(1) end - def update(dict, key, initial, fun) do + @deprecated message + def update(dict, key, default, fun) do case fetch(dict, key) do {:ok, value} -> put(dict, key, fun.(value)) + :error -> - put(dict, key, initial) + put(dict, key, default) end end + @deprecated message def update!(dict, key, fun) do case fetch(dict, key) do {:ok, value} -> put(dict, key, fun.(value)) + :error -> raise KeyError, key: key, term: dict end end + @deprecated message def pop(dict, key, default \\ nil) do case fetch(dict, key) do {:ok, value} -> {value, delete(dict, key)} + :error -> {default, dict} end end + @deprecated message + def pop_lazy(dict, key, fun) when is_function(fun, 0) do + case fetch(dict, key) do + {:ok, value} -> + {value, delete(dict, key)} + + :error -> + {fun.(), dict} + end + end + + @deprecated message def split(dict, keys) do - Enum.reduce(keys, {new, dict}, fn key, {inc, exc} = acc -> + Enum.reduce(keys, {new(), dict}, fn key, {inc, exc} = acc -> case fetch(exc, key) do {:ok, value} -> {put(inc, key, value), delete(exc, key)} + :error -> acc end end) end - defoverridable merge: 2, merge: 3, equal?: 2, to_list: 1, keys: 1, - values: 1, take: 2, drop: 2, get: 2, get: 3, fetch!: 2, - has_key?: 2, put_new: 3, pop: 2, pop: 3, split: 2, - update: 4, update!: 3 + defoverridable merge: 2, + merge: 3, + equal?: 2, + to_list: 1, + keys: 1, + values: 1, + take: 2, + drop: 2, + get: 2, + get: 3, + fetch!: 2, + has_key?: 2, + put_new: 3, + pop: 2, + pop: 3, + split: 2, + update: 4, + update!: 3, + get_and_update: 3, + get_lazy: 3, + pop_lazy: 3, + put_new_lazy: 3 end end - defmacrop target(dict) do quote do case unquote(dict) do - %{__struct__: x} when is_atom(x) -> - x - %{} -> - Map - x when is_list(x) -> - Keyword - x -> - unsupported_dict(x) + %module{} -> module + %{} -> Map + dict when is_list(dict) -> Keyword + dict -> unsupported_dict(dict) end end end - @doc """ - Returns a list of all keys in `dict`. - The keys are not guaranteed to be in any order. - - ## Examples - - iex> d = Enum.into([a: 1, b: 2], dict_impl.new) - iex> Enum.sort(Dict.keys(d)) - [:a,:b] - - """ + @deprecated message @spec keys(t) :: [key] def keys(dict) do target(dict).keys(dict) end - @doc """ - Returns a list of all values in `dict`. - The values are not guaranteed to be in any order. - - ## Examples - - iex> d = Enum.into([a: 1, b: 2], dict_impl.new) - iex> Enum.sort(Dict.values(d)) - [1,2] - - """ + @deprecated message @spec values(t) :: [value] def values(dict) do target(dict).values(dict) end - @doc """ - Returns the number of elements in `dict`. - - ## Examples - - iex> d = Enum.into([a: 1, b: 2], dict_impl.new) - iex> Dict.size(d) - 2 - - """ + @deprecated message @spec size(t) :: non_neg_integer def size(dict) do target(dict).size(dict) end - @doc """ - Returns whether the given `key` exists in the given `dict`. - - ## Examples - - iex> d = Enum.into([a: 1], dict_impl.new) - iex> Dict.has_key?(d, :a) - true - iex> Dict.has_key?(d, :b) - false - - """ + @deprecated message @spec has_key?(t, key) :: boolean def has_key?(dict, key) do target(dict).has_key?(dict, key) end - @doc """ - Returns the value associated with `key` in `dict`. If `dict` does not - contain `key`, returns `default` (or `nil` if not provided). - - ## Examples - - iex> d = Enum.into([a: 1], dict_impl.new) - iex> Dict.get(d, :a) - 1 - iex> Dict.get(d, :b) - nil - iex> Dict.get(d, :b, 3) - 3 - """ + @deprecated message @spec get(t, key, value) :: value def get(dict, key, default \\ nil) do target(dict).get(dict, key, default) end - @doc """ - Returns `{:ok, value}` associated with `key` in `dict`. - If `dict` does not contain `key`, returns `:error`. - - ## Examples + @deprecated message + @spec get_lazy(t, key, (-> value)) :: value + def get_lazy(dict, key, fun) do + target(dict).get_lazy(dict, key, fun) + end - iex> d = Enum.into([a: 1], dict_impl.new) - iex> Dict.fetch(d, :a) - {:ok, 1} - iex> Dict.fetch(d, :b) - :error + @deprecated message + @spec get_and_update(t, key, (value -> {value, value})) :: {value, t} + def get_and_update(dict, key, fun) do + target(dict).get_and_update(dict, key, fun) + end - """ + @deprecated message @spec fetch(t, key) :: value def fetch(dict, key) do target(dict).fetch(dict, key) end - @doc """ - Returns the value associated with `key` in `dict`. If `dict` does not - contain `key`, it raises `KeyError`. - - ## Examples - - iex> d = Enum.into([a: 1], dict_impl.new) - iex> Dict.fetch!(d, :a) - 1 - - """ - @spec fetch!(t, key) :: value | no_return + @deprecated message + @spec fetch!(t, key) :: value def fetch!(dict, key) do target(dict).fetch!(dict, key) end - @doc """ - Stores the given `value` under `key` in `dict`. - If `dict` already has `key`, the stored value is replaced by the new one. - - ## Examples - - iex> d = Enum.into([a: 1, b: 2], dict_impl.new) - iex> d = Dict.put(d, :a, 3) - iex> Dict.get(d, :a) - 3 - - """ + @deprecated message @spec put(t, key, value) :: t def put(dict, key, val) do target(dict).put(dict, key, val) end - @doc """ - Puts the given `value` under `key` in `dict` unless `key` already exists. - - ## Examples - - iex> d = Enum.into([a: 1, b: 2], dict_impl.new) - iex> d = Dict.put_new(d, :a, 3) - iex> Dict.get(d, :a) - 1 - - """ + @deprecated message @spec put_new(t, key, value) :: t def put_new(dict, key, val) do target(dict).put_new(dict, key, val) end - @doc """ - Removes the entry stored under the given `key` from `dict`. - If `dict` does not contain `key`, returns the dictionary unchanged. - - ## Examples - - iex> d = Enum.into([a: 1, b: 2], dict_impl.new) - iex> d = Dict.delete(d, :a) - iex> Dict.get(d, :a) - nil - - iex> d = Enum.into([b: 2], dict_impl.new) - iex> Dict.delete(d, :a) == d - true + @deprecated message + @spec put_new_lazy(t, key, (-> value)) :: t + def put_new_lazy(dict, key, fun) do + target(dict).put_new_lazy(dict, key, fun) + end - """ + @deprecated message @spec delete(t, key) :: t def delete(dict, key) do target(dict).delete(dict, key) end - @doc """ - Merges the dict `b` into dict `a`. - - If one of the dict `b` entries already exists in the `dict`, - the functions in entries in `b` have higher precedence unless a - function is given to resolve conflicts. - - Notice this function is polymorphic as it merges dicts of any - type. Each dict implementation also provides a `merge` function, - but they can only merge dicts of the same type. - - ## Examples - - iex> d1 = Enum.into([a: 1, b: 2], dict_impl.new) - iex> d2 = Enum.into([a: 3, d: 4], dict_impl.new) - iex> d = Dict.merge(d1, d2) - iex> [a: Dict.get(d, :a), b: Dict.get(d, :b), d: Dict.get(d, :d)] - [a: 3, b: 2, d: 4] + @deprecated message + @spec merge(t, t) :: t + def merge(dict1, dict2) do + target1 = target(dict1) + target2 = target(dict2) - iex> d1 = Enum.into([a: 1, b: 2], dict_impl.new) - iex> d2 = Enum.into([a: 3, d: 4], dict_impl.new) - iex> d = Dict.merge(d1, d2, fn(_k, v1, v2) -> - ...> v1 + v2 - ...> end) - iex> [a: Dict.get(d, :a), b: Dict.get(d, :b), d: Dict.get(d, :d)] - [a: 4, b: 2, d: 4] + if target1 == target2 do + target1.merge(dict1, dict2) + else + do_merge(target1, dict1, dict2, fn _k, _v1, v2 -> v2 end) + end + end - """ + @deprecated message @spec merge(t, t, (key, value, value -> value)) :: t - def merge(dict1, dict2, fun \\ fn(_k, _v1, v2) -> v2 end) do + def merge(dict1, dict2, fun) do target1 = target(dict1) target2 = target(dict2) if target1 == target2 do target1.merge(dict1, dict2, fun) else - Enumerable.reduce(dict2, {:cont, dict1}, fn({k, v}, acc) -> - {:cont, target1.update(acc, k, v, fn(other) -> fun.(k, other, v) end)} - end) |> elem(1) + do_merge(target1, dict1, dict2, fun) end end - @doc """ - Returns the value associated with `key` in `dict` as - well as the `dict` without `key`. - - ## Examples - - iex> dict = Enum.into([a: 1], dict_impl.new) - iex> {v, d} = Dict.pop dict, :a - iex> {v, Enum.sort(d)} - {1,[]} - - iex> dict = Enum.into([a: 1], dict_impl.new) - iex> {v, d} = Dict.pop dict, :b - iex> {v, Enum.sort(d)} - {nil,[a: 1]} - - iex> dict = Enum.into([a: 1], dict_impl.new) - iex> {v, d} = Dict.pop dict, :b, 3 - iex> {v, Enum.sort(d)} - {3,[a: 1]} + defp do_merge(target1, dict1, dict2, fun) do + Enumerable.reduce(dict2, {:cont, dict1}, fn {k, v}, acc -> + {:cont, target1.update(acc, k, v, fn other -> fun.(k, other, v) end)} + end) + |> elem(1) + end - """ + @deprecated message @spec pop(t, key, value) :: {value, t} def pop(dict, key, default \\ nil) do target(dict).pop(dict, key, default) end - @doc """ - Update a value in `dict` by calling `fun` on the value to get a new - value. An exception is generated if `key` is not present in the dict. - - ## Examples - - iex> d = Enum.into([a: 1, b: 2], dict_impl.new) - iex> d = Dict.update!(d, :a, fn(val) -> -val end) - iex> Dict.get(d, :a) - -1 + @deprecated message + @spec pop_lazy(t, key, (-> value)) :: {value, t} + def pop_lazy(dict, key, fun) do + target(dict).pop_lazy(dict, key, fun) + end - """ + @deprecated message @spec update!(t, key, (value -> value)) :: t def update!(dict, key, fun) do target(dict).update!(dict, key, fun) end - @doc """ - Update a value in `dict` by calling `fun` on the value to get a new value. If - `key` is not present in `dict` then `initial` will be stored as the first - value. - - ## Examples - - iex> d = Enum.into([a: 1, b: 2], dict_impl.new) - iex> d = Dict.update(d, :c, 3, fn(val) -> -val end) - iex> Dict.get(d, :c) - 3 - - """ + @deprecated message @spec update(t, key, value, (value -> value)) :: t - def update(dict, key, initial, fun) do - target(dict).update(dict, key, initial, fun) + def update(dict, key, default, fun) do + target(dict).update(dict, key, default, fun) end - @doc """ - Returns a tuple of two dicts, where the first dict contains only - entries from `dict` with keys in `keys`, and the second dict - contains only entries from `dict` with keys not in `keys` - - Any non-member keys are ignored. - - ## Examples - - iex> d = Enum.into([a: 1, b: 2, c: 3, d: 4], dict_impl.new) - iex> {d1, d2} = Dict.split(d, [:a, :c, :e]) - iex> {Dict.to_list(d1) |> Enum.sort, Dict.to_list(d2) |> Enum.sort} - {[a: 1, c: 3], [b: 2, d: 4]} - - iex> d = Enum.into([], dict_impl.new) - iex> {d1, d2} = Dict.split(d, [:a, :c]) - iex> {Dict.to_list(d1), Dict.to_list(d2)} - {[], []} - - iex> d = Enum.into([a: 1, b: 2], dict_impl.new) - iex> {d1, d2} = Dict.split(d, [:a, :b, :c]) - iex> {Dict.to_list(d1) |> Enum.sort, Dict.to_list(d2)} - {[a: 1, b: 2], []} - - """ + @deprecated message @spec split(t, [key]) :: {t, t} def split(dict, keys) do target(dict).split(dict, keys) end - @doc """ - Returns a new dict where the given `keys` are removed from `dict`. - Any non-member keys are ignored. - - ## Examples - - iex> d = Enum.into([a: 1, b: 2], dict_impl.new) - iex> d = Dict.drop(d, [:a, :c, :d]) - iex> Dict.to_list(d) - [b: 2] - - iex> d = Enum.into([a: 1, b: 2], dict_impl.new) - iex> d = Dict.drop(d, [:c, :d]) - iex> Dict.to_list(d) |> Enum.sort - [a: 1, b: 2] - - """ + @deprecated message @spec drop(t, [key]) :: t def drop(dict, keys) do target(dict).drop(dict, keys) end - @doc """ - Returns a new dict where only the keys in `keys` from `dict` are included. - - Any non-member keys are ignored. - - ## Examples - - iex> d = Enum.into([a: 1, b: 2], dict_impl.new) - iex> d = Dict.take(d, [:a, :c, :d]) - iex> Dict.to_list(d) - [a: 1] - iex> d = Dict.take(d, [:c, :d]) - iex> Dict.to_list(d) - [] - - """ + @deprecated message @spec take(t, [key]) :: t def take(dict, keys) do target(dict).take(dict, keys) end - @doc false + @deprecated message @spec empty(t) :: t def empty(dict) do target(dict).empty(dict) end - @doc """ - Check if two dicts are equal using `===`. - - Notice this function is polymorphic as it compares dicts of any - type. Each dict implementation also provides an `equal?` function, - but they can only compare dicts of the same type. - - ## Examples - - iex> a = Enum.into([a: 2, b: 3, f: 5, c: 123], dict_impl.new) - iex> b = [a: 2, b: 3, f: 5, c: 123] - iex> Dict.equal?(a, b) - true - - iex> a = Enum.into([a: 2, b: 3, f: 5, c: 123], dict_impl.new) - iex> b = [] - iex> Dict.equal?(a, b) - false - - """ + @deprecated message @spec equal?(t, t) :: boolean def equal?(dict1, dict2) do target1 = target(dict1) @@ -666,28 +421,27 @@ defmodule Dict do target1.equal?(dict1, dict2) target1.size(dict1) == target2.size(dict2) -> - Enumerable.reduce(dict2, {:cont, true}, fn({k, v}, _acc) -> + Enumerable.reduce(dict2, {:cont, true}, fn {k, v}, _acc -> case target1.fetch(dict1, k) do {:ok, ^v} -> {:cont, true} - _ -> {:halt, false} + _ -> {:halt, false} end - end) |> elem(1) + end) + |> elem(1) true -> false end end - @doc """ - Returns a list of key-value pairs stored in `dict`. - No particular order is enforced. - """ + @deprecated message @spec to_list(t) :: list def to_list(dict) do target(dict).to_list(dict) end + @spec unsupported_dict(t) :: no_return defp unsupported_dict(dict) do - raise ArgumentError, "unsupported dict: #{inspect dict}" + raise ArgumentError, "unsupported dict: #{inspect(dict)}" end end diff --git a/lib/elixir/lib/dynamic_supervisor.ex b/lib/elixir/lib/dynamic_supervisor.ex new file mode 100644 index 00000000000..e0f896a97ae --- /dev/null +++ b/lib/elixir/lib/dynamic_supervisor.ex @@ -0,0 +1,1125 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +defmodule DynamicSupervisor do + @moduledoc ~S""" + A supervisor optimized to only start children dynamically. + + The `Supervisor` module was designed to handle mostly static children + that are started in the given order when the supervisor starts. A + `DynamicSupervisor` starts with no children. Instead, children are + started on demand via `start_child/2` and there is no ordering between + children. This allows the `DynamicSupervisor` to hold millions of + children by using efficient data structures and to execute certain + operations, such as shutting down, concurrently. + + ## Examples + + A dynamic supervisor is started with no children and often with a name: + + children = [ + {DynamicSupervisor, name: MyApp.DynamicSupervisor, strategy: :one_for_one} + ] + + Supervisor.start_link(children, strategy: :one_for_one) + + The options given in the child specification are documented in `start_link/1`. + + Once the dynamic supervisor is running, we can use it to start children + on demand. Given this sample `GenServer`: + + defmodule Counter do + use GenServer + + def start_link(initial) do + GenServer.start_link(__MODULE__, initial) + end + + def inc(pid) do + GenServer.call(pid, :inc) + end + + def init(initial) do + {:ok, initial} + end + + def handle_call(:inc, _, count) do + {:reply, count, count + 1} + end + end + + We can use `start_child/2` with a child specification to start a `Counter` + server: + + {:ok, counter1} = DynamicSupervisor.start_child(MyApp.DynamicSupervisor, {Counter, 0}) + Counter.inc(counter1) + #=> 0 + + {:ok, counter2} = DynamicSupervisor.start_child(MyApp.DynamicSupervisor, {Counter, 10}) + Counter.inc(counter2) + #=> 10 + + DynamicSupervisor.count_children(MyApp.DynamicSupervisor) + #=> %{active: 2, specs: 2, supervisors: 0, workers: 2} + + ## Scalability and partitioning + + The `DynamicSupervisor` is a single process responsible for starting + other processes. In some applications, the `DynamicSupervisor` may + become a bottleneck. To address this, you can start multiple instances + of the `DynamicSupervisor` and then pick a "random" instance to start + the child on. + + Instead of: + + children = [ + {DynamicSupervisor, name: MyApp.DynamicSupervisor} + ] + + and: + + DynamicSupervisor.start_child(MyApp.DynamicSupervisor, {Counter, 0}) + + You can do this: + + children = [ + {PartitionSupervisor, + child_spec: DynamicSupervisor, + name: MyApp.DynamicSupervisors} + ] + + and then: + + DynamicSupervisor.start_child( + {:via, PartitionSupervisor, {MyApp.DynamicSupervisors, self()}}, + {Counter, 0} + ) + + In the code above, we start a partition supervisor that will by default + start a dynamic supervisor for each core in your machine. Then, instead + of calling the `DynamicSupervisor` by name, you call it through the + partition supervisor, using `self()` as the routing key. This means each + process will be assigned one of the existing dynamic supervisors. + Read the `PartitionSupervisor` docs for more information. + + ## Module-based supervisors + + Similar to `Supervisor`, dynamic supervisors also support module-based + supervisors. + + defmodule MyApp.DynamicSupervisor do + # Automatically defines child_spec/1 + use DynamicSupervisor + + def start_link(init_arg) do + DynamicSupervisor.start_link(__MODULE__, init_arg, name: __MODULE__) + end + + @impl true + def init(_init_arg) do + DynamicSupervisor.init(strategy: :one_for_one) + end + end + + See the `Supervisor` docs for a discussion of when you may want to use + module-based supervisors. A `@doc` annotation immediately preceding + `use DynamicSupervisor` will be attached to the generated `child_spec/1` + function. + + > #### `use DynamicSupervisor` {: .info} + > + > When you `use DynamicSupervisor`, the `DynamicSupervisor` module will + > set `@behaviour DynamicSupervisor` and define a `child_spec/1` + > function, so your module can be used as a child in a supervision tree. + + ## Name registration + + A supervisor is bound to the same name registration rules as a `GenServer`. + Read more about these rules in the documentation for `GenServer`. + """ + + @behaviour GenServer + + @doc """ + Callback invoked to start the supervisor and during hot code upgrades. + + Developers typically invoke `DynamicSupervisor.init/1` at the end of + their init callback to return the proper supervision flags. + """ + @callback init(init_arg :: term) :: {:ok, sup_flags()} | :ignore + + @typedoc "The supervisor flags returned on init" + @type sup_flags() :: %{ + strategy: strategy(), + intensity: non_neg_integer(), + period: pos_integer(), + max_children: non_neg_integer() | :infinity, + extra_arguments: [term()] + } + + @typedoc "Options given to `start_link/1` and `init/1` functions" + @type init_option :: + {:strategy, strategy()} + | {:max_restarts, non_neg_integer()} + | {:max_seconds, pos_integer()} + | {:max_children, non_neg_integer() | :infinity} + | {:extra_arguments, [term()]} + + @typedoc "Supported strategies" + @type strategy :: :one_for_one + + @typedoc """ + Return values of `start_child` functions. + + Unlike `Supervisor`, this module ignores the child spec ids, + so `{:error, {:already_started, pid}}` is not returned for child specs + given with the same id. `{:error, {:already_started, pid}}` is returned + however if a duplicate name is used when using + [name registration](`m:GenServer#module-name-registration`). + """ + @type on_start_child :: + {:ok, pid} + | {:ok, pid, info :: term} + | :ignore + | {:error, {:already_started, pid} | :max_children | term} + + # In this struct, `args` refers to the arguments passed to init/1 (the `init_arg`). + defstruct [ + :args, + :extra_arguments, + :mod, + :name, + :strategy, + :max_children, + :max_restarts, + :max_seconds, + children: %{}, + restarts: [] + ] + + @doc """ + Returns a specification to start a dynamic supervisor under a supervisor. + + It accepts the same options as `start_link/1`. + + See `Supervisor` for more information about child specifications. + """ + @doc since: "1.6.1" + @spec child_spec([init_option() | GenServer.option()]) :: Supervisor.child_spec() + def child_spec(options) when is_list(options) do + id = + case Keyword.get(options, :name, DynamicSupervisor) do + name when is_atom(name) -> name + {:global, name} -> name + {:via, _module, name} -> name + end + + %{ + id: id, + start: {DynamicSupervisor, :start_link, [options]}, + type: :supervisor + } + end + + @doc false + defmacro __using__(opts) do + quote location: :keep, bind_quoted: [opts: opts] do + @behaviour DynamicSupervisor + if not Module.has_attribute?(__MODULE__, :doc) do + @doc """ + Returns a specification to start this module under a supervisor. + + See `Supervisor`. + """ + end + + def child_spec(arg) do + default = %{ + id: __MODULE__, + start: {__MODULE__, :start_link, [arg]}, + type: :supervisor + } + + Supervisor.child_spec(default, unquote(Macro.escape(opts))) + end + + defoverridable child_spec: 1 + end + end + + @doc """ + Starts a supervisor with the given options. + + This function is typically not invoked directly, instead it is invoked + when using a `DynamicSupervisor` as a child of another supervisor: + + children = [ + {DynamicSupervisor, name: MySupervisor} + ] + + If the supervisor is successfully spawned, this function returns + `{:ok, pid}`, where `pid` is the PID of the supervisor. If the supervisor + is given a name and a process with the specified name already exists, + the function returns `{:error, {:already_started, pid}}`, where `pid` + is the PID of that process. + + Note that a supervisor started with this function is linked to the parent + process and exits not only on crashes but also if the parent process exits + with `:normal` reason. + + ## Options + + * `:name` - registers the supervisor under the given name. + The supported values are described under the "Name registration" + section in the `GenServer` module docs. + + * `:strategy` - the restart strategy option. The only supported + value is `:one_for_one` which means that no other child is + terminated if a child process terminates. You can learn more + about strategies in the `Supervisor` module docs. + + * `:max_restarts` - the maximum number of restarts allowed in + a time frame. Defaults to `3`. + + * `:max_seconds` - the time frame in which `:max_restarts` applies. + Defaults to `5`. + + * `:max_children` - the maximum amount of children to be running + under this supervisor at the same time. When `:max_children` is + exceeded, `start_child/2` returns `{:error, :max_children}`. Defaults + to `:infinity`. + + * `:extra_arguments` - arguments that are prepended to the arguments + specified in the child spec given to `start_child/2`. Defaults to + an empty list. + + * Any of the standard [GenServer options](`t:GenServer.option/0`) + + """ + @doc since: "1.6.0" + @spec start_link([init_option | GenServer.option()]) :: Supervisor.on_start() + def start_link(options) when is_list(options) do + keys = [:extra_arguments, :max_children, :max_seconds, :max_restarts, :strategy] + {sup_opts, start_opts} = Keyword.split(options, keys) + start_link(Supervisor.Default, init(sup_opts), start_opts) + end + + @doc """ + Starts a module-based supervisor process with the given `module` and `init_arg`. + + To start the supervisor, the `c:init/1` callback will be invoked in the given + `module`, with `init_arg` as its argument. The `c:init/1` callback must return a + supervisor specification which can be created with the help of the `init/1` + function. + + If the `c:init/1` callback returns `:ignore`, this function returns + `:ignore` as well and the supervisor terminates with reason `:normal`. + If it fails or returns an incorrect value, this function returns + `{:error, term}` where `term` is a term with information about the + error, and the supervisor terminates with reason `term`. + + The `:name` option can also be given in order to register a supervisor + name, the supported values are described in the "Name registration" + section in the `GenServer` module docs. + + If the supervisor is successfully spawned, this function returns + `{:ok, pid}`, where `pid` is the PID of the supervisor. If the supervisor + is given a name and a process with the specified name already exists, + the function returns `{:error, {:already_started, pid}}`, where `pid` + is the PID of that process. + + Note that a supervisor started with this function is linked to the parent + process and exits not only on crashes but also if the parent process exits + with `:normal` reason. + + ## Options + + This function accepts any regular [`GenServer` options](`t:GenServer.option/0`). + Options specific to `DynamicSupervisor` must be returned from the `c:init/1` + callback. + """ + @doc since: "1.6.0" + @spec start_link(module, term, [GenServer.option()]) :: Supervisor.on_start() + def start_link(module, init_arg, opts \\ []) do + GenServer.start_link(__MODULE__, {module, init_arg, opts[:name]}, opts) + end + + @doc """ + Dynamically adds a child specification to `supervisor` and starts that child. + + `child_spec` should be a valid [child specification](`m:Supervisor#module-child-specification`). + The child process will be started as defined in the child specification. Note that while + the `:id` field is still required in the spec, the value is ignored and + therefore does not need to be unique. Unlike `Supervisor`, this module does not + return `{:error, {:already_started, pid}}` for child specs given with the same id. + `{:error, {:already_started, pid}}` is returned however if a duplicate name is + used when using [name registration](`m:GenServer#module-name-registration`). + + This function will block the `DynamicSupervisor` until the child initializes. + When starting too many processes dynamically, you may want to use a + `PartitionSupervisor` to split the work across multiple processes. + + If the child process start function returns `{:ok, child}` or `{:ok, child, + info}`, then child specification and PID are added to the supervisor and + this function returns the same value. + + If the child process start function returns `:ignore`, then no child is added + to the supervision tree and this function returns `:ignore` too. + + If the child process start function returns an error tuple or an erroneous + value, or if it fails, the child specification is discarded and this function + returns `{:error, error}` where `error` is the error or erroneous value + returned from child process start function, or failure reason if it fails. + + If the supervisor already has N children in a way that N exceeds the amount + of `:max_children` set on the supervisor initialization (see `init/1`), then + this function returns `{:error, :max_children}`. + """ + @doc since: "1.6.0" + @spec start_child( + Supervisor.supervisor(), + Supervisor.child_spec() + | {module, term} + | module + | (old_erlang_child_spec :: :supervisor.child_spec()) + ) :: + on_start_child() + def start_child(supervisor, {_, _, _, _, _, _} = child_spec) do + validate_and_start_child(supervisor, child_spec) + end + + def start_child(supervisor, child_spec) do + validate_and_start_child(supervisor, Supervisor.child_spec(child_spec, [])) + end + + defp validate_and_start_child(supervisor, child_spec) do + case validate_child(child_spec) do + {:ok, child} -> call(supervisor, {:start_child, child}) + error -> {:error, error} + end + end + + defp validate_child(%{id: _, start: {mod, _, _} = start} = child) do + restart = Map.get(child, :restart, :permanent) + type = Map.get(child, :type, :worker) + modules = Map.get(child, :modules, [mod]) + significant = Map.get(child, :significant, false) + + shutdown = + case type do + :worker -> Map.get(child, :shutdown, 5_000) + :supervisor -> Map.get(child, :shutdown, :infinity) + end + + validate_child(start, restart, shutdown, type, modules, significant) + end + + defp validate_child({_, start, restart, shutdown, type, modules}) do + validate_child(start, restart, shutdown, type, modules, false) + end + + defp validate_child(other) do + {:invalid_child_spec, other} + end + + defp validate_child(start, restart, shutdown, type, modules, significant) do + with :ok <- validate_start(start), + :ok <- validate_restart(restart), + :ok <- validate_shutdown(shutdown), + :ok <- validate_type(type), + :ok <- validate_modules(modules), + :ok <- validate_significant(significant) do + {:ok, {start, restart, shutdown, type, modules}} + end + end + + defp validate_start({m, f, args}) when is_atom(m) and is_atom(f) and is_list(args), do: :ok + defp validate_start(mfa), do: {:invalid_mfa, mfa} + + defp validate_type(type) when type in [:supervisor, :worker], do: :ok + defp validate_type(type), do: {:invalid_child_type, type} + + defp validate_restart(restart) when restart in [:permanent, :temporary, :transient], do: :ok + defp validate_restart(restart), do: {:invalid_restart_type, restart} + + defp validate_shutdown(shutdown) when is_integer(shutdown) and shutdown >= 0, do: :ok + defp validate_shutdown(shutdown) when shutdown in [:infinity, :brutal_kill], do: :ok + defp validate_shutdown(shutdown), do: {:invalid_shutdown, shutdown} + + defp validate_significant(false), do: :ok + defp validate_significant(significant), do: {:invalid_significant, significant} + + defp validate_modules(:dynamic), do: :ok + + defp validate_modules(mods) do + if is_list(mods) and Enum.all?(mods, &is_atom/1) do + :ok + else + {:invalid_modules, mods} + end + end + + @doc """ + Terminates the given child identified by `pid`. + + This function will block the `DynamicSupervisor` until the child + terminates, which may take an arbitrary amount of time if the child + is trapping exits and implements its own terminate callback. + For this reason, it is often better to ask the child process + itself to terminate, often by declaring in its child spec it has + a restart strategy of `:transient` (or `:temporary`) and then + sending it a message to stop with reason `:shutdown`. + + If successful, this function returns `:ok`. If there is no process with + the given PID, this function returns `{:error, :not_found}`. + """ + @doc since: "1.6.0" + @spec terminate_child(Supervisor.supervisor(), pid) :: :ok | {:error, :not_found} + def terminate_child(supervisor, pid) when is_pid(pid) do + call(supervisor, {:terminate_child, pid}) + end + + @doc """ + Returns a list with information about all children of the given supervisor. + + Note that calling this function when supervising a large number + of children under low memory conditions can bring the system down due to an + out of memory error. + + This function returns a list of tuples containing: + + * `id` - it is always `:undefined` for dynamic supervisors + + * `child` - the PID of the corresponding child process or the + atom `:restarting` if the process is about to be restarted + + * `type` - `:worker` or `:supervisor` as defined in the child + specification + + * `modules` - as defined in the child specification + + """ + @doc since: "1.6.0" + @spec which_children(Supervisor.supervisor()) :: [ + # module() | :dynamic here because :supervisor.modules() is not exported + {:undefined, pid | :restarting, :worker | :supervisor, [module()] | :dynamic} + ] + def which_children(supervisor) do + call(supervisor, :which_children) + end + + @doc """ + Returns a map containing count values for the supervisor. + + The map contains the following keys: + + * `:specs` - the number of children processes + + * `:active` - the count of all actively running child processes managed by + this supervisor + + * `:supervisors` - the count of all supervisors whether or not the child + process is still alive + + * `:workers` - the count of all workers, whether or not the child process + is still alive + + """ + @doc since: "1.6.0" + @spec count_children(Supervisor.supervisor()) :: %{ + specs: non_neg_integer, + active: non_neg_integer, + supervisors: non_neg_integer, + workers: non_neg_integer + } + def count_children(supervisor) do + call(supervisor, :count_children) |> :maps.from_list() + end + + @doc """ + Synchronously stops the given supervisor with the given `reason`. + + It returns `:ok` if the supervisor terminates with the given + reason. If it terminates with another reason, the call exits. + + This function keeps OTP semantics regarding error reporting. + If the reason is any other than `:normal`, `:shutdown` or + `{:shutdown, _}`, an error report is logged. + """ + @doc since: "1.7.0" + @spec stop(Supervisor.supervisor(), reason :: term, timeout) :: :ok + def stop(supervisor, reason \\ :normal, timeout \\ :infinity) do + GenServer.stop(supervisor, reason, timeout) + end + + @doc """ + Receives a set of `options` that initializes a dynamic supervisor. + + This is typically invoked at the end of the `c:init/1` callback of + module-based supervisors. See the "Module-based supervisors" section + in the module documentation for more information. + + It accepts the same `options` as `start_link/1` (except for `:name`) + and it returns a tuple containing the supervisor options. + + ## Examples + + def init(_arg) do + DynamicSupervisor.init(max_children: 1000) + end + + """ + @doc since: "1.6.0" + @spec init([init_option]) :: {:ok, sup_flags()} + def init(options) when is_list(options) do + strategy = Keyword.get(options, :strategy, :one_for_one) + intensity = Keyword.get(options, :max_restarts, 3) + period = Keyword.get(options, :max_seconds, 5) + max_children = Keyword.get(options, :max_children, :infinity) + extra_arguments = Keyword.get(options, :extra_arguments, []) + + flags = %{ + strategy: strategy, + intensity: intensity, + period: period, + max_children: max_children, + extra_arguments: extra_arguments + } + + {:ok, flags} + end + + ## Callbacks + + @impl true + def init({mod, init_arg, name}) do + Process.put(:"$initial_call", {:supervisor, mod, 1}) + Process.flag(:trap_exit, true) + + case mod.init(init_arg) do + {:ok, flags} when is_map(flags) -> + name = + cond do + is_nil(name) -> {self(), mod} + is_atom(name) -> {:local, name} + is_tuple(name) -> name + end + + state = %DynamicSupervisor{mod: mod, args: init_arg, name: name} + + case init(state, flags) do + {:ok, state} -> {:ok, state} + {:error, reason} -> {:stop, {:supervisor_data, reason}} + end + + :ignore -> + :ignore + + other -> + {:stop, {:bad_return, {mod, :init, other}}} + end + end + + defp init(state, flags) do + extra_arguments = Map.get(flags, :extra_arguments, []) + max_children = Map.get(flags, :max_children, :infinity) + max_restarts = Map.get(flags, :intensity, 1) + max_seconds = Map.get(flags, :period, 5) + strategy = Map.get(flags, :strategy, :one_for_one) + auto_shutdown = Map.get(flags, :auto_shutdown, :never) + + with :ok <- validate_strategy(strategy), + :ok <- validate_restarts(max_restarts), + :ok <- validate_seconds(max_seconds), + :ok <- validate_dynamic(max_children), + :ok <- validate_extra_arguments(extra_arguments), + :ok <- validate_auto_shutdown(auto_shutdown) do + {:ok, + %{ + state + | extra_arguments: extra_arguments, + max_children: max_children, + max_restarts: max_restarts, + max_seconds: max_seconds, + strategy: strategy + }} + end + end + + defp validate_strategy(strategy) when strategy in [:one_for_one], do: :ok + defp validate_strategy(strategy), do: {:error, {:invalid_strategy, strategy}} + + defp validate_restarts(restart) when is_integer(restart) and restart >= 0, do: :ok + defp validate_restarts(restart), do: {:error, {:invalid_intensity, restart}} + + defp validate_seconds(seconds) when is_integer(seconds) and seconds > 0, do: :ok + defp validate_seconds(seconds), do: {:error, {:invalid_period, seconds}} + + defp validate_dynamic(:infinity), do: :ok + defp validate_dynamic(dynamic) when is_integer(dynamic) and dynamic >= 0, do: :ok + defp validate_dynamic(dynamic), do: {:error, {:invalid_max_children, dynamic}} + + defp validate_extra_arguments(list) when is_list(list), do: :ok + defp validate_extra_arguments(extra), do: {:error, {:invalid_extra_arguments, extra}} + + defp validate_auto_shutdown(auto_shutdown) when auto_shutdown in [:never], do: :ok + + defp validate_auto_shutdown(auto_shutdown), + do: {:error, {:invalid_auto_shutdown, auto_shutdown}} + + @impl true + def handle_call(:which_children, _from, state) do + %{children: children} = state + + reply = + for {pid, args} <- children do + case args do + {:restarting, {_, _, _, type, modules}} -> + {:undefined, :restarting, type, modules} + + {_, _, _, type, modules} -> + {:undefined, pid, type, modules} + end + end + + {:reply, reply, state} + end + + def handle_call(:count_children, _from, state) do + %{children: children} = state + specs = map_size(children) + + {active, workers, supervisors} = + Enum.reduce(children, {0, 0, 0}, fn + {_pid, {:restarting, {_, _, _, :worker, _}}}, {active, worker, supervisor} -> + {active, worker + 1, supervisor} + + {_pid, {:restarting, {_, _, _, :supervisor, _}}}, {active, worker, supervisor} -> + {active, worker, supervisor + 1} + + {_pid, {_, _, _, :worker, _}}, {active, worker, supervisor} -> + {active + 1, worker + 1, supervisor} + + {_pid, {_, _, _, :supervisor, _}}, {active, worker, supervisor} -> + {active + 1, worker, supervisor + 1} + end) + + reply = [specs: specs, active: active, supervisors: supervisors, workers: workers] + {:reply, reply, state} + end + + def handle_call({:terminate_child, pid}, _from, %{children: children} = state) do + case children do + %{^pid => info} -> + :ok = terminate_children(%{pid => info}, state) + {:reply, :ok, delete_child(pid, state)} + + %{} -> + {:reply, {:error, :not_found}, state} + end + end + + def handle_call({:start_task, args, restart, shutdown}, from, state) do + {init_restart, init_shutdown} = Process.get(Task.Supervisor) + + if restart == nil and init_restart != :temporary do + {:reply, {:restart, init_restart}, state} + else + restart = restart || init_restart + shutdown = shutdown || init_shutdown + + child = + {{Task.Supervised, :start_link, args}, restart, shutdown, :worker, [Task.Supervised]} + + handle_call({:start_child, child}, from, state) + end + end + + def handle_call({:start_child, child}, _from, state) do + %{children: children, max_children: max_children} = state + + if map_size(children) < max_children do + handle_start_child(child, state) + else + {:reply, {:error, :max_children}, state} + end + end + + defp handle_start_child({{m, f, args} = mfa, restart, shutdown, type, modules}, state) do + %{extra_arguments: extra} = state + + case reply = start_child(m, f, extra ++ args) do + {:ok, pid, _} -> + {:reply, reply, save_child(pid, mfa, restart, shutdown, type, modules, state)} + + {:ok, pid} -> + {:reply, reply, save_child(pid, mfa, restart, shutdown, type, modules, state)} + + _ -> + {:reply, reply, state} + end + end + + defp start_child(m, f, a) do + try do + apply(m, f, a) + catch + kind, reason -> + {:error, exit_reason(kind, reason, __STACKTRACE__)} + else + {:ok, pid, extra} when is_pid(pid) -> {:ok, pid, extra} + {:ok, pid} when is_pid(pid) -> {:ok, pid} + :ignore -> :ignore + {:error, _} = error -> error + other -> {:error, other} + end + end + + defp save_child(pid, mfa, restart, shutdown, type, modules, state) do + mfa = mfa_for_restart(mfa, restart) + put_in(state.children[pid], {mfa, restart, shutdown, type, modules}) + end + + defp mfa_for_restart({m, f, _}, :temporary), do: {m, f, :undefined} + defp mfa_for_restart(mfa, _), do: mfa + + defp exit_reason(:exit, reason, _), do: reason + defp exit_reason(:error, reason, stack), do: {reason, stack} + defp exit_reason(:throw, value, stack), do: {{:nocatch, value}, stack} + + @impl true + def handle_cast(_msg, state) do + {:noreply, state} + end + + @impl true + def handle_info({:EXIT, pid, reason}, state) do + case maybe_restart_child(pid, reason, state) do + {:ok, state} -> {:noreply, state} + {:shutdown, state} -> {:stop, :shutdown, state} + end + end + + def handle_info({:"$gen_restart", pid}, state) do + %{children: children} = state + + case children do + %{^pid => restarting_args} -> + {:restarting, child} = restarting_args + + case restart_child(pid, child, state) do + {:ok, state} -> {:noreply, state} + {:shutdown, state} -> {:stop, :shutdown, state} + end + + # We may hit clause if we send $gen_restart and then + # someone calls terminate_child, removing the child. + %{} -> + {:noreply, state} + end + end + + def handle_info(msg, state) do + :logger.error( + %{ + label: {DynamicSupervisor, :unexpected_msg}, + report: %{ + msg: msg + } + }, + %{ + domain: [:otp, :elixir], + error_logger: %{tag: :error_msg}, + report_cb: &__MODULE__.format_report/1 + } + ) + + {:noreply, state} + end + + @impl true + def code_change(_, %{mod: mod, args: init_arg} = state, _) do + case mod.init(init_arg) do + {:ok, flags} when is_map(flags) -> + case init(state, flags) do + {:ok, state} -> {:ok, state} + {:error, reason} -> {:error, {:supervisor_data, reason}} + end + + :ignore -> + {:ok, state} + + error -> + error + end + end + + @impl true + def terminate(_, %{children: children} = state) do + :ok = terminate_children(children, state) + end + + defp terminate_children(children, state) do + {pids, times, stacks} = monitor_children(children) + size = map_size(pids) + + timers = + Enum.reduce(times, %{}, fn {time, pids}, acc -> + Map.put(acc, :erlang.start_timer(time, self(), :kill), pids) + end) + + stacks = wait_children(pids, size, timers, stacks) + + for {pid, {child, reason}} <- stacks do + report_error(:shutdown_error, reason, pid, child, state) + end + + :ok + end + + defp monitor_children(children) do + Enum.reduce(children, {%{}, %{}, %{}}, fn + {_, {:restarting, _}}, acc -> + acc + + {pid, {_, restart, _, _, _} = child}, {pids, times, stacks} -> + case monitor_child(pid) do + :ok -> + times = exit_child(pid, child, times) + {Map.put(pids, pid, child), times, stacks} + + {:error, :normal} when restart != :permanent -> + {pids, times, stacks} + + {:error, reason} -> + {pids, times, Map.put(stacks, pid, {child, reason})} + end + end) + end + + defp monitor_child(pid) do + ref = Process.monitor(pid) + Process.unlink(pid) + + receive do + {:EXIT, ^pid, reason} -> + receive do + {:DOWN, ^ref, :process, ^pid, _} -> {:error, reason} + end + after + 0 -> :ok + end + end + + defp exit_child(pid, {_, _, shutdown, _, _}, times) do + case shutdown do + :brutal_kill -> + Process.exit(pid, :kill) + times + + :infinity -> + Process.exit(pid, :shutdown) + times + + time -> + Process.exit(pid, :shutdown) + Map.update(times, time, [pid], &[pid | &1]) + end + end + + defp wait_children(_pids, 0, timers, stacks) do + for {timer, _} <- timers do + _ = :erlang.cancel_timer(timer) + + receive do + {:timeout, ^timer, :kill} -> :ok + after + 0 -> :ok + end + end + + stacks + end + + defp wait_children(pids, size, timers, stacks) do + receive do + {:DOWN, _ref, :process, pid, reason} -> + case pids do + %{^pid => child} -> + stacks = wait_child(pid, child, reason, stacks) + wait_children(pids, size - 1, timers, stacks) + + %{} -> + wait_children(pids, size, timers, stacks) + end + + {:timeout, timer, :kill} -> + for pid <- Map.fetch!(timers, timer), do: Process.exit(pid, :kill) + wait_children(pids, size, Map.delete(timers, timer), stacks) + end + end + + defp wait_child(pid, {_, _, :brutal_kill, _, _} = child, reason, stacks) do + case reason do + :killed -> stacks + _ -> Map.put(stacks, pid, {child, reason}) + end + end + + defp wait_child(pid, {_, restart, _, _, _} = child, reason, stacks) do + case reason do + {:shutdown, _} -> stacks + :shutdown -> stacks + :normal when restart != :permanent -> stacks + reason -> Map.put(stacks, pid, {child, reason}) + end + end + + defp maybe_restart_child(pid, reason, %{children: children} = state) do + case children do + %{^pid => {_, restart, _, _, _} = child} -> + maybe_restart_child(restart, reason, pid, child, state) + + %{} -> + {:ok, state} + end + end + + defp maybe_restart_child(:permanent, reason, pid, child, state) do + report_error(:child_terminated, reason, pid, child, state) + restart_child(pid, child, state) + end + + defp maybe_restart_child(_, :normal, pid, _child, state) do + {:ok, delete_child(pid, state)} + end + + defp maybe_restart_child(_, :shutdown, pid, _child, state) do + {:ok, delete_child(pid, state)} + end + + defp maybe_restart_child(_, {:shutdown, _}, pid, _child, state) do + {:ok, delete_child(pid, state)} + end + + defp maybe_restart_child(:transient, reason, pid, child, state) do + report_error(:child_terminated, reason, pid, child, state) + restart_child(pid, child, state) + end + + defp maybe_restart_child(:temporary, reason, pid, child, state) do + report_error(:child_terminated, reason, pid, child, state) + {:ok, delete_child(pid, state)} + end + + defp delete_child(pid, %{children: children} = state) do + %{state | children: Map.delete(children, pid)} + end + + defp restart_child(pid, child, state) do + case add_restart(state) do + {:ok, %{strategy: strategy} = state} -> + case restart_child(strategy, pid, child, state) do + {:ok, state} -> + {:ok, state} + + {:try_again, state} -> + send(self(), {:"$gen_restart", pid}) + {:ok, state} + end + + {:shutdown, state} -> + report_error(:shutdown, :reached_max_restart_intensity, pid, child, state) + {:shutdown, delete_child(pid, state)} + end + end + + defp add_restart(state) do + %{max_seconds: max_seconds, max_restarts: max_restarts, restarts: restarts} = state + + now = :erlang.monotonic_time(1) + restarts = add_restart([now | restarts], now, max_seconds) + state = %{state | restarts: restarts} + + if length(restarts) <= max_restarts do + {:ok, state} + else + {:shutdown, state} + end + end + + defp add_restart(restarts, now, period) do + for then <- restarts, now <= then + period, do: then + end + + defp restart_child(:one_for_one, current_pid, child, state) do + {{m, f, args} = mfa, restart, shutdown, type, modules} = child + %{extra_arguments: extra} = state + + case start_child(m, f, extra ++ args) do + {:ok, pid, _} -> + state = delete_child(current_pid, state) + {:ok, save_child(pid, mfa, restart, shutdown, type, modules, state)} + + {:ok, pid} -> + state = delete_child(current_pid, state) + {:ok, save_child(pid, mfa, restart, shutdown, type, modules, state)} + + :ignore -> + {:ok, delete_child(current_pid, state)} + + {:error, reason} -> + report_error(:start_error, reason, {:restarting, current_pid}, child, state) + state = put_in(state.children[current_pid], {:restarting, child}) + {:try_again, state} + end + end + + defp report_error(error, reason, pid, child, %{name: name, extra_arguments: extra}) do + :logger.error( + %{ + label: {:supervisor, error}, + report: [ + {:supervisor, name}, + {:errorContext, error}, + {:reason, reason}, + {:offender, extract_child(pid, child, extra)} + ] + }, + %{ + domain: [:otp, :sasl], + report_cb: &:logger.format_otp_report/1, + logger_formatter: %{title: "SUPERVISOR REPORT"}, + error_logger: %{tag: :error_report, type: :supervisor_report} + } + ) + end + + defp extract_child(pid, {{m, f, args}, restart, shutdown, type, _modules}, extra) do + [ + pid: pid, + id: :undefined, + mfargs: {m, f, extra ++ args}, + restart_type: restart, + shutdown: shutdown, + child_type: type + ] + end + + ## Helpers + + @compile {:inline, call: 2} + + defp call(supervisor, req) do + GenServer.call(supervisor, req, :infinity) + end + + @doc false + def format_report(%{ + label: {__MODULE__, :unexpected_msg}, + report: %{msg: msg} + }) do + {~c"DynamicSupervisor received unexpected message: ~p~n", [msg]} + end +end diff --git a/lib/elixir/lib/enum.ex b/lib/elixir/lib/enum.ex index 46c97a9bc5a..8a8a3e89937 100644 --- a/lib/elixir/lib/enum.ex +++ b/lib/elixir/lib/enum.ex @@ -1,38 +1,76 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + defprotocol Enumerable do @moduledoc """ Enumerable protocol used by `Enum` and `Stream` modules. When you invoke a function in the `Enum` module, the first argument - is usually a collection that must implement this protocol. For example, - the expression - - Enum.map([1, 2, 3], &(&1 * 2)) - - invokes underneath `Enumerable.reduce/3` to perform the reducing - operation that builds a mapped list by calling the mapping function - `&(&1 * 2)` on every element in the collection and cons'ing the - element with an accumulated list. + is usually a collection that must implement this protocol. + For example, the expression `Enum.map([1, 2, 3], &(&1 * 2))` + invokes `Enumerable.reduce/3` to perform the reducing operation that + builds a mapped list by calling the mapping function `&(&1 * 2)` on + every element in the collection and consuming the element with an + accumulated list. Internally, `Enum.map/2` is implemented as follows: - def map(enum, fun) do - reducer = fn x, acc -> {:cont, [fun.(x)|acc]} end - Enumerable.reduce(enum, {:cont, []}, reducer) |> elem(1) |> :lists.reverse() + def map(enumerable, fun) do + reducer = fn x, acc -> {:cont, [fun.(x) | acc]} end + Enumerable.reduce(enumerable, {:cont, []}, reducer) |> elem(1) |> :lists.reverse() + end + + Note that the user-supplied function is wrapped into a `t:reducer/0` function. + The `t:reducer/0` function must return a tagged tuple after each step, + as described in the `t:acc/0` type. At the end, `Enumerable.reduce/3` + returns `t:result/0`. + + This protocol uses tagged tuples to exchange information between the + reducer function and the data type that implements the protocol. This + allows enumeration of resources, such as files, to be done efficiently + while also guaranteeing the resource will be closed at the end of the + enumeration. This protocol also allows suspension of the enumeration, + which is useful when interleaving between many enumerables is required + (as in the `zip/1` and `zip/2` functions). + + This protocol requires four functions to be implemented, `reduce/3`, + `count/1`, `member?/2`, and `slice/1`. The core of the protocol is the + `reduce/3` function. All other functions exist as optimizations paths + for data structures that can implement certain properties in better + than linear time. + + ## Default implementation for lists + + Sometimes you may want to implement this protocol for a list contained + in struct. This can be done by delegating to the `Enumerable.List` module + in the `reduce/3` implementation and providing a straight-forward + implementation for the remaining ones: + + defimpl Enumerable, for: CustomStruct do + def count(struct), do: {:ok, length(struct.items)} + def member?(struct, value), do: {:ok, value in struct.items} + def slice(struct), do: {:error, __MODULE__} + def reduce(struct, acc, fun), do: Enumerable.List.reduce(struct.items, acc, fun) end + """ + + @typedoc """ + An enumerable of elements of type `element`. - Notice the user given function is wrapped into a `reducer` function. - The `reducer` function must return a tagged tuple after each step, - as described in the `acc/0` type. + This type is equivalent to `t:t/0` but is especially useful for documentation. - The reason the accumulator requires a tagged tuple is to allow the - reducer function to communicate to the underlying enumerable the end - of enumeration, allowing any open resource to be properly closed. It - also allows suspension of the enumeration, which is useful when - interleaving between many enumerables is required (as in zip). + For example, imagine you define a function that expects an enumerable of + integers and returns an enumerable of strings: + + @spec integers_to_strings(Enumerable.t(integer())) :: Enumerable.t(String.t()) + def integers_to_strings(integers) do + Stream.map(integers, &Integer.to_string/1) + end - Finally, `Enumerable.reduce/3` will return another tagged tuple, - as represented by the `result/0` type. """ + @typedoc since: "1.14.0" + @type t(_element) :: t() @typedoc """ The accumulator value for each step. @@ -44,10 +82,10 @@ defprotocol Enumerable do * `:suspend` - the enumeration should be suspended immediately Depending on the accumulator value, the result returned by - `Enumerable.reduce/3` will change. Please check the `result` - type docs for more information. + `Enumerable.reduce/3` will change. Please check the `t:result/0` + type documentation for more information. - In case a reducer function returns a `:suspend` accumulator, + In case a `t:reducer/0` function returns a `:suspend` accumulator, it must be explicitly handled by the caller and never leak. """ @type acc :: {:cont, term} | {:halt, term} | {:suspend, term} @@ -55,28 +93,37 @@ defprotocol Enumerable do @typedoc """ The reducer function. - Should be called with the collection element and the - accumulator contents. Returns the accumulator for - the next enumeration step. + Should be called with the `enumerable` element and the + accumulator contents. + + Returns the accumulator for the next enumeration step. """ - @type reducer :: (term, term -> acc) + @type reducer :: (element :: term, element_acc :: term -> acc) @typedoc """ The result of the reduce operation. It may be *done* when the enumeration is finished by reaching its end, or *halted*/*suspended* when the enumeration was halted - or suspended by the reducer function. - - In case a reducer function returns the `:suspend` accumulator, the - `:suspended` tuple must be explicitly handled by the caller and - never leak. In practice, this means regular enumeration functions - just need to be concerned about `:done` and `:halted` results. - - Furthermore, a `:suspend` call must always be followed by another call, - eventually halting or continuing until the end. + or suspended by the tagged accumulator. + + In case the tagged `:halt` accumulator is given, the `:halted` tuple + with the accumulator must be returned. Functions like `Enum.take_while/2` + use `:halt` underneath and can be used to test halting enumerables. + + In case the tagged `:suspend` accumulator is given, the caller must + return the `:suspended` tuple with the accumulator and a continuation. + The caller is then responsible of managing the continuation and the + caller must always call the continuation, eventually halting or continuing + until the end. `Enum.zip/2` uses suspension, so it can be used to test + whether your implementation handles suspension correctly. You can also use + `Stream.zip/2` with `Enum.take_while/2` to test the combination of + `:suspend` with `:halt`. """ - @type result :: {:done, term} | {:halted, term} | {:suspended, term, continuation} + @type result :: + {:done, term} + | {:halted, term} + | {:suspended, term, continuation} @typedoc """ A partially applied reduce function. @@ -85,192 +132,350 @@ defprotocol Enumerable do the enumeration is suspended. When invoked, it expects a new accumulator and it returns the result. - A continuation is easily implemented as long as the reduce + A continuation can be trivially implemented as long as the reduce function is defined in a tail recursive fashion. If the function is tail recursive, all the state is passed as arguments, so - the continuation would simply be the reducing function partially - applied. + the continuation is the reducing function partially applied. """ @type continuation :: (acc -> result) + @typedoc """ + A slicing function that receives the initial position, + the number of elements in the slice, and the step. + + The `start` position is a number `>= 0` and guaranteed to + exist in the `enumerable`. The length is a number `>= 1` + in a way that `start + length * step <= count`, where + `count` is the maximum amount of elements in the enumerable. + + The function should return a non empty list where + the amount of elements is equal to `length`. + """ + @type slicing_fun :: + (start :: non_neg_integer, length :: pos_integer, step :: pos_integer -> [term()]) + + @typedoc """ + Receives an enumerable and returns a list. + """ + @type to_list_fun :: (t -> [term()]) + @doc """ - Reduces the collection into a value. + Reduces the `enumerable` into an element. Most of the operations in `Enum` are implemented in terms of reduce. - This function should apply the given `reducer` function to each - item in the collection and proceed as expected by the returned accumulator. + This function should apply the given `t:reducer/0` function to each + element in the `enumerable` and proceed as expected by the returned + accumulator. + + See the documentation of the types `t:result/0` and `t:acc/0` for + more information. + + ## Examples As an example, here is the implementation of `reduce` for lists: - def reduce(_, {:halt, acc}, _fun), do: {:halted, acc} - def reduce(list, {:suspend, acc}, fun), do: {:suspended, acc, &reduce(list, &1, fun)} - def reduce([], {:cont, acc}, _fun), do: {:done, acc} - def reduce([h|t], {:cont, acc}, fun), do: reduce(t, fun.(h, acc), fun) + def reduce(_list, {:halt, acc}, _fun), do: {:halted, acc} + def reduce(list, {:suspend, acc}, fun), do: {:suspended, acc, &reduce(list, &1, fun)} + def reduce([], {:cont, acc}, _fun), do: {:done, acc} + def reduce([head | tail], {:cont, acc}, fun), do: reduce(tail, fun.(head, acc), fun) """ @spec reduce(t, acc, reducer) :: result - def reduce(collection, acc, fun) + def reduce(enumerable, acc, fun) @doc """ - Checks if a value exists within the collection. + Retrieves the number of elements in the `enumerable`. - It should return `{:ok, boolean}`. + It should return `{:ok, count}` if you can count the number of elements + in `enumerable` in a faster way than fully traversing it. - If `{:error, __MODULE__}` is returned a default algorithm using `reduce` and - the match (`===`) operator is used. This algorithm runs in linear time. + Otherwise it should return `{:error, __MODULE__}` and a default algorithm + built on top of `reduce/3` that runs in linear time will be used. + """ + @spec count(t) :: {:ok, non_neg_integer} | {:error, module} + def count(enumerable) + + @doc """ + Checks if an `element` exists within the `enumerable`. - Please force use of the default algorithm unless you can implement an - algorithm that is significantly faster. + It should return `{:ok, boolean}` if you can check the membership of a + given element in `enumerable` with `===/2` without traversing the whole + of it. + + Otherwise it should return `{:error, __MODULE__}` and a default algorithm + built on top of `reduce/3` that runs in linear time will be used. + + When called outside guards, the [`in`](`in/2`) and [`not in`](`in/2`) + operators work by using this function. """ @spec member?(t, term) :: {:ok, boolean} | {:error, module} - def member?(collection, value) + def member?(enumerable, element) @doc """ - Retrieves the collection's size. + Returns a function that slices the data structure contiguously. + + It should return either: + + * `{:ok, size, slicing_fun}` - if the `enumerable` has a known + bound and can access a position in the `enumerable` without + traversing all previous elements. The `slicing_fun` will receive + a `start` position, the `amount` of elements to fetch, and a + `step`. - It should return `{:ok, size}`. + * `{:ok, size, to_list_fun}` - if the `enumerable` has a known bound + and can access a position in the `enumerable` by first converting + it to a list via `to_list_fun`. - If `{:error, __MODULE__}` is returned a default algorithm using `reduce` and - the match (`===`) operator is used. This algorithm runs in linear time. + * `{:error, __MODULE__}` - the enumerable cannot be sliced efficiently + and a default algorithm built on top of `reduce/3` that runs in + linear time will be used. - Please force use of the default algorithm unless you can implement an - algorithm that is significantly faster. + ## Differences to `count/1` + + The `size` value returned by this function is used for boundary checks, + therefore it is extremely important that this function only returns `:ok` + if retrieving the `size` of the `enumerable` is cheap, fast, and takes + constant time. Otherwise the simplest of operations, such as + `Enum.at(enumerable, 0)`, will become too expensive. + + On the other hand, the `count/1` function in this protocol should be + implemented whenever you can count the number of elements in the collection + without traversing it. """ - @spec count(t) :: {:ok, non_neg_integer} | {:error, module} - def count(collection) + @spec slice(t) :: + {:ok, size :: non_neg_integer(), slicing_fun() | to_list_fun()} + | {:error, module()} + def slice(enumerable) end defmodule Enum do import Kernel, except: [max: 2, min: 2] @moduledoc """ - Provides a set of algorithms that enumerate over collections according to the - `Enumerable` protocol: - - iex> Enum.map([1, 2, 3], fn(x) -> x * 2 end) - [2,4,6] + Functions for working with collections (known as enumerables). - Some particular types, like dictionaries, yield a specific format on - enumeration. For dicts, the argument is always a `{key, value}` tuple: + In Elixir, an enumerable is any data type that implements the + `Enumerable` protocol. `List`s (`[1, 2, 3]`), `Map`s (`%{foo: 1, bar: 2}`) + and `Range`s (`1..3`) are common data types used as enumerables: - iex> dict = %{a: 1, b: 2} - iex> Enum.map(dict, fn {k, v} -> {k, v * 2} end) - [a: 2, b: 4] + iex> Enum.map([1, 2, 3], fn x -> x * 2 end) + [2, 4, 6] - Note that the functions in the `Enum` module are eager: they always start - the enumeration of the given collection. The `Stream` module allows - lazy enumeration of collections and provides infinite streams. + iex> Enum.sum([1, 2, 3]) + 6 - Since the majority of the functions in `Enum` enumerate the whole - collection and return a list as result, infinite streams need to - be carefully used with such functions, as they can potentially run - forever. For example: + iex> Enum.map(1..3, fn x -> x * 2 end) + [2, 4, 6] - Enum.each Stream.cycle([1,2,3]), &IO.puts(&1) + iex> Enum.sum(1..3) + 6 + iex> map = %{"a" => 1, "b" => 2} + iex> Enum.map(map, fn {k, v} -> {k, v * 2} end) + [{"a", 2}, {"b", 4}] + + Many other enumerables exist in the language, such as `MapSet`s + and the data type returned by `File.stream!/3` which allows a file to be + traversed as if it was an enumerable. + + For a general overview of all functions in the `Enum` module, see + [the `Enum` cheatsheet](enum-cheat.cheatmd). + + The functions in this module work in linear time. This means that, the + time it takes to perform an operation grows at the same rate as the length + of the enumerable. This is expected on operations such as `Enum.map/2`. + After all, if we want to traverse every element on a list, the longer the + list, the more elements we need to traverse, and the longer it will take. + + This linear behavior should also be expected on operations like `count/1`, + `member?/2`, `at/2` and similar. While Elixir does allow data types to + provide performant variants for such operations, you should not expect it + to always be available, since the `Enum` module is meant to work with a + large variety of data types and not all data types can provide optimized + behavior. + + Finally, note the functions in the `Enum` module are eager: they will + traverse the enumerable as soon as they are invoked. This is particularly + dangerous when working with infinite enumerables. In such cases, you should + use the `Stream` module, which allows you to lazily express computations, + without traversing collections, and work with possibly infinite collections. + See the `Stream` module for examples and documentation. """ @compile :inline_list_funcs - @type t :: Enumerable.t + @type t :: Enumerable.t() + @type acc :: any @type element :: any - @type index :: non_neg_integer + + @typedoc "Zero-based index. It can also be a negative integer." + @type index :: integer + @type default :: any - # Require Stream.Reducers and its callbacks require Stream.Reducers, as: R - defmacrop cont(_, entry, acc) do - quote do: {:cont, [unquote(entry)|unquote(acc)]} + defmacrop skip(acc) do + acc + end + + defmacrop next(_, entry, acc) do + quote(do: [unquote(entry) | unquote(acc)]) end - defmacrop acc(h, n, _) do - quote do: {unquote(h), unquote(n)} + defmacrop acc(head, state, _) do + quote(do: {unquote(head), unquote(state)}) end - defmacrop cont_with_acc(f, entry, h, n, _) do + defmacrop next_with_acc(_, entry, head, state, _) do quote do - {:cont, {[unquote(entry)|unquote(h)], unquote(n)}} + {[unquote(entry) | unquote(head)], unquote(state)} end end @doc """ - Invokes the given `fun` for each item in the `collection` and returns `false` - if at least one invocation returns `false`. Otherwise returns `true`. + Returns `true` if all elements in `enumerable` are truthy. + + When an element has a falsy value (`false` or `nil`) iteration stops immediately + and `false` is returned. In all other cases `true` is returned. ## Examples - iex> Enum.all?([2, 4, 6], fn(x) -> rem(x, 2) == 0 end) + iex> Enum.all?([1, 2, 3]) true - iex> Enum.all?([2, 3, 4], fn(x) -> rem(x, 2) == 0 end) + iex> Enum.all?([1, nil, 3]) false - If no function is given, it defaults to checking if - all items in the collection evaluate to `true`. + iex> Enum.all?([]) + true - iex> Enum.all?([1, 2, 3]) + """ + @spec all?(t) :: boolean + def all?(enumerable) when is_list(enumerable) do + all_list(enumerable) + end + + def all?(enumerable) do + Enumerable.reduce(enumerable, {:cont, true}, fn entry, _ -> + if entry, do: {:cont, true}, else: {:halt, false} + end) + |> elem(1) + end + + @doc """ + Returns `true` if `fun.(element)` is truthy for all elements in `enumerable`. + + Iterates over `enumerable` and invokes `fun` on each element. If `fun` ever + returns a falsy value (`false` or `nil`), iteration stops immediately and + `false` is returned. Otherwise, `true` is returned. + + ## Examples + + iex> Enum.all?([2, 4, 6], fn x -> rem(x, 2) == 0 end) true - iex> Enum.all?([1, nil, 3]) + iex> Enum.all?([2, 3, 4], fn x -> rem(x, 2) == 0 end) false + iex> Enum.all?([], fn _ -> nil end) + true + + As the last example shows, `Enum.all?/2` returns `true` if `enumerable` is + empty, regardless of `fun`. In an empty enumerable there is no element for + which `fun` returns a falsy value, so the result must be `true`. This is a + well-defined logical argument for empty collections. + """ - @spec all?(t) :: boolean @spec all?(t, (element -> as_boolean(term))) :: boolean + def all?(enumerable, fun) when is_list(enumerable) do + predicate_list(enumerable, true, fun) + end - def all?(collection, fun \\ fn(x) -> x end) - - def all?(collection, fun) when is_list(collection) do - do_all?(collection, fun) + def all?(first..last//step, fun) do + predicate_range(first, last, step, true, fun) end - def all?(collection, fun) do - Enumerable.reduce(collection, {:cont, true}, fn(entry, _) -> + def all?(enumerable, fun) do + Enumerable.reduce(enumerable, {:cont, true}, fn entry, _ -> if fun.(entry), do: {:cont, true}, else: {:halt, false} - end) |> elem(1) + end) + |> elem(1) end @doc """ - Invokes the given `fun` for each item in the `collection` and returns `true` if - at least one invocation returns `true`. Returns `false` otherwise. + Returns `true` if at least one element in `enumerable` is truthy. + + When an element has a truthy value (neither `false` nor `nil`) iteration stops + immediately and `true` is returned. In all other cases `false` is returned. ## Examples - iex> Enum.any?([2, 4, 6], fn(x) -> rem(x, 2) == 1 end) + iex> Enum.any?([false, false, false]) false - iex> Enum.any?([2, 3, 4], fn(x) -> rem(x, 2) == 1 end) + iex> Enum.any?([false, true, false]) true - If no function is given, it defaults to checking if - at least one item in the collection evaluates to `true`. + iex> Enum.any?([]) + false - iex> Enum.any?([false, false, false]) + """ + @spec any?(t) :: boolean + def any?(enumerable) when is_list(enumerable) do + any_list(enumerable) + end + + def any?(enumerable) do + Enumerable.reduce(enumerable, {:cont, false}, fn entry, _ -> + if entry, do: {:halt, true}, else: {:cont, false} + end) + |> elem(1) + end + + @doc """ + Returns `true` if `fun.(element)` is truthy for at least one element in `enumerable`. + + Iterates over the `enumerable` and invokes `fun` on each element. When an invocation + of `fun` returns a truthy value (neither `false` nor `nil`) iteration stops + immediately and `true` is returned. In all other cases `false` is returned. + + ## Examples + + iex> Enum.any?([2, 4, 6], fn x -> rem(x, 2) == 1 end) false - iex> Enum.any?([false, true, false]) + iex> Enum.any?([2, 3, 4], fn x -> rem(x, 2) == 1 end) true + iex> Enum.any?([], fn x -> x > 0 end) + false + """ - @spec any?(t) :: boolean @spec any?(t, (element -> as_boolean(term))) :: boolean + def any?(enumerable, fun) when is_list(enumerable) do + predicate_list(enumerable, false, fun) + end - def any?(collection, fun \\ fn(x) -> x end) - - def any?(collection, fun) when is_list(collection) do - do_any?(collection, fun) + def any?(first..last//step, fun) do + predicate_range(first, last, step, false, fun) end - def any?(collection, fun) do - Enumerable.reduce(collection, {:cont, false}, fn(entry, _) -> + def any?(enumerable, fun) do + Enumerable.reduce(enumerable, {:cont, false}, fn entry, _ -> if fun.(entry), do: {:halt, true}, else: {:cont, false} - end) |> elem(1) + end) + |> elem(1) end @doc """ - Finds the element at the given index (zero-based). - Returns `default` if index is out of bounds. + Finds the element at the given `index` (zero-based). + + Returns `default` if `index` is out of bounds. + + A negative `index` can be passed, which means the `enumerable` is + enumerated once and the `index` is counted from the end (for example, + `-1` finds the last element). ## Examples @@ -287,67 +492,155 @@ defmodule Enum do :none """ - @spec at(t, integer) :: element | nil - @spec at(t, integer, default) :: element | default - def at(collection, n, default \\ nil) do - case fetch(collection, n) do - {:ok, h} -> h - :error -> default + @spec at(t, index, default) :: element | default + def at(enumerable, index, default \\ nil) when is_integer(index) do + case slice_forward(enumerable, index, 1, 1) do + [value] -> value + [] -> default end end + @doc false + @deprecated "Use Enum.chunk_every/2 instead" + def chunk(enumerable, count), do: chunk(enumerable, count, count, nil) + + @doc false + @deprecated "Use Enum.chunk_every/3 instead" + def chunk(enum, n, step) do + chunk_every(enum, n, step, :discard) + end + + @doc false + @deprecated "Use Enum.chunk_every/4 instead" + def chunk(enumerable, count, step, leftover) do + chunk_every(enumerable, count, step, leftover || :discard) + end + @doc """ - Shortcut to `chunk(coll, n, n)`. + Shortcut to `chunk_every(enumerable, count, count)`. """ - @spec chunk(t, non_neg_integer) :: [list] - def chunk(coll, n), do: chunk(coll, n, n, nil) + @doc since: "1.5.0" + @spec chunk_every(t, pos_integer) :: [list] + def chunk_every(enumerable, count), do: chunk_every(enumerable, count, count, []) @doc """ - Returns a collection of lists containing `n` items each, where - each new chunk starts `step` elements into the collection. + Returns list of lists containing `count` elements each, where + each new chunk starts `step` elements into the `enumerable`. + + `step` is optional and, if not passed, defaults to `count`, i.e. + chunks do not overlap. Chunking will stop as soon as the collection + ends or when we emit an incomplete chunk. + + If the last chunk does not have `count` elements to fill the chunk, + elements are taken from `leftover` to fill in the chunk. If `leftover` + does not have enough elements to fill the chunk, then a partial chunk + is returned with less than `count` elements. - `step` is optional and, if not passed, defaults to `n`, i.e. - chunks do not overlap. If the final chunk does not have `n` - elements to fill the chunk, elements are taken as necessary - from `pad` if it was passed. If `pad` is passed and does not - have enough elements to fill the chunk, then the chunk is - returned anyway with less than `n` elements. If `pad` is not - passed at all or is `nil`, then the partial chunk is discarded - from the result. + If `:discard` is given in `leftover`, the last chunk is discarded + unless it has exactly `count` elements. ## Examples - iex> Enum.chunk([1, 2, 3, 4, 5, 6], 2) + iex> Enum.chunk_every([1, 2, 3, 4, 5, 6], 2) [[1, 2], [3, 4], [5, 6]] - iex> Enum.chunk([1, 2, 3, 4, 5, 6], 3, 2) + iex> Enum.chunk_every([1, 2, 3, 4, 5, 6], 3, 2, :discard) [[1, 2, 3], [3, 4, 5]] - iex> Enum.chunk([1, 2, 3, 4, 5, 6], 3, 2, [7]) + iex> Enum.chunk_every([1, 2, 3, 4, 5, 6], 3, 2, [7]) [[1, 2, 3], [3, 4, 5], [5, 6, 7]] - iex> Enum.chunk([1, 2, 3, 4, 5, 6], 3, 3, []) - [[1, 2, 3], [4, 5, 6]] + iex> Enum.chunk_every([1, 2, 3, 4], 3, 3, []) + [[1, 2, 3], [4]] + + iex> Enum.chunk_every([1, 2, 3, 4], 10) + [[1, 2, 3, 4]] + + iex> Enum.chunk_every([1, 2, 3, 4, 5], 2, 3, []) + [[1, 2], [4, 5]] + + iex> Enum.chunk_every([1, 2, 3, 4], 3, 3, Stream.cycle([0])) + [[1, 2, 3], [4, 0, 0]] """ - @spec chunk(t, non_neg_integer, non_neg_integer) :: [list] - @spec chunk(t, non_neg_integer, non_neg_integer, t | nil) :: [list] - def chunk(coll, n, step, pad \\ nil) when n > 0 and step > 0 do - limit = :erlang.max(n, step) + @doc since: "1.5.0" + @spec chunk_every(t, pos_integer, pos_integer, t | :discard) :: [list] + def chunk_every(enumerable, count, step, leftover \\ []) + when is_integer(count) and count > 0 and is_integer(step) and step > 0 do + R.chunk_every(&chunk_while/4, enumerable, count, step, leftover) + end - {_, {acc, {buffer, i}}} = - Enumerable.reduce(coll, {:cont, {[], {[], 0}}}, R.chunk(n, step, limit)) + @doc """ + Chunks the `enumerable` with fine grained control when every chunk is emitted. - if nil?(pad) || i == 0 do - :lists.reverse(acc) - else - buffer = :lists.reverse(buffer) ++ take(pad, n - i) - :lists.reverse([buffer|acc]) + `chunk_fun` receives the current element and the accumulator and must return: + + * `{:cont, chunk, acc}` to emit a chunk and continue with the accumulator + * `{:cont, acc}` to not emit any chunk and continue with the accumulator + * `{:halt, acc}` to halt chunking over the `enumerable`. + + `after_fun` is invoked with the final accumulator when iteration is + finished (or `halt`ed) to handle any trailing elements that were returned + as part of an accumulator, but were not emitted as a chunk by `chunk_fun`. + It must return: + + * `{:cont, chunk, acc}` to emit a chunk. The chunk will be appended to the + list of already emitted chunks. + * `{:cont, acc}` to not emit a chunk + + The `acc` in `after_fun` is required in order to mirror the tuple format + from `chunk_fun` but it will be discarded since the traversal is complete. + + Returns a list of emitted chunks. + + ## Examples + + iex> chunk_fun = fn element, acc -> + ...> if rem(element, 2) == 0 do + ...> {:cont, Enum.reverse([element | acc]), []} + ...> else + ...> {:cont, [element | acc]} + ...> end + ...> end + iex> after_fun = fn + ...> [] -> {:cont, []} + ...> acc -> {:cont, Enum.reverse(acc), []} + ...> end + iex> Enum.chunk_while(1..10, [], chunk_fun, after_fun) + [[1, 2], [3, 4], [5, 6], [7, 8], [9, 10]] + iex> Enum.chunk_while([1, 2, 3, 5, 7], [], chunk_fun, after_fun) + [[1, 2], [3, 5, 7]] + + """ + @doc since: "1.5.0" + @spec chunk_while( + t, + acc, + (element, acc -> {:cont, chunk, acc} | {:cont, acc} | {:halt, acc}), + (acc -> {:cont, chunk, acc} | {:cont, acc}) + ) :: Enumerable.t() + when chunk: any + def chunk_while(enumerable, acc, chunk_fun, after_fun) do + {_, {res, acc}} = + Enumerable.reduce(enumerable, {:cont, {[], acc}}, fn entry, {buffer, acc} -> + case chunk_fun.(entry, acc) do + {:cont, chunk, acc} -> {:cont, {[chunk | buffer], acc}} + {:cont, acc} -> {:cont, {buffer, acc}} + {:halt, acc} -> {:halt, {buffer, acc}} + end + end) + + case after_fun.(acc) do + {:cont, _acc} -> :lists.reverse(res) + {:cont, chunk, _acc} -> :lists.reverse([chunk | res]) end end @doc """ - Splits `coll` on every element for which `fun` returns a new value. + Splits enumerable on every element for which `fun` returns a new + value. + + Returns a list of lists. ## Examples @@ -356,47 +649,48 @@ defmodule Enum do """ @spec chunk_by(t, (element -> any)) :: [list] - def chunk_by(coll, fun) do - {_, {acc, res}} = - Enumerable.reduce(coll, {:cont, {[], nil}}, R.chunk_by(fun)) - - case res do - {buffer, _} -> - :lists.reverse([:lists.reverse(buffer) | acc]) - nil -> - [] - end + def chunk_by(enumerable, fun) do + R.chunk_by(&chunk_while/4, enumerable, fun) end @doc """ - Given an enumerable of enumerables, concatenate the enumerables into a single list. + Given an enumerable of enumerables, concatenates the `enumerables` into + a single list. ## Examples iex> Enum.concat([1..3, 4..6, 7..9]) - [1,2,3,4,5,6,7,8,9] + [1, 2, 3, 4, 5, 6, 7, 8, 9] iex> Enum.concat([[1, [2], 3], [4], [5, 6]]) - [1,[2],3,4,5,6] + [1, [2], 3, 4, 5, 6] """ @spec concat(t) :: t - def concat(enumerables) do - do_concat(enumerables) + def concat(enumerables) + + def concat(list) when is_list(list) do + concat_list(list) + end + + def concat(enums) do + concat_enum(enums) end @doc """ - Concatenates the enumerable on the right with the enumerable on the left. + Concatenates the enumerable on the `right` with the enumerable on the + `left`. - This function produces the same result as the `Kernel.++/2` operator for lists. + This function produces the same result as the `++/2` operator + for lists. ## Examples iex> Enum.concat(1..3, 4..6) - [1,2,3,4,5,6] + [1, 2, 3, 4, 5, 6] iex> Enum.concat([1, 2, 3], [4, 5, 6]) - [1,2,3,4,5,6] + [1, 2, 3, 4, 5, 6] """ @spec concat(t, t) :: t @@ -405,16 +699,11 @@ defmodule Enum do end def concat(left, right) do - do_concat([left, right]) - end - - defp do_concat(enumerable) do - fun = &[&1|&2] - reduce(enumerable, [], &reduce(&1, &2, fun)) |> :lists.reverse + concat_enum([left, right]) end @doc """ - Returns the collection's size. + Returns the size of the `enumerable`. ## Examples @@ -423,45 +712,156 @@ defmodule Enum do """ @spec count(t) :: non_neg_integer - def count(collection) when is_list(collection) do - :erlang.length(collection) + def count(enumerable) when is_list(enumerable) do + length(enumerable) end - def count(collection) do - case Enumerable.count(collection) do + def count(enumerable) do + case Enumerable.count(enumerable) do {:ok, value} when is_integer(value) -> value + {:error, module} -> - module.reduce(collection, {:cont, 0}, fn - _, acc -> {:cont, acc + 1} - end) |> elem(1) + enumerable |> module.reduce({:cont, 0}, fn _, acc -> {:cont, acc + 1} end) |> elem(1) end end @doc """ - Returns the count of items in the collection for which - `fun` returns `true`. + Returns the count of elements in the `enumerable` for which `fun` returns + a truthy value. ## Examples - iex> Enum.count([1, 2, 3, 4, 5], fn(x) -> rem(x, 2) == 0 end) + iex> Enum.count([1, 2, 3, 4, 5], fn x -> rem(x, 2) == 0 end) 2 """ @spec count(t, (element -> as_boolean(term))) :: non_neg_integer - def count(collection, fun) do - Enumerable.reduce(collection, {:cont, 0}, fn(entry, acc) -> - {:cont, if(fun.(entry), do: acc + 1, else: acc)} - end) |> elem(1) + def count(enumerable, fun) do + reduce(enumerable, 0, fn entry, acc -> + if(fun.(entry), do: acc + 1, else: acc) + end) + end + + @doc """ + Counts the enumerable stopping at `limit`. + + This is useful for checking certain properties of the count of an enumerable + without having to actually count the entire enumerable. For example, if you + wanted to check that the count was exactly, at least, or more than a value. + + If the enumerable implements `c:Enumerable.count/1`, the enumerable is + not traversed and we return the lower of the two numbers. To force + enumeration, use `count_until/3` with `fn _ -> true end` as the second + argument. + + ## Examples + + iex> Enum.count_until(1..20, 5) + 5 + iex> Enum.count_until(1..20, 50) + 20 + iex> Enum.count_until(1..10, 10) == 10 # At least 10 + true + iex> Enum.count_until(1..11, 10 + 1) > 10 # More than 10 + true + iex> Enum.count_until(1..5, 10) < 10 # Less than 10 + true + iex> Enum.count_until(1..10, 10 + 1) == 10 # Exactly ten + true + + """ + @doc since: "1.12.0" + @spec count_until(t, pos_integer) :: non_neg_integer + def count_until(enumerable, limit) when is_integer(limit) and limit > 0 do + case enumerable do + list when is_list(list) -> count_until_list(list, limit, 0) + _ -> count_until_enum(enumerable, limit) + end + end + + @doc """ + Counts the elements in the enumerable for which `fun` returns a truthy value, stopping at `limit`. + + See `count/2` and `count_until/2` for more information. + + ## Examples + + iex> Enum.count_until(1..20, fn x -> rem(x, 2) == 0 end, 7) + 7 + iex> Enum.count_until(1..20, fn x -> rem(x, 2) == 0 end, 11) + 10 + """ + @doc since: "1.12.0" + @spec count_until(t, (element -> as_boolean(term)), pos_integer) :: non_neg_integer + def count_until(enumerable, fun, limit) when is_integer(limit) and limit > 0 do + case enumerable do + list when is_list(list) -> count_until_list(list, fun, limit, 0) + _ -> count_until_enum(enumerable, fun, limit) + end + end + + @doc """ + Enumerates the `enumerable`, returning a list where all consecutive + duplicate elements are collapsed to a single element. + + Elements are compared using `===/2`. + + If you want to remove all duplicate elements, regardless of order, + see `uniq/1`. + + ## Examples + + iex> Enum.dedup([1, 2, 3, 3, 2, 1]) + [1, 2, 3, 2, 1] + + iex> Enum.dedup([1, 1, 2, 2.0, :three, :three]) + [1, 2, 2.0, :three] + + """ + @spec dedup(t) :: list + def dedup(enumerable) when is_list(enumerable) do + dedup_list(enumerable, []) |> :lists.reverse() + end + + def dedup(enumerable) do + reduce(enumerable, [], fn x, acc -> + case acc do + [^x | _] -> acc + _ -> [x | acc] + end + end) + |> :lists.reverse() + end + + @doc """ + Enumerates the `enumerable`, returning a list where all consecutive + duplicate elements are collapsed to a single element. + + The function `fun` maps every element to a term which is used to + determine if two elements are duplicates. + + ## Examples + + iex> Enum.dedup_by([{1, :a}, {2, :b}, {2, :c}, {1, :a}], fn {x, _} -> x end) + [{1, :a}, {2, :b}, {1, :a}] + + iex> Enum.dedup_by([5, 1, 2, 3, 2, 1], fn x -> x > 2 end) + [5, 1, 3, 2] + + """ + @spec dedup_by(t, (element -> term)) :: list + def dedup_by(enumerable, fun) do + {list, _} = reduce(enumerable, {[], []}, R.dedup(fun)) + :lists.reverse(list) end @doc """ - Drops the first `count` items from `collection`. + Drops the `amount` of elements from the `enumerable`. - If a negative value `count` is given, the last `count` - values will be dropped. The collection is enumerated - once to retrieve the proper index and the remaining - calculation is performed from the end. + If a negative `amount` is given, the `amount` of last values will be dropped. + The `enumerable` will be enumerated once to retrieve the proper index and + the remaining calculation is performed from the end. ## Examples @@ -472,79 +872,122 @@ defmodule Enum do [] iex> Enum.drop([1, 2, 3], 0) - [1,2,3] + [1, 2, 3] iex> Enum.drop([1, 2, 3], -1) - [1,2] + [1, 2] """ @spec drop(t, integer) :: list - def drop(collection, count) when is_list(collection) and count >= 0 do - do_drop(collection, count) + def drop(enumerable, amount) + when is_list(enumerable) and is_integer(amount) and amount >= 0 do + drop_list(enumerable, amount) end - def drop(collection, count) when count >= 0 do - res = - reduce(collection, count, fn - x, acc when is_list(acc) -> [x|acc] - x, 0 -> [x] - _, acc when acc > 0 -> acc - 1 - end) - if is_list(res), do: :lists.reverse(res), else: [] + def drop(enumerable, 0) do + to_list(enumerable) + end + + def drop(enumerable, amount) when is_integer(amount) and amount > 0 do + {result, _} = reduce(enumerable, {[], amount}, R.drop()) + if is_list(result), do: :lists.reverse(result), else: [] + end + + def drop(enumerable, amount) when is_integer(amount) and amount < 0 do + {count, fun} = slice_count_and_fun(enumerable, 1) + amount = Kernel.min(amount + count, count) + + if amount > 0 do + fun.(0, amount, 1) + else + [] + end end - def drop(collection, count) when count < 0 do - do_drop(reverse(collection), abs(count)) |> :lists.reverse + @doc """ + Returns a list of every `nth` element in the `enumerable` dropped, + starting with the first element. + + The first element is always dropped, unless `nth` is 0. + + The second argument specifying every `nth` element must be a non-negative + integer. + + ## Examples + + iex> Enum.drop_every(1..10, 2) + [2, 4, 6, 8, 10] + + iex> Enum.drop_every(1..10, 0) + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + + iex> Enum.drop_every([1, 2, 3], 1) + [] + + """ + @spec drop_every(t, non_neg_integer) :: list + def drop_every(enumerable, nth) + + def drop_every(_enumerable, 1), do: [] + def drop_every(enumerable, 0), do: to_list(enumerable) + def drop_every([], nth) when is_integer(nth), do: [] + + def drop_every(enumerable, nth) when is_integer(nth) and nth > 1 do + {res, _} = reduce(enumerable, {[], :first}, R.drop_every(nth)) + :lists.reverse(res) end @doc """ - Drops items at the beginning of `collection` while `fun` returns `true`. + Drops elements at the beginning of the `enumerable` while `fun` returns a + truthy value. ## Examples - iex> Enum.drop_while([1, 2, 3, 4, 5], fn(x) -> x < 3 end) - [3,4,5] + iex> Enum.drop_while([1, 2, 3, 2, 1], fn x -> x < 3 end) + [3, 2, 1] """ @spec drop_while(t, (element -> as_boolean(term))) :: list - def drop_while(collection, fun) when is_list(collection) do - do_drop_while(collection, fun) + def drop_while(enumerable, fun) when is_list(enumerable) do + drop_while_list(enumerable, fun) end - def drop_while(collection, fun) do - {_, {res, _}} = - Enumerable.reduce(collection, {:cont, {[], true}}, R.drop_while(fun)) + def drop_while(enumerable, fun) do + {res, _} = reduce(enumerable, {[], true}, R.drop_while(fun)) :lists.reverse(res) end @doc """ - Invokes the given `fun` for each item in the `collection`. + Invokes the given `fun` for each element in the `enumerable`. + Returns `:ok`. ## Examples - Enum.each(["some", "example"], fn(x) -> IO.puts x end) + Enum.each(["some", "example"], fn x -> IO.puts(x) end) "some" "example" #=> :ok """ @spec each(t, (element -> any)) :: :ok - def each(collection, fun) when is_list(collection) do - :lists.foreach(fun, collection) - :ok + def each(enumerable, fun) when is_list(enumerable) do + :lists.foreach(fun, enumerable) end - def each(collection, fun) do - reduce(collection, nil, fn(entry, _) -> + def each(enumerable, fun) do + reduce(enumerable, nil, fn entry, _ -> fun.(entry) nil end) + :ok end @doc """ - Returns `true` if the collection is empty, otherwise `false`. + Determines if the `enumerable` is empty. + + Returns `true` if `enumerable` is empty, otherwise `false`. ## Examples @@ -556,20 +999,29 @@ defmodule Enum do """ @spec empty?(t) :: boolean - def empty?(collection) when is_list(collection) do - collection == [] + def empty?(enumerable) when is_list(enumerable) do + enumerable == [] end - def empty?(collection) do - Enumerable.reduce(collection, {:cont, true}, fn(_, _) -> {:halt, false} end) |> elem(1) + def empty?(enumerable) do + case Enumerable.slice(enumerable) do + {:ok, value, _} -> + value == 0 + + {:error, module} -> + enumerable + |> module.reduce({:cont, true}, fn _, _ -> {:halt, false} end) + |> elem(1) + end end @doc """ - Finds the element at the given index (zero-based). + Finds the element at the given `index` (zero-based). + Returns `{:ok, element}` if found, otherwise `:error`. - A negative index can be passed, which means the collection is - enumerated once and the index is counted from the end (i.e. + A negative `index` can be passed, which means the `enumerable` is + enumerated once and the `index` is counted from the end (for example, `-1` fetches the last element). ## Examples @@ -577,6 +1029,9 @@ defmodule Enum do iex> Enum.fetch([2, 4, 6], 0) {:ok, 2} + iex> Enum.fetch([2, 4, 6], -3) + {:ok, 2} + iex> Enum.fetch([2, 4, 6], 2) {:ok, 6} @@ -584,35 +1039,19 @@ defmodule Enum do :error """ - @spec fetch(t, integer) :: {:ok, element} | :error - def fetch(collection, n) when is_list(collection) and n >= 0 do - do_fetch(collection, n) - end - - def fetch(collection, n) when n >= 0 do - res = - Enumerable.reduce(collection, {:cont, 0}, fn(entry, acc) -> - if acc == n do - {:halt, entry} - else - {:cont, acc + 1} - end - end) - - case res do - {:halted, entry} -> {:ok, entry} - {:done, _} -> :error + @spec fetch(t, index) :: {:ok, element} | :error + def fetch(enumerable, index) when is_integer(index) do + case slice_forward(enumerable, index, 1, 1) do + [value] -> {:ok, value} + [] -> :error end end - def fetch(collection, n) when n < 0 do - do_fetch(reverse(collection), abs(n + 1)) - end - @doc """ - Finds the element at the given index (zero-based). - Raises `OutOfBoundsError` if the given position - is outside the range of the collection. + Finds the element at the given `index` (zero-based). + + Raises `OutOfBoundsError` if the given `index` is outside the range of + the `enumerable`. ## Examples @@ -623,195 +1062,260 @@ defmodule Enum do 6 iex> Enum.fetch!([2, 4, 6], 4) - ** (Enum.OutOfBoundsError) out of bounds error + ** (Enum.OutOfBoundsError) out of bounds error at position 4 when traversing enumerable [2, 4, 6] """ - @spec fetch!(t, integer) :: element | no_return - def fetch!(collection, n) do - case fetch(collection, n) do - {:ok, h} -> h - :error -> raise Enum.OutOfBoundsError + @spec fetch!(t, index) :: element + def fetch!(enumerable, index) when is_integer(index) do + case slice_forward(enumerable, index, 1, 1) do + [value] -> value + [] -> raise Enum.OutOfBoundsError, index: index, enumerable: enumerable end end @doc """ - Filters the collection, i.e. returns only those elements - for which `fun` returns `true`. + Filters the `enumerable`, i.e. returns only those elements + for which `fun` returns a truthy value. + + See also `reject/2` which discards all elements where the + function returns a truthy value. ## Examples - iex> Enum.filter([1, 2, 3], fn(x) -> rem(x, 2) == 0 end) + iex> Enum.filter([1, 2, 3], fn x -> rem(x, 2) == 0 end) [2] + iex> Enum.filter(["apple", "pear", "banana"], fn fruit -> String.contains?(fruit, "a") end) + ["apple", "pear", "banana"] + iex> Enum.filter([4, 21, 24, 904], fn seconds -> seconds > 1000 end) + [] + + Keep in mind that `filter` is not capable of filtering and + transforming an element at the same time. If you would like + to do so, consider using `flat_map/2`. For example, if you + want to convert all strings that represent an integer and + discard the invalid one in one pass: + + strings = ["1234", "abc", "12ab"] + + Enum.flat_map(strings, fn string -> + case Integer.parse(string) do + # transform to integer + {int, _rest} -> [int] + # skip the value + :error -> [] + end + end) """ @spec filter(t, (element -> as_boolean(term))) :: list - def filter(collection, fun) when is_list(collection) do - for item <- collection, fun.(item), do: item + def filter(enumerable, fun) when is_list(enumerable) do + filter_list(enumerable, fun) end - def filter(collection, fun) do - Enumerable.reduce(collection, {:cont, []}, R.filter(fun)) - |> elem(1) |> :lists.reverse + def filter(enumerable, fun) do + reduce(enumerable, [], R.filter(fun)) |> :lists.reverse() end - @doc """ - Filters the collection and maps its values in one pass. - - ## Examples - - iex> Enum.filter_map([1, 2, 3], fn(x) -> rem(x, 2) == 0 end, &(&1 * 2)) - [4] - - """ - @spec filter_map(t, (element -> as_boolean(term)), (element -> element)) :: list - def filter_map(collection, filter, mapper) when is_list(collection) do - for item <- collection, filter.(item), do: mapper.(item) + @doc false + @deprecated "Use Enum.filter/2 + Enum.map/2 or for comprehensions instead" + def filter_map(enumerable, filter, mapper) when is_list(enumerable) do + for element <- enumerable, filter.(element), do: mapper.(element) end - def filter_map(collection, filter, mapper) do - Enumerable.reduce(collection, {:cont, []}, R.filter_map(filter, mapper)) - |> elem(1) |> :lists.reverse + def filter_map(enumerable, filter, mapper) do + enumerable + |> reduce([], R.filter_map(filter, mapper)) + |> :lists.reverse() end @doc """ - Returns the first item for which `fun` returns a truthy value. If no such - item is found, returns `ifnone`. + Returns the first element for which `fun` returns a truthy value. + If no such element is found, returns `default`. ## Examples - iex> Enum.find([2, 4, 6], fn(x) -> rem(x, 2) == 1 end) - nil + iex> Enum.find([2, 3, 4], fn x -> rem(x, 2) == 1 end) + 3 - iex> Enum.find([2, 4, 6], 0, fn(x) -> rem(x, 2) == 1 end) + iex> Enum.find([2, 4, 6], fn x -> rem(x, 2) == 1 end) + nil + iex> Enum.find([2, 4, 6], 0, fn x -> rem(x, 2) == 1 end) 0 - iex> Enum.find([2, 3, 4], fn(x) -> rem(x, 2) == 1 end) - 3 - """ - @spec find(t, (element -> any)) :: element | nil @spec find(t, default, (element -> any)) :: element | default - def find(collection, ifnone \\ nil, fun) + def find(enumerable, default \\ nil, fun) - def find(collection, ifnone, fun) when is_list(collection) do - do_find(collection, ifnone, fun) + def find(enumerable, default, fun) when is_list(enumerable) do + find_list(enumerable, default, fun) end - def find(collection, ifnone, fun) do - Enumerable.reduce(collection, {:cont, ifnone}, fn(entry, ifnone) -> - if fun.(entry), do: {:halt, entry}, else: {:cont, ifnone} - end) |> elem(1) + def find(enumerable, default, fun) do + Enumerable.reduce(enumerable, {:cont, default}, fn entry, default -> + if fun.(entry), do: {:halt, entry}, else: {:cont, default} + end) + |> elem(1) end @doc """ - Similar to `find/3`, but returns the value of the function - invocation instead of the element itself. + Similar to `find/3`, but returns the index (zero-based) + of the element instead of the element itself. ## Examples - iex> Enum.find_value([2, 4, 6], fn(x) -> rem(x, 2) == 1 end) + iex> Enum.find_index([2, 4, 6], fn x -> rem(x, 2) == 1 end) nil - iex> Enum.find_value([2, 3, 4], fn(x) -> rem(x, 2) == 1 end) - true + iex> Enum.find_index([2, 3, 4], fn x -> rem(x, 2) == 1 end) + 1 """ - @spec find_value(t, (element -> any)) :: any | :nil - @spec find_value(t, any, (element -> any)) :: any | :nil - def find_value(collection, ifnone \\ nil, fun) - - def find_value(collection, ifnone, fun) when is_list(collection) do - do_find_value(collection, ifnone, fun) + @spec find_index(t, (element -> any)) :: non_neg_integer | nil + def find_index(enumerable, fun) when is_list(enumerable) do + find_index_list(enumerable, 0, fun) end - def find_value(collection, ifnone, fun) do - Enumerable.reduce(collection, {:cont, ifnone}, fn(entry, ifnone) -> - fun_entry = fun.(entry) - if fun_entry, do: {:halt, fun_entry}, else: {:cont, ifnone} - end) |> elem(1) + def find_index(enumerable, fun) do + result = + Enumerable.reduce(enumerable, {:cont, {:not_found, 0}}, fn entry, {_, index} -> + if fun.(entry), do: {:halt, {:found, index}}, else: {:cont, {:not_found, index + 1}} + end) + + case elem(result, 1) do + {:found, index} -> index + {:not_found, _} -> nil + end end @doc """ - Similar to `find/3`, but returns the index (zero-based) - of the element instead of the element itself. + Similar to `find/3`, but returns the value of the function + invocation instead of the element itself. + + The return value is considered to be found when the result is truthy + (neither `nil` nor `false`). ## Examples - iex> Enum.find_index([2, 4, 6], fn(x) -> rem(x, 2) == 1 end) + iex> Enum.find_value([2, 3, 4], fn x -> + ...> if x > 2, do: x * x + ...> end) + 9 + + iex> Enum.find_value([2, 4, 6], fn x -> rem(x, 2) == 1 end) nil - iex> Enum.find_index([2, 3, 4], fn(x) -> rem(x, 2) == 1 end) - 1 + iex> Enum.find_value([2, 3, 4], fn x -> rem(x, 2) == 1 end) + true + + iex> Enum.find_value([1, 2, 3], "no bools!", &is_boolean/1) + "no bools!" """ - @spec find_index(t, (element -> any)) :: index | :nil - def find_index(collection, fun) when is_list(collection) do - do_find_index(collection, 0, fun) - end + @spec find_value(t, default, (element -> found_value)) :: found_value | default + when found_value: term + def find_value(enumerable, default \\ nil, fun) - def find_index(collection, fun) do - res = - Enumerable.reduce(collection, {:cont, 0}, fn(entry, acc) -> - if fun.(entry), do: {:halt, acc}, else: {:cont, acc + 1} - end) + def find_value(enumerable, default, fun) when is_list(enumerable) do + find_value_list(enumerable, default, fun) + end - case res do - {:halted, entry} -> entry - {:done, _} -> nil - end + def find_value(enumerable, default, fun) do + Enumerable.reduce(enumerable, {:cont, default}, fn entry, default -> + fun_entry = fun.(entry) + if fun_entry, do: {:halt, fun_entry}, else: {:cont, default} + end) + |> elem(1) end @doc """ - Returns a new collection appending the result of invoking `fun` - on each corresponding item of `collection`. + Maps the given `fun` over `enumerable` and flattens the result only one level deep. - The given function should return an enumerable. + This function returns a new enumerable built by appending the result of invoking `fun` + on each element of `enumerable` together; conceptually, this is similar to a + combination of `map/2` and `concat/1`. ## Examples - iex> Enum.flat_map([:a, :b, :c], fn(x) -> [x, x] end) + iex> Enum.flat_map([:a, :b, :c], fn x -> [x, x] end) [:a, :a, :b, :b, :c, :c] - iex> Enum.flat_map([{1,3}, {4,6}], fn({x,y}) -> x..y end) + iex> Enum.flat_map([{1, 3}, {4, 6}], fn {x, y} -> x..y end) [1, 2, 3, 4, 5, 6] + iex> Enum.flat_map([:a, :b, :c], fn x -> [[x]] end) + [[:a], [:b], [:c]] + + This is frequently used to to transform and filter in one pass, returning empty + lists to exclude results: + + iex> Enum.flat_map([4, 0, 2, 0], fn x -> + ...> if x != 0, do: [1 / x], else: [] + ...> end) + [0.25, 0.5] + """ @spec flat_map(t, (element -> t)) :: list - def flat_map(collection, fun) do - reduce(collection, [], fn(entry, acc) -> - reduce(fun.(entry), acc, &[&1|&2]) - end) |> :lists.reverse + def flat_map(enumerable, fun) when is_list(enumerable) do + flat_map_list(enumerable, fun) + end + + def flat_map(enumerable, fun) do + reduce(enumerable, [], fn entry, acc -> + case fun.(entry) do + [] -> acc + list when is_list(list) -> [list | acc] + other -> [to_list(other) | acc] + end + end) + |> flat_reverse([]) end + # the first clause is an optimization + defp flat_reverse([[elem] | t], acc), do: flat_reverse(t, [elem | acc]) + defp flat_reverse([h | t], acc), do: flat_reverse(t, h ++ acc) + defp flat_reverse([], acc), do: acc + @doc """ - Maps and reduces a collection, flattening the given results. + Maps and reduces an `enumerable`, flattening the results only one level deep. - It expects an accumulator and a function that receives each stream item - and an accumulator, and must return a tuple containing a new stream - (often a list) with the new accumulator or a tuple with `:halt` as first - element and the accumulator as second. + It expects an accumulator and a function that receives each enumerable + element, and must return a tuple containing a new enumerable (often a list) + with the new accumulator or a tuple with `:halt` as first element and + the accumulator as second. + + Returns a 2-element tuple where the first element is the results flattened one level deep and + the second element is the last accumulator. ## Examples - iex> enum = 1..100 + iex> enumerable = 1..100 iex> n = 3 - iex> Enum.flat_map_reduce(enum, 0, fn i, acc -> - ...> if acc < n, do: {[i], acc + 1}, else: {:halt, acc} + iex> Enum.flat_map_reduce(enumerable, 0, fn x, acc -> + ...> if acc < n, do: {[x], acc + 1}, else: {:halt, acc} ...> end) - {[1,2,3], 3} + {[1, 2, 3], 3} + + iex> Enum.flat_map_reduce(1..5, 0, fn x, acc -> {[[x]], acc + x} end) + {[[1], [2], [3], [4], [5]], 15} """ - @spec flat_map_reduce(t, acc, fun) :: {[any], any} when - fun: (element, acc -> {t, acc} | {:halt, acc}), - acc: any - def flat_map_reduce(collection, acc, fun) do + @spec flat_map_reduce(t, acc, fun) :: {[any], acc} + when fun: (element, acc -> {t, acc} | {:halt, acc}) + def flat_map_reduce(enumerable, acc, fun) do {_, {list, acc}} = - Enumerable.reduce(collection, {:cont, {[], acc}}, fn(entry, {list, acc}) -> + Enumerable.reduce(enumerable, {:cont, {[], acc}}, fn entry, {list, acc} -> case fun.(entry, acc) do {:halt, acc} -> {:halt, {list, acc}} + + {[], acc} -> + {:cont, {list, acc}} + + {[entry], acc} -> + {:cont, {[entry | list], acc}} + {entries, acc} -> - {:cont, {reduce(entries, list, &[&1|&2]), acc}} + {:cont, {reduce(entries, list, &[&1 | &2]), acc}} end end) @@ -819,9 +1323,126 @@ defmodule Enum do end @doc """ - Intersperses `element` between each element of the enumeration. + Returns a map with keys as unique elements of `enumerable` and values + as the count of every element. + + ## Examples + + iex> Enum.frequencies(~w{ant buffalo ant ant buffalo dingo}) + %{"ant" => 3, "buffalo" => 2, "dingo" => 1} + + """ + @doc since: "1.10.0" + @spec frequencies(t) :: map + def frequencies(enumerable) do + reduce(enumerable, %{}, fn key, acc -> + case acc do + %{^key => value} -> %{acc | key => value + 1} + %{} -> Map.put(acc, key, 1) + end + end) + end + + @doc """ + Returns a map with keys as unique elements given by `key_fun` and values + as the count of every element. + + ## Examples + + iex> Enum.frequencies_by(~w{aa aA bb cc}, &String.downcase/1) + %{"aa" => 2, "bb" => 1, "cc" => 1} + + iex> Enum.frequencies_by(~w{aaa aA bbb cc c}, &String.length/1) + %{3 => 2, 2 => 2, 1 => 1} + + """ + @doc since: "1.10.0" + @spec frequencies_by(t, (element -> any)) :: map + def frequencies_by(enumerable, key_fun) when is_function(key_fun) do + reduce(enumerable, %{}, fn entry, acc -> + key = key_fun.(entry) + + case acc do + %{^key => value} -> %{acc | key => value + 1} + %{} -> Map.put(acc, key, 1) + end + end) + end + + @doc """ + Splits the `enumerable` into groups based on `key_fun`. + + The result is a map where each key is given by `key_fun` + and each value is a list of elements given by `value_fun`. + The order of elements within each list is preserved from the `enumerable`. + However, like all maps, the resulting map is unordered. + + ## Examples + + iex> Enum.group_by(~w{ant buffalo cat dingo}, &String.length/1) + %{3 => ["ant", "cat"], 5 => ["dingo"], 7 => ["buffalo"]} + + iex> Enum.group_by(~w{ant buffalo cat dingo}, &String.length/1, &String.first/1) + %{3 => ["a", "c"], 5 => ["d"], 7 => ["b"]} + + The key can be any Elixir value. For example, you may use a tuple + to group by multiple keys: + + iex> collection = [ + ...> %{id: 1, lang: "Elixir", seq: 1}, + ...> %{id: 1, lang: "Java", seq: 1}, + ...> %{id: 1, lang: "Ruby", seq: 2}, + ...> %{id: 2, lang: "Python", seq: 1}, + ...> %{id: 2, lang: "C#", seq: 2}, + ...> %{id: 2, lang: "Haskell", seq: 2}, + ...> ] + iex> Enum.group_by(collection, &{&1.id, &1.seq}) + %{ + {1, 1} => [%{id: 1, lang: "Elixir", seq: 1}, %{id: 1, lang: "Java", seq: 1}], + {1, 2} => [%{id: 1, lang: "Ruby", seq: 2}], + {2, 1} => [%{id: 2, lang: "Python", seq: 1}], + {2, 2} => [%{id: 2, lang: "C#", seq: 2}, %{id: 2, lang: "Haskell", seq: 2}] + } + iex> Enum.group_by(collection, &{&1.id, &1.seq}, &{&1.id, &1.lang}) + %{ + {1, 1} => [{1, "Elixir"}, {1, "Java"}], + {1, 2} => [{1, "Ruby"}], + {2, 1} => [{2, "Python"}], + {2, 2} => [{2, "C#"}, {2, "Haskell"}] + } + + """ + @spec group_by(t, (element -> any), (element -> any)) :: map + def group_by(enumerable, key_fun, value_fun \\ fn x -> x end) + + def group_by(enumerable, key_fun, value_fun) when is_function(key_fun) do + reduce(reverse(enumerable), %{}, fn entry, acc -> + key = key_fun.(entry) + value = value_fun.(entry) + + case acc do + %{^key => existing} -> %{acc | key => [value | existing]} + %{} -> Map.put(acc, key, [value]) + end + end) + end + + def group_by(enumerable, dict, fun) do + IO.warn( + "Enum.group_by/3 with a map/dictionary as second element is deprecated. " <> + "A map is used by default and it is no longer required to pass one to this function" + ) + + # Avoid warnings about Dict + dict_module = String.to_atom("Dict") - Complexity: O(n) + reduce(reverse(enumerable), dict, fn entry, categories -> + dict_module.update(categories, fun.(entry), [entry], &[entry | &1]) + end) + end + + @doc """ + Intersperses `separator` between each element of the enumeration. ## Examples @@ -836,91 +1457,204 @@ defmodule Enum do """ @spec intersperse(t, element) :: list - def intersperse(collection, element) do + def intersperse(enumerable, separator) when is_list(enumerable) do + case enumerable do + [] -> [] + list -> intersperse_non_empty_list(list, separator) + end + end + + def intersperse(enumerable, separator) do list = - reduce(collection, [], fn(x, acc) -> - [x, element | acc] - end) |> :lists.reverse() + enumerable + |> reduce([], fn x, acc -> [x, separator | acc] end) + |> :lists.reverse() + # Head is a superfluous separator case list do - [] -> [] - [_|t] -> t # Head is a superfluous intersperser element + [] -> [] + [_ | t] -> t end end @doc """ - Inserts the given enumerable into a collectable. + Inserts the given `enumerable` into a `collectable`. + + Note that passing a non-empty list as the `collectable` is deprecated. + If you're collecting into a non-empty keyword list, consider using + `Keyword.merge(collectable, Enum.to_list(enumerable))`. If you're collecting + into a non-empty list, consider something like `Enum.to_list(enumerable) ++ collectable`. ## Examples - iex> Enum.into([1, 2], [0]) - [0, 1, 2] + iex> Enum.into([1, 2], []) + [1, 2] iex> Enum.into([a: 1, b: 2], %{}) %{a: 1, b: 2} + iex> Enum.into(%{a: 1}, %{b: 2}) + %{a: 1, b: 2} + + iex> Enum.into([a: 1, a: 2], %{}) + %{a: 2} + + iex> Enum.into([a: 2], %{a: 1, b: 3}) + %{a: 2, b: 3} + """ - @spec into(Enumerable.t, Collectable.t) :: Collectable.t - def into(collection, list) when is_list(list) do - list ++ to_list(collection) + @spec into(Enumerable.t(), Collectable.t()) :: Collectable.t() + def into(enumerable, collectable) + + def into(enumerable, []) do + to_list(enumerable) + end + + def into(enumerable, collectable) when is_struct(collectable, MapSet) do + if MapSet.size(collectable) == 0 do + MapSet.new(enumerable) + else + MapSet.new(enumerable) |> MapSet.union(collectable) + end + end + + def into(%_{} = enumerable, collectable) do + into_protocol(enumerable, collectable) + end + + def into(enumerable, %_{} = collectable) do + into_protocol(enumerable, collectable) + end + + def into(enumerable, %{} = collectable) do + if map_size(collectable) == 0 do + into_map(enumerable) + else + into_map(enumerable, collectable) + end end - def into(collection, %{} = map) when is_list(collection) and map_size(map) == 0 do - :maps.from_list(collection) + def into(enumerable, collectable) do + into_protocol(enumerable, collectable) end - def into(collection, collectable) do + defp into_map(%{} = enumerable), do: enumerable + defp into_map(enumerable) when is_list(enumerable), do: :maps.from_list(enumerable) + defp into_map(enumerable), do: enumerable |> Enum.to_list() |> :maps.from_list() + + defp into_map(%{} = enumerable, collectable), do: Map.merge(collectable, enumerable) + + defp into_map(enumerable, collectable) when is_list(enumerable), + do: Map.merge(collectable, :maps.from_list(enumerable)) + + defp into_map(enumerable, collectable), + do: reduce(enumerable, collectable, fn {key, val}, acc -> Map.put(acc, key, val) end) + + defp into_protocol(enumerable, collectable) do {initial, fun} = Collectable.into(collectable) - into(collection, initial, fun, fn x, acc -> - fun.(acc, {:cont, x}) + + try do + reduce_into_protocol(enumerable, initial, fun) + catch + kind, reason -> + fun.(initial, :halt) + :erlang.raise(kind, reason, __STACKTRACE__) + else + acc -> fun.(acc, :done) + end + end + + defp reduce_into_protocol(enumerable, initial, fun) when is_list(enumerable) do + :lists.foldl(fn x, acc -> fun.(acc, {:cont, x}) end, initial, enumerable) + end + + defp reduce_into_protocol(enumerable, initial, fun) do + enumerable + |> Enumerable.reduce({:cont, initial}, fn x, acc -> + {:cont, fun.(acc, {:cont, x})} end) + |> elem(1) end @doc """ - Inserts the given enumerable into a collectable - according to the transformation function. + Inserts the given `enumerable` into a `collectable` according to the + transformation function. ## Examples - iex> Enum.into([2, 3], [3], fn x -> x * 3 end) + iex> Enum.into([1, 2, 3], [], fn x -> x * 3 end) [3, 6, 9] + iex> Enum.into(%{a: 1, b: 2}, %{c: 3}, fn {k, v} -> {k, v * 2} end) + %{a: 2, b: 4, c: 3} + """ - @spec into(Enumerable.t, Collectable.t, (term -> term)) :: Collectable.t + @spec into(Enumerable.t(), Collectable.t(), (term -> term)) :: Collectable.t() + def into(enumerable, [], transform) do + map(enumerable, transform) + end - def into(collection, list, transform) when is_list(list) and is_function(transform, 1) do - list ++ map(collection, transform) + def into(enumerable, collectable, transform) when is_struct(collectable, MapSet) do + if MapSet.size(collectable) == 0 do + MapSet.new(enumerable, transform) + else + MapSet.new(enumerable, transform) |> MapSet.union(collectable) + end end - def into(collection, collectable, transform) when is_function(transform, 1) do - {initial, fun} = Collectable.into(collectable) - into(collection, initial, fun, fn x, acc -> - fun.(acc, {:cont, transform.(x)}) - end) + def into(enumerable, %_{} = collectable, transform) do + into_protocol(enumerable, collectable, transform) end - defp into(collection, initial, fun, callback) do + def into(enumerable, %{} = collectable, transform) do + if map_size(collectable) == 0 do + enumerable |> map(transform) |> :maps.from_list() + else + reduce(enumerable, collectable, fn entry, acc -> + {key, val} = transform.(entry) + Map.put(acc, key, val) + end) + end + end + + def into(enumerable, collectable, transform) do + into_protocol(enumerable, collectable, transform) + end + + defp into_protocol(enumerable, collectable, transform) do + {initial, fun} = Collectable.into(collectable) + try do - reduce(collection, initial, callback) + reduce_into_protocol(enumerable, initial, transform, fun) catch kind, reason -> - stacktrace = System.stacktrace fun.(initial, :halt) - :erlang.raise(kind, reason, stacktrace) + :erlang.raise(kind, reason, __STACKTRACE__) else acc -> fun.(acc, :done) end end + defp reduce_into_protocol(enumerable, initial, transform, fun) when is_list(enumerable) do + :lists.foldl(fn x, acc -> fun.(acc, {:cont, transform.(x)}) end, initial, enumerable) + end + + defp reduce_into_protocol(enumerable, initial, transform, fun) do + enumerable + |> Enumerable.reduce({:cont, initial}, fn x, acc -> + {:cont, fun.(acc, {:cont, transform.(x)})} + end) + |> elem(1) + end + @doc """ - Joins the given `collection` according to `joiner`. - `joiner` can be either a binary or a list and the - result will be of the same type as `joiner`. If - `joiner` is not passed at all, it defaults to an - empty binary. + Joins the given `enumerable` into a string using `joiner` as a + separator. - All items in the collection must be convertible - to a binary, otherwise an error is raised. + If `joiner` is not passed at all, it defaults to an empty string. + + All elements in the `enumerable` must be convertible to a string + or be a binary, otherwise an error is raised. ## Examples @@ -930,347 +1664,887 @@ defmodule Enum do iex> Enum.join([1, 2, 3], " = ") "1 = 2 = 3" + iex> Enum.join([["a", "b"], ["c", "d", "e", ["f", "g"]], "h", "i"], " ") + "ab cdefg h i" + """ - @spec join(t) :: String.t - @spec join(t, String.t) :: String.t - def join(collection, joiner \\ "") + @spec join(t, binary()) :: binary() + def join(enumerable, joiner \\ "") + + def join(enumerable, "") do + enumerable + |> map(&entry_to_string(&1)) + |> IO.iodata_to_binary() + end + + def join(enumerable, joiner) when is_list(enumerable) and is_binary(joiner) do + join_list(enumerable, joiner) + end + + def join(enumerable, joiner) when is_binary(joiner) do + reduced = + reduce(enumerable, :first, fn + entry, :first -> entry_to_string(entry) + entry, acc -> [acc, joiner | entry_to_string(entry)] + end) - def join(collection, joiner) when is_binary(joiner) do - reduced = reduce(collection, :first, fn - entry, :first -> enum_to_string(entry) - entry, acc -> [acc, joiner|enum_to_string(entry)] - end) if reduced == :first do "" else - IO.iodata_to_binary reduced + IO.iodata_to_binary(reduced) end end @doc """ - Returns a new collection, where each item is the result - of invoking `fun` on each corresponding item of `collection`. + Returns a list where each element is the result of invoking + `fun` on each corresponding element of `enumerable`. - For dicts, the function expects a key-value tuple. + For maps, the function expects a key-value tuple. ## Examples - iex> Enum.map([1, 2, 3], fn(x) -> x * 2 end) + iex> Enum.map([1, 2, 3], fn x -> x * 2 end) [2, 4, 6] - iex> Enum.map([a: 1, b: 2], fn({k, v}) -> {k, -v} end) + iex> Enum.map([a: 1, b: 2], fn {k, v} -> {k, -v} end) [a: -1, b: -2] """ @spec map(t, (element -> any)) :: list - def map(collection, fun) when is_list(collection) do - for item <- collection, do: fun.(item) + def map(enumerable, fun) + + def map(enumerable, fun) when is_list(enumerable) do + :lists.map(fun, enumerable) end - def map(collection, fun) do - Enumerable.reduce(collection, {:cont, []}, R.map(fun)) |> elem(1) |> :lists.reverse + def map(first..last//step, fun) do + map_range(first, last, step, fun) + end + + def map(enumerable, fun) do + reduce(enumerable, [], R.map(fun)) |> :lists.reverse() end @doc """ - Maps and joins the given `collection` in one pass. - `joiner` can be either a binary or a list and the - result will be of the same type as `joiner`. If - `joiner` is not passed at all, it defaults to an - empty binary. + Returns a list of results of invoking `fun` on every `nth` + element of `enumerable`, starting with the first element. + + The first element is always passed to the given function, unless `nth` is `0`. + + The second argument specifying every `nth` element must be a non-negative + integer. - All items in the collection must be convertible - to a binary, otherwise an error is raised. + If `nth` is `0`, then `enumerable` is directly converted to a list, + without `fun` being ever applied. ## Examples - iex> Enum.map_join([1, 2, 3], &(&1 * 2)) - "246" + iex> Enum.map_every(1..10, 2, fn x -> x + 1000 end) + [1001, 2, 1003, 4, 1005, 6, 1007, 8, 1009, 10] - iex> Enum.map_join([1, 2, 3], " = ", &(&1 * 2)) - "2 = 4 = 6" + iex> Enum.map_every(1..10, 3, fn x -> x + 1000 end) + [1001, 2, 3, 1004, 5, 6, 1007, 8, 9, 1010] + + iex> Enum.map_every(1..5, 0, fn x -> x + 1000 end) + [1, 2, 3, 4, 5] + + iex> Enum.map_every([1, 2, 3], 1, fn x -> x + 1000 end) + [1001, 1002, 1003] """ - @spec map_join(t, (element -> any)) :: String.t - @spec map_join(t, String.t, (element -> any)) :: String.t - def map_join(collection, joiner \\ "", mapper) + @doc since: "1.4.0" + @spec map_every(t, non_neg_integer, (element -> any)) :: list + def map_every(enumerable, nth, fun) - def map_join(collection, joiner, mapper) when is_binary(joiner) do - reduced = reduce(collection, :first, fn - entry, :first -> enum_to_string(mapper.(entry)) - entry, acc -> [acc, joiner|enum_to_string(mapper.(entry))] - end) + def map_every(enumerable, 1, fun), do: map(enumerable, fun) + def map_every(enumerable, 0, _fun), do: to_list(enumerable) + def map_every([], nth, _fun) when is_integer(nth) and nth > 1, do: [] + + def map_every(enumerable, nth, fun) when is_integer(nth) and nth > 1 do + {res, _} = reduce(enumerable, {[], :first}, R.map_every(nth, fun)) + :lists.reverse(res) + end + + @doc """ + Maps and intersperses the given enumerable in one pass. + + ## Examples + + iex> Enum.map_intersperse([1, 2, 3], :a, &(&1 * 2)) + [2, :a, 4, :a, 6] + """ + @doc since: "1.10.0" + @spec map_intersperse(t, element(), (element -> any())) :: list() + def map_intersperse(enumerable, separator, mapper) + + def map_intersperse(enumerable, separator, mapper) when is_list(enumerable) do + map_intersperse_list(enumerable, separator, mapper) + end + + def map_intersperse(enumerable, separator, mapper) do + reduced = + reduce(enumerable, :first, fn + entry, :first -> [mapper.(entry)] + entry, acc -> [mapper.(entry), separator | acc] + end) if reduced == :first do - "" + [] else - IO.iodata_to_binary reduced + :lists.reverse(reduced) end end @doc """ - Invokes the given `fun` for each item in the `collection` - while also keeping an accumulator. Returns a tuple where - the first element is the mapped collection and the second - one is the final accumulator. + Maps and joins the given `enumerable` in one pass. + + If `joiner` is not passed at all, it defaults to an empty string. + + All elements returned from invoking the `mapper` must be convertible to + a string, otherwise an error is raised. + + ## Examples + + iex> Enum.map_join([1, 2, 3], &(&1 * 2)) + "246" + + iex> Enum.map_join([1, 2, 3], " = ", &(&1 * 2)) + "2 = 4 = 6" + + """ + @spec map_join(t, String.t(), (element -> String.Chars.t())) :: String.t() + def map_join(enumerable, joiner \\ "", mapper) when is_binary(joiner) do + enumerable + |> map_intersperse(joiner, &entry_to_string(mapper.(&1))) + |> IO.iodata_to_binary() + end + + @doc """ + Invokes the given function to each element in the `enumerable` to reduce + it to a single element, while keeping an accumulator. + + Returns a tuple where the first element is the mapped enumerable and + the second one is the final accumulator. - For dicts, the first tuple element must be a `{key, value}` - tuple. + The function, `fun`, receives two arguments: the first one is the + element, and the second one is the accumulator. `fun` must return + a tuple with two elements in the form of `{result, accumulator}`. + + For maps, the first tuple element must be a `{key, value}` tuple. ## Examples - iex> Enum.map_reduce([1, 2, 3], 0, fn(x, acc) -> {x * 2, x + acc} end) + iex> Enum.map_reduce([1, 2, 3], 0, fn x, acc -> {x * 2, x + acc} end) {[2, 4, 6], 6} """ - @spec map_reduce(t, any, (element, any -> any)) :: any - def map_reduce(collection, acc, fun) when is_list(collection) do - :lists.mapfoldl(fun, acc, collection) + @spec map_reduce(t, acc, (element, acc -> {element, acc})) :: {list, acc} + def map_reduce(enumerable, acc, fun) when is_list(enumerable) do + :lists.mapfoldl(fun, acc, enumerable) end - def map_reduce(collection, acc, fun) do - {list, acc} = reduce(collection, {[], acc}, fn(entry, {list, acc}) -> - {new_entry, acc} = fun.(entry, acc) - {[new_entry|list], acc} - end) + def map_reduce(enumerable, acc, fun) do + {list, acc} = + reduce(enumerable, {[], acc}, fn entry, {list, acc} -> + {new_entry, acc} = fun.(entry, acc) + {[new_entry | list], acc} + end) + {:lists.reverse(list), acc} end + @doc false + def max(list = [_ | _]), do: :lists.max(list) + + @doc false + def max(list = [_ | _], empty_fallback) when is_function(empty_fallback, 0) do + :lists.max(list) + end + + @doc false + @spec max(t, (-> empty_result)) :: element | empty_result when empty_result: any + def max(enumerable, empty_fallback) when is_function(empty_fallback, 0) do + max(enumerable, &>=/2, empty_fallback) + end + @doc """ - Returns the maximum value. - Raises `EmptyError` if the collection is empty. + Returns the maximal element in the `enumerable` according + to Erlang's term ordering. + + By default, the comparison is done with the [`>=`](`>=/2`) sorter function. + If multiple elements are considered maximal, the first one that + was found is returned. If you want the last element considered + maximal to be returned, the sorter function should not return true + for equal elements. + + If the enumerable is empty, the provided `empty_fallback` is called. + The default `empty_fallback` raises `Enum.EmptyError`. ## Examples iex> Enum.max([1, 2, 3]) 3 + The fact this function uses Erlang's term ordering means that the comparison + is structural and not semantic. For example: + + iex> Enum.max([~D[2017-03-31], ~D[2017-04-01]]) + ~D[2017-03-31] + + In the example above, `max/2` returned March 31st instead of April 1st + because the structural comparison compares the day before the year. + For this reason, most structs provide a "compare" function, such as + `Date.compare/2`, which receives two structs and returns `:lt` (less-than), + `:eq` (equal to), and `:gt` (greater-than). If you pass a module as the + sorting function, Elixir will automatically use the `compare/2` function + of said module: + + iex> Enum.max([~D[2017-03-31], ~D[2017-04-01]], Date) + ~D[2017-04-01] + + Finally, if you don't want to raise on empty enumerables, you can pass + the empty fallback: + + iex> Enum.max([], &>=/2, fn -> 0 end) + 0 + """ - @spec max(t) :: element | no_return - def max(collection) do - reduce(collection, &Kernel.max(&1, &2)) + @spec max(t, (element, element -> boolean) | module()) :: + element | empty_result + when empty_result: any + @spec max(t, (element, element -> boolean) | module(), (-> empty_result)) :: + element | empty_result + when empty_result: any + def max(enumerable, sorter \\ &>=/2, empty_fallback \\ fn -> raise Enum.EmptyError end) do + aggregate(enumerable, max_sort_fun(sorter), empty_fallback) + end + + defp max_sort_fun(sorter) when is_function(sorter, 2), do: sorter + defp max_sort_fun(module) when is_atom(module), do: &(module.compare(&1, &2) != :lt) + + @doc false + @spec max_by( + t, + (element -> any), + (-> empty_result) | (element, element -> boolean) | module() + ) :: element | empty_result + when empty_result: any + def max_by(enumerable, fun, empty_fallback) + when is_function(fun, 1) and is_function(empty_fallback, 0) do + max_by(enumerable, fun, &>=/2, empty_fallback) end @doc """ - Returns the maximum value as calculated by the given function. - Raises `EmptyError` if the collection is empty. + Returns the maximal element in the `enumerable` as calculated + by the given `fun`. + + By default, the comparison is done with the [`>=`](`>=/2`) sorter function. + If multiple elements are considered maximal, the first one that + was found is returned. If you want the last element considered + maximal to be returned, the sorter function should not return true + for equal elements. + + Calls the provided `empty_fallback` function and returns its value if + `enumerable` is empty. The default `empty_fallback` raises `Enum.EmptyError`. ## Examples - iex> Enum.max_by(["a", "aa", "aaa"], fn(x) -> String.length(x) end) + iex> Enum.max_by(["a", "aa", "aaa"], fn x -> String.length(x) end) "aaa" - """ - @spec max_by(t, (element -> any)) :: element | no_return - def max_by([h|t], fun) do - reduce(t, {h, fun.(h)}, fn(entry, {_, fun_max} = old) -> - fun_entry = fun.(entry) - if(fun_entry > fun_max, do: {entry, fun_entry}, else: old) - end) |> elem(0) - end - - def max_by([], _fun) do - raise Enum.EmptyError - end + iex> Enum.max_by(["a", "aa", "aaa", "b", "bbb"], &String.length/1) + "aaa" - def max_by(collection, fun) do - result = - reduce(collection, :first, fn - entry, {_, fun_max} = old -> - fun_entry = fun.(entry) - if(fun_entry > fun_max, do: {entry, fun_entry}, else: old) - entry, :first -> - {entry, fun.(entry)} - end) + The fact this function uses Erlang's term ordering means that the + comparison is structural and not semantic. Therefore, if you want + to compare structs, most structs provide a "compare" function, such as + `Date.compare/2`, which receives two structs and returns `:lt` (less-than), + `:eq` (equal to), and `:gt` (greater-than). If you pass a module as the + sorting function, Elixir will automatically use the `compare/2` function + of said module: + + iex> users = [ + ...> %{name: "Ellis", birthday: ~D[1943-05-11]}, + ...> %{name: "Lovelace", birthday: ~D[1815-12-10]}, + ...> %{name: "Turing", birthday: ~D[1912-06-23]} + ...> ] + iex> Enum.max_by(users, &(&1.birthday), Date) + %{name: "Ellis", birthday: ~D[1943-05-11]} + + Finally, if you don't want to raise on empty enumerables, you can pass + the empty fallback: + + iex> Enum.max_by([], &String.length/1, fn -> nil end) + nil - case result do - :first -> raise Enum.EmptyError - {entry, _} -> entry - end + """ + @spec max_by( + t, + (element -> any), + (element, element -> boolean) | module(), + (-> empty_result) + ) :: element | empty_result + when empty_result: any + def max_by(enumerable, fun, sorter \\ &>=/2, empty_fallback \\ fn -> raise Enum.EmptyError end) + when is_function(fun, 1) do + aggregate_by(enumerable, fun, max_sort_fun(sorter), empty_fallback) end @doc """ - Checks if `value` exists within the `collection`. + Checks if `element` exists within the `enumerable`. - Membership is tested with the match (`===`) operator, although - enumerables like ranges may include floats inside the given - range. + Membership is tested with the match (`===/2`) operator. ## Examples iex> Enum.member?(1..10, 5) true + iex> Enum.member?(1..10, 5.0) + false + + iex> Enum.member?([1.0, 2.0, 3.0], 2) + false + iex> Enum.member?([1.0, 2.0, 3.0], 2.000) + true iex> Enum.member?([:a, :b, :c], :d) false + + When called outside guards, the [`in`](`in/2`) and [`not in`](`in/2`) + operators work by using this function. """ @spec member?(t, element) :: boolean - def member?(collection, value) when is_list(collection) do - :lists.member(value, collection) + def member?(enumerable, element) when is_list(enumerable) do + :lists.member(element, enumerable) end - def member?(collection, value) do - case Enumerable.member?(collection, value) do - {:ok, value} when is_boolean(value) -> - value + def member?(enumerable, element) do + case Enumerable.member?(enumerable, element) do + {:ok, element} when is_boolean(element) -> + element + {:error, module} -> - module.reduce(collection, {:cont, false}, fn - v, _ when v === value -> {:halt, true} - _, _ -> {:cont, false} - end) |> elem(1) + module.reduce(enumerable, {:cont, false}, fn + v, _ when v === element -> {:halt, true} + _, _ -> {:cont, false} + end) + |> elem(1) end end + @doc false + def min(list = [_ | _]), do: :lists.min(list) + + @doc false + def min(list = [_ | _], empty_fallback) when is_function(empty_fallback, 0) do + :lists.min(list) + end + + @doc false + @spec min(t, (-> empty_result)) :: element | empty_result when empty_result: any + def min(enumerable, empty_fallback) when is_function(empty_fallback, 0) do + min(enumerable, &<=/2, empty_fallback) + end + @doc """ - Returns the minimum value. - Raises `EmptyError` if the collection is empty. + Returns the minimal element in the `enumerable` according + to Erlang's term ordering. + + By default, the comparison is done with the [`<=`](`<=/2`) sorter function. + If multiple elements are considered minimal, the first one that + was found is returned. If you want the last element considered + minimal to be returned, the sorter function should not return true + for equal elements. + + If the enumerable is empty, the provided `empty_fallback` is called. + The default `empty_fallback` raises `Enum.EmptyError`. ## Examples iex> Enum.min([1, 2, 3]) 1 + The fact this function uses Erlang's term ordering means that the comparison + is structural and not semantic. For example: + + iex> Enum.min([~D[2017-03-31], ~D[2017-04-01]]) + ~D[2017-04-01] + + In the example above, `min/2` returned April 1st instead of March 31st + because the structural comparison compares the day before the year. + For this reason, most structs provide a "compare" function, such as + `Date.compare/2`, which receives two structs and returns `:lt` (less-than), + `:eq` (equal to), and `:gt` (greater-than). If you pass a module as the + sorting function, Elixir will automatically use the `compare/2` function + of said module: + + iex> Enum.min([~D[2017-03-31], ~D[2017-04-01]], Date) + ~D[2017-03-31] + + Finally, if you don't want to raise on empty enumerables, you can pass + the empty fallback: + + iex> Enum.min([], fn -> 0 end) + 0 + """ - @spec min(t) :: element | no_return - def min(collection) do - reduce(collection, &Kernel.min(&1, &2)) + @spec min(t, (element, element -> boolean) | module()) :: + element | empty_result + when empty_result: any + @spec min(t, (element, element -> boolean) | module(), (-> empty_result)) :: + element | empty_result + when empty_result: any + def min(enumerable, sorter \\ &<=/2, empty_fallback \\ fn -> raise Enum.EmptyError end) do + aggregate(enumerable, min_sort_fun(sorter), empty_fallback) + end + + defp min_sort_fun(sorter) when is_function(sorter, 2), do: sorter + defp min_sort_fun(module) when is_atom(module), do: &(module.compare(&1, &2) != :gt) + + @doc false + @spec min_by( + t, + (element -> any), + (-> empty_result) | (element, element -> boolean) | module() + ) :: element | empty_result + when empty_result: any + def min_by(enumerable, fun, empty_fallback) + when is_function(fun, 1) and is_function(empty_fallback, 0) do + min_by(enumerable, fun, &<=/2, empty_fallback) end @doc """ - Returns the minimum value as calculated by the given function. - Raises `EmptyError` if the collection is empty. + Returns the minimal element in the `enumerable` as calculated + by the given `fun`. + + By default, the comparison is done with the [`<=`](`<=/2`) sorter function. + If multiple elements are considered minimal, the first one that + was found is returned. If you want the last element considered + minimal to be returned, the sorter function should not return true + for equal elements. + + Calls the provided `empty_fallback` function and returns its value if + `enumerable` is empty. The default `empty_fallback` raises `Enum.EmptyError`. ## Examples - iex> Enum.min_by(["a", "aa", "aaa"], fn(x) -> String.length(x) end) + iex> Enum.min_by(["a", "aa", "aaa"], fn x -> String.length(x) end) "a" - """ - @spec min_by(t, (element -> any)) :: element | no_return - def min_by([h|t], fun) do - reduce(t, {h, fun.(h)}, fn(entry, {_, fun_min} = old) -> - fun_entry = fun.(entry) - if(fun_entry < fun_min, do: {entry, fun_entry}, else: old) - end) |> elem(0) - end - - def min_by([], _fun) do - raise Enum.EmptyError - end + iex> Enum.min_by(["a", "aa", "aaa", "b", "bbb"], &String.length/1) + "a" - def min_by(collection, fun) do - result = - reduce(collection, :first, fn - entry, {_, fun_min} = old -> - fun_entry = fun.(entry) - if(fun_entry < fun_min, do: {entry, fun_entry}, else: old) - entry, :first -> - {entry, fun.(entry)} - end) + The fact this function uses Erlang's term ordering means that the + comparison is structural and not semantic. Therefore, if you want + to compare structs, most structs provide a "compare" function, such as + `Date.compare/2`, which receives two structs and returns `:lt` (less-than), + `:eq` (equal to), and `:gt` (greater-than). If you pass a module as the + sorting function, Elixir will automatically use the `compare/2` function + of said module: + + iex> users = [ + ...> %{name: "Ellis", birthday: ~D[1943-05-11]}, + ...> %{name: "Lovelace", birthday: ~D[1815-12-10]}, + ...> %{name: "Turing", birthday: ~D[1912-06-23]} + ...> ] + iex> Enum.min_by(users, &(&1.birthday), Date) + %{name: "Lovelace", birthday: ~D[1815-12-10]} + + Finally, if you don't want to raise on empty enumerables, you can pass + the empty fallback: + + iex> Enum.min_by([], &String.length/1, fn -> nil end) + nil - case result do - :first -> raise Enum.EmptyError - {entry, _} -> entry - end + """ + @spec min_by( + t, + (element -> any), + (element, element -> boolean) | module(), + (-> empty_result) + ) :: element | empty_result + when empty_result: any + def min_by(enumerable, fun, sorter \\ &<=/2, empty_fallback \\ fn -> raise Enum.EmptyError end) + when is_function(fun, 1) do + aggregate_by(enumerable, fun, min_sort_fun(sorter), empty_fallback) end @doc """ - Returns the sum of all values. + Returns a tuple with the minimal and the maximal elements in the + enumerable. - Raises `ArithmeticError` if collection contains a non-numeric value. + By default, the comparison is done with the [`<`](` Enum.sum([1, 2, 3]) - 6 + iex> Enum.min_max([2, 3, 1]) + {1, 3} + + iex> Enum.min_max(["foo", "bar", "baz"]) + {"bar", "foo"} + + iex> Enum.min_max([], fn -> {nil, nil} end) + {nil, nil} + + The fact this function uses Erlang's term ordering means that the + comparison is structural and not semantic. Therefore, if you want + to compare structs, most structs provide a "compare" function, such as + `Date.compare/2`, which receives two structs and returns `:lt` (less-than), + `:eq` (equal to), and `:gt` (greater-than). If you pass a module as the + sorting function, Elixir will automatically use the `compare/2` function + of said module: + + iex> dates = [ + ...> ~D[2019-01-01], + ...> ~D[2020-01-01], + ...> ~D[2018-01-01] + ...> ] + iex> Enum.min_max(dates, Date) + {~D[2018-01-01], ~D[2020-01-01]} + + You can also pass a custom sorting function: + + iex> Enum.min_max([2, 3, 1], &>/2) + {3, 1} + + Finally, if you don't want to raise on empty enumerables, you can pass + the empty fallback: + + iex> Enum.min_max([], fn -> nil end) + nil """ - @spec sum(t) :: number - def sum(collection) do - reduce(collection, 0, &+/2) + @spec min_max(t, (element, element -> boolean) | module()) :: {element, element} + @spec min_max(t, (-> empty_result)) :: {element, element} | empty_result when empty_result: any + @spec min_max(t, (element, element -> boolean) | module(), (-> empty_result)) :: + {element, element} | empty_result + when empty_result: any + + def min_max(enumerable, sorter_or_empty_fallback \\ fn -> raise Enum.EmptyError end) + + def min_max(first..last//step = range, empty_fallback) + when is_function(empty_fallback, 0) do + case Range.size(range) do + 0 -> + empty_fallback.() + + _ -> + last = last - rem(last - first, step) + {Kernel.min(first, last), Kernel.max(first, last)} + end + end + + def min_max(enumerable, empty_fallback) + when is_function(empty_fallback, 0) do + min_max(enumerable, & raise Enum.EmptyError end) + end + + def min_max(enumerable, sorter, empty_fallback) + when is_atom(sorter) and is_function(empty_fallback, 0) do + min_max(enumerable, min_max_sort_fun(sorter), empty_fallback) + end + + def min_max(enumerable, sorter, empty_fallback) + when is_function(sorter, 2) and is_function(empty_fallback, 0) do + first_fun = &[&1 | &1] + + reduce_fun = fn entry, [min | max] = acc -> + cond do + sorter.(entry, min) -> + [entry | max] + + sorter.(max, entry) -> + [min | entry] + + true -> + acc + end + end + + case reduce_by(enumerable, first_fun, reduce_fun) do + :empty -> empty_fallback.() + [min | max] -> {min, max} + end + end + + @doc false + @spec min_max_by(t, (element -> any), (-> empty_result)) :: {element, element} | empty_result + when empty_result: any + def min_max_by(enumerable, fun, empty_fallback) + when is_function(fun, 1) and is_function(empty_fallback, 0) do + min_max_by(enumerable, fun, & Enum.min_max_by(["aaa", "bb", "c"], fn x -> String.length(x) end) + {"c", "aaa"} + + iex> Enum.min_max_by(["aaa", "a", "bb", "c", "ccc"], &String.length/1) + {"a", "aaa"} + + iex> Enum.min_max_by([], &String.length/1, fn -> {nil, nil} end) + {nil, nil} + + The fact this function uses Erlang's term ordering means that the + comparison is structural and not semantic. Therefore, if you want + to compare structs, most structs provide a "compare" function, such as + `Date.compare/2`, which receives two structs and returns `:lt` (less-than), + `:eq` (equal to), and `:gt` (greater-than). If you pass a module as the + sorting function, Elixir will automatically use the `compare/2` function + of said module: + + iex> users = [ + ...> %{name: "Ellis", birthday: ~D[1943-05-11]}, + ...> %{name: "Lovelace", birthday: ~D[1815-12-10]}, + ...> %{name: "Turing", birthday: ~D[1912-06-23]} + ...> ] + iex> Enum.min_max_by(users, &(&1.birthday), Date) + { + %{name: "Lovelace", birthday: ~D[1815-12-10]}, + %{name: "Ellis", birthday: ~D[1943-05-11]} + } + + Finally, if you don't want to raise on empty enumerables, you can pass + the empty fallback: + + iex> Enum.min_max_by([], &String.length/1, fn -> nil end) + nil + + """ + @spec min_max_by(t, (element -> any), (element, element -> boolean) | module()) :: + {element, element} | empty_result + when empty_result: any + @spec min_max_by( + t, + (element -> any), + (element, element -> boolean) | module(), + (-> empty_result) + ) :: {element, element} | empty_result + when empty_result: any + def min_max_by( + enumerable, + fun, + sorter_or_empty_fallback \\ & raise Enum.EmptyError end + ) + + def min_max_by(enumerable, fun, sorter, empty_fallback) + when is_function(fun, 1) and is_atom(sorter) and is_function(empty_fallback, 0) do + min_max_by(enumerable, fun, min_max_sort_fun(sorter), empty_fallback) + end + + def min_max_by(enumerable, fun, sorter, empty_fallback) + when is_function(fun, 1) and is_function(sorter, 2) and is_function(empty_fallback, 0) do + first_fun = fn entry -> + fun_entry = fun.(entry) + {entry, entry, fun_entry, fun_entry} + end + + reduce_fun = fn entry, {prev_min, prev_max, fun_min, fun_max} = acc -> + fun_entry = fun.(entry) + + cond do + sorter.(fun_entry, fun_min) -> + {entry, prev_max, fun_entry, fun_max} + + sorter.(fun_max, fun_entry) -> + {prev_min, entry, fun_min, fun_entry} + + true -> + acc + end + end + + case reduce_by(enumerable, first_fun, reduce_fun) do + :empty -> empty_fallback.() + {min, max, _, _} -> {min, max} + end end + defp min_max_sort_fun(module) when is_atom(module), do: &(module.compare(&1, &2) == :lt) + @doc """ - Partitions `collection` into two collections, where the first one contains elements - for which `fun` returns a truthy value, and the second one -- for which `fun` - returns `false` or `nil`. + Splits the `enumerable` in two lists according to the given function `fun`. + + Splits the given `enumerable` in two lists by calling `fun` with each element + in the `enumerable` as its only argument. Returns a tuple with the first list + containing all the elements in `enumerable` for which applying `fun` returned + a truthy value, and a second list with all the elements for which applying + `fun` returned a falsy value (`false` or `nil`). + + The elements in both the returned lists are in the same relative order as they + were in the original enumerable (if such enumerable was ordered, like a + list). See the examples below. ## Examples - iex> Enum.partition([1, 2, 3], fn(x) -> rem(x, 2) == 0 end) - {[2], [1,3]} + iex> Enum.split_with([5, 4, 3, 2, 1, 0], fn x -> rem(x, 2) == 0 end) + {[4, 2, 0], [5, 3, 1]} + + iex> Enum.split_with([a: 1, b: -2, c: 1, d: -3], fn {_k, v} -> v < 0 end) + {[b: -2, d: -3], [a: 1, c: 1]} + + iex> Enum.split_with([a: 1, b: -2, c: 1, d: -3], fn {_k, v} -> v > 50 end) + {[], [a: 1, b: -2, c: 1, d: -3]} + + iex> Enum.split_with([], fn {_k, v} -> v > 50 end) + {[], []} """ - @spec partition(t, (element -> any)) :: {list, list} - def partition(collection, fun) do + @doc since: "1.4.0" + @spec split_with(t, (element -> as_boolean(term))) :: {list, list} + def split_with(enumerable, fun) do {acc1, acc2} = - reduce(collection, {[], []}, fn(entry, {acc1, acc2}) -> + reduce(enumerable, {[], []}, fn entry, {acc1, acc2} -> if fun.(entry) do - {[entry|acc1], acc2} + {[entry | acc1], acc2} else - {acc1, [entry|acc2]} + {acc1, [entry | acc2]} end end) {:lists.reverse(acc1), :lists.reverse(acc2)} end + @doc false + @deprecated "Use Enum.split_with/2 instead" + def partition(enumerable, fun) do + split_with(enumerable, fun) + end + @doc """ - Splits `collection` into groups based on `fun`. + Returns a random element of an `enumerable`. - The result is a dict (by default a map) where each key is - a group and each value is a list of elements from `collection` - for which `fun` returned that group. Ordering is not necessarily - preserved. + Raises `Enum.EmptyError` if `enumerable` is empty. + + This function uses Erlang's [`:rand` module](`:rand`) to calculate + the random value. Check its documentation for setting a + different random algorithm or a different seed. + + If a range is passed into the function, this function will pick a + random value between the range limits, without traversing the whole + range (thus executing in constant time and constant memory). ## Examples - iex> Enum.group_by(~w{ant buffalo cat dingo}, &String.length/1) - %{3 => ["cat", "ant"], 7 => ["buffalo"], 5 => ["dingo"]} + The examples below use the `:exsss` pseudorandom algorithm since it's + the default from Erlang/OTP 22: + + # Although not necessary, let's seed the random algorithm + iex> :rand.seed(:exsss, {100, 101, 102}) + iex> Enum.random([1, 2, 3]) + 2 + iex> Enum.random([1, 2, 3]) + 1 + iex> Enum.random(1..1_000) + 309 + ## Implementation + + The random functions in this module implement reservoir sampling, + which allows them to sample infinite collections. In particular, + we implement Algorithm L, as described in by Kim-Hung Li in + "Reservoir-Sampling Algorithms of Time Complexity O(n(1+log(N/n)))". """ - @spec group_by(t, dict, (element -> any)) :: dict when dict: Dict.t - def group_by(collection, dict \\ %{}, fun) do - reduce(collection, dict, fn(entry, categories) -> - Dict.update(categories, fun.(entry), [entry], &[entry|&1]) - end) + @spec random(t) :: element + def random(enumerable) + + def random(enumerable) when is_list(enumerable) do + case length(enumerable) do + 0 -> raise Enum.EmptyError + length -> enumerable |> drop_list(random_count(length)) |> hd() + end end - @doc """ - Invokes `fun` for each element in the collection passing that element and the - accumulator `acc` as arguments. `fun`'s return value is stored in `acc`. - Returns the accumulator. + def random(first.._//step = range) do + case Range.size(range) do + 0 -> raise Enum.EmptyError + size -> first + random_count(size) * step + end + end - ## Examples + def random(enumerable) do + result = + case Enumerable.slice(enumerable) do + {:ok, 0, _} -> + [] - iex> Enum.reduce([1, 2, 3], 0, fn(x, acc) -> x + acc end) - 6 + {:ok, count, fun} when is_function(fun, 1) -> + slice_list(fun.(enumerable), random_count(count), 1, 1) - """ - @spec reduce(t, any, (element, any -> any)) :: any - def reduce(collection, acc, fun) when is_list(collection) do - :lists.foldl(fun, acc, collection) + {:ok, count, fun} when is_function(fun, 3) -> + fun.(random_count(count), 1, 1) + + # TODO: Remove me on v2.0 + {:ok, count, fun} when is_function(fun, 2) -> + IO.warn( + "#{inspect(Enumerable.impl_for(enumerable))} must return a three arity function on slice/1" + ) + + fun.(random_count(count), 1) + + {:error, _} -> + take_random(enumerable, 1) + end + + case result do + [] -> raise Enum.EmptyError + [elem] -> elem + end end - def reduce(collection, acc, fun) do - Enumerable.reduce(collection, {:cont, acc}, - fn x, acc -> {:cont, fun.(x, acc)} end) |> elem(1) + defp random_count(count) do + :rand.uniform(count) - 1 end @doc """ - Invokes `fun` for each element in the collection passing that element and the - accumulator `acc` as arguments. `fun`'s return value is stored in `acc`. - The first element of the collection is used as the initial value of `acc`. - Returns the accumulator. + Invokes `fun` for each element in the `enumerable` with the + accumulator. + + Raises `Enum.EmptyError` if `enumerable` is empty. + + The first element of the `enumerable` is used as the initial value + of the accumulator. Then, the function is invoked with the next + element and the accumulator. The result returned by the function + is used as the accumulator for the next iteration, recursively. + When the `enumerable` is done, the last accumulator is returned. + + Since the first element of the enumerable is used as the initial + value of the accumulator, `fun` will only be executed `n - 1` times + where `n` is the length of the enumerable. This function won't call + the specified function for enumerables that are one-element long. + + If you wish to use another value for the accumulator, use + `Enum.reduce/3`. ## Examples - iex> Enum.reduce([1, 2, 3, 4], fn(x, acc) -> x * acc end) + iex> Enum.reduce([1, 2, 3, 4], fn x, acc -> x * acc end) 24 """ - @spec reduce(t, (element, any -> any)) :: any - def reduce([h|t], fun) do + @spec reduce(t, (element, acc -> acc)) :: acc + def reduce(enumerable, fun) + + def reduce([h | t], fun) do reduce(t, h, fun) end @@ -1278,41 +2552,140 @@ defmodule Enum do raise Enum.EmptyError end - def reduce(collection, fun) do - result = - Enumerable.reduce(collection, {:cont, :first}, fn - x, :first -> - {:cont, {:acc, x}} - x, {:acc, acc} -> - {:cont, {:acc, fun.(x, acc)}} - end) |> elem(1) - - case result do - :first -> raise Enum.EmptyError + def reduce(enumerable, fun) do + Enumerable.reduce(enumerable, {:cont, :first}, fn + x, {:acc, acc} -> {:cont, {:acc, fun.(x, acc)}} + x, :first -> {:cont, {:acc, x}} + end) + |> elem(1) + |> case do + :first -> raise Enum.EmptyError {:acc, acc} -> acc end end @doc """ - Returns elements of collection for which `fun` returns `false`. + Invokes `fun` for each element in the `enumerable` with the accumulator. + + The initial value of the accumulator is `acc`. The function is invoked for + each element in the enumerable with the accumulator. The result returned + by the function is used as the accumulator for the next iteration. + The function returns the last accumulator. + + ## Examples + + iex> Enum.reduce([1, 2, 3], 0, fn x, acc -> x + acc end) + 6 + + iex> Enum.reduce(%{a: 2, b: 3, c: 4}, 0, fn {_key, val}, acc -> acc + val end) + 9 + + ## Reduce as a building block + + Reduce (sometimes called `fold`) is a basic building block in functional + programming. Almost all of the functions in the `Enum` module can be + implemented on top of reduce. Those functions often rely on other operations, + such as `Enum.reverse/1`, which are optimized by the runtime. + + For example, we could implement `map/2` in terms of `reduce/3` as follows: + + def my_map(enumerable, fun) do + enumerable + |> Enum.reduce([], fn x, acc -> [fun.(x) | acc] end) + |> Enum.reverse() + end + + In the example above, `Enum.reduce/3` accumulates the result of each call + to `fun` into a list in reverse order, which is correctly ordered at the + end by calling `Enum.reverse/1`. + + Implementing functions like `map/2`, `filter/2` and others are a good + exercise for understanding the power behind `Enum.reduce/3`. When an + operation cannot be expressed by any of the functions in the `Enum` + module, developers will most likely resort to `reduce/3`. + """ + @spec reduce(t, acc, (element, acc -> acc)) :: acc + def reduce(enumerable, acc, fun) when is_list(enumerable) do + :lists.foldl(fun, acc, enumerable) + end + + def reduce(first..last//step, acc, fun) do + reduce_range(first, last, step, acc, fun) + end + + def reduce(%_{} = enumerable, acc, fun) do + reduce_enumerable(enumerable, acc, fun) + end + + def reduce(%{} = enumerable, acc, fun) do + :maps.fold(fn k, v, acc -> fun.({k, v}, acc) end, acc, enumerable) + end + + def reduce(enumerable, acc, fun) do + reduce_enumerable(enumerable, acc, fun) + end + + @doc """ + Reduces `enumerable` until `fun` returns `{:halt, term}`. + + The return value for `fun` is expected to be + + * `{:cont, acc}` to continue the reduction with `acc` as the new + accumulator or + * `{:halt, acc}` to halt the reduction + + If `fun` returns `{:halt, acc}` the reduction is halted and the function + returns `acc`. Otherwise, if the enumerable is exhausted, the function returns + the accumulator of the last `{:cont, acc}`. + + ## Examples + + iex> Enum.reduce_while(1..100, 0, fn x, acc -> + ...> if x < 5 do + ...> {:cont, acc + x} + ...> else + ...> {:halt, acc} + ...> end + ...> end) + 10 + iex> Enum.reduce_while(1..100, 0, fn x, acc -> + ...> if x > 0 do + ...> {:cont, acc + x} + ...> else + ...> {:halt, acc} + ...> end + ...> end) + 5050 + + """ + @spec reduce_while(t, any, (element, any -> {:cont, any} | {:halt, any})) :: any + def reduce_while(enumerable, acc, fun) do + Enumerable.reduce(enumerable, {:cont, acc}, fun) |> elem(1) + end + + @doc """ + Returns a list of elements in `enumerable` excluding those for which the function `fun` returns + a truthy value. + + See also `filter/2`. ## Examples - iex> Enum.reject([1, 2, 3], fn(x) -> rem(x, 2) == 0 end) + iex> Enum.reject([1, 2, 3], fn x -> rem(x, 2) == 0 end) [1, 3] """ @spec reject(t, (element -> as_boolean(term))) :: list - def reject(collection, fun) when is_list(collection) do - for item <- collection, !fun.(item), do: item + def reject(enumerable, fun) when is_list(enumerable) do + reject_list(enumerable, fun) end - def reject(collection, fun) do - Enumerable.reduce(collection, {:cont, []}, R.reject(fun)) |> elem(1) |> :lists.reverse + def reject(enumerable, fun) do + reduce(enumerable, [], R.reject(fun)) |> :lists.reverse() end @doc """ - Reverses the collection. + Returns a list of elements in `enumerable` in reverse order. ## Examples @@ -1321,18 +2694,20 @@ defmodule Enum do """ @spec reverse(t) :: list - def reverse(collection) when is_list(collection) do - :lists.reverse(collection) - end + def reverse(enumerable) - def reverse(collection) do - reverse(collection, []) - end + def reverse([]), do: [] + def reverse([_] = list), do: list + def reverse([element1, element2]), do: [element2, element1] + def reverse([element1, element2 | rest]), do: :lists.reverse(rest, [element2, element1]) + def reverse(enumerable), do: reduce(enumerable, [], &[&1 | &2]) @doc """ - Reverses the collection and appends the tail. + Reverses the elements in `enumerable`, appends the `tail`, and returns + it as a list. + This is an optimization for - `Enum.concat(Enum.reverse(collection), tail)`. + `enumerable |> Enum.reverse() |> Enum.concat(tail)`. ## Examples @@ -1341,637 +2716,2163 @@ defmodule Enum do """ @spec reverse(t, t) :: list - def reverse(collection, tail) when is_list(collection) and is_list(tail) do - :lists.reverse(collection, tail) + def reverse(enumerable, tail) when is_list(enumerable) do + :lists.reverse(enumerable, to_list(tail)) end - def reverse(collection, tail) do - reduce(collection, to_list(tail), fn(entry, acc) -> - [entry|acc] + def reverse(enumerable, tail) do + reduce(enumerable, to_list(tail), fn entry, acc -> + [entry | acc] end) end @doc """ - Applies the given function to each element in the collection, - storing the result in a list and passing it as the accumulator - for the next computation. + Reverses the `enumerable` in the range from initial `start_index` + through `count` elements. + + If `count` is greater than the size of the rest of the `enumerable`, + then this function will reverse the rest of the enumerable. ## Examples - iex> Enum.scan(1..5, &(&1 + &2)) - [1,3,6,10,15] + iex> Enum.reverse_slice([1, 2, 3, 4, 5, 6], 2, 4) + [1, 2, 6, 5, 4, 3] """ - @spec scan(t, (element, any -> any)) :: list - def scan(enum, fun) do - {_, {res, _}} = - Enumerable.reduce(enum, {:cont, {[], :first}}, R.scan_2(fun)) - :lists.reverse(res) + @spec reverse_slice(t, non_neg_integer, non_neg_integer) :: list + def reverse_slice(enumerable, start_index, count) + when is_integer(start_index) and start_index >= 0 and is_integer(count) and count >= 0 do + list = reverse(enumerable) + length = length(list) + count = Kernel.min(count, length - start_index) + + if count > 0 do + reverse_slice(list, length, start_index + count, count, []) + else + :lists.reverse(list) + end end @doc """ - Applies the given function to each element in the collection, - storing the result in a list and passing it as the accumulator - for the next computation. Uses the given `acc` as the starting value. + Slides a single or multiple elements given by `range_or_single_index` from `enumerable` + to `insertion_index`. - ## Examples + The semantics of the range to be moved match the semantics of `Enum.slice/2`. + Specifically, that means: - iex> Enum.scan(1..5, 0, &(&1 + &2)) - [1,3,6,10,15] + * Indices are normalized, meaning that negative indexes will be counted from the end + (for example, -1 means the last element of the enumerable). This will result in *two* + traversals of your enumerable on types like lists that don't provide a constant-time count. - """ - @spec scan(t, any, (element, any -> any)) :: list - def scan(enum, acc, fun) do - {_, {res, _}} = - Enumerable.reduce(enum, {:cont, {[], acc}}, R.scan_3(fun)) - :lists.reverse(res) - end + * If the normalized index range's `last` is out of bounds, the range is truncated to the last element. - @doc """ - Returns a list of collection elements shuffled. + * If the normalized index range's `first` is out of bounds, the selected range for sliding + will be empty, so you'll get back your input list. - Notice that you need to explicitly call `:random.seed/1` and - set a seed value for the random algorithm. Otherwise, the - default seed will be set which will always return the same - result. For example, one could do the following to set a seed - dynamically: + * Decreasing ranges (such as `5..0//1`) also select an empty range to be moved, + so you'll get back your input list. - :random.seed(:erlang.now) + * Ranges with any step but 1 will raise an error. ## Examples - iex> Enum.shuffle([1, 2, 3]) - [3, 2, 1] - iex> Enum.shuffle([1, 2, 3]) - [3, 1, 2] + # Slide a single element + iex> Enum.slide([:a, :b, :c, :d, :e, :f, :g], 5, 1) + [:a, :f, :b, :c, :d, :e, :g] - """ - @spec shuffle(t) :: list - def shuffle(collection) do - randomized = reduce(collection, [], fn x, acc -> - [{:random.uniform, x}|acc] - end) - unwrap(:lists.keysort(1, randomized), []) - end + # Slide a range of elements towards the head of the list + iex> Enum.slide([:a, :b, :c, :d, :e, :f, :g], 3..5, 1) + [:a, :d, :e, :f, :b, :c, :g] - @doc """ - Returns a subset list of the given collection. Drops elements - until element position `start`, then takes `count` elements. - - If the count is greater than collection length, it returns as - much as possible. If zero, then it returns `[]`. + # Slide a range of elements towards the tail of the list + iex> Enum.slide([:a, :b, :c, :d, :e, :f, :g], 1..3, 5) + [:a, :e, :f, :b, :c, :d, :g] - ## Examples + # Slide with negative indices (counting from the end) + iex> Enum.slide([:a, :b, :c, :d, :e, :f, :g], 3..-1//1, 2) + [:a, :b, :d, :e, :f, :g, :c] + iex> Enum.slide([:a, :b, :c, :d, :e, :f, :g], -4..-2, 1) + [:a, :d, :e, :f, :b, :c, :g] - iex> Enum.slice(1..100, 5, 10) - [6, 7, 8, 9, 10, 11, 12, 13, 14, 15] + # Insert at negative indices (counting from the end) + iex> Enum.slide([:a, :b, :c, :d, :e, :f, :g], 3, -1) + [:a, :b, :c, :e, :f, :g, :d] - iex> Enum.slice(1..10, 5, 100) - [6, 7, 8, 9, 10] + """ + @doc since: "1.13.0" + @spec slide(t, Range.t() | index, index) :: list + def slide(enumerable, range_or_single_index, insertion_index) - iex> Enum.slice(1..10, 5, 0) - [] + def slide(enumerable, single_index, insertion_index) when is_integer(single_index) do + slide(enumerable, single_index..single_index, insertion_index) + end - """ - @spec slice(t, integer, non_neg_integer) :: list - - def slice(_coll, _start, 0), do: [] + # This matches the behavior of Enum.slice/2 + def slide(_, _.._//step = index_range, _insertion_index) when step != 1 do + raise ArgumentError, + "Enum.slide/3 does not accept ranges with custom steps, got: #{inspect(index_range)}" + end - def slice(coll, start, count) when start < 0 do - {list, new_start} = enumerate_and_count(coll, start) - if new_start >= 0 do - slice(list, new_start, count) + # Normalize negative input ranges like Enum.slice/2 + def slide(enumerable, first..last//_, insertion_index) + when first < 0 or last < 0 or insertion_index < 0 do + count = Enum.count(enumerable) + normalized_first = if first >= 0, do: first, else: Kernel.max(first + count, 0) + normalized_last = if last >= 0, do: last, else: last + count + + normalized_insertion_index = + if insertion_index >= 0, do: insertion_index, else: insertion_index + count + + if normalized_first < count and normalized_first != normalized_insertion_index do + normalized_range = normalized_first..normalized_last//1 + slide(enumerable, normalized_range, normalized_insertion_index) else - [] + Enum.to_list(enumerable) end end - def slice(coll, start, count) when is_list(coll) and start >= 0 and count > 0 do - do_slice(coll, start, count) + def slide(enumerable, insertion_index.._//_, insertion_index) do + Enum.to_list(enumerable) end - def slice(coll, start, count) when start >= 0 and count > 0 do - {_, _, list} = Enumerable.reduce(coll, {:cont, {start, count, []}}, fn - _entry, {start, count, _list} when start > 0 -> - {:cont, {start-1, count, []}} - entry, {start, count, list} when count > 1 -> - {:cont, {start, count-1, [entry|list]}} - entry, {start, count, list} -> - {:halt, {start, count, [entry|list]}} - end) |> elem(1) + def slide(_, first..last//_, insertion_index) + when insertion_index > first and insertion_index <= last do + raise ArgumentError, + "insertion index for slide must be outside the range being moved " <> + "(tried to insert #{first}..#{last} at #{insertion_index})" + end - :lists.reverse(list) + def slide(enumerable, first..last//_, _insertion_index) when first > last do + Enum.to_list(enumerable) end - @doc """ - Returns a subset list of the given collection. Drops elements - until element position `range.first`, then takes elements until element - position `range.last` (inclusive). + # Guarantees at this point: step size == 1 and first <= last and (insertion_index < first or insertion_index > last) + def slide(enumerable, first..last//_, insertion_index) do + impl = if is_list(enumerable), do: &slide_list_start/4, else: &slide_any/4 - Positions are calculated by adding the number of items in the collection to - negative positions (so position -3 in a collection with count 5 becomes - position 2). + cond do + insertion_index <= first -> impl.(enumerable, insertion_index, first, last) + insertion_index > last -> impl.(enumerable, first, last + 1, insertion_index) + end + end - The first position (after adding count to negative positions) must be smaller - or equal to the last position. + # Takes the range from middle..last and moves it to be in front of index start + defp slide_any(enumerable, start, middle, last) do + # We're going to deal with 4 "chunks" of the enumerable: + # 0. "Head," before the start index + # 1. "Slide back," between start (inclusive) and middle (exclusive) + # 2. "Slide front," between middle (inclusive) and last (inclusive) + # 3. "Tail," after last + # + # But, we're going to accumulate these into only two lists: pre and post. + # We'll reverse-accumulate the head into our pre list, then "slide back" into post, + # then "slide front" into pre, then "tail" into post. + # + # Then at the end, we're going to reassemble and reverse them, and end up with the + # chunks in the correct order. + {_size, pre, post} = + reduce(enumerable, {0, [], []}, fn item, {index, pre, post} -> + {pre, post} = + cond do + index < start -> {[item | pre], post} + index >= start and index < middle -> {pre, [item | post]} + index >= middle and index <= last -> {[item | pre], post} + true -> {pre, [item | post]} + end + + {index + 1, pre, post} + end) - If the start of the range is not a valid offset for the given - collection or if the range is in reverse order, returns `[]`. + :lists.reverse(pre, :lists.reverse(post)) + end - ## Examples + # Like slide_any/4 above, this optimized implementation of slide for lists depends + # on the indices being sorted such that we're moving middle..last to be in front of start. + defp slide_list_start([h | t], start, middle, last) + when start > 0 and start <= middle and middle <= last do + [h | slide_list_start(t, start - 1, middle - 1, last - 1)] + end - iex> Enum.slice(1..100, 5..10) - [6, 7, 8, 9, 10, 11] + defp slide_list_start(list, 0, middle, last), do: slide_list_middle(list, middle, last, []) + defp slide_list_start([], _start, _middle, _last), do: [] - iex> Enum.slice(1..10, 5..20) - [6, 7, 8, 9, 10] + defp slide_list_middle([h | t], middle, last, acc) when middle > 0 do + slide_list_middle(t, middle - 1, last - 1, [h | acc]) + end - iex> Enum.slice(1..10, 11..20) - [] + defp slide_list_middle(list, 0, last, start_to_middle) do + {slid_range, tail} = slide_list_last(list, last + 1, []) + slid_range ++ :lists.reverse(start_to_middle, tail) + end - iex> Enum.slice(1..10, 6..5) - [] + # You asked for a middle index off the end of the list... you get what we've got + defp slide_list_middle([], _, _, acc) do + :lists.reverse(acc) + end - """ - @spec slice(t, Range.t) :: list - def slice(coll, first..last) when first >= 0 and last >= 0 do - # Simple case, which works on infinite collections - if last - first >= 0 do - slice(coll, first, last - first + 1) - else - [] - end + defp slide_list_last([h | t], last, acc) when last > 0 do + slide_list_last(t, last - 1, [h | acc]) end - def slice(coll, first..last) do - {list, count} = enumerate_and_count(coll, 0) - corr_first = if first >= 0, do: first, else: first + count - corr_last = if last >= 0, do: last, else: last + count - length = corr_last - corr_first + 1 - if corr_first >= 0 and length > 0 do - slice(list, corr_first, length) - else - [] - end + defp slide_list_last(rest, 0, acc) do + {:lists.reverse(acc), rest} + end + + defp slide_list_last([], _, acc) do + {:lists.reverse(acc), []} end @doc """ - Sorts the collection according to Elixir's term ordering. + Passes each element from `enumerable` to the `fun` as the first argument, + stores the `fun` result in a list and passes the result as the second argument + for the next computation. - Uses the merge sort algorithm. + The `fun` isn't applied for the first element of the `enumerable`, + the element is taken as it is. ## Examples - iex> Enum.sort([3, 2, 1]) - [1, 2, 3] + iex> Enum.scan(["a", "b", "c", "d", "e"], fn element, acc -> element <> String.first(acc) end) + ["a", "ba", "cb", "dc", "ed"] + + iex> Enum.scan(1..5, fn element, acc -> element + acc end) + [1, 3, 6, 10, 15] """ - @spec sort(t) :: list - def sort(collection) when is_list(collection) do - :lists.sort(collection) + @spec scan(t, (element, any -> any)) :: list + def scan(enumerable, fun) + + def scan([], _fun), do: [] + + def scan([elem | rest], fun) do + scanned = scan_list(rest, elem, fun) + [elem | scanned] end - def sort(collection) do - sort(collection, &(&1 <= &2)) + def scan(enumerable, fun) do + {res, _} = reduce(enumerable, {[], :first}, R.scan2(fun)) + :lists.reverse(res) end @doc """ - Sorts the collection by the given function. + Passes each element from `enumerable` to the `fun` as the first argument, + stores the `fun` result in a list and passes the result as the second argument + for the next computation. - This function uses the merge sort algorithm. The given function - must return false if the first argument is less than right one. + Passes the given `acc` as the second argument for the `fun` with the first element. ## Examples - iex> Enum.sort([1, 2, 3], &(&1 > &2)) - [3, 2, 1] + iex> Enum.scan(["a", "b", "c", "d", "e"], "_", fn element, acc -> element <> String.first(acc) end) + ["a_", "ba", "cb", "dc", "ed"] - The sorting algorithm will be stable as long as the given function - returns true for values considered equal: + iex> Enum.scan(1..5, 0, fn element, acc -> element + acc end) + [1, 3, 6, 10, 15] - iex> Enum.sort ["some", "kind", "of", "monster"], &(byte_size(&1) <= byte_size(&2)) - ["of", "some", "kind", "monster"] + """ + @spec scan(t, any, (element, any -> any)) :: list + def scan(enumerable, acc, fun) when is_list(enumerable) do + scan_list(enumerable, acc, fun) + end - If the function does not return true, the sorting is not stable and - the order of equal terms may be shuffled: + def scan(enumerable, acc, fun) do + {res, _} = reduce(enumerable, {[], acc}, R.scan3(fun)) + :lists.reverse(res) + end - iex> Enum.sort ["some", "kind", "of", "monster"], &(byte_size(&1) < byte_size(&2)) - ["of", "kind", "some", "monster"] + @doc """ + Returns a list with the elements of `enumerable` shuffled. + + This function uses Erlang's [`:rand` module](`:rand`) to calculate + the random value. Check its documentation for setting a + different random algorithm or a different seed. + + ## Examples + + The examples below use the `:exsss` pseudorandom algorithm since it's + the default from Erlang/OTP 22: + + # Although not necessary, let's seed the random algorithm + iex> :rand.seed(:exsss, {11, 22, 33}) + iex> Enum.shuffle([1, 2, 3]) + [2, 1, 3] + iex> Enum.shuffle([1, 2, 3]) + [2, 3, 1] """ - @spec sort(t, (element, element -> boolean)) :: list - def sort(collection, fun) when is_list(collection) do - :lists.sort(fun, collection) - end + @spec shuffle(t) :: list + def shuffle(enumerable) do + randomized = + reduce(enumerable, [], fn x, acc -> + [{:rand.uniform(), x} | acc] + end) - def sort(collection, fun) do - reduce(collection, [], &sort_reducer(&1, &2, fun)) |> sort_terminator(fun) + shuffle_unwrap(:lists.keysort(1, randomized)) end + defp shuffle_unwrap([{_, h} | rest]), do: [h | shuffle_unwrap(rest)] + defp shuffle_unwrap([]), do: [] + @doc """ - Splits the enumerable into two collections, leaving `count` - elements in the first one. If `count` is a negative number, - it starts counting from the back to the beginning of the - collection. + Returns a subset list of the given `enumerable` by `index_range`. - Be aware that a negative `count` implies the collection - will be enumerated twice: once to calculate the position, and - a second time to do the actual splitting. + `index_range` must be a `Range`. Given an `enumerable`, it drops + elements before `index_range.first` (zero-base), then it takes elements + until element `index_range.last` (inclusively). + + Indexes are normalized, meaning that negative indexes will be counted + from the end (for example, `-1` means the last element of the `enumerable`). + + If `index_range.last` is out of bounds, then it is assigned as the index + of the last element. + + If the normalized `index_range.first` is out of bounds of the given + `enumerable`, or this one is greater than the normalized `index_range.last`, + then `[]` is returned. + + If a step `n` (other than `1`) is used in `index_range`, then it takes + every `n`th element from `index_range.first` to `index_range.last` + (according to the same rules described above). ## Examples - iex> Enum.split([1, 2, 3], 2) - {[1,2], [3]} + iex> Enum.slice([1, 2, 3, 4, 5], 1..3) + [2, 3, 4] - iex> Enum.split([1, 2, 3], 10) - {[1,2,3], []} + iex> Enum.slice([1, 2, 3, 4, 5], 3..10) + [4, 5] - iex> Enum.split([1, 2, 3], 0) - {[], [1,2,3]} + # Last three elements (negative indexes) + iex> Enum.slice([1, 2, 3, 4, 5], -3..-1) + [3, 4, 5] - iex> Enum.split([1, 2, 3], -1) - {[1,2], [3]} + For ranges where `start > stop`, you need to explicit + mark them as increasing: - iex> Enum.split([1, 2, 3], -5) - {[], [1,2,3]} + iex> Enum.slice([1, 2, 3, 4, 5], 1..-2//1) + [2, 3, 4] + + The step can be any positive number. For example, to + get every 2 elements of the collection: + + iex> Enum.slice([1, 2, 3, 4, 5], 0..-1//2) + [1, 3, 5] + + To get every third element of the first ten elements: + + iex> integers = Enum.to_list(1..20) + iex> Enum.slice(integers, 0..9//3) + [1, 4, 7, 10] + + If the first position is after the end of the enumerable + or after the last position of the range, it returns an + empty list: + + iex> Enum.slice([1, 2, 3, 4, 5], 6..10) + [] + + # first is greater than last + iex> Enum.slice([1, 2, 3, 4, 5], 6..5//1) + [] """ - @spec split(t, integer) :: {list, list} - def split(collection, count) when is_list(collection) and count >= 0 do - do_split(collection, count, []) - end + @doc since: "1.6.0" + @spec slice(t, Range.t()) :: list + def slice(enumerable, first..last//step = index_range) do + # TODO: Support negative steps as a reverse on Elixir v2.0. + cond do + step > 0 -> + slice_range(enumerable, first, last, step) - def split(collection, count) when count >= 0 do - {_, list1, list2} = - reduce(collection, {count, [], []}, fn(entry, {counter, acc1, acc2}) -> - if counter > 0 do - {counter - 1, [entry|acc1], acc2} - else - {counter, acc1, [entry|acc2]} - end - end) + step == -1 and first > last -> + IO.warn( + "negative steps are not supported in Enum.slice/2, pass #{first}..#{last}//1 instead" + ) + + slice_range(enumerable, first, last, 1) + + true -> + raise ArgumentError, + "Enum.slice/2 does not accept ranges with negative steps, got: #{inspect(index_range)}" + end + end + + # TODO: Remove me on v2.0 + def slice(enumerable, %{__struct__: Range, first: first, last: last} = index_range) do + step = if first <= last, do: 1, else: -1 + slice(enumerable, Map.put(index_range, :step, step)) + end + + defp slice_range(enumerable, first, -1, step) when first >= 0 do + if step == 1 do + drop(enumerable, first) + else + enumerable |> drop(first) |> take_every_list(step - 1) + end + end + + defp slice_range(enumerable, first, last, step) + when last >= first and last >= 0 and first >= 0 do + slice_forward(enumerable, first, last - first + 1, step) + end + + defp slice_range(enumerable, first, last, step) do + {count, fun} = slice_count_and_fun(enumerable, step) + first = if first >= 0, do: first, else: Kernel.max(first + count, 0) + last = if last >= 0, do: last, else: last + count + amount = last - first + 1 + + if first < count and amount > 0 do + amount = Kernel.min(amount, count - first) + amount = amount_with_step(amount, step) + fun.(first, amount, step) + else + [] + end + end + + defp amount_with_step(amount, 1), do: amount + defp amount_with_step(amount, step), do: div(amount - 1, step) + 1 + + @doc """ + Returns a subset list of the given `enumerable`, from `start_index` (zero-based) + with `amount` number of elements if available. + + Given an `enumerable`, it drops elements right before element `start_index`; + then, it takes `amount` of elements, returning as many elements as possible if + there are not enough elements. + + A negative `start_index` can be passed, which means the `enumerable` is + enumerated once and the index is counted from the end (for example, + `-1` starts slicing from the last element). + + It returns `[]` if `amount` is `0` or if `start_index` is out of bounds. + + ## Examples + + iex> Enum.slice(1..100, 5, 10) + [6, 7, 8, 9, 10, 11, 12, 13, 14, 15] + + # amount to take is greater than the number of elements + iex> Enum.slice(1..10, 5, 100) + [6, 7, 8, 9, 10] + + iex> Enum.slice(1..10, 5, 0) + [] + + # using a negative start index + iex> Enum.slice(1..10, -6, 3) + [5, 6, 7] + iex> Enum.slice(1..10, -11, 5) + [1, 2, 3, 4, 5] + + # out of bound start index + iex> Enum.slice(1..10, 10, 5) + [] + + """ + @spec slice(t, index, non_neg_integer) :: list + def slice(_enumerable, start_index, 0) when is_integer(start_index), do: [] + + def slice(enumerable, start_index, amount) + when is_integer(start_index) and start_index < 0 and is_integer(amount) and amount >= 0 do + {count, fun} = slice_count_and_fun(enumerable, 1) + start_index = Kernel.max(count + start_index, 0) + amount = Kernel.min(amount, count - start_index) + + if amount > 0 do + fun.(start_index, amount, 1) + else + [] + end + end + + def slice(enumerable, start_index, amount) + when is_integer(start_index) and is_integer(amount) and amount >= 0 do + slice_forward(enumerable, start_index, amount, 1) + end + + @doc """ + Sorts the `enumerable` according to Erlang's term ordering. + + This function uses the merge sort algorithm. Do not use this + function to sort structs, see `sort/2` for more information. + + ## Examples + + iex> Enum.sort([3, 2, 1]) + [1, 2, 3] + + """ + @spec sort(t) :: list + def sort(enumerable) when is_list(enumerable) do + :lists.sort(enumerable) + end + + def sort(enumerable) do + sort(enumerable, &(&1 <= &2)) + end + + @doc """ + Sorts the `enumerable` by the given function. + + This function uses the merge sort algorithm. The given function should compare + two arguments, and return `true` if the first argument precedes or is in the + same place as the second one. + + ## Examples + + iex> Enum.sort([1, 2, 3], &(&1 >= &2)) + [3, 2, 1] + + The sorting algorithm will be stable as long as the given function + returns `true` for values considered equal: + + iex> Enum.sort(["some", "kind", "of", "monster"], &(byte_size(&1) <= byte_size(&2))) + ["of", "some", "kind", "monster"] + + If the function does not return `true` for equal values, the sorting + is not stable and the order of equal terms may be shuffled. + For example: + + iex> Enum.sort(["some", "kind", "of", "monster"], &(byte_size(&1) < byte_size(&2))) + ["of", "kind", "some", "monster"] + + ## Ascending and descending (since v1.10.0) + + `sort/2` allows a developer to pass `:asc` or `:desc` as the sorter, which is a convenience for + [`&<=/2`](`<=/2`) and [`&>=/2`](`>=/2`) respectively. + + iex> Enum.sort([2, 3, 1], :asc) + [1, 2, 3] + iex> Enum.sort([2, 3, 1], :desc) + [3, 2, 1] + + ## Sorting structs + + Do not use `/2`, `>=/2` and friends when sorting structs. + That's because the built-in operators above perform structural comparison + and not a semantic one. Imagine we sort the following list of dates: + + iex> dates = [~D[2019-01-01], ~D[2020-03-02], ~D[2019-06-06]] + iex> Enum.sort(dates) + [~D[2019-01-01], ~D[2020-03-02], ~D[2019-06-06]] + + Note that the returned result is incorrect, because `sort/1` by default uses + `<=/2`, which will compare their structure. When comparing structures, the + fields are compared in alphabetical order, which means the dates above will + be compared by `day`, `month` and then `year`, which is the opposite of what + we want. + + For this reason, most structs provide a "compare" function, such as + `Date.compare/2`, which receives two structs and returns `:lt` (less-than), + `:eq` (equal to), and `:gt` (greater-than). If you pass a module as the + sorting function, Elixir will automatically use the `compare/2` function + of said module: + + iex> dates = [~D[2019-01-01], ~D[2020-03-02], ~D[2019-06-06]] + iex> Enum.sort(dates, Date) + [~D[2019-01-01], ~D[2019-06-06], ~D[2020-03-02]] + + To retrieve all dates in descending order, you can wrap the module in + a tuple with `:asc` or `:desc` as first element: + + iex> dates = [~D[2019-01-01], ~D[2020-03-02], ~D[2019-06-06]] + iex> Enum.sort(dates, {:asc, Date}) + [~D[2019-01-01], ~D[2019-06-06], ~D[2020-03-02]] + iex> dates = [~D[2019-01-01], ~D[2020-03-02], ~D[2019-06-06]] + iex> Enum.sort(dates, {:desc, Date}) + [~D[2020-03-02], ~D[2019-06-06], ~D[2019-01-01]] + + """ + @spec sort( + t, + (element, element -> boolean) | :asc | :desc | module() | {:asc | :desc, module()} + ) :: list + def sort(enumerable, sorter) when is_list(enumerable) do + case sorter do + :asc -> :lists.sort(enumerable) + :desc -> :lists.sort(enumerable) |> :lists.reverse() + _ -> :lists.sort(to_sort_fun(sorter), enumerable) + end + end + + def sort(enumerable, sorter) do + fun = to_sort_fun(sorter) + + reduce(enumerable, [], &sort_reducer(&1, &2, fun)) + |> sort_terminator(fun) + end + + defp to_sort_fun(sorter) when is_function(sorter, 2), do: sorter + defp to_sort_fun(:asc), do: &<=/2 + defp to_sort_fun(:desc), do: &>=/2 + defp to_sort_fun(module) when is_atom(module), do: &(module.compare(&1, &2) != :gt) + defp to_sort_fun({:asc, module}) when is_atom(module), do: &(module.compare(&1, &2) != :gt) + defp to_sort_fun({:desc, module}) when is_atom(module), do: &(module.compare(&1, &2) != :lt) + + @doc """ + Sorts the mapped results of the `enumerable` according to the provided `sorter` + function. + + This function maps each element of the `enumerable` using the + provided `mapper` function. The enumerable is then sorted by + the mapped elements using the `sorter`, which defaults to `:asc` + and sorts the elements ascendingly. + + `sort_by/3` differs from `sort/2` in that it only calculates the + comparison value for each element in the enumerable once instead of + once for each element in each comparison. If the same function is + being called on both elements, it's more efficient to use `sort_by/3`. + + ## Ascending and descending (since v1.10.0) + + `sort_by/3` allows a developer to pass `:asc` or `:desc` as the sorter, + which is a convenience for [`&<=/2`](`<=/2`) and [`&>=/2`](`>=/2`) respectively: + iex> Enum.sort_by([2, 3, 1], &(&1), :asc) + [1, 2, 3] + + iex> Enum.sort_by([2, 3, 1], &(&1), :desc) + [3, 2, 1] + + ## Examples + + Using the default `sorter` of `:asc` : + + iex> Enum.sort_by(["some", "kind", "of", "monster"], &byte_size/1) + ["of", "some", "kind", "monster"] + + Sorting by multiple properties - first by size, then by first letter + (this takes advantage of the fact that tuples are compared element-by-element): + + iex> Enum.sort_by(["some", "kind", "of", "monster"], &{byte_size(&1), String.first(&1)}) + ["of", "kind", "some", "monster"] + + Similar to `sort/2`, you can pass a custom sorter: + + iex> Enum.sort_by(["some", "kind", "of", "monster"], &byte_size/1, :desc) + ["monster", "some", "kind", "of"] + + As in `sort/2`, avoid using the default sorting function to sort + structs, as by default it performs structural comparison instead of + a semantic one. In such cases, you shall pass a sorting function as + third element or any module that implements a `compare/2` function. + For example, to sort users by their birthday in both ascending and + descending order respectively: + + iex> users = [ + ...> %{name: "Ellis", birthday: ~D[1943-05-11]}, + ...> %{name: "Lovelace", birthday: ~D[1815-12-10]}, + ...> %{name: "Turing", birthday: ~D[1912-06-23]} + ...> ] + iex> Enum.sort_by(users, &(&1.birthday), Date) + [ + %{name: "Lovelace", birthday: ~D[1815-12-10]}, + %{name: "Turing", birthday: ~D[1912-06-23]}, + %{name: "Ellis", birthday: ~D[1943-05-11]} + ] + iex> Enum.sort_by(users, &(&1.birthday), {:desc, Date}) + [ + %{name: "Ellis", birthday: ~D[1943-05-11]}, + %{name: "Turing", birthday: ~D[1912-06-23]}, + %{name: "Lovelace", birthday: ~D[1815-12-10]} + ] + + ## Performance characteristics + + As detailed in the initial section, `sort_by/3` calculates the comparison + value for each element in the enumerable once instead of once for each + element in each comparison. This implies `sort_by/3` must do an initial + pass on the data to compute those values. + + However, if those values are cheap to compute, for example, you have + already extracted the field you want to sort by into a tuple, then those + extra passes become overhead. In such cases, consider using `List.keysort/3` + instead. + + Let's see an example. Imagine you have a list of products and you have a + list of IDs. You want to keep all products that are in the given IDs and + return their names sorted by their price. You could write it like this: + + for( + product <- products, + product.id in ids, + do: product + ) + |> Enum.sort_by(& &1.price) + |> Enum.map(& &1.name) + + However, you could also write it like this: + + for( + product <- products, + product.id in ids, + do: {product.name, product.price} + ) + |> List.keysort(1) + |> Enum.map(&elem(&1, 0)) + + Using `List.keysort/3` will be a better choice for performance sensitive + code as it avoids additional traversals. + """ + @spec sort_by( + t, + (element -> mapped_element), + (element, element -> boolean) | :asc | :desc | module() | {:asc | :desc, module()} + ) :: + list + when mapped_element: element + def sort_by(enumerable, mapper, sorter \\ :asc) + + def sort_by(enumerable, mapper, :desc) when is_function(mapper, 1) do + enumerable + |> reduce([], &[{&1, mapper.(&1)} | &2]) + |> List.keysort(1, :asc) + |> List.foldl([], &[elem(&1, 0) | &2]) + end + + def sort_by(enumerable, mapper, sorter) when is_function(mapper, 1) do + enumerable + |> map(&{&1, mapper.(&1)}) + |> List.keysort(1, sorter) + |> map(&elem(&1, 0)) + end + + @doc """ + Splits the `enumerable` into two enumerables, leaving `count` + elements in the first one. + + If `count` is a negative number, it starts counting from the + back to the beginning of the `enumerable`. + + Be aware that a negative `count` implies the `enumerable` + will be enumerated twice: once to calculate the position, and + a second time to do the actual splitting. + + ## Examples + + iex> Enum.split([1, 2, 3], 2) + {[1, 2], [3]} + + iex> Enum.split([1, 2, 3], 10) + {[1, 2, 3], []} + + iex> Enum.split([1, 2, 3], 0) + {[], [1, 2, 3]} + + iex> Enum.split([1, 2, 3], -1) + {[1, 2], [3]} + + iex> Enum.split([1, 2, 3], -5) + {[], [1, 2, 3]} + + """ + @spec split(t, integer) :: {list, list} + def split(enumerable, count) when is_list(enumerable) and is_integer(count) and count >= 0 do + split_list(enumerable, count, []) + end + + def split(enumerable, count) when is_integer(count) and count >= 0 do + {_, list1, list2} = + reduce(enumerable, {count, [], []}, fn entry, {counter, acc1, acc2} -> + if counter > 0 do + {counter - 1, [entry | acc1], acc2} + else + {counter, acc1, [entry | acc2]} + end + end) + + {:lists.reverse(list1), :lists.reverse(list2)} + end + + def split(enumerable, count) when is_integer(count) and count < 0 do + split_reverse_list(reverse(enumerable), -count, []) + end + + @doc """ + Splits enumerable in two at the position of the element for which + `fun` returns a falsy value (`false` or `nil`) for the first time. + + It returns a two-element tuple with two lists of elements. + The element that triggered the split is part of the second list. + + ## Examples + + iex> Enum.split_while([1, 2, 3, 4], fn x -> x < 3 end) + {[1, 2], [3, 4]} + + iex> Enum.split_while([1, 2, 3, 4], fn x -> x < 0 end) + {[], [1, 2, 3, 4]} + + iex> Enum.split_while([1, 2, 3, 4], fn x -> x > 0 end) + {[1, 2, 3, 4], []} + + """ + @spec split_while(t, (element -> as_boolean(term))) :: {list, list} + def split_while(enumerable, fun) when is_list(enumerable) do + split_while_list(enumerable, fun, []) + end + + def split_while(enumerable, fun) do + {list1, list2} = + reduce(enumerable, {[], []}, fn + entry, {acc1, []} -> + if(fun.(entry), do: {[entry | acc1], []}, else: {acc1, [entry]}) + + entry, {acc1, acc2} -> + {acc1, [entry | acc2]} + end) + + {:lists.reverse(list1), :lists.reverse(list2)} + end + + @doc """ + Returns the sum of all elements. + + Raises `ArithmeticError` if `enumerable` contains a non-numeric value. + + If you need to apply a transformation first, consider using `Enum.sum_by/2` instead. + + ## Examples + + iex> Enum.sum([1, 2, 3]) + 6 + + iex> Enum.sum(1..10) + 55 + + iex> Enum.sum(1..10//2) + 25 + + """ + @spec sum(t) :: number + def sum(enumerable) + + def sum(first..last//step = range) do + range + |> Range.size() + |> Kernel.*(first + last - rem(last - first, step)) + |> div(2) + end + + def sum(enumerable) do + reduce(enumerable, 0, &+/2) + end + + @doc """ + Maps and sums the given `enumerable` in one pass. + + Raises `ArithmeticError` if `mapper` returns a non-numeric value. + + ## Examples + + iex> Enum.sum_by([%{count: 1}, %{count: 2}, %{count: 3}], fn x -> x.count end) + 6 + + iex> Enum.sum_by(1..3, fn x -> x ** 2 end) + 14 + + iex> Enum.sum_by([], fn x -> x.count end) + 0 + + Filtering can be achieved by returning `0` to ignore elements: + + iex> Enum.sum_by([1, -2, 3], fn x -> if x > 0, do: x, else: 0 end) + 4 + + """ + @doc since: "1.18.0" + @spec sum_by(t, (element -> number)) :: number + def sum_by(enumerable, mapper) + + def sum_by(list, mapper) when is_list(list) and is_function(mapper, 1) do + sum_by_list(list, mapper, 0) + end + + def sum_by(enumerable, mapper) when is_function(mapper, 1) do + reduce(enumerable, 0, fn x, acc -> acc + mapper.(x) end) + end + + @doc """ + Returns the product of all elements. + + Raises `ArithmeticError` if `enumerable` contains a non-numeric value. + + If you need to apply a transformation first, consider using `Enum.product_by/2` instead. + + ## Examples + + iex> Enum.product([]) + 1 + iex> Enum.product([2, 3, 4]) + 24 + iex> Enum.product([2.0, 3.0, 4.0]) + 24.0 + + """ + @doc since: "1.12.0" + @spec product(t) :: number + def product(enumerable) do + reduce(enumerable, 1, &*/2) + end + + @doc """ + Maps and computes the product of the given `enumerable` in one pass. + + Raises `ArithmeticError` if `mapper` returns a non-numeric value. + + ## Examples + + iex> Enum.product_by([%{count: 2}, %{count: 4}, %{count: 3}], fn x -> x.count end) + 24 + + iex> Enum.product_by(1..3, fn x -> x ** 2 end) + 36 + + iex> Enum.product_by([], fn x -> x.count end) + 1 + + Filtering can be achieved by returning `1` to ignore elements: + + iex> Enum.product_by([2, -1, 3], fn x -> if x > 0, do: x, else: 1 end) + 6 + + """ + @doc since: "1.18.0" + @spec product_by(t, (element -> number)) :: number + def product_by(enumerable, mapper) + + def product_by(list, mapper) when is_list(list) and is_function(mapper, 1) do + product_by_list(list, mapper, 1) + end + + def product_by(enumerable, mapper) when is_function(mapper, 1) do + reduce(enumerable, 1, fn x, acc -> acc * mapper.(x) end) + end + + @doc """ + Takes an `amount` of elements from the beginning or the end of the `enumerable`. + + If a positive `amount` is given, it takes the `amount` elements from the + beginning of the `enumerable`. + + If a negative `amount` is given, the `amount` of elements will be taken from the end. + The `enumerable` will be enumerated once to retrieve the proper index and + the remaining calculation is performed from the end. + + If amount is `0`, it returns `[]`. + + ## Examples + + iex> Enum.take([1, 2, 3], 2) + [1, 2] + + iex> Enum.take([1, 2, 3], 10) + [1, 2, 3] + + iex> Enum.take([1, 2, 3], 0) + [] + + iex> Enum.take([1, 2, 3], -1) + [3] + + """ + @spec take(t, integer) :: list + def take(enumerable, amount) + + def take(_enumerable, 0), do: [] + + def take(enumerable, amount) + when is_list(enumerable) and is_integer(amount) and amount > 0 do + take_list(enumerable, amount) + end + + def take(enumerable, amount) when is_integer(amount) and amount > 0 do + {_, {res, _}} = + Enumerable.reduce(enumerable, {:cont, {[], amount}}, fn entry, {list, n} -> + case n do + 1 -> {:halt, {[entry | list], n - 1}} + _ -> {:cont, {[entry | list], n - 1}} + end + end) + + :lists.reverse(res) + end + + def take(enumerable, amount) when is_integer(amount) and amount < 0 do + case slice_count_and_fun(enumerable, 1) do + {0, _fun} -> + [] + + {count, fun} -> + first = Kernel.max(amount + count, 0) + fun.(first, count - first, 1) + end + end + + @doc """ + Returns a list of every `nth` element in the `enumerable`, + starting with the first element. + + The first element is always included, unless `nth` is 0. + + The second argument specifying every `nth` element must be a non-negative + integer. + + ## Examples + + iex> Enum.take_every(1..10, 2) + [1, 3, 5, 7, 9] + + iex> Enum.take_every(1..10, 0) + [] + + iex> Enum.take_every([1, 2, 3], 1) + [1, 2, 3] + + """ + @spec take_every(t, non_neg_integer) :: list + def take_every(enumerable, nth) + + def take_every(_enumerable, 0), do: [] + def take_every(enumerable, 1), do: to_list(enumerable) + + def take_every(list, nth) when is_list(list) and is_integer(nth) and nth > 1 do + take_every_list(list, nth - 1) + end + + def take_every(enumerable, nth) when is_integer(nth) and nth > 1 do + {res, _} = reduce(enumerable, {[], :first}, R.take_every(nth)) + :lists.reverse(res) + end + + @doc """ + Takes `count` random elements from `enumerable`. + + Note that this function will traverse the whole `enumerable` to + get the random sublist. + + See `random/1` for notes on implementation and random seed. + + ## Examples + + # Although not necessary, let's seed the random algorithm + iex> :rand.seed(:exsss, {1, 2, 3}) + iex> Enum.take_random(1..10, 2) + [6, 1] + iex> Enum.take_random(?a..?z, 5) + ~c"bkzmt" + + """ + @spec take_random(t, non_neg_integer) :: list + def take_random(enumerable, count) + def take_random(_enumerable, 0), do: [] + def take_random([], _), do: [] + + def take_random(enumerable, 1) do + enumerable + |> reduce({0, 0, 1.0, nil}, fn + elem, {idx, idx, w, _current} -> + {jdx, w} = take_jdx_w(idx, w, 1) + {idx + 1, jdx, w, elem} + + _elem, {idx, jdx, w, current} -> + {idx + 1, jdx, w, current} + end) + |> case do + {0, 0, 1.0, nil} -> [] + {_idx, _jdx, _w, current} -> [current] + end + end + + def take_random(enumerable, count) when count in 0..128 do + sample = Tuple.duplicate(nil, count) + + reducer = fn + elem, {idx, jdx, w, sample} when idx < count -> + rand = take_index(idx) + sample = sample |> put_elem(idx, elem(sample, rand)) |> put_elem(rand, elem) + + if idx == jdx do + {jdx, w} = take_jdx_w(idx, w, count) + {idx + 1, jdx, w, sample} + else + {idx + 1, jdx, w, sample} + end + + elem, {idx, idx, w, sample} -> + pos = :rand.uniform(count) - 1 + {jdx, w} = take_jdx_w(idx, w, count) + {idx + 1, jdx, w, put_elem(sample, pos, elem)} + + _elem, {idx, jdx, w, sample} -> + {idx + 1, jdx, w, sample} + end + + {size, _, _, sample} = reduce(enumerable, {0, count - 1, 1.0, sample}, reducer) + + if count < size do + Tuple.to_list(sample) + else + take_tupled(sample, size, []) + end + end + + def take_random(enumerable, count) when is_integer(count) and count >= 0 do + reducer = fn + elem, {idx, jdx, w, sample} when idx < count -> + rand = take_index(idx) + sample = sample |> Map.put(idx, Map.get(sample, rand)) |> Map.put(rand, elem) + + if idx == jdx do + {jdx, w} = take_jdx_w(idx, w, count) + {idx + 1, jdx, w, sample} + else + {idx + 1, jdx, w, sample} + end + + elem, {idx, idx, w, sample} -> + pos = :rand.uniform(count) - 1 + {jdx, w} = take_jdx_w(idx, w, count) + {idx + 1, jdx, w, %{sample | pos => elem}} + + _elem, {idx, jdx, w, sample} -> + {idx + 1, jdx, w, sample} + end + + {size, _, _, sample} = reduce(enumerable, {0, count - 1, 1.0, %{}}, reducer) + take_mapped(sample, Kernel.min(count, size), []) + end + + @compile {:inline, take_jdx_w: 3, take_index: 1} + defp take_jdx_w(idx, w, count) do + w = w * :math.exp(:math.log(:rand.uniform()) / count) + jdx = idx + floor(:math.log(:rand.uniform()) / :math.log(1 - w)) + 1 + {jdx, w} + end + + defp take_index(0), do: 0 + defp take_index(idx), do: :rand.uniform(idx + 1) - 1 + + defp take_tupled(_sample, 0, acc), do: acc + + defp take_tupled(sample, position, acc) do + position = position - 1 + take_tupled(sample, position, [elem(sample, position) | acc]) + end + + defp take_mapped(_sample, 0, acc), do: acc + + defp take_mapped(sample, position, acc) do + position = position - 1 + take_mapped(sample, position, [Map.fetch!(sample, position) | acc]) + end + + @doc """ + Takes the elements from the beginning of the `enumerable` while `fun` returns + a truthy value. + + ## Examples + + iex> Enum.take_while([1, 2, 3], fn x -> x < 3 end) + [1, 2] + + """ + @spec take_while(t, (element -> as_boolean(term))) :: list + def take_while(enumerable, fun) when is_list(enumerable) do + take_while_list(enumerable, fun) + end + + def take_while(enumerable, fun) do + {_, res} = + Enumerable.reduce(enumerable, {:cont, []}, fn entry, acc -> + if fun.(entry) do + {:cont, [entry | acc]} + else + {:halt, acc} + end + end) + + :lists.reverse(res) + end + + @doc """ + Converts `enumerable` to a list. + + ## Examples + + iex> Enum.to_list(1..3) + [1, 2, 3] + + """ + @spec to_list(t) :: [element] + def to_list(enumerable) when is_list(enumerable), do: enumerable + def to_list(%{__struct__: Range} = range), do: Range.to_list(range) + def to_list(%_{} = enumerable), do: reverse(enumerable) |> :lists.reverse() + def to_list(%{} = enumerable), do: Map.to_list(enumerable) + def to_list(enumerable), do: reverse(enumerable) |> :lists.reverse() + + @doc """ + Enumerates the `enumerable`, removing all duplicate elements. + + The first occurrence of each element is kept and all following + duplicates are removed. The overall order is preserved. + + ## Examples + + iex> Enum.uniq([1, 2, 3, 3, 2, 1]) + [1, 2, 3] + + """ + @spec uniq(t) :: list + def uniq(enumerable) do + uniq_by(enumerable, fn x -> x end) + end + + @doc false + @deprecated "Use Enum.uniq_by/2 instead" + def uniq(enumerable, fun) do + uniq_by(enumerable, fun) + end + + @doc """ + Enumerates the `enumerable`, by removing the elements for which + function `fun` returned duplicate elements. + + The function `fun` maps every element to a term. Two elements are + considered duplicates if the return value of `fun` is equal for + both of them. + + The first occurrence of each element is kept and all following + duplicates are removed. The overall order is preserved. + + ## Example + + iex> Enum.uniq_by([{1, :x}, {2, :y}, {1, :z}], fn {x, _} -> x end) + [{1, :x}, {2, :y}] + + iex> Enum.uniq_by([a: {:tea, 2}, b: {:tea, 2}, c: {:coffee, 1}], fn {_, y} -> y end) + [a: {:tea, 2}, c: {:coffee, 1}] + + """ + @spec uniq_by(t, (element -> term)) :: list + + def uniq_by(enumerable, fun) when is_list(enumerable) do + uniq_list(enumerable, %{}, fun) + end + + def uniq_by(enumerable, fun) do + {list, _} = reduce(enumerable, {[], %{}}, R.uniq_by(fun)) + :lists.reverse(list) + end + + @doc """ + Opposite of `zip/2`. Extracts two-element tuples from the + given `enumerable` and groups them together. + + It takes an `enumerable` with elements being two-element tuples and returns + a tuple with two lists, each of which is formed by the first and + second element of each tuple, respectively. + + This function fails unless `enumerable` is or can be converted into a + list of tuples with *exactly* two elements in each tuple. + + ## Examples + + iex> Enum.unzip([{:a, 1}, {:b, 2}, {:c, 3}]) + {[:a, :b, :c], [1, 2, 3]} + + """ + @spec unzip(t) :: {[element], [element]} + def unzip(enumerable) + + def unzip([_ | _] = list) do + :lists.reverse(list) |> unzip([], []) + end + + def unzip([]) do + {[], []} + end + + def unzip(enumerable) do + {list1, list2} = + reduce(enumerable, {[], []}, fn {el1, el2}, {list1, list2} -> + {[el1 | list1], [el2 | list2]} + end) {:lists.reverse(list1), :lists.reverse(list2)} end - def split(collection, count) when count < 0 do - do_split_reverse(reverse(collection), abs(count), []) + defp unzip([{el1, el2} | reversed_list], list1, list2) do + unzip(reversed_list, [el1 | list1], [el2 | list2]) + end + + defp unzip([], list1, list2) do + {list1, list2} + end + + @doc """ + Returns the `enumerable` with each element wrapped in a tuple + alongside its index or according to a given function. + + If an integer offset is given as `fun_or_offset`, it will index from the given + offset instead of from zero. + + If a 2-arity function is given as `fun_or_offset`, the function will be invoked + for each element in `enumerable` as the first argument and with a zero-based + index as the second. `with_index/2` returns a list with the result of each invocation. + + ## Examples + + iex> Enum.with_index([:a, :b, :c]) + [a: 0, b: 1, c: 2] + + iex> Enum.with_index([:a, :b, :c], 3) + [a: 3, b: 4, c: 5] + + iex> Enum.with_index([:a, :b, :c], fn element, index -> {index, element} end) + [{0, :a}, {1, :b}, {2, :c}] + + """ + @spec with_index(t, integer) :: [{term, integer}] + @spec with_index(t, (element, index -> value)) :: [value] when value: any + def with_index(enumerable, fun_or_offset \\ 0) + + def with_index(enumerable, offset) when is_list(enumerable) and is_integer(offset) do + with_index_list(enumerable, offset) + end + + def with_index(enumerable, fun) when is_list(enumerable) and is_function(fun, 2) do + with_index_list(enumerable, 0, fun) + end + + def with_index(enumerable, offset) when is_integer(offset) do + enumerable + |> map_reduce(offset, fn x, i -> {{x, i}, i + 1} end) + |> elem(0) + end + + def with_index(enumerable, fun) when is_function(fun, 2) do + enumerable + |> map_reduce(0, fn x, i -> {fun.(x, i), i + 1} end) + |> elem(0) + end + + @doc """ + Zips corresponding elements from two enumerables into a list + of tuples. + + Because a list of two-element tuples with atoms as the first + tuple element is a keyword list (`Keyword`), zipping a first list + of atoms with a second list of any kind creates a keyword list. + + The zipping finishes as soon as either enumerable completes. + + ## Examples + + iex> Enum.zip([1, 2, 3], [:a, :b, :c]) + [{1, :a}, {2, :b}, {3, :c}] + + iex> Enum.zip([:a, :b, :c], [1, 2, 3]) + [a: 1, b: 2, c: 3] + + iex> Enum.zip([1, 2, 3, 4, 5], [:a, :b, :c]) + [{1, :a}, {2, :b}, {3, :c}] + + """ + @spec zip(t, t) :: [{any, any}] + def zip(enumerable1, enumerable2) when is_list(enumerable1) and is_list(enumerable2) do + zip_list(enumerable1, enumerable2, []) + end + + def zip(enumerable1, enumerable2) do + zip([enumerable1, enumerable2]) + end + + @doc """ + Zips corresponding elements from a finite collection of enumerables + into a list of tuples. + + The zipping finishes as soon as any enumerable in the given collection completes. + + ## Examples + + iex> Enum.zip([[1, 2, 3], [:a, :b, :c], ["foo", "bar", "baz"]]) + [{1, :a, "foo"}, {2, :b, "bar"}, {3, :c, "baz"}] + + iex> Enum.zip([[1, 2, 3, 4, 5], [:a, :b, :c]]) + [{1, :a}, {2, :b}, {3, :c}] + + """ + @doc since: "1.4.0" + @spec zip(enumerables) :: [tuple()] when enumerables: [t()] | t() + def zip([]), do: [] + + def zip(enumerables) do + zip_reduce(enumerables, [], &[List.to_tuple(&1) | &2]) + |> :lists.reverse() + end + + @doc """ + Zips corresponding elements from two enumerables into a list, transforming them with + the `zip_fun` function as it goes. + + The corresponding elements from each collection are passed to the provided two-arity `zip_fun` + function in turn. Returns a list that contains the result of calling `zip_fun` for each pair of + elements. + + The zipping finishes as soon as either enumerable runs out of elements. + + ## Zipping Maps + + It's important to remember that zipping inherently relies on order. + If you zip two lists you get the element at the index from each list in turn. + If we zip two maps together it's tempting to think that you will get the given + key in the left map and the matching key in the right map, but there is no such + guarantee because map keys are not ordered! Consider the following: + + left = %{:a => 1, 1 => 3} + right = %{:a => 1, :b => :c} + Enum.zip(left, right) + #=> [{{1, 3}, {:a, 1}}, {{:a, 1}, {:b, :c}}] + + As you can see `:a` does not get paired with `:a`. If this is what you want, + you should use `Map.merge/3`. + + ## Examples + + iex> Enum.zip_with([1, 2], [3, 4], fn x, y -> x + y end) + [4, 6] + + iex> Enum.zip_with([1, 2], [3, 4, 5, 6], fn x, y -> x + y end) + [4, 6] + + iex> Enum.zip_with([1, 2, 5, 6], [3, 4], fn x, y -> x + y end) + [4, 6] + + """ + @doc since: "1.12.0" + @spec zip_with(t, t, (enum1_elem :: term, enum2_elem :: term -> term)) :: [term] + def zip_with(enumerable1, enumerable2, zip_fun) + when is_list(enumerable1) and is_list(enumerable2) and is_function(zip_fun, 2) do + zip_with_list(enumerable1, enumerable2, zip_fun) + end + + def zip_with(enumerable1, enumerable2, zip_fun) when is_function(zip_fun, 2) do + zip_reduce(enumerable1, enumerable2, [], fn l, r, acc -> [zip_fun.(l, r) | acc] end) + |> :lists.reverse() + end + + @doc """ + Zips corresponding elements from a finite collection of enumerables + into list, transforming them with the `zip_fun` function as it goes. + + The first element from each of the enums in `enumerables` will be put + into a list which is then passed to the one-arity `zip_fun` function. + Then, the second elements from each of the enums are put into a list + and passed to `zip_fun`, and so on until any one of the enums in + `enumerables` runs out of elements. + + Returns a list with all the results of calling `zip_fun`. + + ## Examples + + iex> Enum.zip_with([[1, 2], [3, 4], [5, 6]], fn [x, y, z] -> x + y + z end) + [9, 12] + + iex> Enum.zip_with([[1, 2], [3, 4]], fn [x, y] -> x + y end) + [4, 6] + + """ + @doc since: "1.12.0" + @spec zip_with(t, ([term] -> term)) :: [term] + def zip_with([], _fun), do: [] + + def zip_with(enumerables, zip_fun) do + zip_reduce(enumerables, [], fn values, acc -> [zip_fun.(values) | acc] end) + |> :lists.reverse() + end + + @doc """ + Reduces over two enumerables halting as soon as either enumerable is empty. + + In practice, the behavior provided by this function can be achieved with: + + Enum.reduce(Stream.zip(left, right), acc, reducer) + + But `zip_reduce/4` exists for convenience purposes. + + ## Examples + + iex> Enum.zip_reduce([1, 2], [3, 4], 0, fn x, y, acc -> x + y + acc end) + 10 + + If one of the lists has more entries than the others, + those entries are discarded: + + iex> Enum.zip_reduce([1, 2, 3], [4, 5], [], fn x, y, acc -> [x + y | acc] end) + [7, 5] + """ + @doc since: "1.12.0" + @spec zip_reduce(t, t, acc, (enum1_elem :: term, enum2_elem :: term, acc -> acc)) :: acc + when acc: term + def zip_reduce(left, right, acc, reducer) + when is_list(left) and is_list(right) and is_function(reducer, 3) do + zip_reduce_list(left, right, acc, reducer) + end + + def zip_reduce(left, right, acc, reducer) when is_function(reducer, 3) do + reduce = fn [l, r], acc -> {:cont, reducer.(l, r, acc)} end + R.zip_with([left, right], & &1).({:cont, acc}, reduce) |> elem(1) + end + + @doc """ + Reduces over all of the given enumerables, halting as soon as any enumerable is + empty. + + The reducer will receive 2 args: a list of elements (one from each enum) and the + accumulator. + + In practice, the behavior provided by this function can be achieved with: + + Enum.reduce(Stream.zip(enums), acc, reducer) + + But `zip_reduce/3` exists for convenience purposes. + + ## Examples + + iex> enums = [[1, 1], [2, 2], [3, 3]] + ...> Enum.zip_reduce(enums, [], fn elements, acc -> + ...> [List.to_tuple(elements) | acc] + ...> end) + [{1, 2, 3}, {1, 2, 3}] + + If one of the lists has more entries than the others, + those entries are discarded: + + iex> enums = [[1, 2], [a: 3, b: 4], [5, 6, 7]] + ...> Enum.zip_reduce(enums, [], fn elements, acc -> + ...> [List.to_tuple(elements) | acc] + ...> end) + [{2, {:b, 4}, 6}, {1, {:a, 3}, 5}] + """ + @doc since: "1.12.0" + @spec zip_reduce(t, acc, ([term], acc -> acc)) :: acc when acc: term + def zip_reduce([], acc, reducer) when is_function(reducer, 2), do: acc + + def zip_reduce(enumerables, acc, reducer) when is_function(reducer, 2) do + R.zip_with(enumerables, & &1).({:cont, acc}, &{:cont, reducer.(&1, &2)}) |> elem(1) + end + + ## Helpers + + @compile {:inline, + entry_to_string: 1, + reduce: 3, + reduce_by: 3, + reduce_enumerable: 3, + reduce_range: 5, + map_range: 4} + + defp entry_to_string(entry) when is_binary(entry), do: entry + defp entry_to_string(entry), do: String.Chars.to_string(entry) + + defp aggregate([head | tail], fun, _empty) do + aggregate_list(tail, head, fun) + end + + defp aggregate([], _fun, empty) do + empty.() + end + + defp aggregate(first..last//step = range, fun, empty) do + case Range.size(range) do + 0 -> + empty.() + + _ -> + last = last - rem(last - first, step) + + case fun.(first, last) do + true -> first + false -> last + end + end + end + + defp aggregate(enumerable, fun, empty) do + ref = make_ref() + + enumerable + |> reduce(ref, fn + element, ^ref -> + element + + element, acc -> + case fun.(acc, element) do + true -> acc + false -> element + end + end) + |> case do + ^ref -> empty.() + result -> result + end + end + + defp aggregate_list([head | tail], acc, fun) do + acc = + case fun.(acc, head) do + true -> acc + false -> head + end + + aggregate_list(tail, acc, fun) + end + + defp aggregate_list([], acc, _fun), do: acc + + defp aggregate_by(enumerable, fun, sorter, empty_fallback) do + first_fun = &[&1 | fun.(&1)] + + reduce_fun = fn entry, [_ | fun_ref] = old -> + fun_entry = fun.(entry) + + case sorter.(fun_ref, fun_entry) do + true -> old + false -> [entry | fun_entry] + end + end + + case reduce_by(enumerable, first_fun, reduce_fun) do + :empty -> empty_fallback.() + [entry | _] -> entry + end + end + + defp reduce_by([head | tail], first, fun) do + :lists.foldl(fun, first.(head), tail) + end + + defp reduce_by([], _first, _fun) do + :empty + end + + defp reduce_by(enumerable, first, fun) do + reduce(enumerable, :empty, fn + element, :empty -> first.(element) + element, acc -> fun.(element, acc) + end) + end + + ## Implementations + + ## all?/1 + + defp all_list([h | t]) do + if h do + all_list(t) + else + false + end + end + + defp all_list([]) do + true + end + + ## any?/1 + + defp any_list([h | t]) do + if h do + true + else + any_list(t) + end + end + + defp any_list([]) do + false end - @doc """ - Splits `collection` in two while `fun` returns `true`. - - ## Examples + ## any?/2 all?/2 - iex> Enum.split_while([1, 2, 3, 4], fn(x) -> x < 3 end) - {[1, 2], [3, 4]} + defp predicate_list([h | t], initial, fun) do + if !!fun.(h) == initial do + predicate_list(t, initial, fun) + else + not initial + end + end - """ - @spec split_while(t, (element -> as_boolean(term))) :: {list, list} - def split_while(collection, fun) when is_list(collection) do - do_split_while(collection, fun, []) + defp predicate_list([], initial, _) do + initial end - def split_while(collection, fun) do - {list1, list2} = - reduce(collection, {[], []}, fn - entry, {acc1, []} -> - if(fun.(entry), do: {[entry|acc1], []}, else: {acc1, [entry]}) - entry, {acc1, acc2} -> - {acc1, [entry|acc2]} - end) + defp predicate_range(first, last, step, initial, fun) + when step > 0 and first <= last + when step < 0 and first >= last do + if !!fun.(first) == initial do + predicate_range(first + step, last, step, initial, fun) + else + not initial + end + end - {:lists.reverse(list1), :lists.reverse(list2)} + defp predicate_range(_first, _last, _step, initial, _fun) do + initial end - @doc """ - Takes the first `count` items from the collection. + ## concat - If a negative `count` is given, the last `count` values will - be taken. For such, the collection is fully enumerated keeping up - to `2 * count` elements in memory. Once the end of the collection is - reached, the last `count` elements are returned. + defp concat_list([h | t]) when is_list(h), do: h ++ concat_list(t) + defp concat_list([h | t]), do: concat_enum([h | t]) + defp concat_list([]), do: [] - ## Examples + defp concat_enum(enum) do + fun = &[&1 | &2] + enum |> reduce([], &reduce(&1, &2, fun)) |> :lists.reverse() + end - iex> Enum.take([1, 2, 3], 2) - [1,2] + # count_until - iex> Enum.take([1, 2, 3], 10) - [1,2,3] + @compile {:inline, count_until_list: 3} - iex> Enum.take([1, 2, 3], 0) - [] + defp count_until_list([], _limit, acc), do: acc - iex> Enum.take([1, 2, 3], -1) - [3] + defp count_until_list([_head | tail], limit, acc) do + case acc + 1 do + ^limit -> limit + acc -> count_until_list(tail, limit, acc) + end + end - """ - @spec take(t, integer) :: list + defp count_until_enum(enumerable, limit) do + case Enumerable.count(enumerable) do + {:ok, value} -> + Kernel.min(value, limit) - def take(_collection, 0) do - [] + {:error, module} -> + module.reduce(enumerable, {:cont, 0}, fn _entry, acc -> + case acc + 1 do + ^limit -> {:halt, limit} + acc -> {:cont, acc} + end + end) + |> elem(1) + end end - def take(collection, count) when is_list(collection) and count > 0 do - do_take(collection, count) - end + @compile {:inline, count_until_list: 4} - def take(collection, count) when count > 0 do - {_, {res, _}} = - Enumerable.reduce(collection, {:cont, {[], count}}, fn(entry, {list, count}) -> - if count > 1 do - {:cont, {[entry|list], count - 1}} - else - {:halt, {[entry|list], count}} - end - end) - :lists.reverse(res) + defp count_until_list([], _fun, _limit, acc), do: acc + + defp count_until_list([head | tail], fun, limit, acc) do + if fun.(head) do + case acc + 1 do + ^limit -> limit + acc -> count_until_list(tail, fun, limit, acc) + end + else + count_until_list(tail, fun, limit, acc) + end end - def take(collection, count) when count < 0 do - Stream.take(collection, count).({:cont, []}, &{:cont, [&1|&2]}) - |> elem(1) |> :lists.reverse + defp count_until_enum(enumerable, fun, limit) do + Enumerable.reduce(enumerable, {:cont, 0}, fn entry, acc -> + if fun.(entry) do + case acc + 1 do + ^limit -> {:halt, limit} + acc -> {:cont, acc} + end + else + {:cont, acc} + end + end) + |> elem(1) end - @doc """ - Returns a collection of every `nth` item in the collection, - starting with the first element. + # dedup - ## Examples + defp dedup_list([value | tail], acc) do + acc = + case acc do + [^value | _] -> acc + _ -> [value | acc] + end - iex> Enum.take_every(1..10, 2) - [1, 3, 5, 7, 9] + dedup_list(tail, acc) + end - """ - @spec take_every(t, integer) :: list - def take_every(_collection, 0), do: [] - def take_every(collection, nth) do - {_, {res, _}} = - Enumerable.reduce(collection, {:cont, {[], :first}}, R.take_every(nth)) - :lists.reverse(res) + defp dedup_list([], acc) do + acc end - @doc """ - Takes the items at the beginning of `collection` while `fun` returns `true`. + ## drop - ## Examples + defp drop_list(list, 0), do: list + defp drop_list([_ | tail], counter), do: drop_list(tail, counter - 1) + defp drop_list([], _), do: [] - iex> Enum.take_while([1, 2, 3], fn(x) -> x < 3 end) - [1, 2] + ## drop_while - """ - @spec take_while(t, (element -> as_boolean(term))) :: list - def take_while(collection, fun) when is_list(collection) do - do_take_while(collection, fun) + defp drop_while_list([head | tail], fun) do + if fun.(head) do + drop_while_list(tail, fun) + else + [head | tail] + end end - def take_while(collection, fun) do - Enumerable.reduce(collection, {:cont, []}, R.take_while(fun)) - |> elem(1) |> :lists.reverse + defp drop_while_list([], _) do + [] end - @doc """ - Convert `collection` to a list. - - ## Examples + ## filter - iex> Enum.to_list(1 .. 3) - [1, 2, 3] + defp filter_list([head | tail], fun) do + if fun.(head) do + [head | filter_list(tail, fun)] + else + filter_list(tail, fun) + end + end - """ - @spec to_list(t) :: [term] - def to_list(collection) when is_list(collection) do - collection + defp filter_list([], _fun) do + [] end - def to_list(collection) do - reverse(collection) |> :lists.reverse + ## find + + defp find_list([head | tail], default, fun) do + if fun.(head) do + head + else + find_list(tail, default, fun) + end end + defp find_list([], default, _) do + default + end - @doc """ - Traverses the given enumerable keeping its shape. + ## find_index - It also expects the enumerable to implement the `Collectable` protocol. + defp find_index_list([head | tail], counter, fun) do + if fun.(head) do + counter + else + find_index_list(tail, counter + 1, fun) + end + end - ## Examples + defp find_index_list([], _, _) do + nil + end - iex> Enum.traverse(%{a: 1, b: 2}, fn {k, v} -> {k, v * 2} end) - %{a: 2, b: 4} + ## find_value - """ - @spec traverse(Enumerable.t, (term -> term)) :: Collectable.t - def traverse(collection, transform) when is_list(collection) do - :lists.map(transform, collection) + defp find_value_list([head | tail], default, fun) do + fun.(head) || find_value_list(tail, default, fun) end - def traverse(collection, transform) do - into(collection, Collectable.empty(collection), transform) + defp find_value_list([], default, _) do + default end - @doc """ - Enumerates the collection, removing all duplicated items. + ## flat_map - ## Examples + defp flat_map_list([head | tail], fun) do + case fun.(head) do + # the two first clauses are an optimization + [] -> flat_map_list(tail, fun) + [elem] -> [elem | flat_map_list(tail, fun)] + list when is_list(list) -> list ++ flat_map_list(tail, fun) + other -> to_list(other) ++ flat_map_list(tail, fun) + end + end - iex> Enum.uniq([1, 2, 3, 2, 1]) - [1, 2, 3] + defp flat_map_list([], _fun) do + [] + end - iex> Enum.uniq([{1, :x}, {2, :y}, {1, :z}], fn {x, _} -> x end) - [{1,:x}, {2,:y}] + ## intersperse - """ - @spec uniq(t) :: list - @spec uniq(t, (element -> term)) :: list - def uniq(collection, fun \\ fn x -> x end) + defp intersperse_non_empty_list([head], _separator), do: [head] - def uniq(collection, fun) when is_list(collection) do - do_uniq(collection, [], fun) + defp intersperse_non_empty_list([head | rest], separator) do + [head, separator | intersperse_non_empty_list(rest, separator)] end - def uniq(collection, fun) do - {_, {list, _}} = - Enumerable.reduce(collection, {:cont, {[], []}}, R.uniq(fun)) - :lists.reverse(list) - end + ## join - @doc """ - Zips corresponding elements from two collections into one list - of tuples. + defp join_list([], _joiner), do: "" - The zipping finishes as soon as any enumerable completes. + defp join_list(list, joiner) do + join_non_empty_list(list, joiner, []) + |> :lists.reverse() + |> IO.iodata_to_binary() + end - ## Examples + defp join_non_empty_list([first], _joiner, acc), do: [entry_to_string(first) | acc] - iex> Enum.zip([1, 2, 3], [:a, :b, :c]) - [{1,:a},{2,:b},{3,:c}] + defp join_non_empty_list([first | rest], joiner, acc) do + join_non_empty_list(rest, joiner, [joiner, entry_to_string(first) | acc]) + end - iex> Enum.zip([1,2,3,4,5], [:a, :b, :c]) - [{1,:a},{2,:b},{3,:c}] + ## map - """ - @spec zip(t, t) :: [{any, any}] - def zip(coll1, coll2) when is_list(coll1) and is_list(coll2) do - do_zip(coll1, coll2) + defp map_range(first, last, step, fun) + when step > 0 and first <= last + when step < 0 and first >= last do + [fun.(first) | map_range(first + step, last, step, fun)] end - def zip(coll1, coll2) do - Stream.zip(coll1, coll2).({:cont, []}, &{:cont, [&1|&2]}) |> elem(1) |> :lists.reverse + defp map_range(_first, _last, _step, _fun) do + [] end - @doc """ - Returns the collection with each element wrapped in a tuple - alongside its index. - - ## Examples + ## map_intersperse - iex> Enum.with_index [1,2,3] - [{1,0},{2,1},{3,2}] + defp map_intersperse_list([], _, _), + do: [] - """ - @spec with_index(t) :: list({element, non_neg_integer}) - def with_index(collection) do - map_reduce(collection, 0, fn x, acc -> - {{x, acc}, acc + 1} - end) |> elem(0) - end + defp map_intersperse_list([last], _, mapper), + do: [mapper.(last)] - ## Helpers + defp map_intersperse_list([head | rest], separator, mapper), + do: [mapper.(head), separator | map_intersperse_list(rest, separator, mapper)] - @compile {:inline, enum_to_string: 1} + ## reduce - defp enumerate_and_count(collection, count) when is_list(collection) do - {collection, length(collection) - abs(count)} + defp reduce_range(first, last, step, acc, fun) + when step > 0 and first <= last + when step < 0 and first >= last do + reduce_range(first + step, last, step, fun.(first, acc), fun) end - defp enumerate_and_count(collection, count) do - map_reduce(collection, -abs(count), fn(x, acc) -> {x, acc + 1} end) + defp reduce_range(_first, _last, _step, acc, _fun) do + acc end - defp enum_to_string(entry) when is_binary(entry), do: entry - defp enum_to_string(entry), do: String.Chars.to_string(entry) - - ## Implementations + defp reduce_enumerable(enumerable, acc, fun) do + Enumerable.reduce(enumerable, {:cont, acc}, fn x, acc -> {:cont, fun.(x, acc)} end) |> elem(1) + end - ## all? + ## reject - defp do_all?([h|t], fun) do - if fun.(h) do - do_all?(t, fun) + defp reject_list([head | tail], fun) do + if fun.(head) do + reject_list(tail, fun) else - false + [head | reject_list(tail, fun)] end end - defp do_all?([], _) do - true + defp reject_list([], _fun) do + [] end - ## any? + ## reverse_slice - defp do_any?([h|t], fun) do - if fun.(h) do - true - else - do_any?(t, fun) - end + defp reverse_slice(rest, idx, idx, count, acc) do + {slice, rest} = head_slice(rest, count, []) + :lists.reverse(rest, :lists.reverse(slice, acc)) end - defp do_any?([], _) do - false + defp reverse_slice([elem | rest], idx, start, count, acc) do + reverse_slice(rest, idx - 1, start, count, [elem | acc]) end - ## fetch + defp head_slice(rest, 0, acc), do: {acc, rest} - defp do_fetch([h|_], 0), do: {:ok, h} - defp do_fetch([_|t], n), do: do_fetch(t, n - 1) - defp do_fetch([], _), do: :error + defp head_slice([elem | rest], count, acc) do + head_slice(rest, count - 1, [elem | acc]) + end - ## drop + ## scan - defp do_drop([_|t], counter) when counter > 0 do - do_drop(t, counter - 1) - end + defp scan_list([], _acc, _fun), do: [] - defp do_drop(list, 0) do - list + defp scan_list([elem | rest], acc, fun) do + acc = fun.(elem, acc) + [acc | scan_list(rest, acc, fun)] end - defp do_drop([], _) do - [] - end + ## slice - ## drop_while + defp slice_forward(enumerable, start, amount, step) when start < 0 do + {count, fun} = slice_count_and_fun(enumerable, step) + start = count + start - defp do_drop_while([h|t], fun) do - if fun.(h) do - do_drop_while(t, fun) + if start >= 0 do + amount = Kernel.min(amount, count - start) + amount = amount_with_step(amount, step) + fun.(start, amount, step) else - [h|t] + [] end end - defp do_drop_while([], _) do - [] + defp slice_forward(list, start, amount, step) when is_list(list) do + amount = amount_with_step(amount, step) + slice_list(list, start, amount, step) end - ## find + defp slice_forward(enumerable, start, amount, step) do + case Enumerable.slice(enumerable) do + {:ok, count, _} when start >= count -> + [] - defp do_find([h|t], ifnone, fun) do - if fun.(h) do - h - else - do_find(t, ifnone, fun) - end - end + {:ok, count, fun} when is_function(fun, 1) -> + amount = Kernel.min(amount, count - start) |> amount_with_step(step) + enumerable |> fun.() |> slice_exact(start, amount, step, count) - defp do_find([], ifnone, _) do - ifnone - end + {:ok, count, fun} when is_function(fun, 3) -> + amount = Kernel.min(amount, count - start) |> amount_with_step(step) + fun.(start, amount, step) - ## find_index + # TODO: Remove me on v2.0 + {:ok, count, fun} when is_function(fun, 2) -> + IO.warn( + "#{inspect(Enumerable.impl_for(enumerable))} must return a three arity function on slice/1" + ) - defp do_find_index([h|t], counter, fun) do - if fun.(h) do - counter + amount = Kernel.min(amount, count - start) + + if step == 1 do + fun.(start, amount) + else + fun.(start, Kernel.min(amount * step, count - start)) + |> take_every_list(amount, step - 1) + end + + {:error, module} -> + slice_enum(enumerable, module, start, amount, step) + end + end + + defp slice_list(list, start, amount, step) do + if step == 1 do + list |> drop_list(start) |> take_list(amount) else - do_find_index(t, counter + 1, fun) + list |> drop_list(start) |> take_every_list(amount, step - 1) end end - defp do_find_index([], _, _) do - nil + defp slice_enum(enumerable, module, start, amount, 1) do + {_, {_, _, slice}} = + module.reduce(enumerable, {:cont, {start, amount, []}}, fn + _entry, {start, amount, _list} when start > 0 -> + {:cont, {start - 1, amount, []}} + + entry, {start, amount, list} when amount > 1 -> + {:cont, {start, amount - 1, [entry | list]}} + + entry, {start, amount, list} -> + {:halt, {start, amount, [entry | list]}} + end) + + :lists.reverse(slice) end - ## find_value + defp slice_enum(enumerable, module, start, amount, step) do + {_, {_, _, _, slice}} = + module.reduce(enumerable, {:cont, {start, amount, 1, []}}, fn + _entry, {start, amount, to_drop, _list} when start > 0 -> + {:cont, {start - 1, amount, to_drop, []}} + + entry, {start, amount, to_drop, list} when amount > 1 -> + case to_drop do + 1 -> {:cont, {start, amount - 1, step, [entry | list]}} + _ -> {:cont, {start, amount - 1, to_drop - 1, list}} + end + + entry, {start, amount, to_drop, list} -> + case to_drop do + 1 -> {:halt, {start, amount, to_drop, [entry | list]}} + _ -> {:halt, {start, amount, to_drop, list}} + end + end) - defp do_find_value([h|t], ifnone, fun) do - fun.(h) || do_find_value(t, ifnone, fun) + :lists.reverse(slice) end - defp do_find_value([], ifnone, _) do - ifnone + defp slice_count_and_fun(list, _step) when is_list(list) do + length = length(list) + {length, &slice_exact(list, &1, &2, &3, length)} end - ## shuffle + defp slice_count_and_fun(enumerable, step) do + case Enumerable.slice(enumerable) do + {:ok, count, fun} when is_function(fun, 1) -> + {count, &slice_exact(fun.(enumerable), &1, &2, &3, count)} + + {:ok, count, fun} when is_function(fun, 3) -> + {count, fun} + + # TODO: Remove me on v2.0 + {:ok, count, fun} when is_function(fun, 2) -> + IO.warn( + "#{inspect(Enumerable.impl_for(enumerable))} must return a three arity function on slice/1" + ) + + if step == 1 do + {count, fn start, amount, 1 -> fun.(start, amount) end} + else + {count, + fn start, amount, step -> + fun.(start, Kernel.min(amount * step, count - start)) + |> take_every_list(amount, step - 1) + end} + end - defp unwrap([{_, h} | collection], t) do - unwrap(collection, [h|t]) + {:error, module} -> + {list, count} = + enumerable + |> module.reduce({:cont, {[], 0}}, fn elem, {acc, count} -> + {:cont, {[elem | acc], count + 1}} + end) + |> elem(1) + + {count, + fn start, amount, step -> + list |> :lists.reverse() |> slice_exact(start, amount, step, count) + end} + end end - defp unwrap([], t), do: t + # Slice a list when we know the bounds + defp slice_exact(_list, _start, 0, _step, _), do: [] + + defp slice_exact(list, start, amount, 1, size) when start + amount == size, + do: list |> drop_exact(start) + + defp slice_exact(list, start, amount, 1, _), + do: list |> drop_exact(start) |> take_exact(amount) + + defp slice_exact(list, start, amount, step, _), + do: list |> drop_exact(start) |> take_every_list(amount, step - 1) + + defp drop_exact(list, 0), do: list + defp drop_exact([_ | tail], amount), do: drop_exact(tail, amount - 1) + + defp take_exact(_list, 0), do: [] + defp take_exact([head | tail], amount), do: [head | take_exact(tail, amount - 1)] ## sort defp sort_reducer(entry, {:split, y, x, r, rs, bool}, fun) do cond do fun.(y, entry) == bool -> - {:split, entry, y, [x|r], rs, bool} + {:split, entry, y, [x | r], rs, bool} + fun.(x, entry) == bool -> - {:split, y, entry, [x|r], rs, bool} + {:split, y, entry, [x | r], rs, bool} + r == [] -> {:split, y, x, [entry], rs, bool} + true -> {:pivot, y, x, r, rs, entry, bool} end @@ -1981,10 +4882,13 @@ defmodule Enum do cond do fun.(y, entry) == bool -> {:pivot, entry, y, [x | r], rs, s, bool} + fun.(x, entry) == bool -> {:pivot, y, entry, [x | r], rs, s, bool} + fun.(s, entry) == bool -> {:split, entry, s, [], [[y, x | r] | rs], bool} + true -> {:split, s, entry, [], [[y, x | r] | rs], bool} end @@ -1995,7 +4899,7 @@ defmodule Enum do end defp sort_reducer(entry, acc, _fun) do - [entry|acc] + [entry | acc] end defp sort_terminator({:split, y, x, r, rs, bool}, fun) do @@ -2010,214 +4914,306 @@ defmodule Enum do acc end - defp sort_merge(list, fun, true), do: - reverse_sort_merge(list, [], fun, true) + defp sort_merge(list, fun, true), do: reverse_sort_merge(list, [], fun, true) - defp sort_merge(list, fun, false), do: - sort_merge(list, [], fun, false) + defp sort_merge(list, fun, false), do: sort_merge(list, [], fun, false) + defp sort_merge([t1, [h2 | t2] | l], acc, fun, true), + do: sort_merge(l, [sort_merge1(t1, h2, t2, [], fun, false) | acc], fun, true) - defp sort_merge([t1, [h2 | t2] | l], acc, fun, true), do: - sort_merge(l, [sort_merge_1(t1, h2, t2, [], fun, false) | acc], fun, true) - - defp sort_merge([[h2 | t2], t1 | l], acc, fun, false), do: - sort_merge(l, [sort_merge_1(t1, h2, t2, [], fun, false) | acc], fun, false) + defp sort_merge([[h2 | t2], t1 | l], acc, fun, false), + do: sort_merge(l, [sort_merge1(t1, h2, t2, [], fun, false) | acc], fun, false) defp sort_merge([l], [], _fun, _bool), do: l - defp sort_merge([l], acc, fun, bool), do: - reverse_sort_merge([:lists.reverse(l, []) | acc], [], fun, bool) - - defp sort_merge([], acc, fun, bool), do: - reverse_sort_merge(acc, [], fun, bool) + defp sort_merge([l], acc, fun, bool), + do: reverse_sort_merge([:lists.reverse(l, []) | acc], [], fun, bool) + defp sort_merge([], acc, fun, bool), do: reverse_sort_merge(acc, [], fun, bool) - defp reverse_sort_merge([[h2 | t2], t1 | l], acc, fun, true), do: - reverse_sort_merge(l, [sort_merge_1(t1, h2, t2, [], fun, true) | acc], fun, true) + defp reverse_sort_merge([[h2 | t2], t1 | l], acc, fun, true), + do: reverse_sort_merge(l, [sort_merge1(t1, h2, t2, [], fun, true) | acc], fun, true) - defp reverse_sort_merge([t1, [h2 | t2] | l], acc, fun, false), do: - reverse_sort_merge(l, [sort_merge_1(t1, h2, t2, [], fun, true) | acc], fun, false) + defp reverse_sort_merge([t1, [h2 | t2] | l], acc, fun, false), + do: reverse_sort_merge(l, [sort_merge1(t1, h2, t2, [], fun, true) | acc], fun, false) - defp reverse_sort_merge([l], acc, fun, bool), do: - sort_merge([:lists.reverse(l, []) | acc], [], fun, bool) + defp reverse_sort_merge([l], acc, fun, bool), + do: sort_merge([:lists.reverse(l, []) | acc], [], fun, bool) - defp reverse_sort_merge([], acc, fun, bool), do: - sort_merge(acc, [], fun, bool) + defp reverse_sort_merge([], acc, fun, bool), do: sort_merge(acc, [], fun, bool) - - defp sort_merge_1([h1 | t1], h2, t2, m, fun, bool) do + defp sort_merge1([h1 | t1], h2, t2, m, fun, bool) do if fun.(h1, h2) == bool do - sort_merge_2(h1, t1, t2, [h2 | m], fun, bool) + sort_merge2(h1, t1, t2, [h2 | m], fun, bool) else - sort_merge_1(t1, h2, t2, [h1 | m], fun, bool) + sort_merge1(t1, h2, t2, [h1 | m], fun, bool) end end - defp sort_merge_1([], h2, t2, m, _fun, _bool), do: - :lists.reverse(t2, [h2 | m]) - + defp sort_merge1([], h2, t2, m, _fun, _bool), do: :lists.reverse(t2, [h2 | m]) - defp sort_merge_2(h1, t1, [h2 | t2], m, fun, bool) do + defp sort_merge2(h1, t1, [h2 | t2], m, fun, bool) do if fun.(h1, h2) == bool do - sort_merge_2(h1, t1, t2, [h2 | m], fun, bool) + sort_merge2(h1, t1, t2, [h2 | m], fun, bool) else - sort_merge_1(t1, h2, t2, [h1 | m], fun, bool) + sort_merge1(t1, h2, t2, [h1 | m], fun, bool) end end - defp sort_merge_2(h1, t1, [], m, _fun, _bool), do: - :lists.reverse(t1, [h1 | m]) + defp sort_merge2(h1, t1, [], m, _fun, _bool), do: :lists.reverse(t1, [h1 | m]) ## split - defp do_split([h|t], counter, acc) when counter > 0 do - do_split(t, counter - 1, [h|acc]) + defp split_list([head | tail], counter, acc) when counter > 0 do + split_list(tail, counter - 1, [head | acc]) end - defp do_split(list, 0, acc) do + defp split_list(list, 0, acc) do {:lists.reverse(acc), list} end - defp do_split([], _, acc) do + defp split_list([], _, acc) do {:lists.reverse(acc), []} end - defp do_split_reverse([h|t], counter, acc) when counter > 0 do - do_split_reverse(t, counter - 1, [h|acc]) + defp split_reverse_list([head | tail], counter, acc) when counter > 0 do + split_reverse_list(tail, counter - 1, [head | acc]) end - defp do_split_reverse(list, 0, acc) do + defp split_reverse_list(list, 0, acc) do {:lists.reverse(list), acc} end - defp do_split_reverse([], _, acc) do + defp split_reverse_list([], _, acc) do {[], acc} end ## split_while - defp do_split_while([h|t], fun, acc) do - if fun.(h) do - do_split_while(t, fun, [h|acc]) + defp split_while_list([head | tail], fun, acc) do + if fun.(head) do + split_while_list(tail, fun, [head | acc]) else - {:lists.reverse(acc), [h|t]} + {:lists.reverse(acc), [head | tail]} end end - defp do_split_while([], _, acc) do + defp split_while_list([], _, acc) do {:lists.reverse(acc), []} end + ## sum_by + + defp sum_by_list([], _, acc), do: acc + defp sum_by_list([h | t], mapper, acc), do: sum_by_list(t, mapper, acc + mapper.(h)) + + ## product_by + + defp product_by_list([], _, acc), do: acc + defp product_by_list([h | t], mapper, acc), do: product_by_list(t, mapper, acc * mapper.(h)) + ## take - defp do_take([h|t], counter) when counter > 0 do - [h|do_take(t, counter - 1)] - end + defp take_list(_list, 0), do: [] + defp take_list([head | tail], counter), do: [head | take_list(tail, counter - 1)] + defp take_list([], _counter), do: [] - defp do_take(_list, 0) do - [] - end + defp take_every_list([head | tail], to_drop), + do: [head | tail |> drop_list(to_drop) |> take_every_list(to_drop)] - defp do_take([], _) do - [] - end + defp take_every_list([], _to_drop), do: [] + + defp take_every_list(_list, 0, _to_drop), do: [] + + defp take_every_list([head | tail], counter, to_drop), + do: [head | tail |> drop_list(to_drop) |> take_every_list(counter - 1, to_drop)] + + defp take_every_list([], _counter, _to_drop), do: [] ## take_while - defp do_take_while([h|t], fun) do - if fun.(h) do - [h|do_take_while(t, fun)] + defp take_while_list([head | tail], fun) do + if fun.(head) do + [head | take_while_list(tail, fun)] else [] end end - defp do_take_while([], _) do + defp take_while_list([], _) do [] end ## uniq - defp do_uniq([h|t], acc, fun) do - fun_h = fun.(h) - case :lists.member(fun_h, acc) do - true -> do_uniq(t, acc, fun) - false -> [h|do_uniq(t, [fun_h|acc], fun)] + defp uniq_list([head | tail], set, fun) do + value = fun.(head) + + case set do + %{^value => true} -> uniq_list(tail, set, fun) + %{} -> [head | uniq_list(tail, Map.put(set, value, true), fun)] end end - defp do_uniq([], _acc, _fun) do + defp uniq_list([], _set, _fun) do [] end - ## zip + ## with_index - defp do_zip([h1|next1], [h2|next2]) do - [{h1, h2}|do_zip(next1, next2)] + defp with_index_list([head | tail], offset) do + [{head, offset} | with_index_list(tail, offset + 1)] end - defp do_zip(_, []), do: [] - defp do_zip([], _), do: [] + defp with_index_list([], _offset), do: [] - ## slice - - defp do_slice([], _start, _count) do - [] + defp with_index_list([head | tail], offset, fun) do + [fun.(head, offset) | with_index_list(tail, offset + 1, fun)] end - defp do_slice(_list, _start, 0) do - [] + defp with_index_list([], _offset, _fun), do: [] + + ## zip + + defp zip_list([head1 | next1], [head2 | next2], acc) do + zip_list(next1, next2, [{head1, head2} | acc]) end - defp do_slice([h|t], 0, count) do - [h|do_slice(t, 0, count-1)] + defp zip_list([], _, acc), do: :lists.reverse(acc) + defp zip_list(_, [], acc), do: :lists.reverse(acc) + + defp zip_with_list([head1 | next1], [head2 | next2], fun) do + [fun.(head1, head2) | zip_with_list(next1, next2, fun)] end - defp do_slice([_|t], start, count) do - do_slice(t, start-1, count) + defp zip_with_list(_, [], _fun), do: [] + defp zip_with_list([], _, _fun), do: [] + + defp zip_reduce_list([head1 | next1], [head2 | next2], acc, fun) do + zip_reduce_list(next1, next2, fun.(head1, head2, acc), fun) end + + defp zip_reduce_list(_, [], acc, _fun), do: acc + defp zip_reduce_list([], _, acc, _fun), do: acc end defimpl Enumerable, for: List do - def reduce(_, {:halt, acc}, _fun), do: {:halted, acc} - def reduce(list, {:suspend, acc}, fun), do: {:suspended, acc, &reduce(list, &1, fun)} - def reduce([], {:cont, acc}, _fun), do: {:done, acc} - def reduce([h|t], {:cont, acc}, fun), do: reduce(t, fun.(h, acc), fun) - - def member?(_list, _value), - do: {:error, __MODULE__} - def count(_list), - do: {:error, __MODULE__} + def count(list), do: {:ok, length(list)} + + def member?(list, value), do: {:ok, :lists.member(value, list)} + + def slice([]), do: {:ok, 0, fn _, _, _ -> [] end} + def slice(_list), do: {:error, __MODULE__} + + def reduce(_list, {:halt, acc}, _fun), do: {:halted, acc} + def reduce(list, {:suspend, acc}, fun), do: {:suspended, acc, &reduce(list, &1, fun)} + def reduce([], {:cont, acc}, _fun), do: {:done, acc} + def reduce([head | tail], {:cont, acc}, fun), do: reduce(tail, fun.(head, acc), fun) end defimpl Enumerable, for: Map do - def reduce(map, acc, fun) do - do_reduce(:maps.to_list(map), acc, fun) + def count(map) do + {:ok, map_size(map)} end - defp do_reduce(_, {:halt, acc}, _fun), do: {:halted, acc} - defp do_reduce(list, {:suspend, acc}, fun), do: {:suspended, acc, &do_reduce(list, &1, fun)} - defp do_reduce([], {:cont, acc}, _fun), do: {:done, acc} - defp do_reduce([h|t], {:cont, acc}, fun), do: do_reduce(t, fun.(h, acc), fun) - def member?(map, {key, value}) do - {:ok, match?({:ok, ^value}, :maps.find(key, map))} + {:ok, match?(%{^key => ^value}, map)} end def member?(_map, _other) do {:ok, false} end - def count(map) do - {:ok, map_size(map)} + def slice(map) do + size = map_size(map) + {:ok, size, &:maps.to_list/1} + end + + def reduce(map, acc, fun) do + Enumerable.List.reduce(:maps.to_list(map), acc, fun) end end defimpl Enumerable, for: Function do - def reduce(function, acc, fun) when is_function(function, 2), - do: function.(acc, fun) - def member?(_function, _value), - do: {:error, __MODULE__} - def count(_function), - do: {:error, __MODULE__} + def count(_function), do: {:error, __MODULE__} + def member?(_function, _value), do: {:error, __MODULE__} + def slice(_function), do: {:error, __MODULE__} + + def reduce(function, acc, fun) when is_function(function, 2), do: function.(acc, fun) + + def reduce(function, _acc, _fun) do + raise Protocol.UndefinedError, + protocol: @protocol, + value: function, + description: "only anonymous functions of arity 2 are enumerable" + end +end + +defimpl Enumerable, for: Range do + def reduce(first..last//step, acc, fun) do + reduce(first, last, acc, fun, step) + end + + # TODO: Remove me on v2.0 + def reduce(%{__struct__: Range, first: first, last: last} = range, acc, fun) do + step = if first <= last, do: 1, else: -1 + reduce(Map.put(range, :step, step), acc, fun) + end + + defp reduce(_first, _last, {:halt, acc}, _fun, _step) do + {:halted, acc} + end + + defp reduce(first, last, {:suspend, acc}, fun, step) do + {:suspended, acc, &reduce(first, last, &1, fun, step)} + end + + defp reduce(first, last, {:cont, acc}, fun, step) + when step > 0 and first <= last + when step < 0 and first >= last do + reduce(first + step, last, fun.(first, acc), fun, step) + end + + defp reduce(_, _, {:cont, acc}, _fun, _up) do + {:done, acc} + end + + def member?(first..last//step, value) when is_integer(value) do + if step > 0 do + {:ok, first <= value and value <= last and rem(value - first, step) == 0} + else + {:ok, last <= value and value <= first and rem(value - first, step) == 0} + end + end + + # TODO: Remove me on v2.0 + def member?(%{__struct__: Range, first: first, last: last} = range, value) + when is_integer(value) do + step = if first <= last, do: 1, else: -1 + member?(Map.put(range, :step, step), value) + end + + def member?(_, _value) do + {:ok, false} + end + + def count(range) do + {:ok, Range.size(range)} + end + + def slice(first.._//step = range) do + {:ok, Range.size(range), &slice(first + &1 * step, step + &3 - 1, &2)} + end + + # TODO: Remove me on v2.0 + def slice(%{__struct__: Range, first: first, last: last} = range) do + step = if first <= last, do: 1, else: -1 + slice(Map.put(range, :step, step)) + end + + defp slice(current, _step, 1), do: [current] + + defp slice(current, step, remaining) when remaining > 1 do + [current | slice(current + step, step, remaining - 1)] + end end diff --git a/lib/elixir/lib/exception.ex b/lib/elixir/lib/exception.ex index 398f0f1f5a6..209980d5e0c 100644 --- a/lib/elixir/lib/exception.ex +++ b/lib/elixir/lib/exception.ex @@ -1,53 +1,110 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + defmodule Exception do @moduledoc """ - Functions to format throw/catch/exit and exceptions. + Functions for dealing with throw/catch/exit and exceptions. + + This module also defines the behaviour required by custom + exceptions. To define your own, see `defexception/1`. - Note that stacktraces in Elixir are updated on throw, - errors and exits. For example, at any given moment, - `System.stacktrace` will return the stacktrace for the - last throw/error/exit that ocurred in the current process. + ## Formatting functions - Do not rely on the particular format returned by the `format` + Several functions in this module help format exceptions. + Some of these functions expect the stacktrace as argument. + The stacktrace is typically available inside catch and + rescue by using the `__STACKTRACE__/0` variable. + + Do not rely on the particular format returned by the functions in this module. They may be changed in future releases in order to better suit Elixir's tool chain. In other words, - by using the functions in this module it is guarantee you will + by using the functions in this module it is guaranteed you will format exceptions as in the current Elixir version being used. """ - @typedoc "The exception type (as generated by defexception)" - @type t :: %{__struct__: module, __exception__: true} + @typedoc "The exception type" + @type t :: %{ + required(:__struct__) => module, + required(:__exception__) => true, + optional(atom) => any + } @typedoc "The kind handled by formatting functions" - @type kind :: :error | :exit | :throw | {:EXIT, pid} + @type kind :: :error | non_error_kind + @type non_error_kind :: :exit | :throw | {:EXIT, pid} @type stacktrace :: [stacktrace_entry] @type stacktrace_entry :: - {module, function, arity_or_args, location} | - {function, arity_or_args, location} + {module, atom, arity_or_args, location} + | {(... -> any), arity_or_args, location} + + @type arity_or_args :: non_neg_integer | list + @type location :: keyword - @typep arity_or_args :: non_neg_integer | list - @typep location :: Keyword.t + @doc """ + Receives the arguments given to `raise/2` and returns the exception struct. + The default implementation accepts either a set of keyword arguments + that is merged into the struct or a string to be used as the exception's message. + """ @callback exception(term) :: t - @callback message(t) :: String.t @doc """ - Returns true if the given argument is an exception. + Receives the exception struct and must return its message. + + Many exceptions have a message field which by default is accessed + by this function. However, if an exception does not have a message field, + this function must be explicitly implemented. + """ + @callback message(t) :: String.t() + + @doc """ + Called from `Exception.blame/3` to augment the exception struct. + + Can be used to collect additional information about the exception + or do some additional expensive computation. """ - def exception?(%{__struct__: struct, __exception__: true}) when is_atom(struct), do: true + @callback blame(t, stacktrace) :: {t, stacktrace} + @optional_callbacks [blame: 2] + + @doc false + # Callback for formatting Erlang exceptions + def format_error(%struct{} = exception, _stacktrace) do + %{general: message(exception), reason: "#" <> Atom.to_string(struct)} + end + + @doc false + @deprecated "Use Kernel.is_exception/1 instead" + def exception?(term) + def exception?(%_{__exception__: true}), do: true def exception?(_), do: false @doc """ - Gets the message for an exception. + Gets the message for an `exception`. + + This function will invoke the `c:message/1` callback on the exception + module to retrieve the message. If the callback raises an exception or + returns a non-binary value, this function will rescue the error and + return a descriptive error message instead. """ - def message(%{__struct__: module, __exception__: true} = exception) when is_atom(module) do + @spec message(t) :: String.t() + def message(%module{__exception__: true} = exception) do try do module.message(exception) rescue - e -> - raise ArgumentError, - "Got #{inspect e.__struct__} with message " <> - "\"#{message(e)}\" while retrieving message for #{inspect(exception)}" + caught_exception -> + "got #{inspect(caught_exception.__struct__)} with message " <> + "#{inspect(message(caught_exception))} while retrieving Exception.message/1 " <> + "for #{inspect(exception)}. Stacktrace:\n#{format_stacktrace(__STACKTRACE__)}" + else + result when is_binary(result) -> + result + + result -> + "got #{inspect(result)} " <> + "while retrieving Exception.message/1 for #{inspect(exception)} " <> + "(expected a string)" end end @@ -59,47 +116,29 @@ defmodule Exception do normalizes only `:error`, returning the untouched payload for others. - The third argument, a stacktrace, is optional. If it is - not supplied `System.stacktrace/0` will sometimes be used - to get additional information for the `kind` `:error`. If - the stacktrace is unknown and `System.stacktrace/0` would - not return the stacktrace corresponding to the exception - an empty stacktrace, `[]`, must be used. + The third argument is the stacktrace which is used to enrich + a normalized error with more information. It is only used when + the kind is an error. """ @spec normalize(:error, any, stacktrace) :: t - @spec normalize(kind, payload, stacktrace) :: payload when payload: var - - # Generating a stacktrace is expensive, default to nil - # to only fetch it when needed. - def normalize(kind, payload, stacktrace \\ nil) - - def normalize(:error, exception, stacktrace) do - if exception?(exception) do - exception - else - ErlangError.normalize(exception, stacktrace) - end - end - - def normalize(_kind, payload, _stacktrace) do - payload - end + @spec normalize(non_error_kind, payload, stacktrace) :: payload when payload: var + def normalize(kind, payload, stacktrace \\ []) + def normalize(:error, %_{__exception__: true} = payload, _stacktrace), do: payload + def normalize(:error, payload, stacktrace), do: ErlangError.normalize(payload, stacktrace) + def normalize(_kind, payload, _stacktrace), do: payload @doc """ - Normalizes and formats any throw, error and exit. + Normalizes and formats any throw/error/exit. The message is formatted and displayed in the same format as used by Elixir's CLI. - The third argument, a stacktrace, is optional. If it is - not supplied `System.stacktrace/0` will sometimes be used - to get additional information for the `kind` `:error`. If - the stacktrace is unknown and `System.stacktrace/0` would - not return the stacktrace corresponding to the exception - an empty stacktrace, `[]`, must be used. + The third argument is the stacktrace which is used to enrich + a normalized error with more information. It is only used when + the kind is an error. """ - @spec format_banner(kind, any, stacktrace | nil) :: String.t - def format_banner(kind, exception, stacktrace \\ nil) + @spec format_banner(kind, any, stacktrace) :: String.t() + def format_banner(kind, exception, stacktrace \\ []) def format_banner(:error, exception, stacktrace) do exception = normalize(:error, exception, stacktrace) @@ -107,7 +146,7 @@ defmodule Exception do end def format_banner(:throw, reason, _stacktrace) do - "** (throw) " <> inspect(reason) + "** (throw) " <> inspect(reason) end def format_banner(:exit, reason, _stacktrace) do @@ -115,98 +154,355 @@ defmodule Exception do end def format_banner({:EXIT, pid}, reason, _stacktrace) do - "** (EXIT from #{inspect pid}) " <> format_exit(reason, <<"\n ">>) + "** (EXIT from #{inspect(pid)}) " <> format_exit(reason, <<"\n ">>) end @doc """ - Normalizes and formats throw/errors/exits and stacktrace. + Normalizes and formats throws/errors/exits and stacktraces. It relies on `format_banner/3` and `format_stacktrace/1` to generate the final format. - Note that `{:EXIT, pid}` do not generate a stacktrace though - (as they are retrieved as messages without stacktraces). + If `kind` is `{:EXIT, pid}`, it does not generate a stacktrace, + as such exits are retrieved as messages without stacktraces. """ - - @spec format(kind, any, stacktrace | nil) :: String.t - - def format(kind, payload, stacktrace \\ nil) + @spec format(kind, any, stacktrace) :: String.t() + def format(kind, payload, stacktrace \\ []) def format({:EXIT, _} = kind, any, _) do format_banner(kind, any) end def format(kind, payload, stacktrace) do - stacktrace = stacktrace || System.stacktrace message = format_banner(kind, payload, stacktrace) + case stacktrace do [] -> message - _ -> message <> "\n" <> format_stacktrace(stacktrace) + _ -> message <> "\n" <> format_stacktrace(stacktrace) + end + end + + @doc false + def __format_message_with_term__(message, term) do + inspected = + term + |> inspect(pretty: true) + |> String.split("\n") + |> Enum.map_intersperse("\n", fn + "" -> "" + line -> " " <> line + end) + + IO.iodata_to_binary([message, "\n\n", inspected, "\n"]) + end + + @doc """ + Attaches information to throws/errors/exits for extra debugging. + + This operation is potentially expensive, as it reads data + from the file system, parses beam files, evaluates code and + so on. + + If `kind` argument is `:error` and the `error` is an Erlang exception, this function will + normalize it. If the `error` argument is an Elixir exception, this function will invoke + the optional `c:blame/2` callback on the exception module if it is implemented. + Unlike `message/1`, this function will not rescue errors - if the callback raises an exception, + the error will propagate to the caller. It is your choice if you want to rescue and return + the original exception, return a different exception, or let it cascade. + """ + @doc since: "1.5.0" + @spec blame(:error, any, stacktrace) :: {t, stacktrace} + @spec blame(non_error_kind, payload, stacktrace) :: {payload, stacktrace} when payload: var + def blame(kind, error, stacktrace) + + def blame(:error, error, stacktrace) do + %module{} = struct = normalize(:error, error, stacktrace) + + if Code.ensure_loaded?(module) and function_exported?(module, :blame, 2) do + module.blame(struct, stacktrace) + else + {struct, stacktrace} + end + end + + def blame(_kind, reason, stacktrace) do + {reason, stacktrace} + end + + @doc """ + Blames the invocation of the given module, function and arguments. + + This function will retrieve the available clauses from bytecode + and evaluate them against the given arguments. The clauses are + returned as a list of `{args, guards}` pairs where each argument + and each top-level condition in a guard separated by `and`/`or` + is wrapped in a tuple with blame metadata. + + This function returns either `{:ok, definition, clauses}` or `:error`. + Where `definition` is `:def`, `:defp`, `:defmacro` or `:defmacrop`. + """ + @doc since: "1.5.0" + @spec blame_mfa(module, function :: atom, args :: [term]) :: + {:ok, :def | :defp | :defmacro | :defmacrop, [{args :: [term], guards :: [term]}]} + | :error + def blame_mfa(module, function, args) + when is_atom(module) and is_atom(function) and is_list(args) do + try do + blame_mfa(module, function, length(args), args) + rescue + _ -> :error + end + end + + defp blame_mfa(module, function, arity, call_args) do + with [_ | _] = path <- :code.which(module), + {:ok, {_, [debug_info: debug_info]}} <- :beam_lib.chunks(path, [:debug_info]), + {:debug_info_v1, backend, data} <- debug_info, + {:ok, %{definitions: defs}} <- backend.debug_info(:elixir_v1, module, data, []), + {_, kind, _, clauses} <- List.keyfind(defs, {function, arity}, 0) do + clauses = + for {meta, ex_args, guards, _block} <- clauses do + scope = :elixir_erl.scope(meta, true) + ann = :elixir_erl.get_ann(meta) + + {erl_args, scope} = + :elixir_erl_clauses.match(ann, &:elixir_erl_pass.translate_args/3, ex_args, scope) + + {args, binding} = + [call_args, ex_args, erl_args] + |> Enum.zip() + |> Enum.map_reduce([], &blame_arg/2) + + guards = + guards + |> Enum.map(&blame_guard(&1, ann, scope, binding)) + |> Enum.map(&Macro.prewalk(&1, fn guard -> translate_guard(guard) end)) + + {args, guards} + end + + {:ok, kind, clauses} + else + _ -> :error + end + end + + defp map_node?({:is_map, _, [_]}), do: true + defp map_node?(_), do: false + defp map_key_node?({:is_map_key, _, [_, _]}), do: true + defp map_key_node?(_), do: false + + defp struct_validation_node?( + {:is_atom, _, [{{:., [], [:erlang, :map_get]}, _, [:__struct__, _]}]} + ), + do: true + + defp struct_validation_node?( + {:==, _, [{{:., [], [:erlang, :map_get]}, _, [:__struct__, _]}, _module]} + ), + do: true + + defp struct_validation_node?(_), do: false + + defp struct_macro?( + {:and, _, + [ + {:and, _, [%{node: node_1 = {_, _, [arg]}}, %{node: node_2 = {_, _, [arg, _]}}]}, + %{node: node_3 = {_, _, [{_, _, [_, arg]}]}} + ]} + ), + do: map_node?(node_1) and map_key_node?(node_2) and struct_validation_node?(node_3) + + defp struct_macro?( + {:and, _, + [ + {:and, _, + [ + {:and, _, + [ + %{node: node_1 = {_, _, [arg]}}, + {:or, _, [%{node: {:is_atom, _, [_]}}, %{node: :fail}]} + ]}, + %{node: node_2 = {_, _, [arg, _]}} + ]}, + %{node: node_3 = {_, _, [{_, _, [_, arg]}, _]}} + ]} + ), + do: map_node?(node_1) and map_key_node?(node_2) and struct_validation_node?(node_3) + + defp struct_macro?(_), do: false + + defp translate_guard(guard) do + if struct_macro?(guard) do + undo_is_struct_guard(guard) + else + guard + end + end + + defp undo_is_struct_guard({:and, meta, [_, %{node: {_, _, [{_, _, [_, arg]} | optional]}}]}) do + args = + case optional do + [] -> [arg] + [module] -> [arg, module] + end + + %{match?: meta[:value], node: {:is_struct, meta, args}} + end + + defp blame_arg({call_arg, ex_arg, erl_arg}, binding) do + {match?, binding} = blame_arg(erl_arg, call_arg, binding) + {blame_wrap(match?, rewrite_arg(ex_arg)), binding} + end + + defp blame_arg(erl_arg, call_arg, binding) do + binding = :orddict.store(:VAR, call_arg, binding) + + try do + ann = :erl_anno.new(0) + + {:value, _, binding} = + :erl_eval.expr({:match, ann, erl_arg, {:var, ann, :VAR}}, binding, :none) + + {true, binding} + rescue + _ -> {false, binding} + end + end + + defp rewrite_arg(arg) do + Macro.prewalk(arg, fn + {:%{}, meta, [__struct__: Range, first: first, last: last, step: step]} -> + {:..//, meta, [first, last, step]} + + other -> + other + end) + end + + defp blame_guard({{:., _, [:erlang, op]}, meta, [left, right]}, ann, scope, binding) + when op == :andalso or op == :orelse do + guards = [ + blame_guard(left, ann, scope, binding), + blame_guard(right, ann, scope, binding) + ] + + kernel_op = + case op do + :orelse -> :or + :andalso -> :and + end + + evaluate_guard(kernel_op, meta, guards) + end + + defp blame_guard(ex_guard, ann, scope, binding) do + ex_guard + |> blame_guard?(binding, ann, scope) + |> blame_wrap(rewrite_guard(ex_guard)) + end + + defp blame_guard?(ex_guard, binding, ann, scope) do + {erl_guard, _} = :elixir_erl_pass.translate(ex_guard, ann, scope) + {:value, true, _} = :erl_eval.expr(erl_guard, binding, :none) + true + rescue + _ -> false + end + + defp evaluate_guard(kernel_op, meta, guards = [_, _]) do + [x, y] = Enum.map(guards, &evaluate_guard/1) + + logic_value = + case kernel_op do + :or -> x or y + :and -> x and y + end + + {kernel_op, Keyword.put(meta, :value, logic_value), guards} + end + + defp evaluate_guard(%{match?: value}), do: value + defp evaluate_guard({_, meta, _}) when is_list(meta), do: meta[:value] + + defp rewrite_guard(guard) do + Macro.prewalk(guard, fn + {{:., _, [mod, fun]}, meta, args} -> erl_to_ex(mod, fun, args, meta) + other -> other + end) + end + + defp erl_to_ex(mod, fun, args, meta) do + case :elixir_rewrite.erl_to_ex(mod, fun, args) do + {Kernel, fun, args, _} -> {fun, meta, args} + {mod, fun, args, _} -> {{:., [], [mod, fun]}, meta, args} end end + defp blame_wrap(match?, ast), do: %{match?: match?, node: ast} + @doc """ - Formats an exit, returns a string. + Formats an exit. It returns a string. Often there are errors/exceptions inside exits. Exits are often wrapped by the caller and provide stacktraces too. This function formats exits in a way to nicely show the exit reason, caller and stacktrace. """ - @spec format_exit(any) :: String.t + @spec format_exit(any) :: String.t() def format_exit(reason) do format_exit(reason, <<"\n ">>) end # 2-Tuple could be caused by an error if the second element is a stacktrace. defp format_exit({exception, maybe_stacktrace} = reason, joiner) - when is_list(maybe_stacktrace) and maybe_stacktrace !== [] do + when is_list(maybe_stacktrace) and maybe_stacktrace !== [] do try do Enum.map(maybe_stacktrace, &format_stacktrace_entry/1) + catch + :error, _ -> + # Not a stacktrace, was an exit. + format_exit_reason(reason) else formatted_stacktrace -> # Assume a non-empty list formattable as stacktrace is a # stacktrace, so exit was caused by an error. - message = "an exception was raised:" <> joiner <> - format_banner(:error, exception, maybe_stacktrace) + message = + "an exception was raised:" <> + joiner <> format_banner(:error, exception, maybe_stacktrace) + Enum.join([message | formatted_stacktrace], joiner <> <<" ">>) - catch - :error, _ -> - # Not a stacktrace, was an exit. - format_exit_reason(reason) end end # :supervisor.start_link returns this error reason when it fails to init # because a child's start_link raises. - defp format_exit({:shutdown, - {:failed_to_start_child, child, {:EXIT, reason}}}, joiner) do + defp format_exit({:shutdown, {:failed_to_start_child, child, {:EXIT, reason}}}, joiner) do format_start_child(child, reason, joiner) end # :supervisor.start_link returns this error reason when it fails to init # because a child's start_link returns {:error, reason}. - defp format_exit({:shutdown, {:failed_to_start_child, child, reason}}, - joiner) do + defp format_exit({:shutdown, {:failed_to_start_child, child, reason}}, joiner) do format_start_child(child, reason, joiner) end # 2-Tuple could be an exit caused by mfa if second element is mfa, args # must be a list of arguments - max length 255 due to max arity. defp format_exit({reason2, {mod, fun, args}} = reason, joiner) - when length(args) < 256 do + when length(args) < 256 do try do format_mfa(mod, fun, args) - else - mfa -> - # Assume tuple formattable as an mfa is an mfa, so exit was caused by - # failed mfa. - "exited in: " <> mfa <> joiner <> - "** (EXIT) " <> format_exit(reason2, joiner <> <<" ">>) catch :error, _ -> # Not an mfa, was an exit. format_exit_reason(reason) + else + mfa -> + # Assume tuple formattable as an mfa is an mfa, + # so exit was caused by failed mfa. + "exited in: " <> + mfa <> joiner <> "** (EXIT) " <> format_exit(reason2, joiner <> <<" ">>) end end @@ -221,12 +517,13 @@ defmodule Exception do "shutdown: #{inspect(reason)}" end + defp format_exit_reason(:calling_self), do: "process attempted to call itself" defp format_exit_reason(:timeout), do: "time out" defp format_exit_reason(:killed), do: "killed" defp format_exit_reason(:noconnection), do: "no connection" defp format_exit_reason(:noproc) do - "no process" + "no process: the process is not alive or there's no process currently associated with the given name, possibly because its application isn't started" end defp format_exit_reason({:nodedown, node_name}) when is_atom(node_name) do @@ -253,29 +550,29 @@ defmodule Exception do # :supervisor.start_link error reasons - # If value is a list will be be formatted by mfa exit in format_exit/1 + # If value is a list will be formatted by mfa exit in format_exit/1 defp format_exit_reason({:bad_return, {mod, :init, value}}) - when is_atom(mod) do + when is_atom(mod) do format_mfa(mod, :init, 1) <> " returned a bad value: " <> inspect(value) end defp format_exit_reason({:bad_start_spec, start_spec}) do - "bad start spec: invalid children: " <> inspect(start_spec) + "bad child specification, invalid children: " <> inspect(start_spec) end defp format_exit_reason({:start_spec, start_spec}) do - "bad start spec: " <> format_sup_spec(start_spec) + "bad child specification, " <> format_sup_spec(start_spec) end defp format_exit_reason({:supervisor_data, data}) do - "bad supervisor data: " <> format_sup_data(data) + "bad supervisor configuration, " <> format_sup_data(data) end defp format_exit_reason(reason), do: inspect(reason) defp format_start_child(child, reason, joiner) do - "shutdown: failed to start child: " <> inspect(child) <> joiner <> - "** (EXIT) " <> format_exit(reason, joiner <> <<" ">>) + "shutdown: failed to start child: " <> + inspect(child) <> joiner <> "** (EXIT) " <> format_exit(reason, joiner <> <<" ">>) end defp format_sup_data({:invalid_type, type}) do @@ -287,49 +584,70 @@ defmodule Exception do end defp format_sup_data({:invalid_intensity, intensity}) do - "invalid intensity: " <> inspect(intensity) + "invalid max_restarts (intensity): " <> inspect(intensity) end defp format_sup_data({:invalid_period, period}) do - "invalid period: " <> inspect(period) + "invalid max_seconds (period): " <> inspect(period) + end + + defp format_sup_data({:invalid_max_children, max_children}) do + "invalid max_children: " <> inspect(max_children) + end + + defp format_sup_data({:invalid_extra_arguments, extra}) do + "invalid extra_arguments: " <> inspect(extra) end - defp format_sup_data(other), do: inspect(other) + defp format_sup_data(other), do: "got: #{inspect(other)}" + + defp format_sup_spec({:duplicate_child_name, id}) do + """ + more than one child specification has the id: #{inspect(id)}. + If using maps as child specifications, make sure the :id keys are unique. + If using a module or {module, arg} as child, use Supervisor.child_spec/2 to change the :id, for example: + + children = [ + Supervisor.child_spec({MyWorker, arg}, id: :my_worker_1), + Supervisor.child_spec({MyWorker, arg}, id: :my_worker_2) + ] + """ + end defp format_sup_spec({:invalid_child_spec, child_spec}) do - "invalid child spec: " <> inspect(child_spec) + "invalid child specification: #{inspect(child_spec)}" end defp format_sup_spec({:invalid_child_type, type}) do - "invalid child type: " <> inspect(type) + "invalid child type: #{inspect(type)}. Must be :worker or :supervisor." end defp format_sup_spec({:invalid_mfa, mfa}) do - "invalid mfa: " <> inspect(mfa) + "invalid mfa: #{inspect(mfa)}" end defp format_sup_spec({:invalid_restart_type, restart}) do - "invalid restart type: " <> inspect(restart) + "invalid restart type: #{inspect(restart)}. Must be :permanent, :transient or :temporary." end defp format_sup_spec({:invalid_shutdown, shutdown}) do - "invalid shutdown: " <> inspect(shutdown) + "invalid shutdown: #{inspect(shutdown)}. Must be an integer >= 0, :infinity or :brutal_kill." end defp format_sup_spec({:invalid_module, mod}) do - "invalid module: " <> inspect(mod) + "invalid module: #{inspect(mod)}. Must be an atom." end defp format_sup_spec({:invalid_modules, modules}) do - "invalid modules: " <> inspect(modules) + "invalid modules: #{inspect(modules)}. Must be a list of atoms or :dynamic." end - defp format_sup_spec(other), do: inspect(other) + defp format_sup_spec(other), do: "got: #{inspect(other)}" @doc """ Receives a stacktrace entry and formats it into a string. """ - @spec format_stacktrace_entry(stacktrace_entry) :: String.t + @spec format_stacktrace_entry(stacktrace_entry) :: String.t() def format_stacktrace_entry(entry) # From Macro.Env.stacktrace @@ -356,9 +674,19 @@ defmodule Exception do end defp format_application(module) do + # We cannot use Application due to bootstrap issues case :application.get_application(module) do - {:ok, app} -> "(" <> Atom.to_string(app) <> ") " - :undefined -> "" + {:ok, app} -> + case :application.get_key(app, :vsn) do + {:ok, vsn} when is_list(vsn) -> + "(" <> Atom.to_string(app) <> " " <> List.to_string(vsn) <> ") " + + _ -> + "(" <> Atom.to_string(app) <> ") " + end + + :undefined -> + "" end end @@ -368,14 +696,20 @@ defmodule Exception do A stacktrace must be given as an argument. If not, the stacktrace is retrieved from `Process.info/2`. """ + @spec format_stacktrace(stacktrace | nil) :: String.t() def format_stacktrace(trace \\ nil) do - trace = trace || case Process.info(self, :current_stacktrace) do - {:current_stacktrace, t} -> Enum.drop(t, 3) - end + trace = + if trace do + trace + else + case Process.info(self(), :current_stacktrace) do + {:current_stacktrace, t} -> Enum.drop(t, 3) + end + end case trace do [] -> "\n" - s -> " " <> Enum.map_join(s, "\n ", &format_stacktrace_entry(&1)) <> "\n" + _ -> " " <> Enum.map_join(trace, "\n ", &format_stacktrace_entry(&1)) <> "\n" end end @@ -385,12 +719,13 @@ defmodule Exception do ## Examples - Exception.format_fa(fn -> end, 1) + Exception.format_fa(fn -> nil end, 1) #=> "#Function<...>/1" """ + @spec format_fa(fun, arity) :: String.t() def format_fa(fun, arity) when is_function(fun) do - "#{inspect fun}#{format_arity(arity)}" + "#{inspect(fun)}#{format_arity(arity)}" end @doc """ @@ -400,31 +735,30 @@ defmodule Exception do ## Examples - iex> Exception.format_mfa Foo, :bar, 1 + iex> Exception.format_mfa(Foo, :bar, 1) "Foo.bar/1" - iex> Exception.format_mfa Foo, :bar, [] + iex> Exception.format_mfa(Foo, :bar, []) "Foo.bar()" - iex> Exception.format_mfa nil, :bar, [] + iex> Exception.format_mfa(nil, :bar, []) "nil.bar()" Anonymous functions are reported as -func/arity-anonfn-count-, where func is the name of the enclosing function. Convert to "anonymous fn in func/arity" """ + @spec format_mfa(module, atom, arity_or_args) :: String.t() def format_mfa(module, fun, arity) when is_atom(module) and is_atom(fun) do - fun = - case inspect(fun) do - ":" <> fun -> fun - fun -> fun - end - - case match?("\"-" <> _, fun) and String.split(fun, "-") do - [ "\"", outer_fun, "fun", _count, "\"" ] -> - "anonymous fn#{format_arity(arity)} in #{inspect module}.#{outer_fun}" - _ -> - "#{inspect module}.#{fun}#{format_arity(arity)}" + case Code.Identifier.extract_anonymous_fun_parent(fun) do + {outer_name, outer_arity} -> + "anonymous fn#{format_arity(arity)} in " <> + "#{Macro.inspect_atom(:literal, module)}." <> + "#{Macro.inspect_atom(:remote_call, outer_name)}/#{outer_arity}" + + :error -> + "#{Macro.inspect_atom(:literal, module)}." <> + "#{Macro.inspect_atom(:remote_call, fun)}#{format_arity(arity)}" end end @@ -438,8 +772,9 @@ defmodule Exception do end @doc """ - Formats the given file and line as shown in stacktraces. - If any of the values are nil, they are omitted. + Formats the given `file` and `line` as shown in stacktraces. + + If any of the values are `nil`, they are omitted. ## Examples @@ -453,292 +788,1795 @@ defmodule Exception do "" """ - def format_file_line(file, line) do - format_file_line(file, line, "") + @spec format_file_line(String.t() | nil, non_neg_integer | nil, String.t()) :: String.t() + def format_file_line(file, line, suffix \\ "") do + cond do + is_nil(file) -> "" + is_nil(line) or line == 0 -> "#{file}:#{suffix}" + true -> "#{file}:#{line}:#{suffix}" + end end - defp format_file_line(file, line, suffix) do - if file do - if line && line != 0 do - "#{file}:#{line}:#{suffix}" - else - "#{file}:#{suffix}" - end - else + @doc """ + Formats the given `file`, `line`, and `column` as shown in stacktraces. + + If any of the values are `nil`, they are omitted. + + ## Examples + + iex> Exception.format_file_line_column("foo", 1, 2) + "foo:1:2:" + + iex> Exception.format_file_line_column("foo", 1, nil) + "foo:1:" + + iex> Exception.format_file_line_column("foo", nil, nil) + "foo:" + + iex> Exception.format_file_line_column("foo", nil, 2) + "foo:" + + iex> Exception.format_file_line_column(nil, nil, nil) "" + + """ + @spec format_file_line_column( + String.t() | nil, + non_neg_integer | nil, + non_neg_integer | nil, + String.t() + ) :: String.t() + def format_file_line_column(file, line, column, suffix \\ "") do + cond do + is_nil(file) -> "" + is_nil(line) or line == 0 -> "#{file}:#{suffix}" + is_nil(column) or column == 0 -> "#{file}:#{line}:#{suffix}" + true -> "#{file}:#{line}:#{column}:#{suffix}" end end defp format_location(opts) when is_list(opts) do - format_file_line Keyword.get(opts, :file), Keyword.get(opts, :line), " " + case opts[:column] do + nil -> format_file_line(Keyword.get(opts, :file), Keyword.get(opts, :line), " ") + col -> format_file_line_column(Keyword.get(opts, :file), Keyword.get(opts, :line), col, " ") + end + end + + @doc false + def format_delimiter(delimiter) do + if delimiter |> Atom.to_string() |> String.contains?(["\"", "'"]), + do: delimiter, + else: ~s("#{delimiter}") + end + + @doc false + def format_snippet( + {start_line, _start_column} = start_pos, + {end_line, end_column} = end_pos, + description, + file, + lines, + start_message, + end_message + ) + when start_line < end_line do + max_digits = digits(end_line) + general_padding = max(2, max_digits) + 1 + padding = n_spaces(general_padding) + + relevant_lines = + if end_line - start_line < 5 do + line_range(lines, start_pos, end_pos, padding, max_digits, start_message, end_message) + else + trimmed_inbetween_lines( + lines, + start_pos, + end_pos, + padding, + max_digits, + start_message, + end_message + ) + end + + """ + #{padding}#{red("error:")} #{pad_message(description, padding)} + #{padding}│ + #{relevant_lines} + #{padding}│ + #{padding}└─ #{Path.relative_to_cwd(file)}:#{end_line}:#{end_column}\ + """ + end + + def format_snippet( + {start_line, start_column}, + {end_line, end_column}, + description, + file, + lines, + start_message, + end_message + ) + when start_line == end_line do + max_digits = digits(end_line) + general_padding = max(2, max_digits) + 1 + padding = n_spaces(general_padding) + formatted_line = [line_padding(end_line, max_digits), to_string(end_line), " │ ", hd(lines)] + + mismatched_closing_line = + [ + n_spaces(start_column - 1), + red("│"), + format_end_message(end_column - start_column, end_message) + ] + + unclosed_delimiter_line = + [padding, " │ ", format_start_message(start_column, start_message)] + + below_line = [padding, " │ ", mismatched_closing_line, "\n", unclosed_delimiter_line] + + """ + #{padding}#{red("error:")} #{pad_message(description, padding)} + #{padding}│ + #{formatted_line} + #{below_line} + #{padding}│ + #{padding}└─ #{Path.relative_to_cwd(file)}:#{end_line}:#{end_column}\ + """ + end + + defp line_padding(line_number, max_digits) do + line_digits = digits(line_number) + + spacing = + if line_digits == 1 do + max(2, max_digits) + else + max_digits - line_digits + 1 + end + + n_spaces(spacing) + end + + defp n_spaces(n), do: String.duplicate(" ", n) + + defp digits(number, acc \\ 1) + defp digits(number, acc) when number < 10, do: acc + defp digits(number, acc), do: digits(div(number, 10), acc + 1) + + defp trimmed_inbetween_lines( + lines, + {start_line, start_column}, + {end_line, end_column}, + padding, + max_digits, + start_message, + end_message + ) do + start_padding = line_padding(start_line, max_digits) + end_padding = line_padding(end_line, max_digits) + first_line = hd(lines) + last_line = List.last(lines) + + """ + #{start_padding}#{start_line} │ #{first_line} + #{padding}│ #{format_start_message(start_column, start_message)} + ... + #{end_padding}#{end_line} │ #{last_line} + #{padding}│ #{format_end_message(end_column, end_message)}\ + """ + end + + defp line_range( + lines, + {start_line, start_column}, + {end_line, end_column}, + padding, + max_digits, + start_message, + end_message + ) do + Enum.zip_with(lines, start_line..end_line, fn line, line_number -> + line_padding = line_padding(line_number, max_digits) + + cond do + line_number == start_line -> + [ + line_padding, + to_string(line_number), + " │ ", + line, + "\n", + padding, + " │ ", + format_start_message(start_column, start_message) + ] + + line_number == end_line -> + [ + line_padding, + to_string(line_number), + " │ ", + line, + "\n", + padding, + " │ ", + format_end_message(end_column, end_message) + ] + + true -> + [line_padding, to_string(line_number), " │ ", line] + end + end) + |> Enum.intersperse("\n") + end + + defp format_end_message(end_column, message), + do: [ + n_spaces(end_column - 1), + red(message) + ] + + defp format_start_message(start_column, message), + do: [n_spaces(start_column - 1), red(message)] + + defp pad_message(message, padding), do: String.replace(message, "\n", "\n #{padding}") + + defp red(string) do + if IO.ANSI.enabled?() do + [IO.ANSI.red(), string, IO.ANSI.reset()] + else + string + end end end -# Some exceptions implement `message/1` instead of `exception/1` mostly +# Some exceptions implement "message/1" instead of "exception/1" mostly # for bootstrap reasons. It is recommended for applications to implement -# `exception/1` instead of `message/1` as described in `defexception/1` +# "exception/1" instead of "message/1" as described in "defexception/1" # docs. defmodule RuntimeError do - defexception message: "runtime error" + @moduledoc """ + An exception for a generic runtime error. - def exception(msg) when is_binary(msg) do - %RuntimeError{message: msg} - end + This is the exception that `raise/1` raises when you pass it only a string as + a message: - def exception(arg) do - super(arg) - end + iex> raise "oops!" + ** (RuntimeError) oops! + + You should use this exceptions sparingly, since most of the time it might be + better to define your own exceptions specific to your application or library. + Sometimes, however, there are situations in which you don't expect a condition to + happen, but you want to give a meaningful error message if it does. In those cases, + `RuntimeError` can be a good choice. + + ## Fields + + `RuntimeError` exceptions have a single field, `:message` (a `t:String.t/0`), + which is public and can be accessed freely when reading or creating `RuntimeError` + exceptions. + """ + + defexception message: "runtime error" end defmodule ArgumentError do - defexception message: "argument error" + @moduledoc """ + An exception raised when an argument to a function is invalid. - def exception(msg) when is_binary(msg) do - %ArgumentError{message: msg} - end + You can raise this exception when you want to signal that an argument to + a function is invalid. For example, this exception is raised when calling + `Integer.to_string/1` with an invalid argument: - def exception(arg) do - super(arg) - end + iex> Integer.to_string(1.0) + ** (ArgumentError) errors were found at the given arguments: + ... + + `ArgumentError` exceptions have a single field, `:message` (a `t:String.t/0`), + which is public and can be accessed freely when reading or creating `ArgumentError` + exceptions. + """ + + defexception message: "argument error" end defmodule ArithmeticError do - defexception [] + @moduledoc """ + An exception raised on invalid arithmetic operations. - def message(_) do - "bad argument in arithmetic expression" - end -end + For example, this exception is raised if you divide by `0`: -defmodule SystemLimitError do - defexception [] + iex> 1 / 0 + ** (ArithmeticError) bad argument in arithmetic expression + """ - def message(_) do - "a system limit has been reached" - end -end + defexception message: "bad argument in arithmetic expression" -defmodule SyntaxError do - defexception [file: nil, line: nil, description: "syntax error"] + @unary_ops [:+, :-] + @binary_ops [:+, :-, :*, :/] + @binary_funs [:div, :rem] + @bitwise_binary_funs [:band, :bor, :bxor, :bsl, :bsr] - def message(exception) do - Exception.format_file_line(Path.relative_to_cwd(exception.file), exception.line) <> - " " <> exception.description - end -end + @impl true + def blame(%{message: message} = exception, [{:erlang, fun, args, _} | _] = stacktrace) do + message = + message <> + case {fun, args} do + {op, [a]} when op in @unary_ops -> + ": #{op}(#{inspect(a)})" -defmodule TokenMissingError do - defexception [file: nil, line: nil, description: "expression is incomplete"] + {op, [a, b]} when op in @binary_ops -> + ": #{inspect(a)} #{op} #{inspect(b)}" - def message(exception) do - Exception.format_file_line(Path.relative_to_cwd(exception.file), exception.line) <> - " " <> exception.description - end -end + {fun, [a, b]} when fun in @binary_funs -> + ": #{fun}(#{inspect(a)}, #{inspect(b)})" -defmodule CompileError do - defexception [file: nil, line: nil, description: "compile error"] + {fun, [a, b]} when fun in @bitwise_binary_funs -> + ": Bitwise.#{fun}(#{inspect(a)}, #{inspect(b)})" - def message(exception) do - Exception.format_file_line(Path.relative_to_cwd(exception.file), exception.line) <> - " " <> exception.description - end -end + {:bnot, [a]} -> + ": Bitwise.bnot(#{inspect(a)})" -defmodule BadFunctionError do - defexception [term: nil] + _ -> + "" + end - def message(exception) do - "expected a function, got: #{inspect(exception.term)}" + {%{exception | message: message}, stacktrace} end -end - -defmodule BadStructError do - defexception [struct: nil, term: nil] - def message(exception) do - "expected a struct named #{inspect(exception.struct)}, got: #{inspect(exception.term)}" + def blame(exception, stacktrace) do + {exception, stacktrace} end end -defmodule MatchError do - defexception [term: nil] +defmodule SystemLimitError do + @moduledoc """ + An exception raised when a system limit has been reached. - def message(exception) do - "no match of right hand side value: #{inspect(exception.term)}" - end -end + For example, this can happen if you try to create an atom that is too large: -defmodule CaseClauseError do - defexception [term: nil] + iex> String.to_atom(String.duplicate("a", 100_000)) + ** (SystemLimitError) a system limit has been reached + """ - def message(exception) do - "no case clause matching: #{inspect(exception.term)}" - end + defexception message: "a system limit has been reached" end -defmodule CondClauseError do - defexception [] +defmodule MismatchedDelimiterError do + @moduledoc """ + An exception raised when a mismatched delimiter is found when parsing code. + + For example: + + iex> Code.eval_string("[1, 2, 3}") + ** (MismatchedDelimiterError) mismatched delimiter found on nofile:1:9: + ... + + The following fields of this exceptions are public and can be accessed freely: + + * `:file` (`t:Path.t/0` or `nil`) - the file where the error occurred, or `nil` if + the error occurred in code that did not come from a file + * `:line` - the line for the opening delimiter + * `:column` - the column for the opening delimiter + * `:end_line` - the line for the mismatched closing delimiter + * `:end_column` - the column for the mismatched closing delimiter + * `:opening_delimiter` - an atom representing the opening delimiter + * `:closing_delimiter` - an atom representing the mismatched closing delimiter + * `:expected_delimiter` - an atom representing the closing delimiter + * `:description` - a description of the mismatched delimiter error + """ - def message(_exception) do - "no cond clause evaluated to a true value" + defexception [ + :file, + :line, + :column, + :end_line, + :end_column, + :opening_delimiter, + :closing_delimiter, + :expected_delimiter, + :snippet, + description: "mismatched delimiter error" + ] + + @impl true + def message(%{ + line: start_line, + column: start_column, + end_line: end_line, + end_column: end_column, + description: description, + expected_delimiter: expected_delimiter, + file: file, + snippet: snippet + }) do + start_pos = {start_line, start_column} + end_pos = {end_line, end_column} + lines = String.split(snippet, "\n") + expected_delimiter = Exception.format_delimiter(expected_delimiter) + + start_message = "└ unclosed delimiter" + end_message = ~s/└ mismatched closing delimiter (expected #{expected_delimiter})/ + + snippet = + Exception.format_snippet( + start_pos, + end_pos, + description, + file, + lines, + start_message, + end_message + ) + + format_message(file, end_line, end_column, snippet) + end + + defp format_message(file, line, column, message) do + location = Exception.format_file_line_column(Path.relative_to_cwd(file), line, column) + "mismatched delimiter found on " <> location <> "\n" <> message end end -defmodule TryClauseError do - defexception [term: nil] +defmodule SyntaxError do + @moduledoc """ + An exception raised when there's a syntax error when parsing code. - def message(exception) do - "no try clause matching: #{inspect(exception.term)}" - end -end + For example: -defmodule BadArityError do - defexception [function: nil, args: nil] + iex> Code.eval_string("5 + 5h") + ** (SyntaxError) invalid syntax found on nofile:1:5: + ... - def message(exception) do - fun = exception.function - args = exception.args - insp = Enum.map_join(args, ", ", &inspect/1) - {:arity, arity} = :erlang.fun_info(fun, :arity) - "#{inspect(fun)} with arity #{arity} called with #{count(length(args), insp)}" + The following fields of this exceptions are public and can be accessed freely: + + * `:file` (`t:Path.t/0` or `nil`) - the file where the error occurred, or `nil` if + the error occurred in code that did not come from a file + * `:line` - the line where the error occurred + * `:column` - the column where the error occurred + * `:description` - a description of the syntax error + """ + + defexception [:file, :line, :column, :snippet, description: "syntax error"] + + @impl true + def message(%{ + file: file, + line: line, + column: column, + description: description, + snippet: snippet + }) + when not is_nil(snippet) and not is_nil(column) do + snippet = + :elixir_errors.format_snippet(:error, {line, column}, file, description, snippet, %{}) + + format_message(file, line, column, snippet) end - defp count(0, _insp), do: "no arguments" - defp count(1, insp), do: "1 argument (#{insp})" - defp count(x, insp), do: "#{x} arguments (#{insp})" -end + @impl true + def message(%{ + file: file, + line: line, + column: column, + description: description + }) do + snippet = + :elixir_errors.format_snippet(:error, {line, column}, file, description, nil, %{}) -defmodule UndefinedFunctionError do - defexception [module: nil, function: nil, arity: nil] + padded = " " <> String.replace(snippet, "\n", "\n ") + format_message(file, line, column, padded) + end - def message(exception) do - if exception.function do - formatted = Exception.format_mfa exception.module, exception.function, exception.arity - "undefined function: #{formatted}" - else - "undefined function" - end + defp format_message(file, line, column, message) do + location = Exception.format_file_line_column(Path.relative_to_cwd(file), line, column) + "invalid syntax found on " <> location <> "\n" <> message end end -defmodule FunctionClauseError do - defexception [module: nil, function: nil, arity: nil] +defmodule TokenMissingError do + @moduledoc """ + An exception raised when a token is missing when parsing code. + + For example: + + iex> Code.eval_string("[1, 2, 3") + ** (TokenMissingError) token missing on nofile:1:9: + ... + + The following fields of this exceptions are public and can be accessed freely: + + * `:file` (`t:Path.t/0` or `nil`) - the file where the error occurred, or `nil` if + the error occurred in code that did not come from a file + * `:line` - the line for the opening delimiter + * `:column` - the column for the opening delimiter + * `:end_line` - the line for the end of the string + * `:end_column` - the column for the end of the string + * `:opening_delimiter` - an atom representing the opening delimiter + * `:expected_delimiter` - an atom representing the expected delimiter + * `:description` - a description of the missing token error + + This is mostly raised by Elixir tooling when compiling and evaluating code. + """ + + defexception [ + :file, + :line, + :column, + :end_line, + :end_column, + :snippet, + :opening_delimiter, + :expected_delimiter, + description: "expression is incomplete" + ] + + @impl true + def message(%{ + file: file, + line: line, + column: column, + end_line: end_line, + description: description, + expected_delimiter: expected_delimiter, + snippet: snippet + }) + when not is_nil(snippet) and not is_nil(column) and not is_nil(end_line) do + {trimmed, [last_line | _] = reversed_lines} = + snippet + |> String.split("\n") + |> Enum.reverse() + |> Enum.split_while(&(&1 == "")) + + end_line = end_line - length(trimmed) + end_column = String.length(last_line) + 1 + + start_pos = {line, column} + end_pos = {end_line, end_column} + expected_delimiter = Exception.format_delimiter(expected_delimiter) + + start_message = ~s/└ unclosed delimiter/ + end_message = ~s/└ missing closing delimiter (expected #{expected_delimiter})/ + + snippet = + Exception.format_snippet( + start_pos, + end_pos, + description, + file, + Enum.reverse(reversed_lines), + start_message, + end_message + ) + + format_message(file, end_line, end_column, snippet) + end + + @impl true + def message(%{ + file: file, + line: line, + column: column, + snippet: snippet, + description: description + }) do + snippet = + :elixir_errors.format_snippet(:error, {line, column}, file, description, snippet, %{}) + + format_message(file, line, column, snippet) + end + + defp format_message(file, line, column, message) do + location = Exception.format_file_line_column(Path.relative_to_cwd(file), line, column) + "token missing on " <> location <> "\n" <> message + end +end + +defmodule CompileError do + @moduledoc """ + An exception raised when there's an error when compiling code. + + For example: + + 1 = y + ** (CompileError) iex:1: undefined variable "y" + + The following fields of this exceptions are public and can be accessed freely: + + * `:file` (`t:Path.t/0` or `nil`) - the file where the error occurred, or `nil` if + the error occurred in code that did not come from a file + * `:line` (`t:non_neg_integer/0`) - the line where the error occurred + * `:description` (`t:String.t/0`) - a description of the compile error + + This is mostly raised by Elixir tooling when compiling and evaluating code. + """ + + defexception [:file, :line, description: "compile error"] + + @impl true + def message(%{file: file, line: line, description: description}) do + case Exception.format_file_line(file && Path.relative_to_cwd(file), line) do + "" -> description + formatted -> formatted <> " " <> description + end + end +end + +defmodule Kernel.TypespecError do + @moduledoc """ + An exception raised when there's an error in a typespec. + + For example, if your typespec definition points to an invalid type, you get an exception: + + @type my_type :: intger() + + will raise: + + ** (Kernel.TypespecError) type intger/0 undefined + + The following fields of this exceptions are public and can be accessed freely: + + * `:file` (`t:Path.t/0` or `nil`) - the file where the error occurred, or `nil` if + the error occurred in code that did not come from a file + * `:line` (`t:non_neg_integer/0`) - the line where the error occurred + """ + + defexception [:file, :line, :description] + + @impl true + def message(%{file: file, line: line, description: description}) do + case Exception.format_file_line(file && Path.relative_to_cwd(file), line) do + "" -> description + formatted -> formatted <> " " <> description + end + end +end + +defmodule BadFunctionError do + @moduledoc """ + An exception raised when a function is expected, but something else was given. + + For example: + + iex> value = "hello" + iex> value.() + ** (BadFunctionError) expected a function, got: "hello" + """ + + defexception [:term] + + @impl true + def message(%{term: term}) when is_function(term) do + "function #{inspect(term)} is invalid, likely because it points to an old version of the code" + end def message(exception) do - if exception.function do - formatted = Exception.format_mfa exception.module, exception.function, exception.arity - "no function clause matching in #{formatted}" + "expected a function, got: #{inspect(exception.term)}" + end +end + +defmodule BadMapError do + @moduledoc """ + An exception raised when a map is expected, but something else was given. + + For example: + + iex> value = "hello" + iex> %{value | key: "value"} + ** (BadMapError) expected a map, got: + ... + """ + + defexception [:term] + + @impl true + def message(exception) do + Exception.__format_message_with_term__( + "expected a map, got:", + exception.term + ) + end +end + +defmodule BadBooleanError do + @moduledoc """ + An exception raised when a boolean is expected, but something else was given. + + This exception is raised by `and` and `or` when the first argument is not a boolean: + + iex> 123 and true + ** (BadBooleanError) expected a boolean on left-side of "and", got: + ... + """ + + defexception [:term, :operator] + + @impl true + def message(exception) do + Exception.__format_message_with_term__( + "expected a boolean on left-side of \"#{exception.operator}\", got:", + exception.term + ) + end +end + +defmodule MatchError do + @moduledoc """ + An exception raised when a pattern match (`=/2`) fails. + + For example: + + iex> [_ | _] = [] + ** (MatchError) no match of right hand side value: + ... + + The following fields of this exception are public and can be accessed freely: + + * `:term` (`t:term/0`) - the term that did not match the pattern + + """ + + defexception [:term] + + @impl true + def message(exception) do + Exception.__format_message_with_term__( + "no match of right hand side value:", + exception.term + ) + end +end + +defmodule CaseClauseError do + @moduledoc """ + An exception raised when a term in a `case/2` expression + does not match any of the defined `->` clauses. + + For example: + + iex> case System.unique_integer() do + ...> bin when is_binary(bin) -> :oops + ...> :ok -> :neither_this_one + ...> end + ** (CaseClauseError) no case clause matching: + ... + + The following fields of this exception are public and can be accessed freely: + + * `:term` (`t:term/0`) - the term that did not match any of the clauses + """ + + defexception [:term] + + @impl true + def message(exception) do + Exception.__format_message_with_term__( + "no case clause matching:", + exception.term + ) + end +end + +defmodule WithClauseError do + @moduledoc """ + An exception raised when a term in a `with/1` expression + does not match any of the defined `->` clauses in its `else`. + + For example, this exception gets raised for a `with/1` like the following, because + the `{:ok, 2}` term does not match the `:error` or `{:error, _}` clauses in the + `else`: + + iex> with {:ok, 1} <- {:ok, 2} do + ...> :woah + ...> else + ...> :error -> :error + ...> {:error, _} -> :error + ...> end + ** (WithClauseError) no with clause matching: + ... + + The following fields of this exception are public and can be accessed freely: + + * `:term` (`t:term/0`) - the term that did not match any of the clauses + """ + + defexception [:term] + + @impl true + def message(exception) do + Exception.__format_message_with_term__( + "no with clause matching:", + exception.term + ) + end +end + +defmodule CondClauseError do + @moduledoc """ + An exception raised when no clauses in a `cond/1` expression evaluate to a truthy value. + + For example, this exception gets raised for a `cond/1` like the following: + + iex> cond do + ...> 1 + 1 == 3 -> :woah + ...> nil -> "yeah this won't happen" + ...> end + ** (CondClauseError) no cond clause evaluated to a truthy value + """ + + defexception [] + + @impl true + def message(_exception) do + "no cond clause evaluated to a truthy value" + end +end + +defmodule TryClauseError do + @moduledoc """ + An exception raised when none of the `else` clauses in a `try/1` match. + + For example: + + iex> try do + ...> :ok + ...> rescue + ...> e -> e + ...> else + ...> # :ok -> :ok is missing + ...> :not_ok -> :not_ok + ...> end + ** (TryClauseError) no try clause matching: + ... + + The following fields of this exception are public and can be accessed freely: + + * `:term` (`t:term/0`) - the term that did not match any of the clauses + + """ + defexception [:term] + + @impl true + def message(exception) do + Exception.__format_message_with_term__( + "no try clause matching:", + exception.term + ) + end +end + +defmodule BadArityError do + @moduledoc """ + An exception raised when a function is called with the wrong number of arguments. + + For example: + + my_function = fn x, y -> x + y end + my_function.(42) + ** (BadArityError) #Function<41.39164016/2 in :erl_eval.expr/6> with arity 2 called with 1 argument (42) + """ + + defexception [:function, :args] + + @impl true + def message(exception) do + fun = exception.function + args = exception.args + insp = Enum.map_join(args, ", ", &inspect/1) + {:arity, arity} = Function.info(fun, :arity) + "#{inspect(fun)} with arity #{arity} called with #{count(length(args), insp)}" + end + + defp count(0, _insp), do: "no arguments" + defp count(1, insp), do: "1 argument (#{insp})" + defp count(x, insp), do: "#{x} arguments (#{insp})" +end + +defmodule UndefinedFunctionError do + @moduledoc """ + An exception raised when a function is invoked that is not defined. + + For example: + + # Let's use apply/3 as otherwise Elixir emits a compile-time warning + iex> apply(String, :non_existing_fun, ["hello"]) + ** (UndefinedFunctionError) function String.non_existing_fun/1 is undefined or private + + The following fields of this exception are public and can be accessed freely: + + * `:module` (`t:module/0`) - the module name + * `:function` (`t:atom/0`) - the function name + * `:arity` (`t:non_neg_integer/0`) - the arity of the function + """ + + @function_threshold 0.77 + @max_suggestions 5 + defexception [:module, :function, :arity, :reason, :message] + + @impl true + def message(%{message: nil} = exception) do + %{reason: reason, module: module, function: function, arity: arity} = exception + {message, hint_type} = message(reason, module, function, arity) + message <> if(hint_type == :suggest_module, do: hint_for_missing_alias(module), else: "") + end + + def message(%{message: message}) do + message + end + + defp message(nil, module, function, arity) do + cond do + is_nil(function) or is_nil(arity) -> + {"undefined function", :suggest_module} + + is_nil(module) -> + formatted_fun = Exception.format_mfa(module, function, arity) + {"function #{formatted_fun} is undefined", :suggest_module} + + function_exported?(module, :module_info, 0) -> + message(:"function not exported", module, function, arity) + + true -> + message(:"module could not be loaded", module, function, arity) + end + end + + defp message(:"module could not be loaded", module, function, arity) do + formatted_fun = Exception.format_mfa(module, function, arity) + + {"function #{formatted_fun} is undefined (module #{inspect(module)} is not available)", + :suggest_module} + end + + defp message(:"function not exported", module, function, arity) do + formatted_fun = Exception.format_mfa(module, function, arity) + {"function #{formatted_fun} is undefined or private", :suggest_function} + end + + defp message(:"undefined local", nil, function, arity) do + {"function #{function}/#{arity} is undefined (there is no such import)", :no_hint} + end + + defp message(reason, module, function, arity) do + formatted_fun = Exception.format_mfa(module, function, arity) + {"function #{formatted_fun} is undefined (#{reason})", :suggest_module} + end + + @impl true + def blame(exception, stacktrace) do + %{reason: reason, module: module, function: function, arity: arity} = exception + {message, hint_type} = message(reason, module, function, arity) + message = message <> hint(module, function, arity, hint_type) + {%{exception | message: message}, stacktrace} + end + + defp hint(_, _, _, :no_hint), do: "" + + defp hint(nil, _function, 0, _hint_type) do + ". If you are using the dot syntax, such as module.function(), " <> + "make sure the left-hand side of the dot is a module atom" + end + + defp hint(module, function, arity, :suggest_function) do + hint_for_behaviour(module, function, arity) <> + hint_for_loaded_module(module, function, arity) + end + + defp hint(module, function, arity, :suggest_module) do + hint_for_missing_module(module, function, arity) + end + + defp hint_for_missing_alias(module) do + with "Elixir." <> rest <- Atom.to_string(module), + false <- rest =~ "." do + ". Make sure the module name is correct and has been specified in full (or that an alias has been defined)" + else + _ -> "" + end + end + + @doc false + def hint_for_missing_module(module, function, arity) do + downcased_module = downcase_module_name(module) + stripped_module = module |> Atom.to_string() |> String.replace_leading("Elixir.", "") + + candidates = + for {name, _, _} = candidate <- :code.all_available(), + downcase_module_name(name) == downcased_module or + String.ends_with?(List.to_string(name), stripped_module), + {:module, module} <- [load_module(candidate)], + function_exported?(module, function, arity), + do: module + + if candidates != [] do + suggestions = + candidates + |> Enum.take(@max_suggestions) + |> Enum.sort(:asc) + |> Enum.map(fn module -> + ["\n * ", Exception.format_mfa(module, function, arity)] + end) + + ". Did you mean:\n#{suggestions}\n" + else + hint_for_missing_alias(module) + end + end + + defp load_module({name, _path, _loaded?}) do + name + |> List.to_atom() + |> Code.ensure_loaded() + end + + defp downcase_module_name(module) do + module + |> to_string() + |> String.downcase(:ascii) + end + + @doc false + def hint_for_loaded_module(module, function, arity) do + cond do + macro_exported?(module, function, arity) -> + ". However, there is a macro with the same name and arity. " <> + "Be sure to require #{inspect(module)} if you intend to invoke this macro" + + message = otp_obsolete(module, function, arity) -> + ", #{message}" + + true -> + IO.iodata_to_binary(did_you_mean(module, function)) + end + end + + defp otp_obsolete(module, function, arity) do + case :otp_internal.obsolete(module, function, arity) do + {:removed, [_ | _] = string} -> string + _ -> nil + end + end + + defp hint_for_behaviour(module, function, arity) do + case behaviours_for(module) do + [] -> + "" + + behaviours -> + case Enum.find(behaviours, &expects_callback?(&1, function, arity)) do + nil -> "" + behaviour -> ", but the behaviour #{inspect(behaviour)} expects it to be present" + end + end + rescue + # In case the module was removed while we are computing this + UndefinedFunctionError -> "" + end + + defp behaviours_for(module) do + :attributes + |> module.module_info() + |> Keyword.get(:behaviour, []) + end + + defp expects_callback?(behaviour, function, arity) do + callbacks = + behaviour.behaviour_info(:callbacks) -- behaviour.behaviour_info(:optional_callbacks) + + Enum.member?(callbacks, {function, arity}) + end + + ## Shared helpers across hints + + defp did_you_mean(module, function) do + exports = exports_for(module) + + result = + case Keyword.take(exports, [function]) do + [] -> + candidates = exports -- deprecated_functions_for(module) + base = Atom.to_string(function) + + for {key, val} <- candidates, + dist = String.jaro_distance(base, Atom.to_string(key)), + dist >= @function_threshold, + do: {dist, key, val} + + arities -> + for {key, val} <- arities, do: {1.0, key, val} + end + |> Enum.sort(&(elem(&1, 0) >= elem(&2, 0))) + |> Enum.take(@max_suggestions) + |> Enum.sort(&(elem(&1, 1) <= elem(&2, 1))) + + case result do + [] -> [] + suggestions -> [". Did you mean:\n\n" | Enum.map(suggestions, &format_fa/1)] + end + end + + defp format_fa({_dist, fun, arity}) do + [" * ", Macro.inspect_atom(:remote_call, fun), ?/, Integer.to_string(arity), ?\n] + end + + defp exports_for(module) do + if function_exported?(module, :__info__, 1) do + module.__info__(:macros) ++ module.__info__(:functions) else - "no function clause matches" + module.module_info(:exports) + end + rescue + # In case the module was removed while we are computing this + UndefinedFunctionError -> [] + end + + defp deprecated_functions_for(module) do + if function_exported?(module, :__info__, 1) do + for {name_arity, _message} <- module.__info__(:deprecated), do: name_arity + else + [] + end + rescue + # In case the module was removed while we are computing this + UndefinedFunctionError -> [] + end +end + +defmodule FunctionClauseError do + @moduledoc """ + An exception raised when a function call doesn't match any defined clause. + + For example: + + iex> URI.parse(:wrong_argument) + ** (FunctionClauseError) no function clause matching in URI.parse/1 + + The following fields of this exception are public and can be accessed freely: + + * `:module` (`t:module/0`) - the module name + * `:function` (`t:atom/0`) - the function name + * `:arity` (`t:non_neg_integer/0`) - the arity of the function + """ + + defexception [:module, :function, :arity, :kind, :args, :clauses] + + @clause_limit 10 + + @impl true + def message(exception) do + case exception do + %{function: nil} -> + "no function clause matches" + + %{module: module, function: function, arity: arity} -> + formatted = Exception.format_mfa(module, function, arity) + blamed = blame(exception, &inspect/1, &blame_match/1) + "no function clause matching in #{formatted}" <> blamed + end + end + + @impl true + def blame(%{module: module, function: function, arity: arity} = exception, stacktrace) do + case stacktrace do + [{^module, ^function, args, meta} | rest] when length(args) == arity -> + exception = + case Exception.blame_mfa(module, function, args) do + {:ok, kind, clauses} -> %{exception | args: args, kind: kind, clauses: clauses} + :error -> %{exception | args: args} + end + + {exception, [{module, function, arity, meta} | rest]} + + stacktrace -> + {exception, stacktrace} end end + + defp blame_match(%{match?: true, node: node}), do: Macro.to_string(node) + defp blame_match(%{match?: false, node: node}), do: "-" <> Macro.to_string(node) <> "-" + + @doc false + def blame(%{args: nil}, _, _) do + "" + end + + def blame(exception, inspect_fun, fun) do + %{module: module, function: function, arity: arity, kind: kind, args: args, clauses: clauses} = + exception + + mfa = Exception.format_mfa(module, function, arity) + + format_clause_fun = fn {args, guards} -> + args = Enum.map_join(args, ", ", fun) + base = " #{kind} #{function}(#{args})" + Enum.reduce(guards, base, &"#{&2} when #{clause_to_string(&1, fun, 0)}") <> "\n" + end + + "\n\nThe following arguments were given to #{mfa}:\n" <> + "#{format_args(args, inspect_fun)}" <> + "#{format_clauses(clauses, format_clause_fun, @clause_limit)}" + end + + defp clause_to_string({op, _, [left, right]} = node, fun, parent) do + case Code.Identifier.binary_op(op) do + {_side, precedence} -> + left = clause_to_string(left, fun, precedence) + right = clause_to_string(right, fun, precedence) + + if parent > precedence do + "(" <> left <> " #{op} " <> right <> ")" + else + left <> " #{op} " <> right + end + + _ -> + fun.(node) + end + end + + defp clause_to_string(node, fun, _precedence), do: fun.(node) + + defp format_args(args, inspect_fun) do + args + |> Enum.with_index(1) + |> Enum.map(fn {arg, i} -> + [pad("\n# "), Integer.to_string(i), pad("\n"), pad(inspect_fun.(arg)), "\n"] + end) + end + + defp format_clauses(clauses, format_clause_fun, limit) + defp format_clauses(nil, _, _), do: "" + defp format_clauses([], _, _), do: "" + + defp format_clauses(clauses, format_clause_fun, limit) do + top_clauses = + clauses + |> Enum.take(limit) + |> Enum.map(format_clause_fun) + + [ + "\nAttempted function clauses (showing #{length(top_clauses)} out of #{length(clauses)}):", + "\n\n", + top_clauses, + non_visible_clauses(length(clauses) - limit) + ] + end + + defp non_visible_clauses(n) when n <= 0, do: [] + defp non_visible_clauses(1), do: [" ...\n (1 clause not shown)\n"] + defp non_visible_clauses(n), do: [" ...\n (#{n} clauses not shown)\n"] + + defp pad(string) do + String.replace(string, "\n", "\n ") + end end defmodule Code.LoadError do - defexception [:file, :message] + @moduledoc """ + An exception raised when a file cannot be loaded. + + This is typically raised by functions in the `Code` module, for example: + + Code.require_file("missing_file.exs") + ** (Code.LoadError) could not load missing_file.exs. Reason: enoent + + The following fields of this exception are public and can be accessed freely: + + * `:file` (`t:String.t/0`) - the file name + * `:reason` (`t:term/0`) - the reason why the file could not be loaded + """ + + defexception [:file, :message, :reason] def exception(opts) do file = Keyword.fetch!(opts, :file) - %Code.LoadError{message: "could not load #{file}", file: file} + reason = Keyword.fetch!(opts, :reason) + message = "could not load #{file}. Reason: #{reason}" + %Code.LoadError{message: message, file: file, reason: reason} end end defmodule Protocol.UndefinedError do - defexception [protocol: nil, value: nil, description: nil] + @moduledoc """ + An exception raised when a protocol is not implemented for a given value. - def message(exception) do - msg = "protocol #{inspect exception.protocol} not implemented for #{inspect exception.value}" - if exception.description do - msg <> ", " <> exception.description - else - msg + For example: + + iex> Enum.at("A string!", 0) + ** (Protocol.UndefinedError) protocol Enumerable not implemented for BitString + ... + + The following fields of this exception are public and can be accessed freely: + + * `:protocol` (`t:module/0`) - the protocol that is not implemented + * `:value` (`t:term/0`) - the value that does not implement the protocol + """ + + defexception [:protocol, :value, description: ""] + + @impl true + def message(%{protocol: protocol, value: value, description: description}) do + inspected = + value + |> inspect(pretty: true) + # Indent only lines with contents on them + |> String.replace(~r/^(?=.+)/m, " ") + + "protocol #{inspect(protocol)} not implemented for " <> + value_type(value) <> + maybe_description(description) <> + maybe_available(protocol) <> + """ + + + Got value: + + #{inspected} + """ + end + + defp value_type(%{__struct__: struct}), do: "#{inspect(struct)} (a struct)" + defp value_type(value) when is_atom(value), do: "Atom" + defp value_type(value) when is_bitstring(value), do: "BitString" + defp value_type(value) when is_float(value), do: "Float" + defp value_type(value) when is_function(value), do: "Function" + defp value_type(value) when is_integer(value), do: "Integer" + defp value_type(value) when is_list(value), do: "List" + defp value_type(value) when is_map(value), do: "Map" + defp value_type(value) when is_pid(value), do: "PID" + defp value_type(value) when is_port(value), do: "Port" + defp value_type(value) when is_reference(value), do: "Reference" + defp value_type(value) when is_tuple(value), do: "Tuple" + + defp maybe_description(""), do: "" + defp maybe_description(description), do: ", " <> description + + defp maybe_available(protocol) do + case protocol.__protocol__(:impls) do + {:consolidated, []} -> + ". There are no implementations for this protocol." + + {:consolidated, types} -> + ". This protocol is implemented for: " <> + Enum.map_join(types, ", ", &inspect/1) + + :not_consolidated -> + "" end end end defmodule KeyError do - defexception key: nil, term: nil + @moduledoc """ + An exception raised when a key is not found in a data structure. - def message(exception) do - "key #{inspect exception.key} not found in: #{inspect exception.term}" + For example, this is raised by `Map.fetch!/2` when the given key + cannot be found in the given map: + + iex> map = %{name: "Alice", age: 25} + iex> Map.fetch!(map, :first_name) + ** (KeyError) key :first_name not found in: + ... + + The following fields of this exception are public and can be accessed freely: + + * `:term` (`t:term/0`) - the data structure that was searched + * `:key` (`t:term/0`) - the key that was not found + """ + + defexception [:key, :term, :message] + + @impl true + def message(exception = %{message: nil}), do: message(exception.key, exception.term) + def message(%{message: message}), do: message + + defp message(key, term) do + message = "key #{inspect(key)} not found" + + cond do + term == nil -> + message + + is_atom(term) and is_atom(key) -> + message <> + " in: #{inspect(term)} (if instead you want to invoke #{inspect(term)}.#{key}(), " <> + "make sure to add parentheses after the function name)" + + true -> + Exception.__format_message_with_term__( + message <> " in:", + term + ) + end + end + + @impl true + def blame(exception = %{message: message}, stacktrace) when is_binary(message) do + {exception, stacktrace} + end + + def blame(exception, stacktrace) do + %{term: term, key: key} = exception + message = message(key, term) + + if is_atom(key) and (map_with_atom_keys_only?(term) or Keyword.keyword?(term)) do + hint = did_you_mean(key, available_keys(term)) + message = message <> IO.iodata_to_binary(hint) + {%{exception | message: message}, stacktrace} + else + {%{exception | message: message}, stacktrace} + end + end + + defp map_with_atom_keys_only?(term) do + is_map(term) and Enum.all?(Map.to_list(term), fn {k, _} -> is_atom(k) end) + end + + defp available_keys(term) when is_map(term), do: Map.keys(term) + defp available_keys(term) when is_list(term), do: Keyword.keys(term) + + @threshold 0.77 + @max_suggestions 5 + defp did_you_mean(missing_key, available_keys) do + stringified_key = Atom.to_string(missing_key) + + suggestions = + for key <- available_keys, + distance = String.jaro_distance(stringified_key, Atom.to_string(key)), + distance >= @threshold, + do: {distance, key} + + case suggestions do + [] -> [] + suggestions -> ["\nDid you mean:\n\n" | format_suggestions(suggestions)] + end + end + + defp format_suggestions(suggestions) do + suggestions + |> Enum.sort(&(elem(&1, 0) >= elem(&2, 0))) + |> Enum.take(@max_suggestions) + |> Enum.sort(&(elem(&1, 1) <= elem(&2, 1))) + |> Enum.map(fn {_, key} -> [" * ", inspect(key), ?\n] end) end end defmodule UnicodeConversionError do + @moduledoc """ + An exception raised when converting data to or from Unicode. + + For example: + + iex> String.to_charlist(<<0xFF>>) + ** (UnicodeConversionError) invalid encoding starting at <<255>> + + """ defexception [:encoded, :message] def exception(opts) do %UnicodeConversionError{ encoded: Keyword.fetch!(opts, :encoded), - message: "#{Keyword.fetch!(opts, :kind)} #{detail Keyword.fetch!(opts, :rest)}" + message: "#{Keyword.fetch!(opts, :kind)} #{detail(Keyword.fetch!(opts, :rest))}" } end defp detail(rest) when is_binary(rest) do - "encoding starting at #{inspect rest}" + "encoding starting at #{inspect(rest)}" end - defp detail([h|_]) do + defp detail([h | _]) when is_integer(h) do "code point #{h}" end + + defp detail([h | _]) do + detail(h) + end +end + +defmodule MissingApplicationsError do + @moduledoc """ + An exception that is raised when an application depends on one or more + missing applications. + + This exception is used by Mix and other tools. It can also be used by library authors + when their library only requires an external application (like a dependency) for a subset + of features. + + The fields of this exception are public. See `t:t/0`. + + *Available since v1.18.0.* + + ## Examples + + unless Application.spec(:plug, :vsn) do + raise MissingApplicationsError, + description: "application :plug is required for testing Plug-related functionality", + apps: [{:plug, "~> 1.0"}] + end + + """ + + @moduledoc since: "1.18.0" + + @type t() :: %__MODULE__{ + apps: [{Application.app(), Version.requirement()}, ...], + description: String.t() + } + + defexception apps: [], description: "missing applications found" + + @impl true + def message(%__MODULE__{apps: apps, description: description}) do + # We explicitly format these as tuples so that they're easier to copy-paste + # into dependencies. + formatted_apps = + Enum.map(apps, fn {app_name, requirement} -> + ~s(\n {#{inspect(app_name)}, "#{requirement}"}) + end) + + """ + #{description} + + To address this, include these applications as your dependencies: + #{formatted_apps}\ + """ + end end defmodule Enum.OutOfBoundsError do - defexception [] + @moduledoc """ + An exception that is raised when a function expects an enumerable to have + a certain size but finds that it is too small. + + For example: + + iex> Enum.fetch!([1, 2, 3], 5) + ** (Enum.OutOfBoundsError) out of bounds error at position 5 when traversing enumerable [1, 2, 3] + """ - def message(_) do - "out of bounds error" + defexception [:enumerable, :index, :message] + + @impl true + def message(exception = %{message: nil}), do: message(exception.index, exception.enumerable) + def message(%{message: message}), do: message + + def message(index, enumerable) do + "out of bounds error" <> + if index do + " at position #{index}" + else + "" + end <> + if enumerable do + " when traversing enumerable #{inspect(enumerable)}" + else + "" + end end end defmodule Enum.EmptyError do - defexception [] + @moduledoc """ + An exception that is raised when something expects a non-empty enumerable + but finds an empty one. - def message(_) do - "empty error" - end + For example: + + iex> Enum.min([]) + ** (Enum.EmptyError) empty error + + """ + + defexception message: "empty error" end defmodule File.Error do - defexception [reason: nil, action: "", path: nil] + @moduledoc """ + An exception that is raised when a file operation fails. + + For example, this exception is raised, when trying to read a non existent file: + + iex> File.read!("nonexistent_file.txt") + ** (File.Error) could not read file "nonexistent_file.txt": no such file or directory + + The following fields of this exception are public and can be accessed freely: + + * `:path` (`t:Path.t/0`) - the path of the file that caused the error + * `:reason` (`t:File.posix/0`) - the reason for the error + + """ + + defexception [:reason, :path, action: ""] + + @impl true + def message(%{action: action, reason: reason, path: path}) do + formatted = + case {action, reason} do + {"remove directory", :eexist} -> + "directory is not empty" + + _ -> + IO.iodata_to_binary(:file.format_error(reason)) + end + + "could not #{action} #{inspect(path)}: #{formatted}" + end +end + +defmodule File.CopyError do + @moduledoc """ + An exception that is raised when copying a file fails. + + For example, this exception is raised when trying to copy to file or directory that isn't present: + iex> File.cp_r!("non_existent", "source_dir/subdir") + ** (File.CopyError) could not copy recursively from "non_existent" to "source_dir/subdir". non_existent: no such file or directory + + The following fields of this exception are public and can be accessed freely: + + * `:source` (`t:Path.t/0`) - the source path + * `:destination` (`t:Path.t/0`) - the destination path + * `:reason` (`t:File.posix/0`) - the reason why the file could not be copied + + """ + + defexception [:reason, :source, :destination, on: "", action: ""] + + @impl true def message(exception) do formatted = IO.iodata_to_binary(:file.format_error(exception.reason)) - "could not #{exception.action} #{exception.path}: #{formatted}" + + location = + case exception.on do + "" -> "" + on -> ". #{on}" + end + + "could not #{exception.action} from #{inspect(exception.source)} to " <> + "#{inspect(exception.destination)}#{location}: #{formatted}" end end -defmodule File.CopyError do - defexception [reason: nil, action: "", source: nil, destination: nil, on: nil] +defmodule File.RenameError do + @moduledoc """ + An exception that is raised when renaming a file fails. + + For example, this exception is raised when trying to rename a file that isn't present: + + iex> File.rename!("source.txt", "target.txt") + ** (File.RenameError) could not rename from "source.txt" to "target.txt": no such file or directory + + The following fields of this exception are public and can be accessed freely: + + * `:source` (`t:Path.t/0`) - the source path + * `:destination` (`t:Path.t/0`) - the destination path + * `:reason` (`t:File.posix/0`) - the reason why the file could not be renamed + """ + + defexception [:reason, :source, :destination, on: "", action: ""] + + @impl true def message(exception) do formatted = IO.iodata_to_binary(:file.format_error(exception.reason)) - location = if on = exception.on, do: ". #{on}", else: "" - "could not #{exception.action} from #{exception.source} to " <> - "#{exception.destination}#{location}: #{formatted}" + + location = + case exception.on do + "" -> "" + on -> ". #{on}" + end + + "could not #{exception.action} from #{inspect(exception.source)} to " <> + "#{inspect(exception.destination)}#{location}: #{formatted}" end end -defmodule ErlangError do - defexception [original: nil] +defmodule File.LinkError do + @moduledoc """ + An exception that is raised when linking a file fails. + + For example, this exception is raised when trying to link to file that isn't present: + + iex> File.ln!("existing.txt", "link.txt") + ** (File.LinkError) could not create hard link from "link.txt" to "existing.txt": no such file or directory + + The following fields of this exception are public and can be accessed freely: + + * `:existing` (`t:Path.t/0`) - the existing file to link + * `:new` (`t:Path.t/0`) - the link destination + * `:reason` (`t:File.posix/0`) - the reason why the file could not be linked + + """ + + defexception [:reason, :existing, :new, action: ""] + @impl true def message(exception) do - "erlang error: #{inspect(exception.original)}" + formatted = IO.iodata_to_binary(:file.format_error(exception.reason)) + + "could not #{exception.action} from #{inspect(exception.new)} to " <> + "#{inspect(exception.existing)}: #{formatted}" + end +end + +defmodule ErlangError do + @moduledoc """ + An exception raised when invoking an Erlang code that errors + with a value not handled by Elixir. + + Most common error reasons, such as `:badarg` are automatically + converted into exceptions by Elixir. However, you may invoke some + code that emits a custom error reason and those get wrapped into + `ErlangError`: + + iex> :erlang.error(:some_invalid_error) + ** (ErlangError) Erlang error: :some_invalid_error + """ + + defexception [:original, :reason] + + @impl true + def message(exception) + + def message(%__MODULE__{original: original, reason: nil}) do + "Erlang error: #{inspect(original)}" + end + + def message(%__MODULE__{original: original, reason: reason}) do + IO.iodata_to_binary(["Erlang error: ", inspect(original), reason]) end @doc false - def normalize(:badarg, _stacktrace) do - %ArgumentError{} + def normalize(:badarg, stacktrace) do + case stacktrace do + [{:erlang, :apply, [module, function, args], _} | _] when not is_atom(module) -> + message = + cond do + is_map(module) and is_atom(function) and is_map_key(module, function) -> + "you attempted to apply a function named #{inspect(function)} on a map/struct. " <> + "If you are using Kernel.apply/3, make sure the module is an atom. " <> + if is_function(Map.get(module, function)) do + "If you are trying to invoke an anonymous function in a map/struct, " <> + "add a dot between the function name and the parenthesis: map.#{function}.()" + else + "If you are using the dot syntax, ensure there are no parentheses " <> + "after the field name, such as map.#{function}" + end + + is_atom(function) and args == [] -> + "you attempted to apply a function named #{inspect(function)} on #{inspect(module)}. " <> + "If you are using Kernel.apply/3, make sure the module is an atom. " <> + "If you are using the dot syntax, such as module.function(), " <> + "make sure the left-hand side of the dot is an atom representing a module" + + true -> + "you attempted to apply a function on #{inspect(module)}. " <> + "Modules (the first argument of apply) must always be an atom" + end + + %ArgumentError{message: message} + + _ -> + case error_info(:badarg, stacktrace, "errors were found at the given arguments") do + {:ok, reason, details} -> %ArgumentError{message: reason <> details} + :error -> %ArgumentError{} + end + end end def normalize(:badarith, _stacktrace) do %ArithmeticError{} end - def normalize(:system_limit, _stacktrace) do - %SystemLimitError{} + def normalize(:system_limit, stacktrace) do + default_reason = "a system limit has been reached due to errors at the given arguments" + + case error_info(:system_limit, stacktrace, default_reason) do + {:ok, reason, details} -> %SystemLimitError{message: reason <> details} + :error -> %SystemLimitError{} + end end def normalize(:cond_clause, _stacktrace) do @@ -753,29 +2591,65 @@ defmodule ErlangError do %BadFunctionError{term: term} end - def normalize({:badstruct, struct, term}, _stacktrace) do - %BadStructError{struct: struct, term: term} - end - def normalize({:badmatch, term}, _stacktrace) do %MatchError{term: term} end + def normalize({:badmap, term}, _stacktrace) do + %BadMapError{term: term} + end + + def normalize({:badbool, op, term}, _stacktrace) do + %BadBooleanError{operator: op, term: term} + end + + def normalize({:badkey, key}, stacktrace) do + term = + case stacktrace do + [{Map, :get_and_update!, [map, _, _], _} | _] -> map + [{Map, :update!, [map, _, _], _} | _] -> map + [{:maps, :update, [_, _, map], _} | _] -> map + [{:maps, :get, [_, map], _} | _] -> map + [{:erlang, :map_get, [_, map], _} | _] -> map + _ -> nil + end + + %KeyError{key: key, term: term} + end + + def normalize({:badkey, key, map}, _stacktrace) when is_map(map) do + %KeyError{key: key, term: map} + end + + def normalize({:badkey, key, term}, _stacktrace) do + message = + "key #{inspect(key)} not found in: #{inspect(term, pretty: true, limit: :infinity)}\n\n" <> + "If you are using the dot syntax, such as map.field, " <> + "make sure the left-hand side of the dot is a map" + + %KeyError{key: key, term: term, message: message} + end + def normalize({:case_clause, term}, _stacktrace) do %CaseClauseError{term: term} end + # :else_clause is aligned on what Erlang returns for `maybe` + def normalize({:else_clause, term}, _stacktrace) do + %WithClauseError{term: term} + end + def normalize({:try_clause, term}, _stacktrace) do %TryClauseError{term: term} end def normalize(:undef, stacktrace) do - {mod, fun, arity} = from_stacktrace(stacktrace || :erlang.get_stacktrace) + {mod, fun, arity} = from_stacktrace(stacktrace) %UndefinedFunctionError{module: mod, function: fun, arity: arity} end def normalize(:function_clause, stacktrace) do - {mod, fun, arity} = from_stacktrace(stacktrace || :erlang.get_stacktrace) + {mod, fun, arity} = from_stacktrace(stacktrace) %FunctionClauseError{module: mod, function: fun, arity: arity} end @@ -783,19 +2657,64 @@ defmodule ErlangError do %ArgumentError{message: "argument error: #{inspect(payload)}"} end - def normalize(other, _stacktrace) do - %ErlangError{original: other} + def normalize(other, stacktrace) do + case error_info(other, stacktrace, "") do + {:ok, _reason, details} -> %ErlangError{original: other, reason: details} + :error -> %ErlangError{original: other} + end end - defp from_stacktrace([{module, function, args, _}|_]) when is_list(args) do + defp from_stacktrace([{module, function, args, _} | _]) when is_list(args) do {module, function, length(args)} end - defp from_stacktrace([{module, function, arity, _}|_]) do + defp from_stacktrace([{module, function, arity, _} | _]) do {module, function, arity} end defp from_stacktrace(_) do {nil, nil, nil} end + + defp error_info(erl_exception, stacktrace, default_reason) do + with [{module, fun, args_or_arity, opts} | tail] <- stacktrace, + %{} = error_info <- opts[:error_info] do + error_module = Map.get(error_info, :module, module) + error_fun = Map.get(error_info, :function, :format_error) + + error_info = Map.put(error_info, :pretty_printer, &inspect/1) + head = {module, fun, args_or_arity, Keyword.put(opts, :error_info, error_info)} + + extra = + try do + apply(error_module, error_fun, [erl_exception, [head | tail]]) + rescue + _ -> %{} + end + + arity = if is_integer(args_or_arity), do: args_or_arity, else: length(args_or_arity) + args_errors = Map.take(extra, Enum.to_list(1..arity//1)) + reason = Map.get(extra, :reason, default_reason) + + cond do + map_size(args_errors) > 0 -> + {:ok, reason, IO.iodata_to_binary([":\n\n" | Enum.map(args_errors, &arg_error/1)])} + + general = extra[:general] -> + {:ok, reason, ": " <> IO.chardata_to_string(general)} + + true -> + :error + end + else + _ -> :error + end + end + + defp arg_error({n, message}), do: " * #{nth(n)} argument: #{message}\n" + + defp nth(1), do: "1st" + defp nth(2), do: "2nd" + defp nth(3), do: "3rd" + defp nth(n), do: "#{n}th" end diff --git a/lib/elixir/lib/file.ex b/lib/elixir/lib/file.ex index 23cc854db4c..255df12a0b5 100644 --- a/lib/elixir/lib/file.ex +++ b/lib/elixir/lib/file.ex @@ -1,37 +1,47 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + defmodule File do @moduledoc ~S""" This module contains functions to manipulate files. Some of those functions are low-level, allowing the user - to interact with the file or IO devices, like `open/2`, + to interact with files or IO devices, like `open/2`, `copy/3` and others. This module also provides higher level functions that work with filenames and have their naming - based on UNIX variants. For example, one can copy a file + based on Unix variants. For example, one can copy a file via `cp/3` and remove files and directories recursively - via `rm_rf/1` + via `rm_rf/1`. + + Paths given to functions in this module can be either relative to the + current working directory (as returned by `File.cwd/0`), or absolute + paths. Shell conventions like `~` are not expanded automatically. + To use paths like `~/Downloads`, you can use `Path.expand/1` or + `Path.expand/2` to expand your path to an absolute path. ## Encoding In order to write and read files, one must use the functions - in the `IO` module. By default, a file is opened in binary mode + in the `IO` module. By default, a file is opened in binary mode, which requires the functions `IO.binread/2` and `IO.binwrite/2` to interact with the file. A developer may pass `:utf8` as an option when opening the file, then the slower `IO.read/2` and `IO.write/2` functions must be used as they are responsible for - doing the proper conversions and data guarantees. + doing the proper conversions and providing the proper data guarantees. - Note that filenames when given as char lists in Elixir are + Note that filenames when given as charlists in Elixir are always treated as UTF-8. In particular, we expect that the - shell and the operating system are configured to use UTF8 - encoding. Binary filenames are considering raw and passed - to the OS as is. + shell and the operating system are configured to use UTF-8 + encoding. Binary filenames are considered raw and passed + to the operating system as is. ## API Most of the functions in this module return `:ok` or `{:ok, result}` in case of success, `{:error, reason}` - otherwise. Those function are also followed by a variant - that ends with `!` which returns the result (without the + otherwise. Those functions also have a variant + that ends with `!` which returns the result (instead of the `{:ok, result}` tuple) in case of success or raises an exception in case it fails. For example: @@ -47,15 +57,15 @@ defmodule File do File.read!("invalid.txt") #=> raises File.Error - In general, a developer should use the former in case he wants + In general, a developer should use the former in case they want to react if the file does not exist. The latter should be used - when the developer expects his software to fail in case the + when the developer expects their software to fail in case the file cannot be read (i.e. it is literally an exception). ## Processes and raw files Every time a file is opened, Elixir spawns a new process. Writing - to a file is equivalent to sending messages to that process that + to a file is equivalent to sending messages to the process that writes to the file descriptor. This means files can be passed between nodes and message passing @@ -63,44 +73,163 @@ defmodule File do However, you may not always want to pay the price for this abstraction. In such cases, a file can be opened in `:raw` mode. The options `:read_ahead` - and `:delayed_write` are also useful when operating large files or + and `:delayed_write` are also useful when operating on large files or working with files in tight loops. - Check http://www.erlang.org/doc/man/file.html#open-2 for more information - about such options and other performance considerations. - """ + Check `:file.open/2` for more information about such options and + other performance considerations. + + ## Seeking within a file + + You may also use any of the functions from the [`:file`](`:file`) + module to interact with files returned by Elixir. For example, + to read from a specific position in a file, use `:file.pread/3`: + + File.write!("example.txt", "Eats, Shoots & Leaves") + file = File.open!("example.txt") + :file.pread(file, 15, 6) + #=> {:ok, "Leaves"} - alias :file, as: F + Alternatively, if you need to keep track of the current position, + use `:file.position/2` and `:file.read/2`: + + :file.position(file, 6) + #=> {:ok, 6} + :file.read(file, 6) + #=> {:ok, "Shoots"} + :file.position(file, {:cur, -12}) + #=> {:ok, 0} + :file.read(file, 4) + #=> {:ok, "Eats"} + """ @type posix :: :file.posix() @type io_device :: :file.io_device() + @type file_descriptor :: :file.fd() @type stat_options :: [time: :local | :universal | :posix] + @type mode :: + :append + | :binary + | :charlist + | :compressed + | :delayed_write + | :exclusive + | :raw + | :read + | :read_ahead + | :sync + | :write + | {:read_ahead, pos_integer} + | {:delayed_write, non_neg_integer, non_neg_integer} + | encoding_mode() + + @type encoding_mode :: + :utf8 + | { + :encoding, + :latin1 + | :unicode + | :utf8 + | :utf16 + | :utf32 + | {:utf16, :big | :little} + | {:utf32, :big | :little} + } + + @type stream_mode :: + encoding_mode() + | read_offset_mode() + | :append + | :compressed + | :delayed_write + | :trim_bom + | {:read_ahead, pos_integer | false} + | {:delayed_write, non_neg_integer, non_neg_integer} + + @type read_offset_mode :: {:read_offset, non_neg_integer()} + + @type erlang_time :: + {{year :: non_neg_integer(), month :: 1..12, day :: 1..31}, + {hour :: 0..23, minute :: 0..59, second :: 0..59}} + + @type posix_time :: integer() + + @type on_conflict_callback :: (Path.t(), Path.t() -> boolean) @doc """ Returns `true` if the path is a regular file. + This function follows symbolic links, so if a symbolic link points to a + regular file, `true` is returned. + + ## Options + + The supported options are: + + * `:raw` - a single atom to bypass the file server and only check + for the file locally + ## Examples - File.regular? __ENV__.file #=> true + File.regular?(__ENV__.file) + #=> true """ - @spec regular?(Path.t) :: boolean - def regular?(path) do - :elixir_utils.read_file_type(IO.chardata_to_string(path)) == {:ok, :regular} + @spec regular?(Path.t(), [regular_option]) :: boolean + when regular_option: :raw + def regular?(path, opts \\ []) do + :elixir_utils.read_file_type(IO.chardata_to_string(path), opts) == {:ok, :regular} end @doc """ - Returns `true` if the path is a directory. + Returns `true` if the given path is a directory. + + This function follows symbolic links, so if a symbolic link points to a + directory, `true` is returned. + + ## Options + + The supported options are: + + * `:raw` - a single atom to bypass the file server and only check + for the file locally + + ## Examples + + File.dir?("./test") + #=> true + + File.dir?("test") + #=> true + + File.dir?("/usr/bin") + #=> true + + File.dir?("~/Downloads") + #=> false + + "~/Downloads" |> Path.expand() |> File.dir?() + #=> true + """ - @spec dir?(Path.t) :: boolean - def dir?(path) do - :elixir_utils.read_file_type(IO.chardata_to_string(path)) == {:ok, :directory} + @spec dir?(Path.t(), [dir_option]) :: boolean + when dir_option: :raw + def dir?(path, opts \\ []) do + :elixir_utils.read_file_type(IO.chardata_to_string(path), opts) == {:ok, :directory} end @doc """ Returns `true` if the given path exists. - It can be regular file, directory, socket, - symbolic link, named pipe or device file. + + It can be a regular file, directory, socket, symbolic link, named pipe, or device file. + Returns `false` for symbolic links pointing to non-existing targets. + + ## Options + + The supported options are: + + * `:raw` - a single atom to bypass the file server and only check + for the file locally ## Examples @@ -114,55 +243,91 @@ defmodule File do #=> true """ - @spec exists?(Path.t) :: boolean - def exists?(path) do - match?({:ok, _}, F.read_file_info(IO.chardata_to_string(path))) + @spec exists?(Path.t(), [exists_option]) :: boolean + when exists_option: :raw + def exists?(path, opts \\ []) do + opts = [{:time, :posix}] ++ opts + match?({:ok, _}, :file.read_file_info(IO.chardata_to_string(path), opts)) end @doc """ - Tries to create the directory `path`. Missing parent directories are not created. + Tries to create the directory `path`. + + Missing parent directories are not created. Returns `:ok` if successful, or `{:error, reason}` if an error occurs. Typical error reasons are: * `:eacces` - missing search or write permissions for the parent - directories of `path` + directories of `path` * `:eexist` - there is already a file or directory named `path` * `:enoent` - a component of `path` does not exist - * `:enospc` - there is a no space left on the device + * `:enospc` - there is no space left on the device * `:enotdir` - a component of `path` is not a directory; - on some platforms, `:enoent` is returned instead + on some platforms, `:enoent` is returned instead + + ## Examples + + File.mkdir("test/unit") + #=> :ok + + File.mkdir("non/existing") + #=> {:error, :enoent} """ - @spec mkdir(Path.t) :: :ok | {:error, posix} + @spec mkdir(Path.t()) :: :ok | {:error, posix | :badarg} def mkdir(path) do - F.make_dir(IO.chardata_to_string(path)) + :file.make_dir(IO.chardata_to_string(path)) end @doc """ - Same as `mkdir/1`, but raises an exception in case of failure. Otherwise `:ok`. + Same as `mkdir/1`, but raises a `File.Error` exception in case of failure. + Otherwise `:ok`. + + ## Examples + + File.mkdir!("test/unit") + #=> :ok + + File.mkdir!("non/existing") + ** (File.Error) could not make directory "non/existing": no such file or directory """ - @spec mkdir!(Path.t) :: :ok | no_return + @spec mkdir!(Path.t()) :: :ok def mkdir!(path) do - path = IO.chardata_to_string(path) case mkdir(path) do - :ok -> :ok + :ok -> + :ok + {:error, reason} -> - raise File.Error, reason: reason, action: "make directory", path: path + raise File.Error, + reason: reason, + action: "make directory", + path: IO.chardata_to_string(path) end end @doc """ - Tries to create the directory `path`. Missing parent directories are created. - Returns `:ok` if successful, or `{:error, reason}` if an error occurs. + Tries to create the directory `path`. + + Missing parent directories are created. Returns `:ok` if successful, or + `{:error, reason}` if an error occurs. Typical error reasons are: * `:eacces` - missing search or write permissions for the parent - directories of `path` - * `:enospc` - there is a no space left on the device + directories of `path` + * `:enospc` - there is no space left on the device * `:enotdir` - a component of `path` is not a directory + * `:eperm` - missed required permissions + + ## Examples + + File.mkdir_p("non/existing/parents") + #=> :ok + + File.mkdir_p("/usr/sbin/temp") + #=> {:error, :eperm} """ - @spec mkdir_p(Path.t) :: :ok | {:error, posix} + @spec mkdir_p(Path.t()) :: :ok | {:error, posix | :badarg} def mkdir_p(path) do do_mkdir_p(IO.chardata_to_string(path)) end @@ -172,35 +337,50 @@ defmodule File do end defp do_mkdir_p(path) do - if dir?(path) do + parent = Path.dirname(path) + + if parent == path do :ok else - parent = Path.dirname(path) - if parent == path do - # Protect against infinite loop - {:error, :einval} - else - _ = do_mkdir_p(parent) - case :file.make_dir(path) do - {:error, :eexist} = error -> - if dir?(path), do: :ok, else: error - other -> - other - end + case do_mkdir_p(parent) do + :ok -> + case :file.make_dir(path) do + {:error, :eexist} -> + if dir?(path), do: :ok, else: {:error, :enotdir} + + other -> + other + end + + e -> + e end end end @doc """ - Same as `mkdir_p/1`, but raises an exception in case of failure. Otherwise `:ok`. + Same as `mkdir_p/1`, but raises a `File.Error` exception in case of failure. + Otherwise `:ok`. + + ## Examples + + File.mkdir_p!("non/existing/parents") + #=> :ok + + File.mkdir_p!("/usr/sbin/temp") + ** (File.Error) could not make directory (with -p) "/usr/sbin/temp": not owner """ - @spec mkdir_p!(Path.t) :: :ok | no_return + @spec mkdir_p!(Path.t()) :: :ok def mkdir_p!(path) do - path = IO.chardata_to_string(path) case mkdir_p(path) do - :ok -> :ok + :ok -> + :ok + {:error, reason} -> - raise File.Error, reason: reason, action: "make directory (with -p)", path: path + raise File.Error, + reason: reason, + action: "make directory (with -p)", + path: IO.chardata_to_string(path) end end @@ -212,31 +392,47 @@ defmodule File do * `:enoent` - the file does not exist * `:eacces` - missing permission for reading the file, - or for searching one of the parent directories + or for searching one of the parent directories * `:eisdir` - the named file is a directory * `:enotdir` - a component of the file name is not a directory; - on some platforms, `:enoent` is returned instead + on some platforms, `:enoent` is returned instead * `:enomem` - there is not enough memory for the contents of the file You can use `:file.format_error/1` to get a descriptive string of the error. + + ## Examples + + File.read("hello.txt") + #=> {:ok, "world"} + + File.read("non_existing.txt") + #=> {:error, :enoent} """ - @spec read(Path.t) :: {:ok, binary} | {:error, posix} + @spec read(Path.t()) :: {:ok, binary} | {:error, posix | :badarg | :terminated | :system_limit} def read(path) do - F.read_file(IO.chardata_to_string(path)) + :file.read_file(IO.chardata_to_string(path)) end @doc """ - Returns binary with the contents of the given filename or raises - `File.Error` if an error occurs. + Returns a binary with the contents of the given filename, + or raises a `File.Error` exception if an error occurs. + + ## Examples + + File.read!("hello.txt") + #=> "world" + + File.read!("non_existing.txt") + ** (File.Error) could not read file "non_existing.txt": no such file or directory """ - @spec read!(Path.t) :: binary | no_return + @spec read!(Path.t()) :: binary def read!(path) do - path = IO.chardata_to_string(path) case read(path) do {:ok, binary} -> binary + {:error, reason} -> - raise File.Error, reason: reason, action: "read file", path: path + raise File.Error, reason: reason, action: "read file", path: IO.chardata_to_string(path) end end @@ -250,83 +446,353 @@ defmodule File do The accepted options are: - * `:time` - `:local | :universal | :posix`; default: `:local` + * `:time` - configures how the file timestamps are returned + + The values for `:time` can be: + + * `:universal` - returns a `{date, time}` tuple in UTC (default) + * `:local` - returns a `{date, time}` tuple using the same time zone as the + machine + * `:posix` - returns the time as integer seconds since epoch + + Note: Since file times are stored in POSIX time format on most operating systems, + it is faster to retrieve file information with the `time: :posix` option. + + ## Examples + + File.stat("hello.txt") + #=> {:ok, %File.Stat{...}} + File.stat("non_existing.txt", time: :posix) + #=> {:error, :enoent} """ - @spec stat(Path.t, stat_options) :: {:ok, File.Stat.t} | {:error, posix} + @spec stat(Path.t(), stat_options) :: {:ok, File.Stat.t()} | {:error, posix | :badarg} def stat(path, opts \\ []) do - case F.read_file_info(IO.chardata_to_string(path), opts) do + opts = Keyword.put_new(opts, :time, :universal) + + case :file.read_file_info(IO.chardata_to_string(path), opts) do {:ok, fileinfo} -> {:ok, File.Stat.from_record(fileinfo)} + error -> error end end @doc """ - Same as `stat/2` but returns the `File.Stat` directly and - throws `File.Error` if an error is returned. + Same as `stat/2` but returns the `File.Stat` directly, + or raises a `File.Error` exception if an error is returned. + + ## Examples + + File.stat!("hello.txt") + #=> %File.Stat{...} + + File.stat!("non_existing.txt", time: :posix) + ** (File.Error) could not read file stats "non_existing.txt": no such file or directory """ - @spec stat!(Path.t, stat_options) :: File.Stat.t | no_return + @spec stat!(Path.t(), stat_options) :: File.Stat.t() def stat!(path, opts \\ []) do - path = IO.chardata_to_string(path) case stat(path, opts) do - {:ok, info} -> info + {:ok, info} -> + info + {:error, reason} -> - raise File.Error, reason: reason, action: "read file stats", path: path + raise File.Error, + reason: reason, + action: "read file stats", + path: IO.chardata_to_string(path) + end + end + + @doc """ + Returns information about the `path`. If the file is a symlink, sets + the `type` to `:symlink` and returns a `File.Stat` struct for the link. For any + other file, returns exactly the same values as `stat/2`. + + For more details, see `:file.read_link_info/2`. + + ## Options + + The accepted options are: + + * `:time` - configures how the file timestamps are returned + + The values for `:time` can be: + + * `:universal` - returns a `{date, time}` tuple in UTC (default) + * `:local` - returns a `{date, time}` tuple using the machine time + * `:posix` - returns the time as integer seconds since epoch + + Note: Since file times are stored in POSIX time format on most operating systems, + it is faster to retrieve file information with the `time: :posix` option. + + ## Examples + + File.lstat("link_to_hello") + #=> {:ok, %File.Stat{type: :symlink, ...}} + + File.lstat("non_existing.txt", time: :posix) + #=> {:error, :enoent} + """ + @spec lstat(Path.t(), stat_options) :: {:ok, File.Stat.t()} | {:error, posix | :badarg} + def lstat(path, opts \\ []) do + opts = Keyword.put_new(opts, :time, :universal) + + case :file.read_link_info(IO.chardata_to_string(path), opts) do + {:ok, fileinfo} -> + {:ok, File.Stat.from_record(fileinfo)} + + error -> + error end end @doc """ - Writes the given `File.Stat` back to the filesystem at the given + Same as `lstat/2` but returns the `File.Stat` struct directly, + or raises a `File.Error` exception if an error is returned. + + ## Examples + + File.lstat!("link_to_hello") + #=> %File.Stat{type: :symlink, ...} + + File.lstat!("non_existing.txt", time: :posix) + ** (File.Error) could not read file stats "non_existing.txt": no such file or directory + """ + @spec lstat!(Path.t(), stat_options) :: File.Stat.t() + def lstat!(path, opts \\ []) do + case lstat(path, opts) do + {:ok, info} -> + info + + {:error, reason} -> + raise File.Error, + reason: reason, + action: "read file stats", + path: IO.chardata_to_string(path) + end + end + + @doc """ + Reads the symbolic link at `path`. + + If `path` exists and is a symlink, returns `{:ok, target}`, otherwise returns + `{:error, reason}`. + + For more details, see `:file.read_link/1`. + + Typical error reasons are: + + * `:einval` - path is not a symbolic link + * `:enoent` - path does not exist + * `:enotsup` - symbolic links are not supported on the current platform + + ## Examples + + File.read_link("link_to_hello") + #=> {:ok, "hello.txt"} + + File.read_link("hello.txt") + #=> {:error, :einval} + """ + @doc since: "1.5.0" + @spec read_link(Path.t()) :: {:ok, binary} | {:error, posix | :badarg} + def read_link(path) do + case path |> IO.chardata_to_string() |> :file.read_link() do + {:ok, target} -> {:ok, IO.chardata_to_string(target)} + error -> error + end + end + + @doc """ + Same as `read_link/1` but returns the target directly, + or raises a `File.Error` exception if an error is returned. + + ## Examples + + File.read_link!("link_to_hello") + #=> "hello.txt" + + File.read_link!("hello.txt") + ** (File.Error) could not read link "hello.txt": invalid argument + """ + @doc since: "1.5.0" + @spec read_link!(Path.t()) :: binary + def read_link!(path) do + case read_link(path) do + {:ok, resolved} -> + resolved + + {:error, reason} -> + raise File.Error, reason: reason, action: "read link", path: IO.chardata_to_string(path) + end + end + + @doc """ + Writes the given `File.Stat` back to the file system at the given path. Returns `:ok` or `{:error, reason}`. + + ## Examples + + File.write_stat("hello.txt", new_stat) + #=> :ok + + File.write_stat("non_existing.txt", new_stat) + #=> {:error, :enoent} """ - @spec write_stat(Path.t, File.Stat.t, stat_options) :: :ok | {:error, posix} + @spec write_stat(Path.t(), File.Stat.t(), stat_options) :: :ok | {:error, posix | :badarg} def write_stat(path, stat, opts \\ []) do - F.write_file_info(IO.chardata_to_string(path), File.Stat.to_record(stat), opts) + opts = Keyword.put_new(opts, :time, :universal) + :file.write_file_info(IO.chardata_to_string(path), File.Stat.to_record(stat), opts) end @doc """ - Same as `write_stat/3` but raises an exception if it fails. + Same as `write_stat/3` but raises a `File.Error` exception if it fails. Returns `:ok` otherwise. + + ## Examples + + File.write_stat!("hello.txt", new_stat) + #=> :ok + + File.write_stat!("non_existing.txt", new_stat) + ** (File.Error) could not write file stats "non_existing.txt": no such file or directory """ - @spec write_stat!(Path.t, File.Stat.t, stat_options) :: :ok | no_return + @spec write_stat!(Path.t(), File.Stat.t(), stat_options) :: :ok def write_stat!(path, stat, opts \\ []) do - path = IO.chardata_to_string(path) case write_stat(path, stat, opts) do - :ok -> :ok + :ok -> + :ok + {:error, reason} -> - raise File.Error, reason: reason, action: "write file stats", path: path + raise File.Error, + reason: reason, + action: "write file stats", + path: IO.chardata_to_string(path) end end @doc """ Updates modification time (mtime) and access time (atime) of - the given file. File is created if it doesn’t exist. + the given file. + + The file is created if it doesn't exist. Requires datetime in UTC + (as returned by `:erlang.universaltime()`) or an integer + representing the POSIX timestamp (as returned by `System.os_time(:second)`). + + In Unix-like systems, changing the modification time may require + you to be either `root` or the owner of the file. Having write + access may not be enough. In those cases, touching the file the + first time (to create it) will succeed, but touching an existing + file with fail with `{:error, :eperm}`. + + ## Examples + + File.touch("/tmp/a.txt", {{2018, 1, 30}, {13, 59, 59}}) + #=> :ok + File.touch("/fakedir/b.txt", {{2018, 1, 30}, {13, 59, 59}}) + {:error, :enoent} + + File.touch("/tmp/a.txt", 1_544_519_753) + #=> :ok + """ - @spec touch(Path.t, :calendar.datetime) :: :ok | {:error, posix} - def touch(path, time \\ :calendar.local_time) do + @spec touch(Path.t(), erlang_time() | posix_time()) :: + :ok | {:error, posix | :badarg | :terminated | :system_limit} + def touch(path, time \\ System.os_time(:second)) + + def touch(path, time) when is_tuple(time) do path = IO.chardata_to_string(path) - case F.change_time(path, time) do - {:error, :enoent} -> - write(path, "") - F.change_time(path, time) - other -> - other - end + + with {:error, :enoent} <- :elixir_utils.change_universal_time(path, time), + :ok <- write(path, "", [:raw, :append]), + do: :elixir_utils.change_universal_time(path, time) + end + + def touch(path, time) when is_integer(time) do + path = IO.chardata_to_string(path) + + with {:error, :enoent} <- :elixir_utils.change_posix_time(path, time), + :ok <- write(path, "", [:raw, :append]), + do: :elixir_utils.change_posix_time(path, time) end @doc """ - Same as `touch/2` but raises an exception if it fails. + Same as `touch/2` but raises a `File.Error` exception if it fails. Returns `:ok` otherwise. + + The file is created if it doesn't exist. Requires datetime in UTC + (as returned by `:erlang.universaltime()`) or an integer + representing the POSIX timestamp (as returned by `System.os_time(:second)`). + + ## Examples + + File.touch!("/tmp/a.txt", {{2018, 1, 30}, {13, 59, 59}}) + #=> :ok + File.touch!("/fakedir/b.txt", {{2018, 1, 30}, {13, 59, 59}}) + ** (File.Error) could not touch "/fakedir/b.txt": no such file or directory + + File.touch!("/tmp/a.txt", 1_544_519_753) + """ - @spec touch!(Path.t, :calendar.datetime) :: :ok | no_return - def touch!(path, time \\ :calendar.local_time) do - path = IO.chardata_to_string(path) + @spec touch!(Path.t(), erlang_time() | posix_time()) :: :ok + def touch!(path, time \\ System.os_time(:second)) do case touch(path, time) do - :ok -> :ok + :ok -> + :ok + {:error, reason} -> - raise File.Error, reason: reason, action: "touch", path: path + raise File.Error, reason: reason, action: "touch", path: IO.chardata_to_string(path) + end + end + + @doc """ + Creates a hard link `new` to the file `existing`. + + Returns `:ok` if successful, `{:error, reason}` otherwise. + If the operating system does not support hard links, returns + `{:error, :enotsup}`. + + ## Examples + + File.ln("hello.txt", "hard_link_to_hello") + #=> :ok + + File.ln("non_existing.txt", "link") + #=> {:error, :enoent} + """ + @doc since: "1.5.0" + @spec ln(Path.t(), Path.t()) :: :ok | {:error, posix | :badarg} + def ln(existing, new) do + :file.make_link(IO.chardata_to_string(existing), IO.chardata_to_string(new)) + end + + @doc """ + Same as `ln/2` but raises a `File.LinkError` exception if it fails. + Returns `:ok` otherwise. + + ## Examples + + File.ln!("hello.txt", "hard_link_to_hello") + #=> :ok + + File.ln!("non_existing.txt", "link") + ** (File.LinkError) could not create hard link from "non_existing.txt" to "link": no such file or directory + """ + @doc since: "1.5.0" + @spec ln!(Path.t(), Path.t()) :: :ok + def ln!(existing, new) do + case ln(existing, new) do + :ok -> + :ok + + {:error, reason} -> + raise File.LinkError, + reason: reason, + action: "create hard link", + existing: IO.chardata_to_string(existing), + new: IO.chardata_to_string(new) end end @@ -336,15 +802,59 @@ defmodule File do Returns `:ok` if successful, `{:error, reason}` otherwise. If the operating system does not support symlinks, returns `{:error, :enotsup}`. + + Creates a symlink even if the `existing` target actually doesn't exist + + ## Examples + + File.ln_s("hello.txt", "link_to_hello") + #=> :ok + + File.ln_s("non_existing.txt", "link") + #=> :ok + + # Returns error if `new` file exists + File.ln_s("non_existing.txt", "existed_link") + #=> {:error, :eexist} """ + @doc since: "1.5.0" + @spec ln_s(Path.t(), Path.t()) :: :ok | {:error, posix | :badarg} def ln_s(existing, new) do - F.make_symlink(existing, new) + :file.make_symlink(IO.chardata_to_string(existing), IO.chardata_to_string(new)) + end + + @doc """ + Same as `ln_s/2` but raises a `File.LinkError` exception if it fails. + Returns `:ok` otherwise. + + ## Examples + + File.ln_s!("hello.txt", "link_to_hello") + #=> :ok + + # Raises if `new` file exists + File.ln_s!("non_existing.txt", "existed_link") + ** (File.LinkError) could not create symlink from "non_existing.txt" to "existed_link": file already exists + """ + @spec ln_s!(Path.t(), Path.t()) :: :ok + def ln_s!(existing, new) do + case ln_s(existing, new) do + :ok -> + :ok + + {:error, reason} -> + raise File.LinkError, + reason: reason, + action: "create symlink", + existing: IO.chardata_to_string(existing), + new: IO.chardata_to_string(new) + end end @doc """ Copies the contents of `source` to `destination`. - Both parameters can be a filename or an io device opened + Both parameters can be a filename or an IO device opened with `open/2`. `bytes_count` specifies the number of bytes to copy, the default being `:infinity`. @@ -362,222 +872,472 @@ defmodule File do Typical error reasons are the same as in `open/2`, `read/1` and `write/3`. + + ## Examples + + File.copy("hello.txt", "hello_copy.txt") + #=> {:ok, 6} + + File.copy("non_existing.txt", "copy.txt") + #=> {:error, :enoent} """ - @spec copy(Path.t, Path.t, pos_integer | :infinity) :: {:ok, non_neg_integer} | {:error, posix} + @spec copy(Path.t() | io_device, Path.t() | io_device, pos_integer | :infinity) :: + {:ok, non_neg_integer} | {:error, posix | :badarg | :terminated} def copy(source, destination, bytes_count \\ :infinity) do - F.copy(IO.chardata_to_string(source), IO.chardata_to_string(destination), bytes_count) + source = normalize_path_or_io_device(source) + destination = normalize_path_or_io_device(destination) + + :file.copy(source, destination, bytes_count) end @doc """ - The same as `copy/3` but raises an `File.CopyError` if it fails. + The same as `copy/3` but raises a `File.CopyError` exception if it fails. Returns the `bytes_copied` otherwise. + + ## Examples + + File.copy!("hello.txt", "hello_copy.txt") + #=> 6 + + File.copy!("non_existing.txt", "copy.txt") + ** (File.CopyError) could not copy from "non_existing.txt" to "copy.txt": no such file or directory """ - @spec copy!(Path.t, Path.t, pos_integer | :infinity) :: non_neg_integer | no_return + @spec copy!(Path.t() | io_device, Path.t() | io_device, pos_integer | :infinity) :: + non_neg_integer def copy!(source, destination, bytes_count \\ :infinity) do - source = IO.chardata_to_string(source) - destination = IO.chardata_to_string(destination) case copy(source, destination, bytes_count) do - {:ok, bytes_count} -> bytes_count + {:ok, bytes_count} -> + bytes_count + {:error, reason} -> - raise File.CopyError, reason: reason, action: "copy", - source: source, destination: destination + raise File.CopyError, + reason: reason, + action: "copy", + source: normalize_path_or_io_device(source), + destination: normalize_path_or_io_device(destination) end end @doc """ - Copies the contents in `source` to `destination` preserving its mode. + Renames the `source` file to `destination` file. It can be used to move files + (and directories) between directories. If moving a file, you must fully + specify the `destination` filename, it is not sufficient to simply specify + its directory. - If a file already exists in the destination, it invokes a - callback which should return `true` if the existing file - should be overwritten, `false` otherwise. It defaults to return `true`. + Returns `:ok` in case of success, `{:error, reason}` otherwise. - It returns `:ok` in case of success, returns - `{:error, reason}` otherwise. + Note: The command `mv` in Unix-like systems behaves differently depending on + whether `source` is a file and the `destination` is an existing directory. + We have chosen to explicitly disallow this behavior. - If you want to copy contents from an io device to another device - or do a straight copy from a source to a destination without - preserving modes, check `copy/3` instead. + ## Examples + + # Rename file "a.txt" to "b.txt" + File.rename("a.txt", "b.txt") + #=> :ok + + # Rename directory "samples" to "tmp" + File.rename("samples", "tmp") + #=> :ok - Note: The command `cp` in Unix systems behaves differently depending - if `destination` is an existing directory or not. We have chosen to - explicitly disallow this behaviour. If destination is a directory, an - error will be returned. + File.rename("non_existing.txt", "existing.txt") + #=> {:error, :enoent} """ - @spec cp(Path.t, Path.t, (Path.t, Path.t -> boolean)) :: :ok | {:error, posix} - def cp(source, destination, callback \\ fn(_, _) -> true end) do + @doc since: "1.1.0" + @spec rename(Path.t(), Path.t()) :: :ok | {:error, posix | :badarg} + def rename(source, destination) do source = IO.chardata_to_string(source) destination = IO.chardata_to_string(destination) + :file.rename(source, destination) + end + + @doc """ + The same as `rename/2` but raises a `File.RenameError` exception if it fails. + Returns `:ok` otherwise. + + ## Examples + + File.rename!("samples", "tmp") + #=> :ok + + File.rename!("non_existing.txt", "existing.txt") + ** (File.RenameError) could not rename from "non_existing.txt" to "existing.txt": no such file or directory + """ + @doc since: "1.9.0" + @spec rename!(Path.t(), Path.t()) :: :ok + def rename!(source, destination) do + case rename(source, destination) do + :ok -> + :ok + + {:error, reason} -> + raise File.RenameError, + reason: reason, + action: "rename", + source: IO.chardata_to_string(source), + destination: IO.chardata_to_string(destination) + end + end - case do_cp_file(source, destination, callback, []) do + @doc ~S""" + Copies the contents of `source_file` to `destination_file` preserving its modes. + + `source_file` must be a file or a symbolic link to one. `destination_file` must + be a path to a non-existent file. If either is a directory, `{:error, :eisdir}` + will be returned. + + The function returns `:ok` in case of success. Otherwise, it returns + `{:error, reason}`. + + If you want to copy contents from an IO device to another device + or do a straight copy from a source to a destination without + preserving modes, check `copy/3` instead. + + Note: The command `cp` in Unix-like systems behaves differently depending on + whether the destination is an existing directory or not. We have chosen to + explicitly disallow copying to a destination which is a directory, + and an error will be returned if tried. + + ## Options + + * `:on_conflict` - (since v1.14.0) Invoked when a file already exists in the destination. + The function receives arguments for `source_file` and `destination_file`. It should + return `true` if the existing file should be overwritten, `false` if otherwise. + The default callback returns `true`. On earlier versions, this callback could be + given as third argument, but such behavior is now deprecated. + + ## Examples + + File.cp("hello.txt", "hello_copy.txt") + #=> :ok + + File.cp("hello.txt", "hello_copy.txt", on_conflict: fn source, destination -> + IO.gets("Overwriting #{destination} by #{source}. Type y to confirm. ") == "y\n" + end) + #=> :ok + + File.cp("non_existing.txt", "copy.txt") + #=> {:error, :enoent} + """ + @spec cp(Path.t(), Path.t(), on_conflict: on_conflict_callback) :: + :ok | {:error, posix | :badarg | :terminated} + def cp(source_file, destination_file, options \\ []) + + # TODO: Deprecate me on Elixir v1.19 + def cp(source_file, destination_file, callback) when is_function(callback, 2) do + IO.warn_once( + {__MODULE__, :cp}, + fn -> + "passing a callback to File.cp/3 is deprecated, pass it as a on_conflict: callback option instead" + end, + 3 + ) + + cp(source_file, destination_file, on_conflict: callback) + end + + def cp(source_file, destination_file, options) when is_list(options) do + on_conflict = Keyword.get(options, :on_conflict, fn _, _ -> true end) + source_file = IO.chardata_to_string(source_file) + destination_file = IO.chardata_to_string(destination_file) + + case do_cp_file(source_file, destination_file, on_conflict, []) do {:error, reason, _} -> {:error, reason} _ -> :ok end end - @doc """ - The same as `cp/3`, but raises `File.CopyError` if it fails. - Returns the list of copied files otherwise. + defp path_differs?(path, path), do: false + + defp path_differs?(p1, p2) do + Path.expand(p1) !== Path.expand(p2) + end + + @doc ~S""" + The same as `cp/3`, but raises a `File.CopyError` exception if it fails. + Returns `:ok` otherwise. + + ## Examples + + File.cp!("hello.txt", "hello_copy.txt") + #=> :ok + + File.cp!("hello.txt", "hello_copy.txt", on_conflict: fn source, destination -> + IO.gets("Overwriting #{destination} by #{source}. Type y to confirm. ") == "y\n" + end) + #=> :ok + + File.cp!("non_existing.txt", "copy.txt") + ** (File.CopyError) could not copy from "non_existing.txt" to "copy.txt": no such file or directory """ - @spec cp!(Path.t, Path.t, (Path.t, Path.t -> boolean)) :: :ok | no_return - def cp!(source, destination, callback \\ fn(_, _) -> true end) do - source = IO.chardata_to_string(source) - destination = IO.chardata_to_string(destination) + @spec cp!(Path.t(), Path.t(), on_conflict: on_conflict_callback) :: :ok + def cp!(source_file, destination_file, options \\ []) do + case cp(source_file, destination_file, options) do + :ok -> + :ok - case cp(source, destination, callback) do - :ok -> :ok {:error, reason} -> - raise File.CopyError, reason: reason, action: "copy recursively", - source: source, destination: destination + raise File.CopyError, + reason: reason, + action: "copy", + source: IO.chardata_to_string(source_file), + destination: IO.chardata_to_string(destination_file) end end @doc ~S""" - Copies the contents in source to destination. + Copies the contents in `source` to `destination` recursively, maintaining the + source directory structure and modes. - If the source is a file, it copies `source` to - `destination`. If the source is a directory, it copies - the contents inside source into the destination. + If `source` is a file or a symbolic link to it, `destination` must be a path + to an existent file, a symbolic link to one, or a path to a non-existent file. - If a file already exists in the destination, - it invokes a callback which should return - `true` if the existing file should be overwritten, - `false` otherwise. It defaults to return `true`. + If `source` is a directory, or a symbolic link to it, then `destination` must + be an existent `directory` or a symbolic link to one, or a path to a non-existent directory. - If a directory already exists in the destination - where a file is meant to be (or otherwise), this - function will fail. + If the source is a file, it copies `source` to `destination`. If the `source` + is a directory, it copies the contents inside source into the `destination` directory. - This function may fail while copying files, - in such cases, it will leave the destination - directory in a dirty state, where already - copied files won't be removed. + If a file already exists in the destination, it invokes the optional `on_conflict` + callback given as an option. See "Options" for more information. - It returns `{:ok, files_and_directories}` in case of - success with all files and directories copied in no - specific order, `{:error, reason, file}` otherwise. + This function may fail while copying files, in such cases, it will leave the + destination directory in a dirty state, where file which have already been + copied won't be removed. - Note: The command `cp` in Unix systems behaves differently - depending if `destination` is an existing directory or not. - We have chosen to explicitly disallow this behaviour. + The function returns `{:ok, files_and_directories}` in case of + success, `files_and_directories` lists all files and directories copied in no + specific order. It returns `{:error, reason, file}` otherwise. + + Note: The command `cp` in Unix-like systems behaves differently depending on + whether `destination` is an existing directory or not. We have chosen to + explicitly disallow this behavior. If `source` is a `file` and `destination` + is a directory, `{:error, :eisdir}` will be returned. + + ## Options + + * `:on_conflict` - (since v1.14.0) Invoked when a file already exists in the destination. + The function receives arguments for `source` and `destination`. It should return + `true` if the existing file should be overwritten, `false` if otherwise. The default + callback returns `true`. On earlier versions, this callback could be given as third + argument, but such behavior is now deprecated. + + * `:dereference_symlinks` - (since v1.14.0) By default, this function will copy symlinks + by creating symlinks that point to the same location. This option forces symlinks to be + dereferenced and have their contents copied instead when set to `true`. If the dereferenced + files do not exist, than the operation fails. The default is `false`. ## Examples - # Copies "a.txt" to "tmp" - File.cp_r "a.txt", "tmp.txt" + # Copies file "a.txt" to "b.txt" + File.cp_r("a.txt", "b.txt") + #=> {:ok, ["b.txt"]} # Copies all files in "samples" to "tmp" - File.cp_r "samples", "tmp" + File.cp_r("samples", "tmp") + #=> {:ok, ["z.txt", "y.txt", "x.txt]} # Same as before, but asks the user how to proceed in case of conflicts - File.cp_r "samples", "tmp", fn(source, destination) -> - IO.gets("Overwriting #{destination} by #{source}. Type y to confirm.") == "y" - end + File.cp_r("samples", "tmp", on_conflict: fn source, destination -> + IO.gets("Overwriting #{destination} by #{source}. Type y to confirm. ") == "y\n" + end) + #=> {:ok, ["z.txt", "y.txt", "x.txt]} + File.cp_r("non_existing.txt", "copy.txt") + #=> {:error, :enoent} """ - @spec cp_r(Path.t, Path.t, (Path.t, Path.t -> boolean)) :: {:ok, [binary]} | {:error, posix, binary} - def cp_r(source, destination, callback \\ fn(_, _) -> true end) when is_function(callback) do - source = IO.chardata_to_string(source) - destination = IO.chardata_to_string(destination) - - case do_cp_r(source, destination, callback, []) do + @spec cp_r(Path.t(), Path.t(), + on_conflict: on_conflict_callback, + dereference_symlinks: boolean() + ) :: + {:ok, [binary]} | {:error, posix | :badarg | :terminated, binary} + + def cp_r(source, destination, options \\ []) + + # TODO: Deprecate me on Elixir v1.19 + def cp_r(source, destination, callback) when is_function(callback, 2) do + IO.warn_once( + {__MODULE__, :cp_r}, + fn -> + "passing a callback to File.cp_r/3 is deprecated, pass it as a on_conflict: callback option instead" + end, + 3 + ) + + cp_r(source, destination, on_conflict: callback) + end + + def cp_r(source, destination, options) when is_list(options) do + on_conflict = Keyword.get(options, :on_conflict, fn _, _ -> true end) + dereference? = Keyword.get(options, :dereference_symlinks, false) + + source = + source + |> IO.chardata_to_string() + |> assert_no_null_byte!("File.cp_r/3") + + destination = + destination + |> IO.chardata_to_string() + |> assert_no_null_byte!("File.cp_r/3") + + case do_cp_r(source, destination, on_conflict, dereference?, []) do {:error, _, _} = error -> error res -> {:ok, res} end end @doc """ - The same as `cp_r/3`, but raises `File.CopyError` if it fails. + The same as `cp_r/3`, but raises a `File.CopyError` exception if it fails. Returns the list of copied files otherwise. + + ## Examples + + File.cp_r!("a.txt", "b.txt") + #=> ["b.txt"] + + File.cp_r!("samples", "tmp") + #=> ["z.txt", "y.txt", "x.txt] + + File.cp_r!("non_existing.txt", "copy.txt") + ** (File.CopyError) could not copy recursively from "non_existing.txt" to "copy.txt". non_existing.txt: no such file or directory """ - @spec cp_r!(Path.t, Path.t, (Path.t, Path.t -> boolean)) :: [binary] | no_return - def cp_r!(source, destination, callback \\ fn(_, _) -> true end) do - source = IO.chardata_to_string(source) - destination = IO.chardata_to_string(destination) + @spec cp_r!(Path.t(), Path.t(), + on_conflict: on_conflict_callback, + dereference_symlinks: boolean() + ) :: [binary] + def cp_r!(source, destination, options \\ []) do + case cp_r(source, destination, options) do + {:ok, files} -> + files - case cp_r(source, destination, callback) do - {:ok, files} -> files {:error, reason, file} -> - raise File.CopyError, reason: reason, action: "copy recursively", - source: source, destination: destination, on: file + raise File.CopyError, + reason: reason, + action: "copy recursively", + on: file, + source: IO.chardata_to_string(source), + destination: IO.chardata_to_string(destination) end end - # src may be a file or a directory, dest is definitely - # a directory. Returns nil unless an error is found. - defp do_cp_r(src, dest, callback, acc) when is_list(acc) do + defp do_cp_r(src, dest, on_conflict, dereference?, acc) when is_list(acc) do case :elixir_utils.read_link_type(src) do {:ok, :regular} -> - do_cp_file(src, dest, callback, acc) + case do_cp_file(src, dest, on_conflict, acc) do + # we don't have a way to make a distinction between a non-existing src + # or dest being a non-existing dir in the case of :enoent, + # but we already know that src exists here. + {:error, :enoent, _} -> {:error, :enoent, dest} + other -> other + end + {:ok, :symlink} -> - case F.read_link(src) do - {:ok, link} -> do_cp_link(link, src, dest, callback, acc) - {:error, reason} -> {:error, reason, src} + case :file.read_link(src) do + {:ok, link} when dereference? -> + do_cp_r(Path.expand(link, Path.dirname(src)), dest, on_conflict, dereference?, acc) + + {:ok, link} -> + do_cp_link(link, src, dest, on_conflict, acc) + + {:error, reason} -> + {:error, reason, src} end + {:ok, :directory} -> - case F.list_dir(src) do + case :file.list_dir(src) do {:ok, files} -> case mkdir(dest) do success when success in [:ok, {:error, :eexist}] -> - Enum.reduce(files, [dest|acc], fn(x, acc) -> - do_cp_r(Path.join(src, x), Path.join(dest, x), callback, acc) + Enum.reduce(files, [dest | acc], fn x, acc -> + do_cp_r(Path.join(src, x), Path.join(dest, x), on_conflict, dereference?, acc) end) - {:error, reason} -> {:error, reason, dest} + + {:error, reason} -> + {:error, reason, dest} end - {:error, reason} -> {:error, reason, src} + + {:error, reason} -> + {:error, reason, src} end - {:ok, _} -> {:error, :eio, src} - {:error, reason} -> {:error, reason, src} + + {:ok, _} -> + {:error, :eio, src} + + {:error, reason} -> + {:error, reason, src} end end - # If we reach this clause, there was an error while - # processing a file. - defp do_cp_r(_, _, _, acc) do + # If we reach this clause, there was an error while processing a file. + defp do_cp_r(_, _, _, _, acc) do acc end - defp copy_file_mode!(src, dest) do - write_stat!(dest, %{stat!(dest) | mode: stat!(src).mode}) + defp copy_file_mode(src, dest) do + with {:ok, dest_fileinfo} <- stat(dest), + {:ok, src_fileinfo} <- stat(src) do + write_stat(dest, %{dest_fileinfo | mode: src_fileinfo.mode}) + end end # Both src and dest are files. - defp do_cp_file(src, dest, callback, acc) do - case F.copy(src, {dest, [:exclusive]}) do + defp do_cp_file(src, dest, on_conflict, acc) do + case :file.copy(src, {dest, [:exclusive]}) do {:ok, _} -> - copy_file_mode!(src, dest) - [dest|acc] + case copy_file_mode(src, dest) do + :ok -> + [dest | acc] + + {:error, reason} -> + {:error, reason, src} + end + {:error, :eexist} -> - if callback.(src, dest) do - rm(dest) + if path_differs?(src, dest) and on_conflict.(src, dest) do case copy(src, dest) do {:ok, _} -> - copy_file_mode!(src, dest) - [dest|acc] - {:error, reason} -> {:error, reason, src} + case copy_file_mode(src, dest) do + :ok -> + [dest | acc] + + {:error, reason} -> + {:error, reason, src} + end + + {:error, reason} -> + {:error, reason, src} end else acc end - {:error, reason} -> {:error, reason, src} + + {:error, reason} -> + {:error, reason, src} end end # Both src and dest are files. - defp do_cp_link(link, src, dest, callback, acc) do - case F.make_symlink(link, dest) do + defp do_cp_link(link, src, dest, on_conflict, acc) do + case :file.make_symlink(link, dest) do :ok -> - [dest|acc] + [dest | acc] + {:error, :eexist} -> - if callback.(src, dest) do - rm(dest) - case F.make_symlink(link, dest) do - :ok -> [dest|acc] + if path_differs?(src, dest) and on_conflict.(src, dest) do + # If rm/1 fails, :file.make_symlink/2 will fail + _ = rm(dest) + + case :file.make_symlink(link, dest) do + :ok -> [dest | acc] {:error, reason} -> {:error, reason, src} end else acc end - {:error, reason} -> {:error, reason, src} + + {:error, reason} -> + {:error, reason, src} end end @@ -588,39 +1348,66 @@ defmodule File do contents are overwritten. Returns `:ok` if successful, or `{:error, reason}` if an error occurs. + `content` must be `iodata` (a list of bytes or a binary). Setting the + encoding for this function has no effect. + **Warning:** Every time this function is invoked, a file descriptor is opened and a new process is spawned to write to the file. For this reason, if you are doing multiple writes in a loop, opening the file via `File.open/2` and using the functions in `IO` to write to the file will yield much better performance - then calling this function multiple times. + than calling this function multiple times. Typical error reasons are: * `:enoent` - a component of the file name does not exist * `:enotdir` - a component of the file name is not a directory; - on some platforms, enoent is returned instead - * `:enospc` - there is a no space left on the device + on some platforms, `:enoent` is returned instead + * `:enospc` - there is no space left on the device * `:eacces` - missing permission for writing the file or searching one of - the parent directories + the parent directories * `:eisdir` - the named file is a directory - Check `File.open/2` for other available options. + Check `File.open/2` for the list of available `modes`. + + ## Examples + + File.write("hello.txt", "world!") + #=> :ok + + File.write("temp", "world!") + #=> {:error, :eisdir} + """ - @spec write(Path.t, iodata, list) :: :ok | {:error, posix} + @spec write(Path.t(), iodata, [mode]) :: + :ok | {:error, posix | :badarg | :terminated | :system_limit} def write(path, content, modes \\ []) do - F.write_file(IO.chardata_to_string(path), content, modes) + modes = normalize_modes(modes, false) + :file.write_file(IO.chardata_to_string(path), content, modes) end @doc """ - Same as `write/3` but raises an exception if it fails, returns `:ok` otherwise. + Same as `write/3` but raises a `File.Error` exception if it fails. + Returns `:ok` otherwise. + + ## Examples + + File.write!("hello.txt", "world!") + #=> :ok + + File.write!("temp", "world!") + ** (File.Error) could not write to file "temp": illegal operation on a directory """ - @spec write!(Path.t, iodata, list) :: :ok | no_return + @spec write!(Path.t(), iodata, [mode]) :: :ok def write!(path, content, modes \\ []) do - path = IO.chardata_to_string(path) - case F.write_file(path, content, modes) do - :ok -> :ok + case write(path, content, modes) do + :ok -> + :ok + {:error, reason} -> - raise File.Error, reason: reason, action: "write to file", path: path + raise File.Error, + reason: reason, + action: "write to file", + path: IO.chardata_to_string(path) end end @@ -628,6 +1415,7 @@ defmodule File do Tries to delete the file `path`. Returns `:ok` if successful, or `{:error, reason}` if an error occurs. + Note the file is deleted even if in read-only mode. Typical error reasons are: @@ -636,89 +1424,128 @@ defmodule File do * `:eacces` - missing permission for the file or one of its parents * `:eperm` - the file is a directory and user is not super-user * `:enotdir` - a component of the file name is not a directory; - on some platforms, enoent is returned instead + on some platforms, `:enoent` is returned instead * `:einval` - filename had an improper type, such as tuple ## Examples - File.rm('file.txt') + File.rm("file.txt") #=> :ok - File.rm('tmp_dir/') + File.rm("tmp_dir/") #=> {:error, :eperm} - """ - @spec rm(Path.t) :: :ok | {:error, posix} + @spec rm(Path.t()) :: :ok | {:error, posix | :badarg} def rm(path) do path = IO.chardata_to_string(path) - case F.delete(path) do + + case :file.delete(path) do :ok -> :ok + {:error, :eacces} = e -> change_mode_windows(path) || e + {:error, _} = e -> e end end defp change_mode_windows(path) do - if match? {:win32, _}, :os.type do - case F.read_file_info(IO.chardata_to_string(path)) do + if match?({:win32, _}, :os.type()) do + case :file.read_file_info(path) do {:ok, file_info} when elem(file_info, 3) in [:read, :none] -> - File.chmod(path, (elem(file_info, 7) + 0200)) - F.delete(path) + change_mode_windows(path, file_info) + _ -> nil end end end + defp change_mode_windows(path, file_info) do + case chmod(path, elem(file_info, 7) + 0o200) do + :ok -> :file.delete(path) + {:error, _reason} = error -> error + end + end + @doc """ - Same as `rm/1`, but raises an exception in case of failure. Otherwise `:ok`. + Same as `rm/1`, but raises a `File.Error` exception in case of failure. + Otherwise `:ok`. + + ## Examples + + File.rm!("file.txt") + #=> :ok + + File.rm!("non_existing/") + ** (File.Error) could not remove file "non_existing/": no such file or directory """ - @spec rm!(Path.t) :: :ok | no_return + @spec rm!(Path.t()) :: :ok def rm!(path) do - path = IO.chardata_to_string(path) case rm(path) do - :ok -> :ok + :ok -> + :ok + {:error, reason} -> - raise File.Error, reason: reason, action: "remove file", path: path + raise File.Error, reason: reason, action: "remove file", path: IO.chardata_to_string(path) end end @doc """ Tries to delete the dir at `path`. + Returns `:ok` if successful, or `{:error, reason}` if an error occurs. + It returns `{:error, :eexist}` if the directory is not empty. ## Examples - File.rmdir('tmp_dir') + File.rmdir("tmp_dir") #=> :ok - File.rmdir('file.txt') - #=> {:error, :enotdir} + File.rmdir("non_empty_dir") + #=> {:error, :eexist} + File.rmdir("file.txt") + #=> {:error, :enotdir} """ - @spec rmdir(Path.t) :: :ok | {:error, posix} + @spec rmdir(Path.t()) :: :ok | {:error, posix | :badarg} def rmdir(path) do - F.del_dir(IO.chardata_to_string(path)) + :file.del_dir(IO.chardata_to_string(path)) end @doc """ - Same as `rmdir/1`, but raises an exception in case of failure. Otherwise `:ok`. + Same as `rmdir/1`, but raises a `File.Error` exception in case of failure. + Otherwise `:ok`. + + ## Examples + + File.rmdir!("tmp_dir") + #=> :ok + + File.rmdir!("non_empty_dir") + ** (File.Error) could not remove directory "non_empty_dir": directory is not empty + + File.rmdir!("file.txt") + ** (File.Error) could not remove directory "file.txt": not a directory """ - @spec rmdir!(Path.t) :: :ok | {:error, posix} + @spec rmdir!(Path.t()) :: :ok def rmdir!(path) do - path = IO.chardata_to_string(path) case rmdir(path) do - :ok -> :ok + :ok -> + :ok + {:error, reason} -> - raise File.Error, reason: reason, action: "remove directory", path: path + raise File.Error, + reason: reason, + action: "remove directory", + path: IO.chardata_to_string(path) end end @doc """ - Remove files and directories recursively at the given `path`. + Removes files and directories recursively at the given `path`. Symlinks are not followed but simply removed, non-existing files are simply ignored (i.e. doesn't make this function fail). @@ -728,117 +1555,156 @@ defmodule File do ## Examples - File.rm_rf "samples" + File.rm_rf("samples") #=> {:ok, ["samples", "samples/1.txt"]} - File.rm_rf "unknown" + File.rm_rf("unknown") #=> {:ok, []} + File.rm_rf("/tmp") + #=> {:error, :eperm, "/tmp"} """ - @spec rm_rf(Path.t) :: {:ok, [binary]} | {:error, posix, binary} + @spec rm_rf(Path.t()) :: {:ok, [binary]} | {:error, posix | :badarg, binary} def rm_rf(path) do - do_rm_rf(IO.chardata_to_string(path), {:ok, []}) + {major, _} = :os.type() + + path + |> IO.chardata_to_string() + |> assert_no_null_byte!("File.rm_rf/1") + |> do_rm_rf([], major) end - defp do_rm_rf(path, {:ok, _} = entry) do - case safe_list_dir(path) do + defp do_rm_rf(path, acc, major) do + case safe_list_dir(path, major) do {:ok, files} when is_list(files) -> - res = - Enum.reduce files, entry, fn(file, tuple) -> - do_rm_rf(Path.join(path, file), tuple) - end - - case res do - {:ok, acc} -> - case rmdir(path) do - :ok -> {:ok, [path|acc]} - {:error, :enoent} -> res - {:error, reason} -> {:error, reason, path} + acc = + Enum.reduce(files, acc, fn file, acc -> + # In case we can't delete, continue anyway, we might succeed + # to delete it on Windows due to how they handle symlinks. + case do_rm_rf(Path.join(path, file), acc, major) do + {:ok, acc} -> acc + {:error, _, _} -> acc end - reason -> - reason + end) + + case rmdir(path) do + :ok -> {:ok, [path | acc]} + {:error, :enoent} -> {:ok, acc} + {:error, reason} -> {:error, reason, path} end - {:ok, :directory} -> do_rm_directory(path, entry) - {:ok, :regular} -> do_rm_regular(path, entry) - {:error, reason} when reason in [:enoent, :enotdir] -> entry - {:error, reason} -> {:error, reason, path} - end - end - defp do_rm_rf(_, reason) do - reason + {:ok, :directory} -> + do_rm_directory(path, acc) + + {:ok, :regular} -> + do_rm_regular(path, acc) + + {:error, reason} when reason in [:enoent, :enotdir] -> + {:ok, acc} + + {:error, reason} -> + {:error, reason, path} + end end - defp do_rm_regular(path, {:ok, acc} = entry) do + defp do_rm_regular(path, acc) do case rm(path) do - :ok -> {:ok, [path|acc]} - {:error, :enoent} -> entry + :ok -> {:ok, [path | acc]} + {:error, :enoent} -> {:ok, acc} {:error, reason} -> {:error, reason, path} end end - # On windows, symlinks are treated as directory and must be removed - # with rmdir/1. But on Unix, we remove them via rm/1. So we first try - # to remove it as a directory and, if we get :enotdir, we fallback to - # a file removal. - defp do_rm_directory(path, {:ok, acc} = entry) do + # On Windows, symlinks are treated as directory and must be removed + # with rmdir/1. But on Unix-like systems, we remove them via rm/1. + # So we first try to remove it as a directory and, if we get :enotdir, + # we fall back to a file removal. + defp do_rm_directory(path, acc) do case rmdir(path) do - :ok -> {:ok, [path|acc]} - {:error, :enotdir} -> do_rm_regular(path, entry) - {:error, :enoent} -> entry + :ok -> {:ok, [path | acc]} + {:error, :enotdir} -> do_rm_regular(path, acc) + {:error, :enoent} -> {:ok, acc} {:error, reason} -> {:error, reason, path} end end - defp safe_list_dir(path) do + defp safe_list_dir(path, major) do case :elixir_utils.read_link_type(path) do - {:ok, :symlink} -> + {:ok, :directory} -> + # If we cannot read the files, try to delete it anyway + case :file.list_dir_all(path) do + {:ok, files} -> {:ok, files} + {:error, _} -> {:ok, :directory} + end + + {:ok, :symlink} when major == :win32 -> case :elixir_utils.read_file_type(path) do {:ok, :directory} -> {:ok, :directory} _ -> {:ok, :regular} end - {:ok, :directory} -> - F.list_dir(path) + {:ok, _} -> {:ok, :regular} + + {:error, :eio} when major == :win32 -> + # unix domain socket returns `{:error, :eio}` + # on other platforms the result is `{:ok, :regular}` + {:ok, :regular} + {:error, reason} -> {:error, reason} end end @doc """ - Same as `rm_rf/1` but raises `File.Error` in case of failures, - otherwise the list of files or directories removed. + Same as `rm_rf/1` but raises a `File.Error` exception in case of failures, + otherwise returns the list of files or directories removed. + + ## Examples + + File.rm_rf!("samples") + #=> ["samples", "samples/1.txt"] + + File.rm_rf!("unknown") + #=> [] + + File.rm_rf!("/tmp") + ** (File.Error) could not remove files and directories recursively from "/tmp": not owner """ - @spec rm_rf!(Path.t) :: [binary] | no_return + @spec rm_rf!(Path.t()) :: [binary] def rm_rf!(path) do - path = IO.chardata_to_string(path) case rm_rf(path) do - {:ok, files} -> files + {:ok, files} -> + files + {:error, reason, _} -> - raise File.Error, reason: reason, path: path, + raise File.Error, + reason: reason, + path: IO.chardata_to_string(path), action: "remove files and directories recursively from" end end @doc ~S""" - Opens the given `path` according to the given list of modes. + Opens the given `path`. - In order to write and read files, one must use the functions - in the `IO` module. By default, a file is opened in binary mode - which requires the functions `IO.binread/2` and `IO.binwrite/2` - to interact with the file. A developer may pass `:utf8` as an - option when opening the file and then all other functions from - `IO` are available, since they work directly with Unicode data. + `modes_or_function` can either be a list of modes or a function. If it's a + list, it's considered to be a list of modes (that are documented below). If + it's a function, then it's equivalent to calling `open(path, [], + modes_or_function)`. See the documentation for `open/3` for more information + on this function. The allowed modes: + * `:binary` - opens the file in binary mode, disabling special handling of + Unicode sequences (default mode). + * `:read` - the file, which must exist, is opened for reading. * `:write` - the file is opened for writing. It is created if it does not exist. - If the file does exists, and if write is not combined with read, the file + If the file does exist, and if write is not combined with read, the file will be truncated. * `:append` - the file will be opened for writing, and it will be created @@ -848,8 +1714,8 @@ defmodule File do * `:exclusive` - the file, when opened for writing, is created if it does not exist. If the file exists, open will return `{:error, :eexist}`. - * `:char_list` - when this term is given, read operations on the file will - return char lists rather than binaries. + * `:charlist` - when this term is given, read operations on the file will + return charlists rather than binaries. * `:compressed` - makes it possible to read or write gzip compressed files. @@ -859,29 +1725,46 @@ defmodule File do * `:utf8` - this option denotes how data is actually stored in the disk file and makes the file perform automatic translation of characters to - and from utf-8. + and from UTF-8. If data is sent to a file in a format that cannot be converted to the - utf-8 or if data is read by a function that returns data in a format that + UTF-8 or if data is read by a function that returns data in a format that cannot cope with the character range of the data, an error occurs and the file will be closed. - Check http://www.erlang.org/doc/man/file.html#open-2 for more information about - other options like `:read_ahead` and `:delayed_write`. + * `:delayed_write`, `:raw`, `:ram`, `:read_ahead`, `:sync`, `{:encoding, ...}`, + `{:read_ahead, pos_integer}`, `{:delayed_write, non_neg_integer, non_neg_integer}` - + for more information about these options see `:file.open/2`. This function returns: - * `{:ok, io_device}` - the file has been opened in the requested mode. + * `{:ok, io_device | file_descriptor}` - the file has been opened in + the requested mode. We explore the differences between these two results + in the following section + + * `{:error, reason}` - the file could not be opened due to `reason`. + + ## IO devices + + By default, this function returns an IO device. An `io_device` is + a process which handles the file and you can interact with it using + the functions in the `IO` module. By default, a file is opened in + `:binary` mode, which requires the functions `IO.binread/2` and + `IO.binwrite/2` to interact with the file. A developer may pass `:utf8` + as a mode when opening the file and then all other functions from + `IO` are available, since they work directly with Unicode data. - `io_device` is actually the pid of the process which handles the file. - This process is linked to the process which originally opened the file. - If any process to which the `io_device` is linked terminates, the file - will be closed and the process itself will be terminated. + Given the IO device is a file, if the owner process terminates, + the file is closed and the process itself terminates too. If any + process to which the `io_device` is linked terminates, the file + will be closed and the process itself will be terminated. - An `io_device` returned from this call can be used as an argument to the - `IO` module functions. + ## File descriptors - * `{:error, reason}` - the file could not be opened. + When the `:raw` or `:ram` modes are given, this function returns + a low-level file descriptors. This avoids creating a process but + requires using the functions in the [`:file`](`:file`) module to + interact with it. ## Examples @@ -890,25 +1773,29 @@ defmodule File do File.close(file) """ - @spec open(Path.t, list) :: {:ok, io_device} | {:error, posix} - def open(path, modes \\ []) + @spec open(Path.t(), [mode | :ram]) :: + {:ok, io_device | file_descriptor} | {:error, posix | :badarg | :system_limit} + @spec open(Path.t(), (io_device | file_descriptor -> res)) :: + {:ok, res} | {:error, posix | :badarg | :system_limit} + when res: var + def open(path, modes_or_function \\ []) def open(path, modes) when is_list(modes) do - F.open(IO.chardata_to_string(path), open_defaults(modes, true)) + :file.open(IO.chardata_to_string(path), normalize_modes(modes, true)) end - def open(path, function) when is_function(function) do + def open(path, function) when is_function(function, 1) do open(path, [], function) end @doc """ - Similar to `open/2` but expects a function as last argument. + Similar to `open/2` but expects a function as its last argument. - The file is opened, given to the function as argument and + The file is opened, given to the function as an argument and automatically closed after the function returns, regardless if there was an error when executing the function. - It returns `{:ok, function_result}` in case of success, + Returns `{:ok, function_result}` in case of success, `{:error, reason}` otherwise. This function expects the file to be closed with success, @@ -916,119 +1803,212 @@ defmodule File do is given. For this reason, we do not recommend passing `:delayed_write` to this function. + See `open/2` for the list of available `modes`. + ## Examples - File.open("file.txt", [:read, :write], fn(file) -> + File.open("file.txt", [:read, :write], fn file -> IO.read(file, :line) end) - + #=> {:ok, "file content"} """ - @spec open(Path.t, list, (io_device -> res)) :: {:ok, res} | {:error, posix} when res: var - def open(path, modes, function) do + @spec open(Path.t(), [mode | :ram], (io_device | file_descriptor -> res)) :: + {:ok, res} | {:error, posix | :badarg | :system_limit} + when res: var + def open(path, modes, function) when is_list(modes) and is_function(function, 1) do case open(path, modes) do - {:ok, device} -> + {:ok, io_device} -> try do - {:ok, function.(device)} + {:ok, function.(io_device)} after - :ok = close(device) + :ok = close(io_device) end - other -> other + + other -> + other end end @doc """ - Same as `open/2` but raises an error if file could not be opened. + Similar to `open/2` but raises a `File.Error` exception if the file + could not be opened. Returns the IO device otherwise. + + See `open/2` for the list of available modes. + + ## Examples - Returns the `io_device` otherwise. + File.open!("file.txt", fn file -> + IO.read(file, :line) + end) + #=> "file content" """ - @spec open!(Path.t, list) :: io_device | no_return - def open!(path, modes \\ []) do - path = IO.chardata_to_string(path) - case open(path, modes) do - {:ok, device} -> device + @spec open!(Path.t(), [mode | :ram]) :: io_device | file_descriptor + @spec open!(Path.t(), (io_device | file_descriptor -> res)) :: res when res: var + def open!(path, modes_or_function \\ []) do + case open(path, modes_or_function) do + {:ok, io_device_or_function_result} -> + io_device_or_function_result + {:error, reason} -> - raise File.Error, reason: reason, action: "open", path: path + raise File.Error, reason: reason, action: "open", path: IO.chardata_to_string(path) end end @doc """ - Same as `open/3` but raises an error if file could not be opened. + Similar to `open/3` but raises a `File.Error` exception if the file + could not be opened. + + If it succeeds opening the file, it returns the `function` result on the IO device. + + See `open/2` for the list of available `modes`. - Returns the function result otherwise. + ## Examples + + File.open!("file.txt", [:read, :write], fn file -> + IO.read(file, :line) + end) + #=> "file content" """ - @spec open!(Path.t, list, (io_device -> res)) :: res | no_return when res: var + @spec open!(Path.t(), [mode | :ram], (io_device | file_descriptor -> res)) :: res when res: var def open!(path, modes, function) do - path = IO.chardata_to_string(path) case open(path, modes, function) do - {:ok, device} -> device + {:ok, function_result} -> + function_result + {:error, reason} -> - raise File.Error, reason: reason, action: "open", path: path + raise File.Error, reason: reason, action: "open", path: IO.chardata_to_string(path) end end @doc """ Gets the current working directory. - In rare circumstances, this function can fail on Unix. It may happen - if read permission does not exist for the parent directories of the + In rare circumstances, this function can fail on Unix-like systems. It may happen + if read permissions do not exist for the parent directories of the current directory. For this reason, returns `{:ok, cwd}` in case of success, `{:error, reason}` otherwise. + + ## Examples + + File.cwd() + #=> {:ok, "/Users/user/elixir/elixir_lang"} + + # Missing read permission for one of the parents of the current directory + File.cwd() + #=> {:error, :eacces} """ - @spec cwd() :: {:ok, binary} | {:error, posix} + @spec cwd() :: {:ok, binary} | {:error, posix | :badarg} def cwd() do - case F.get_cwd do - {:ok, base} -> {:ok, IO.chardata_to_string(base)} + case :file.get_cwd() do + {:ok, base} -> {:ok, IO.chardata_to_string(fix_drive_letter(base))} {:error, _} = error -> error end end + defp fix_drive_letter([l, ?:, ?/ | rest] = original) when l in ?A..?Z do + case :os.type() do + {:win32, _} -> [l + ?a - ?A, ?:, ?/ | rest] + _ -> original + end + end + + defp fix_drive_letter(original), do: original + @doc """ - The same as `cwd/0`, but raises an exception if it fails. + The same as `cwd/0`, but raises a `File.Error` exception if it fails. + + ## Examples + + File.cwd!() + #=> "/Users/user/elixir/elixir_lang" """ - @spec cwd!() :: binary | no_return + @spec cwd!() :: binary def cwd!() do - case F.get_cwd do - {:ok, cwd} -> IO.chardata_to_string(cwd) + case cwd() do + {:ok, cwd} -> + cwd + {:error, reason} -> - raise File.Error, reason: reason, action: "get current working directory" + raise File.Error, reason: reason, action: "get current working directory" end end @doc """ Sets the current working directory. + The current working directory is set for the BEAM globally. This can lead to + race conditions if multiple processes are changing the current working + directory concurrently. To run an external command in a given directory + without changing the global current working directory, use the `:cd` option + of `System.cmd/3` and `Port.open/2`. + Returns `:ok` if successful, `{:error, reason}` otherwise. + + ## Examples + + File.cd("bin") + #=> :ok + + File.cd("non_existing_dir") + #=> {:error, :enoent} """ - @spec cd(Path.t) :: :ok | {:error, posix} + @spec cd(Path.t()) :: :ok | {:error, posix | :badarg | :no_translation} def cd(path) do - F.set_cwd(IO.chardata_to_string(path)) + :file.set_cwd(IO.chardata_to_string(path)) end @doc """ - The same as `cd/1`, but raises an exception if it fails. + The same as `cd/1`, but raises a `File.Error` exception if it fails. + + ## Examples + + File.cd!("bin") + #=> :ok + + File.cd!("non_existing_dir") + ** (File.Error) could not set current working directory to "non_existing_dir": no such file or directory """ - @spec cd!(Path.t) :: :ok | no_return + @spec cd!(Path.t()) :: :ok def cd!(path) do - path = IO.chardata_to_string(path) - case F.set_cwd(path) do - :ok -> :ok + case cd(path) do + :ok -> + :ok + {:error, reason} -> - raise File.Error, reason: reason, action: "set current working directory to", path: path + raise File.Error, + reason: reason, + action: "set current working directory to", + path: IO.chardata_to_string(path) end end @doc """ Changes the current directory to the given `path`, - executes the given function and then revert back - to the previous path regardless if there is an exception. + executes the given function and then reverts back + to the previous path regardless of whether there is an exception. + + The current working directory is temporarily set for the BEAM globally. This + can lead to race conditions if multiple processes are changing the current + working directory concurrently. To run an external command in a given + directory without changing the global current working directory, use the + `:cd` option of `System.cmd/3` and `Port.open/2`. Raises an error if retrieving or changing the current directory fails. + + ## Examples + + File.cd!("bin", fn -> do_something() end) + #=> :result_of_do_something + + File.cd!("non_existing_dir", fn -> do_something() end) + ** (File.Error) could not set current working directory to "non_existing_dir": no such file or directory """ - @spec cd!(Path.t, (() -> res)) :: res | no_return when res: var + @spec cd!(Path.t(), (-> res)) :: res when res: var def cd!(path, function) do - old = cwd! + old = cwd!() cd!(path) + try do function.() after @@ -1037,30 +2017,54 @@ defmodule File do end @doc """ - Returns list of files in the given directory. + Returns the list of files in the given directory. - It returns `{:ok, [files]}` in case of success, + Hidden files are not ignored and the results are *not* sorted. + + Since directories are considered files by the file system, + they are also included in the returned value. + + Returns `{:ok, files}` in case of success, `{:error, reason}` otherwise. + + ## Examples + + File.ls("bin") + #=> {:ok, ["iex", "elixir"]} + + File.ls("non_existing_dir") + #=> {:error, :enoent} """ - @spec ls(Path.t) :: {:ok, [binary]} | {:error, posix} + @spec ls(Path.t()) :: {:ok, [binary]} | {:error, posix | :badarg | {:no_translation, binary}} def ls(path \\ ".") do - case F.list_dir(IO.chardata_to_string(path)) do + case :file.list_dir(IO.chardata_to_string(path)) do {:ok, file_list} -> {:ok, Enum.map(file_list, &IO.chardata_to_string/1)} {:error, _} = error -> error end end @doc """ - The same as `ls/1` but raises `File.Error` - in case of an error. + The same as `ls/1` but raises a `File.Error` exception in case of an error. + + ## Examples + + File.ls!("bin") + #=> ["iex", "elixir"] + + File.ls!("non_existing_dir") + ** (File.Error) could not list directory "non_existing_dir": no such file or directory """ - @spec ls!(Path.t) :: [binary] | no_return + @spec ls!(Path.t()) :: [binary] def ls!(path \\ ".") do - path = IO.chardata_to_string(path) case ls(path) do - {:ok, value} -> value + {:ok, value} -> + value + {:error, reason} -> - raise File.Error, reason: reason, action: "list directory", path: path + raise File.Error, + reason: reason, + action: "list directory", + path: IO.chardata_to_string(path) end end @@ -1070,132 +2074,305 @@ defmodule File do Note that if the option `:delayed_write` was used when opening the file, `close/1` might return an old write error and not even try to close the file. - See `open/2`. + See `open/2` for more information. + + ## Examples + + {:ok, file} = File.open("hello.txt") + File.close(file) + #=> :ok + + File.close(:not_an_io_device) + #=> {:error, :badarg} """ @spec close(io_device) :: :ok | {:error, posix | :badarg | :terminated} def close(io_device) do - F.close(io_device) + :file.close(io_device) end @doc """ + Shortcut for `File.stream!/3`. + """ + @spec stream!(Path.t(), :line | pos_integer | [stream_mode]) :: File.Stream.t() + def stream!(path, line_or_bytes_modes \\ []) + + def stream!(path, modes) when is_list(modes), + do: stream!(path, :line, modes) + + def stream!(path, line_or_bytes) when is_integer(line_or_bytes) or line_or_bytes == :line, + do: stream!(path, line_or_bytes, []) + + @doc ~S""" Returns a `File.Stream` for the given `path` with the given `modes`. The stream implements both `Enumerable` and `Collectable` protocols, which means it can be used both for read and write. - The `line_or_byte` argument configures how the file is read when - streaming, by `:line` (default) or by a given number of bytes. + The `line_or_bytes` argument configures how the file is read when + streaming, by `:line` (default) or by a given number of bytes. When + using the `:line` option, CRLF line breaks (`"\r\n"`) are normalized + to LF (`"\n"`). + + Similar to other file operations, a stream can be created in one node + and forwarded to another node. Once the stream is opened in another node, + a request will be sent to the creator node to spawn a process for file + streaming. Operating the stream can fail on open for the same reasons as - `File.open!/2`. Note that the file is automatically opened only and - every time streaming begins. There is no need to pass `:read` and - `:write` modes, as those are automatically set by Elixir. + `File.open!/2`. Note that the file is automatically opened each time streaming + begins. There is no need to pass `:read` and `:write` modes, as those are + automatically set by Elixir. ## Raw files Since Elixir controls when the streamed file is opened, the underlying device cannot be shared and as such it is convenient to open the file in raw mode for performance reasons. Therefore, Elixir **will** open - streams in `:raw` mode with the `:read_ahead` option unless an encoding - is specified. + streams in `:raw` mode with the `:read_ahead` option if the stream is + open in the same node as it is created and no encoding has been specified. + This means any data streamed into the file must be converted to `t:iodata/0` + type. If you pass, for example, `[encoding: :utf8]` or + `[encoding: {:utf16, :little}]` in the modes parameter, the underlying stream + will use `IO.write/2` and the `String.Chars` protocol to convert the data. + See `IO.binwrite/2` and `IO.write/2` . One may also consider passing the `:delayed_write` option if the stream is meant to be written to under a tight loop. + + ## Byte order marks and read offset + + If you pass `:trim_bom` in the modes parameter, the stream will + trim UTF-8, UTF-16 and UTF-32 byte order marks when reading from file. + + Note that this function does not try to discover the file encoding + based on BOM. From Elixir v1.16.0, you may also pass a `:read_offset` + that is skipped whenever enumerating the stream (if both `:read_offset` + and `:trim_bom` are given, the offset is skipped after the BOM). + + See `Stream.run/1` for an example of streaming into a file. + + ## Examples + + # Read a utf8 text file which may include BOM + File.stream!("./test/test.txt", [:trim_bom, encoding: :utf8]) + #=> %File.Stream{path: "./test/test.txt", ...} + + # Read in 2048 byte chunks rather than lines + File.stream!("./test/test.data", 2048) + #=> %File.Stream{path: "./test/test.data", ...} """ - def stream!(path, modes \\ [], line_or_bytes \\ :line) do - modes = open_defaults(modes, true) - File.Stream.__build__(IO.chardata_to_string(path), modes, line_or_bytes) + @spec stream!(Path.t(), :line | pos_integer, [stream_mode]) :: File.Stream.t() + def stream!(path, line_or_bytes, modes) + + def stream!(path, modes, line_or_bytes) when is_list(modes) do + # TODO: Remove me on Elixir 2.0 + IO.warn( + "File.stream!(path, modes, line_or_byte) is deprecated, " <> + "invoke File.stream!(path, line_or_bytes, modes) instead" + ) + + stream!(path, line_or_bytes, modes) + end + + def stream!(path, line_or_bytes, modes) do + modes = normalize_modes(modes, true) + File.Stream.__build__(IO.chardata_to_string(path), line_or_bytes, modes) end @doc """ - Changes the unix file `mode` for a given `file`. - Returns `:ok` on success, or `{:error, reason}` - on failure. + Changes the `mode` for a given `file`. + + Returns `:ok` on success, or `{:error, reason}` on failure. + + ## Permissions + + File permissions are specified by adding together the following octal modes: + + * `0o400` - read permission: owner + * `0o200` - write permission: owner + * `0o100` - execute permission: owner + + * `0o040` - read permission: group + * `0o020` - write permission: group + * `0o010` - execute permission: group + + * `0o004` - read permission: other + * `0o002` - write permission: other + * `0o001` - execute permission: other + + For example, setting the mode `0o755` gives it + write, read and execute permission to the owner + and both read and execute permission to group + and others. + + ## Examples + + File.chmod("hello.txt", 0o755) + #=> :ok + + File.chmod("non_existing.txt", 0o755) + #=> {:error, :enoent} """ - @spec chmod(Path.t, integer) :: :ok | {:error, posix} + @spec chmod(Path.t(), non_neg_integer) :: :ok | {:error, posix | :badarg} def chmod(path, mode) do - F.change_mode(IO.chardata_to_string(path), mode) + :file.change_mode(IO.chardata_to_string(path), mode) end @doc """ - Same as `chmod/2`, but raises an exception in case of failure. Otherwise `:ok`. + Same as `chmod/2`, but raises a `File.Error` exception in case of failure. + Otherwise `:ok`. + + ## Examples + + File.chmod!("hello.txt", 0o755) + #=> :ok + + File.chmod!("non_existing.txt", 0o755) + ** (File.Error) could not change mode for "non_existing.txt": no such file or directory """ - @spec chmod!(Path.t, integer) :: :ok | no_return + @spec chmod!(Path.t(), non_neg_integer) :: :ok def chmod!(path, mode) do - path = IO.chardata_to_string(path) case chmod(path, mode) do - :ok -> :ok + :ok -> + :ok + {:error, reason} -> - raise File.Error, reason: reason, action: "change mode for", path: path + raise File.Error, + reason: reason, + action: "change mode for", + path: IO.chardata_to_string(path) end end @doc """ - Changes the user group given by the group id `gid` + Changes the group given by the group ID `gid` for a given `file`. Returns `:ok` on success, or `{:error, reason}` on failure. + + ## Examples + + File.chgrp("hello.txt", 10) + #=> :ok + + File.chgrp("non_existing.txt", 10) + #=> {:error, :enoent} """ - @spec chgrp(Path.t, integer) :: :ok | {:error, posix} + @spec chgrp(Path.t(), non_neg_integer) :: :ok | {:error, posix | :badarg} def chgrp(path, gid) do - F.change_group(IO.chardata_to_string(path), gid) + :file.change_group(IO.chardata_to_string(path), gid) end @doc """ - Same as `chgrp/2`, but raises an exception in case of failure. Otherwise `:ok`. + Same as `chgrp/2`, but raises a `File.Error` exception in case of failure. + Otherwise `:ok`. + + ## Examples + + File.chgrp!("hello.txt", 10) + #=> :ok + + File.chgrp!("non_existing.txt", 10) + ** (File.Error) could not change group for "non_existing.txt": no such file or directory """ - @spec chgrp!(Path.t, integer) :: :ok | no_return + @spec chgrp!(Path.t(), non_neg_integer) :: :ok def chgrp!(path, gid) do - path = IO.chardata_to_string(path) case chgrp(path, gid) do - :ok -> :ok + :ok -> + :ok + {:error, reason} -> - raise File.Error, reason: reason, action: "change group for", path: path + raise File.Error, + reason: reason, + action: "change group for", + path: IO.chardata_to_string(path) end end @doc """ - Changes the owner given by the user id `uid` + Changes the owner given by the user ID `uid` for a given `file`. Returns `:ok` on success, or `{:error, reason}` on failure. + + ## Examples + + File.chown("hello.txt", 15) + #=> :ok + + File.chown("secret.txt", 15) + #=> {:error, :eperm} """ - @spec chown(Path.t, integer) :: :ok | {:error, posix} + @spec chown(Path.t(), non_neg_integer) :: :ok | {:error, posix | :badarg} def chown(path, uid) do - F.change_owner(IO.chardata_to_string(path), uid) + :file.change_owner(IO.chardata_to_string(path), uid) end @doc """ - Same as `chown/2`, but raises an exception in case of failure. Otherwise `:ok`. + Same as `chown/2`, but raises a `File.Error` exception in case of failure. + Otherwise `:ok`. + + ## Examples + + File.chown!("hello.txt", 15) + #=> :ok + + File.chown!("secret.txt", 15) + ** (File.Error) could not change owner for "secret.txt": not owner """ - @spec chown!(Path.t, integer) :: :ok | no_return + @spec chown!(Path.t(), non_neg_integer) :: :ok def chown!(path, uid) do - path = IO.chardata_to_string(path) case chown(path, uid) do - :ok -> :ok + :ok -> + :ok + {:error, reason} -> - raise File.Error, reason: reason, action: "change owner for", path: path + raise File.Error, + reason: reason, + action: "change owner for", + path: IO.chardata_to_string(path) end end ## Helpers - @read_ahead 64*1024 + @read_ahead_size 64 * 1024 - defp open_defaults([:char_list|t], _add_binary) do - open_defaults(t, false) + defp assert_no_null_byte!(binary, operation) do + case :binary.match(binary, "\0") do + {_, _} -> + raise ArgumentError, + "cannot execute #{operation} for path with null byte, got: #{inspect(binary)}" + + :nomatch -> + binary + end + end + + defp normalize_modes([:utf8 | rest], binary?) do + [encoding: :utf8] ++ normalize_modes(rest, binary?) end - defp open_defaults([:utf8|t], add_binary) do - open_defaults([{:encoding, :utf8}|t], add_binary) + defp normalize_modes([:read_ahead | rest], binary?) do + [read_ahead: @read_ahead_size] ++ normalize_modes(rest, binary?) end - defp open_defaults([:read_ahead|t], add_binary) do - open_defaults([{:read_ahead, @read_ahead}|t], add_binary) + # TODO: Remove :char_list mode on v2.0 + defp normalize_modes([mode | rest], _binary?) when mode in [:charlist, :char_list] do + if mode == :char_list do + IO.warn("the :char_list mode is deprecated, use :charlist") + end + + normalize_modes(rest, false) end - defp open_defaults([h|t], add_binary) do - [h|open_defaults(t, add_binary)] + defp normalize_modes([mode | rest], binary?) do + [mode | normalize_modes(rest, binary?)] end - defp open_defaults([], true), do: [:binary] - defp open_defaults([], false), do: [] + defp normalize_modes([], true), do: [:binary] + defp normalize_modes([], false), do: [] + + defp normalize_path_or_io_device(path) when is_list(path), do: IO.chardata_to_string(path) + defp normalize_path_or_io_device(path) when is_binary(path), do: path + defp normalize_path_or_io_device(io_device) when is_pid(io_device), do: io_device + defp normalize_path_or_io_device(io_device = {:file_descriptor, _, _}), do: io_device end diff --git a/lib/elixir/lib/file/stat.ex b/lib/elixir/lib/file/stat.ex index 5a684ff58ca..ec85fc980ec 100644 --- a/lib/elixir/lib/file/stat.ex +++ b/lib/elixir/lib/file/stat.ex @@ -1,18 +1,22 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + require Record defmodule File.Stat do @moduledoc """ - A struct responsible to hold file information. + A struct that holds file information. In Erlang, this struct is represented by a `:file_info` record. Therefore this module also provides functions for converting - in between the Erlang record and the Elixir struct. + between the Erlang record and the Elixir struct. Its fields are: * `size` - size of file in bytes. - * `type` - `:device | :directory | :regular | :other`; the type of the + * `type` - `:device | :directory | :regular | :other | :symlink`; the type of the file. * `access` - `:read | :write | :read_write | :none`; the current system @@ -23,7 +27,7 @@ defmodule File.Stat do * `mtime` - the last time the file was written. * `ctime` - the interpretation of this time field depends on the operating - system. On Unix, it is the last time the file or the inode was changed. + system. On Unix-like operating systems, it is the last time the file or the inode was changed. In Windows, it is the time of creation. * `mode` - the file permissions. @@ -32,35 +36,53 @@ defmodule File.Stat do systems which have no concept of links. * `major_device` - identifies the file system where the file is located. - In windows, the number indicates a drive as follows: 0 means A:, 1 means + In Windows, the number indicates a drive as follows: 0 means A:, 1 means B:, and so on. - * `minor_device` - only valid for character devices on Unix. In all other + * `minor_device` - only valid for character devices on Unix-like systems. In all other cases, this field is zero. - * `inode` - gives the inode number. On non-Unix file systems, this field + * `inode` - gives the inode number. On non-Unix-like file systems, this field will be zero. - * `uid` - indicates the owner of the file. + * `uid` - indicates the owner of the file. Will be zero for non-Unix-like file + systems. - * `gid` - gives the group that the owner of the file belongs to. Will be - zero for non-Unix file systems. + * `gid` - indicates the group that owns the file. Will be zero for + non-Unix-like file systems. The time type returned in `atime`, `mtime`, and `ctime` is dependent on the time type set in options. `{:time, type}` where type can be `:local`, - `:universal`, or `:posix`. Default is `:local`. + `:universal`, or `:posix`. Default is `:universal`. """ record = Record.extract(:file_info, from_lib: "kernel/include/file.hrl") - keys = :lists.map(&elem(&1, 0), record) - vals = :lists.map(&{&1, [], nil}, keys) - pairs = :lists.zip(keys, vals) + keys = :lists.map(&elem(&1, 0), record) + vals = :lists.map(&{&1, [], nil}, keys) + pairs = :lists.zip(keys, vals) defstruct keys + @type t :: %__MODULE__{ + size: non_neg_integer() | :undefined, + type: :device | :directory | :regular | :other | :symlink | :undefined, + access: :read | :write | :read_write | :none | :undefined, + atime: :calendar.datetime() | integer() | :undefined, + mtime: :calendar.datetime() | integer() | :undefined, + ctime: :calendar.datetime() | integer() | :undefined, + mode: non_neg_integer() | :undefined, + links: non_neg_integer() | :undefined, + major_device: non_neg_integer() | :undefined, + minor_device: non_neg_integer() | :undefined, + inode: non_neg_integer() | :undefined, + uid: non_neg_integer() | :undefined, + gid: non_neg_integer() | :undefined + } + @doc """ Converts a `File.Stat` struct to a `:file_info` record. """ + @spec to_record(t()) :: :file.file_info() def to_record(%File.Stat{unquote_splicing(pairs)}) do {:file_info, unquote_splicing(vals)} end @@ -68,6 +90,9 @@ defmodule File.Stat do @doc """ Converts a `:file_info` record into a `File.Stat`. """ + @spec from_record(:file.file_info()) :: t() + def from_record(file_info) + def from_record({:file_info, unquote_splicing(vals)}) do %File.Stat{unquote_splicing(pairs)} end diff --git a/lib/elixir/lib/file/stream.ex b/lib/elixir/lib/file/stream.ex index 3c7ffed9190..94b2ad20b5b 100644 --- a/lib/elixir/lib/file/stream.ex +++ b/lib/elixir/lib/file/stream.ex @@ -1,3 +1,7 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + defmodule File.Stream do @moduledoc """ Defines a `File.Stream` struct returned by `File.stream!/3`. @@ -7,43 +11,61 @@ defmodule File.Stream do * `path` - the file path * `modes` - the file modes * `raw` - a boolean indicating if bin functions should be used - * `line_or_bytes` - if reading should read lines or a given amount of bytes + * `line_or_bytes` - if reading should read lines or a given number of bytes + * `node` - the node the file belongs to """ - defstruct path: nil, modes: [], line_or_bytes: :line, raw: true + defstruct path: nil, modes: [], line_or_bytes: :line, raw: true, node: nil + + @type t :: %__MODULE__{} @doc false - def __build__(path, modes, line_or_bytes) do + def __build__(path, line_or_bytes, modes) do + with {:read_offset, offset} <- :lists.keyfind(:read_offset, 1, modes), + false <- is_integer(offset) and offset >= 0 do + raise ArgumentError, + "expected :read_offset to be a non-negative integer, got: #{inspect(offset)}" + end + raw = :lists.keyfind(:encoding, 1, modes) == false modes = - if raw do - if :lists.keyfind(:read_ahead, 1, modes) == {:read_ahead, false} do - [:raw|modes] - else - [:raw, :read_ahead|modes] - end - else - modes + case raw do + true -> + case :lists.keyfind(:read_ahead, 1, modes) do + {:read_ahead, false} -> [:raw | :lists.keydelete(:read_ahead, 1, modes)] + {:read_ahead, _} -> [:raw | modes] + false -> [:raw, :read_ahead | modes] + end + + false -> + modes end - %File.Stream{path: path, modes: modes, raw: raw, line_or_bytes: line_or_bytes} + %File.Stream{path: path, modes: modes, raw: raw, line_or_bytes: line_or_bytes, node: node()} end - defimpl Collectable do - def empty(stream) do - stream - end + @doc false + def __open__(%File.Stream{path: path, node: node}, modes) when node == node() do + :file.open(path, modes) + end + + @doc false + def __open__(%File.Stream{path: path, node: node}, modes) do + :erpc.call(node, :file_io_server, :start, [self(), path, List.delete(modes, :raw)]) + end - def into(%{path: path, modes: modes, raw: raw} = stream) do - modes = for mode <- modes, not mode in [:read], do: mode + defimpl Collectable do + def into(%{modes: modes, raw: raw} = stream) do + modes = for mode <- modes, mode not in [:read], do: mode - case :file.open(path, [:write|modes]) do + case File.Stream.__open__(stream, [:write | modes]) do {:ok, device} -> {:ok, into(device, stream, raw)} + {:error, reason} -> - raise File.Error, reason: reason, action: "stream", path: path + raise File.Error, reason: reason, action: "stream", path: stream.path end end @@ -51,40 +73,75 @@ defmodule File.Stream do fn :ok, {:cont, x} -> case raw do - true -> IO.binwrite(device, x) + true -> IO.binwrite(device, x) false -> IO.write(device, x) end + :ok, :done -> - :file.close(device) + # If delayed_write option is used and the last write failed will + # MatchError here as {:error, _} is returned. + :ok = :file.close(device) stream + :ok, :halt -> - :file.close(device) + # If delayed_write option is used and the last write failed will + # MatchError here as {:error, _} is returned. + :ok = :file.close(device) end end end defimpl Enumerable do - def reduce(%{path: path, modes: modes, line_or_bytes: line_or_bytes, raw: raw}, acc, fun) do - modes = for mode <- modes, not mode in [:write, :append], do: mode - - start_fun = - fn -> - case :file.open(path, modes) do - {:ok, device} -> device - {:error, reason} -> - raise File.Error, reason: reason, action: "stream", path: path - end + @read_ahead_size 64 * 1024 + + def reduce(%{modes: modes, line_or_bytes: line_or_bytes, raw: raw} = stream, acc, fun) do + start_fun = fn -> + case File.Stream.__open__(stream, read_modes(modes)) do + {:ok, device} -> + skip_bom_and_offset(device, raw, modes) + + {:error, reason} -> + raise File.Error, reason: reason, action: "stream", path: stream.path end + end next_fun = case raw do - true -> &IO.each_binstream(&1, line_or_bytes) + true -> &IO.each_binstream(&1, line_or_bytes) false -> &IO.each_stream(&1, line_or_bytes) end Stream.resource(start_fun, next_fun, &:file.close/1).(acc, fun) end + def count(%{modes: modes, line_or_bytes: :line, path: path, raw: raw} = stream) do + pattern = :binary.compile_pattern("\n") + + counter = fn device -> + device = skip_bom_and_offset(device, raw, modes) + count_lines(device, path, pattern, read_function(stream), 0) + end + + {:ok, open!(stream, modes, counter)} + end + + def count(%{path: path, line_or_bytes: bytes, raw: true, modes: modes, node: node} = stream) do + case :erpc.call(node, File, :stat, [path]) do + {:ok, %{size: 0}} -> + {:error, __MODULE__} + + {:ok, %{size: size}} -> + bom_offset = count_raw_bom(stream, modes) + offset = get_read_offset(modes) + size = max(size - bom_offset - offset, 0) + remainder = if rem(size, bytes) == 0, do: 0, else: 1 + {:ok, div(size, bytes) + remainder} + + {:error, reason} -> + raise File.Error, reason: reason, action: "stream", path: path + end + end + def count(_stream) do {:error, __MODULE__} end @@ -92,5 +149,102 @@ defmodule File.Stream do def member?(_stream, _term) do {:error, __MODULE__} end + + def slice(_stream) do + {:error, __MODULE__} + end + + defp open!(stream, modes, fun) do + case File.Stream.__open__(stream, read_modes(modes)) do + {:ok, device} -> + try do + fun.(device) + after + :file.close(device) + end + + {:error, reason} -> + raise File.Error, reason: reason, action: "stream", path: stream.path + end + end + + defp count_raw_bom(stream, modes) do + if :trim_bom in modes do + open!(stream, read_modes(modes), &(&1 |> trim_bom(true) |> elem(1))) + else + 0 + end + end + + defp skip_bom_and_offset(device, raw, modes) do + device = + if :trim_bom in modes do + device |> trim_bom(raw) |> elem(0) + else + device + end + + offset = get_read_offset(modes) + + if offset > 0 do + {:ok, _} = :file.position(device, {:cur, offset}) + end + + device + end + + defp trim_bom(device, true) do + bom_length = device |> IO.binread(4) |> bom_length() + {:ok, new_pos} = :file.position(device, bom_length) + {device, new_pos} + end + + defp trim_bom(device, false) do + # Or we read the bom in the correct amount or it isn't there + case bom_length(IO.read(device, 1)) do + 0 -> + {:ok, _} = :file.position(device, 0) + {device, 0} + + _ -> + {device, 1} + end + end + + defp bom_length(<<239, 187, 191, _rest::binary>>), do: 3 + defp bom_length(<<254, 255, _rest::binary>>), do: 2 + defp bom_length(<<255, 254, _rest::binary>>), do: 2 + defp bom_length(<<0, 0, 254, 255, _rest::binary>>), do: 4 + defp bom_length(<<254, 255, 0, 0, _rest::binary>>), do: 4 + defp bom_length(_binary), do: 0 + + def get_read_offset(modes) do + case :lists.keyfind(:read_offset, 1, modes) do + {:read_offset, offset} -> offset + false -> 0 + end + end + + defp read_modes(modes) do + for mode <- modes, mode not in [:write, :append, :trim_bom], do: mode + end + + defp count_lines(device, path, pattern, read, count) do + case read.(device) do + data when is_binary(data) -> + count_lines(device, path, pattern, read, count + count_lines(data, pattern)) + + :eof -> + count + + {:error, reason} -> + raise File.Error, reason: reason, action: "stream", path: path + end + end + + defp count_lines(data, pattern), do: length(:binary.matches(data, pattern)) + + defp read_function(%{raw: true}), do: &IO.binread(&1, @read_ahead_size) + defp read_function(%{raw: false}), do: &IO.read(&1, @read_ahead_size) end end diff --git a/lib/elixir/lib/float.ex b/lib/elixir/lib/float.ex index 720d043958f..6466bd0b54d 100644 --- a/lib/elixir/lib/float.ex +++ b/lib/elixir/lib/float.ex @@ -1,254 +1,666 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +import Kernel, except: [round: 1] + defmodule Float do @moduledoc """ - Functions for working with floating point numbers. + Functions for working with floating-point numbers. + + For mathematical operations on top of floating-points, + see Erlang's [`:math`](`:math`) module. + + ## Kernel functions + + There are functions related to floating-point numbers on the `Kernel` module + too. Here is a list of them: + + * `Kernel.round/1`: rounds a number to the nearest integer. + * `Kernel.trunc/1`: returns the integer part of a number. + + ## Known issues + + There are some very well known problems with floating-point numbers + and arithmetic due to the fact most decimal fractions cannot be + represented by a floating-point binary and most operations are not exact, + but operate on approximations. Those issues are not specific + to Elixir, they are a property of floating point representation itself. + + For example, the numbers 0.1 and 0.01 are two of them, what means the result + of squaring 0.1 does not give 0.01 neither the closest representable. Here is + what happens in this case: + + * The closest representable number to 0.1 is 0.1000000014 + * The closest representable number to 0.01 is 0.0099999997 + * Doing 0.1 * 0.1 should return 0.01, but because 0.1 is actually 0.1000000014, + the result is 0.010000000000000002, and because this is not the closest + representable number to 0.01, you'll get the wrong result for this operation + + There are also other known problems like flooring or rounding numbers. See + `round/2` and `floor/2` for more details about them. + + To learn more about floating-point arithmetic visit: + + * [0.30000000000000004.com](http://0.30000000000000004.com/) + * [What Every Programmer Should Know About Floating-Point Arithmetic](https://floating-point-gui.de/) + + """ + + import Bitwise + + @power_of_2_to_52 4_503_599_627_370_496 + @precision_range 0..15 + @type precision_range :: 0..15 + + @min_finite then(<<0xFFEFFFFFFFFFFFFF::64>>, fn <> -> num end) + @max_finite then(<<0x7FEFFFFFFFFFFFFF::64>>, fn <> -> num end) + + @doc """ + Returns the maximum finite value for a float. + + ## Examples + + iex> Float.max_finite() + 1.7976931348623157e308 + + """ + @spec max_finite() :: float + def max_finite, do: @max_finite + + @doc """ + Returns the minimum finite value for a float. + + ## Examples + + iex> Float.min_finite() + -1.7976931348623157e308 + + """ + @spec min_finite() :: float + def min_finite, do: @min_finite + + @doc """ + Computes `base` raised to power of `exponent`. + + `base` must be a float and `exponent` can be any number. + However, if a negative base and a fractional exponent + are given, it raises `ArithmeticError`. + + It always returns a float. See `Integer.pow/2` for + exponentiation that returns integers. + + ## Examples + + iex> Float.pow(2.0, 0) + 1.0 + iex> Float.pow(2.0, 1) + 2.0 + iex> Float.pow(2.0, 10) + 1024.0 + iex> Float.pow(2.0, -1) + 0.5 + iex> Float.pow(2.0, -3) + 0.125 + + iex> Float.pow(3.0, 1.5) + 5.196152422706632 + + iex> Float.pow(-2.0, 3) + -8.0 + iex> Float.pow(-2.0, 4) + 16.0 + + iex> Float.pow(-1.0, 0.5) + ** (ArithmeticError) bad argument in arithmetic expression + """ + @doc since: "1.12.0" + @spec pow(float, number) :: float + def pow(base, exponent) when is_float(base) and is_number(exponent), + do: :math.pow(base, exponent) @doc """ Parses a binary into a float. - If successful, returns a tuple of the form `{float, remainder_of_binary}`. - Otherwise `:error`. + If successful, returns a tuple in the form of `{float, remainder_of_binary}`; + when the binary cannot be coerced into a valid float, the atom `:error` is + returned. + + If the size of float exceeds the maximum size of `1.7976931348623157e+308`, + `:error` is returned even though the textual representation itself might be + well formed. + + If you want to convert a string-formatted float directly to a float, + `String.to_float/1` can be used instead. ## Examples iex> Float.parse("34") - {34.0,""} - + {34.0, ""} iex> Float.parse("34.25") - {34.25,""} - + {34.25, ""} iex> Float.parse("56.5xyz") - {56.5,"xyz"} + {56.5, "xyz"} + iex> Float.parse(".12") + :error iex> Float.parse("pi") :error + iex> Float.parse("1.7976931348623159e+308") + :error """ @spec parse(binary) :: {float, binary} | :error def parse("-" <> binary) do - case parse_unsign(binary) do + case parse_unsigned(binary) do :error -> :error {number, remainder} -> {-number, remainder} end end - def parse(binary) do - parse_unsign(binary) + def parse("+" <> binary) do + parse_unsigned(binary) end - defp parse_unsign("-" <> _), do: :error - defp parse_unsign(binary) when is_binary(binary) do - case Integer.parse binary do - :error -> :error - {integer_part, after_integer} -> parse_unsign after_integer, integer_part - end + def parse(binary) do + parse_unsigned(binary) end - # Dot followed by digit is required afterwards or we are done - defp parse_unsign(<< ?., char, rest :: binary >>, int) when char in ?0..?9 do - parse_unsign(rest, char - ?0, 1, int) + defp parse_unsigned(<>) when digit in ?0..?9, + do: parse_unsigned(rest, false, false, [digit]) + + defp parse_unsigned(binary) when is_binary(binary), do: :error + + defp parse_unsigned(<>, dot?, e?, acc) when digit in ?0..?9, + do: parse_unsigned(rest, dot?, e?, [digit | acc]) + + defp parse_unsigned(<>, false, false, acc) when digit in ?0..?9, + do: parse_unsigned(rest, true, false, [digit, ?. | acc]) + + defp parse_unsigned(<>, dot?, false, acc) + when exp_marker in ~c"eE" and digit in ?0..?9, + do: parse_unsigned(rest, true, true, [digit, ?e | add_dot(acc, dot?)]) + + defp parse_unsigned(<>, dot?, false, acc) + when exp_marker in ~c"eE" and sign in ~c"-+" and digit in ?0..?9, + do: parse_unsigned(rest, true, true, [digit, sign, ?e | add_dot(acc, dot?)]) + + # When floats are expressed in scientific notation, :erlang.binary_to_float/1 can raise an + # ArgumentError if the e exponent is too big. For example, "1.0e400". Because of this, we + # rescue the ArgumentError here and return an error. + defp parse_unsigned(rest, dot?, true = _e?, acc) do + acc + |> add_dot(dot?) + |> :lists.reverse() + |> :erlang.list_to_float() + rescue + ArgumentError -> :error + else + float -> {float, rest} end - defp parse_unsign(rest, int) do - {:erlang.float(int), rest} - end + defp parse_unsigned(rest, dot?, false = _e?, acc) do + float = + acc + |> add_dot(dot?) + |> :lists.reverse() + |> :erlang.list_to_float() - # Handle decimal points - defp parse_unsign(<< char, rest :: binary >>, float, decimal, int) when char in ?0..?9 do - parse_unsign rest, 10 * float + (char - ?0), decimal + 1, int + {float, rest} end - defp parse_unsign(<< ?e, after_e :: binary >>, float, decimal, int) do - case Integer.parse after_e do - :error -> - # Note we rebuild the binary here instead of breaking it apart at - # the function clause because the current approach copies a binary - # just on this branch. If we broke it apart in the function clause, - # the copy would happen when calling Integer.parse/1. - {floatify(int, float, decimal), << ?e, after_e :: binary >>} - {exponential, after_exponential} -> - {floatify(int, float, decimal, exponential), after_exponential} - end - end + defp add_dot(acc, true), do: acc + defp add_dot(acc, false), do: [?0, ?. | acc] - defp parse_unsign(bitstring, float, decimal, int) do - {floatify(int, float, decimal), bitstring} - end + @doc """ + Rounds a float to the largest float less than or equal to `number`. - defp floatify(int, float, decimal, exponential \\ 0) do - multiplier = if int < 0, do: -1.0, else: 1.0 + `floor/2` also accepts a precision to round a floating-point value down + to an arbitrary number of fractional digits (between 0 and 15). + The operation is performed on the binary floating point, without a + conversion to decimal. - # Try to ensure the minimum amount of rounding errors - result = multiplier * (abs(int) * :math.pow(10, decimal) + float) * :math.pow(10, exponential - decimal) + This function always returns a float. `Kernel.trunc/1` may be used instead to + truncate the result to an integer afterwards. - # Try avoiding stuff like this: - # iex(1)> 0.0001 * 75 - # 0.007500000000000001 - # Due to IEEE 754 floating point standard - # http://docs.oracle.com/cd/E19957-01/806-3568/ncg_goldberg.html + ## Known issues - final_decimal_places = decimal - exponential - if final_decimal_places > 0 do - decimal_power_round = :math.pow(10, final_decimal_places) - trunc(result * decimal_power_round) / decimal_power_round - else - result - end - end + The behavior of `floor/2` for floats can be surprising. For example: - @doc """ - Rounds a float to the largest integer less than or equal to `num`. + iex> Float.floor(12.52, 2) + 12.51 - ## Examples + One may have expected it to floor to 12.52. This is not a bug. + Most decimal fractions cannot be represented as a binary floating point + and therefore the number above is internally represented as 12.51999999, + which explains the behavior above. - iex> Float.floor(34) - 34 + ## Examples iex> Float.floor(34.25) - 34 - + 34.0 iex> Float.floor(-56.5) - -57 + -57.0 + iex> Float.floor(34.259, 2) + 34.25 """ - @spec floor(float | integer) :: integer - def floor(num) when is_integer(num), do: num - def floor(num) when is_float(num) do - truncated = :erlang.trunc(num) - case :erlang.abs(num - truncated) do - x when x > 0 and num < 0 -> truncated - 1 - _ -> truncated - end + @spec floor(float, precision_range) :: float + def floor(number, precision \\ 0) + + def floor(number, 0) when is_float(number) do + :math.floor(number) + end + + def floor(number, precision) when is_float(number) and precision in @precision_range do + round(number, precision, :floor) + end + + def floor(number, precision) when is_float(number) do + raise ArgumentError, invalid_precision_message(precision) end @doc """ - Rounds a float to the largest integer greater than or equal to `num`. + Rounds a float to the smallest float greater than or equal to `number`. - ## Examples + `ceil/2` also accepts a precision to round a floating-point value down + to an arbitrary number of fractional digits (between 0 and 15). - iex> Float.ceil(34) - 34 + The operation is performed on the binary floating point, without a + conversion to decimal. - iex> Float.ceil(34.25) - 35 + The behavior of `ceil/2` for floats can be surprising. For example: + + iex> Float.ceil(-12.52, 2) + -12.51 + + One may have expected it to ceil to -12.52. This is not a bug. + Most decimal fractions cannot be represented as a binary floating point + and therefore the number above is internally represented as -12.51999999, + which explains the behavior above. + + This function always returns floats. `Kernel.trunc/1` may be used instead to + truncate the result to an integer afterwards. + ## Examples + + iex> Float.ceil(34.25) + 35.0 iex> Float.ceil(-56.5) - -56 + -56.0 + iex> Float.ceil(34.251, 2) + 34.26 + iex> Float.ceil(-0.01) + -0.0 """ - @spec ceil(float | integer) :: integer - def ceil(num) when is_integer(num), do: num - def ceil(num) when is_float(num) do - truncated = :erlang.trunc(num) - case :erlang.abs(num - truncated) do - x when x > 0 and num > 0 -> truncated + 1 - _ -> truncated - end + @spec ceil(float, precision_range) :: float + def ceil(number, precision \\ 0) + + def ceil(number, 0) when is_float(number) do + :math.ceil(number) + end + + def ceil(number, precision) when is_float(number) and precision in @precision_range do + round(number, precision, :ceil) + end + + def ceil(number, precision) when is_float(number) do + raise ArgumentError, invalid_precision_message(precision) end @doc """ - Rounds a floating point value to an arbitrary number of fractional digits - (between 0 and 15). + Rounds a floating-point value to an arbitrary number of fractional + digits (between 0 and 15). + + The rounding direction always ties to half up. The operation is + performed on the binary floating point, without a conversion to decimal. + + This function only accepts floats and always returns a float. Use + `Kernel.round/1` if you want a function that accepts both floats + and integers and always returns an integer. + + ## Known issues + + The behavior of `round/2` for floats can be surprising. For example: + + iex> Float.round(5.5675, 3) + 5.567 + + One may have expected it to round to the half up 5.568. This is not a bug. + Most decimal fractions cannot be represented as a binary floating point + and therefore the number above is internally represented as 5.567499999, + which explains the behavior above. If you want exact rounding for decimals, + you must use a decimal library. The behavior above is also in accordance + to reference implementations, such as "Correctly Rounded Binary-Decimal and + Decimal-Binary Conversions" by David M. Gay. ## Examples + iex> Float.round(12.5) + 13.0 iex> Float.round(5.5674, 3) 5.567 - iex> Float.round(5.5675, 3) - 5.568 - + 5.567 iex> Float.round(-5.5674, 3) -5.567 - - iex> Float.round(-5.5675, 3) - -5.568 + iex> Float.round(-5.5675) + -6.0 + iex> Float.round(12.341444444444441, 15) + 12.341444444444441 + iex> Float.round(-0.01) + -0.0 """ - @spec round(float, integer) :: float - def round(number, precision) when is_float(number) and is_integer(precision) and precision in 0..15 do - Kernel.round(number * :math.pow(10, precision)) / :math.pow(10, precision) + @spec round(float, precision_range) :: float + # This implementation is slow since it relies on big integers. + # Faster implementations are available on more recent papers + # and could be implemented in the future. + def round(float, precision \\ 0) + + def round(float, 0) when float == 0.0, do: float + + def round(float, 0) when is_float(float) do + case float |> :erlang.round() |> :erlang.float() do + zero when zero == 0.0 and float < 0.0 -> -0.0 + rounded -> rounded + end end - @doc """ - Returns a char list which corresponds to the text representation of the given float. + def round(float, precision) when is_float(float) and precision in @precision_range do + round(float, precision, :half_up) + end - Inlined by the compiler. + def round(float, precision) when is_float(float) do + raise ArgumentError, invalid_precision_message(precision) + end - ## Examples + defp round(num, _precision, _rounding) when is_float(num) and num == 0.0, do: num + + defp round(float, precision, rounding) do + <> = <> + {num, count} = decompose(significant, 1) + count = count - exp + 1023 + + cond do + # Precision beyond 15 digits + count >= 104 -> + case rounding do + :ceil when sign === 0 -> 1 / power_of_10(precision) + :floor when sign === 1 -> -1 / power_of_10(precision) + :ceil when sign === 1 -> minus_zero() + :half_up when sign === 1 -> minus_zero() + _ -> 0.0 + end + + # We are asking more precision than we have + count <= precision -> + float + + true -> + # Difference in precision between float and asked precision + # We subtract 1 because we need to calculate the remainder too + diff = count - precision - 1 + + # Get up to latest so we calculate the remainder + power_of_10 = power_of_10(diff) + + # Convert the numerand to decimal base + num = num * power_of_5(count) + + # Move to the given precision - 1 + num = div(num, power_of_10) + div = div(num, 10) + num = rounding(rounding, sign, num, div) + + # Convert back to float without loss + # https://www.exploringbinary.com/correct-decimal-to-floating-point-using-big-integers/ + den = power_of_10(precision) + boundary = den <<< 52 + + cond do + num == 0 and sign == 1 -> + minus_zero() + + num == 0 -> + 0.0 + + num >= boundary -> + {den, exp} = scale_down(num, boundary, 52) + decimal_to_float(sign, num, den, exp) + + true -> + {num, exp} = scale_up(num, boundary, 52) + decimal_to_float(sign, num, den, exp) + end + end + end - iex> Float.to_char_list(7.0) - '7.00000000000000000000e+00' + # TODO remove once we require Erlang/OTP 27+ + # This function tricks the compiler to avoid this bug in previous versions: + # https://github.com/elixir-lang/elixir/blob/main/lib/elixir/lib/float.ex#L408-L412 + defp minus_zero, do: -0.0 - """ - @spec to_char_list(float) :: char_list - def to_char_list(number) do - :erlang.float_to_list(number) + defp decompose(significant, initial) do + decompose(significant, 1, 0, initial) end - @doc """ - Returns a list which corresponds to the text representation - of `float`. + defp decompose(<<1::1, bits::bitstring>>, count, last_count, acc) do + decompose(bits, count + 1, count, (acc <<< (count - last_count)) + 1) + end - ## Options + defp decompose(<<0::1, bits::bitstring>>, count, last_count, acc) do + decompose(bits, count + 1, last_count, acc) + end - * `:decimals` — number of decimal points to show - * `:scientific` — number of decimal points to show, in scientific format - * `:compact` — when true, use the most compact representation (ignored - with the `scientific` option) + defp decompose(<<>>, _count, last_count, acc) do + {acc, last_count} + end + + defp scale_up(num, boundary, exp) when num >= boundary, do: {num, exp} + defp scale_up(num, boundary, exp), do: scale_up(num <<< 1, boundary, exp - 1) + + defp scale_down(num, den, exp) do + new_den = den <<< 1 + + if num < new_den do + {den >>> 52, exp} + else + scale_down(num, new_den, exp + 1) + end + end + + defp decimal_to_float(sign, num, den, exp) do + quo = div(num, den) + rem = num - quo * den + + tmp = + case den >>> 1 do + den when rem > den -> quo + 1 + den when rem < den -> quo + _ when (quo &&& 1) === 1 -> quo + 1 + _ -> quo + end + + tmp = tmp - @power_of_2_to_52 + <> = <> + tmp + end + + defp rounding(:floor, 1, _num, div), do: div + 1 + defp rounding(:ceil, 0, _num, div), do: div + 1 + + defp rounding(:half_up, _sign, num, div) do + case rem(num, 10) do + rem when rem < 5 -> div + rem when rem >= 5 -> div + 1 + end + end + + defp rounding(_, _, _, div), do: div + + Enum.reduce(0..104, 1, fn x, acc -> + defp power_of_10(unquote(x)), do: unquote(acc) + acc * 10 + end) + + Enum.reduce(0..104, 1, fn x, acc -> + defp power_of_5(unquote(x)), do: unquote(acc) + acc * 5 + end) + + @doc """ + Returns a pair of integers whose ratio is exactly equal + to the original float and with a positive denominator. ## Examples - iex> Float.to_char_list 7.1, [decimals: 2, compact: true] - '7.1' + iex> Float.ratio(0.0) + {0, 1} + iex> Float.ratio(3.14) + {7070651414971679, 2251799813685248} + iex> Float.ratio(-3.14) + {-7070651414971679, 2251799813685248} + iex> Float.ratio(1.5) + {3, 2} + iex> Float.ratio(-1.5) + {-3, 2} + iex> Float.ratio(16.0) + {16, 1} + iex> Float.ratio(-16.0) + {-16, 1} """ - @spec to_char_list(float, list) :: char_list - def to_char_list(float, options) do - :erlang.float_to_list(float, expand_compact(options)) + @doc since: "1.4.0" + @spec ratio(float) :: {integer, pos_integer} + def ratio(float) when is_float(float) and float == 0.0, do: {0, 1} + + def ratio(float) when is_float(float) do + <> = <> + + {num, den_exp} = + if exp != 0 do + # Floats are expressed like this: + # (2**52 + mantissa) * 2**(-52 + exp - 1023) + # + # We compute the root factors of the mantissa so we have this: + # (2**52 + mantissa * 2**count) * 2**(-52 + exp - 1023) + {mantissa, count} = root_factors(mantissa, 0) + + # Now we can move the count around so we have this: + # (2**(52-count) + mantissa) * 2**(count + -52 + exp - 1023) + if mantissa == 0 do + {1, exp - 1023} + else + num = (1 <<< (52 - count)) + mantissa + den_exp = count - 52 + exp - 1023 + {num, den_exp} + end + else + # Subnormals are expressed like this: + # (mantissa) * 2**(-52 + 1 - 1023) + # + # So we compute it to this: + # (mantissa * 2**(count)) * 2**(-52 + 1 - 1023) + # + # Which becomes: + # mantissa * 2**(count-1074) + root_factors(mantissa, -1074) + end + + if den_exp > 0 do + {sign(sign, num <<< den_exp), 1} + else + {sign(sign, num), 1 <<< -den_exp} + end end + defp root_factors(mantissa, count) when mantissa != 0 and (mantissa &&& 1) == 0, + do: root_factors(mantissa >>> 1, count + 1) + + defp root_factors(mantissa, count), + do: {mantissa, count} + + @compile {:inline, sign: 2} + defp sign(0, num), do: num + defp sign(1, num), do: -num + @doc """ - Returns a binary which corresponds to the text representation - of `some_float`. + Returns a charlist which corresponds to the shortest text representation + of the given float. + + It uses the algorithm presented in "Ryū: fast float-to-string conversion" + in Proceedings of the SIGPLAN '2018 Conference on Programming Language + Design and Implementation. + + For a configurable representation, use `:erlang.float_to_list/2`. Inlined by the compiler. ## Examples - iex> Float.to_string(7.0) - "7.00000000000000000000e+00" + iex> Float.to_charlist(7.0) + ~c"7.0" """ - @spec to_string(float) :: String.t - def to_string(some_float) do - :erlang.float_to_binary(some_float) + @spec to_charlist(float) :: charlist + def to_charlist(float) do + :erlang.float_to_list(float, [:short]) end @doc """ - Returns a binary which corresponds to the text representation - of `float`. + Returns a binary which corresponds to the shortest text representation + of the given float. + + The underlying algorithm changes depending on the Erlang/OTP version: - ## Options + * For OTP >= 24, it uses the algorithm presented in "Ryū: fast + float-to-string conversion" in Proceedings of the SIGPLAN '2018 + Conference on Programming Language Design and Implementation. - * `:decimals` — number of decimal points to show - * `:scientific` — number of decimal points to show, in scientific format - * `:compact` — when true, use the most compact representation (ignored - with the `scientific` option) + * For OTP < 24, it uses the algorithm presented in "Printing Floating-Point + Numbers Quickly and Accurately" in Proceedings of the SIGPLAN '1996 + Conference on Programming Language Design and Implementation. + + For a configurable representation, use `:erlang.float_to_binary/2`. + + Inlined by the compiler. ## Examples - iex> Float.to_string 7.1, [decimals: 2, compact: true] - "7.1" + iex> Float.to_string(7.0) + "7.0" """ - @spec to_string(float, list) :: String.t + @spec to_string(float) :: String.t() + def to_string(float) do + :erlang.float_to_binary(float, [:short]) + end + + @doc false + @deprecated "Use Float.to_charlist/1 instead" + def to_char_list(float), do: Float.to_charlist(float) + + @doc false + @deprecated "Use :erlang.float_to_list/2 instead" + def to_char_list(float, options) do + :erlang.float_to_list(float, expand_compact(options)) + end + + @doc false + @deprecated "Use :erlang.float_to_binary/2 instead" def to_string(float, options) do :erlang.float_to_binary(float, expand_compact(options)) end - defp expand_compact([{:compact, false}|t]), do: expand_compact(t) - defp expand_compact([{:compact, true}|t]), do: [:compact|expand_compact(t)] - defp expand_compact([h|t]), do: [h|expand_compact(t)] - defp expand_compact([]), do: [] + defp invalid_precision_message(precision) do + "precision #{precision} is out of valid range of #{inspect(@precision_range)}" + end + + defp expand_compact([{:compact, false} | t]), do: expand_compact(t) + defp expand_compact([{:compact, true} | t]), do: [:compact | expand_compact(t)] + defp expand_compact([h | t]), do: [h | expand_compact(t)] + defp expand_compact([]), do: [] end diff --git a/lib/elixir/lib/function.ex b/lib/elixir/lib/function.ex new file mode 100644 index 00000000000..654666a1a89 --- /dev/null +++ b/lib/elixir/lib/function.ex @@ -0,0 +1,212 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +defmodule Function do + @moduledoc """ + A set of functions for working with functions. + + Anonymous functions are typically created by using `fn`: + + iex> add = fn a, b -> a + b end + iex> add.(1, 2) + 3 + + Anonymous functions can also have multiple clauses. All clauses + should expect the same number of arguments: + + iex> negate = fn + ...> true -> false + ...> false -> true + ...> end + iex> negate.(false) + true + + ## The capture operator + + It is also possible to capture public module functions and pass them + around as if they were anonymous functions by using the capture + operator `&/1`: + + iex> add = &Kernel.+/2 + iex> add.(1, 2) + 3 + + iex> length = &String.length/1 + iex> length.("hello") + 5 + + To capture a definition within the current module, you can skip the + module prefix, such as `&my_fun/2`. In those cases, the captured + function can be public (`def`) or private (`defp`). + + The capture operator can also be used to create anonymous functions + that expect at least one argument: + + iex> add = &(&1 + &2) + iex> add.(1, 2) + 3 + + In such cases, using the capture operator is no different than using `fn`. + + ## Internal and external functions + + We say that functions that point to definitions residing in modules, such + as `&String.length/1`, are **external** functions. All other functions are + **local** and they are always bound to the file or module that defined them. + + Besides the functions in this module to work with functions, `Kernel` also + has an `apply/2` function that invokes a function with a dynamic number of + arguments, as well as `is_function/1` and `is_function/2`, to check + respectively if a given value is a function or a function of a given arity. + """ + + @type information :: + :arity + | :env + | :index + | :module + | :name + | :new_index + | :new_uniq + | :pid + | :type + | :uniq + + @doc """ + Captures the given function. + + Inlined by the compiler. + + ## Examples + + iex> Function.capture(String, :length, 1) + &String.length/1 + + """ + @doc since: "1.7.0" + @spec capture(module, atom, arity) :: fun + def capture(module, function_name, arity) do + :erlang.make_fun(module, function_name, arity) + end + + @doc """ + Returns a keyword list with information about a function. + + The returned keys (with the corresponding possible values) for + all types of functions (local and external) are the following: + + * `:type` - `:local` (for anonymous functions) or `:external` (for + named functions). + + * `:module` - an atom which is the module where the function is defined when + anonymous or the module which the function refers to when it's a named function. + + * `:arity` - (integer) the number of arguments the function is to be called with. + + * `:name` - (atom) the name of the function. + + * `:env` - a list of the environment or free variables. For named + functions, the returned list is always empty. + + When `fun` is an anonymous function (that is, the type is `:local`), the following + additional keys are returned: + + * `:pid` - PID of the process that originally created the function. + + * `:index` - (integer) an index into the module function table. + + * `:new_index` - (integer) an index into the module function table. + + * `:new_uniq` - (binary) a unique value for this function. It's + calculated from the compiled code for the entire module. + + * `:uniq` - (integer) a unique value for this function. This integer is + calculated from the compiled code for the entire module. + + **Note**: this function must be used only for debugging purposes. + + Inlined by the compiler. + + ## Examples + + iex> fun = fn x -> x end + iex> info = Function.info(fun) + iex> Keyword.get(info, :arity) + 1 + iex> Keyword.get(info, :type) + :local + + iex> fun = &String.length/1 + iex> info = Function.info(fun) + iex> Keyword.get(info, :type) + :external + iex> Keyword.get(info, :name) + :length + + """ + @doc since: "1.7.0" + @spec info(fun) :: [{information, term}] + def info(fun), do: :erlang.fun_info(fun) + + @doc """ + Returns a specific information about the function. + + The returned information is a two-element tuple in the shape of + `{info, value}`. + + For any function, the information asked for can be any of the atoms + `:module`, `:name`, `:arity`, `:env`, or `:type`. + + For anonymous functions, there is also information about any of the + atoms `:index`, `:new_index`, `:new_uniq`, `:uniq`, and `:pid`. + For a named function, the value of any of these items is always the + atom `:undefined`. + + For more information on each of the possible returned values, see + `info/1`. + + Inlined by the compiler. + + ## Examples + + iex> f = fn x -> x end + iex> Function.info(f, :arity) + {:arity, 1} + iex> Function.info(f, :type) + {:type, :local} + + iex> fun = &String.length/1 + iex> Function.info(fun, :name) + {:name, :length} + iex> Function.info(fun, :pid) + {:pid, :undefined} + + """ + @doc since: "1.7.0" + @spec info(fun, item) :: {item, term} when item: information + def info(fun, item), do: :erlang.fun_info(fun, item) + + @doc """ + Returns its input `value`. This function can be passed as an anonymous function + to transformation functions. + + ## Examples + + iex> Function.identity("Hello world!") + "Hello world!" + + iex> ~c"abcdaabccc" |> Enum.sort() |> Enum.chunk_by(&Function.identity/1) + [~c"aaa", ~c"bb", ~c"cccc", ~c"d"] + + iex> Enum.group_by(~c"abracadabra", &Function.identity/1) + %{97 => ~c"aaaaa", 98 => ~c"bb", 99 => ~c"c", 100 => ~c"d", 114 => ~c"rr"} + + iex> Enum.map([1, 2, 3, 4], &Function.identity/1) + [1, 2, 3, 4] + + """ + @doc since: "1.10.0" + @spec identity(value) :: value when value: var + def identity(value), do: value +end diff --git a/lib/elixir/lib/gen_event.ex b/lib/elixir/lib/gen_event.ex index 99ab1c48388..4763739a8f3 100644 --- a/lib/elixir/lib/gen_event.ex +++ b/lib/elixir/lib/gen_event.ex @@ -1,185 +1,102 @@ -defmodule GenEvent do - @moduledoc """ - A behaviour module for implementing event handling functionality. - - The event handling model consists of a generic event manager - process with an arbitrary number of event handlers which are - added and deleted dynamically. - - An event manager implemented using this module will have a standard - set of interface functions and include functionality for tracing and - error reporting. It will also fit into an supervision tree. - - ## Example - - There are many use cases for event handlers. For example, a logging - system can be built using event handlers where which log message is - an event and different event handlers can be plugged to handle the - log messages. One handler may print error messages on the terminal, - another can write it to a file, while a third one can keep the - messages in memory (like a buffer) until they are read. - - As an example, let's have a GenEvent that accumulates messages until - they are collected by an explicit call. - - defmodule LoggerHandler do - use GenEvent - - # Callbacks - - def handle_event({:log, x}, messages) do - {:ok, [x|messages]} - end - - def handle_call(:messages, messages) do - {:ok, Enum.reverse(messages), []} - end - end - - {:ok, pid} = GenEvent.start_link() - - GenEvent.add_handler(pid, LoggerHandler, []) - #=> :ok - - GenEvent.notify(pid, {:log, 1}) - #=> :ok - - GenEvent.notify(pid, {:log, 2}) - #=> :ok - - GenEvent.call(pid, LoggerHandler, :messages) - #=> [1, 2] - - GenEvent.call(pid, LoggerHandler, :messages) - #=> [] - - We start a new event manager by calling `GenEvent.start_link/0`. - Notifications can be sent to the event manager which will then - invoke `handle_event/0` for each registered handler. - - We can add new handlers with `add_handler/4`. Calls can also - be made to specific handlers by using `call/3`. - - ## Callbacks - - There are 6 callbacks required to be implemented in a `GenEvent`. By - adding `use GenEvent` to your module, Elixir will automatically define - all 6 callbacks for you, leaving it up to you to implement the ones - you want to customize. The callbacks are: - - * `init(args)` - invoked when the event handler is added. - - It must return: - - - `{:ok, state}` - - `{:ok, state, :hibernate}` - - `{:error, reason}` - - * `handle_event(msg, state)` - invoked whenever an event is sent via - `notify/2` or `sync_notify/2`. - - It must return: - - - `{:ok, new_state}` - - `{:ok, new_state, :hibernate}` - - `{:swap_handler, args1, new_state, handler2, args2}` - - `:remove_handler` - - * `handle_call(msg, state)` - invoked when a `call/3` is done to a specific - handler. - - It must return: - - - `{:ok, reply, new_state}` - - `{:ok, reply, new_state, :hibernate}` - - `{:swap_handler, reply, args1, new_state, handler2, args2}` - - `{:remove_handler, reply}` +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec - * `handle_info(msg, state)` - invoked to handle all other messages which - are received by the process. Must return the same values as - `handle_event/2`. +defmodule GenEvent do + # Functions from this module are deprecated in elixir_dispatch. - It must return: + @moduledoc """ + An event manager with event handlers behaviour. - - `{:noreply, state}` - - `{:noreply, state, timeout}` - - `{:stop, reason, state}` + If you are interested in implementing an event manager, please read the + "Alternatives" section below. If you have to implement an event handler to + integrate with an existing system, such as Elixir's Logger, please use + [`:gen_event`](`:gen_event`) instead. - * `terminate(reason, state)` - called when the event handler is removed or - the event manager is terminating. It can return any term. + ## Alternatives - * `code_change(old_vsn, state, extra)` - called when the application - code is being upgraded live (hot code swapping). + There are a few suitable alternatives to replace GenEvent. Each of them can be + the most beneficial based on the use case. - It must return: + ### Supervisor and GenServers - - `{:ok, new_state}` + One alternative to GenEvent is a very minimal solution consisting of using a + supervisor and multiple GenServers started under it. The supervisor acts as + the "event manager" and the children GenServers act as the "event handlers". + This approach has some shortcomings (it provides no back-pressure for example) + but can still replace GenEvent for low-profile usages of it. [This blog post + by José + Valim](https://dashbit.co/blog/replacing-genevent-by-a-supervisor-plus-genserver) + has more detailed information on this approach. - ## Name Registration + ### GenStage - A GenEvent is bound to the same name registration rules as a `GenServer`. - Read more about it in the `GenServer` docs. + If the use case where you were using GenEvent requires more complex logic, + [GenStage](https://github.com/elixir-lang/gen_stage) provides a great + alternative. GenStage is an external Elixir library maintained by the Elixir + team; it provides a tool to implement systems that exchange events in a + demand-driven way with built-in support for back-pressure. See the [GenStage + documentation](https://hexdocs.pm/gen_stage) for more information. - ## Streaming + ### `:gen_event` - `GenEvent`s can be streamed from and streamed with the help of `stream/2`. - Here are some examples: + If your use case requires exactly what GenEvent provided, or you have to + integrate with an existing `:gen_event`-based system, you can still use the + [`:gen_event`](`:gen_event`) Erlang module. + """ - stream = GenEvent.stream(pid) + @moduledoc deprecated: "Use Erlang/OTP's :gen_event module instead" - # Take the next 10 events - Enum.take(stream, 10) + @callback init(args :: term) :: + {:ok, state} + | {:ok, state, :hibernate} + | {:error, reason :: term} + when state: term - # Print all remaining events - for event <- stream do - IO.inspect event - end + @callback handle_event(event :: term, state :: term) :: + {:ok, new_state} + | {:ok, new_state, :hibernate} + | :remove_handler + when new_state: term - A stream may also be given an id, which allows all streams with the given - id to be cancelled at any moment via `cancel_streams/1`. + @callback handle_call(request :: term, state :: term) :: + {:ok, reply, new_state} + | {:ok, reply, new_state, :hibernate} + | {:remove_handler, reply} + when reply: term, new_state: term - ## Learn more + @callback handle_info(msg :: term, state :: term) :: + {:ok, new_state} + | {:ok, new_state, :hibernate} + | :remove_handler + when new_state: term - If you wish to find out more about gen events, Elixir getting started - guides provide a tutorial-like introduction. The documentation and links - in Erlang can also provide extra insight. + @callback terminate(reason, state :: term) :: term + when reason: :stop | {:stop, term} | :remove_handler | {:error, term} | term - * http://elixir-lang.org/getting_started/mix/1.html - * http://www.erlang.org/doc/man/gen_event.html - * http://learnyousomeerlang.com/event-handlers - """ + @callback code_change(old_vsn, state :: term, extra :: term) :: {:ok, new_state :: term} + when old_vsn: term | {:down, term} - @typedoc "Return values of `start*` functions" @type on_start :: {:ok, pid} | {:error, {:already_started, pid}} - @typedoc "The GenEvent manager name" @type name :: atom | {:global, term} | {:via, module, term} - @typedoc "Options used by the `start*` functions" @type options :: [name: name] - @typedoc "The event manager reference" @type manager :: pid | name | {atom, node} - @typedoc "Supported values for new handlers" - @type handler :: module | {module, term} - - @doc """ - Defines a `GenEvent` stream. + @type handler :: atom | {atom, term} - This is a struct returned by `stream/2`. The struct is public and - contains the following fields: - - * `:manager` - the manager reference given to `GenEvent.stream/2` - * `:id` - the event stream id for cancellation - * `:timeout` - the timeout in between events, defaults to `:infinity` - * `:duration` - the duration of the subscription, defaults to `:infinity` - * `:mode` - if the subscription mode is sync or async, defaults to `:sync` - """ - defstruct manager: nil, id: nil, timeout: :infinity, duration: :infinity, mode: :sync + message = "Use one of the alternatives described in the documentation for the GenEvent module" + @deprecated message @doc false defmacro __using__(_) do + deprecation_message = + "the GenEvent module is deprecated, see its documentation for alternatives" + + IO.warn(deprecation_message, __CALLER__) + quote location: :keep do @behaviour :gen_event @@ -194,8 +111,21 @@ defmodule GenEvent do end @doc false - def handle_call(_request, state) do - {:ok, {:error, :bad_call}, state} + def handle_call(msg, state) do + proc = + case Process.info(self(), :registered_name) do + {_, []} -> self() + {_, name} -> name + end + + # We do this to trick Dialyzer to not complain about non-local returns. + case :erlang.phash2(1, 1) do + 0 -> + raise "attempted to call GenEvent #{inspect(proc)} but no handle_call/2 clause was provided" + + 1 -> + {:remove_handler, {:bad_call, msg}} + end end @doc false @@ -204,7 +134,7 @@ defmodule GenEvent do end @doc false - def terminate(reason, state) do + def terminate(_reason, _state) do :ok end @@ -213,452 +143,767 @@ defmodule GenEvent do {:ok, state} end - defoverridable [init: 1, handle_event: 2, handle_call: 2, - handle_info: 2, terminate: 2, code_change: 3] + defoverridable init: 1, + handle_event: 2, + handle_call: 2, + handle_info: 2, + terminate: 2, + code_change: 3 end end - @doc """ - Starts an event manager linked to the current process. - - This is often used to start the `GenEvent` as part of a supervision tree. - - It accepts the `:name` option which is described under the `Name Registration` - section in the `GenServer` module docs. - - If the event manager is successfully created and initialized, the function - returns `{:ok, pid}`, where pid is the pid of the server. If there already - exists a process with the specified server name, the function returns - `{:error, {:already_started, pid}}` with the pid of that process. - - Note that a `GenEvent` started with `start_link/1` is linked to the - parent process and will exit not only on crashes but also if the parent - process exits with `:normal` reason. - """ + @doc false + @deprecated message @spec start_link(options) :: on_start def start_link(options \\ []) when is_list(options) do do_start(:link, options) end - @doc """ - Starts an event manager process without links (outside of a supervision tree). - - See `start_link/1` for more information. - """ + @doc false + @deprecated message @spec start(options) :: on_start def start(options \\ []) when is_list(options) do do_start(:nolink, options) end + @no_callback :"no callback module" + defp do_start(mode, options) do case Keyword.get(options, :name) do nil -> - :gen.start(:gen_event, mode, :"no callback module", [], []) + :gen.start(GenEvent, mode, @no_callback, [], []) + atom when is_atom(atom) -> - :gen.start(:gen_event, mode, {:local, atom}, :"no callback module", [], []) - other when is_tuple(other) -> - :gen.start(:gen_event, mode, other, :"no callback module", [], []) + :gen.start(GenEvent, mode, {:local, atom}, @no_callback, [], []) + + {:global, _term} = tuple -> + :gen.start(GenEvent, mode, tuple, @no_callback, [], []) + + {:via, via_module, _term} = tuple when is_atom(via_module) -> + :gen.start(GenEvent, mode, tuple, @no_callback, [], []) + + other -> + raise ArgumentError, """ + expected :name option to be one of the following: + + * nil + * atom + * {:global, term} + * {:via, module, term} + + Got: #{inspect(other)} + """ + end + end + + @doc false + @deprecated message + @spec stream(manager, keyword) :: GenEvent.Stream.t() + def stream(manager, options \\ []) do + %GenEvent.Stream{manager: manager, timeout: Keyword.get(options, :timeout, :infinity)} + end + + @doc false + @deprecated message + @spec add_handler(manager, handler, term) :: :ok | {:error, term} + def add_handler(manager, handler, args) do + rpc(manager, {:add_handler, handler, args}) + end + + @doc false + @deprecated message + @spec add_mon_handler(manager, handler, term) :: :ok | {:error, term} + def add_mon_handler(manager, handler, args) do + rpc(manager, {:add_mon_handler, handler, args, self()}) + end + + @doc false + @deprecated message + @spec notify(manager, term) :: :ok + def notify(manager, event) + + def notify({:global, name}, msg) do + try do + :global.send(name, {:notify, msg}) + :ok + catch + _, _ -> :ok end end - @doc """ - Returns a stream that consumes and notifies events to the `manager`. + def notify({:via, mod, name}, msg) when is_atom(mod) do + try do + mod.send(name, {:notify, msg}) + :ok + catch + _, _ -> :ok + end + end - The stream is a `GenEvent` struct that implements the `Enumerable` - protocol. The supported options are: + def notify(manager, msg) + when is_pid(manager) + when is_atom(manager) + when tuple_size(manager) == 2 and is_atom(elem(manager, 0)) and is_atom(elem(manager, 1)) do + send(manager, {:notify, msg}) + :ok + end - * `:id` - an id to identify all live stream instances; when an `:id` is - given, existing streams can be called with via `cancel_streams`. + @doc false + @deprecated message + @spec sync_notify(manager, term) :: :ok + def sync_notify(manager, event) do + rpc(manager, {:sync_notify, event}) + end - * `:timeout` (Enumerable) - raises if no event arrives in X milliseconds. + @doc false + @deprecated message + @spec ack_notify(manager, term) :: :ok + def ack_notify(manager, event) do + rpc(manager, {:ack_notify, event}) + end - * `:duration` (Enumerable) - only consume events during the X milliseconds - from the streaming start. + @doc false + @deprecated message + @spec call(manager, handler, term, timeout) :: term | {:error, term} + def call(manager, handler, request, timeout \\ 5000) do + try do + :gen.call(manager, self(), {:call, handler, request}, timeout) + catch + :exit, reason -> + exit({reason, {__MODULE__, :call, [manager, handler, request, timeout]}}) + else + {:ok, res} -> res + end + end - * `:mode` - the mode to consume events, can be `:sync` (default) or - `:async`. On sync, the event manager waits for the event to be consumed - before moving on to the next event handler. + @doc false + @deprecated message + @spec remove_handler(manager, handler, term) :: term | {:error, term} + def remove_handler(manager, handler, args) do + rpc(manager, {:delete_handler, handler, args}) + end - """ - def stream(manager, options \\ []) do - %GenEvent{manager: manager, - id: Keyword.get(options, :id), - timeout: Keyword.get(options, :timeout, :infinity), - duration: Keyword.get(options, :duration, :infinity), - mode: Keyword.get(options, :mode, :sync)} + @doc false + @deprecated message + @spec swap_handler(manager, handler, term, handler, term) :: :ok | {:error, term} + def swap_handler(manager, handler1, args1, handler2, args2) do + rpc(manager, {:swap_handler, handler1, args1, handler2, args2}) end - @doc """ - Adds a new event handler to the event `manager`. + @doc false + @deprecated message + @spec swap_mon_handler(manager, handler, term, handler, term) :: :ok | {:error, term} + def swap_mon_handler(manager, handler1, args1, handler2, args2) do + rpc(manager, {:swap_mon_handler, handler1, args1, handler2, args2, self()}) + end + + @doc false + @deprecated message + @spec which_handlers(manager) :: [handler] + def which_handlers(manager) do + rpc(manager, :which_handlers) + end - The event manager will call the `init/1` callback with `args` to - initiate the event handler and its internal state. + @doc false + @deprecated message + @spec stop(manager, reason :: term, timeout) :: :ok + def stop(manager, reason \\ :normal, timeout \\ :infinity) do + :gen.stop(manager, reason, timeout) + end - If `init/1` returns a correct value indicating successful completion, - the event manager adds the event handler and this function returns - `:ok`. If the callback fails with `reason` or returns `{:error, reason}`, - the event handler is ignored and this function returns `{:EXIT, reason}` - or `{:error, reason}`, respectively. + defp rpc(module, cmd) do + {:ok, reply} = :gen.call(module, self(), cmd, :infinity) + reply + end - ## Linked handlers + ## Init callbacks - When adding a handler, a `:link` option with value `true` can be given. - This means the event handler and the calling process are now linked. + require Record + Record.defrecordp(:handler, [:module, :id, :state, :pid, :ref]) - If the calling process later terminates with `reason`, the event manager - will delete the event handler by calling the `terminate/2` callback with - `{:stop, reason}` as argument. If the event handler later is deleted, - the event manager sends a message `{:gen_event_EXIT, handler, reason}` - to the calling process. Reason is one of the following: + @doc false + def init_it(starter, :self, name, mod, args, options) do + init_it(starter, self(), name, mod, args, options) + end - * `:normal` - if the event handler has been removed due to a call to - `remove_handler/3`, or `:remove_handler` has been returned by a callback - function + def init_it(starter, parent, name, _mod, _args, options) do + Process.put(:"$initial_call", {__MODULE__, :init_it, 6}) + debug = :gen.debug_options(name, options) + :proc_lib.init_ack(starter, {:ok, self()}) + loop(parent, name(name), [], debug, false) + end - * `:shutdown` - if the event handler has been removed because the event - manager is terminating + @doc false + def init_hib(parent, name, handlers, debug) do + fetch_msg(parent, name, handlers, debug, true) + end - * `{:swapped, new_handler, pid}` - if the process pid has replaced the - event handler by another + defp name({:local, name}), do: name + defp name({:global, name}), do: name + defp name({:via, _, name}), do: name + defp name(pid) when is_pid(pid), do: pid - * a term - if the event handler is removed due to an error. Which term - depends on the error + ## Loop - """ - @spec add_handler(manager, handler, term, [link: boolean]) :: :ok | {:EXIT, term} | {:error, term} - def add_handler(manager, handler, args, options \\ []) do - case Keyword.get(options, :link, false) do - true -> :gen_event.add_sup_handler(manager, handler, args) - false -> :gen_event.add_handler(manager, handler, args) + defp loop(parent, name, handlers, debug, true) do + :proc_lib.hibernate(__MODULE__, :init_hib, [parent, name, handlers, debug]) + end + + defp loop(parent, name, handlers, debug, false) do + fetch_msg(parent, name, handlers, debug, false) + end + + defp fetch_msg(parent, name, handlers, debug, hib) do + receive do + {:system, from, req} -> + :sys.handle_system_msg(req, from, parent, __MODULE__, debug, [name, handlers, hib], hib) + + {:EXIT, ^parent, reason} -> + server_terminate(reason, parent, handlers, name) + + msg when debug == [] -> + handle_msg(msg, parent, name, handlers, []) + + msg -> + debug = :sys.handle_debug(debug, &print_event/3, name, {:in, msg}) + handle_msg(msg, parent, name, handlers, debug) end end - @doc """ - Sends an event notification to the event `manager`. + defp handle_msg(msg, parent, name, handlers, debug) do + case msg do + {:notify, event} -> + {hib, handlers} = server_event(:async, event, handlers, name) + loop(parent, name, handlers, debug, hib) + + {_from, _tag, {:notify, event}} -> + {hib, handlers} = server_event(:async, event, handlers, name) + loop(parent, name, handlers, debug, hib) + + {_from, tag, {:ack_notify, event}} -> + reply(tag, :ok) + {hib, handlers} = server_event(:ack, event, handlers, name) + loop(parent, name, handlers, debug, hib) + + {_from, tag, {:sync_notify, event}} -> + {hib, handlers} = server_event(:sync, event, handlers, name) + reply(tag, :ok) + loop(parent, name, handlers, debug, hib) + + {:DOWN, ref, :process, _pid, reason} = other -> + case handle_down(ref, reason, handlers, name) do + {:ok, handlers} -> + loop(parent, name, handlers, debug, false) + + :error -> + {hib, handlers} = server_info(other, handlers, name) + loop(parent, name, handlers, debug, hib) + end + + {_from, tag, {:call, handler, query}} -> + {hib, reply, handlers} = server_call(handler, query, handlers, name) + reply(tag, reply) + loop(parent, name, handlers, debug, hib) - The event manager will call `handle_event/2` for each installed event handler. + {_from, tag, {:add_handler, handler, args}} -> + {hib, reply, handlers} = server_add_handler(handler, args, handlers) + reply(tag, reply) + loop(parent, name, handlers, debug, hib) - `notify` is asynchronous and will return immediately after the notification is - sent. `notify` will not fail even if the specified event manager does not exist, - unless it is specified as `name` (atom). - """ - @spec notify(manager, term) :: :ok - defdelegate notify(manager, event), to: :gen_event + {_from, tag, {:add_mon_handler, handler, args, notify}} -> + {hib, reply, handlers} = server_add_mon_handler(handler, args, handlers, notify) + reply(tag, reply) + loop(parent, name, handlers, debug, hib) - @doc """ - Sends a sync event notification to the event `manager`. + {_from, tag, {:add_process_handler, pid, notify}} -> + {hib, reply, handlers} = server_add_process_handler(pid, handlers, notify) + reply(tag, reply) + loop(parent, name, handlers, debug, hib) - In other words, this function only returns `:ok` after the event manager - invokes the `handle_event/2` on each installed event handler. + {_from, tag, {:delete_handler, handler, args}} -> + {reply, handlers} = server_remove_handler(handler, args, handlers, name) + reply(tag, reply) + loop(parent, name, handlers, debug, false) - See `notify/2` for more info. - """ - @spec sync_notify(manager, term) :: :ok - defdelegate sync_notify(manager, event), to: :gen_event + {_from, tag, {:swap_handler, handler1, args1, handler2, args2}} -> + {hib, reply, handlers} = + server_swap_handler(handler1, args1, handler2, args2, handlers, nil, name) - @doc """ - Makes a synchronous call to the event `handler` installed in `manager`. + reply(tag, reply) + loop(parent, name, handlers, debug, hib) - The given `request` is sent and the caller waits until a reply arrives or - a timeout occurs. The event manager will call `handle_call/2` to handle - the request. + {_from, tag, {:swap_mon_handler, handler1, args1, handler2, args2, mon}} -> + {hib, reply, handlers} = + server_swap_handler(handler1, args1, handler2, args2, handlers, mon, name) - The return value `reply` is defined in the return value of `handle_call/2`. - If the specified event handler is not installed, the function returns - `{:error, :bad_module}`. - """ - @spec call(manager, handler, term, timeout) :: term | {:error, term} - def call(manager, handler, request, timeout \\ 5000) do - :gen_event.call(manager, handler, request, timeout) + reply(tag, reply) + loop(parent, name, handlers, debug, hib) + + {_from, tag, :which_handlers} -> + reply(tag, server_which_handlers(handlers)) + loop(parent, name, handlers, debug, false) + + {_from, tag, :get_modules} -> + reply(tag, server_get_modules(handlers)) + loop(parent, name, handlers, debug, false) + + other -> + {hib, handlers} = server_info(other, handlers, name) + loop(parent, name, handlers, debug, hib) + end end - @doc """ - Cancels all streams currently running with the given `:id`. + ## System callbacks - In order for a stream to be cancelled, an `:id` must be passed - when the stream is created via `stream/2`. Passing a stream without - an id leads to an argument error. - """ - @spec cancel_streams(t) :: :ok - def cancel_streams(%GenEvent{id: nil}) do - raise ArgumentError, "cannot cancel streams without an id" + @doc false + def system_continue(parent, debug, [name, handlers, hib]) do + loop(parent, name, handlers, debug, hib) end - def cancel_streams(%GenEvent{manager: manager, id: id}) do - handlers = :gen_event.which_handlers(manager) + @doc false + def system_terminate(reason, parent, _debug, [name, handlers, _hib]) do + server_terminate(reason, parent, handlers, name) + end - for {Enumerable.GenEvent, {handler_id, _}} = ref <- handlers, - handler_id === id do - :gen_event.delete_handler(manager, ref, :remove_handler) - end + @doc false + def system_code_change([name, handlers, hib], module, old_vsn, extra) do + handlers = + for handler <- handlers do + if handler(handler, :module) == module do + {:ok, state} = module.code_change(old_vsn, handler(handler, :state), extra) + handler(handler, state: state) + else + handler + end + end - :ok + {:ok, [name, handlers, hib]} end - @doc """ - Removes an event handler from the event `manager`. + @doc false + def system_get_state([_name, handlers, _hib]) do + tuples = + for handler(module: mod, id: id, state: state) <- handlers do + {mod, id, state} + end - The event manager will call `terminate/2` to terminate the event handler - and return the callback value. If the specified event handler is not - installed, the function returns `{:error, :module_not_found}`. - """ - @spec remove_handler(manager, handler, term) :: term | {:error, term} - def remove_handler(manager, handler, args) do - :gen_event.delete_handler(manager, handler, args) + {:ok, tuples} end - @doc """ - Replaces an old event handler with a new one in the event `manager`. + @doc false + def system_replace_state(fun, [name, handlers, hib]) do + {handlers, states} = + :lists.unzip( + for handler <- handlers do + handler(module: mod, id: id, state: state) = handler + cur = {mod, id, state} + + try do + new = {^mod, ^id, new_state} = fun.(cur) + {handler(handler, state: new_state), new} + catch + _, _ -> + {handler, cur} + end + end + ) + + {:ok, states, [name, handlers, hib]} + end + + # Keeping deprecated format_status/2 since the current implementation is not + # compatible with format_status/1 and GenEvent is deprecated anyway + @doc false + def format_status(opt, status_data) do + [pdict, sys_state, parent, _debug, [name, handlers, _hib]] = status_data + header = :gen.format_status_header(~c"Status for event handler", name) + + formatted = + for handler <- handlers do + handler(module: module, state: state) = handler + + if function_exported?(module, :format_status, 2) do + try do + state = module.format_status(opt, [pdict, state]) + handler(handler, state: state) + catch + _, _ -> handler + end + else + handler + end + end - First, the old event handler is deleted by calling `terminate/2` with - the given `args1` and collects the return value. Then the new event handler - is added and initiated by calling `init({args2, term}), where term is the - return value of calling `terminate/2` in the old handler. This makes it - possible to transfer information from one handler to another. + [ + header: header, + data: [{~c"Status", sys_state}, {~c"Parent", parent}], + items: {~c"Installed handlers", formatted} + ] + end - The new handler will be added even if the specified old event handler - is not installed in which case `term = :error` or if the handler fails to - terminate with a given reason. + ## Loop helpers - If there was a linked connection between handler1 and a process pid, there - will be a link connection between handler2 and pid instead. A new link in - between the caller process and the new handler can also be set with by - giving `link: true` as option. See `add_handler/4` for more information. + defp print_event(dev, {:in, msg}, name) do + case msg do + {:notify, event} -> + IO.puts(dev, "*DBG* #{inspect(name)} got event #{inspect(event)}") - If `init/1` in the second handler returns a correct value, this function - returns `:ok`. - """ - @spec swap_handler(manager, handler, term, handler, term, [link: boolean]) :: :ok | {:error, term} - def swap_handler(manager, handler1, args1, handler2, args2, options \\ []) do - case Keyword.get(options, :link, false) do - true -> :gen_event.swap_sup_handler(manager, {handler1, args1}, {handler2, args2}) - false -> :gen_event.swap_handler(manager, {handler1, args1}, {handler2, args2}) + {_, _, {:call, handler, query}} -> + IO.puts( + dev, + "*DBG* #{inspect(name)} (handler #{inspect(handler)}) got call #{inspect(query)}" + ) + + _ -> + IO.puts(dev, "*DBG* #{inspect(name)} got #{inspect(msg)}") end end - @doc """ - Returns a list of all event handlers installed in the `manager`. - """ - @spec which_handlers(manager) :: [handler] - defdelegate which_handlers(manager), to: :gen_event + defp print_event(dev, dbg, name) do + IO.puts(dev, "*DBG* #{inspect(name)}: #{inspect(dbg)}") + end - @doc """ - Terminates the event `manager`. + defp server_add_handler({module, id}, args, handlers) do + handler = handler(module: module, id: {module, id}) + do_add_handler(module, handler, args, handlers, :ok) + end - Before terminating, the event manager will call `terminate(:stop, ...)` - for each installed event handler. - """ - @spec stop(manager) :: :ok - defdelegate stop(manager), to: :gen_event -end + defp server_add_handler(module, args, handlers) do + handler = handler(module: module, id: module) + do_add_handler(module, handler, args, handlers, :ok) + end -defimpl Enumerable, for: GenEvent do - use GenEvent + defp server_add_mon_handler({module, id}, args, handlers, notify) do + ref = Process.monitor(notify) + handler = handler(module: module, id: {module, id}, pid: notify, ref: ref) + do_add_handler(module, handler, args, handlers, :ok) + end - @doc false - def init({_mode, mon_pid, _pid, ref} = state) do - # Tell the mon_pid we are good to go, and send self() so that this handler - # can be removed later without using the managers name. - send(mon_pid, {:UP, ref, self()}) - {:ok, state} + defp server_add_mon_handler(module, args, handlers, notify) do + ref = Process.monitor(notify) + handler = handler(module: module, id: module, pid: notify, ref: ref) + do_add_handler(module, handler, args, handlers, :ok) end - @doc false - def handle_event(event, {:sync, mon_pid, pid, ref} = state) do - sync = Process.monitor(mon_pid) - send pid, {ref, sync, event} - receive do - {^sync, :done} -> - Process.demonitor(sync, [:flush]) - :remove_handler - {^sync, :next} -> - Process.demonitor(sync, [:flush]) - {:ok, state} - {:DOWN, ^sync, _, _, _} -> - {:ok, state} + defp server_add_process_handler(pid, handlers, notify) do + ref = Process.monitor(pid) + handler = handler(module: GenEvent.Stream, id: {self(), ref}, pid: notify, ref: ref) + do_add_handler(GenEvent.Stream, handler, {pid, ref}, handlers, {self(), ref}) + end + + defp server_remove_handler(module, args, handlers, name) do + do_take_handler(module, args, handlers, name, :remove, :normal) + end + + defp server_swap_handler(module1, args1, module2, args2, handlers, sup, name) do + {state, handlers} = + do_take_handler(module1, args1, handlers, name, :swapped, {:swapped, module2, sup}) + + if sup do + server_add_mon_handler(module2, {args2, state}, handlers, sup) + else + server_add_handler(module2, {args2, state}, handlers) + end + end + + defp server_info(event, handlers, name) do + handlers = :lists.reverse(handlers) + server_notify(event, :handle_info, handlers, name, handlers, [], false) + end + + defp server_event(mode, event, handlers, name) do + {handlers, streams} = server_split_process_handlers(mode, event, handlers, [], []) + {hib, handlers} = server_notify(event, :handle_event, handlers, name, handlers, [], false) + {hib, server_collect_process_handlers(mode, event, streams, handlers, name)} + end + + defp server_split_process_handlers(mode, event, [handler | t], handlers, streams) do + case handler(handler, :id) do + {pid, _ref} when is_pid(pid) -> + server_process_notify(mode, event, handler) + server_split_process_handlers(mode, event, t, handlers, [handler | streams]) + + _ -> + server_split_process_handlers(mode, event, t, [handler | handlers], streams) end end - def handle_event(event, {:async, _mon_pid, pid, ref} = state) do - send pid, {ref, nil, event} - {:ok, state} + defp server_split_process_handlers(_mode, _event, [], handlers, streams) do + {handlers, streams} end - def reduce(stream, acc, fun) do - start_fun = fn() -> start(stream) end - next_fun = &next(stream, &1) - stop_fun = &stop(stream, &1) - Stream.resource(start_fun, next_fun, stop_fun).(acc, wrap_reducer(fun)) + defp server_process_notify(mode, event, handler(state: {pid, ref})) do + send(pid, {self(), {self(), ref}, {mode_to_tag(mode), event}}) end - def count(_stream) do - {:error, __MODULE__} + defp mode_to_tag(:ack), do: :ack_notify + defp mode_to_tag(:sync), do: :sync_notify + defp mode_to_tag(:async), do: :notify + + defp server_notify(event, fun, [handler | t], name, handlers, acc, hib) do + case server_update(handler, fun, event, name, handlers) do + {new_hib, handler} -> + server_notify(event, fun, t, name, handlers, [handler | acc], hib or new_hib) + + :error -> + server_notify(event, fun, t, name, handlers, acc, hib) + end end - def member?(_stream, _item) do - {:error, __MODULE__} + defp server_notify(_, _, [], _, _, acc, hib) do + {hib, acc} end - defp wrap_reducer(fun) do - fn - {nil, _manager, event}, acc -> - fun.(event, acc) - {ref, manager, event}, acc -> - try do - fun.(event, acc) - after - send manager, {ref, :next} + defp server_update(handler, fun, event, name, _handlers) do + handler(module: module, state: state) = handler + + case do_handler(module, fun, [event, state]) do + {:ok, res} -> + case res do + {:ok, state} -> + {false, handler(handler, state: state)} + + {:ok, state, :hibernate} -> + {true, handler(handler, state: state)} + + :remove_handler -> + do_terminate(handler, :remove_handler, event, name, :normal) + :error + + other -> + reason = {:bad_return_value, other} + do_terminate(handler, {:error, reason}, event, name, reason) + :error end + + {:error, reason} -> + do_terminate(handler, {:error, reason}, event, name, reason) + :error end end - defp start(%{manager: manager, id: id, duration: duration, mode: mode} = stream) do - {mon_pid, mon_ref} = add_handler(mode, manager, id, duration) - send mon_pid, {:UP, mon_ref, self()} + defp server_collect_process_handlers(:async, event, [handler | t], handlers, name) do + server_collect_process_handlers(:async, event, t, [handler | handlers], name) + end + + defp server_collect_process_handlers(mode, event, [handler | t], handlers, name) + when mode in [:sync, :ack] do + handler(ref: ref, id: id) = handler receive do - # The subscription process gave us a go. - {:UP, ^mon_ref, manager_pid} -> - {mon_ref, mon_pid, manager_pid} - # The subscription process died due to an abnormal reason. - {:DOWN, ^mon_ref, _, _, reason} -> - exit({reason, {__MODULE__, :start, [stream]}}) + {^ref, :ok} -> + server_collect_process_handlers(mode, event, t, [handler | handlers], name) + + {_from, tag, {:delete_handler, ^id, args}} -> + do_terminate(handler, args, :remove, name, :normal) + reply(tag, :ok) + server_collect_process_handlers(mode, event, t, handlers, name) + + {:DOWN, ^ref, _, _, reason} -> + do_terminate(handler, {:stop, reason}, :DOWN, name, :shutdown) + server_collect_process_handlers(mode, event, t, handlers, name) end end - defp next(%{timeout: timeout} = stream, {mon_ref, mon_pid, manager_pid} = acc) do - # If :DOWN is received must resend it to self so that stop/2 can receive it - # and know that the handler has been removed. - receive do - {:DOWN, ^mon_ref, _, _, :normal} -> - send(self(), {:DOWN, mon_ref, :process, mon_pid, :normal}) - nil - {:DOWN, ^mon_ref, _, _, reason} -> - send(self(), {:DOWN, mon_ref, :process, mon_pid, :normal}) - exit({reason, {__MODULE__, :next, [stream, acc]}}) - {^mon_ref, sync_ref, event} -> - {{sync_ref, manager_pid, event}, acc} - after - timeout -> - exit({:timeout, {__MODULE__, :next, [stream, acc]}}) + defp server_collect_process_handlers(_mode, _event, [], handlers, _name) do + handlers + end + + defp server_call(module, query, handlers, name) do + case :lists.keyfind(module, handler(:id) + 1, handlers) do + false -> + {false, {:error, :not_found}, handlers} + + handler -> + case server_call_update(handler, query, name, handlers) do + {{hib, handler}, reply} -> + {hib, reply, :lists.keyreplace(module, handler(:id) + 1, handlers, handler)} + + {:error, reply} -> + {false, reply, :lists.keydelete(module, handler(:id) + 1, handlers)} + end end end - defp stop(%{mode: mode} = stream, {mon_ref, mon_pid, manager_pid} = acc) do - case remove_handler(mon_ref, mon_pid, manager_pid) do - :ok when mode == :async -> - flush_events(mon_ref) - :ok -> - :ok + defp server_call_update(handler, query, name, _handlers) do + handler(module: module, state: state) = handler + + case do_handler(module, :handle_call, [query, state]) do + {:ok, res} -> + case res do + {:ok, reply, state} -> + {{false, handler(handler, state: state)}, reply} + + {:ok, reply, state, :hibernate} -> + {{true, handler(handler, state: state)}, reply} + + {:remove_handler, reply} -> + do_terminate(handler, :remove_handler, query, name, :normal) + {:error, reply} + + other -> + reason = {:bad_return_value, other} + do_terminate(handler, {:error, reason}, query, name, reason) + {:error, {:error, reason}} + end + {:error, reason} -> - exit({reason, {__MODULE__, :stop, [stream, acc]}}) + do_terminate(handler, {:error, reason}, query, name, reason) + {:error, {:error, reason}} end end - defp add_handler(mode, manager, id, duration) do - parent = self() - - # The subscription is managed by another process, that dies if - # the handler dies, and is killed when there is a need to remove - # the subscription. - spawn_monitor(fn -> - # It is possible that the handler could be removed, and then the GenEvent - # could exit before this process has exited normally. Because the removal - # does not cause an unlinking this process would exit with the same - # reason. Trapping exits ensures that no errors is raised in this case. - Process.flag(:trap_exit, true) - parent_ref = Process.monitor(parent) - - # Receive the notification from the parent, unless it died. - mon_ref = receive do - {:UP, ref, ^parent} -> ref - {:DOWN, ^parent_ref, _, _, _} -> exit(:normal) - end + defp server_get_modules(handlers) do + for(handler(module: module) <- handlers, do: module) + |> :ordsets.from_list() + |> :ordsets.to_list() + end - cancel = cancel_ref(id, mon_ref) - :ok = :gen_event.add_sup_handler(manager, {__MODULE__, cancel}, - {mode, self(), parent, mon_ref}) - - receive do - # This message is already in the mailbox if we got this far. - {:UP, ^mon_ref, manager_pid} -> - send(parent, {:UP, mon_ref, manager_pid}) - receive do - # The stream has finished, remove the handler. - {:DONE, ^mon_ref} -> - exit_handler(manager_pid, parent_ref, cancel) - - # If the parent died, we can exit normally. - {:DOWN, ^parent_ref, _, _, _} -> - exit(:normal) - - # reason should be normal unless the handler is swapped. - {:gen_event_EXIT, {__MODULE__, ^cancel}, reason} -> - exit(reason) - - # Exit if the manager dies, so the streamer is notified. - {:EXIT, ^manager_pid, :noconnection} -> - exit({:nodedown, node(manager_pid)}) - - {:EXIT, ^manager_pid, reason} -> - exit(reason) - after - # Our time is over, notify the parent. - duration -> exit(:normal) - end + defp server_which_handlers(handlers) do + for handler(id: id) <- handlers, do: id + end + + defp server_terminate(reason, _parent, handlers, name) do + _ = + for handler <- handlers do + do_terminate(handler, :stop, :stop, name, :shutdown) end - end) + + exit(reason) end - defp cancel_ref(nil, mon_ref), do: mon_ref - defp cancel_ref(id, mon_ref), do: {id, mon_ref} + defp reply({from, ref}, msg) do + send(from, {ref, msg}) + end - defp exit_handler(manager_pid, parent_ref, cancel) do - # Send exit signal so manager removes handler. - Process.exit(manager_pid, :shutdown) - receive do - # If the parent died, we can exit normally. - {:DOWN, ^parent_ref, _, _, _} -> - exit(:normal) - - # Probably the reason is :shutdown, which occurs when the manager receives - # an exit signal from a handler supervising process. However whatever the - # reason the handler has been removed so it is ok. - {:gen_event_EXIT, {__MODULE__, ^cancel}, _} -> - exit(:normal) - - # The connection broke, perhaps the handler might try to forward events - # before it removes the handler, so must exit abnormally. - {:EXIT, ^manager_pid, :noconnection} -> - exit({:nodedown, node(manager_pid)}) - - # The manager has exited but don't exit abnormally as the handler has died - # with the manager and all expected events have been handled. This is ok. - {:EXIT, ^manager_pid, _} -> - exit(:normal) + defp handle_down(ref, reason, handlers, name) do + case :lists.keyfind(ref, handler(:ref) + 1, handlers) do + false -> + :error + + handler -> + do_terminate(handler, {:stop, reason}, :DOWN, name, :shutdown) + {:ok, :lists.keydelete(ref, handler(:ref) + 1, handlers)} end end - defp remove_handler(mon_ref, mon_pid, manager_pid) do - send(mon_pid, {:DONE, mon_ref}) - receive do - {^mon_ref, sync, _} when sync != nil -> - send(manager_pid, {sync, :done}) - Process.demonitor(mon_ref, [:flush]) - :ok - {:DOWN, ^mon_ref, _, _, :normal} -> - :ok - {:DOWN, ^mon_ref, _, _, reason} -> - {:error, reason} + defp do_add_handler(module, handler, arg, handlers, succ) do + case :lists.keyfind(handler(handler, :id), handler(:id) + 1, handlers) do + false -> + case do_handler(module, :init, [arg]) do + {:ok, res} -> + case res do + {:ok, state} -> + {false, succ, [handler(handler, state: state) | handlers]} + + {:ok, state, :hibernate} -> + {true, succ, [handler(handler, state: state) | handlers]} + + {:error, _} = error -> + {false, error, handlers} + + other -> + {false, {:error, {:bad_return_value, other}}, handlers} + end + + {:error, _} = error -> + {false, error, handlers} + end + + _ -> + {false, {:error, :already_present}, handlers} end end - defp flush_events(mon_ref) do - receive do - {^mon_ref, _, _} -> - flush_events(mon_ref) - after - 0 -> :ok + defp do_take_handler(module, args, handlers, name, last_in, reason) do + case :lists.keytake(module, handler(:id) + 1, handlers) do + {:value, handler, handlers} -> + {do_terminate(handler, args, last_in, name, reason), handlers} + + false -> + {{:error, :not_found}, handlers} + end + end + + defp do_terminate(handler, arg, last_in, name, reason) do + handler(module: module, state: state) = handler + + res = + case do_handler(module, :terminate, [arg, state]) do + {:ok, res} -> res + {:error, _} = error -> error + end + + report_terminate(handler, reason, state, last_in, name) + res + end + + defp do_handler(mod, fun, args) do + try do + apply(mod, fun, args) + catch + :throw, val -> {:ok, val} + :error, val -> {:error, {val, __STACKTRACE__}} + :exit, val -> {:error, val} + else + res -> {:ok, res} + end + end + + defp report_terminate(handler, reason, state, last_in, name) do + report_error(handler, reason, state, last_in, name) + + if ref = handler(handler, :ref) do + Process.demonitor(ref, [:flush]) + end + + if pid = handler(handler, :pid) do + send(pid, {:gen_event_EXIT, handler(handler, :id), reason}) + end + end + + defp report_error(_handler, :normal, _, _, _), do: :ok + defp report_error(_handler, :shutdown, _, _, _), do: :ok + defp report_error(_handler, {:swapped, _, _}, _, _, _), do: :ok + + defp report_error(handler, reason, state, last_in, name) do + reason = + case reason do + {:undef, [{m, f, a, _} | _] = mfas} -> + cond do + :code.is_loaded(m) == false -> + {:"module could not be loaded", mfas} + + function_exported?(m, f, length(a)) -> + reason + + true -> + {:"function not exported", mfas} + end + + _ -> + reason + end + + formatted = report_status(handler, state) + + :error_logger.error_msg( + ~c"** gen_event handler ~p crashed.~n" ++ + ~c"** Was installed in ~p~n" ++ + ~c"** Last event was: ~p~n" ++ ~c"** When handler state == ~p~n" ++ ~c"** Reason == ~p~n", + [handler(handler, :id), name, last_in, formatted, reason] + ) + end + + defp report_status(handler(module: module), state) do + if function_exported?(module, :format_status, 2) do + try do + module.format_status(:terminate, [Process.get(), state]) + catch + _, _ -> state + end + else + state end end end diff --git a/lib/elixir/lib/gen_event/stream.ex b/lib/elixir/lib/gen_event/stream.ex new file mode 100644 index 00000000000..2b906bf54cd --- /dev/null +++ b/lib/elixir/lib/gen_event/stream.ex @@ -0,0 +1,179 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +defmodule GenEvent.Stream do + @moduledoc false + @moduledoc deprecated: "This functionality is no longer supported" + defstruct manager: nil, timeout: :infinity + + @type t :: %__MODULE__{manager: GenEvent.manager(), timeout: timeout} + + @doc false + def init({_pid, _ref} = state) do + {:ok, state} + end + + @doc false + def handle_event(event, _state) do + # We do this to trick Dialyzer to not complain about non-local returns. + case :erlang.phash2(1, 1) do + 0 -> exit({:bad_event, event}) + 1 -> :remove_handler + end + end + + @doc false + def handle_call(msg, _state) do + # We do this to trick Dialyzer to not complain about non-local returns. + reason = {:bad_call, msg} + + case :erlang.phash2(1, 1) do + 0 -> exit(reason) + 1 -> {:remove_handler, reason} + end + end + + @doc false + def handle_info(_msg, state) do + {:ok, state} + end + + @doc false + def terminate(_reason, _state) do + :ok + end + + @doc false + def code_change(_old, state, _extra) do + {:ok, state} + end +end + +defimpl Enumerable, for: GenEvent.Stream do + @moduledoc false + @moduledoc deprecated: "This functionality is no longer supported" + + def reduce(stream, acc, fun) do + start_fun = fn -> start(stream) end + next_fun = &next(stream, &1) + stop_fun = &stop(stream, &1) + Stream.resource(start_fun, next_fun, stop_fun).(acc, wrap_reducer(fun)) + end + + def count(_stream) do + {:error, __MODULE__} + end + + def member?(_stream, _item) do + {:error, __MODULE__} + end + + def slice(_stream) do + {:error, __MODULE__} + end + + defp wrap_reducer(fun) do + fn + {:ack, manager, ref, event}, acc -> + send(manager, {ref, :ok}) + fun.(event, acc) + + {:async, _manager, _ref, event}, acc -> + fun.(event, acc) + + {:sync, manager, ref, event}, acc -> + try do + fun.(event, acc) + after + send(manager, {ref, :ok}) + end + end + end + + defp start(%{manager: manager} = stream) do + try do + {:ok, {pid, ref}} = + :gen.call(manager, self(), {:add_process_handler, self(), self()}, :infinity) + + mon_ref = Process.monitor(pid) + {pid, ref, mon_ref} + catch + :exit, reason -> exit({reason, {__MODULE__, :start, [stream]}}) + end + end + + defp next(%{timeout: timeout} = stream, {pid, ref, mon_ref} = acc) do + self = self() + + receive do + # Got an async event. + {_from, {^pid, ^ref}, {:notify, event}} -> + {[{:async, pid, ref, event}], acc} + + # Got a sync event. + {_from, {^pid, ^ref}, {:sync_notify, event}} -> + {[{:sync, pid, ref, event}], acc} + + # Got an ack event. + {_from, {^pid, ^ref}, {:ack_notify, event}} -> + {[{:ack, pid, ref, event}], acc} + + # The handler was removed. Stop iteration, resolve the + # event later. We need to demonitor now, otherwise DOWN + # appears with higher priority in the shutdown process. + {:gen_event_EXIT, {^pid, ^ref}, _reason} = event -> + Process.demonitor(mon_ref, [:flush]) + send(self, event) + {:halt, {:removed, acc}} + + # The manager died. Stop iteration, resolve the event later. + {:DOWN, ^mon_ref, _, _, _} = event -> + send(self, event) + {:halt, {:removed, acc}} + after + timeout -> + exit({:timeout, {__MODULE__, :next, [stream, acc]}}) + end + end + + # If we reach this branch, we know the handler was already + # removed, so we don't trigger a request for doing so. + defp stop(stream, {:removed, {pid, ref, mon_ref} = acc}) do + case wait_for_handler_removal(pid, ref, mon_ref) do + :ok -> + flush_events(ref) + + {:error, reason} -> + exit({reason, {__MODULE__, :stop, [stream, acc]}}) + end + end + + # If we reach this branch, the handler was not removed yet, + # so we trigger a request for doing so. + defp stop(stream, {pid, ref, _} = acc) do + _ = :gen_event.delete_handler(pid, {pid, ref}, :shutdown) + stop(stream, {:removed, acc}) + end + + defp wait_for_handler_removal(pid, ref, mon_ref) do + receive do + {:gen_event_EXIT, {^pid, ^ref}, _reason} -> + Process.demonitor(mon_ref, [:flush]) + :ok + + {:DOWN, ^mon_ref, _, _, reason} -> + {:error, reason} + end + end + + defp flush_events(ref) do + receive do + {_from, {_pid, ^ref}, {notify, _event}} + when notify in [:notify, :ack_notify, :sync_notify] -> + flush_events(ref) + after + 0 -> :ok + end + end +end diff --git a/lib/elixir/lib/gen_server.ex b/lib/elixir/lib/gen_server.ex index faeb3344aab..7aec82f93ba 100644 --- a/lib/elixir/lib/gen_server.ex +++ b/lib/elixir/lib/gen_server.ex @@ -1,207 +1,857 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + defmodule GenServer do @moduledoc """ A behaviour module for implementing the server of a client-server relation. - A GenServer is a process as any other Elixir process and it can be used + A GenServer is a process like any other Elixir process and it can be used to keep state, execute code asynchronously and so on. The advantage of using a generic server process (GenServer) implemented using this module is that it will have a standard set of interface functions and include functionality for tracing and error reporting. It will also fit into a supervision tree. + ```mermaid + graph BT + C(Client #3) ~~~ B(Client #2) ~~~ A(Client #1) + A & B & C -->|request| GenServer + GenServer -.->|reply| A & B & C + ``` + ## Example The GenServer behaviour abstracts the common client-server interaction. - Developer are only required to implement the callbacks and functionality they are - interested in. + Developers are only required to implement the callbacks and functionality + they are interested in. Let's start with a code example and then explore the available callbacks. - Imagine we want a GenServer that works like a stack, allowing us to push - and pop items: + Imagine we want to implement a service with a GenServer that works + like a stack, allowing us to push and pop elements. We'll customize a + generic GenServer with our own module by implementing three callbacks. + + `c:init/1` transforms our initial argument to the initial state for the + GenServer. `c:handle_call/3` fires when the server receives a synchronous + `pop` message, popping an element from the stack and returning it to the + user. `c:handle_cast/2` will fire when the server receives an asynchronous + `push` message, pushing an element onto the stack: defmodule Stack do use GenServer # Callbacks - def handle_call(:pop, _from, [h|t]) do - {:reply, h, t} + @impl true + def init(elements) do + initial_state = String.split(elements, ",", trim: true) + {:ok, initial_state} + end + + @impl true + def handle_call(:pop, _from, state) do + [to_caller | new_state] = state + {:reply, to_caller, new_state} end - def handle_cast({:push, item}, state) do - {:noreply, [item|state]} + @impl true + def handle_cast({:push, element}, state) do + new_state = [element | state] + {:noreply, new_state} end end + We leave the process machinery of startup, message passing, and the message + loop to the GenServer behaviour and focus only on the stack + implementation. We can now use the GenServer API to interact with + the service by creating a process and sending it messages: + # Start the server - {:ok, pid} = GenServer.start_link(Stack, [:hello]) + {:ok, pid} = GenServer.start_link(Stack, "hello,world") # This is the client GenServer.call(pid, :pop) - #=> :hello + #=> "hello" - GenServer.cast(pid, {:push, :world}) + GenServer.cast(pid, {:push, "elixir"}) #=> :ok GenServer.call(pid, :pop) - #=> :world + #=> "elixir" + + We start our `Stack` by calling `start_link/2`, passing the module + with the server implementation and its initial argument with a + comma-separated list of elements. The GenServer behaviour calls the + `c:init/1` callback to establish the initial GenServer state. From + this point on, the GenServer has control so we interact with it by + sending two types of messages on the client. **call** messages expect + a reply from the server (and are therefore synchronous) while **cast** + messages do not. + + Each call to `GenServer.call/3` results in a message + that must be handled by the `c:handle_call/3` callback in the GenServer. + A `cast/2` message must be handled by `c:handle_cast/2`. `GenServer` + supports 8 callbacks, but only `c:init/1` is required. + + > #### `use GenServer` {: .info} + > + > When you `use GenServer`, the `GenServer` module will + > set `@behaviour GenServer` and define a `child_spec/1` + > function, so your module can be used as a child + > in a supervision tree. - We start our `Stack` by calling `start_link/3`, passing the module - with the server implementation and its initial argument (a list - representing the stack containing the item `:hello`). We can primarily - interact with the server by sending two types of messages. **call** - messages expect a reply from the server (and are therefore synchronous) - while **cast** messages do not. + ## Client / Server APIs - Every time you do a `GenServer.call/3`, the client will send a message - that must be handled by the `handle_call/3` callback in the GenServer. - A `cast/2` message must be handled by `handle_cast/2`. + Although in the example above we have used `GenServer.start_link/3` and + friends to directly start and communicate with the server, most of the + time we don't call the `GenServer` functions directly. Instead, we wrap + the calls in new functions representing the public API of the server. + These thin wrappers are called the **client API**. - ## Callbacks + Here is a better implementation of our Stack module: - There are 6 callbacks required to be implemented in a `GenServer`. By - adding `use GenServer` to your module, Elixir will automatically define - all 6 callbacks for you, leaving it up to you to implement the ones - you want to customize. The callbacks are: + defmodule Stack do + use GenServer - * `init(args)` - invoked when the server is started. + # Client - It must return: + def start_link(default) when is_binary(default) do + GenServer.start_link(__MODULE__, default) + end - - `{:ok, state}` - - `{:ok, state, timeout}` - - `:ignore` - - `{:stop, reason}` + def push(pid, element) do + GenServer.cast(pid, {:push, element}) + end + + def pop(pid) do + GenServer.call(pid, :pop) + end + + # Server (callbacks) + + @impl true + def init(elements) do + initial_state = String.split(elements, ",", trim: true) + {:ok, initial_state} + end + + @impl true + def handle_call(:pop, _from, state) do + [to_caller | new_state] = state + {:reply, to_caller, new_state} + end + + @impl true + def handle_cast({:push, element}, state) do + new_state = [element | state] + {:noreply, new_state} + end + end + + In practice, it is common to have both server and client functions in + the same module. If the server and/or client implementations are growing + complex, you may want to have them in different modules. - * `handle_call(msg, {from, ref}, state)` and `handle_cast(msg, state)` - - invoked to handle call (sync) and cast (async) messages. + The following diagram summarizes the interactions between client and server. + Both Client and Server are processes and communication happens via messages + (continuous line). The Server <-> Module interaction happens when the + GenServer process calls your code (dotted lines): - It must return: + ```mermaid + sequenceDiagram + participant C as Client (Process) + participant S as Server (Process) + participant M as Module (Code) - - `{:reply, reply, new_state}` - - `{:reply, reply, new_state, timeout}` - - `{:reply, reply, new_state, :hibernate}` - - `{:noreply, new_state}` - - `{:noreply, new_state, timeout}` - - `{:noreply, new_state, :hibernate}` - - `{:stop, reason, new_state}` - - `{:stop, reason, reply, new_state}` + note right of C: Typically started by a supervisor + C->>+S: GenServer.start_link(module, arg, options) + S-->>+M: init(arg) + M-->>-S: {:ok, state} | :ignore | {:error, reason} + S->>-C: {:ok, pid} | :ignore | {:error, reason} - * `handle_info(msg, state)` - invoked to handle all other messages which - are received by the process. + note right of C: call is synchronous + C->>+S: GenServer.call(pid, message) + S-->>+M: handle_call(message, from, state) + M-->>-S: {:reply, reply, state} | {:stop, reason, reply, state} + S->>-C: reply - It must return: + note right of C: cast is asynchronous + C-)S: GenServer.cast(pid, message) + S-->>+M: handle_cast(message, state) + M-->>-S: {:noreply, state} | {:stop, reason, state} - - `{:noreply, state}` - - `{:noreply, state, timeout}` - - `{:stop, reason, state}` + note right of C: send is asynchronous + C-)S: Kernel.send(pid, message) + S-->>+M: handle_info(message, state) + M-->>-S: {:noreply, state} | {:stop, reason, state} + ``` - * `terminate(reason, state)` - called when the server is about to - terminate, useful for cleaning up. It must return `:ok`. + ## How to supervise - * `code_change(old_vsn, state, extra)` - called when the application - code is being upgraded live (hot code swapping). + A `GenServer` is most commonly started under a supervision tree. + When we invoke `use GenServer`, it automatically defines a `child_spec/1` + function that allows us to start the `Stack` directly under a supervisor. + To start a default stack of `["hello", "world"]` under a supervisor, + we can do: - It must return: + children = [ + {Stack, "hello,world"} + ] - - `{:ok, new_state}` - - `{:error, reason}` + Supervisor.start_link(children, strategy: :one_for_all) - ## Name Registration + Note that specifying a module `MyServer` would be the same as specifying + the tuple `{MyServer, []}`. + + `use GenServer` also accepts a list of options which configures the + child specification and therefore how it runs under a supervisor. + The generated `child_spec/1` can be customized with the following options: + + * `:id` - the child specification identifier, defaults to the current module + * [`:restart`](`m:Supervisor#module-restart-values-restart`) - when the + child should be restarted, defaults to `:permanent` + * [`:shutdown`](`m:Supervisor#module-shutdown-values-shutdown`) - how to + shut down the child, either immediately or by giving it time to shut down + + For example: + + use GenServer, restart: :transient, shutdown: 10_000 + + See the ["Child specification"](`m:Supervisor#module-child_spec-1-function`) section in the `Supervisor` module for more + detailed information. The `@doc` annotation immediately preceding + `use GenServer` will be attached to the generated `child_spec/1` function. + + When stopping the GenServer, for example by returning a `{:stop, reason, new_state}` + tuple from a callback, the exit reason is used by the supervisor to determine + whether the GenServer needs to be restarted. See the "Exit reasons and restarts" + section in the `Supervisor` module. + + ## Name registration Both `start_link/3` and `start/3` support the `GenServer` to register a name on start via the `:name` option. Registered names are also automatically cleaned up on termination. The supported values are: - * an atom - the GenServer is registered locally with the given name - using `Process.register/2`. + * an atom - the GenServer is registered locally (to the current node) + with the given name using `Process.register/2`. - * `{:global, term}`- the GenServer is registered globally with the given - term using the functions in the `:global` module. + * `{:global, term}` - the GenServer is registered globally with the given + term using the functions in the [`:global` module](`:global`). * `{:via, module, term}` - the GenServer is registered with the given - mechanism and name. The `:via` option expects a module name to control - the registration mechanism alongside a name which can be any term. + mechanism and name. The `:via` option expects a module that exports + `register_name/2`, `unregister_name/1`, `whereis_name/1` and `send/2`. + One such example is the [`:global` module](`:global`) which uses these functions + for keeping the list of names of processes and their associated PIDs + that are available globally for a network of Elixir nodes. Elixir also + ships with a local, decentralized and scalable registry called `Registry` + for locally storing names that are generated dynamically. - For example, we could start and register our Stack server locally as follows: + For example, we could start and register our `Stack` server locally as follows: # Start the server and register it locally with name MyStack - {:ok, _} = GenServer.start_link(Stack, [:hello], name: MyStack) + {:ok, _} = GenServer.start_link(Stack, "hello", name: MyStack) # Now messages can be sent directly to MyStack - GenServer.call(MyStack, :pop) #=> :hello + GenServer.call(MyStack, :pop) + #=> "hello" Once the server is started, the remaining functions in this module (`call/3`, - `cast/2`, and friends) will also accept an atom, or any `:global` or `:via` - tuples. In general, the following formats are supported: + `cast/2`, and friends) will also accept an atom, or any `{:global, ...}` or + `{:via, ...}` tuples. In general, the following formats are supported: - * a `pid` - * an `atom` if the server is locally registered + * a PID + * an atom if the server is locally registered * `{atom, node}` if the server is locally registered at another node * `{:global, term}` if the server is globally registered * `{:via, module, name}` if the server is registered through an alternative registry - ## Client / Server APIs + If there is an interest to register dynamic names locally, do not use + atoms, as atoms are never garbage-collected and therefore dynamically + generated atoms won't be garbage-collected. For such cases, you can + set up your own local registry by using the `Registry` module. - Although in the example above we have used `GenServer.start_link/3` and - friends to directly start and communicate with the server, most of the - time we don't call the `GenServer` functions directly. Instead, we wrap - the calls in new functions representing the public API of the server. + For example: - Here is a better implementation of our Stack module: + {:ok, _} = Registry.start_link(keys: :unique, name: :stacks) + name = {:via, Registry, {:stacks, "stack 1"}} + {:ok, _pid} = GenServer.start_link(Stack, "hello", name: name) + GenServer.whereis(name) + #=> #PID<0.150.0> - defmodule Stack do + ## Receiving "regular" messages + + The goal of a `GenServer` is to abstract the "receive" loop for developers, + automatically handling system messages, supporting code change, synchronous + calls and more. Therefore, you should never call your own "receive" inside + the GenServer callbacks as doing so will cause the GenServer to misbehave. + + Besides the synchronous and asynchronous communication provided by `call/3` + and `cast/2`, "regular" messages sent by functions such as `send/2`, + `Process.send_after/4` and similar, can be handled inside the `c:handle_info/2` + callback. + + `c:handle_info/2` can be used in many situations, such as handling monitor + DOWN messages sent by `Process.monitor/1`. Another use case for `c:handle_info/2` + is to perform periodic work, with the help of `Process.send_after/4`: + + defmodule MyApp.Periodically do use GenServer - # Client + def start_link(_) do + GenServer.start_link(__MODULE__, %{}) + end - def start_link(default) do - GenServer.start_link(__MODULE__, default) + @impl true + def init(state) do + # Schedule work to be performed on start + schedule_work() + + {:ok, state} end - def push(pid, item) do - GenServer.cast(pid, {:push, item}) + @impl true + def handle_info(:work, state) do + # Do the desired work here + # ... + + # Reschedule once more + schedule_work() + + {:noreply, state} end - def pop(pid) do - GenServer.call(pid, :pop) + defp schedule_work do + # We schedule the work to happen in 2 hours (written in milliseconds). + # Alternatively, one might write :timer.hours(2) + Process.send_after(self(), :work, 2 * 60 * 60 * 1000) end + end - # Server (callbacks) + ## Timeouts - def handle_call(:pop, _from, [h|t]) do - {:reply, h, t} - end + The return value of `c:init/1` or any of the `handle_*` callbacks may include + a timeout value in milliseconds; if not, `:infinity` is assumed. + The timeout can be used to detect a lull in incoming messages. + + The `timeout()` value is used as follows: + + * If the process has any message already waiting when the `timeout()` value + is returned, the timeout is ignored and the waiting message is handled as + usual. This means that even a timeout of `0` milliseconds is not guaranteed + to execute (if you want to take another action immediately and unconditionally, + use a `:continue` instruction instead). + + * If any message arrives before the specified number of milliseconds + elapse, the timeout is cleared and that message is handled as usual. + + * Otherwise, when the specified number of milliseconds have elapsed with no + message arriving, `handle_info/2` is called with `:timeout` as the first + argument. - def handle_call(request, from, state) do - # Call the default implementation from GenServer - super(request, from, state) + For example: + + defmodule Counter do + use GenServer + + @timeout to_timeout(second: 5) + + @impl true + def init(count) do + {:ok, count, @timeout} end - def handle_cast({:push, item}, state) do - {:noreply, [item|state]} + @impl true + def handle_call(:increment, _from, count) do + new_count = count + 1 + {:reply, new_count, new_count, @timeout} end - def handle_cast(request, state) do - super(request, state) + @impl true + def handle_info(:timeout, count) do + {:stop, :normal, count} end end - In practice, it is common to have both server and client functions in - the same module. If the server and/or client implementations are growing - complex, you may want to have them in different modules. + A `Counter` server will exit with `:normal` if there are no messages in 5 seconds + after the initialization or after the last `:increment` call: + + {:ok, counter_pid} = GenServer.start(Counter, 50) + GenServer.call(counter_pid, :increment) + #=> 51 + + # After 5 seconds + Process.alive?(counter_pid) + #=> false + + ## When (not) to use a GenServer + + So far, we have learned that a `GenServer` can be used as a supervised process + that handles sync and async calls. It can also handle system messages, such as + periodic messages and monitoring events. GenServer processes may also be named. + + A GenServer, or a process in general, must be used to model runtime characteristics + of your system. A GenServer must never be used for code organization purposes. + + In Elixir, code organization is done by modules and functions, processes are not + necessary. For example, imagine you are implementing a calculator and you decide + to put all the calculator operations behind a GenServer: + + def add(a, b) do + GenServer.call(__MODULE__, {:add, a, b}) + end + + def subtract(a, b) do + GenServer.call(__MODULE__, {:subtract, a, b}) + end + + def handle_call({:add, a, b}, _from, state) do + {:reply, a + b, state} + end + + def handle_call({:subtract, a, b}, _from, state) do + {:reply, a - b, state} + end + + This is an anti-pattern not only because it convolutes the calculator logic but + also because you put the calculator logic behind a single process that will + potentially become a bottleneck in your system, especially as the number of + calls grow. Instead just define the functions directly: + + def add(a, b) do + a + b + end + + def subtract(a, b) do + a - b + end + + If you don't need a process, then you don't need a process. Use processes only to + model runtime properties, such as mutable state, concurrency and failures, never + for code organization. + + ## Debugging with the :sys module + + GenServers, as [special processes](https://www.erlang.org/doc/design_principles/spec_proc.html), + can be debugged using the [`:sys` module](`:sys`). + Through various hooks, this module allows developers to introspect the state of + the process and trace system events that happen during its execution, such as + received messages, sent replies and state changes. + + Let's explore the basic functions from the + [`:sys` module](`:sys`) used for debugging: + + * `:sys.get_state/2` - allows retrieval of the state of the process. + In the case of a GenServer process, it will be the callback module state, + as passed into the callback functions as last argument. + * `:sys.get_status/2` - allows retrieval of the status of the process. + This status includes the process dictionary, if the process is running + or is suspended, the parent PID, the debugger state, and the state of + the behaviour module, which includes the callback module state + (as returned by `:sys.get_state/2`). It's possible to change how this + status is represented by defining the optional `c:GenServer.format_status/1` + callback. + * `:sys.trace/3` - prints all the system events to `:stdio`. + * `:sys.statistics/3` - manages collection of process statistics. + * `:sys.no_debug/2` - turns off all debug handlers for the given process. + It is very important to switch off debugging once we're done. Excessive + debug handlers or those that should be turned off, but weren't, can + seriously damage the performance of the system. + * `:sys.suspend/2` - allows to suspend a process so that it only + replies to system messages but no other messages. A suspended process + can be reactivated via `:sys.resume/2`. + + Let's see how we could use those functions for debugging the stack server + we defined earlier. + + iex> {:ok, pid} = Stack.start_link("") + iex> :sys.statistics(pid, true) # turn on collecting process statistics + iex> :sys.trace(pid, true) # turn on event printing + iex> Stack.push(pid, 1) + *DBG* <0.122.0> got cast {push,1} + *DBG* <0.122.0> new state [1] + :ok + + iex> :sys.get_state(pid) + [1] + + iex> Stack.pop(pid) + *DBG* <0.122.0> got call pop from <0.80.0> + *DBG* <0.122.0> sent 1 to <0.80.0>, new state [] + 1 + + iex> :sys.statistics(pid, :get) + {:ok, + [ + start_time: {{2016, 7, 16}, {12, 29, 41}}, + current_time: {{2016, 7, 16}, {12, 29, 50}}, + reductions: 117, + messages_in: 2, + messages_out: 0 + ]} + + iex> :sys.no_debug(pid) # turn off all debug handlers + :ok + + iex> :sys.get_status(pid) + {:status, #PID<0.122.0>, {:module, :gen_server}, + [ + [ + "$initial_call": {Stack, :init, 1}, # process dictionary + "$ancestors": [#PID<0.80.0>, #PID<0.51.0>] + ], + :running, # :running | :suspended + #PID<0.80.0>, # parent + [], # debugger state + [ + header: 'Status for generic server <0.122.0>', # module status + data: [ + {'Status', :running}, + {'Parent', #PID<0.80.0>}, + {'Logged events', []} + ], + data: [{'State', [1]}] + ] + ]} ## Learn more - If you wish to find out more about gen servers, Elixir getting started - guides provide a tutorial-like introduction. The documentation and links + If you wish to find out more about GenServers, the Elixir Getting Started + guide provides a tutorial-like introduction. The documentation and links in Erlang can also provide extra insight. - * http://elixir-lang.org/getting_started/mix/1.html - * http://www.erlang.org/doc/man/gen_server.html - * http://www.erlang.org/doc/design_principles/gen_server_concepts.html - * http://learnyousomeerlang.com/clients-and-servers + * [GenServer - Elixir's Getting Started Guide](genservers.md) + * [`:gen_server` module documentation](`:gen_server`) + * [gen_server Behaviour - OTP Design Principles](https://www.erlang.org/doc/design_principles/gen_server_concepts.html) + * [Clients and Servers - Learn You Some Erlang for Great Good!](http://learnyousomeerlang.com/clients-and-servers) + + """ + + @doc """ + Invoked when the server is started. `start_link/3` or `start/3` will + block until it returns. + + `init_arg` is the argument term (second argument) passed to `start_link/3`. + + Returning `{:ok, state}` will cause `start_link/3` to return + `{:ok, pid}` and the process to enter its loop. + + Returning `{:ok, state, timeout}` is similar to `{:ok, state}`, + except that it also sets a timeout. See the "Timeouts" section + in the module documentation for more information. + + Returning `{:ok, state, :hibernate}` is similar to `{:ok, state}` + except the process is hibernated before entering the loop. See + `c:handle_call/3` for more information on hibernation. + + Returning `{:ok, state, {:continue, continue_arg}}` is similar to + `{:ok, state}` except that immediately after entering the loop, + the `c:handle_continue/2` callback will be invoked with `continue_arg` + as the first argument and `state` as the second one. + + Returning `:ignore` will cause `start_link/3` to return `:ignore` and + the process will exit normally without entering the loop or calling + `c:terminate/2`. If used when part of a supervision tree the parent + supervisor will not fail to start nor immediately try to restart the + `GenServer`. The remainder of the supervision tree will be started + and so the `GenServer` should not be required by other processes. + It can be started later with `Supervisor.restart_child/2` as the child + specification is saved in the parent supervisor. The main use cases for + this are: + + * The `GenServer` is disabled by configuration but might be enabled later. + * An error occurred and it will be handled by a different mechanism than the + `Supervisor`. Likely this approach involves calling `Supervisor.restart_child/2` + after a delay to attempt a restart. + + Returning `{:stop, reason}` will cause `start_link/3` to return + `{:error, reason}` and the process to exit with reason `reason` without + entering the loop or calling `c:terminate/2`. + """ + @callback init(init_arg :: term) :: + {:ok, state} + | {:ok, state, timeout | :hibernate | {:continue, continue_arg :: term}} + | :ignore + | {:stop, reason :: term} + when state: term + + @doc """ + Invoked to handle synchronous `call/3` messages. `call/3` will block until a + reply is received (unless the call times out or nodes are disconnected). + + `request` is the request message sent by a `call/3`, `from` is a 2-tuple + containing the caller's PID and a term that uniquely identifies the call, and + `state` is the current state of the `GenServer`. + + Returning `{:reply, reply, new_state}` sends the response `reply` to the + caller and continues the loop with new state `new_state`. + + Returning `{:reply, reply, new_state, timeout}` is similar to + `{:reply, reply, new_state}` except that it also sets a timeout. + See the "Timeouts" section in the module documentation for more information. + + Returning `{:reply, reply, new_state, :hibernate}` is similar to + `{:reply, reply, new_state}` except the process is hibernated and will + continue the loop once a message is in its message queue. However, if a message is + already in the message queue, the process will continue the loop immediately. + Hibernating a `GenServer` causes garbage collection and leaves a continuous + heap that minimises the memory used by the process. + + Hibernating should not be used aggressively as too much time could be spent + garbage collecting, which would delay the processing of incoming messages. + Normally it should only be used when you are not expecting new messages to + immediately arrive and minimising the memory of the process is shown to be + beneficial. + + Returning `{:reply, reply, new_state, {:continue, continue_arg}}` is similar to + `{:reply, reply, new_state}` except that `c:handle_continue/2` will be invoked + immediately after with `continue_arg` as the first argument and + `state` as the second one. + + Returning `{:noreply, new_state}` does not send a response to the caller and + continues the loop with new state `new_state`. The response must be sent with + `reply/2`. + + There are three main use cases for not replying using the return value: + + * To reply before returning from the callback because the response is known + before calling a slow function. + * To reply after returning from the callback because the response is not yet + available. + * To reply from another process, such as a task. + + When replying from another process the `GenServer` should exit if the other + process exits without replying as the caller will be blocking awaiting a + reply. + + Returning `{:noreply, new_state, timeout | :hibernate | {:continue, continue_arg}}` + is similar to `{:noreply, new_state}` except a timeout, hibernation or continue + occurs as with a `:reply` tuple. + + Returning `{:stop, reason, reply, new_state}` stops the loop and `c:terminate/2` + is called with reason `reason` and state `new_state`. Then, the `reply` is sent + as the response to call and the process exits with reason `reason`. + + Returning `{:stop, reason, new_state}` is similar to + `{:stop, reason, reply, new_state}` except a reply is not sent. + + This callback is optional. If one is not implemented, the server will fail + if a call is performed against it. + """ + @callback handle_call(request :: term, from, state :: term) :: + {:reply, reply, new_state} + | {:reply, reply, new_state, + timeout | :hibernate | {:continue, continue_arg :: term}} + | {:noreply, new_state} + | {:noreply, new_state, timeout | :hibernate | {:continue, continue_arg :: term}} + | {:stop, reason, reply, new_state} + | {:stop, reason, new_state} + when reply: term, new_state: term, reason: term + + @doc """ + Invoked to handle asynchronous `cast/2` messages. + + `request` is the request message sent by a `cast/2` and `state` is the current + state of the `GenServer`. + + Returning `{:noreply, new_state}` continues the loop with new state `new_state`. + + Returning `{:noreply, new_state, timeout}` is similar to `{:noreply, new_state}` + except that it also sets a timeout. See the "Timeouts" section in the module + documentation for more information. + + Returning `{:noreply, new_state, :hibernate}` is similar to + `{:noreply, new_state}` except the process is hibernated before continuing the + loop. See `c:handle_call/3` for more information. + + Returning `{:noreply, new_state, {:continue, continue_arg}}` is similar to + `{:noreply, new_state}` except `c:handle_continue/2` will be invoked + immediately after with `continue_arg` as the first argument and + `state` as the second one. + + Returning `{:stop, reason, new_state}` stops the loop and `c:terminate/2` is + called with the reason `reason` and state `new_state`. The process exits with + reason `reason`. + + This callback is optional. If one is not implemented, the server will fail + if a cast is performed against it. + """ + @callback handle_cast(request :: term, state :: term) :: + {:noreply, new_state} + | {:noreply, new_state, timeout | :hibernate | {:continue, continue_arg :: term}} + | {:stop, reason :: term, new_state} + when new_state: term + + @doc """ + Invoked to handle all other messages. + + `msg` is the message and `state` is the current state of the `GenServer`. When + a timeout occurs the message is `:timeout`. + + Return values are the same as `c:handle_cast/2`. + + This callback is optional. If one is not implemented, the received message + will be logged. + """ + @callback handle_info(msg :: :timeout | term, state :: term) :: + {:noreply, new_state} + | {:noreply, new_state, timeout | :hibernate | {:continue, continue_arg :: term}} + | {:stop, reason :: term, new_state} + when new_state: term + + @doc """ + Invoked to handle continue instructions. + + It is useful for performing work after initialization or for splitting the work + in a callback in multiple steps, updating the process state along the way. + + Return values are the same as `c:handle_cast/2`. + + This callback is optional. If one is not implemented, the server will fail + if a continue instruction is used. + """ + @callback handle_continue(continue_arg, state :: term) :: + {:noreply, new_state} + | {:noreply, new_state, timeout | :hibernate | {:continue, continue_arg}} + | {:stop, reason :: term, new_state} + when new_state: term, continue_arg: term + + @doc """ + Invoked when the server is about to exit. It should do any cleanup required. + + `reason` is exit reason and `state` is the current state of the `GenServer`. + The return value is ignored. + + `c:terminate/2` is useful for cleanup that requires access to the + `GenServer`'s state. However, it is **not guaranteed** that `c:terminate/2` + is called when a `GenServer` exits. Therefore, important cleanup should be + done using process links and/or monitors. A monitoring process will receive the + same exit `reason` that would be passed to `c:terminate/2`. + + `c:terminate/2` is called if: + + * the `GenServer` traps exits (using `Process.flag/2`) *and* the parent + process (the one which called `start_link/1`) sends an exit signal + + * a callback (except `c:init/1`) does one of the following: + + * returns a `:stop` tuple + + * raises (via `raise/2`) or exits (via `exit/1`) + + * returns an invalid value + + If part of a supervision tree, a `GenServer` will receive an exit signal from + its parent process (its supervisor) when the tree is shutting down. The exit + signal is based on the shutdown strategy in the child's specification, where + this value can be: + + * `:brutal_kill`: the `GenServer` is killed and so `c:terminate/2` is not called. + + * a timeout value, where the supervisor will send the exit signal `:shutdown` and + the `GenServer` will have the duration of the timeout to terminate. + If after duration of this timeout the process is still alive, it will be killed + immediately. + + For a more in-depth explanation, please read the "Shutdown values (:shutdown)" + section in the `Supervisor` module. + + If the `GenServer` receives an exit signal (that is not `:normal`) from any + process when it is not trapping exits it will exit abruptly with the same + reason and so not call `c:terminate/2`. Note that a process does *NOT* trap + exits by default and an exit signal is sent when a linked process exits or its + node is disconnected. + + `c:terminate/2` is only called after the `GenServer` finishes processing all + messages which arrived in its mailbox prior to the exit signal. If it + receives a `:kill` signal before it finishes processing those, + `c:terminate/2` will not be called. If `c:terminate/2` is called, any + messages received after the exit signal will still be in the mailbox. + + There is no cleanup needed when the `GenServer` controls a `port` (for example, + `:gen_tcp.socket`) or `t:File.io_device/0`, because these will be closed on + receiving a `GenServer`'s exit signal and do not need to be closed manually + in `c:terminate/2`. + + If `reason` is neither `:normal`, `:shutdown`, nor `{:shutdown, term}` an error is + logged. + + This callback is optional. """ + @callback terminate(reason, state :: term) :: term + when reason: :normal | :shutdown | {:shutdown, term} | term + + @doc """ + Invoked to change the state of the `GenServer` when a different version of a + module is loaded (hot code swapping) and the state's term structure should be + changed. + + `old_vsn` is the previous version of the module (defined by the `@vsn` + attribute) when upgrading. When downgrading the previous version is wrapped in + a 2-tuple with first element `:down`. `state` is the current state of the + `GenServer` and `extra` is any extra data required to change the state. + + Returning `{:ok, new_state}` changes the state to `new_state` and the code + change is successful. + + Returning `{:error, reason}` fails the code change with reason `reason` and + the state remains as the previous state. + + If `c:code_change/3` raises the code change fails and the loop will continue + with its previous state. Therefore this callback does not usually contain side effects. + + This callback is optional. + """ + @callback code_change(old_vsn, state :: term, extra :: term) :: + {:ok, new_state :: term} + | {:error, reason :: term} + when old_vsn: term | {:down, term} + + @doc """ + This function is called by a `GenServer` process in the following situations: + + * [`:sys.get_status/1,2`](`:sys.get_status/1`) is invoked to get the `GenServer` status. + * The `GenServer` process terminates abnormally and logs an error. + + This callback is used to limit the status of the process returned by + [`:sys.get_status/1,2`](`:sys.get_status/1`) or sent to logger. + + The callback gets a map `status` describing the current status and shall return + a map `new_status` with the same keys, but it may transform some values. + + Two possible use cases for this callback is to remove sensitive information + from the state to prevent it from being printed in log files, or to compact + large irrelevant status items that would only clutter the logs. + + ## Example + + @impl GenServer + def format_status(status) do + Map.new(status, fn + {:state, state} -> {:state, Map.delete(state, :private_key)} + {:message, {:password, _}} -> {:message, {:password, "redacted"}} + key_value -> key_value + end) + end + + """ + @doc since: "1.17.0" + @callback format_status(status :: :gen_server.format_status()) :: + new_status :: :gen_server.format_status() + + # TODO: Remove this on v2.0 + @doc deprecated: "Use format_status/1 callback instead" + @callback format_status(reason, pdict_and_state :: list) :: term + when reason: :normal | :terminate + + @optional_callbacks code_change: 3, + terminate: 2, + handle_info: 2, + handle_cast: 2, + handle_call: 3, + format_status: 1, + format_status: 2, + handle_continue: 2 @typedoc "Return values of `start*` functions" @type on_start :: {:ok, pid} | :ignore | {:error, {:already_started, pid} | term} @@ -210,40 +860,123 @@ defmodule GenServer do @type name :: atom | {:global, term} | {:via, module, term} @typedoc "Options used by the `start*` functions" - @type options :: [debug: debug, - name: name, - timeout: timeout, - spawn_opt: Process.spawn_opt] + @type options :: [option] - @typedoc "debug options supported by the `start*` functions" - @type debug :: [:trace | :log | :statistics | {:log_to_file, Path.t}] + @typedoc "Option values used by the `start*` functions" + @type option :: + {:debug, debug} + | {:name, name} + | {:timeout, timeout} + | {:spawn_opt, [Process.spawn_opt()]} + | {:hibernate_after, timeout} - @typedoc "The server reference" + @typedoc "Debug options supported by the `start*` functions" + @type debug :: [:trace | :log | :statistics | {:log_to_file, Path.t()}] + + @typedoc """ + The server reference. + + This is either a plain PID or a value representing a registered name. + See the "Name registration" section of this document for more information. + """ @type server :: pid | name | {atom, node} + @typedoc """ + Tuple describing the client of a call request. + + `pid` is the PID of the caller and `tag` is a unique term used to identify the + call. + """ + @type from :: {pid, tag :: term} + @doc false - defmacro __using__(_) do - quote location: :keep do - @behaviour :gen_server + defmacro __using__(opts) do + quote location: :keep, bind_quoted: [opts: opts] do + @behaviour GenServer - @doc false - def init(args) do - {:ok, args} + if not Module.has_attribute?(__MODULE__, :doc) do + @doc """ + Returns a specification to start this module under a supervisor. + + See `Supervisor`. + """ end + def child_spec(init_arg) do + default = %{ + id: __MODULE__, + start: {__MODULE__, :start_link, [init_arg]} + } + + Supervisor.child_spec(default, unquote(Macro.escape(opts))) + end + + defoverridable child_spec: 1 + + # TODO: Remove this on v2.0 + @before_compile GenServer + @doc false def handle_call(msg, _from, state) do - {:stop, {:bad_call, msg}, state} + proc = + case Process.info(self(), :registered_name) do + {_, []} -> self() + {_, name} -> name + end + + # We do this to trick Dialyzer to not complain about non-local returns. + case :erlang.phash2(1, 1) do + 0 -> + raise "attempted to call GenServer #{inspect(proc)} but no handle_call/3 clause was provided" + + 1 -> + {:stop, {:bad_call, msg}, state} + end end @doc false - def handle_info(_msg, state) do + def handle_info(msg, state) do + proc = + case Process.info(self(), :registered_name) do + {_, []} -> self() + {_, name} -> name + end + + :logger.error( + %{ + label: {GenServer, :no_handle_info}, + report: %{ + module: __MODULE__, + message: msg, + name: proc + } + }, + %{ + domain: [:otp, :elixir], + error_logger: %{tag: :error_msg}, + report_cb: &GenServer.format_report/1 + } + ) + {:noreply, state} end @doc false def handle_cast(msg, state) do - {:stop, {:bad_cast, msg}, state} + proc = + case Process.info(self(), :registered_name) do + {_, []} -> self() + {_, name} -> name + end + + # We do this to trick Dialyzer to not complain about non-local returns. + case :erlang.phash2(1, 1) do + 0 -> + raise "attempted to cast GenServer #{inspect(proc)} but no handle_cast/2 clause was provided" + + 1 -> + {:stop, {:bad_cast, msg}, state} + end end @doc false @@ -256,8 +989,36 @@ defmodule GenServer do {:ok, state} end - defoverridable [init: 1, handle_call: 3, handle_info: 2, - handle_cast: 2, terminate: 2, code_change: 3] + defoverridable code_change: 3, terminate: 2, handle_info: 2, handle_cast: 2, handle_call: 3 + end + end + + defmacro __before_compile__(env) do + if not Module.defines?(env.module, {:init, 1}) do + message = """ + function init/1 required by behaviour GenServer is not implemented \ + (in module #{inspect(env.module)}). + + We will inject a default implementation for now: + + def init(init_arg) do + {:ok, init_arg} + end + + You can copy the implementation above or define your own that converts \ + the arguments given to GenServer.start_link/3 to the server state. + """ + + IO.warn(message, env) + + quote do + @doc false + def init(init_arg) do + {:ok, init_arg} + end + + defoverridable init: 1 + end end end @@ -266,43 +1027,49 @@ defmodule GenServer do This is often used to start the `GenServer` as part of a supervision tree. - Once the server is started, it calls the `init/1` function in the given `module` - passing the given `args` to initialize it. To ensure a synchronized start-up - procedure, this function does not return until `init/1` has returned. + Once the server is started, the `c:init/1` function of the given `module` is + called with `init_arg` as its argument to initialize the server. To ensure a + synchronized start-up procedure, this function does not return until `c:init/1` + has returned. Note that a `GenServer` started with `start_link/3` is linked to the - parent process and will exit in case of crashes. The GenServer will also - exit due to the `:normal` reasons in case it is configured to trap exits - in the `init/1` callback. + parent process and will exit in case of crashes from the parent. The GenServer + will also exit due to the `:normal` reasons in case it is configured to trap + exits in the `c:init/1` callback. ## Options - The `:name` option is used for name registration as described in the module - documentation. If the option `:timeout` option is present, the server is - allowed to spend the given milliseconds initializing or it will be - terminated and the start function will return `{:error, :timeout}`. + * `:name` - used for name registration as described in the "Name + registration" section in the documentation for `GenServer` + + * `:timeout` - if present, the server is allowed to spend the given number of + milliseconds initializing or it will be terminated and the start function + will return `{:error, :timeout}` - If the `:debug` option is present, the corresponding function in the - [`:sys` module](http://www.erlang.org/doc/man/sys.html) will be invoked. + * `:debug` - if present, the corresponding function in the [`:sys` module](`:sys`) is invoked - If the `:spawn_opt` option is present, its value will be passed as options - to the underlying process as in `Process.spawn/4`. + * `:spawn_opt` - if present, its value is passed as options to the + underlying process as in `Process.spawn/4` + + * `:hibernate_after` - if present, the GenServer process awaits any message for + the given number of milliseconds and if no message is received, the process goes + into hibernation automatically (by calling `:proc_lib.hibernate/3`). ## Return values - If the server is successfully created and initialized, the function returns - `{:ok, pid}`, where pid is the pid of the server. If there already exists a - process with the specified server name, the function returns - `{:error, {:already_started, pid}}` with the pid of that process. + If the server is successfully created and initialized, this function returns + `{:ok, pid}`, where `pid` is the PID of the server. If a process with the + specified server name already exists, this function returns + `{:error, {:already_started, pid}}` with the PID of that process. - If the `init/1` callback fails with `reason`, the function returns + If the `c:init/1` callback fails with `reason`, this function returns `{:error, reason}`. Otherwise, if it returns `{:stop, reason}` - or `:ignore`, the process is terminated and the function returns + or `:ignore`, the process is terminated and this function returns `{:error, reason}` or `:ignore`, respectively. """ - @spec start_link(module, any, options) :: on_start - def start_link(module, args, options \\ []) when is_atom(module) and is_list(options) do - do_start(:link, module, args, options) + @spec start_link(module, term, options) :: on_start + def start_link(module, init_arg, options \\ []) when is_atom(module) and is_list(options) do + do_start(:link, module, init_arg, options) end @doc """ @@ -310,19 +1077,66 @@ defmodule GenServer do See `start_link/3` for more information. """ - @spec start(module, any, options) :: on_start - def start(module, args, options \\ []) when is_atom(module) and is_list(options) do - do_start(:nolink, module, args, options) + @spec start(module, term, options) :: on_start + def start(module, init_arg, options \\ []) when is_atom(module) and is_list(options) do + do_start(:nolink, module, init_arg, options) end - defp do_start(link, module, args, options) do + defp do_start(link, module, init_arg, options) do case Keyword.pop(options, :name) do {nil, opts} -> - :gen.start(:gen_server, link, module, args, opts) + :gen.start(:gen_server, link, module, init_arg, opts) + {atom, opts} when is_atom(atom) -> - :gen.start(:gen_server, link, {:local, atom}, module, args, opts) - {other, opts} when is_tuple(other) -> - :gen.start(:gen_server, link, other, module, args, opts) + :gen.start(:gen_server, link, {:local, atom}, module, init_arg, opts) + + {{:global, _term} = tuple, opts} -> + :gen.start(:gen_server, link, tuple, module, init_arg, opts) + + {{:via, via_module, _term} = tuple, opts} when is_atom(via_module) -> + :gen.start(:gen_server, link, tuple, module, init_arg, opts) + + {other, _} -> + raise ArgumentError, """ + expected :name option to be one of the following: + + * nil + * atom + * {:global, term} + * {:via, module, term} + + Got: #{inspect(other)} + """ + end + end + + @doc """ + Synchronously stops the server with the given `reason`. + + The `c:terminate/2` callback of the given `server` will be invoked before + exiting. This function returns `:ok` if the server terminates with the + given reason; if it terminates with another reason, the call exits. + + This function keeps OTP semantics regarding error reporting. + If the reason is any other than `:normal`, `:shutdown` or + `{:shutdown, _}`, an error report is logged. + """ + @spec stop(server, reason :: term, timeout) :: :ok + def stop(server, reason \\ :normal, timeout \\ :infinity) do + case whereis(server) do + nil -> + exit({:noproc, {__MODULE__, :stop, [server, reason, timeout]}}) + + pid when pid == self() -> + exit({:calling_self, {__MODULE__, :stop, [server, reason, timeout]}}) + + pid -> + try do + :proc_lib.stop(pid, reason, timeout) + catch + :exit, err -> + exit({err, {__MODULE__, :stop, [server, reason, timeout]}}) + end end end @@ -330,101 +1144,227 @@ defmodule GenServer do Makes a synchronous call to the `server` and waits for its reply. The client sends the given `request` to the server and waits until a reply - arrives or a timeout occurs. `handle_call/3` will be called on the server + arrives or a timeout occurs. `c:handle_call/3` will be called on the server to handle the request. - The server can be any of the values described in the `Name Registration` - section of the module documentation. + `server` can be a PID or any of the other values described in the + "Name registration" section of the documentation for this module. ## Timeouts - The `timeout` is an integer greater than zero which specifies how many + `timeout` is an integer greater than zero which specifies how many milliseconds to wait for a reply, or the atom `:infinity` to wait - indefinitely. The default value is 5000. If no reply is received within - the specified time, the function call fails. If the caller catches the - failure and continues running, and the server is just late with the reply, - it may arrive at any time later into the caller's message queue. The caller - must in this case be prepared for this and discard any such garbage messages - that are two element tuples with a reference as the first element. + indefinitely. The default value is `5000`. If no reply is received within + the specified time, the function call fails and the caller exits. If the + caller catches the failure and continues running, and the server is just late + with the reply, it may arrive at any time later into the caller's message + queue. The caller must in this case be prepared for this and discard any such + garbage messages that are two-element tuples with a reference as the first + element. """ @spec call(server, term, timeout) :: term - def call(server, request, timeout \\ 5000) do - :gen_server.call(server, request, timeout) + def call(server, request, timeout \\ 5000) + when (is_integer(timeout) and timeout >= 0) or timeout == :infinity do + case whereis(server) do + nil -> + exit({:noproc, {__MODULE__, :call, [server, request, timeout]}}) + + pid -> + try do + :gen.call(pid, :"$gen_call", request, timeout) + catch + :exit, reason -> + exit({reason, {__MODULE__, :call, [server, request, timeout]}}) + else + {:ok, res} -> res + end + end end @doc """ - Sends an asynchronous request to the `server`. + Casts a request to the `server` without waiting for a response. - This function returns `:ok` immediately, regardless of whether the - destination node or server does exists. `handle_cast/2` will be called on the - server to handle the request. + This function always returns `:ok` regardless of whether + the destination `server` (or node) exists. Therefore it + is unknown whether the destination `server` successfully + handled the request. + + `server` can be any of the values described in the "Name registration" + section of the documentation for this module. """ @spec cast(server, term) :: :ok - defdelegate cast(server, request), to: :gen_server + def cast(server, request) + + def cast({:global, name}, request) do + try do + :global.send(name, cast_msg(request)) + :ok + catch + _, _ -> :ok + end + end + + def cast({:via, mod, name}, request) do + try do + mod.send(name, cast_msg(request)) + :ok + catch + _, _ -> :ok + end + end + + def cast({name, node}, request) when is_atom(name) and is_atom(node), + do: do_send({name, node}, cast_msg(request)) + + def cast(dest, request) when is_atom(dest) or is_pid(dest), do: do_send(dest, cast_msg(request)) @doc """ Casts all servers locally registered as `name` at the specified nodes. - The function returns immediately and ignores nodes that do not exist, or where the + This function returns immediately and ignores nodes that do not exist, or where the server name does not exist. See `multi_call/4` for more information. """ @spec abcast([node], name :: atom, term) :: :abcast - def abcast(nodes \\ nodes(), name, request) do - :gen_server.abcast(nodes, name, request) + def abcast(nodes \\ [node() | Node.list()], name, request) + when is_list(nodes) and is_atom(name) do + msg = cast_msg(request) + _ = for node <- nodes, do: do_send({name, node}, msg) + :abcast + end + + defp cast_msg(req) do + {:"$gen_cast", req} + end + + defp do_send(dest, msg) do + try do + send(dest, msg) + :ok + catch + _, _ -> :ok + end end @doc """ Calls all servers locally registered as `name` at the specified `nodes`. - The `request` is first sent to every node and then we wait for the - replies. This function returns a tuple containing the node and its reply - as first element and all bad nodes as second element. The bad nodes is a - list of nodes that either did not exist, or where a server with the given - `name` did not exist or did not reply. + First, the `request` is sent to every node in `nodes`; then, the caller waits + for the replies. This function returns a two-element tuple `{replies, + bad_nodes}` where: + + * `replies` - is a list of `{node, reply}` tuples where `node` is the node + that replied and `reply` is its reply + * `bad_nodes` - is a list of nodes that either did not exist or where a + server with the given `name` did not exist or did not reply + + `nodes` is a list of node names to which the request is sent. The default + value is the list of all known nodes (including this node). - Nodes is a list of node names to which the request is sent. The default - value is the list of all known nodes. + ## Examples + + Assuming the `Stack` GenServer mentioned in the docs for the `GenServer` + module is registered as `Stack` in the `:"foo@my-machine"` and + `:"bar@my-machine"` nodes: + + GenServer.multi_call(Stack, :pop) + #=> {[{:"foo@my-machine", :hello}, {:"bar@my-machine", :world}], []} - To avoid that late answers (after the timeout) pollute the caller's message - queue, a middleman process is used to do the actual calls. Late answers will - then be discarded when they arrive to a terminated process. """ @spec multi_call([node], name :: atom, term, timeout) :: - {replies :: [{node, term}], bad_nodes :: [node]} - def multi_call(nodes \\ nodes(), name, request, timeout \\ :infinity) do + {replies :: [{node, term}], bad_nodes :: [node]} + def multi_call(nodes \\ [node() | Node.list()], name, request, timeout \\ :infinity) do :gen_server.multi_call(nodes, name, request, timeout) end @doc """ Replies to a client. - This function can be used by a server to explicitly send a reply to a - client that called `call/3` or `multi_call/4`. When the reply cannot be - defined in the return value of `handle_call/3`. + This function can be used to explicitly send a reply to a client that called + `call/3` or `multi_call/4` when the reply cannot be specified in the return + value of `c:handle_call/3`. + + `client` must be the `from` argument (the second argument) accepted by + `c:handle_call/3` callbacks. `reply` is an arbitrary term which will be given + back to the client as the return value of the call. - The `client` must be the `from` argument (the second argument) received - in `handle_call/3` callbacks. Reply is an arbitrary term which will be - given back to the client as the return value of the call. + Note that `reply/2` can be called from any process, not just the GenServer + that originally received the call (as long as that GenServer communicated the + `from` argument somehow). This function always returns `:ok`. + + ## Examples + + def handle_call(:reply_in_one_second, from, state) do + Process.send_after(self(), {:reply, from}, 1_000) + {:noreply, state} + end + + def handle_info({:reply, from}, state) do + GenServer.reply(from, :one_second_has_passed) + {:noreply, state} + end + """ - @spec reply({pid, reference}, term) :: :ok - def reply(client, reply) + @spec reply(from, term) :: :ok + def reply(client, reply) do + :gen.reply(client, reply) + end - def reply({to, tag}, reply) do - try do - send(to, {tag, reply}) - :ok - catch - _, _ -> :ok + @doc """ + Returns the `pid` or `{name, node}` of a GenServer process, `nil` otherwise. + + To be precise, `nil` is returned whenever a `pid` or `{name, node}` cannot + be returned. Note there is no guarantee the returned `pid` or `{name, node}` + is alive, as a process could terminate immediately after it is looked up. + + ## Examples + + For example, to lookup a server process, monitor it and send a cast to it: + + process = GenServer.whereis(server) + monitor = Process.monitor(process) + GenServer.cast(process, :hello) + + """ + @spec whereis(server) :: pid | {atom, node} | nil + def whereis(server) + + def whereis(pid) when is_pid(pid), do: pid + + def whereis(name) when is_atom(name) do + Process.whereis(name) + end + + def whereis({:global, name}) do + case :global.whereis_name(name) do + pid when is_pid(pid) -> pid + :undefined -> nil + end + end + + def whereis({:via, mod, name}) do + case apply(mod, :whereis_name, [name]) do + pid when is_pid(pid) -> pid + :undefined -> nil end end - @compile {:inline, [nodes: 0]} + def whereis({name, local}) when is_atom(name) and local == node() do + Process.whereis(name) + end + + def whereis({name, node} = server) when is_atom(name) and is_atom(node) do + server + end - defp nodes do - [node()|:erlang.nodes()] + @doc false + def format_report(%{ + label: {GenServer, :no_handle_info}, + report: %{module: mod, message: msg, name: proc} + }) do + {~c"~p ~p received unexpected message in handle_info/2: ~p~n", [mod, proc, msg]} end end diff --git a/lib/elixir/lib/hash_dict.ex b/lib/elixir/lib/hash_dict.ex index edcb8e86329..d3d56d7601e 100644 --- a/lib/elixir/lib/hash_dict.ex +++ b/lib/elixir/lib/hash_dict.ex @@ -1,19 +1,16 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + defmodule HashDict do @moduledoc """ - A key-value store. - - The `HashDict` is represented internally as a struct, therefore - `%HashDict{}` can be used whenever there is a need to match - on any `HashDict`. Note though the struct fields are private and - must not be accessed directly. Instead, use the functions on this - or in the `Dict` module. - - Implementation-wise, `HashDict` is implemented using tries, which - grows in space as the number of keys grows, working well with both - small and large set of keys. For more information about the - functions and their APIs, please consult the `Dict` module. + Tuple-based HashDict implementation. + + This module is deprecated. Use the `Map` module instead. """ + @moduledoc deprecated: "Use Map instead" + use Dict @node_bitmap 0b111 @@ -21,7 +18,7 @@ defmodule HashDict do @node_size 8 @node_template :erlang.make_tuple(@node_size, []) - @opaque t :: map + @opaque t :: %__MODULE__{size: non_neg_integer, root: term} @doc false defstruct size: 0, root: @node_template @@ -29,58 +26,70 @@ defmodule HashDict do @compile :inline_list_funcs @compile {:inline, key_hash: 1, key_mask: 1, key_shift: 1} + message = "Use maps and the Map module instead" + @doc """ Creates a new empty dict. """ - @spec new :: Dict.t + @spec new :: Dict.t() + @deprecated message def new do %HashDict{} end + @deprecated message def put(%HashDict{root: root, size: size}, key, value) do {root, counter} = do_put(root, key, value, key_hash(key)) %HashDict{root: root, size: size + counter} end + @deprecated message def update!(%HashDict{root: root, size: size} = dict, key, fun) when is_function(fun, 1) do - {root, counter} = do_update(root, key, fn -> raise KeyError, key: key, term: dict end, - fun, key_hash(key)) + {root, counter} = + do_update(root, key, fn -> raise KeyError, key: key, term: dict end, fun, key_hash(key)) + %HashDict{root: root, size: size + counter} end - def update(%HashDict{root: root, size: size}, key, initial, fun) when is_function(fun, 1) do - {root, counter} = do_update(root, key, fn -> initial end, fun, key_hash(key)) + @deprecated message + def update(%HashDict{root: root, size: size}, key, default, fun) when is_function(fun, 1) do + {root, counter} = do_update(root, key, fn -> default end, fun, key_hash(key)) %HashDict{root: root, size: size + counter} end + @deprecated message def fetch(%HashDict{root: root}, key) do do_fetch(root, key, key_hash(key)) end + @deprecated message def delete(dict, key) do case dict_delete(dict, key) do {dict, _value} -> dict - :error -> dict + :error -> dict end end + @deprecated message def pop(dict, key, default \\ nil) do case dict_delete(dict, key) do {dict, value} -> {value, dict} - :error -> {default, dict} + :error -> {default, dict} end end + @deprecated message def size(%HashDict{size: size}) do size end @doc false + @deprecated message def reduce(%HashDict{root: root}, acc, fun) do do_reduce(root, acc, fun, @node_size, fn {:suspend, acc} -> {:suspended, acc, &{:done, elem(&1, 1)}} - {:halt, acc} -> {:halted, acc} - {:cont, acc} -> {:done, acc} + {:halt, acc} -> {:halted, acc} + {:cont, acc} -> {:done, acc} end) end @@ -90,7 +99,7 @@ defmodule HashDict do def dict_delete(%HashDict{root: root, size: size}, key) do case do_delete(root, key, key_hash(key)) do {root, value} -> {%HashDict{root: root, size: size - 1}, value} - :error -> :error + :error -> :error end end @@ -98,86 +107,105 @@ defmodule HashDict do defp do_fetch(node, key, hash) do index = key_mask(hash) + case elem(node, index) do - [^key|v] -> {:ok, v} + [^key | v] -> {:ok, v} {^key, v, _} -> {:ok, v} - {_, _, n} -> do_fetch(n, key, key_shift(hash)) - _ -> :error + {_, _, n} -> do_fetch(n, key, key_shift(hash)) + _ -> :error end end defp do_put(node, key, value, hash) do index = key_mask(hash) + case elem(node, index) do [] -> - {put_elem(node, index, [key|value]), 1} - [^key|_] -> - {put_elem(node, index, [key|value]), 0} - [k|v] -> - n = put_elem(@node_template, key_mask(key_shift(hash)), [key|value]) + {put_elem(node, index, [key | value]), 1} + + [^key | _] -> + {put_elem(node, index, [key | value]), 0} + + [k | v] -> + n = put_elem(@node_template, key_mask(key_shift(hash)), [key | value]) {put_elem(node, index, {k, v, n}), 1} + {^key, _, n} -> {put_elem(node, index, {key, value, n}), 0} + {k, v, n} -> {n, counter} = do_put(n, key, value, key_shift(hash)) {put_elem(node, index, {k, v, n}), counter} end end - defp do_update(node, key, initial, fun, hash) do + defp do_update(node, key, default, fun, hash) do index = key_mask(hash) + case elem(node, index) do [] -> - {put_elem(node, index, [key|initial.()]), 1} - [^key|value] -> - {put_elem(node, index, [key|fun.(value)]), 0} - [k|v] -> - n = put_elem(@node_template, key_mask(key_shift(hash)), [key|initial.()]) + {put_elem(node, index, [key | default.()]), 1} + + [^key | value] -> + {put_elem(node, index, [key | fun.(value)]), 0} + + [k | v] -> + n = put_elem(@node_template, key_mask(key_shift(hash)), [key | default.()]) {put_elem(node, index, {k, v, n}), 1} + {^key, value, n} -> {put_elem(node, index, {key, fun.(value), n}), 0} + {k, v, n} -> - {n, counter} = do_update(n, key, initial, fun, key_shift(hash)) + {n, counter} = do_update(n, key, default, fun, key_shift(hash)) {put_elem(node, index, {k, v, n}), counter} end end defp do_delete(node, key, hash) do index = key_mask(hash) + case elem(node, index) do [] -> :error - [^key|value] -> + + [^key | value] -> {put_elem(node, index, []), value} - [_|_] -> + + [_ | _] -> :error + {^key, value, n} -> {put_elem(node, index, do_compact_node(n)), value} + {k, v, n} -> case do_delete(n, key, key_shift(hash)) do {@node_template, value} -> - {put_elem(node, index, [k|v]), value} + {put_elem(node, index, [k | v]), value} + {n, value} -> {put_elem(node, index, {k, v, n}), value} + :error -> :error end end end - Enum.each 0..(@node_size - 1), fn index -> + Enum.each(0..(@node_size - 1), fn index -> defp do_compact_node(node) when elem(node, unquote(index)) != [] do case elem(node, unquote(index)) do - [k|v] -> + [k | v] -> case put_elem(node, unquote(index), []) do - @node_template -> [k|v] + @node_template -> [k | v] n -> {k, v, n} end + {k, v, n} -> {k, v, put_elem(node, unquote(index), do_compact_node(n))} end end - end + end) ## Dict reduce @@ -193,8 +221,8 @@ defmodule HashDict do next.(acc) end - defp do_reduce_each([k|v], {:cont, acc}, fun, next) do - next.(fun.({k,v}, acc)) + defp do_reduce_each([k | v], {:cont, acc}, fun, next) do + next.(fun.({k, v}, acc)) end defp do_reduce_each({k, v, n}, {:cont, acc}, fun, next) do @@ -202,7 +230,12 @@ defmodule HashDict do end defp do_reduce(node, acc, fun, count, next) when count > 0 do - do_reduce_each(:erlang.element(count, node), acc, fun, &do_reduce(node, &1, fun, count - 1, next)) + do_reduce_each( + :erlang.element(count, node), + acc, + fun, + &do_reduce(node, &1, fun, count - 1, next) + ) end defp do_reduce(_node, acc, _fun, 0, next) do @@ -227,41 +260,62 @@ defmodule HashDict do end defimpl Enumerable, for: HashDict do - def reduce(dict, acc, fun), do: HashDict.reduce(dict, acc, fun) - def member?(dict, {k, v}), do: {:ok, match?({:ok, ^v}, HashDict.fetch(dict, k))} - def member?(_dict, _), do: {:ok, false} - def count(dict), do: {:ok, HashDict.size(dict)} -end + @moduledoc false + @moduledoc deprecated: "Use Map instead" + + def reduce(dict, acc, fun) do + # Avoid warnings about HashDict being deprecated. + module = String.to_atom("HashDict") + module.reduce(dict, acc, fun) + end + + def member?(dict, {key, value}) do + # Avoid warnings about HashDict being deprecated. + module = String.to_atom("HashDict") + {:ok, match?({:ok, ^value}, module.fetch(dict, key))} + end + + def member?(_dict, _) do + {:ok, false} + end -defimpl Access, for: HashDict do - def get(dict, key) do - HashDict.get(dict, key, nil) + def count(dict) do + # Avoid warnings about HashDict being deprecated. + module = String.to_atom("HashDict") + {:ok, module.size(dict)} end - def get_and_update(dict, key, fun) do - {get, update} = fun.(HashDict.get(dict, key, nil)) - {get, HashDict.put(dict, key, update)} + def slice(_dict) do + {:error, __MODULE__} end end defimpl Collectable, for: HashDict do - def empty(_dict) do - HashDict.new - end + @moduledoc false + @moduledoc deprecated: "Use Map instead" def into(original) do - {original, fn - dict, {:cont, {k, v}} -> Dict.put(dict, k, v) + # Avoid warnings about HashDict being deprecated. + module = String.to_atom("HashDict") + + collector_fun = fn + dict, {:cont, {key, value}} -> module.put(dict, key, value) dict, :done -> dict _, :halt -> :ok - end} + end + + {original, collector_fun} end end defimpl Inspect, for: HashDict do + @moduledoc false + @moduledoc deprecated: "Use Map instead" import Inspect.Algebra def inspect(dict, opts) do - concat ["#HashDict<", Inspect.List.inspect(HashDict.to_list(dict), opts), ">"] + # Avoid warnings about HashDict being deprecated. + module = String.to_atom("HashDict") + concat(["#HashDict<", Inspect.List.inspect(module.to_list(dict), opts), ">"]) end end diff --git a/lib/elixir/lib/hash_set.ex b/lib/elixir/lib/hash_set.ex index acbc21ae943..aedd38dd55c 100644 --- a/lib/elixir/lib/hash_set.ex +++ b/lib/elixir/lib/hash_set.ex @@ -1,27 +1,24 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + defmodule HashSet do @moduledoc """ - A set store. - - The `HashSet` is represented internally as a struct, therefore - `%HashSet{}` can be used whenever there is a need to match - on any `HashSet`. Note though the struct fields are private and - must not be accessed directly. Instead, use the functions on this - or in the `Set` module. - - The `HashSet` is implemented using tries, which grows in - space as the number of keys grows, working well with both - small and large set of keys. For more information about the - functions and their APIs, please consult the `Set` module. + Tuple-based HashSet implementation. + + This module is deprecated. Use the `MapSet` module instead. """ - @behaviour Set + @moduledoc deprecated: "Use MapSet instead" @node_bitmap 0b111 @node_shift 3 @node_size 8 @node_template :erlang.make_tuple(@node_size, []) - @opaque t :: map + message = "Use the MapSet module instead" + + @opaque t :: %__MODULE__{size: non_neg_integer, root: term} @doc false defstruct size: 0, root: @node_template @@ -29,74 +26,85 @@ defmodule HashSet do @compile :inline_list_funcs @compile {:inline, key_hash: 1, key_mask: 1, key_shift: 1} - @doc """ - Creates a new empty set. - """ - @spec new :: Set.t + @deprecated message + @spec new :: Set.t() def new do %HashSet{} end + @deprecated message def union(%HashSet{size: size1} = set1, %HashSet{size: size2} = set2) when size1 <= size2 do - set_fold set1, set2, fn v, acc -> put(acc, v) end + set_fold(set1, set2, fn v, acc -> put(acc, v) end) end + @deprecated message def union(%HashSet{} = set1, %HashSet{} = set2) do - set_fold set2, set1, fn v, acc -> put(acc, v) end + set_fold(set2, set1, fn v, acc -> put(acc, v) end) end + @deprecated message def intersection(%HashSet{} = set1, %HashSet{} = set2) do - set_fold set1, %HashSet{}, fn v, acc -> + set_fold(set1, %HashSet{}, fn v, acc -> if member?(set2, v), do: put(acc, v), else: acc - end + end) end + @deprecated message def difference(%HashSet{} = set1, %HashSet{} = set2) do - set_fold set2, set1, fn v, acc -> delete(acc, v) end + set_fold(set2, set1, fn v, acc -> delete(acc, v) end) end + @deprecated message def to_list(set) do - set_fold(set, [], &[&1|&2]) |> :lists.reverse + set_fold(set, [], &[&1 | &2]) |> :lists.reverse() end + @deprecated message def equal?(%HashSet{size: size1} = set1, %HashSet{size: size2} = set2) do case size1 do ^size2 -> subset?(set1, set2) - _ -> false + _ -> false end end + @deprecated message def subset?(%HashSet{} = set1, %HashSet{} = set2) do reduce(set1, {:cont, true}, fn member, acc -> case member?(set2, member) do true -> {:cont, acc} - _ -> {:halt, false} + _ -> {:halt, false} end - end) |> elem(1) + end) + |> elem(1) end + @deprecated message def disjoint?(%HashSet{} = set1, %HashSet{} = set2) do reduce(set2, {:cont, true}, fn member, acc -> case member?(set1, member) do false -> {:cont, acc} - _ -> {:halt, false} + _ -> {:halt, false} end - end) |> elem(1) + end) + |> elem(1) end + @deprecated message def member?(%HashSet{root: root}, term) do do_member?(root, term, key_hash(term)) end + @deprecated message def put(%HashSet{root: root, size: size}, term) do {root, counter} = do_put(root, term, key_hash(term)) %HashSet{root: root, size: size + counter} end + @deprecated message def delete(%HashSet{root: root, size: size} = set, term) do case do_delete(root, term, key_hash(term)) do {:ok, root} -> %HashSet{root: root, size: size - 1} - :error -> set + :error -> set end end @@ -104,11 +112,12 @@ defmodule HashSet do def reduce(%HashSet{root: root}, acc, fun) do do_reduce(root, acc, fun, @node_size, fn {:suspend, acc} -> {:suspended, acc, &{:done, elem(&1, 1)}} - {:halt, acc} -> {:halted, acc} - {:cont, acc} -> {:done, acc} + {:halt, acc} -> {:halted, acc} + {:cont, acc} -> {:done, acc} end) end + @deprecated message def size(%HashSet{size: size}) do size end @@ -123,72 +132,85 @@ defmodule HashSet do defp do_member?(node, term, hash) do index = key_mask(hash) + case elem(node, index) do - [] -> false - [^term|_] -> true - [_] -> false - [_|n] -> do_member?(n, term, key_shift(hash)) + [] -> false + [^term | _] -> true + [_] -> false + [_ | n] -> do_member?(n, term, key_shift(hash)) end end defp do_put(node, term, hash) do index = key_mask(hash) + case elem(node, index) do [] -> {put_elem(node, index, [term]), 1} - [^term|_] -> + + [^term | _] -> {node, 0} + [t] -> n = put_elem(@node_template, key_mask(key_shift(hash)), [term]) - {put_elem(node, index, [t|n]), 1} - [t|n] -> + {put_elem(node, index, [t | n]), 1} + + [t | n] -> {n, counter} = do_put(n, term, key_shift(hash)) - {put_elem(node, index, [t|n]), counter} + {put_elem(node, index, [t | n]), counter} end end defp do_delete(node, term, hash) do index = key_mask(hash) + case elem(node, index) do [] -> :error + [^term] -> {:ok, put_elem(node, index, [])} + [_] -> :error - [^term|n] -> + + [^term | n] -> {:ok, put_elem(node, index, do_compact_node(n))} - [t|n] -> + + [t | n] -> case do_delete(n, term, key_shift(hash)) do {:ok, @node_template} -> {:ok, put_elem(node, index, [t])} + {:ok, n} -> - {:ok, put_elem(node, index, [t|n])} + {:ok, put_elem(node, index, [t | n])} + :error -> :error end end end - Enum.each 0..(@node_size - 1), fn index -> + Enum.each(0..(@node_size - 1), fn index -> defp do_compact_node(node) when elem(node, unquote(index)) != [] do case elem(node, unquote(index)) do [t] -> case put_elem(node, unquote(index), []) do @node_template -> [t] - n -> [t|n] + n -> [t | n] end - [t|n] -> - [t|put_elem(node, unquote(index), do_compact_node(n))] + + [t | n] -> + [t | put_elem(node, unquote(index), do_compact_node(n))] end end - end + end) ## Set fold - defp do_fold_each([], acc, _fun), do: acc - defp do_fold_each([t], acc, fun), do: fun.(t, acc) - defp do_fold_each([t|n], acc, fun), do: do_fold(n, fun.(t, acc), fun, @node_size) + defp do_fold_each([], acc, _fun), do: acc + defp do_fold_each([t], acc, fun), do: fun.(t, acc) + defp do_fold_each([t | n], acc, fun), do: do_fold(n, fun.(t, acc), fun, @node_size) defp do_fold(node, acc, fun, count) when count > 0 do acc = do_fold_each(:erlang.element(count, node), acc, fun) @@ -217,12 +239,17 @@ defmodule HashSet do next.(fun.(t, acc)) end - defp do_reduce_each([t|n], {:cont, acc}, fun, next) do + defp do_reduce_each([t | n], {:cont, acc}, fun, next) do do_reduce(n, fun.(t, acc), fun, @node_size, next) end defp do_reduce(node, acc, fun, count, next) when count > 0 do - do_reduce_each(:erlang.element(count, node), acc, fun, &do_reduce(node, &1, fun, count - 1, next)) + do_reduce_each( + :erlang.element(count, node), + acc, + fun, + &do_reduce(node, &1, fun, count - 1, next) + ) end defp do_reduce(_node, acc, _fun, 0, next) do @@ -247,29 +274,58 @@ defmodule HashSet do end defimpl Enumerable, for: HashSet do - def reduce(set, acc, fun), do: HashSet.reduce(set, acc, fun) - def member?(set, v), do: {:ok, HashSet.member?(set, v)} - def count(set), do: {:ok, HashSet.size(set)} + @moduledoc false + @moduledoc deprecated: "Use MapSet instead" + + def reduce(set, acc, fun) do + # Avoid warnings about HashSet being deprecated. + module = String.to_atom("HashSet") + module.reduce(set, acc, fun) + end + + def member?(set, term) do + # Avoid warnings about HashSet being deprecated. + module = String.to_atom("HashSet") + {:ok, module.member?(set, term)} + end + + def count(set) do + # Avoid warnings about HashSet being deprecated. + module = String.to_atom("HashSet") + {:ok, module.size(set)} + end + + def slice(_set) do + {:error, __MODULE__} + end end defimpl Collectable, for: HashSet do - def empty(_dict) do - HashSet.new - end + @moduledoc false + @moduledoc deprecated: "Use MapSet instead" def into(original) do - {original, fn - set, {:cont, x} -> HashSet.put(set, x) + # Avoid warnings about HashSet being deprecated. + module = String.to_atom("HashSet") + + collector_fun = fn + set, {:cont, term} -> module.put(set, term) set, :done -> set _, :halt -> :ok - end} + end + + {original, collector_fun} end end defimpl Inspect, for: HashSet do + @moduledoc false + @moduledoc deprecated: "Use MapSet instead" import Inspect.Algebra def inspect(set, opts) do - concat ["#HashSet<", Inspect.List.inspect(HashSet.to_list(set), opts), ">"] + # Avoid warnings about HashSet being deprecated. + module = String.to_atom("HashSet") + concat(["#HashSet<", Inspect.List.inspect(module.to_list(set), opts), ">"]) end end diff --git a/lib/elixir/lib/inspect.ex b/lib/elixir/lib/inspect.ex index 26f1dbd5977..169ddfac73a 100644 --- a/lib/elixir/lib/inspect.ex +++ b/lib/elixir/lib/inspect.ex @@ -1,471 +1,624 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + import Kernel, except: [inspect: 1] import Inspect.Algebra +alias Code.Identifier + defprotocol Inspect do @moduledoc """ - The `Inspect` protocol is responsible for converting any Elixir - data structure into an algebra document. This document is then - formatted, either in pretty printing format or a regular one. + The `Inspect` protocol converts an Elixir data structure into an + algebra document. + + This is typically done when you want to customize how your own + structs are inspected in logs and the terminal. + + This documentation refers to implementing the `Inspect` protocol + for your own data structures. To learn more about using inspect, + see `Kernel.inspect/2` and `IO.inspect/2`. + + ## Inspect representation + + There are typically three choices of inspect representation. In order + to understand them, let's imagine we have the following `User` struct: + + defmodule User do + defstruct [:id, :name, :address] + end + + Our choices are: + + 1. Print the struct using Elixir's struct syntax, for example: + `%User{address: "Earth", id: 13, name: "Jane"}`. This is the + default representation and best choice if all struct fields + are public. + + 2. Print using the `#User<...>` notation, for example: `#User`. + This notation does not emit valid Elixir code and is typically + used when the struct has private fields (for example, you may want + to hide the field `:address` to redact person identifiable information). + + 3. Print the struct using the expression syntax, for example: + `User.new(13, "Jane", "Earth")`. This assumes there is a `User.new/3` + function. This option is mostly used as an alternative to option 2 + for representing custom data structures, such as `MapSet`, `Date.Range`, + and others. + + You can implement the Inspect protocol for your own structs while + adhering to the conventions above. Option 1 is the default representation + and you can quickly achieve option 2 by deriving the `Inspect` protocol. + For option 3, you need your custom implementation. + + ## Deriving + + The `Inspect` protocol can be derived to customize the order of fields + (the default is alphabetical) and hide certain fields from structs, + so they don't show up in logs, inspects and similar. The latter is + especially useful for fields containing private information. + + The supported options are: + + * `:only` - only include the given fields when inspecting. + + * `:except` - remove the given fields when inspecting. + + * `:optional` - (since v1.14.0) a list of fields that should not be + included when they match their default value. This can be used to + simplify the struct representation at the cost of hiding + information. Since v1.19.0, the `:all` atom can be passed to + mark all fields as optional. + + Whenever `:only` or `:except` are used to restrict fields, + the struct will be printed using the `#User<...>` notation, + as the struct can no longer be copy and pasted as valid Elixir + code. Let's see an example: + + defmodule User do + @derive {Inspect, only: [:id, :name]} + defstruct [:id, :name, :address] + end + + inspect(%User{id: 1, name: "Jane", address: "Earth"}) + #=> #User + + If you use only the `:optional` option, the struct will still be + printed as a valid struct. - The `inspect/2` function receives the entity to be inspected - followed by the inspecting options, represented by the struct - `Inspect.Opts`. + defmodule Point do + @derive {Inspect, optional: [:z]} + defstruct [x: 0, y: 0, z: 0] + end + + inspect(%Point{x: 1}) + %Point{x: 1, y: 0} - Inspection is done using the functions available in `Inspect.Algebra`. + ## Custom implementation - ## Examples + You can also define your custom protocol implementation by + defining the `inspect/2` function. The function receives the + entity to be inspected followed by the inspecting options, + represented by the struct `Inspect.Opts` and it must return + an algebra document alongside the updated options (or, optionally, + just the algebra document). Building of the algebra document + is done with `Inspect.Algebra`. Many times, inspecting a structure can be implemented in function - of existing entities. For example, here is `HashSet`'s `inspect` + of existing entities. For example, here is `MapSet`'s `inspect/2` implementation: - defimpl Inspect, for: HashSet do + defimpl Inspect, for: MapSet do import Inspect.Algebra - def inspect(dict, opts) do - concat ["#HashSet<", to_doc(HashSet.to_list(dict), opts), ">"] + def inspect(map_set, opts) do + {doc, opts} = to_doc_with_opts(MapSet.to_list(map_set), opts) + {concat(["MapSet.new(", doc, ")"]), opts} end end - The `concat` function comes from `Inspect.Algebra` and it - concatenates algebra documents together. In the example above, - it is concatenating the string `"HashSet<"` (all strings are - valid algebra documents that keep their formatting when pretty - printed), the document returned by `Inspect.Algebra.to_doc/2` and the - other string `">"`. + First [`to_doc_with_opts/2`](`Inspect.Algebra.to_doc_with_opts/2`) is + used to convert another data structure into its algebra document and + then [`concat/1`](`Inspect.Algebra.concat/1`) concatenates algebra + documents together. + + In the example above it is concatenating the string `"MapSet.new("`, + the document returned by `to_doc_with_opts/2`, and the final string `")"`. + Therefore, the MapSet with the numbers 1, 2, and 3 will be printed as: - Since regular strings are valid entities in an algebra document, - an implementation of inspect may simply return a string, - although that will devoid it of any pretty-printing. + iex> MapSet.new([1, 2, 3], fn x -> x * 2 end) + MapSet.new([2, 4, 6]) - ## Error handling + In other words, `MapSet`'s inspect representation returns an expression + that, when evaluated, builds the `MapSet` itself. + + ### Error handling In case there is an error while your structure is being inspected, - Elixir will automatically fall back to a raw representation. + Elixir will raise an `ArgumentError` error and will automatically fall back + to a raw representation for printing the structure. Furthermore, you + must be careful when debugging your own Inspect implementation, as calls + to `IO.inspect/2` or `dbg/1` may trigger an infinite loop (as in order to + inspect/debug the data structure, you must call `inspect` itself). + + Here are some tips: + + * For debugging, use `IO.inspect/2` with the `structs: false` option, + which disables custom printing and avoids calling the Inspect + implementation recursively - You can however access the underlying error by invoking the Inspect - implementation directly. For example, to test Inspect.HashSet above, - you can invoke it as: + * To access the underlying error on your custom `Inspect` implementation, + you may invoke the protocol directly. For example, we could invoke the + `Inspect.MapSet` implementation above as: - Inspect.HashSet.inspect(HashSet.new, Inspect.Opts.new) + Inspect.MapSet.inspect(MapSet.new(), %Inspect.Opts{}) + Note that, from Elixir v1.19, the inspect protocol was augmented to + allow a two-element tuple with the document and the updated options + to be returned from the protocol. """ # Handle structs in Any @fallback_to_any true - def inspect(thing, opts) -end + @impl true + defmacro __deriving__(module, options) do + info = Macro.struct_info!(module, __CALLER__) + fields = Enum.sort(Enum.map(info, & &1.field) -- [:__exception__, :__struct__]) -defimpl Inspect, for: Atom do - require Macro + only = Keyword.get(options, :only, fields) + except = Keyword.get(options, :except, []) - def inspect(atom, _opts) do - inspect(atom) - end - - def inspect(false), do: "false" - def inspect(true), do: "true" - def inspect(nil), do: "nil" - def inspect(:""), do: ":\"\"" - - def inspect(atom) do - binary = Atom.to_string(atom) + :ok = validate_option(:only, only, fields, module) + :ok = validate_option(:except, except, fields, module) - cond do - valid_ref_identifier?(binary) -> - if only_elixir?(binary) do - binary - else - "Elixir." <> rest = binary - rest - end - valid_atom_identifier?(binary) -> - ":" <> binary - atom in [:%{}, :{}, :<<>>, :..., :%] -> - ":" <> binary - atom in Macro.binary_ops or atom in Macro.unary_ops -> - ":" <> binary - true -> - << ?:, ?", Inspect.BitString.escape(binary, ?") :: binary, ?" >> - end - end + optional = + case Keyword.get(options, :optional, []) do + :all -> + fields - defp only_elixir?("Elixir." <> rest), do: only_elixir?(rest) - defp only_elixir?("Elixir"), do: true - defp only_elixir?(_), do: false + optional -> + :ok = validate_option(:optional, optional, fields, module) + optional + end - # Detect if atom is an atom alias (Elixir.Foo.Bar.Baz) + inspect_module = + if fields == Enum.sort(only) and except == [] do + Inspect.Map + else + Inspect.Any + end - defp valid_ref_identifier?("Elixir" <> rest) do - valid_ref_piece?(rest) - end + filtered_fields = + fields + |> Enum.reject(&(&1 in except)) + |> Enum.filter(&(&1 in only)) - defp valid_ref_identifier?(_), do: false + filtered_guard = + quote do + var!(field) in unquote(filtered_fields) + end - defp valid_ref_piece?(<>) when h in ?A..?Z do - valid_ref_piece? valid_identifier?(t) - end + field_guard = + if optional == [] do + filtered_guard + else + optional_map = + for field <- optional, into: %{} do + default = Enum.find(info, %{}, &(&1.field == field)) |> Map.get(:default, nil) + {field, default} + end - defp valid_ref_piece?(<<>>), do: true - defp valid_ref_piece?(_), do: false + quote do + unquote(filtered_guard) and + not case unquote(Macro.escape(optional_map)) do + %{^var!(field) => var!(default)} -> + var!(default) == Map.get(var!(struct), var!(field)) - # Detect if atom + %{} -> + false + end + end + end - defp valid_atom_identifier?(<>) when h in ?a..?z or h in ?A..?Z or h == ?_ do - valid_atom_piece?(t) + quote do + defimpl Inspect, for: unquote(module) do + def inspect(var!(struct), var!(opts)) do + var!(infos) = + for %{field: var!(field)} = var!(info) <- unquote(module).__info__(:struct), + unquote(field_guard), + do: var!(info) + + var!(name) = Macro.inspect_atom(:literal, unquote(module)) + + unquote(inspect_module).inspect_as_struct( + var!(struct), + var!(name), + var!(infos), + var!(opts) + ) + end + end + end end - defp valid_atom_identifier?(_), do: false + defp validate_option(option, option_list, fields, module) do + if not is_list(option_list) do + raise ArgumentError, + "invalid value #{Kernel.inspect(option_list)} in #{Kernel.inspect(option)} " <> + "when deriving the Inspect protocol for #{Kernel.inspect(module)} " <> + "(expected a list)" + end - defp valid_atom_piece?(t) do - case valid_identifier?(t) do - <<>> -> true - <> -> true - <> -> true - <> -> valid_atom_piece?(t) - _ -> false + case option_list -- fields do + [] -> + :ok + + unknown_fields -> + raise ArgumentError, + "unknown fields #{Kernel.inspect(unknown_fields)} in #{Kernel.inspect(option)} " <> + "when deriving the Inspect protocol for #{Kernel.inspect(module)}" end end - defp valid_identifier?(<>) - when h in ?a..?z - when h in ?A..?Z - when h in ?0..?9 - when h == ?_ do - valid_identifier? t - end + @doc """ + Converts `term` into an algebra document. - defp valid_identifier?(other), do: other + This function shouldn't be invoked directly, unless when implementing + a custom `inspect_fun` to be given to `Inspect.Opts`. Everywhere else, + `Inspect.Algebra.to_doc/2` should be preferred as it handles structs + and exceptions. + """ + @spec inspect(t, Inspect.Opts.t()) :: + Inspect.Algebra.t() | {Inspect.Algebra.t(), Inspect.Opts.t()} + def inspect(term, opts) end -defimpl Inspect, for: BitString do - def inspect(thing, %Inspect.Opts{binaries: bins} = opts) when is_binary(thing) do - if bins == :as_strings or (bins == :infer and String.printable?(thing)) do - <> - else - inspect_bitstring(thing, opts) - end - end - - def inspect(thing, opts) do - inspect_bitstring(thing, opts) +defimpl Inspect, for: Atom do + def inspect(atom, opts) do + color_doc(Macro.inspect_atom(:literal, atom), color_key(atom), opts) end - ## Escaping + defp color_key(atom) when is_boolean(atom), do: :boolean + defp color_key(nil), do: nil + defp color_key(_), do: :atom +end - @doc false - def escape(other, char) do - escape(other, char, <<>>) - end +defimpl Inspect, for: BitString do + def inspect(term, opts) when is_binary(term) do + %Inspect.Opts{binaries: bins, base: base, printable_limit: printable_limit} = opts + + if bins == :as_strings or + (bins == :infer and String.printable?(term, printable_limit) and base == :decimal) do + inspected = + case Identifier.escape(term, ?", printable_limit) do + {escaped, ""} -> [?", escaped, ?"] + {escaped, _} -> [?", escaped, ?", " <> ..."] + end - defp escape(<< char, t :: binary >>, char, binary) do - escape(t, char, << binary :: binary, ?\\, char >>) - end - defp escape(<>, char, binary) do - escape(t, char, << binary :: binary, ?\\, ?#, ?{>>) - end - defp escape(<>, char, binary) do - escape(t, char, << binary :: binary, ?\\, ?a >>) - end - defp escape(<>, char, binary) do - escape(t, char, << binary :: binary, ?\\, ?b >>) - end - defp escape(<>, char, binary) do - escape(t, char, << binary :: binary, ?\\, ?d >>) - end - defp escape(<>, char, binary) do - escape(t, char, << binary :: binary, ?\\, ?e >>) - end - defp escape(<>, char, binary) do - escape(t, char, << binary :: binary, ?\\, ?f >>) - end - defp escape(<>, char, binary) do - escape(t, char, << binary :: binary, ?\\, ?n >>) - end - defp escape(<>, char, binary) do - escape(t, char, << binary :: binary, ?\\, ?r >>) - end - defp escape(<>, char, binary) do - escape(t, char, << binary :: binary, ?\\, ?\\ >>) - end - defp escape(<>, char, binary) do - escape(t, char, << binary :: binary, ?\\, ?t >>) - end - defp escape(<>, char, binary) do - escape(t, char, << binary :: binary, ?\\, ?v >>) - end - defp escape(<>, char, binary) do - head = << h :: utf8 >> - if String.printable?(head) do - escape(t, char, append(head, binary)) + color_doc(IO.iodata_to_binary(inspected), :string, opts) else - << byte :: size(8), h :: binary >> = head - t = << h :: binary, t :: binary >> - escape(t, char, << binary :: binary, escape_char(byte) :: binary >>) + inspect_bitstring(term, opts) end end - defp escape(<>, char, binary) do - escape(t, char, << binary :: binary, escape_char(h) :: binary >>) - end - defp escape(<<>>, _char, binary), do: binary - - @doc false - # Also used by Regex - def escape_char(char) when char in ?\000..?\377, - do: octify(char) - - def escape_char(char), do: hexify(char) - defp octify(byte) do - << hi :: size(2), mi :: size(3), lo :: size(3) >> = << byte >> - << ?\\, ?0 + hi, ?0 + mi, ?0 + lo >> + def inspect(term, opts) do + inspect_bitstring(term, opts) end - defp hexify(char) when char < 0x10000 do - <> = <> - <> + defp inspect_bitstring("", opts) do + color_doc("<<>>", :binary, opts) end - defp hexify(char) when char < 0x1000000 do - <> = <> - <> + defp inspect_bitstring(bitstring, %{limit: limit} = opts) do + left = color_doc("<<", :binary, opts) + right = color_doc(">>", :binary, opts) + inner = each_bit(bitstring, limit, opts) + doc = group(concat(concat(left, nest(inner, 2)), right)) + new_limit = if limit == :infinity, do: limit, else: max(0, limit - byte_size(bitstring)) + {doc, %{opts | limit: new_limit}} end - defp to_hex(c) when c in 0..9, do: ?0+c - defp to_hex(c) when c in 10..15, do: ?a+c-10 - - defp append(<>, binary), do: append(t, << binary :: binary, h >>) - defp append(<<>>, binary), do: binary - - ## Bitstrings - - defp inspect_bitstring(bitstring, opts) do - each_bit(bitstring, opts.limit, "<<") <> ">>" + defp each_bit(_, 0, _) do + "..." end - defp each_bit(_, 0, acc) do - acc <> "..." + defp each_bit(<<>>, _counter, _opts) do + Inspect.Algebra.empty() end - defp each_bit(<>, counter, acc) when t != <<>> do - each_bit(t, decrement(counter), acc <> Integer.to_string(h) <> ", ") + defp each_bit(<>, _counter, opts) do + Inspect.Integer.inspect(h, opts) end - defp each_bit(<>, _counter, acc) do - acc <> Integer.to_string(h) - end - - defp each_bit(<<>>, _counter, acc) do - acc + defp each_bit(<>, counter, opts) do + flex_glue( + concat(Inspect.Integer.inspect(h, opts), ","), + each_bit(t, decrement(counter), opts) + ) end - defp each_bit(bitstring, _counter, acc) do + defp each_bit(bitstring, _counter, opts) do size = bit_size(bitstring) - <> = bitstring - acc <> Integer.to_string(h) <> "::size(" <> Integer.to_string(size) <> ")" + <> = bitstring + concat(Inspect.Integer.inspect(h, opts), "::size(" <> Integer.to_string(size) <> ")") end + @compile {:inline, decrement: 1} defp decrement(:infinity), do: :infinity - defp decrement(counter), do: counter - 1 + defp decrement(counter), do: counter - 1 end defimpl Inspect, for: List do - @doc ~S""" - Represents a list, checking if it can be printed or not. - If so, a single-quoted representation is returned, - otherwise the brackets syntax is used. Keywords are - printed in keywords syntax. - - ## Examples - - iex> inspect('bar') - "'bar'" + def inspect([], opts) do + color_doc("[]", :list, opts) + end + + # TODO: Remove :char_list and :as_char_lists handling on v2.0 + def inspect(term, opts) do + %Inspect.Opts{ + charlists: lists, + char_lists: lists_deprecated, + printable_limit: printable_limit + } = opts + + lists = + if lists == :infer and lists_deprecated != :infer do + case lists_deprecated do + :as_char_lists -> + IO.warn( + "the :char_lists inspect option and its :as_char_lists " <> + "value are deprecated, use the :charlists option and its " <> + ":as_charlists value instead" + ) + + :as_charlists + + _ -> + IO.warn("the :char_lists inspect option is deprecated, use :charlists instead") + lists_deprecated + end + else + lists + end - iex> inspect([0|'bar']) - "[0, 98, 97, 114]" + open = color_doc("[", :list, opts) + sep = color_doc(",", :list, opts) + close = color_doc("]", :list, opts) - iex> inspect([:foo,:bar]) - "[:foo, :bar]" + cond do + lists == :as_charlists or (lists == :infer and List.ascii_printable?(term, printable_limit)) -> + inspected = + case Identifier.escape(IO.chardata_to_string(term), ?", printable_limit) do + {escaped, ""} -> [?~, ?c, ?", escaped, ?"] + {escaped, _} -> [?~, ?c, ?", escaped, ?", " ++ ..."] + end - """ + color_doc(IO.iodata_to_binary(inspected), :charlist, opts) - def inspect([], _opts), do: "[]" + keyword?(term) -> + container_doc_with_opts(open, term, close, opts, &keyword/2, + separator: sep, + break: :strict + ) - def inspect(thing, %Inspect.Opts{char_lists: lists} = opts) do - cond do - lists == :as_char_lists or (lists == :infer and printable?(thing)) -> - << ?', Inspect.BitString.escape(IO.chardata_to_string(thing), ?') :: binary, ?' >> - keyword?(thing) -> - surround_many("[", thing, "]", opts.limit, &keyword(&1, opts)) true -> - surround_many("[", thing, "]", opts.limit, &to_doc(&1, opts)) + container_doc_with_opts(open, term, close, opts, &to_doc_with_opts/2, separator: sep) end end + @doc false def keyword({key, value}, opts) do - concat( - key_to_binary(key) <> ": ", - to_doc(value, opts) - ) + key = color_doc(Macro.inspect_atom(:key, key), :atom, opts) + {doc, opts} = to_doc_with_opts(value, opts) + {concat(key, concat(" ", doc)), opts} end + @doc false def keyword?([{key, _value} | rest]) when is_atom(key) do - case Atom.to_char_list(key) do - 'Elixir.' ++ _ -> false + case Atom.to_charlist(key) do + [?E, ?l, ?i, ?x, ?i, ?r, ?.] ++ _ -> false _ -> keyword?(rest) end end - def keyword?([]), do: true + def keyword?([]), do: true def keyword?(_other), do: false - - ## Private - - defp key_to_binary(key) do - case Inspect.Atom.inspect(key) do - ":" <> right -> right - other -> other - end - end - - defp printable?([c|cs]) when is_integer(c) and c in 32..126, do: printable?(cs) - defp printable?([?\n|cs]), do: printable?(cs) - defp printable?([?\r|cs]), do: printable?(cs) - defp printable?([?\t|cs]), do: printable?(cs) - defp printable?([?\v|cs]), do: printable?(cs) - defp printable?([?\b|cs]), do: printable?(cs) - defp printable?([?\f|cs]), do: printable?(cs) - defp printable?([?\e|cs]), do: printable?(cs) - defp printable?([?\a|cs]), do: printable?(cs) - defp printable?([]), do: true - defp printable?(_), do: false end defimpl Inspect, for: Tuple do - def inspect({}, _opts), do: "{}" - def inspect(tuple, opts) do - surround_many("{", Tuple.to_list(tuple), "}", opts.limit, &to_doc(&1, opts)) + open = color_doc("{", :tuple, opts) + sep = color_doc(",", :tuple, opts) + close = color_doc("}", :tuple, opts) + container_opts = [separator: sep, break: :flex] + + container_doc_with_opts( + open, + Tuple.to_list(tuple), + close, + opts, + &to_doc_with_opts/2, + container_opts + ) end end defimpl Inspect, for: Map do def inspect(map, opts) do - nest inspect(map, "", opts), 1 + inspect_as_map(map, opts) end - def inspect(map, name, opts) do - map = :maps.to_list(map) - surround_many("%" <> name <> "{", map, "}", opts.limit, traverse_fun(map, opts)) - end + def inspect_as_map(map, opts) do + list = + if Keyword.get(opts.custom_options, :sort_maps) do + map |> Map.to_list() |> :lists.sort() + else + Map.to_list(map) + end - defp traverse_fun(list, opts) do - if Inspect.List.keyword?(list) do - &Inspect.List.keyword(&1, opts) - else - &to_map(&1, opts) - end + fun = + if Inspect.List.keyword?(list) do + &Inspect.List.keyword/2 + else + sep = color_doc(" => ", :map, opts) + &to_assoc(&1, &2, sep) + end + + map_container_doc(list, "", opts, fun) end - defp to_map({key, value}, opts) do - concat( - concat(to_doc(key, opts), " => "), - to_doc(value, opts) - ) + def inspect_as_struct(map, name, infos, opts) do + fun = fn %{field: field}, opts -> Inspect.List.keyword({field, Map.get(map, field)}, opts) end + map_container_doc(infos, name, opts, fun) end -end -defimpl Inspect, for: Integer do - def inspect(thing, _opts) do - Integer.to_string(thing) + defp to_assoc({key, value}, opts, sep) do + {key_doc, opts} = to_doc_with_opts(key, opts) + {value_doc, opts} = to_doc_with_opts(value, opts) + {concat(concat(key_doc, sep), value_doc), opts} end -end -defimpl Inspect, for: Float do - def inspect(thing, _opts) do - IO.iodata_to_binary(:io_lib_format.fwrite_g(thing)) + defp map_container_doc(list, name, opts, fun) do + open = color_doc("%" <> name <> "{", :map, opts) + sep = color_doc(",", :map, opts) + close = color_doc("}", :map, opts) + container_doc_with_opts(open, list, close, opts, fun, separator: sep, break: :strict) end end -defimpl Inspect, for: Regex do - def inspect(regex, _opts) do - delim = ?/ - concat ["~r", - <>, - regex.opts] +defimpl Inspect, for: Integer do + def inspect(term, %Inspect.Opts{base: base} = opts) do + inspected = Integer.to_string(term, base_to_value(base)) |> prepend_prefix(base) + color_doc(inspected, :number, opts) end - defp escape(bin, term), - do: escape(bin, <<>>, term) - - defp escape(<> <> rest, buf, term), - do: escape(rest, buf <> <>, term) - - defp escape(<> <> rest, buf, term), - do: escape(rest, buf <> <>, term) - - # the list of characters is from `String.printable?` impl - # minus characters treated specially by regex: \s, \d, \b, \e + defp base_to_value(base) do + case base do + :binary -> 2 + :decimal -> 10 + :octal -> 8 + :hex -> 16 + end + end - defp escape(<> <> rest, buf, term), - do: escape(rest, <>, term) + defp prepend_prefix(value, :decimal), do: value - defp escape(<> <> rest, buf, term), - do: escape(rest, <>, term) + defp prepend_prefix(<>, base) do + "-" <> prepend_prefix(value, base) + end - defp escape(<> <> rest, buf, term), - do: escape(rest, <>, term) + defp prepend_prefix(value, base) do + prefix = + case base do + :binary -> "0b" + :octal -> "0o" + :hex -> "0x" + end - defp escape(<> <> rest, buf, term), - do: escape(rest, <>, term) + prefix <> value + end +end - defp escape(<> <> rest, buf, term), - do: escape(rest, <>, term) +defimpl Inspect, for: Float do + def inspect(float, opts) do + abs = abs(float) + + formatted = + if abs >= 1.0 and abs < 1.0e16 and trunc(float) == float do + [Integer.to_string(trunc(float)), ?., ?0] + else + Float.to_charlist(float) + end - defp escape(<> <> rest, buf, term), - do: escape(rest, <>, term) + color_doc(IO.iodata_to_binary(formatted), :number, opts) + end +end - defp escape(<> <> rest, buf, term) do - charstr = <> - if String.printable?(charstr) and not c in [?\d, ?\b, ?\e] do - escape(rest, buf <> charstr, term) - else - escape(rest, buf <> Inspect.BitString.escape_char(c), term) +defimpl Inspect, for: Regex do + def inspect(regex = %{opts: regex_opts}, opts) when is_list(regex_opts) do + case translate_options(regex_opts, []) do + :error -> + concat([ + "Regex.compile!(", + to_doc(regex.source, opts), + ", ", + to_doc(regex_opts, opts), + ")" + ]) + + translated_opts -> + {escaped, _} = + regex.source + |> normalize(<<>>) + |> Identifier.escape(?/, :infinity, &escape_map/1) + + source = IO.iodata_to_binary([?~, ?r, ?/, escaped, ?/, translated_opts]) + color_doc(source, :regex, opts) end end - defp escape(<> <> rest, buf, term), - do: escape(rest, <>, term) - - defp escape(<<>>, buf, _), do: buf + defp translate_options([:dotall, {:newline, :anycrlf} | t], acc), + do: translate_options(t, [?s | acc]) + + defp translate_options([:unicode, :ucp | t], acc), do: translate_options(t, [?u | acc]) + defp translate_options([:caseless | t], acc), do: translate_options(t, [?i | acc]) + defp translate_options([:extended | t], acc), do: translate_options(t, [?x | acc]) + defp translate_options([:firstline | t], acc), do: translate_options(t, [?f | acc]) + defp translate_options([:ungreedy | t], acc), do: translate_options(t, [?U | acc]) + defp translate_options([:multiline | t], acc), do: translate_options(t, [?m | acc]) + defp translate_options([:export | t], acc), do: translate_options(t, [?E | acc]) + defp translate_options([], acc), do: acc + defp translate_options(_t, _acc), do: :error + + defp normalize(<>, acc), do: normalize(rest, <>) + defp normalize(<>, acc), do: normalize(rest, <>) + defp normalize(<>, acc), do: normalize(rest, <>) + defp normalize(<>, acc), do: normalize(rest, <>) + defp normalize(<<>>, acc), do: acc + + defp escape_map(?\a), do: [?\\, ?a] + defp escape_map(?\f), do: [?\\, ?f] + defp escape_map(?\n), do: [?\\, ?n] + defp escape_map(?\r), do: [?\\, ?r] + defp escape_map(?\t), do: [?\\, ?t] + defp escape_map(?\v), do: [?\\, ?v] + defp escape_map(_), do: false end defimpl Inspect, for: Function do + @elixir_compiler :binary.bin_to_list("elixir_compiler_") + def inspect(function, _opts) do - fun_info = :erlang.fun_info(function) + fun_info = Function.info(function) mod = fun_info[:module] + name = fun_info[:name] - if fun_info[:type] == :external and fun_info[:env] == [] do - "&#{Inspect.Atom.inspect(mod)}.#{fun_info[:name]}/#{fun_info[:arity]}" - else - case Atom.to_char_list(mod) do - 'elixir_compiler_' ++ _ -> - if function_exported?(mod, :__RELATIVE__, 0) do - "#Function<#{uniq(fun_info)} in file:#{mod.__RELATIVE__}>" - else - default_inspect(mod, fun_info) - end - _ -> + cond do + not is_atom(mod) -> + "#Function<#{uniq(fun_info)}/#{fun_info[:arity]}>" + + fun_info[:type] == :external and fun_info[:env] == [] -> + inspected_as_atom = Macro.inspect_atom(:literal, mod) + inspected_as_function = Macro.inspect_atom(:remote_call, name) + "&#{inspected_as_atom}.#{inspected_as_function}/#{fun_info[:arity]}" + + match?(@elixir_compiler ++ _, Atom.to_charlist(mod)) -> + if function_exported?(mod, :__RELATIVE__, 0) do + "#Function<#{uniq(fun_info)} in file:#{mod.__RELATIVE__()}>" + else default_inspect(mod, fun_info) - end + end + + true -> + default_inspect(mod, fun_info) end end defp default_inspect(mod, fun_info) do - "#Function<#{uniq(fun_info)}/#{fun_info[:arity]} in " <> - "#{Inspect.Atom.inspect(mod)}#{extract_name(fun_info[:name])}>" + inspected_as_atom = Macro.inspect_atom(:literal, mod) + extracted_name = extract_name(fun_info[:name]) + "#Function<#{uniq(fun_info)}/#{fun_info[:arity]} in #{inspected_as_atom}#{extracted_name}>" end defp extract_name([]) do @@ -473,16 +626,17 @@ defimpl Inspect, for: Function do end defp extract_name(name) do - name = Atom.to_string(name) - case :binary.split(name, "-", [:global]) do - ["", name | _] -> "." <> name - _ -> "." <> name + case Identifier.extract_anonymous_fun_parent(name) do + {name, arity} -> + "." <> Macro.inspect_atom(:remote_call, name) <> "/" <> arity + + :error -> + "." <> Macro.inspect_atom(:remote_call, name) end end defp uniq(fun_info) do - Integer.to_string(fun_info[:new_index]) <> "." <> - Integer.to_string(fun_info[:uniq]) + Integer.to_string(fun_info[:new_index]) <> "." <> Integer.to_string(fun_info[:uniq]) end end @@ -494,31 +648,77 @@ end defimpl Inspect, for: Port do def inspect(port, _opts) do - IO.iodata_to_binary :erlang.port_to_list(port) + IO.iodata_to_binary(:erlang.port_to_list(port)) end end defimpl Inspect, for: Reference do def inspect(ref, _opts) do - '#Ref' ++ rest = :erlang.ref_to_list(ref) + [?#, ?R, ?e, ?f] ++ rest = :erlang.ref_to_list(ref) "#Reference" <> IO.iodata_to_binary(rest) end end defimpl Inspect, for: Any do - def inspect(%{__struct__: struct} = map, opts) do - try do - struct.__struct__ - rescue - _ -> Inspect.Map.inspect(map, opts) - else - dunder -> - if :maps.keys(dunder) == :maps.keys(map) do - pruned = :maps.remove(:__exception__, :maps.remove(:__struct__, map)) - Inspect.Map.inspect(pruned, Inspect.Atom.inspect(struct, opts), opts) - else - Inspect.Map.inspect(map, opts) - end + def inspect(%module{} = struct, opts) do + info = + for %{field: field} = map <- module.__info__(:struct), + field != :__exception__, + do: map + + Inspect.Map.inspect_as_struct(struct, Macro.inspect_atom(:literal, module), info, opts) + end + + def inspect_as_struct(map, name, infos, opts) do + open = color_doc("#" <> name <> "<", :map, opts) + sep = color_doc(",", :map, opts) + close = color_doc(">", :map, opts) + + fun = fn + %{field: field}, opts -> Inspect.List.keyword({field, Map.get(map, field)}, opts) + :..., _opts -> "..." end + + container_doc(open, infos ++ [:...], close, opts, fun, separator: sep, break: :strict) + end +end + +defimpl Inspect, for: Range do + import Inspect.Algebra + import Kernel, except: [inspect: 2] + + def inspect(first..last//1, opts) when last >= first do + concat([to_doc(first, opts), "..", to_doc(last, opts)]) + end + + def inspect(first..last//step, opts) do + concat([to_doc(first, opts), "..", to_doc(last, opts), "//", to_doc(step, opts)]) + end + + # TODO: Remove me on v2.0 + def inspect(%{__struct__: Range, first: first, last: last} = range, opts) do + step = if first <= last, do: 1, else: -1 + inspect(Map.put(range, :step, step), opts) end end + +require Protocol + +Protocol.derive( + Inspect, + Macro.Env, + only: [ + :module, + :file, + :line, + :function, + :context, + :aliases, + :requires, + :functions, + :macros, + :macro_aliases, + :context_modules, + :lexical_tracker + ] +) diff --git a/lib/elixir/lib/inspect/algebra.ex b/lib/elixir/lib/inspect/algebra.ex index 3a3b915d786..99b8d5cce55 100644 --- a/lib/elixir/lib/inspect/algebra.ex +++ b/lib/elixir/lib/inspect/algebra.ex @@ -1,55 +1,205 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + defmodule Inspect.Opts do @moduledoc """ - Defines the Inspect.Opts used by the Inspect protocol. + Defines the options used by the `Inspect` protocol. The following fields are available: - * `:structs` - when false, structs are not formatted by the inspect - protocol, they are instead printed as maps, defaults to true. + * `:base` - prints integers and binaries as `:binary`, `:octal`, `:decimal`, + or `:hex`. Defaults to `:decimal`. - * `:binaries` - when `:as_strings` all binaries will be printed as strings, - non-printable bytes will be escaped. + * `:binaries` - when `:as_binaries` all binaries will be printed in bit + syntax. - When `:as_binaries` all binaries will be printed in bit syntax. + When `:as_strings` all binaries will be printed as strings, non-printable + bytes will be escaped. - When the default `:infer`, the binary will be printed as a string if it - is printable, otherwise in bit syntax. + When the default `:infer`, the binary will be printed as a string if `:base` + is `:decimal` and if it is printable, otherwise in bit syntax. See + `String.printable?/1` to learn when a string is printable. - * `:char_lists` - when `:as_char_lists` all lists will be printed as char - lists, non-printable elements will be escaped. + * `:charlists` - when `:as_charlists` all lists will be printed as charlists, + non-printable elements will be escaped. When `:as_lists` all lists will be printed as lists. - When the default `:infer`, the list will be printed as a char list if it - is printable, otherwise as list. + When the default `:infer`, the list will be printed as a charlist if it + is printable, otherwise as list. See `List.ascii_printable?/1` to learn + when a charlist is printable. + + * `:custom_options` (since v1.9.0) - a keyword list storing custom user-defined + options. Useful when implementing the `Inspect` protocol for nested structs + to pass the custom options through. + + It supports some pre-defined keys: + + - `:sort_maps` (since v1.14.4) - if set to `true`, sorts key-value pairs + in maps. This can be helpful to make map inspection deterministic for + testing, given maps key order is random. + + * `:inspect_fun` (since v1.9.0) - a function to build algebra documents. + Defaults to `Inspect.Opts.default_inspect_fun/0`. + + * `:limit` - limits the number of items that are inspected for tuples, + bitstrings, maps, lists and any other collection of items, with the exception of + printable strings and printable charlists which use the `:printable_limit` option. + It accepts a positive integer or `:infinity`. It defaults to `100` since + `Elixir v1.19.0`, as it has better defaults to deal with nested collections. + + * `:pretty` - if set to `true` enables pretty printing. Defaults to `false`. + + * `:printable_limit` - limits the number of characters that are inspected + on printable strings and printable charlists. You can use `String.printable?/1` + and `List.ascii_printable?/1` to check if a given string or charlist is + printable. If you don't want to limit the number of characters to a particular + number, use `:infinity`. It accepts a positive integer or `:infinity`. + Defaults to `4096`. + + * `:safe` - when `false`, failures while inspecting structs will be raised + as errors instead of being wrapped in the `Inspect.Error` exception. This + is useful when debugging failures and crashes for custom inspect + implementations. Defaults to `true`. + + * `:structs` - when `false`, structs are not formatted by the inspect + protocol, they are instead printed as maps. Defaults to `true`. + + * `:syntax_colors` - when set to a keyword list of colors the output is + colorized. The keys are types and the values are the colors to use for + each type (for example, `[number: :red, atom: :blue]`). Types can include + `:atom`, `:binary`, `:boolean`, `:list`, `:map`, `:number`, `:regex`, + `:string`, `:tuple`, or some types to represent AST like `:variable`, + `:call`, and `:operator`. + Custom data types may provide their own options. + Colors can be any `t:IO.ANSI.ansidata/0` as accepted by `IO.ANSI.format/1`. + A default list of colors can be retrieved from `IO.ANSI.syntax_colors/0`. + + * `:width` - number of characters per line used when pretty is `true` or when + printing to IO devices. Set to `0` to force each item to be printed on its + own line. If you don't want to limit the number of items to a particular + number, use `:infinity`. Defaults to `80`. - * `:limit` - limits the number of items that are printed for tuples, - bitstrings, and lists, does not apply to strings nor char lists, defaults - to 50. + """ - * `:pretty` - if set to true enables pretty printing, defaults to false. + # TODO: Remove :char_lists key on v2.0 + defstruct base: :decimal, + binaries: :infer, + char_lists: :infer, + charlists: :infer, + custom_options: [], + inspect_fun: &Inspect.inspect/2, + limit: 100, + pretty: false, + printable_limit: 4096, + safe: true, + structs: true, + syntax_colors: [], + width: 80 + + @type color_key :: atom + + @type t :: %__MODULE__{ + base: :decimal | :binary | :hex | :octal, + binaries: :infer | :as_binaries | :as_strings, + charlists: :infer | :as_lists | :as_charlists, + custom_options: keyword, + inspect_fun: (any, t -> Inspect.Algebra.t()), + limit: non_neg_integer | :infinity, + pretty: boolean, + printable_limit: non_neg_integer | :infinity, + safe: boolean, + structs: boolean, + syntax_colors: [{color_key, IO.ANSI.ansidata()}], + width: non_neg_integer | :infinity + } + + @typedoc """ + Options for building an `Inspect.Opts` struct with `new/1`. + """ + @type new_opt :: + {:base, :decimal | :binary | :hex | :octal} + | {:binaries, :infer | :as_binaries | :as_strings} + | {:charlists, :infer | :as_lists | :as_charlists} + | {:custom_options, keyword} + | {:inspect_fun, (any, t -> Inspect.Algebra.t())} + | {:limit, non_neg_integer | :infinity} + | {:pretty, boolean} + | {:printable_limit, non_neg_integer | :infinity} + | {:safe, boolean} + | {:structs, boolean} + | {:syntax_colors, [{color_key, IO.ANSI.ansidata()}]} + | {:width, non_neg_integer | :infinity} + + @doc """ + Builds an `Inspect.Opts` struct. + """ + @doc since: "1.13.0" + @spec new([new_opt()]) :: t + def new(opts) do + struct(%Inspect.Opts{inspect_fun: default_inspect_fun()}, opts) + end - * `:width` - defaults to the 80 characters. + @doc """ + Returns the default inspect function. """ + @doc since: "1.13.0" + @spec default_inspect_fun() :: (term, t -> Inspect.Algebra.t()) + def default_inspect_fun do + :persistent_term.get({__MODULE__, :inspect_fun}, &Inspect.inspect/2) + end + + @doc """ + Sets the default inspect function. + + Set this option with care as it will change how all values + in the system are inspected. The main use of this functionality + is to provide an entry point to filter inspected values, + in order for entities to comply with rules and legislations + on data security and data privacy. + + It is **extremely discouraged** for libraries to set their own + function as this must be controlled by applications. Libraries + should instead define their own structs with custom inspect + implementations. If a library must change the default inspect + function, then it is best to ask users of your library to explicitly + call `default_inspect_fun/1` with your function of choice. + + The default is `Inspect.inspect/2`. + + ## Examples + + previous_fun = Inspect.Opts.default_inspect_fun() + + Inspect.Opts.default_inspect_fun(fn + %{address: _} = map, opts -> + previous_fun.(%{map | address: "[REDACTED]"}, opts) + + value, opts -> + previous_fun.(value, opts) + end) - defstruct structs: true :: boolean, - binaries: :infer :: :infer | :as_binaries | :as_strings, - char_lists: :infer :: :infer | :as_lists | :as_char_lists, - limit: 50 :: pos_integer, - width: 80 :: pos_integer | :infinity, - pretty: false :: boolean + """ + @doc since: "1.13.0" + @spec default_inspect_fun((term, t -> Inspect.Algebra.t())) :: :ok + def default_inspect_fun(fun) when is_function(fun, 2) do + :persistent_term.put({__MODULE__, :inspect_fun}, fun) + end end defmodule Inspect.Algebra do @moduledoc ~S""" A set of functions for creating and manipulating algebra - documents, as described in ["Strictly Pretty" (2000) by Christian Lindig][0]. + documents. - An algebra document is represented by an `Inspect.Algebra` node - or a regular string. + This module implements the functionality described in + ["Strictly Pretty" (2000) by Christian Lindig][0] with small + additions, like support for binary nodes and a break mode that + maximises use of horizontal space. - iex> Inspect.Algebra.empty - :doc_nil + iex> Inspect.Algebra.line() + :doc_line iex> "foo" "foo" @@ -57,445 +207,1191 @@ defmodule Inspect.Algebra do With the functions in this module, we can concatenate different elements together and render them: - iex> doc = Inspect.Algebra.concat(Inspect.Algebra.empty, "foo") - iex> Inspect.Algebra.pretty(doc, 80) + iex> doc = Inspect.Algebra.concat(Inspect.Algebra.empty(), "foo") + iex> Inspect.Algebra.format(doc, 80) "foo" The functions `nest/2`, `space/2` and `line/2` help you put the document together into a rigid structure. However, the document - algebra gets interesting when using functions like `break/2`, which - converts the given string into a line break depending on how much space - there is to print. Let's glue two docs together with a break and then - render it: + algebra gets interesting when using functions like `glue/3` and + `group/1`. A glue inserts a break between two documents. A group + indicates a document that must fit the current line, otherwise + breaks are rendered as new lines. Let's glue two docs together + with a break, group it and then render it: iex> doc = Inspect.Algebra.glue("a", " ", "b") - iex> Inspect.Algebra.pretty(doc, 80) + iex> doc = Inspect.Algebra.group(doc) + iex> Inspect.Algebra.format(doc, 80) "a b" - Notice the break was represented as is, because we haven't reached + Note that the break was represented as is, because we haven't reached a line limit. Once we do, it is replaced by a newline: iex> doc = Inspect.Algebra.glue(String.duplicate("a", 20), " ", "b") - iex> Inspect.Algebra.pretty(doc, 10) + iex> doc = Inspect.Algebra.group(doc) + iex> Inspect.Algebra.format(doc, 10) "aaaaaaaaaaaaaaaaaaaa\nb" + This module uses the byte size to compute how much space there is + left. If your document contains strings, then those need to be + wrapped in `string/1`, which then relies on `String.length/1` to + precompute the document size. + Finally, this module also contains Elixir related functions, a bit - tied to Elixir formatting, namely `surround/3` and `surround_many/5`. + tied to Elixir formatting, such as `to_doc/2`. ## Implementation details - The original Haskell implementation of the algorithm by [Wadler][1] - relies on lazy evaluation to unfold document groups on two alternatives: - `:flat` (breaks as spaces) and `:break` (breaks as newlines). - Implementing the same logic in a strict language such as Elixir leads - to an exponential growth of possible documents, unless document groups - are encoded explictly as `:flat` or `:break`. Those groups are then reduced - to a simple document, where the layout is already decided, per [Lindig][0]. + The implementation of `Inspect.Algebra` is based on the Strictly Pretty + paper by [Lindig][0] which builds on top of previous pretty printing + algorithms but is tailored to strict languages, such as Elixir. + The core idea in the paper is the use of explicit document groups which + are rendered as flat (breaks as spaces) or as break (breaks as newlines). - This implementation slightly changes the semantic of Lindig's algorithm - to allow elements that belong to the same group to be printed together - in the same line, even if they do not fit the line fully. This was achieved - by changing `:break` to mean a possible break and `:flat` to force a flat - structure. Then deciding if a break works as a newline is just a matter - of checking if we have enough space until the next break that is not - inside a group (which is still flat). + This implementation provides two types of breaks: `:strict` and `:flex`. + When a group does not fit, all strict breaks are treated as newlines. + Flex breaks, however, are re-evaluated on every occurrence and may still + be rendered flat. See `break/1` and `flex_break/1` for more information. - Custom pretty printers can be implemented using the documents returned - by this module and by providing their own rendering functions. + This implementation also adds `force_unfit/1` and optimistic/pessimistic + groups which give more control over the document fitting. - [0]: http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.34.2200 - [1]: http://homepages.inf.ed.ac.uk/wadler/papers/prettier/prettier.pdf + [0]: https://lindig.github.io/papers/strictly-pretty-2000.pdf """ - @surround_separator "," + @container_separator "," @tail_separator " |" @newline "\n" - @nesting 1 - @break " " - # Functional interface to `doc` records + # Functional interface to "doc" records + + @type t :: + binary + | doc_nil + | doc_cons + | doc_line + | doc_break + | doc_collapse + | doc_color + | doc_fits + | doc_force + | doc_group + | doc_nest + | doc_string + | doc_limit + + @typep doc_nil :: [] + defmacrop doc_nil do + [] + end - @type t :: :doc_nil | :doc_line | doc_cons | doc_nest | doc_break | doc_group | binary + @typep doc_line :: :doc_line + defmacrop doc_line do + :doc_line + end - @typep doc_cons :: {:doc_cons, t, t} + @typep doc_cons :: nonempty_improper_list(t, t) defmacrop doc_cons(left, right) do - quote do: {:doc_cons, unquote(left), unquote(right)} + quote do: [unquote(left) | unquote(right)] end - @typep doc_nest :: {:doc_nest, t, non_neg_integer} - defmacrop doc_nest(doc, indent) do - quote do: {:doc_nest, unquote(doc), unquote(indent) } + @typep doc_string :: {:doc_string, binary, non_neg_integer} + defmacrop doc_string(string, length) do + quote do: {:doc_string, unquote(string), unquote(length)} end - @typep doc_break :: {:doc_break, binary} - defmacrop doc_break(break) do - quote do: {:doc_break, unquote(break)} + @typep doc_limit :: {:doc_limit, t, pos_integer | :infinity} + defmacrop doc_limit(doc, limit) do + quote do: {:doc_limit, unquote(doc), unquote(limit)} end - @typep doc_group :: {:doc_group, t} - defmacrop doc_group(group) do - quote do: {:doc_group, unquote(group)} + @typep doc_nest :: {:doc_nest, t, :cursor | :reset | non_neg_integer, :always | :break} + defmacrop doc_nest(doc, indent, always_or_break) do + quote do: {:doc_nest, unquote(doc), unquote(indent), unquote(always_or_break)} end - defmacrop is_doc(doc) do - if Macro.Env.in_guard?(__CALLER__) do - do_is_doc(doc) - else - var = quote do: doc - quote do - unquote(var) = unquote(doc) - unquote(do_is_doc(var)) - end - end + @typep doc_break :: {:doc_break, binary, :flex | :strict} + defmacrop doc_break(break, mode) do + quote do: {:doc_break, unquote(break), unquote(mode)} end - defp do_is_doc(doc) do - quote do - is_binary(unquote(doc)) or - unquote(doc) in [:doc_nil, :doc_line] or - (is_tuple(unquote(doc)) and - elem(unquote(doc), 0) in [:doc_cons, :doc_nest, :doc_break, :doc_group]) - end + @typep doc_group :: {:doc_group, t, :normal | :optimistic | :pessimistic | :inherit} + defmacrop doc_group(group, mode) do + quote do: {:doc_group, unquote(group), unquote(mode)} + end + + @typep doc_fits :: {:doc_fits, t, :enabled | :disabled} + defmacrop doc_fits(group, mode) do + quote do: {:doc_fits, unquote(group), unquote(mode)} + end + + @typep doc_force :: {:doc_force, t} + defmacrop doc_force(group) do + quote do: {:doc_force, unquote(group)} + end + + @typep doc_collapse :: {:doc_collapse, pos_integer()} + defmacrop doc_collapse(count) do + quote do: {:doc_collapse, unquote(count)} + end + + @typep doc_color :: {:doc_color, t, IO.ANSI.ansidata()} + defmacrop doc_color(doc, color) do + quote do: {:doc_color, unquote(doc), unquote(color)} + end + + @typedoc """ + Options for container documents. + """ + @type container_opts :: [ + separator: String.t(), + break: :strict | :flex | :maybe + ] + + @docs [ + :doc_break, + :doc_collapse, + :doc_color, + :doc_cons, + :doc_fits, + :doc_force, + :doc_group, + :doc_nest, + :doc_string, + :doc_limit + ] + + defguard is_doc(doc) + when is_list(doc) or is_binary(doc) or doc == doc_line() or + (is_tuple(doc) and elem(doc, 0) in @docs) + + defguardp is_width(width) when width == :infinity or (is_integer(width) and width >= 0) + + # Elixir + Inspect.Opts conveniences + # These have the _doc suffix. + + @doc """ + Converts an Elixir term to an algebra document + according to the `Inspect` protocol. + + In practice, one must prefer to use `to_doc_with_opts/2` + over this function, as `to_doc_with_opts/2` returns the + updated options from inspection. + """ + @spec to_doc(any, Inspect.Opts.t()) :: t + def to_doc(term, opts) do + to_doc_with_opts(term, opts) |> elem(0) end @doc """ - Converts an Elixir structure to an algebra document - according to the inspect protocol. + Converts an Elixir term to an algebra document + according to the `Inspect` protocol, alongside the updated options. + + This function is used when implementing the inspect protocol for + a given type and you must convert nested terms to documents too. """ - @spec to_doc(any, Inspect.Opts.t) :: t - def to_doc(%{__struct__: struct} = map, %Inspect.Opts{} = opts) when is_atom(struct) do - if opts.structs do + @doc since: "1.19.0" + @spec to_doc_with_opts(any, Inspect.Opts.t()) :: {t, Inspect.Opts.t()} + def to_doc_with_opts(term, opts) + + def to_doc_with_opts(%_{} = struct, %Inspect.Opts{inspect_fun: fun} = opts) do + if opts.structs and valid_struct?(struct) do try do - Inspect.inspect(map, opts) + fun.(struct, opts) rescue - e -> - res = Inspect.Map.inspect(map, opts) - raise ArgumentError, - "Got #{inspect e.__struct__} with message " <> - "\"#{Exception.message(e)}\" while inspecting #{pretty(res, opts.width)}" + caught_exception -> + # Because we try to raise a nice error message in case + # we can't inspect a struct, there is a chance the error + # message itself relies on the struct being printed, so + # we need to trap the inspected messages to guarantee + # we won't try to render any failed instruct when building + # the error message. + if Process.get(:inspect_trap) do + Inspect.Map.inspect_as_map(struct, opts) + else + try do + Process.put(:inspect_trap, true) + + {doc_struct, _opts} = + Inspect.Map.inspect_as_map(struct, %{ + opts + | syntax_colors: [], + inspect_fun: Inspect.Opts.default_inspect_fun() + }) + + inspected_struct = + doc_struct + |> format(opts.width) + |> IO.iodata_to_binary() + + inspect_error = + Inspect.Error.exception( + exception: caught_exception, + stacktrace: __STACKTRACE__, + inspected_struct: inspected_struct + ) + + if opts.safe do + opts = %{opts | inspect_fun: Inspect.Opts.default_inspect_fun()} + Inspect.inspect(inspect_error, opts) + else + reraise(inspect_error, __STACKTRACE__) + end + after + Process.delete(:inspect_trap) + end + end end else - Inspect.Map.inspect(map, opts) + Inspect.Map.inspect_as_map(struct, opts) + end + |> pack_opts(opts) + end + + def to_doc_with_opts(arg, %Inspect.Opts{inspect_fun: fun} = opts) do + fun.(arg, opts) |> pack_opts(opts) + end + + defp valid_struct?(%module{} = struct) do + try do + module.__info__(:struct) + rescue + _ -> false + else + info -> + valid_struct?(info, struct, map_size(struct) - 1) + end + end + + defp valid_struct?([%{field: field} | info], struct, count) when is_map_key(struct, field), + do: valid_struct?(info, struct, count - 1) + + defp valid_struct?([], _struct, 0), + do: true + + defp valid_struct?(_fields, _struct, _count), + do: false + + defp pack_opts({_doc, %Inspect.Opts{}} = doc_opts, _opts), do: doc_opts + defp pack_opts(doc, opts), do: {doc, opts} + + @doc ~S""" + Wraps `collection` in `left` and `right` according to limit and contents + and returns only the container document. + + In practice, one must prefer to use `container_doc_with_opts/6` + over this function, as `container_doc_with_opts/6` returns the + updated options from inspection. + """ + @doc since: "1.6.0" + @spec container_doc( + t, + [term], + t, + Inspect.Opts.t(), + (term, Inspect.Opts.t() -> t), + container_opts() + ) :: + t + def container_doc(left, collection, right, inspect_opts, fun, opts \\ []) do + container_doc_with_opts(left, collection, right, inspect_opts, fun, opts) |> elem(0) + end + + @doc ~S""" + Wraps `collection` in `left` and `right` according to limit and contents. + + It uses the given `left` and `right` documents as surrounding and the + separator document `separator` to separate items in `docs`. If all entries + in the collection are simple documents (texts or strings), then this function + attempts to put as much as possible on the same line. If they are not simple, + only one entry is shown per line if they do not fit. + + The limit in the given `inspect_opts` is respected and when reached this + function stops processing and outputs `"..."` instead. + + It returns a tuple with the algebra document and the updated options. + + ## Options + + * `:separator` - the separator used between each doc + * `:break` - If `:strict`, always break between each element. If `:flex`, + breaks only when necessary. If `:maybe`, chooses `:flex` only if all + elements are text-based, otherwise is `:strict` + + ## Examples + + iex> inspect_opts = %Inspect.Opts{limit: :infinity} + iex> fun = fn i, _opts -> to_string(i) end + iex> {doc, _opts} = Inspect.Algebra.container_doc_with_opts("[", Enum.to_list(1..5), "]", inspect_opts, fun) + iex> Inspect.Algebra.format(doc, 5) |> IO.iodata_to_binary() + "[1,\n 2,\n 3,\n 4,\n 5]" + + iex> inspect_opts = %Inspect.Opts{limit: 3} + iex> fun = fn i, _opts -> to_string(i) end + iex> {doc, _opts} = Inspect.Algebra.container_doc_with_opts("[", Enum.to_list(1..5), "]", inspect_opts, fun) + iex> Inspect.Algebra.format(doc, 20) |> IO.iodata_to_binary() + "[1, 2, 3, ...]" + + iex> inspect_opts = %Inspect.Opts{limit: 3} + iex> fun = fn i, _opts -> to_string(i) end + iex> opts = [separator: "!"] + iex> {doc, _opts} = Inspect.Algebra.container_doc_with_opts("[", Enum.to_list(1..5), "]", inspect_opts, fun, opts) + iex> Inspect.Algebra.format(doc, 20) |> IO.iodata_to_binary() + "[1! 2! 3! ...]" + + """ + @doc since: "1.19.0" + @spec container_doc_with_opts( + t, + [term], + t, + Inspect.Opts.t(), + (term, Inspect.Opts.t() -> t), + container_opts() + ) :: + {t, Inspect.Opts.t()} + def container_doc_with_opts(left, collection, right, inspect_opts, fun, opts \\ []) + when is_doc(left) and is_list(collection) and is_doc(right) and is_function(fun, 2) and + is_list(opts) do + case collection do + [] -> + {concat(left, right), inspect_opts} + + _ -> + break = Keyword.get(opts, :break, :maybe) + separator = Keyword.get(opts, :separator, @container_separator) + + {docs, simple?, inspect_opts} = + container_each(collection, inspect_opts, fun, [], break == :maybe) + + flex? = simple? or break == :flex + docs = fold(docs, &join(&1, &2, flex?, separator)) + + group = + case flex? do + true -> doc_group(concat(concat(left, nest(docs, 1)), right), :normal) + false -> doc_group(glue(nest(glue(left, "", docs), 2), "", right), :normal) + end + + {group, inspect_opts} + end + end + + defp container_each([], opts, _fun, acc, simple?) do + {:lists.reverse(acc), simple?, opts} + end + + defp container_each(_, opts, _fun, acc, simple?) when opts.limit <= 0 do + {:lists.reverse(["..." | acc]), simple?, opts} + end + + defp container_each([term | terms], opts, fun, acc, simple?) when is_list(terms) do + {doc, opts} = call_container_fun(fun, term, opts) + container_each(terms, opts, fun, [doc | acc], simple? and simple?(doc)) + end + + defp container_each([left | right], opts, fun, acc, simple?) do + {left, opts} = call_container_fun(fun, left, opts) + {right, _opts} = call_container_fun(fun, right, opts) + simple? = simple? and simple?(left) and simple?(right) + doc = join(left, right, simple?, @tail_separator) + {:lists.reverse([doc | acc]), simple?, opts} + end + + defp call_container_fun(fun, term, %{limit: bounded} = opts) + when bounded <= 0 or bounded == :infinity do + case fun.(term, opts) do + {doc, %Inspect.Opts{} = opts} -> {doc, opts} + doc -> {doc, opts} + end + end + + defp call_container_fun(fun, term, %{limit: limit} = opts) do + changed_opts = %{opts | limit: limit - 1} + + case fun.(term, changed_opts) do + {doc, %Inspect.Opts{} = opts} -> {doc, opts} + doc_nil() -> {doc_nil(), opts} + doc -> {doc, changed_opts} + end + end + + defp join(doc_nil(), doc_nil(), _, _), do: doc_nil() + defp join(left, doc_nil(), _, _), do: left + defp join(doc_nil(), right, _, _), do: right + defp join(left, right, true, sep), do: flex_glue(concat(left, sep), right) + defp join(left, right, false, sep), do: glue(concat(left, sep), right) + + defp simple?(doc_cons(left, right)), do: simple?(left) and simple?(right) + defp simple?(doc_color(doc, _)), do: simple?(doc) + defp simple?(doc_string(_, _)), do: true + defp simple?(doc_nil()), do: true + defp simple?(other), do: is_binary(other) + + @doc false + @deprecated "Use a combination of concat/2 and nest/2 instead" + def surround(left, doc, right) when is_doc(left) and is_doc(doc) and is_doc(right) do + concat(concat(left, nest(doc, 1)), right) + end + + @doc false + @deprecated "Use Inspect.Algebra.container_doc/6 instead" + def surround_many( + left, + docs, + right, + %Inspect.Opts{} = inspect, + fun, + separator \\ @container_separator + ) + when is_doc(left) and is_list(docs) and is_doc(right) and is_function(fun, 2) do + container_doc(left, docs, right, inspect, fun, separator: separator) + end + + # TODO: Deprecate me on Elixir v1.23 + @doc deprecated: "Use color_doc/3 instead" + def color(doc, key, opts) do + color_doc(doc, key, opts) + end + + @doc ~S""" + Colors a document if the `color_key` has a color in the options. + """ + @doc since: "1.18.0" + @spec color_doc(t, Inspect.Opts.color_key(), Inspect.Opts.t()) :: t + def color_doc(doc, color_key, %Inspect.Opts{syntax_colors: syntax_colors}) when is_doc(doc) do + if precolor = Keyword.get(syntax_colors, color_key) do + postcolor = Keyword.get(syntax_colors, :reset, :reset) + concat(doc_color(doc, ansi(precolor)), doc_color(empty(), ansi(postcolor))) + else + doc end end - def to_doc(arg, %Inspect.Opts{} = opts) do - Inspect.inspect(arg, opts) + defp ansi(color) do + color + |> IO.ANSI.format_fragment(true) + |> IO.iodata_to_binary() end + # Algebra API + + @compile {:inline, + empty: 0, + concat: 2, + break: 0, + break: 1, + glue: 2, + glue: 3, + flex_break: 0, + flex_break: 1, + flex_glue: 2, + flex_glue: 3} + @doc """ Returns a document entity used to represent nothingness. ## Examples - iex> Inspect.Algebra.empty - :doc_nil + iex> Inspect.Algebra.empty() + [] """ - @spec empty() :: :doc_nil - def empty, do: :doc_nil + @spec empty() :: doc_nil() + def empty, do: doc_nil() - @doc """ - Concatenates two document entities. + @doc ~S""" + Creates a document represented by string. + + While `Inspect.Algebra` accepts binaries as documents, + those are counted by binary size. On the other hand, + `string` documents are measured in terms of graphemes + towards the document size. ## Examples - iex> doc = Inspect.Algebra.concat "Tasteless", "Artosis" - iex> Inspect.Algebra.pretty(doc, 80) - "TastelessArtosis" + The following document has 10 bytes and therefore it + does not format to width 9 without breaks: + + iex> doc = Inspect.Algebra.glue("olá", " ", "mundo") + iex> doc = Inspect.Algebra.group(doc) + iex> Inspect.Algebra.format(doc, 9) + "olá\nmundo" + + However, if we use `string`, then the string length is + used, instead of byte size, correctly fitting: + + iex> string = Inspect.Algebra.string("olá") + iex> doc = Inspect.Algebra.glue(string, " ", "mundo") + iex> doc = Inspect.Algebra.group(doc) + iex> Inspect.Algebra.format(doc, 9) + "olá mundo" """ - @spec concat(t, t) :: doc_cons - def concat(x, y) when is_doc(x) and is_doc(y) do - doc_cons(x, y) + @doc since: "1.6.0" + @spec string(String.t()) :: doc_string + def string(string) when is_binary(string) do + doc_string(string, String.length(string)) end - @doc """ - Concatenates a list of documents. + @doc ~S""" + Concatenates two document entities returning a new document. + + ## Examples + + iex> doc = Inspect.Algebra.concat("hello", "world") + iex> Inspect.Algebra.format(doc, 80) + "helloworld" + """ - @spec concat([t]) :: doc_cons - def concat(docs) do - folddoc(docs, &concat(&1, &2)) + @spec concat(t, t) :: t + def concat(doc1, doc2) when is_doc(doc1) and is_doc(doc2) do + doc_cons(doc1, doc2) end @doc ~S""" - Nests document entity `x` positions deep. + Disable any rendering limit while rendering the given document. + + ## Examples - Nesting will be appended to the line breaks. + iex> doc = Inspect.Algebra.glue("hello", "world") |> Inspect.Algebra.group() + iex> Inspect.Algebra.format(doc, 10) + "hello\nworld" + iex> doc = Inspect.Algebra.no_limit(doc) + iex> Inspect.Algebra.format(doc, 10) + "hello world" + + """ + @doc since: "1.14.0" + @spec no_limit(t) :: t + def no_limit(doc) do + doc_limit(doc, :infinity) + end + + @doc ~S""" + Concatenates a list of documents returning a new document. + + ## Examples + + iex> doc = Inspect.Algebra.concat(["a", "b", "c"]) + iex> Inspect.Algebra.format(doc, 80) + "abc" + + """ + @spec concat([t]) :: t + def concat(docs) when is_list(docs) do + fold(docs, &concat(&1, &2)) + end + + @doc ~S""" + Colors a document with the given color (preceding the document itself). + """ + @doc since: "1.18.0" + @spec color(t, binary) :: t + def color(doc, color) when is_doc(doc) and is_binary(color) do + doc_color(doc, color) + end + + @doc ~S""" + Nests the given document at the given `level`. + + If `level` is an integer, that's the indentation appended + to line breaks whenever they occur. If the level is `:cursor`, + the current position of the "cursor" in the document becomes + the nesting. If the level is `:reset`, it is set back to 0. + + `mode` can be `:always`, which means nesting always happen, + or `:break`, which means nesting only happens inside a group + that has been broken. ## Examples iex> doc = Inspect.Algebra.nest(Inspect.Algebra.glue("hello", "world"), 5) - iex> Inspect.Algebra.pretty(doc, 5) + iex> doc = Inspect.Algebra.group(doc) + iex> Inspect.Algebra.format(doc, 5) "hello\n world" """ - @spec nest(t, non_neg_integer) :: doc_nest - def nest(x, 0) when is_doc(x) do - x + @spec nest(t, non_neg_integer | :cursor | :reset, :always | :break) :: doc_nest | t + def nest(doc, level, mode \\ :always) + + def nest(doc, :cursor, mode) when is_doc(doc) and mode in [:always, :break] do + doc_nest(doc, :cursor, mode) + end + + def nest(doc, :reset, mode) when is_doc(doc) and mode in [:always, :break] do + doc_nest(doc, :reset, mode) end - def nest(x, i) when is_doc(x) and is_integer(i) do - doc_nest(x, i) + def nest(doc, 0, _mode) when is_doc(doc) do + doc + end + + def nest(doc, level, mode) + when is_doc(doc) and is_integer(level) and level > 0 and mode in [:always, :break] do + doc_nest(doc, level, mode) end @doc ~S""" - Document entity representing a break. + Returns a break document based on the given `string`. - This break can be rendered as a linebreak or as spaces, - depending on the `mode` of the chosen layout or the provided - separator. + This break can be rendered as a linebreak or as the given `string`, + depending on the `mode` of the chosen layout. ## Examples - Let's glue two docs together with a break and then render it: + Let's create a document by concatenating two strings with a break between + them: - iex> doc = Inspect.Algebra.glue("a", " ", "b") - iex> Inspect.Algebra.pretty(doc, 80) - "a b" + iex> doc = Inspect.Algebra.concat(["a", Inspect.Algebra.break("\t"), "b"]) + iex> Inspect.Algebra.format(doc, 80) + "a\tb" - Notice the break was represented as is, because we haven't reached - a line limit. Once we do, it is replaced by a newline: + Note that the break was represented with the given string, because we didn't + reach a line limit. Once we do, it is replaced by a newline: - iex> doc = Inspect.Algebra.glue(String.duplicate("a", 20), " ", "b") - iex> Inspect.Algebra.pretty(doc, 10) + iex> break = Inspect.Algebra.break("\t") + iex> doc = Inspect.Algebra.concat([String.duplicate("a", 20), break, "b"]) + iex> doc = Inspect.Algebra.group(doc) + iex> Inspect.Algebra.format(doc, 10) "aaaaaaaaaaaaaaaaaaaa\nb" """ @spec break(binary) :: doc_break - def break(s) when is_binary(s), do: doc_break(s) + def break(string \\ " ") when is_binary(string) do + doc_break(string, :strict) + end - @spec break() :: doc_break - def break(), do: doc_break(@break) + @doc """ + Collapse any new lines and whitespace following this + node, emitting up to `max` new lines. + """ + @doc since: "1.6.0" + @spec collapse_lines(pos_integer) :: doc_collapse + def collapse_lines(max) when is_integer(max) and max > 0 do + doc_collapse(max) + end @doc """ - Inserts a break between two docs. See `break/1` for more info. + Considers the next break as fit. """ - @spec glue(t, t) :: doc_cons - def glue(x, y), do: concat(x, concat(break, y)) + # TODO: Deprecate me on Elixir v1.23 + @doc deprecated: "Pass the optimistic/pessimistic type to group/2 instead" + @spec next_break_fits(t, :enabled | :disabled) :: doc_fits + def next_break_fits(doc, mode \\ :enabled) + when is_doc(doc) and mode in [:enabled, :disabled] do + doc_fits(doc, mode) + end @doc """ - Inserts a break, passed as the second argument, between two docs, - the first and the third arguments. + Forces the current group to be unfit. """ - @spec glue(t, binary, t) :: doc_cons - def glue(x, g, y) when is_binary(g), do: concat(x, concat(break(g), y)) + @doc since: "1.6.0" + @spec force_unfit(t) :: doc_force + def force_unfit(doc) when is_doc(doc) do + doc_force(doc) + end + + @doc """ + Returns a flex break document based on the given `string`. + + A flex break still causes a group to break, like `break/1`, + but it is re-evaluated when the documented is rendered. + + For example, take a group document represented as `[1, 2, 3]` + where the space after every comma is a break. When the document + above does not fit a single line, all breaks are enabled, + causing the document to be rendered as: + + [1, + 2, + 3] + + However, if flex breaks are used, then each break is re-evaluated + when rendered, so the document could be possible rendered as: + + [1, 2, + 3] + + Hence the name "flex". they are more flexible when it comes + to the document fitting. On the other hand, they are more expensive + since each break needs to be re-evaluated. + + This function is used by `container_doc/6` and friends to the + maximum number of entries on the same line. + """ + @doc since: "1.6.0" + @spec flex_break(binary) :: doc_break + def flex_break(string \\ " ") when is_binary(string) do + doc_break(string, :flex) + end + + @doc """ + Glues two documents (`doc1` and `doc2`) inserting a + `flex_break/1` given by `break_string` between them. + + This function is used by `container_doc/6` and friends + to the maximum number of entries on the same line. + """ + @doc since: "1.6.0" + @spec flex_glue(t, binary, t) :: t + def flex_glue(doc1, break_string \\ " ", doc2) when is_binary(break_string) do + concat(doc1, concat(flex_break(break_string), doc2)) + end @doc ~S""" - Returns a group containing the specified document. + Glues two documents (`doc1` and `doc2`) inserting the given + break `break_string` between them. + + For more information on how the break is inserted, see `break/1`. ## Examples - iex> doc = Inspect.Algebra.group( - ...> Inspect.Algebra.concat( - ...> Inspect.Algebra.group( - ...> Inspect.Algebra.concat( - ...> "Hello,", + iex> doc = Inspect.Algebra.glue("hello", "world") + iex> Inspect.Algebra.format(doc, 80) + "hello world" + + iex> doc = Inspect.Algebra.glue("hello", "\t", "world") + iex> Inspect.Algebra.format(doc, 80) + "hello\tworld" + + """ + @spec glue(t, binary, t) :: t + def glue(doc1, break_string \\ " ", doc2) when is_binary(break_string) do + concat(doc1, concat(break(break_string), doc2)) + end + + @doc ~S""" + Returns a group containing the specified document `doc`. + + Documents in a group are attempted to be rendered together + to the best of the renderer ability. If there are `break/1`s + in the group and the group does not fit the given width, + the breaks are converted into lines. Otherwise the breaks + are rendered as text based on their string contents. + + There are three types of groups, described next. + + ## Group modes + + * `:normal` - the group fits if it fits within the given width + + * `:optimistic` - the group fits if it fits within the given + width. However, when nested within another group, the parent + group will assume this group fits as long as it has a single + break, even if the optimistic group has a `force_unfit/1` + document within it. Overall, this has an effect similar + to swapping the order groups break. For example, if you have + a `parent_group(child_group)` and they do not fit, the parent + converts breaks into newlines first, allowing the child to compute + if it fits. However, if the child group is optimistic and it + has breaks, then the parent assumes it fits, leaving the overall + fitting decision to the child + + * `:pessimistic` - the group fits if it fits within the given + width. However it disables any optimistic group within it + + ## Examples + + iex> doc = + ...> Inspect.Algebra.group( + ...> Inspect.Algebra.concat( + ...> Inspect.Algebra.group( ...> Inspect.Algebra.concat( - ...> Inspect.Algebra.break, - ...> "A" + ...> "Hello,", + ...> Inspect.Algebra.concat( + ...> Inspect.Algebra.break(), + ...> "A" + ...> ) ...> ) + ...> ), + ...> Inspect.Algebra.concat( + ...> Inspect.Algebra.break(), + ...> "B" ...> ) - ...> ), - ...> Inspect.Algebra.concat( - ...> Inspect.Algebra.break, - ...> "B" ...> ) - ...> )) - iex> Inspect.Algebra.pretty(doc, 80) + ...> ) + iex> Inspect.Algebra.format(doc, 80) "Hello, A B" - iex> Inspect.Algebra.pretty(doc, 6) - "Hello,\nA B" + iex> Inspect.Algebra.format(doc, 6) + "Hello,\nA\nB" + + ## Mode examples + + The different groups modes are used by Elixir's code formatter + to avoid breaking code at some specific locations. For example, + consider this code: + + some_function_call(%{..., key: value, ...}) + + Now imagine that this code does not fit its line. The code + formatter introduces breaks inside `(` and `)` and inside + `%{` and `}`, each within their own group. Therefore the + document would break as: + + some_function_call( + %{ + ..., + key: value, + ... + } + ) + + To address this, the formatter marks the inner group as optimistic. + This means the first group, which is `(...)` will consider the document + fits and avoids adding breaks around the parens. So overall the code + is formatted as: + + some_function_call(%{ + ..., + key: value, + ... + }) """ - @spec group(t) :: doc_group - def group(d) when is_doc(d) do - doc_group(d) + @spec group(t, :normal | :optimistic | :pessimistic) :: doc_group + def group(doc, mode \\ :normal) when is_doc(doc) do + doc_group( + doc, + case mode do + # TODO: Deprecate :self and :inherit on Elixir v1.23 + :self -> :normal + :inherit -> :inherit + mode when mode in [:normal, :optimistic, :pessimistic] -> mode + end + ) end - @doc """ - Inserts a mandatory single space between two document entities. + @doc ~S""" + Inserts a mandatory single space between two documents. ## Examples - iex> doc = Inspect.Algebra.space "Hughes", "Wadler" - iex> Inspect.Algebra.pretty(doc, 80) + iex> doc = Inspect.Algebra.space("Hughes", "Wadler") + iex> Inspect.Algebra.format(doc, 5) "Hughes Wadler" """ - @spec space(t, t) :: doc_cons - def space(x, y), do: concat(x, concat(" ", y)) + @spec space(t, t) :: t + def space(doc1, doc2), do: concat(doc1, concat(" ", doc2)) @doc ~S""" - Inserts a mandatory linebreak between two document entities. + A mandatory linebreak. + + A group with linebreaks will fit if all lines in the group fit. ## Examples - iex> doc = Inspect.Algebra.line "Hughes", "Wadler" - iex> Inspect.Algebra.pretty(doc, 80) + iex> doc = + ...> Inspect.Algebra.concat( + ...> Inspect.Algebra.concat( + ...> "Hughes", + ...> Inspect.Algebra.line() + ...> ), + ...> "Wadler" + ...> ) + iex> Inspect.Algebra.format(doc, 80) "Hughes\nWadler" """ - @spec line(t, t) :: doc_cons - def line(x, y), do: concat(x, concat(:doc_line, y)) + @doc since: "1.6.0" + @spec line() :: t + def line(), do: doc_line() - @doc """ - Folds a list of document entities into a document entity - using a function that is passed as the first argument. + @doc ~S""" + Inserts a mandatory linebreak between two documents. + + See `line/0`. ## Examples - iex> doc = ["A", "B"] - iex> doc = Inspect.Algebra.folddoc(doc, fn(x,y) -> - ...> Inspect.Algebra.concat [x, "!", y] - ...> end) - iex> Inspect.Algebra.pretty(doc, 80) - "A!B" + iex> doc = Inspect.Algebra.line("Hughes", "Wadler") + iex> Inspect.Algebra.format(doc, 80) + "Hughes\nWadler" """ - @spec folddoc([t], ((t, t) -> t)) :: t - def folddoc([], _), do: empty - def folddoc([doc], _), do: doc - def folddoc([d|ds], f), do: f.(d, folddoc(ds, f)) + @spec line(t, t) :: t + def line(doc1, doc2), do: concat(doc1, concat(line(), doc2)) - # Elixir conveniences + # TODO: Deprecate me on Elixir v1.23 + @doc deprecated: "Use fold/2 instead" + def fold_doc(docs, folder_fun), do: fold(docs, folder_fun) @doc ~S""" - Surrounds a document with characters. + Folds a list of documents into a document using the given folder function. - Puts the document between left and right enclosing and nesting it. - The document is marked as a group, to show the maximum as possible - concisely together. + The list of documents is folded "from the right"; in that, this function is + similar to `List.foldr/3`, except that it doesn't expect an initial + accumulator and uses the last element of `docs` as the initial accumulator. ## Examples - iex> doc = Inspect.Algebra.surround "[", Inspect.Algebra.glue("a", "b"), "]" - iex> Inspect.Algebra.pretty(doc, 3) - "[a\n b]" + iex> docs = ["A", "B", "C"] + iex> docs = + ...> Inspect.Algebra.fold(docs, fn doc, acc -> + ...> Inspect.Algebra.concat([doc, "!", acc]) + ...> end) + iex> Inspect.Algebra.format(docs, 80) + "A!B!C" """ - @spec surround(binary, t, binary) :: t - def surround(left, doc, right) do - group concat left, concat(nest(doc, @nesting), right) - end + @doc since: "1.18.0" + @spec fold([t], (t, t -> t)) :: t + def fold(docs, folder_fun) + + def fold([], _folder_fun), do: empty() + def fold([doc], _folder_fun), do: doc + + def fold([doc | docs], folder_fun) when is_function(folder_fun, 2), + do: folder_fun.(doc, fold(docs, folder_fun)) @doc ~S""" - Maps and glues a collection of items together using the given separator - and surrounds them. A limit can be passed which, once reached, stops - gluing and outputs "..." instead. + Formats a given document for a given width. - ## Examples + Takes the maximum width and a document to print as its arguments + and returns an IO data representation of the best layout for the + document to fit in the given width. - iex> doc = Inspect.Algebra.surround_many("[", Enum.to_list(1..5), "]", :infinity, &Integer.to_string(&1)) - iex> Inspect.Algebra.pretty(doc, 5) - "[1,\n 2,\n 3,\n 4,\n 5]" + The document starts flat (without breaks) until a group is found. - iex> doc = Inspect.Algebra.surround_many("[", Enum.to_list(1..5), "]", 3, &Integer.to_string(&1)) - iex> Inspect.Algebra.pretty(doc, 20) - "[1, 2, 3, ...]" + ## Examples - iex> doc = Inspect.Algebra.surround_many("[", Enum.to_list(1..5), "]", 3, &Integer.to_string(&1), "!") - iex> Inspect.Algebra.pretty(doc, 20) - "[1! 2! 3! ...]" + iex> doc = Inspect.Algebra.glue("hello", " ", "world") + iex> doc = Inspect.Algebra.group(doc) + iex> doc |> Inspect.Algebra.format(30) |> IO.iodata_to_binary() + "hello world" + iex> doc |> Inspect.Algebra.format(10) |> IO.iodata_to_binary() + "hello\nworld" """ - @spec surround_many(binary, [any], binary, integer | :infinity, (term -> t), binary) :: t - def surround_many(left, docs, right, limit, fun, separator \\ @surround_separator) - - def surround_many(left, [], right, _, _fun, _) do - concat(left, right) + @spec format(t, non_neg_integer | :infinity) :: iodata + def format(doc, width) when is_doc(doc) and is_width(width) do + format(width, 0, [{0, :flat, doc}], <<>>) end - def surround_many(left, docs, right, limit, fun, sep) do - surround(left, surround_many(docs, limit, fun, sep), right) + # Type representing the document mode to be rendered: + # + # * flat - represents a document with breaks as flats (a break may fit, as it may break) + # * break - represents a document with breaks as breaks (a break always fits, since it breaks) + # + # These other two modes only affect fitting: + # + # * flat_no_break - represents a document with breaks as flat not allowed to enter in break mode + # * break_no_flat - represents a document with breaks as breaks not allowed to enter in flat mode + # + @typep mode :: :flat | :flat_no_break | :break | :break_no_flat + + @spec fits?( + width :: non_neg_integer() | :infinity, + column :: non_neg_integer(), + break? :: boolean(), + entries + ) :: boolean() + when entries: + maybe_improper_list( + {integer(), mode(), t()} | :group_over, + {:tail, boolean(), entries} | [] + ) + + # We need at least a break to consider the document does not fit since a + # large document without breaks has no option but fitting its current line. + # + # In case we have groups and the group fits, we need to consider the group + # parent without the child breaks, hence {:tail, b?, t} below. + defp fits?(w, k, b?, _) when k > w and b?, do: false + defp fits?(_, _, _, []), do: true + defp fits?(w, k, _, {:tail, b?, t}), do: fits?(w, k, b?, t) + + ## Group over + # If we get to the end of the group and if fits, it is because + # something already broke elsewhere, so we can consider the group + # fits. This only appears when checking if a flex break and fitting. + + defp fits?(_w, _k, b?, [:group_over | _]), + do: b? + + ## Flat no break + + defp fits?(w, k, b?, [{i, _, doc_fits(x, :disabled)} | t]), + do: fits?(w, k, b?, [{i, :flat_no_break, x} | t]) + + defp fits?(w, k, b?, [{i, :flat_no_break, doc_fits(x, _)} | t]), + do: fits?(w, k, b?, [{i, :flat_no_break, x} | t]) + + defp fits?(w, k, b?, [{i, _, doc_group(x, :pessimistic)} | t]), + do: fits?(w, k, b?, [{i, :flat_no_break, x} | t]) + + defp fits?(w, k, b?, [{i, :flat_no_break, doc_group(x, _)} | t]), + do: fits?(w, k, b?, [{i, :flat_no_break, x} | t]) + + ## Breaks no flat + + defp fits?(w, k, b?, [{i, _, doc_fits(x, :enabled)} | t]), + do: fits?(w, k, b?, [{i, :break_no_flat, x} | t]) + + defp fits?(w, k, b?, [{i, _, doc_group(x, :optimistic)} | t]), + do: fits?(w, k, b?, [{i, :break_no_flat, x} | t]) + + defp fits?(w, k, b?, [{i, :break_no_flat, doc_force(x)} | t]), + do: fits?(w, k, b?, [{i, :break_no_flat, x} | t]) + + defp fits?(_, _, _, [{_, :break_no_flat, doc_break(_, _)} | _]), do: true + defp fits?(_, _, _, [{_, :break_no_flat, doc_line()} | _]), do: true + + ## Breaks + + defp fits?(_, _, _, [{_, :break, doc_break(_, _)} | _]), do: true + defp fits?(_, _, _, [{_, :break, doc_line()} | _]), do: true + + defp fits?(w, k, b?, [{i, :break, doc_group(x, _)} | t]), + do: fits?(w, k, b?, [{i, :flat, x} | {:tail, b?, t}]) + + ## Catch all + + defp fits?(w, _, _, [{i, _, doc_line()} | t]), do: fits?(w, i, false, t) + defp fits?(w, k, b?, [{_, _, doc_nil()} | t]), do: fits?(w, k, b?, t) + defp fits?(w, _, b?, [{i, _, doc_collapse(_)} | t]), do: fits?(w, i, b?, t) + defp fits?(w, k, b?, [{i, m, doc_color(x, _)} | t]), do: fits?(w, k, b?, [{i, m, x} | t]) + defp fits?(w, k, b?, [{_, _, doc_string(_, l)} | t]), do: fits?(w, k + l, b?, t) + defp fits?(w, k, b?, [{_, _, s} | t]) when is_binary(s), do: fits?(w, k + byte_size(s), b?, t) + defp fits?(_, _, _, [{_, _, doc_force(_)} | _]), do: false + defp fits?(w, k, _, [{_, _, doc_break(s, _)} | t]), do: fits?(w, k + byte_size(s), true, t) + defp fits?(w, k, b?, [{i, m, doc_nest(x, _, :break)} | t]), do: fits?(w, k, b?, [{i, m, x} | t]) + + defp fits?(w, k, b?, [{i, m, doc_nest(x, j, _)} | t]), + do: fits?(w, k, b?, [{apply_nesting(i, k, j), m, x} | t]) + + defp fits?(w, k, b?, [{i, m, doc_cons(x, y)} | t]), + do: fits?(w, k, b?, [{i, m, x}, {i, m, y} | t]) + + defp fits?(w, k, b?, [{i, m, doc_group(x, _)} | t]), + do: fits?(w, k, b?, [{i, m, x} | {:tail, b?, t}]) + + defp fits?(w, k, b?, [{i, m, doc_limit(x, :infinity)} | t]) when w != :infinity, + do: fits?(:infinity, k, b?, [{i, :flat, x}, {i, m, doc_limit(empty(), w)} | t]) + + defp fits?(_w, k, b?, [{i, m, doc_limit(x, w)} | t]), + do: fits?(w, k, b?, [{i, m, x} | t]) + + @spec format( + width :: non_neg_integer() | :infinity, + column :: non_neg_integer(), + [{integer, mode, t} | :group_over], + binary + ) :: iodata + defp format(_, _, [], acc), do: acc + + defp format(w, k, [{_, _, doc_nil()} | t], acc), + do: format(w, k, t, acc) + + defp format(w, _, [{i, _, doc_line()} | t], acc), + do: format(w, i, t, <>) + + defp format(w, k, [{i, m, doc_cons(x, y)} | t], acc), + do: format(w, k, [{i, m, x}, {i, m, y} | t], acc) + + defp format(w, k, [{i, m, doc_color(x, c)} | t], acc), + do: format(w, k, [{i, m, x} | t], <>) + + defp format(w, k, [{_, _, doc_string(s, l)} | t], acc), + do: format(w, k + l, t, <>) + + defp format(w, k, [{_, _, s} | t], acc) when is_binary(s), + do: format(w, k + byte_size(s), t, <>) + + defp format(w, k, [{i, m, doc_force(x)} | t], acc), + do: format(w, k, [{i, m, x} | t], acc) + + defp format(w, k, [{i, m, doc_fits(x, _)} | t], acc), + do: format(w, k, [{i, m, x} | t], acc) + + defp format(w, _, [{i, _, doc_collapse(max)} | t], acc), + do: [acc | collapse(List.wrap(format(w, i, t, <<>>)), max, 0, i)] + + # Flex breaks are conditional to the document and the mode + defp format(w, k, [{i, m, doc_break(s, :flex)} | t], acc) do + k = k + byte_size(s) + + if w == :infinity or m == :flat or fits?(w, k, true, t) do + format(w, k, t, <>) + else + format(w, i, t, <>) + end end - defp surround_many(_, 0, _fun, _sep) do - "..." + # Strict breaks are conditional to the mode + defp format(w, k, [{i, mode, doc_break(s, :strict)} | t], acc) do + if mode == :break do + format(w, i, t, <>) + else + format(w, k + byte_size(s), t, <>) + end end - defp surround_many([h], _limit, fun, _sep) do - fun.(h) + # Nesting is conditional to the mode. + defp format(w, k, [{i, mode, doc_nest(x, j, nest)} | t], acc) do + if nest == :always or (nest == :break and mode == :break) do + format(w, k, [{apply_nesting(i, k, j), mode, x} | t], acc) + else + format(w, k, [{i, mode, x} | t], acc) + end end - defp surround_many([h|t], limit, fun, sep) when is_list(t) do - glue( - concat(fun.(h), sep), - surround_many(t, decrement(limit), fun, sep) - ) + # Groups must do the fitting decision. + defp format(w, k, [:group_over | t], acc) do + format(w, k, t, acc) end - defp surround_many([h|t], _limit, fun, _sep) do - glue( - concat(fun.(h), @tail_separator), - fun.(t) - ) + # TODO: Deprecate me in Elixir v1.23 + defp format(w, k, [{i, :break, doc_group(x, :inherit)} | t], acc) do + format(w, k, [{i, :break, x} | t], acc) end - defp decrement(:infinity), do: :infinity - defp decrement(counter), do: counter - 1 + defp format(w, k, [{i, :flat, doc_group(x, :optimistic)} | t], acc) do + if w == :infinity or fits?(w, k, false, [{i, :flat, x} | t]) do + format(w, k, [{i, :flat, x}, :group_over | t], acc) + else + format(w, k, [{i, :break, x}, :group_over | t], acc) + end + end - @doc """ - The pretty printing function. + defp format(w, k, [{i, _, doc_group(x, _)} | t], acc) do + if w == :infinity or fits?(w, k, false, [{i, :flat, x}]) do + format(w, k, [{i, :flat, x}, :group_over | t], acc) + else + format(w, k, [{i, :break, x}, :group_over | t], acc) + end + end - Takes the maximum width and a document to print as its arguments - and returns the string representation of the best layout for the - document to fit in the given width. - """ - @spec pretty(t, non_neg_integer | :infinity) :: binary - def pretty(d, w) do - sdoc = format w, 0, [{0, default_mode(w), doc_group(d)}] - render(sdoc) + # Limit is set to infinity and then reverts + defp format(w, k, [{i, m, doc_limit(x, :infinity)} | t], acc) when w != :infinity do + format(:infinity, k, [{i, :flat, x}, {i, m, doc_limit(empty(), w)} | t], acc) end - defp default_mode(:infinity), do: :flat - defp default_mode(_), do: :break + defp format(_w, k, [{i, m, doc_limit(x, w)} | t], acc) do + format(w, k, [{i, m, x} | t], acc) + end - # Rendering and internal helpers + defp collapse(["\n" <> rest | t], max, count, i) do + collapse([strip_whitespace(rest) | t], max, count + 1, i) + end - # Record representing the document mode to be rendered: flat or broken - @typep mode :: :flat | :break + defp collapse(["" | t], max, count, i) do + collapse(t, max, count, i) + end - @doc false - @spec fits?(integer, [{integer, mode, t}]) :: boolean - def fits?(w, _) when w < 0, do: false - def fits?(_, []), do: true - def fits?(_, [{_, _, :doc_line} | _]), do: true - def fits?(w, [{_, _, :doc_nil} | t]), do: fits?(w, t) - def fits?(w, [{i, m, doc_cons(x, y)} | t]), do: fits?(w, [{i, m, x} | [{i, m, y} | t]]) - def fits?(w, [{i, m, doc_nest(x, j)} | t]), do: fits?(w, [{i + j, m, x} | t]) - def fits?(w, [{i, _, doc_group(x)} | t]), do: fits?(w, [{i, :flat, x} | t]) - def fits?(w, [{_, _, s} | t]) when is_binary(s), do: fits?((w - byte_size s), t) - def fits?(w, [{_, :flat, doc_break(s)} | t]), do: fits?((w - byte_size s), t) - def fits?(_, [{_, :break, doc_break(_)} | _]), do: true + defp collapse(t, max, count, i) do + [:binary.copy("\n", min(max, count)), :binary.copy(" ", i) | t] + end - @doc false - @spec format(integer | :infinity, integer, [{integer, mode, t}]) :: [binary] - def format(_, _, []), do: [] - def format(w, _, [{i, _, :doc_line} | t]), do: [indent(i) | format(w, i, t)] - def format(w, k, [{_, _, :doc_nil} | t]), do: format(w, k, t) - def format(w, k, [{i, m, doc_cons(x, y)} | t]), do: format(w, k, [{i, m, x} | [{i, m, y} | t]]) - def format(w, k, [{i, m, doc_nest(x, j)} | t]), do: format(w, k, [{i + j, m, x} | t]) - def format(w, k, [{i, m, doc_group(x)} | t]), do: format(w, k, [{i, m, x} | t]) - def format(w, k, [{_, _, s} | t]) when is_binary(s), do: [s | format(w, (k + byte_size s), t)] - def format(w, k, [{_, :flat, doc_break(s)} | t]), do: [s | format(w, (k + byte_size s), t)] - def format(w, k, [{i, :break, doc_break(s)} | t]) do - k = k + byte_size(s) + defp strip_whitespace(" " <> rest), do: strip_whitespace(rest) + defp strip_whitespace(rest), do: rest - if w == :infinity or fits?(w - k, t) do - [s | format(w, k, t)] - else - [indent(i) | format(w, i, t)] - end - end + defp apply_nesting(_, k, :cursor), do: k + defp apply_nesting(_, _, :reset), do: 0 + defp apply_nesting(i, _, j), do: i + j defp indent(0), do: @newline defp indent(i), do: @newline <> :binary.copy(" ", i) - - @doc false - @spec render([binary]) :: binary - def render(sdoc) do - IO.iodata_to_binary sdoc - end end diff --git a/lib/elixir/lib/inspect/error.ex b/lib/elixir/lib/inspect/error.ex new file mode 100644 index 00000000000..47a00213ad0 --- /dev/null +++ b/lib/elixir/lib/inspect/error.ex @@ -0,0 +1,90 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team + +defmodule Inspect.Error do + @moduledoc """ + Raised when a struct cannot be inspected. + """ + @enforce_keys [:exception_module, :exception_message, :stacktrace, :inspected_struct] + defexception @enforce_keys + + @impl true + def exception(arguments) when is_list(arguments) do + exception = Keyword.fetch!(arguments, :exception) + exception_module = exception.__struct__ + exception_message = Exception.message(exception) |> String.trim_trailing("\n") + stacktrace = Keyword.fetch!(arguments, :stacktrace) + inspected_struct = Keyword.fetch!(arguments, :inspected_struct) + + %Inspect.Error{ + exception_module: exception_module, + exception_message: exception_message, + stacktrace: stacktrace, + inspected_struct: inspected_struct + } + end + + @impl true + def message(%__MODULE__{ + exception_module: exception_module, + exception_message: exception_message, + inspected_struct: inspected_struct + }) do + ~s''' + got #{inspect(exception_module)} with message: + + """ + #{pad(exception_message, 4)} + """ + + while inspecting: + + #{pad(inspected_struct, 4)} + ''' + end + + @doc false + def pad(message, padding_length) + when is_binary(message) and is_integer(padding_length) and padding_length >= 0 do + padding = String.duplicate(" ", padding_length) + + message + |> String.split("\n") + |> Enum.map(fn + "" -> "\n" + line -> [padding, line, ?\n] + end) + |> IO.iodata_to_binary() + |> String.trim_trailing("\n") + end +end + +defimpl Inspect, for: Inspect.Error do + @impl true + def inspect(%{stacktrace: stacktrace} = inspect_error, _opts) do + message = Exception.message(inspect_error) + format_output(message, stacktrace) + end + + defp format_output(message, [_ | _] = stacktrace) do + stacktrace = Exception.format_stacktrace(stacktrace) + + """ + #Inspect.Error< + #{Inspect.Error.pad(message, 2)} + + Stacktrace: + + #{stacktrace} + >\ + """ + end + + defp format_output(message, []) do + """ + #Inspect.Error< + #{Inspect.Error.pad(message, 2)} + >\ + """ + end +end diff --git a/lib/elixir/lib/integer.ex b/lib/elixir/lib/integer.ex index 85177480fae..25b658cd546 100644 --- a/lib/elixir/lib/integer.ex +++ b/lib/elixir/lib/integer.ex @@ -1,78 +1,359 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + defmodule Integer do @moduledoc """ Functions for working with integers. + + Some functions that work on integers are found in `Kernel`: + + * `Kernel.abs/1` + * `Kernel.div/2` + * `Kernel.max/2` + * `Kernel.min/2` + * `Kernel.rem/2` + """ import Bitwise @doc """ - Determines if an integer is odd. + Determines if `integer` is odd. + + Returns `true` if the given `integer` is an odd number, + otherwise it returns `false`. + + Allowed in guard clauses. + + ## Examples + + iex> Integer.is_odd(5) + true + + iex> Integer.is_odd(6) + false + + iex> Integer.is_odd(-5) + true + + iex> Integer.is_odd(0) + false + + """ + defguard is_odd(integer) when is_integer(integer) and (integer &&& 1) == 1 + + @doc """ + Determines if an `integer` is even. + + Returns `true` if the given `integer` is an even number, + otherwise it returns `false`. + + Allowed in guard clauses. + + ## Examples + + iex> Integer.is_even(10) + true + + iex> Integer.is_even(5) + false + + iex> Integer.is_even(-10) + true + + iex> Integer.is_even(0) + true + + """ + defguard is_even(integer) when is_integer(integer) and (integer &&& 1) == 0 + + @doc """ + Computes `base` raised to power of `exponent`. + + Both `base` and `exponent` must be integers. + The exponent must be zero or positive. + + See `Float.pow/2` for exponentiation of negative + exponents as well as floats. + + ## Examples + + iex> Integer.pow(2, 0) + 1 + iex> Integer.pow(2, 1) + 2 + iex> Integer.pow(2, 10) + 1024 + iex> Integer.pow(2, 11) + 2048 + iex> Integer.pow(2, 64) + 0x10000000000000000 + + iex> Integer.pow(3, 4) + 81 + iex> Integer.pow(4, 3) + 64 + + iex> Integer.pow(-2, 3) + -8 + iex> Integer.pow(-2, 4) + 16 + + iex> Integer.pow(2, -2) + ** (ArithmeticError) bad argument in arithmetic expression + + """ + @doc since: "1.12.0" + @spec pow(integer, non_neg_integer) :: integer + def pow(base, exponent) when is_integer(base) and is_integer(exponent) do + if exponent < 0, do: :erlang.error(:badarith, [base, exponent]) + base ** exponent + end + + @doc """ + Computes the modulo remainder of an integer division. + + This function performs a [floored division](`floor_div/2`), which means that + the result will always have the sign of the `divisor`. + + Raises an `ArithmeticError` exception if one of the arguments is not an + integer, or when the `divisor` is `0`. + + ## Examples + + iex> Integer.mod(5, 2) + 1 + iex> Integer.mod(6, -4) + -2 + + """ + @doc since: "1.4.0" + @spec mod(integer, neg_integer | pos_integer) :: integer + def mod(dividend, divisor) do + remainder = rem(dividend, divisor) + + if remainder * divisor < 0 do + remainder + divisor + else + remainder + end + end + + @doc """ + Performs a floored integer division. + + Raises an `ArithmeticError` exception if one of the arguments is not an + integer, or when the `divisor` is `0`. + + This function performs a *floored* integer division, which means that + the result will always be rounded towards negative infinity. + + If you want to perform truncated integer division (rounding towards zero), + use `Kernel.div/2` instead. + + ## Examples + + iex> Integer.floor_div(5, 2) + 2 + iex> Integer.floor_div(6, -4) + -2 + iex> Integer.floor_div(-99, 2) + -50 + + """ + @doc since: "1.4.0" + @spec floor_div(integer, neg_integer | pos_integer) :: integer + def floor_div(dividend, divisor) do + if :erlang.xor(dividend < 0, divisor < 0) and rem(dividend, divisor) != 0 do + div(dividend, divisor) - 1 + else + div(dividend, divisor) + end + end + + @doc """ + Performs a ceiled integer division. + + Raises an `ArithmeticError` exception if one of the arguments is not an + integer, or when the `divisor` is `0`. + + This function performs a *ceiled* integer division, which means that + the result will always be rounded towards positive infinity. + + ## Examples + + iex> Integer.ceil_div(5, 2) + 3 + iex> Integer.ceil_div(6, -4) + -1 + iex> Integer.ceil_div(-99, 2) + -49 + + """ + @doc since: "1.20.0" + @spec ceil_div(integer, neg_integer | pos_integer) :: integer + def ceil_div(dividend, divisor) do + if not :erlang.xor(dividend < 0, divisor < 0) and rem(dividend, divisor) != 0 do + div(dividend, divisor) + 1 + else + div(dividend, divisor) + end + end + + @doc """ + Returns the ordered digits for the given `integer`. + + An optional `base` value may be provided representing the radix for the returned + digits. This one must be an integer >= 2. + + ## Examples + + iex> Integer.digits(123) + [1, 2, 3] + + iex> Integer.digits(170, 2) + [1, 0, 1, 0, 1, 0, 1, 0] + + iex> Integer.digits(-170, 2) + [-1, 0, -1, 0, -1, 0, -1, 0] - Returns `true` if `n` is an odd number, otherwise `false`. - Implemented as a macro so it is allowed in guard clauses. """ - defmacro odd?(n) do - quote do: (unquote(n) &&& 1) == 1 + @spec digits(integer, pos_integer) :: [integer, ...] + def digits(integer, base \\ 10) + when is_integer(integer) and is_integer(base) and base >= 2 do + case integer do + 0 -> [0] + _integer -> digits(integer, base, []) + end end + defp digits(0, _base, acc), do: acc + + defp digits(integer, base, acc), + do: digits(div(integer, base), base, [rem(integer, base) | acc]) + @doc """ - Determines if an integer is even. + Returns the integer represented by the ordered `digits`. + + An optional `base` value may be provided representing the radix for the `digits`. + Base has to be an integer greater than or equal to `2`. + + ## Examples + + iex> Integer.undigits([1, 2, 3]) + 123 + + iex> Integer.undigits([1, 4], 16) + 20 + + iex> Integer.undigits([]) + 0 - Returns `true` if `n` is an even number, otherwise `false`. - Implemented as a macro so it is allowed in guard clauses. """ - defmacro even?(n) do - quote do: (unquote(n) &&& 1) == 0 + @spec undigits([integer], pos_integer) :: integer + def undigits(digits, base \\ 10) when is_list(digits) and is_integer(base) and base >= 2 do + undigits(digits, base, 0) end + defp undigits([], _base, acc), do: acc + + defp undigits([digit | _], base, _) when is_integer(digit) and digit >= base, + do: raise(ArgumentError, "invalid digit #{digit} in base #{base}") + + defp undigits([digit | tail], base, acc) when is_integer(digit), + do: undigits(tail, base, acc * base + digit) + @doc """ - Converts a binary to an integer. + Parses a text representation of an integer. - If successful, returns a tuple of the form `{integer, remainder_of_binary}`. + An optional `base` to the corresponding integer can be provided. + If `base` is not given, 10 will be used. + + If successful, returns a tuple in the form of `{integer, remainder_of_binary}`. Otherwise `:error`. + Raises an error if `base` is less than 2 or more than 36. + + If you want to convert a string-formatted integer directly to an integer, + `String.to_integer/1` or `String.to_integer/2` can be used instead. + ## Examples iex> Integer.parse("34") - {34,""} + {34, ""} iex> Integer.parse("34.5") - {34,".5"} + {34, ".5"} iex> Integer.parse("three") :error + iex> Integer.parse("34", 10) + {34, ""} + + iex> Integer.parse("f4", 16) + {244, ""} + + iex> Integer.parse("Awww++", 36) + {509216, "++"} + + iex> Integer.parse("fab", 10) + :error + + iex> Integer.parse("a2", 38) + ** (ArgumentError) invalid base 38 + """ - @spec parse(binary) :: {integer, binary} | :error - def parse(<< ?-, bin :: binary >>) do - case do_parse(bin) do - :error -> :error - {number, remainder} -> {-number, remainder} + @spec parse(binary, 2..36) :: {integer, remainder_of_binary :: binary} | :error + def parse(binary, base \\ 10) + + def parse(_binary, base) when base not in 2..36 do + raise ArgumentError, "invalid base #{inspect(base)}" + end + + def parse(binary, base) when is_binary(binary) do + case count_digits(binary, base) do + 0 -> + :error + + count -> + {digits, rem} = :erlang.split_binary(binary, count) + {:erlang.binary_to_integer(digits, base), rem} end end - def parse(<< ?+, bin :: binary >>) do - do_parse(bin) + defp count_digits(<>, base) when sign in ~c"+-" do + case count_digits_nosign(rest, base, 1) do + 1 -> 0 + count -> count + end end - def parse(bin) when is_binary(bin) do - do_parse(bin) + defp count_digits(<>, base) do + count_digits_nosign(rest, base, 0) end - defp do_parse(<< char, bin :: binary >>) when char in ?0..?9, do: do_parse(bin, char - ?0) - defp do_parse(_), do: :error + digits = [{?0..?9, -?0}, {?A..?Z, 10 - ?A}, {?a..?z, 10 - ?a}] - defp do_parse(<< char, rest :: binary >>, acc) when char in ?0..?9 do - do_parse rest, 10 * acc + (char - ?0) - end + for {chars, diff} <- digits, + char <- chars do + digit = char + diff - defp do_parse(bitstring, acc) do - {acc, bitstring} + defp count_digits_nosign(<>, base, count) + when base > unquote(digit) do + count_digits_nosign(rest, base, count + 1) + end end + defp count_digits_nosign(<<_::bits>>, _, count), do: count + @doc """ Returns a binary which corresponds to the text representation - of `some_integer`. + of `integer` in the given `base`. + + `base` can be an integer between 2 and 36. If no `base` is given, + it defaults to `10`. Inlined by the compiler. @@ -81,59 +362,164 @@ defmodule Integer do iex> Integer.to_string(123) "123" + iex> Integer.to_string(+456) + "456" + + iex> Integer.to_string(-789) + "-789" + + iex> Integer.to_string(0123) + "123" + + iex> Integer.to_string(100, 16) + "64" + + iex> Integer.to_string(-100, 16) + "-64" + + iex> Integer.to_string(882_681_651, 36) + "ELIXIR" + """ - @spec to_string(integer) :: String.t - def to_string(some_integer) do - :erlang.integer_to_binary(some_integer) + @spec to_string(integer, 2..36) :: String.t() + def to_string(integer, base \\ 10) do + :erlang.integer_to_binary(integer, base) end @doc """ - Returns a binary which corresponds to the text representation - of `some_integer` in base `base`. + Returns a charlist which corresponds to the text representation + of `integer` in the given `base`. + + `base` can be an integer between 2 and 36. If no `base` is given, + it defaults to `10`. Inlined by the compiler. ## Examples - iex> Integer.to_string(100, 16) - "64" + iex> Integer.to_charlist(123) + ~c"123" + + iex> Integer.to_charlist(+456) + ~c"456" + + iex> Integer.to_charlist(-789) + ~c"-789" + + iex> Integer.to_charlist(0123) + ~c"123" + + iex> Integer.to_charlist(100, 16) + ~c"64" + + iex> Integer.to_charlist(-100, 16) + ~c"-64" + + iex> Integer.to_charlist(882_681_651, 36) + ~c"ELIXIR" """ - @spec to_string(integer, pos_integer) :: String.t - def to_string(some_integer, base) do - :erlang.integer_to_binary(some_integer, base) + @spec to_charlist(integer, 2..36) :: charlist + def to_charlist(integer, base \\ 10) do + :erlang.integer_to_list(integer, base) end @doc """ - Returns a char list which corresponds to the text representation of the given integer. + Returns the greatest common divisor of the two given integers. - Inlined by the compiler. + The greatest common divisor (GCD) of `integer1` and `integer2` is the largest positive + integer that divides both `integer1` and `integer2` without leaving a remainder. + + By convention, `gcd(0, 0)` returns `0`. ## Examples - iex> Integer.to_char_list(7) - '7' + iex> Integer.gcd(2, 3) + 1 + + iex> Integer.gcd(8, 12) + 4 + + iex> Integer.gcd(8, -12) + 4 + + iex> Integer.gcd(10, 0) + 10 + + iex> Integer.gcd(7, 7) + 7 + + iex> Integer.gcd(0, 0) + 0 """ - @spec to_char_list(integer) :: list - def to_char_list(number) do - :erlang.integer_to_list(number) + @doc since: "1.5.0" + @spec gcd(integer, integer) :: non_neg_integer + def gcd(integer1, integer2) when is_integer(integer1) and is_integer(integer2) do + gcd_positive(abs(integer1), abs(integer2)) end + defp gcd_positive(0, integer2), do: integer2 + defp gcd_positive(integer1, 0), do: integer1 + defp gcd_positive(integer1, integer2), do: gcd_positive(integer2, rem(integer1, integer2)) + @doc """ - Returns a char list which corresponds to the text representation of the - given integer in the given case. + Returns the extended greatest common divisor of the two given integers. - Inlined by the compiler. + This function uses the extended Euclidean algorithm to return a three-element tuple with the `gcd` + and the coefficients `m` and `n` of Bézout's identity such that: + + gcd(a, b) = m*a + n*b + + By convention, `extended_gcd(0, 0)` returns `{0, 0, 0}`. ## Examples - iex> Integer.to_char_list(1023, 16) - '3FF' + iex> Integer.extended_gcd(240, 46) + {2, -9, 47} + iex> Integer.extended_gcd(46, 240) + {2, 47, -9} + iex> Integer.extended_gcd(-46, 240) + {2, -47, -9} + iex> Integer.extended_gcd(-46, -240) + {2, -47, 9} + + iex> Integer.extended_gcd(14, 21) + {7, -1, 1} + + iex> Integer.extended_gcd(10, 0) + {10, 1, 0} + iex> Integer.extended_gcd(0, 10) + {10, 0, 1} + iex> Integer.extended_gcd(0, 0) + {0, 0, 0} """ - @spec to_char_list(integer, pos_integer) :: list - def to_char_list(number, base) do - :erlang.integer_to_list(number, base) + @doc since: "1.12.0" + @spec extended_gcd(integer, integer) :: {non_neg_integer, integer, integer} + def extended_gcd(0, 0), do: {0, 0, 0} + def extended_gcd(0, b), do: {b, 0, 1} + def extended_gcd(a, 0), do: {a, 1, 0} + + def extended_gcd(integer1, integer2) when is_integer(integer1) and is_integer(integer2) do + extended_gcd(integer2, integer1, 0, 1, 1, 0) end + + defp extended_gcd(r1, r0, s1, s0, t1, t0) do + div = div(r0, r1) + + case r0 - div * r1 do + 0 when r1 > 0 -> {r1, s1, t1} + 0 when r1 < 0 -> {-r1, -s1, -t1} + r2 -> extended_gcd(r2, r1, s0 - div * s1, s1, t0 - div * t1, t1) + end + end + + @doc false + @deprecated "Use Integer.to_charlist/1 instead" + def to_char_list(integer), do: Integer.to_charlist(integer) + + @doc false + @deprecated "Use Integer.to_charlist/2 instead" + def to_char_list(integer, base), do: Integer.to_charlist(integer, base) end diff --git a/lib/elixir/lib/io.ex b/lib/elixir/lib/io.ex index 947b6ffa61f..24804339c5f 100644 --- a/lib/elixir/lib/io.ex +++ b/lib/elixir/lib/io.ex @@ -1,84 +1,247 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + defmodule IO do - @moduledoc """ - Functions handling IO. + @moduledoc ~S""" + Functions handling input/output (IO). - Many functions in this module expects an IO device as argument. - An IO device must be a pid or an atom representing a process. + Many functions in this module expect an IO device as an argument. + An IO device must be a PID or an atom representing a process. For convenience, Elixir provides `:stdio` and `:stderr` as shortcuts to Erlang's `:standard_io` and `:standard_error`. - The majority of the functions expect char data, i.e. strings or - lists of characters and strings. In case another type is given, - it will do a conversion to string via the `String.Chars` protocol - (as shown in typespecs). + The majority of the functions expect chardata. In case another type is given, + functions will convert those types to string via the `String.Chars` protocol + (as shown in typespecs). For more information on chardata, see the + "IO data" section below. - The functions starting with `bin*` expects iodata as argument, - i.e. binaries or lists of bytes and binaries. + The functions of this module use UNIX-style naming where possible. ## IO devices - An IO device may be an atom or a pid. In case it is an atom, - the atom must be the name of a registered process. However, - there are three exceptions for this rule: + An IO device may be an atom or a PID. In case it is an atom, + the atom must be the name of a registered process. In addition, + Elixir provides two shortcuts: + + * `:stdio` - a shortcut for `:standard_io`, which maps to + the current `Process.group_leader/0` in Erlang + + * `:stderr` - a shortcut for the named process `:standard_error` + provided in Erlang + + IO devices maintain their position, which means subsequent calls to any + reading or writing functions will start from the place where the device + was last accessed. The position of files can be changed using the + `:file.position/2` function. + + ## IO data + + IO data is a data type that can be used as a more efficient alternative to binaries + in certain situations. + + A term of type **IO data** is a binary or a list containing bytes (integers within the `0..255` range) + or nested IO data. The type is recursive. Let's see an example of one of + the possible IO data representing the binary `"hello"`: + + [?h, "el", ["l", [?o]]] + + The built-in `t:iodata/0` type is defined in terms of `t:iolist/0`. An IO list is + the same as IO data but it doesn't allow for a binary at the top level (but binaries + are still allowed in the list itself). + + ### Use cases for IO data + + IO data exists because often you need to do many append operations + on smaller chunks of binaries in order to create a bigger binary. However, in + Erlang and Elixir concatenating binaries will copy the concatenated binaries + into a new binary. + + def email(username, domain) do + username <> "@" <> domain + end + + In this function, creating the email address will copy the `username` and `domain` + binaries. Now imagine you want to use the resulting email inside another binary: + + def welcome_message(name, username, domain) do + "Welcome #{name}, your email is: #{email(username, domain)}" + end + + IO.puts(welcome_message("Meg", "meg", "example.com")) + #=> "Welcome Meg, your email is: meg@example.com" + + Every time you concatenate binaries or use interpolation (`#{}`) you are making + copies of those binaries. However, in many cases you don't need the complete + binary while you create it, but only at the end to print it out or send it + somewhere. In such cases, you can construct the binary by creating IO data: + + def email(username, domain) do + [username, ?@, domain] + end + + def welcome_message(name, username, domain) do + ["Welcome ", name, ", your email is: ", email(username, domain)] + end + + IO.puts(welcome_message("Meg", "meg", "example.com")) + #=> "Welcome Meg, your email is: meg@example.com" + + Building IO data is cheaper than concatenating binaries. Concatenating multiple + pieces of IO data just means putting them together inside a list since IO data + can be arbitrarily nested, and that's a cheap and efficient operation. Most of + the IO-based APIs, such as `:gen_tcp` and `IO`, receive IO data and write it + to the socket directly without converting it to binary. + + One drawback of IO data is that you can't do things like pattern match on the + first part of a piece of IO data like you can with a binary, because you usually + don't know the shape of the IO data. In those cases, you may need to convert it + to a binary by calling `iodata_to_binary/1`, which is reasonably efficient + since it's implemented natively in C. Other functionality, like computing the + length of IO data, can be computed directly on the iodata by calling `iodata_length/1`. + + ### Chardata + + Erlang and Elixir also have the idea of `t:chardata/0`. Chardata is very + similar to IO data: the only difference is that integers in IO data represent + bytes while integers in chardata represent Unicode code points. Bytes + (`t:byte/0`) are integers within the `0..255` range, while Unicode code points + (`t:char/0`) are integers within the `0..0x10FFFF` range. The `IO` module provides + the `chardata_to_string/1` function for chardata as the "counter-part" of the + `iodata_to_binary/1` function for IO data. + + If you try to use `iodata_to_binary/1` on chardata, it will result in an + argument error. For example, let's try to put a code point that is not + representable with one byte, like `?π`, inside IO data: - * `:standard_io` - when the `:standard_io` atom is given, - it is treated as a shortcut for `Process.group_leader` + IO.iodata_to_binary(["The symbol for pi is: ", ?π]) + #=> ** (ArgumentError) argument error - * `:stdio` - is a shortcut for `:standard_io` + If we use chardata instead, it will work as expected: - * `:stderr` - is a shortcut for `:standard_error` + iex> IO.chardata_to_string(["The symbol for pi is: ", ?π]) + "The symbol for pi is: π" """ @type device :: atom | pid @type nodata :: {:error, term} | :eof - @type chardata() :: :unicode.chardata() + @type chardata :: String.t() | maybe_improper_list(char | chardata, String.t() | []) - import :erlang, only: [group_leader: 0] + @type inspect_opts :: [Inspect.Opts.new_opt() | {:label, term}] - defmacrop is_iodata(data) do - quote do - is_list(unquote(data)) or is_binary(unquote(data)) - end - end + @typedoc """ + Stacktrace information as keyword options for `warn/2`. - @doc """ - Reads `count` characters from the IO device or until - the end of the line if `:line` is given. It returns: + At least `:file` is required. Other options are optional and used + to provide more precise location information. + """ + @type warn_stacktrace_opts :: [ + file: String.t(), + line: pos_integer(), + column: pos_integer(), + module: module(), + function: {atom(), arity()} + ] - * `data` - the input characters + defguardp is_device(term) when is_atom(term) or is_pid(term) + defguardp is_iodata(data) when is_list(data) or is_binary(data) + + @doc ~S""" + Reads from the IO `device`. + + The `device` is iterated as specified by the `line_or_chars` argument: + + * if `line_or_chars` is an integer, it represents a number of bytes. The device is + iterated by that number of bytes. This should be the preferred mode for reading + non-textual inputs. + + * if `line_or_chars` is `:line`, the device is iterated line by line. + CRLF newlines ("\r\n") are automatically normalized to "\n". + + * if `line_or_chars` is `:eof` (since v1.13), the device is iterated until `:eof`. + If the device is already at the end, it returns `:eof` itself. + + It returns: + + * `data` - the output characters * `:eof` - end of file was encountered * `{:error, reason}` - other (rare) error condition; for instance, `{:error, :estale}` if reading from an NFS volume + """ - @spec read(device, :line | non_neg_integer) :: chardata | nodata - def read(device \\ group_leader, chars_or_line) + @spec read(device, :eof | :line | non_neg_integer) :: chardata | nodata + def read(device \\ :stdio, line_or_chars) + + # TODO: Remove me on v2.0 + def read(device, :all) do + IO.warn("IO.read(device, :all) is deprecated, use IO.read(device, :eof) instead") + + with :eof <- read(device, :eof) do + with [_ | _] = opts <- :io.getopts(device), + false <- Keyword.get(opts, :binary, true) do + ~c"" + else + _ -> "" + end + end + end + + def read(device, :eof) do + getn(device, ~c"", :eof) + end def read(device, :line) do - :io.get_line(map_dev(device), '') + :io.get_line(map_dev(device), ~c"") end - def read(device, count) when count >= 0 do - :io.get_chars(map_dev(device), '', count) + def read(device, count) when is_integer(count) and count >= 0 do + :io.get_chars(map_dev(device), ~c"", count) end - @doc """ - Reads `count` bytes from the IO device or until - the end of the line if `:line` is given. It returns: + @doc ~S""" + Reads from the IO `device`. The operation is Unicode unsafe. - * `data` - the input characters + The `device` is iterated as specified by the `line_or_chars` argument: + + * if `line_or_chars` is an integer, it represents a number of bytes. The device is + iterated by that number of bytes. This should be the preferred mode for reading + non-textual inputs. + + * if `line_or_chars` is `:line`, the device is iterated line by line. + CRLF newlines ("\r\n") are automatically normalized to "\n". + + * if `line_or_chars` is `:eof` (since v1.13), the device is iterated until `:eof`. + If the device is already at the end, it returns `:eof` itself. + + It returns: + + * `data` - the output bytes * `:eof` - end of file was encountered * `{:error, reason}` - other (rare) error condition; for instance, `{:error, :estale}` if reading from an NFS volume + + Note: do not use this function on IO devices in Unicode mode + as it will return the wrong result. """ - @spec binread(device, :line | non_neg_integer) :: iodata | nodata - def binread(device \\ group_leader, chars_or_line) + @spec binread(device, :eof | :line | non_neg_integer) :: iodata | nodata + def binread(device \\ :stdio, line_or_chars) + + # TODO: Remove me on v2.0 + def binread(device, :all) do + IO.warn("IO.binread(device, :all) is deprecated, use IO.binread(device, :eof) instead") + with :eof <- binread(device, :eof), do: "" + end + + def binread(device, :eof) do + binread_eof(map_dev(device), "") + end def binread(device, :line) do case :file.read_line(map_dev(device)) do @@ -87,91 +250,311 @@ defmodule IO do end end - def binread(device, count) when count >= 0 do + def binread(device, count) when is_integer(count) and count >= 0 do case :file.read(map_dev(device), count) do {:ok, data} -> data other -> other end end + @read_all_size 4096 + defp binread_eof(mapped_dev, acc) do + case :file.read(mapped_dev, @read_all_size) do + {:ok, data} -> binread_eof(mapped_dev, acc <> data) + :eof -> if acc == "", do: :eof, else: acc + other -> other + end + end + @doc """ - Writes the given argument to the given device. + Writes `chardata` to the given `device`. - By default the device is the standard output. - It returns `:ok` if it succeeds. + By default, the `device` is the standard output. + + ## Examples + + IO.write("sample") + #=> sample + + IO.write(:stderr, "error") + #=> error + + """ + @spec write(device, chardata | String.Chars.t()) :: :ok + def write(device \\ :stdio, chardata) do + :io.put_chars(map_dev(device), to_chardata(chardata)) + end + + @doc """ + Writes `iodata` to the given `device`. + + This operation is meant to be used with "raw" devices + that are started without an encoding. The given `iodata` + is written as is to the device, without conversion. For + more information on IO data, see the "IO data" section in + the module documentation. + + Use `write/2` for devices with encoding. + + Important: do **not** use this function on IO devices in + Unicode mode as it will write the wrong data. In particular, + the standard IO device is set to Unicode by default, so writing + to stdio with this function will likely result in the wrong data + being sent down the wire. + """ + @spec binwrite(device, iodata) :: :ok + def binwrite(device \\ :stdio, iodata) when is_iodata(iodata) do + with {:error, reason} <- :file.write(map_dev(device), iodata) do + :erlang.error(reason) + end + end + + @doc """ + Writes `item` to the given `device`, similar to `write/2`, + but adds a newline at the end. + + By default, the `device` is the standard output. It returns `:ok` + if it succeeds. + + Trivia: `puts` is shorthand for `put string`. ## Examples - IO.write "sample" - #=> "sample" + IO.puts("Hello World!") + #=> Hello World! - IO.write :stderr, "error" - #=> "error" + IO.puts(:stderr, "error") + #=> error """ - @spec write(device, chardata | String.Chars.t) :: :ok - def write(device \\ group_leader(), item) do - :io.put_chars map_dev(device), to_chardata(item) + @spec puts(device, chardata | String.Chars.t()) :: :ok + def puts(device \\ :stdio, item) when is_device(device) do + :io.put_chars(map_dev(device), [to_chardata(item), ?\n]) end @doc """ - Writes the given argument to the given device - as a binary, no unicode conversion happens. + Writes a `message` to stderr, along with the given `stacktrace_info`. + + The `stacktrace_info` must be one of: + + * a `__STACKTRACE__`, where all entries in the stacktrace will be + included in the error message + + * a `Macro.Env` structure (since v1.14.0), where a single stacktrace + entry from the compilation environment will be used + + * a keyword list with at least the `:file` option representing + a single stacktrace entry (since v1.14.0). The `:line`, `:column`, + `:module`, and `:function` options are also supported + + This function notifies the compiler a warning was printed + and emits a compiler diagnostic (`t:Code.diagnostic/1`). + The diagnostic will include precise file and location information + if a `Macro.Env` is given or those values have been passed as + keyword list, but not for stacktraces, as they are often imprecise. + + It returns `:ok` if it succeeds. + + ## Examples + + IO.warn("variable bar is unused", module: MyApp, function: {:main, 1}, line: 4, file: "my_app.ex") + #=> warning: variable bar is unused + #=> my_app.ex:4: MyApp.main/1 - Check `write/2` for more information. """ - @spec binwrite(device, iodata) :: :ok | {:error, term} - def binwrite(device \\ group_leader(), item) when is_iodata(item) do - :file.write map_dev(device), item + @spec warn( + chardata | String.Chars.t(), + Exception.stacktrace() | warn_stacktrace_opts() | Macro.Env.t() + ) :: + :ok + def warn(message, stacktrace_info) + + def warn(message, %Macro.Env{line: line, file: file} = env) do + message = to_chardata(message) + + :elixir_errors.emit_diagnostic(:warning, line, file, message, Macro.Env.stacktrace(env), + read_snippet: true + ) + end + + def warn(message, [{_, _} | _] = keyword) do + if file = keyword[:file] do + line = keyword[:line] + column = keyword[:column] + position = if line && column, do: {line, column}, else: line + message = to_chardata(message) + + stacktrace = + Macro.Env.stacktrace(%{ + __ENV__ + | module: keyword[:module], + function: keyword[:function], + line: line, + file: file + }) + + :elixir_errors.emit_diagnostic(:warning, position, file, message, stacktrace, + read_snippet: true + ) + else + warn(message, []) + end + end + + def warn(message, []) do + message = to_chardata(message) + :elixir_errors.emit_diagnostic(:warning, 0, nil, message, [], read_snippet: false) + end + + def warn(message, [{_, _, _, _} | _] = stacktrace) do + message = to_chardata(message) + :elixir_errors.emit_diagnostic(:warning, 0, nil, message, stacktrace, read_snippet: false) + end + + @doc false + def warn_once(key, message, stacktrace_drop_levels) do + {:current_stacktrace, stacktrace} = Process.info(self(), :current_stacktrace) + stacktrace = Enum.drop(stacktrace, stacktrace_drop_levels) + + if :elixir_config.warn(key, stacktrace) do + warn(message.(), stacktrace) + else + :ok + end end @doc """ - Writes the argument to the device, similar to `write/2`, - but adds a newline at the end. The argument is expected - to be a chardata. + Writes a `message` to stderr, along with the current stacktrace. + + It returns `:ok` if it succeeds. + + Do not call this function at the tail of another function. Due to tail + call optimization, a stacktrace entry would not be added and the + stacktrace would be incorrectly trimmed. Therefore make sure at least + one expression (or an atom such as `:ok`) follows the `IO.warn/1` call. + + ## Examples + + IO.warn("variable bar is unused") + #=> warning: variable bar is unused + #=> (iex) evaluator.ex:108: IEx.Evaluator.eval/4 + """ - @spec puts(device, chardata | String.Chars.t) :: :ok - def puts(device \\ group_leader(), item) do - erl_dev = map_dev(device) - :io.put_chars erl_dev, [to_chardata(item), ?\n] + @spec warn(chardata | String.Chars.t()) :: :ok + def warn(message) do + {:current_stacktrace, stacktrace} = Process.info(self(), :current_stacktrace) + warn(message, Enum.drop(stacktrace, 2)) end @doc """ - Inspects and writes the given argument to the device. + Inspects and writes the given `item` to the standard output. + + It's important to note that it returns the given `item` unchanged. + This makes it possible to "spy" on values by inserting an + `IO.inspect/2` call almost anywhere in your code, for example, + in the middle of a pipeline. - It sets by default pretty printing to true and returns - the item itself. + It enables pretty printing by default with width of + 80 characters. The width can be changed by explicitly + passing the `:width` option. - Note this function does not use the IO device width - because some IO devices does not implement the - appropriate functions. Setting the width must be done - explicitly by passing the `:width` option. + The output can be decorated with a label, by providing the `:label` + option to easily distinguish it from other `IO.inspect/2` calls. + The label will be printed before the inspected `item`. + + See `Inspect.Opts` for a full list of remaining formatting options. + To print to other IO devices, see `IO.inspect/3` ## Examples - IO.inspect Process.list + The following code: + + IO.inspect(<<0, 1, 2>>, width: 40) + + Prints: + + <<0, 1, 2>> + + You can use the `:label` option to decorate the output: + + IO.inspect(1..100, label: "a wonderful range") + + Prints: + + a wonderful range: 1..100 + + Inspect truncates large inputs by default. The `:printable_limit` controls + the limit for strings and other string-like constructs (such as charlists): + + "abc" + |> String.duplicate(9001) + |> IO.inspect(printable_limit: :infinity) + + For containers such as lists, maps, and tuples, the number of entries + is managed by the `:limit` option: + + 1..100 + |> Enum.map(& {&1, &1}) + |> Enum.into(%{}) + |> IO.inspect(limit: :infinity) """ - @spec inspect(term, Keyword.t) :: term + @spec inspect(item, inspect_opts) :: item when item: var def inspect(item, opts \\ []) do - inspect group_leader(), item, opts + inspect(:stdio, item, opts) end @doc """ - Inspects the item with options using the given device. + Inspects `item` according to the given options using the IO `device`. + + See `inspect/2` for a full list of options. """ - @spec inspect(device, term, Keyword.t) :: term - def inspect(device, item, opts) when is_list(opts) do - opts = Keyword.put_new(opts, :pretty, true) - puts device, Kernel.inspect(item, opts) + @spec inspect(device, item, inspect_opts) :: item when item: var + def inspect(device, item, opts) when is_device(device) and is_list(opts) do + {label, opts} = Keyword.pop(opts, :label) + label = if label, do: [to_chardata(label), ": "], else: [] + opts = Inspect.Opts.new(opts) + doc = Inspect.Algebra.group(Inspect.Algebra.to_doc(item, opts)) + chardata = Inspect.Algebra.format(doc, opts.width) + puts(device, [label, chardata]) item end @doc """ - Gets a number of bytes from the io device. If the - io device is a unicode device, `count` implies - the number of unicode codepoints to be retrieved. + Gets a number of bytes from IO device `:stdio`. + + If `:stdio` is a Unicode device, `count` implies + the number of Unicode code points to be retrieved. Otherwise, `count` is the number of raw bytes to be retrieved. + + See `IO.getn/3` for a description of return values. + """ + @spec getn( + device | chardata | String.Chars.t(), + pos_integer | :eof | chardata | String.Chars.t() + ) :: + chardata | nodata + def getn(prompt, count \\ 1) + + def getn(prompt, :eof) do + getn(:stdio, prompt, :eof) + end + + def getn(prompt, count) when is_integer(count) and count > 0 do + getn(:stdio, prompt, count) + end + + def getn(device, prompt) when not is_integer(prompt) do + getn(device, prompt, 1) + end + + @doc """ + Gets a number of bytes from the IO `device`. + + If the IO `device` is a Unicode device, `count` implies + the number of Unicode code points to be retrieved. + Otherwise, `count` is the number of raw bytes to be retrieved. + It returns: * `data` - the input characters @@ -181,100 +564,161 @@ defmodule IO do * `{:error, reason}` - other (rare) error condition; for instance, `{:error, :estale}` if reading from an NFS volume - """ - @spec getn(chardata | String.Chars.t, pos_integer) :: chardata | nodata - @spec getn(device, chardata | String.Chars.t) :: chardata | nodata - def getn(prompt, count \\ 1) - def getn(prompt, count) when is_integer(count) do - getn(group_leader, prompt, count) + """ + @spec getn(device, chardata | String.Chars.t(), pos_integer | :eof) :: chardata | nodata + def getn(device, prompt, :eof) do + getn_eof(map_dev(device), to_chardata(prompt), []) end - def getn(device, prompt) do - getn(device, prompt, 1) + def getn(device, prompt, count) when is_integer(count) and count > 0 do + :io.get_chars(map_dev(device), to_chardata(prompt), count) end - @doc """ - Gets a number of bytes from the io device. If the - io device is a unicode device, `count` implies - the number of unicode codepoints to be retrieved. - Otherwise, `count` is the number of raw bytes to be retrieved. - """ - @spec getn(device, chardata | String.Chars.t, pos_integer) :: chardata | nodata - def getn(device, prompt, count) do - :io.get_chars(map_dev(device), to_chardata(prompt), count) + defp getn_eof(device, prompt, acc) do + case :io.get_line(device, prompt) do + line when is_binary(line) or is_list(line) -> getn_eof(device, ~c"", [line | acc]) + :eof -> wrap_eof(:lists.reverse(acc)) + other -> other + end end - @doc """ - Reads a line from the IO device. It returns: + defp wrap_eof([h | _] = acc) when is_binary(h), do: IO.iodata_to_binary(acc) + defp wrap_eof([h | _] = acc) when is_list(h), do: :lists.flatten(acc) + defp wrap_eof([]), do: :eof + + @doc ~S""" + Reads a line from the IO `device`. + + It returns: * `data` - the characters in the line terminated - by a LF (or end of file) + by a line-feed (LF) or end of file (EOF) * `:eof` - end of file was encountered * `{:error, reason}` - other (rare) error condition; for instance, `{:error, :estale}` if reading from an NFS volume + + Trivia: `gets` is shorthand for `get string`. + + ## Examples + + To display "What is your name?" as a prompt and await user input: + + IO.gets("What is your name?\n") + """ - @spec gets(device, chardata | String.Chars.t) :: chardata | nodata - def gets(device \\ group_leader(), prompt) do + @spec gets(device, chardata | String.Chars.t()) :: chardata | nodata + def gets(device \\ :stdio, prompt) do :io.get_line(map_dev(device), to_chardata(prompt)) end @doc """ - Converts the io device into a `IO.Stream`. + Returns a line-based `IO.Stream` on `:stdio`. + + This is equivalent to: + + IO.stream(:stdio, :line) + + """ + @doc since: "1.12.0" + @spec stream() :: Enumerable.t(String.t()) + def stream, do: stream(:stdio, :line) + + @doc ~S""" + Converts the IO `device` into an `IO.Stream`. An `IO.Stream` implements both `Enumerable` and `Collectable`, allowing it to be used for both read and write. - The device is iterated line by line if `:line` is given or - by a given number of codepoints. + The `device` is iterated by the given number of characters + or line by line if `:line` is given. In case `:line` is given, + "\r\n" is automatically normalized to "\n". - This reads the IO as utf-8. Check out + This reads from the IO as UTF-8. Check out `IO.binstream/2` to handle the IO as a raw binary. Note that an IO stream has side effects and every time you go over the stream you may get different results. + `stream/0` has been introduced in Elixir v1.12.0, + while `stream/2` has been available since v1.0.0. + ## Examples Here is an example on how we mimic an echo server from the command line: - Enum.each IO.stream(:stdio, :line), &IO.write(&1) + Enum.each(IO.stream(:stdio, :line), &IO.write(&1)) + + Another example where you might want to collect a user input + every new line and break on an empty line, followed by removing + redundant new line characters (`"\n"`): + + IO.stream(:stdio, :line) + |> Enum.take_while(&(&1 != "\n")) + |> Enum.map(&String.replace(&1, "\n", "")) """ - @spec stream(device, :line | pos_integer) :: Enumerable.t - def stream(device, line_or_codepoints) do + @spec stream(device, :line | pos_integer) :: Enumerable.t() + def stream(device \\ :stdio, line_or_codepoints) + when line_or_codepoints == :line + when is_integer(line_or_codepoints) and line_or_codepoints > 0 do IO.Stream.__build__(map_dev(device), false, line_or_codepoints) end @doc """ - Converts the IO device into a `IO.Stream`. + Returns a raw, line-based `IO.Stream` on `:stdio`. The operation is Unicode unsafe. + + This is equivalent to: + + IO.binstream(:stdio, :line) + + """ + @doc since: "1.12.0" + @spec binstream() :: Enumerable.t(binary) + def binstream, do: binstream(:stdio, :line) + + @doc ~S""" + Converts the IO `device` into an `IO.Stream`. The operation is Unicode unsafe. An `IO.Stream` implements both `Enumerable` and `Collectable`, allowing it to be used for both read and write. - The device is iterated line by line or by a number of bytes. - This reads the IO device as a raw binary. + The `device` is iterated by the given number of bytes or line + by line if `:line` is given. In case `:line` is given, "\r\n" + is automatically normalized to "\n". Passing the number of bytes + should be the preferred mode for reading non-textual inputs. Note that an IO stream has side effects and every time you go over the stream you may get different results. + + This reads from the IO device as a raw binary. Therefore, + do not use this function on IO devices in Unicode mode as + it will return the wrong result. + + `binstream/0` has been introduced in Elixir v1.12.0, + while `binstream/2` has been available since v1.0.0. """ - @spec binstream(device, :line | pos_integer) :: Enumerable.t - def binstream(device, line_or_bytes) do + @spec binstream(device, :line | pos_integer) :: Enumerable.t() + def binstream(device \\ :stdio, line_or_bytes) + when line_or_bytes == :line + when is_integer(line_or_bytes) and line_or_bytes > 0 do IO.Stream.__build__(map_dev(device), true, line_or_bytes) end @doc """ - Converts chardata (a list of integers representing codepoints, - lists and strings) into a string. + Converts chardata into a string. + + For more information about chardata, see the ["Chardata"](#module-chardata) + section in the module documentation. - In case the conversion fails, it raises a `UnicodeConversionError`. - If a string is given, returns the string itself. + In case the conversion fails, it raises an `UnicodeConversionError`. + If a string is given, it returns the string itself. ## Examples @@ -284,33 +728,32 @@ defmodule IO do iex> IO.chardata_to_string([0x0061, "bc"]) "abc" + iex> IO.chardata_to_string("string") + "string" + """ - @spec chardata_to_string(chardata) :: String.t | no_return + @spec chardata_to_string(chardata) :: String.t() + def chardata_to_string(chardata) + def chardata_to_string(string) when is_binary(string) do string end def chardata_to_string(list) when is_list(list) do - case :unicode.characters_to_binary(list) do - result when is_binary(result) -> - result - - {:error, encoded, rest} -> - raise UnicodeConversionError, encoded: encoded, rest: rest, kind: :invalid - - {:incomplete, encoded, rest} -> - raise UnicodeConversionError, encoded: encoded, rest: rest, kind: :incomplete - end + List.to_string(list) end @doc """ - Converts iodata (a list of integers representing bytes, lists - and binaries) into a binary. + Converts IO data into a binary + + The operation is Unicode unsafe. - Notice that this function treats lists of integers as raw bytes - and does not perform any kind of encoding conversion. If you want - to convert from a char list to a string (UTF-8 encoded), please - use `chardata_to_string/1` instead. + Note that this function treats integers in the given IO data as + raw bytes and does not perform any kind of encoding conversion. + If you want to convert from a charlist to a UTF-8-encoded string, + use `chardata_to_string/1` instead. For more information about + IO data and chardata, see the ["IO data"](#module-io-data) section in the + module documentation. If this function receives a binary, the same binary is returned. @@ -321,63 +764,70 @@ defmodule IO do iex> bin1 = <<1, 2, 3>> iex> bin2 = <<4, 5>> iex> bin3 = <<6>> - iex> IO.iodata_to_binary([bin1, 1, [2, 3, bin2], 4|bin3]) - <<1,2,3,1,2,3,4,5,4,6>> + iex> IO.iodata_to_binary([bin1, 1, [2, 3, bin2], 4 | bin3]) + <<1, 2, 3, 1, 2, 3, 4, 5, 4, 6>> iex> bin = <<1, 2, 3>> iex> IO.iodata_to_binary(bin) - <<1,2,3>> + <<1, 2, 3>> """ @spec iodata_to_binary(iodata) :: binary - def iodata_to_binary(item) do - :erlang.iolist_to_binary(item) + def iodata_to_binary(iodata) do + :erlang.iolist_to_binary(iodata) end @doc """ - Returns the size of an iodata. + Returns the size of an IO data. + + For more information about IO data, see the ["IO data"](#module-io-data) + section in the module documentation. Inlined by the compiler. ## Examples - iex> IO.iodata_length([1, 2|<<3, 4>>]) + iex> IO.iodata_length([1, 2 | <<3, 4>>]) 4 """ @spec iodata_length(iodata) :: non_neg_integer - def iodata_length(item) do - :erlang.iolist_size(item) + def iodata_length(iodata) do + :erlang.iolist_size(iodata) end @doc false - def each_stream(device, what) do - case read(device, what) do + def each_stream(device, line_or_codepoints) do + case read(device, line_or_codepoints) do :eof -> - nil + {:halt, device} + {:error, reason} -> raise IO.StreamError, reason: reason + data -> - {data, device} + {[data], device} end end @doc false - def each_binstream(device, what) do - case binread(device, what) do + def each_binstream(device, line_or_chars) do + case binread(device, line_or_chars) do :eof -> - nil + {:halt, device} + {:error, reason} -> raise IO.StreamError, reason: reason + data -> - {data, device} + {[data], device} end end @compile {:inline, map_dev: 1, to_chardata: 1} - # Map the Elixir names for standard io and error to Erlang names - defp map_dev(:stdio), do: :standard_io + # Map the Elixir names for standard IO and error to Erlang names + defp map_dev(:stdio), do: :standard_io defp map_dev(:stderr), do: :standard_error defp map_dev(other) when is_atom(other) or is_pid(other) or is_tuple(other), do: other diff --git a/lib/elixir/lib/io/ansi.ex b/lib/elixir/lib/io/ansi.ex index bb52036357d..f808760ff49 100644 --- a/lib/elixir/lib/io/ansi.ex +++ b/lib/elixir/lib/io/ansi.ex @@ -1,228 +1,341 @@ -defmodule IO.ANSI.Sequence do - @moduledoc false - - defmacro defsequence(name, code \\ "", terminator \\ "m") do - quote bind_quoted: [name: name, code: code, terminator: terminator] do - def unquote(name)() do - "\e[#{unquote(code)}#{unquote(terminator)}" - end - - defp escape_sequence(unquote(Atom.to_char_list(name))) do - unquote(name)() - end - end - end -end +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec defmodule IO.ANSI do @moduledoc """ - Functionality to render ANSI escape sequences - (http://en.wikipedia.org/wiki/ANSI_escape_code) — characters embedded - in text used to control formatting, color, and other output options - on video text terminals. + Functionality to render ANSI escape sequences. + + [ANSI escape sequences](https://en.wikipedia.org/wiki/ANSI_escape_code) + are characters embedded in text used to control formatting, color, and + other output options on video text terminals. + + ANSI escapes are typically enabled on all Unix terminals. They are also + available on Windows consoles from Windows 10, although it must be + explicitly enabled for the current user in the registry by running the + following command: + + reg add HKCU\\Console /v VirtualTerminalLevel /t REG_DWORD /d 1 + + After running the command above, you must restart your current console. + + ## Examples + + Because the ANSI escape sequences are embedded in text, the normal usage of + these functions is to concatenate their output with text. + + formatted_text = IO.ANSI.blue_background() <> "Example" <> IO.ANSI.reset() + IO.puts(formatted_text) + + A higher level and more convenient API is also available via `IO.ANSI.format/1`, + where you use atoms to represent each ANSI escape sequence and by default + checks if ANSI is enabled: + + IO.puts(IO.ANSI.format([:blue_background, "Example"])) + + In case ANSI is disabled, the ANSI escape sequences are simply discarded. """ - import IO.ANSI.Sequence + @type ansicode :: atom + @type ansilist :: + maybe_improper_list(char | ansicode | binary | ansilist, binary | ansicode | []) + @type ansidata :: ansilist | ansicode | binary + + @doc """ + Checks if ANSI coloring is supported and enabled on this machine. + + This function simply reads the configuration value for + `:ansi_enabled` in the `:elixir` application. The value is by + default `false` unless Elixir can detect during startup that + both `stdout` and `stderr` are terminals. + """ + @spec enabled? :: boolean + def enabled? do + Application.get_env(:elixir, :ansi_enabled, false) + end @doc """ - Checks whether the default I/O device is a terminal or a file. + Syntax colors to be used by `Inspect`. + + Those colors are used throughout Elixir's standard library, + such as `dbg/2` and `IEx`. + + The colors can be changed by setting the `:ansi_syntax_colors` + in the `:elixir` application configuration. Configuration for + most built-in data types are supported: `:atom`, `:binary`, + `:boolean`, `:charlist`, `:list`, `:map`, `:nil`, `:number`, + `:string`, and `:tuple`. The default is: + + [ + atom: :cyan + boolean: :magenta, + charlist: :yellow, + nil: :magenta, + number: :yellow, + string: :green + ] + + """ + @doc since: "1.14.0" + @spec syntax_colors :: Keyword.t(ansidata) + def syntax_colors do + Application.fetch_env!(:elixir, :ansi_syntax_colors) + end + + @doc "Sets foreground color." + @spec color(0..255) :: String.t() + def color(code) when code in 0..255, do: "\e[38;5;#{code}m" + + @doc ~S""" + Sets the foreground color from individual RGB values. + + Valid values for each color are in the range 0 to 5. + """ + @spec color(0..5, 0..5, 0..5) :: String.t() + def color(r, g, b) when r in 0..5 and g in 0..5 and b in 0..5 do + color(16 + 36 * r + 6 * g + b) + end + + @doc "Sets background color." + @spec color_background(0..255) :: String.t() + def color_background(code) when code in 0..255, do: "\e[48;5;#{code}m" + + @doc ~S""" + Sets the background color from individual RGB values. - Used to identify whether printing ANSI escape sequences will likely - be displayed as intended. This is checked by sending a message to - the group leader. In case the group leader does not support the message, - it will likely lead to a timeout (and a slow down on execution time). + Valid values for each color are in the range 0 to 5. """ - @spec terminal? :: boolean - @spec terminal?(:io.device) :: boolean - def terminal?(device \\ :erlang.group_leader) do - !match?({:win32, _}, :os.type()) and - match?({:ok, _}, :io.columns(device)) + @spec color_background(0..5, 0..5, 0..5) :: String.t() + def color_background(r, g, b) when r in 0..5 and g in 0..5 and b in 0..5 do + color_background(16 + 36 * r + 6 * g + b) + end + + defsequence = fn name, code, terminator -> + @spec unquote(name)() :: String.t() + def unquote(name)() do + "\e[#{unquote(code)}#{unquote(terminator)}" + end + + defp format_sequence(unquote(name)) do + unquote(name)() + end end - @doc "Resets all attributes" - defsequence :reset, 0 + @doc "Resets all attributes." + defsequence.(:reset, 0, "m") - @doc "Bright (increased intensity) or Bold" - defsequence :bright, 1 + @doc "Bright (increased intensity) or bold." + defsequence.(:bright, 1, "m") - @doc "Faint (decreased intensity), not widely supported" - defsequence :faint, 2 + @doc "Faint (decreased intensity). Not widely supported." + defsequence.(:faint, 2, "m") @doc "Italic: on. Not widely supported. Sometimes treated as inverse." - defsequence :italic, 3 + defsequence.(:italic, 3, "m") - @doc "Underline: Single" - defsequence :underline, 4 + @doc "Underline: single." + defsequence.(:underline, 4, "m") - @doc "Blink: Slow. Less than 150 per minute" - defsequence :blink_slow, 5 + @doc "Blink: slow. Less than 150 per minute." + defsequence.(:blink_slow, 5, "m") - @doc "Blink: Rapid. MS-DOS ANSI.SYS; 150 per minute or more; not widely supported" - defsequence :blink_rapid, 6 + @doc "Blink: rapid. MS-DOS ANSI.SYS; 150 per minute or more; not widely supported." + defsequence.(:blink_rapid, 6, "m") - @doc "Image: Negative. Swap foreground and background" - defsequence :inverse, 7 + @doc "Image: negative. Swap foreground and background." + defsequence.(:inverse, 7, "m") - @doc "Image: Negative. Swap foreground and background" - defsequence :reverse, 7 + @doc "Image: negative. Swap foreground and background." + defsequence.(:reverse, 7, "m") - @doc "Conceal. Not widely supported" - defsequence :conceal, 8 + @doc "Conceal. Not widely supported." + defsequence.(:conceal, 8, "m") @doc "Crossed-out. Characters legible, but marked for deletion. Not widely supported." - defsequence :crossed_out, 9 + defsequence.(:crossed_out, 9, "m") - @doc "Sets primary (default) font" - defsequence :primary_font, 10 + @doc "Sets primary (default) font." + defsequence.(:primary_font, 10, "m") for font_n <- [1, 2, 3, 4, 5, 6, 7, 8, 9] do - @doc "Sets alternative font #{font_n}" - defsequence :"font_#{font_n}", font_n + 10 + @doc "Sets alternative font #{font_n}." + defsequence.(:"font_#{font_n}", font_n + 10, "m") end - @doc "Normal color or intensity" - defsequence :normal, 22 + @doc "Normal color or intensity." + defsequence.(:normal, 22, "m") + + @doc "Not italic." + defsequence.(:not_italic, 23, "m") + + @doc "Underline: none." + defsequence.(:no_underline, 24, "m") - @doc "Not italic" - defsequence :not_italic, 23 + @doc "Blink: off." + defsequence.(:blink_off, 25, "m") - @doc "Underline: None" - defsequence :no_underline, 24 + @doc "Image: positive. Normal foreground and background." + defsequence.(:inverse_off, 27, "m") - @doc "Blink: off" - defsequence :blink_off, 25 + @doc "Image: positive. Normal foreground and background." + defsequence.(:reverse_off, 27, "m") + + @doc "Reveal: Not concealed." + defsequence.(:reveal, 28, "m") + + @doc "Not crossed-out." + defsequence.(:not_crossed_out, 29, "m") colors = [:black, :red, :green, :yellow, :blue, :magenta, :cyan, :white] - colors = Enum.zip(0..(length(colors)-1), colors) - for {code, color} <- colors do - @doc "Sets foreground color to #{color}" - defsequence color, code + 30 + for {color, code} <- Enum.with_index(colors) do + @doc "Sets foreground color to #{color}." + defsequence.(color, code + 30, "m") + + @doc "Sets foreground color to light #{color}." + defsequence.(:"light_#{color}", code + 90, "m") + + @doc "Sets background color to #{color}." + defsequence.(:"#{color}_background", code + 40, "m") - @doc "Sets background color to #{color}" - defsequence :"#{color}_background", code + 40 + @doc "Sets background color to light #{color}." + defsequence.(:"light_#{color}_background", code + 100, "m") end - @doc "Default text color" - defsequence :default_color, 39 + @doc "Default text color." + defsequence.(:default_color, 39, "m") + + @doc "Default background color." + defsequence.(:default_background, 49, "m") + + @doc "Framed." + defsequence.(:framed, 51, "m") + + @doc "Encircled." + defsequence.(:encircled, 52, "m") - @doc "Default background color" - defsequence :default_background, 49 + @doc "Overlined." + defsequence.(:overlined, 53, "m") - @doc "Framed" - defsequence :framed, 51 + @doc "Not framed or encircled." + defsequence.(:not_framed_encircled, 54, "m") - @doc "Encircled" - defsequence :encircled, 52 + @doc "Not overlined." + defsequence.(:not_overlined, 55, "m") - @doc "Overlined" - defsequence :overlined, 53 + @doc "Clears screen." + defsequence.(:clear, "2", "J") - @doc "Not framed or encircled" - defsequence :not_framed_encircled, 54 + @doc "Clears line." + defsequence.(:clear_line, "2", "K") - @doc "Not overlined" - defsequence :not_overlined, 55 + @doc "Sends cursor home." + defsequence.(:home, "", "H") - @doc "Send cursor home" - defsequence :home, "", "H" + @doc """ + Sends cursor to the absolute position specified by `line` and `column`. + + Line `0` and column `0` would mean the top left corner. + """ + @spec cursor(non_neg_integer, non_neg_integer) :: String.t() + def cursor(line, column) + when is_integer(line) and line >= 0 and is_integer(column) and column >= 0 do + "\e[#{line};#{column}H" + end - @doc "Clear screen" - defsequence :clear, "2", "J" + @doc "Sends cursor `lines` up." + @spec cursor_up(pos_integer) :: String.t() + def cursor_up(lines \\ 1) when is_integer(lines) and lines >= 1, do: "\e[#{lines}A" - defp escape_sequence(other) do - raise ArgumentError, "invalid ANSI sequence specification: #{other}" + @doc "Sends cursor `lines` down." + @spec cursor_down(pos_integer) :: String.t() + def cursor_down(lines \\ 1) when is_integer(lines) and lines >= 1, do: "\e[#{lines}B" + + @doc "Sends cursor `columns` to the right." + @spec cursor_right(pos_integer) :: String.t() + def cursor_right(columns \\ 1) when is_integer(columns) and columns >= 1, do: "\e[#{columns}C" + + @doc "Sends cursor `columns` to the left." + @spec cursor_left(pos_integer) :: String.t() + def cursor_left(columns \\ 1) when is_integer(columns) and columns >= 1, do: "\e[#{columns}D" + + defp format_sequence(other) do + raise ArgumentError, "invalid ANSI sequence specification: #{inspect(other)}" end @doc ~S""" - Escapes a string by converting named ANSI sequences into actual ANSI codes. + Formats a chardata-like argument by converting named ANSI sequences into actual + ANSI codes. - The format for referring to sequences is `%{red}` and `%{red,bright}` (for - multiple sequences). + The named sequences are represented by atoms. - It will also append a `%{reset}` to the string. If you don't want this - behaviour, use `escape_fragment/2`. + It will also append an `IO.ANSI.reset/0` to the chardata when a conversion is + performed. If you don't want this behavior, use `format_fragment/2`. An optional boolean parameter can be passed to enable or disable - emitting actual ANSI codes. When `false`, no ANSI codes will emitted. - By default, standard output will be checked if it is a terminal capable - of handling these sequences (using `terminal?/1` function) + emitting actual ANSI codes. When `false`, no ANSI codes will be emitted. + By default checks if ANSI is enabled using the `enabled?/0` function. + + An `ArgumentError` will be raised if an invalid ANSI code is provided. ## Examples - iex> IO.ANSI.escape("Hello %{red,bright,green}yes", true) - "Hello \e[31m\e[1m\e[32myes\e[0m" + iex> IO.ANSI.format(["Hello, ", :red, :bright, "world!"], true) + [[[[[[], "Hello, "] | "\e[31m"] | "\e[1m"], "world!"] | "\e[0m"] """ - @spec escape(String.t, emit :: boolean) :: String.t - def escape(string, emit \\ terminal?) when is_binary(string) and is_boolean(emit) do - {rendered, emitted} = do_escape(string, emit, false, nil, []) - if emitted do - rendered <> reset - else - rendered - end + @spec format(ansidata, boolean) :: IO.chardata() + def format(ansidata, emit? \\ enabled?()) when is_boolean(emit?) do + do_format(ansidata, [], [], emit?, :maybe) end @doc ~S""" - Escapes a string by converting named ANSI sequences into actual ANSI codes. + Formats a chardata-like argument by converting named ANSI sequences into actual + ANSI codes. - The format for referring to sequences is `%{red}` and `%{red,bright}` (for - multiple sequences). + The named sequences are represented by atoms. An optional boolean parameter can be passed to enable or disable - emitting actual ANSI codes. When `false`, no ANSI codes will emitted. - By default, standard output will be checked if it is a terminal capable - of handling these sequences (using `terminal?/1` function) + emitting actual ANSI codes. When `false`, no ANSI codes will be emitted. + By default checks if ANSI is enabled using the `enabled?/0` function. ## Examples - iex> IO.ANSI.escape_fragment("Hello %{red,bright,green}yes", true) - "Hello \e[31m\e[1m\e[32myes" - - iex> IO.ANSI.escape_fragment("%{reset}bye", true) - "\e[0mbye" + iex> IO.ANSI.format_fragment([:bright, ~c"Word"], true) + [[[[[[] | "\e[1m"], 87], 111], 114], 100] """ - @spec escape_fragment(String.t, emit :: boolean) :: String.t - def escape_fragment(string, emit \\ terminal?) when is_binary(string) and is_boolean(emit) do - {escaped, _emitted} = do_escape(string, emit, false, nil, []) - escaped - end - - defp do_escape(<>, emit, emitted, buffer, acc) when is_list(buffer) do - sequences = - buffer - |> Enum.reverse() - |> :string.tokens(',') - |> Enum.map(&(&1 |> :string.strip |> escape_sequence)) - |> Enum.reverse() - - if emit and sequences != [] do - do_escape(t, emit, true, nil, sequences ++ acc) - else - do_escape(t, emit, emitted, nil, acc) - end + @spec format_fragment(ansidata, boolean) :: IO.chardata() + def format_fragment(ansidata, emit? \\ enabled?()) when is_boolean(emit?) do + do_format(ansidata, [], [], emit?, false) + end + + defp do_format([term | rest], rem, acc, emit?, append_reset) do + do_format(term, [rest | rem], acc, emit?, append_reset) + end + + defp do_format(term, rem, acc, true, append_reset) when is_atom(term) do + do_format([], rem, [acc | format_sequence(term)], true, !!append_reset) end - defp do_escape(<>, emit, emitted, buffer, acc) when is_list(buffer) do - do_escape(t, emit, emitted, [h|buffer], acc) + defp do_format(term, rem, acc, false, append_reset) when is_atom(term) do + format_sequence(term) + do_format([], rem, acc, false, append_reset) end - defp do_escape(<<>>, _emit, _emitted, buffer, _acc) when is_list(buffer) do - buffer = IO.iodata_to_binary Enum.reverse(buffer) - raise ArgumentError, "missing } for escape fragment #{buffer}" + defp do_format(term, rem, acc, emit?, append_reset) when not is_list(term) do + do_format([], rem, [acc, term], emit?, append_reset) end - defp do_escape(<>, emit, emitted, nil, acc) do - do_escape(t, emit, emitted, [], acc) + defp do_format([], [next | rest], acc, emit?, append_reset) do + do_format(next, rest, acc, emit?, append_reset) end - defp do_escape(<>, emit, emitted, nil, acc) do - do_escape(t, emit, emitted, nil, [h|acc]) + defp do_format([], [], acc, true, true) do + [acc | IO.ANSI.reset()] end - defp do_escape(<<>>, _emit, emitted, nil, acc) do - {IO.iodata_to_binary(Enum.reverse(acc)), emitted} + defp do_format([], [], acc, _emit?, _append_reset) do + acc end end diff --git a/lib/elixir/lib/io/ansi/docs.ex b/lib/elixir/lib/io/ansi/docs.ex index 4dfcfb21ac0..fe3a04bc1e4 100644 --- a/lib/elixir/lib/io/ansi/docs.ex +++ b/lib/elixir/lib/io/ansi/docs.ex @@ -1,34 +1,64 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + defmodule IO.ANSI.Docs do @moduledoc false + @type print_opts :: [ + enabled: boolean(), + doc_bold: [IO.ANSI.ansicode()], + doc_code: [IO.ANSI.ansicode()], + doc_headings: [IO.ANSI.ansicode()], + doc_metadata: [IO.ANSI.ansicode()], + doc_quote: [IO.ANSI.ansicode()], + doc_inline_code: [IO.ANSI.ansicode()], + doc_table_heading: [IO.ANSI.ansicode()], + doc_title: [IO.ANSI.ansicode()], + doc_underline: [IO.ANSI.ansicode()], + width: pos_integer() + ] + + @bullet_text_unicode "• " + @bullet_text_ascii "* " @bullets [?*, ?-, ?+] + @spaces [" ", "\n", "\t"] @doc """ The default options used by this module. - The supported values are: + The supported keys are: - * `:enabled` - toggles coloring on and off (true) - * `:doc_code` - code blocks (cyan, bright) - * `:doc_inline_code` - inline code (cyan) - * `:doc_headings` - h1 and h2 headings (yellow, bright) - * `:doc_title` - top level heading (reverse, yellow, bright) - * `:doc_bold` - bold text (bright) - * `:doc_underline` - underlined text (underline) - * `:width` - the width to format the text (80) + * `:enabled` - toggles coloring on and off (true) + * `:doc_bold` - bold text (bright) + * `:doc_code` - code blocks (cyan) + * `:doc_headings` - h1, h2, h3, h4, h5, h6 headings (yellow) + * `:doc_metadata` - documentation metadata keys (yellow) + * `:doc_quote` - leading quote character `> ` (light black) + * `:doc_inline_code` - inline code (cyan) + * `:doc_table_heading` - the style for table headings + * `:doc_title` - top level heading (reverse, yellow) + * `:doc_underline` - underlined text (underline) + * `:width` - the width to format the text (80) Values for the color settings are strings with comma-separated ANSI values. """ + @spec default_options() :: print_opts def default_options do - [enabled: true, - doc_code: "cyan,bright", - doc_inline_code: "cyan", - doc_headings: "yellow,bright", - doc_title: "reverse,yellow,bright", - doc_bold: "bright", - doc_underline: "underline", - width: 80] + [ + enabled: true, + doc_bold: [:bright], + doc_code: [:cyan], + doc_headings: [:yellow], + doc_metadata: [:yellow], + doc_quote: [:light_black], + doc_inline_code: [:cyan], + doc_table_heading: [:reverse], + doc_title: [:reverse, :yellow], + doc_underline: [:underline], + width: 80 + ] end @doc """ @@ -36,215 +66,536 @@ defmodule IO.ANSI.Docs do See `default_options/0` for docs on the supported options. """ - def print_heading(heading, options \\ []) do - IO.puts IO.ANSI.reset - options = Keyword.merge(default_options, options) - width = options[:width] - padding = div(width + String.length(heading), 2) - heading = heading |> String.rjust(padding) |> String.ljust(width) - write(:doc_title, heading, options) + @spec print_headings([String.t()], print_opts) :: :ok + def print_headings(headings, options \\ []) do + # It's possible for some of the headings to contain newline characters (`\n`), so in order to prevent it from + # breaking the output from `print_headings/2`, as `print_headings/2` tries to pad the whole heading, we first split + # any heading containgin newline characters into multiple headings, that way each one is padded on its own. + headings = Enum.flat_map(headings, fn heading -> String.split(heading, "\n") end) + options = Keyword.merge(default_options(), options) + newline_after_block(options) + width = options[:width] + + for heading <- headings do + padding = div(width + String.length(heading), 2) + heading = String.pad_leading(heading, padding) + heading = if options[:enabled], do: String.pad_trailing(heading, width), else: heading + write(:doc_title, heading, options) + end + + newline_after_block(options) end @doc """ - Prints the documentation body. + Prints documentation metadata (only `delegate_to`, `deprecated`, `guard`, and `since` for now). - In addition to the priting string, takes a set of options - defined in `default_options/1`. + See `default_options/0` for docs on the supported options. """ - def print(doc, options \\ []) do - options = Keyword.merge(default_options, options) + @spec print_metadata(map, print_opts) :: :ok + def print_metadata(metadata, options \\ []) when is_map(metadata) do + options = Keyword.merge(default_options(), options) + print_each_metadata(metadata, options) && IO.write("\n") + end + + @metadata_filter [:deprecated, :guard, :since] + + defp print_each_metadata(metadata, options) do + metadata + |> Enum.sort() + |> Enum.reduce(false, fn + {key, value}, _printed when is_binary(value) and key in @metadata_filter -> + label = metadata_label(key, options) + indent = String.duplicate(" ", length_without_escape(label, 0) + 1) + write_with_wrap([label | String.split(value, @spaces)], options[:width], indent, true, "") + + {key, value}, _printed when is_boolean(value) and key in @metadata_filter -> + IO.puts([metadata_label(key, options), ?\s, to_string(value)]) + + {:delegate_to, {m, f, a}}, _printed -> + label = metadata_label(:delegate_to, options) + IO.puts([label, ?\s, Exception.format_mfa(m, f, a)]) + + _metadata, printed -> + printed + end) + end + + defp metadata_label(key, options) do + "#{color(:doc_metadata, options)}#{key}:#{maybe_reset(options)}" + end + + @doc """ + Prints the documentation body `doc` according to `format`. + + It takes a set of `options` defined in `default_options/0`. + """ + @spec print(term(), String.t(), print_opts) :: :ok + def print(doc, format, options \\ []) + + def print(doc, "text/markdown", options) when is_binary(doc) and is_list(options) do + print_markdown(doc, options) + end + + def print(_doc, format, options) when is_binary(format) and is_list(options) do + IO.puts("\nUnknown documentation format #{inspect(format)}\n") + end + + ## Markdown + + def print_markdown(doc, options) do + options = Keyword.merge(default_options(), options) + doc - |> String.split(["\r\n","\n"], trim: false) - |> Enum.map(&String.rstrip/1) - |> process("", options) + |> String.split(["\r\n", "\n"], trim: false) + |> Enum.map(&String.trim_trailing/1) + |> process([], "", options) end - defp process([], _indent, _options), do: nil + defp process([], text, indent, options) do + write_text(text, indent, options) + end - defp process(["# " <> heading | rest], _indent, options) do - write_h1(String.strip(heading), options) - process(rest, "", options) + defp process(["# " <> _ = heading | rest], text, indent, options) do + write_heading(heading, rest, text, indent, options) end - defp process(["## " <> heading | rest], _indent, options) do - write_h2(String.strip(heading), options) - process(rest, "", options) + defp process(["## " <> _ = heading | rest], text, indent, options) do + write_heading(heading, rest, text, indent, options) end - defp process(["### " <> heading | rest], indent, options) do - write_h3(String.strip(heading), indent, options) - process(rest, indent, options) + defp process(["### " <> _ = heading | rest], text, indent, options) do + write_heading(heading, rest, text, indent, options) end - defp process(["" | rest], indent, options) do - process(rest, indent, options) + defp process(["#### " <> _ = heading | rest], text, indent, options) do + write_heading(heading, rest, text, indent, options) end - defp process([" " <> line | rest], indent, options) do - process_code(rest, [line], indent, options) + defp process(["##### " <> _ = heading | rest], text, indent, options) do + write_heading(heading, rest, text, indent, options) end - defp process([line | rest], indent, options) do - {stripped, count} = strip_spaces(line, 0) - case stripped do - <> when bullet in @bullets -> - process_list(item, rest, count, indent, options) - _ -> - process_text(rest, [line], indent, false, options) - end + defp process(["###### " <> _ = heading | rest], text, indent, options) do + write_heading(heading, rest, text, indent, options) end - defp strip_spaces(" " <> line, acc) do - strip_spaces(line, acc + 1) + defp process([">" <> line | rest], text, indent, options) do + write_text(text, indent, options) + process_quote(rest, [line], indent, options) end - defp strip_spaces(rest, acc) do - {rest, acc} + defp process(["" | rest], text, indent, options) do + write_text(text, indent, options) + process(rest, [], indent, options) end - ## Headings + defp process([" " <> line | rest], text, indent, options) do + write_text(text, indent, options) + process_code(rest, [line], indent, options) + end + + defp process(["```mermaid" <> _line | rest], text, indent, options) do + write_text(text, indent, options) - defp write_h1(heading, options) do - write_h2(String.upcase(heading), options) + rest + |> Enum.drop_while(&(&1 != "```")) + |> Enum.drop(1) + |> process([], indent, options) end - defp write_h2(heading, options) do - write(:doc_headings, heading, options) + defp process(["```" <> _line | rest], text, indent, options) do + process_fenced_code_block(rest, text, indent, options, _delimiter = "```") + end + + defp process(["") + rest + end + + defp drop_comment([line | rest]) do + case :binary.split(line, "-->") do + [_] -> drop_comment(rest) + [_, line] -> [line | rest] + end + end + + defp drop_comment([]) do + [] + end + + # We have four entries: **, __, *, _ and `. + # + # The first four behave the same while the last one is simpler + # when it comes to delimiters as it ignores spaces and escape + # characters. But, since the first two has two characters, + # we need to handle 3 cases: + # + # 1. __ and ** + # 2. _ and * + # 3. ` + # + # Where the first two should have the same code but match differently. + @single [?_, ?*] + + # Characters that can mark the beginning or the end of a word. + # Only support the most common ones at this moment. + @delimiters [?\s, ?', ?", ?!, ?@, ?#, ?$, ?%, ?^, ?&] ++ + [?-, ?+, ?(, ?), ?[, ?], ?{, ?}, ?<, ?>, ?.] + + ### Inline start + + defp handle_inline(<>, options) when mark in @single do + handle_inline(rest, [mark | mark], [<>], [], options) + end + + defp handle_inline(<>, options) when mark in @single do + handle_inline(rest, mark, [<>], [], options) end - # Single inline quotes. - @single [?`, ?_, ?*] + defp handle_inline(rest, options) do + handle_inline(rest, nil, [], [], options) + end + + ### Inline delimiters - # ` does not require space in between - @spaced [?_, ?*] + defp handle_inline(">+M: module.child_spec(arg) + M-->>-S: %{id: term, start: {module, :start_link, [arg]}} + S-->>+M: module.start_link(arg) + M->>M: Spawns child process (child_pid) + M-->>-S: {:ok, child_pid} | :ignore | {:error, reason} + S->>-C: {:ok, supervisor_pid} | {:error, reason} + ``` + + Luckily for us, `use GenServer` already defines a `Counter.child_spec/1` + exactly like above, so you don't need to write the definition above yourself. + If you want to customize the automatically generated `child_spec/1` function, + you can pass the options directly to `use GenServer`: + + use GenServer, restart: :transient + + Finally, note it is also possible to simply pass the `Counter` module as + a child: - * You need to do some particular action on supervisor - initialization, like setting up a ETS table. + children = [ + Counter + ] + + When only the module name is given, it is equivalent to `{Counter, []}`, + which in our case would be invalid, which is why we always pass the initial + counter explicitly. + + By replacing the child specification with `{Counter, 0}`, we keep it + encapsulated in the `Counter` module. We could now share our + `Counter` implementation with other developers and they can add it directly + to their supervision tree without worrying about the low-level details of + the counter. + + Overall, a child specification can be one of the following: + + * a map representing the child specification itself - as outlined in the + "Child specification" section + + * a tuple with a module as first element and the start argument as second - + such as `{Counter, 0}`. In this case, `Counter.child_spec(0)` is called + to retrieve the child specification - * You want to perform partial hot-code swapping of the - tree. For example, if you add or remove a children, - the module-based supervision will add and remove the - new children directly, while the dynamic supervision - requires the whole tree to be restarted in order to - perform such swaps. + * a module - such as `Counter`. In this case, `Counter.child_spec([])` + would be called, which is invalid for the counter, but it is useful in + many other cases, especially when you want to pass a list of options + to the child process - ## Strategies + If you need to convert a `{module, arg}` tuple or a module child specification to a + [child specification](`t:child_spec/0`) or modify a child specification itself, + you can use the `Supervisor.child_spec/2` function. + For example, to run the counter with a different `:id` and a `:shutdown` value of + 10 seconds (10_000 milliseconds): + + children = [ + Supervisor.child_spec({Counter, 0}, id: MyCounter, shutdown: 10_000) + ] + + ## Supervisor strategies and options + + So far we have started the supervisor passing a single child as a tuple + as well as a strategy called `:one_for_one`: + + children = [ + {Counter, 0} + ] + + Supervisor.start_link(children, strategy: :one_for_one) + + The first argument given to `start_link/2` is a list of child + specifications as defined in the "child_spec/1" section above. + + The second argument is a keyword list of options: + + * `:strategy` - the supervision strategy option. It can be either + `:one_for_one`, `:rest_for_one` or `:one_for_all`. Required. + See the "Strategies" section. + + * `:max_restarts` - the maximum number of restarts allowed in + a time frame. Defaults to `3`. + + * `:max_seconds` - the time frame in which `:max_restarts` applies. + Defaults to `5`. + + * `:auto_shutdown` - the automatic shutdown option. It can be + `:never`, `:any_significant`, or `:all_significant`. Optional. + See the "Automatic shutdown" section. + + * `:name` - a name to register the supervisor process. Supported values are + explained in the "Name registration" section in the documentation for + `GenServer`. Optional. + + ### Strategies + + Supervisors support different supervision strategies (through the + `:strategy` option, as seen above): * `:one_for_one` - if a child process terminates, only that process is restarted. @@ -125,188 +355,729 @@ defmodule Supervisor do processes are terminated and then all child processes (including the terminated one) are restarted. - * `:rest_for_one` - if a child process terminates, the "rest" of - the child processes, i.e. the child processes after the terminated - one in start order, are terminated. Then the terminated child - process and the rest of the child processes are restarted. + * `:rest_for_one` - if a child process terminates, the terminated child + process and the rest of the children started after it, are terminated and + restarted. + + In the above, process termination refers to unsuccessful termination, which + is determined by the `:restart` option. + + To efficiently supervise children started dynamically, see `DynamicSupervisor`. + + ### Automatic shutdown - * `:simple_one_for_one` - similar to `:one_for_one` but suits better - when dynamically attaching children. This strategy requires the - supervisor specification to contain only one children. Many functions - in this module behave slightly differently when this strategy is - used. + Supervisors have the ability to automatically shut themselves down when child + processes marked as `:significant` exit. - ## Name Registration + Supervisors support different automatic shutdown options (through + the `:auto_shutdown` option, as seen above): + + * `:never` - this is the default, automatic shutdown is disabled. + + * `:any_significant` - if any significant child process exits, the supervisor + will automatically shut down its children, then itself. + + * `:all_significant` - when all significant child processes have exited, + the supervisor will automatically shut down its children, then itself. + + Only `:transient` and `:temporary` child processes can be marked as significant, + and this configuration affects the behavior. Significant `:transient` child + processes must exit normally for automatic shutdown to be considered, where + `:temporary` child processes may exit for any reason. + + ### Name registration A supervisor is bound to the same name registration rules as a `GenServer`. - Read more about it in the `GenServer` docs. + Read more about these rules in the documentation for `GenServer`. + + ## Module-based supervisors + + In the example so far, the supervisor was started by passing the supervision + structure to `start_link/2`. However, supervisors can also be created by + explicitly defining a supervision module: + + defmodule MyApp.Supervisor do + # Automatically defines child_spec/1 + use Supervisor + + def start_link(init_arg) do + Supervisor.start_link(__MODULE__, init_arg, name: __MODULE__) + end + + @impl true + def init(_init_arg) do + children = [ + {Counter, 0} + ] + + Supervisor.init(children, strategy: :one_for_one) + end + end + + The difference between the two approaches is that a module-based + supervisor gives you more direct control over how the supervisor + is initialized. Instead of calling `Supervisor.start_link/2` with + a list of child specifications that are implicitly initialized for us, + we must explicitly initialize the children by calling `Supervisor.init/2` + inside its `c:init/1` callback. `Supervisor.init/2` accepts the same + `:strategy`, `:max_restarts`, and `:max_seconds` options as `start_link/2`. + + > #### `use Supervisor` {: .info} + > + > When you `use Supervisor`, the `Supervisor` module will + > set `@behaviour Supervisor` and define a `child_spec/1` + > function, so your module can be used as a child + > in a supervision tree. + + `use Supervisor` also defines a `child_spec/1` function which allows + us to run `MyApp.Supervisor` as a child of another supervisor or + at the top of your supervision tree as: + + children = [ + MyApp.Supervisor + ] + + Supervisor.start_link(children, strategy: :one_for_one) + + A general guideline is to use the supervisor without a callback + module only at the top of your supervision tree, generally in the + `c:Application.start/2` callback. We recommend using module-based + supervisors for any other supervisor in your application, so they + can run as a child of another supervisor in the tree. The `child_spec/1` + generated automatically by `Supervisor` can be customized with the + following options: + + * `:id` - the child specification identifier, defaults to the current module + * `:restart` - when the supervisor should be restarted, defaults to `:permanent` + + The `@doc` annotation immediately preceding `use Supervisor` will be + attached to the generated `child_spec/1` function. + + ## Start and shutdown + + When the supervisor starts, it traverses all child specifications and + then starts each child in the order they are defined. This is done by + calling the function defined under the `:start` key in the child + specification and typically defaults to `start_link/1`. + + The `start_link/1` (or a custom) is then called for each child process. + The `start_link/1` function must return `{:ok, pid}` where `pid` is the + process identifier of a new process that is linked to the supervisor. + The child process usually starts its work by executing the `c:init/1` + callback. Generally speaking, the `init` callback is where we initialize + and configure the child process. + + The shutdown process happens in reverse order. + + When a supervisor shuts down, it terminates all children in the opposite + order they are listed. The termination happens by sending a shutdown exit + signal, via `Process.exit(child_pid, :shutdown)`, to the child process and + then awaiting for a time interval for the child process to terminate. This + interval defaults to 5000 milliseconds. If the child process does not + terminate in this interval, the supervisor abruptly terminates the child + with reason `:kill`. The shutdown time can be configured in the child + specification which is fully detailed in the next section. + + If the child process is not trapping exits, it will shutdown immediately + when it receives the first exit signal. If the child process is trapping + exits, then the `terminate` callback is invoked, and the child process + must terminate in a reasonable time interval before being abruptly + terminated by the supervisor. + + In other words, if it is important that a process cleans after itself + when your application or the supervision tree is shutting down, then + this process must trap exits and its child specification should specify + the proper `:shutdown` value, ensuring it terminates within a reasonable + interval. + + ## Exit reasons and restarts + + A supervisor restarts a child process depending on its `:restart` configuration. + For example, when `:restart` is set to `:transient`, the supervisor does not + restart the child in case it exits with reason `:normal`, `:shutdown` or + `{:shutdown, term}`. + + Those exits also impact logging. By default, behaviours such as GenServers + do not emit error logs when the exit reason is `:normal`, `:shutdown` or + `{:shutdown, term}`. + + So one may ask: which exit reason should I choose? There are three options: + + * `:normal` - in such cases, the exit won't be logged, there is no restart + in transient mode, and linked processes do not exit + + * `:shutdown` or `{:shutdown, term}` - in such cases, the exit won't be + logged, there is no restart in transient mode, and linked processes exit + with the same reason unless they're trapping exits + + * any other term - in such cases, the exit will be logged, there are + restarts in transient mode, and linked processes exit with the same + reason unless they're trapping exits + + Generally speaking, if you are exiting for expected reasons, you want to use + `:shutdown` or `{:shutdown, term}`. + + Note that the supervisor that reaches maximum restart intensity will exit with + `:shutdown` reason. In this case the supervisor will only be restarted if its + child specification was defined with the `:restart` option set to `:permanent` + (the default). """ @doc false - defmacro __using__(_) do - quote location: :keep do - @behaviour :supervisor + defmacro __using__(opts) do + quote location: :keep, bind_quoted: [opts: opts] do import Supervisor.Spec + @behaviour Supervisor + + if not Module.has_attribute?(__MODULE__, :doc) do + @doc """ + Returns a specification to start this module under a supervisor. + + See `Supervisor`. + """ + end + + def child_spec(init_arg) do + default = %{ + id: __MODULE__, + start: {__MODULE__, :start_link, [init_arg]}, + type: :supervisor + } + + Supervisor.child_spec(default, unquote(Macro.escape(opts))) + end + + defoverridable child_spec: 1 end end - @typedoc "Return values of `start_link` functions" - @type on_start :: {:ok, pid} | :ignore | - {:error, {:already_started, pid} | {:shutdown, term} | term} - - @typedoc "Return values of `start_child` functions" - @type on_start_child :: {:ok, child} | {:ok, child, info :: term} | - {:error, {:already_started, child} | :already_present | term} + @doc """ + Callback invoked to start the supervisor and during hot code upgrades. + Developers typically invoke `Supervisor.init/2` at the end of their + init callback to return the proper supervision flags. + """ + @callback init(init_arg :: term) :: + {:ok, + {sup_flags(), [child_spec() | (old_erlang_child_spec :: :supervisor.child_spec())]}} + | :ignore + + @typedoc "Return values of `start_link/2` and `start_link/3`." + @type on_start :: + {:ok, pid} + | :ignore + | {:error, {:already_started, pid} | {:shutdown, term} | term} + + @typedoc "Return values of `start_child/2`." + @type on_start_child :: + {:ok, child} + | {:ok, child, info :: term} + | {:error, {:already_started, child} | :already_present | term} + + @typedoc """ + A child process. + + It can be a PID when the child process was started, or `:undefined` when + the child was created by a [dynamic supervisor](`DynamicSupervisor`). + """ @type child :: pid | :undefined - @typedoc "The Supervisor name" + @typedoc "The supervisor name." @type name :: atom | {:global, term} | {:via, module, term} - @typedoc "Options used by the `start*` functions" - @type options :: [name: name, - strategy: Supervisor.Spec.strategy, - max_restarts: non_neg_integer, - max_seconds: non_neg_integer] + @typedoc "Option values used by the `start_link/2` and `start_link/3` functions." + @type option :: {:name, name} - @typedoc "The supervisor reference" + @typedoc "The supervisor flags returned on init." + @type sup_flags() :: %{ + strategy: strategy(), + intensity: non_neg_integer(), + period: pos_integer(), + auto_shutdown: auto_shutdown() + } + + @typedoc "The supervisor reference." @type supervisor :: pid | name | {atom, node} + @typedoc "Options given to `start_link/2` and `init/2`." + @type init_option :: + {:strategy, strategy} + | {:max_restarts, non_neg_integer} + | {:max_seconds, pos_integer} + | {:auto_shutdown, auto_shutdown} + + @typedoc "Supported restart options." + @type restart :: :permanent | :transient | :temporary + + @typedoc "Supported shutdown options." + @type shutdown :: timeout() | :brutal_kill + + @typedoc "Supported strategies." + @type strategy :: :one_for_one | :one_for_all | :rest_for_one + + @typedoc "Supported automatic shutdown options." + @type auto_shutdown :: :never | :any_significant | :all_significant + + @typedoc """ + Type of a supervised child. + + Whether the supervised child is a worker or a supervisor. + """ + @type type :: :worker | :supervisor + + # Note we have inlined all types for readability + @typedoc """ + The supervisor child specification. + + It defines how the supervisor should start, stop and restart each of its children. + """ + @type child_spec :: %{ + required(:id) => atom() | term(), + required(:start) => {module(), function_name :: atom(), args :: [term()]}, + optional(:restart) => restart(), + optional(:shutdown) => shutdown(), + optional(:type) => type(), + optional(:modules) => [module()] | :dynamic, + optional(:significant) => boolean() + } + + @typedoc """ + A module-based child spec. + + This is a form of child spec that you can pass to functions such as `child_spec/2`, + `start_child/2`, and `start_link/2`, in addition to the normalized `t:child_spec/0`. + + A module-based child spec can be: + + * a **module** — the supervisor calls `module.child_spec([])` to retrieve the + child specification + + * a **two-element tuple** in the shape of `{module, arg}` — the supervisor + calls `module.child_spec(arg)` to retrieve the child specification + + """ + @typedoc since: "1.16.0" + @type module_spec :: {module(), args :: term()} | module() + + @typedoc """ + Options for overriding child specification fields. + """ + @type child_spec_overrides :: [ + id: atom() | term(), + start: {module(), atom(), [term()]}, + restart: restart(), + shutdown: shutdown(), + type: type(), + modules: [module()] | :dynamic, + significant: boolean() + ] + @doc """ Starts a supervisor with the given children. - A strategy is required to be given as an option. Furthermore, - the `:max_restarts` and `:max_seconds` value can be configured - as described in `Supervisor.Spec.supervise/2` docs. + `children` is a list of the following forms: + + * a child specification (see `t:child_spec/0`) + + * a module, where the supervisor calls `module.child_spec([])` + to retrieve the child specification (see `t:module_spec/0`) + + * a `{module, arg}` tuple, where the supervisor calls `module.child_spec(arg)` + to retrieve the child specification (see `t:module_spec/0`) + + * a (old) Erlang-style child specification (see + [`:supervisor.child_spec()`](`t::supervisor.child_spec/0`)) + + A strategy is required to be provided through the `:strategy` option. See + "Supervisor strategies and options" for examples and other options. The options can also be used to register a supervisor name. - the supported values are described under the `Name Registration` + The supported values are described under the "Name registration" section in the `GenServer` module docs. - If the supervisor and its child processes are successfully created - (i.e. if the start function of all child processes returns `{:ok, child}`, - `{:ok, child, info}`, or `:ignore`) the function returns - `{:ok, pid}`, where `pid` is the pid of the supervisor. If there - already exists a process with the specified name, the function returns - `{:error, {:already_started, pid}}`, where pid is the pid of that - process. - - If any of the child process start functions fail or return an error tuple or - an erroneous value, the supervisor will first terminate all already - started child processes with reason `:shutdown` and then terminate - itself and return `{:error, {:shutdown, reason}}`. - - Note that the `Supervisor` is linked to the parent process - and will exit not only on crashes but also if the parent process - exits with `:normal` reason. + If the supervisor and all child processes are successfully spawned + (if the start function of each child process returns `{:ok, child}`, + `{:ok, child, info}`, or `:ignore`), this function returns + `{:ok, pid}`, where `pid` is the PID of the supervisor. If the supervisor + is given a name and a process with the specified name already exists, + the function returns `{:error, {:already_started, pid}}`, where `pid` + is the PID of that process. + + If the start function of any of the child processes fails or returns an error + tuple or an erroneous value, the supervisor first terminates with reason + `:shutdown` all the child processes that have already been started, and then + terminates itself and returns `{:error, {:shutdown, reason}}`. + + Note that a supervisor started with this function is linked to the parent + process and exits not only on crashes but also if the parent process exits + with `:normal` reason. """ - @spec start_link([tuple], options) :: on_start + @spec start_link( + [child_spec | module_spec | (old_erlang_child_spec :: :supervisor.child_spec())], + [option | init_option] + ) :: + {:ok, pid} | {:error, {:already_started, pid} | {:shutdown, term} | term} def start_link(children, options) when is_list(children) do - spec = Supervisor.Spec.supervise(children, options) - start_link(Supervisor.Default, spec, options) + {sup_opts, start_opts} = + Keyword.split(options, [:strategy, :max_seconds, :max_restarts, :auto_shutdown]) + + start_link(Supervisor.Default, init(children, sup_opts), start_opts) + end + + @doc """ + Receives a list of child specifications to initialize and a set of `options`. + + This is typically invoked at the end of the `c:init/1` callback of + module-based supervisors. See the sections "Supervisor strategies and options" and + "Module-based supervisors" in the module documentation for more information. + + This function returns a tuple containing the supervisor + flags and child specifications. + + ## Examples + + def init(_init_arg) do + children = [ + {Counter, 0} + ] + + Supervisor.init(children, strategy: :one_for_one) + end + + ## Options + + * `:strategy` - the supervision strategy option. It can be either + `:one_for_one`, `:rest_for_one`, or `:one_for_all` + + * `:max_restarts` - the maximum number of restarts allowed in + a time frame. Defaults to `3`. + + * `:max_seconds` - the time frame in seconds in which `:max_restarts` + applies. Defaults to `5`. + + * `:auto_shutdown` - the automatic shutdown option. It can be either + `:never`, `:any_significant`, or `:all_significant` + + The `:strategy` option is required and by default a maximum of 3 restarts + is allowed within 5 seconds. Check the `Supervisor` module for a detailed + description of the available strategies. + """ + @doc since: "1.5.0" + @spec init( + [child_spec | module_spec | (old_erlang_child_spec :: :supervisor.child_spec())], + [init_option] + ) :: + {:ok, + {sup_flags(), [child_spec() | (old_erlang_child_spec :: :supervisor.child_spec())]}} + def init(children, options) when is_list(children) and is_list(options) do + strategy = + case options[:strategy] do + nil -> + raise ArgumentError, "expected :strategy option to be given" + + :simple_one_for_one -> + IO.warn( + ":simple_one_for_one strategy is deprecated, please use DynamicSupervisor instead" + ) + + :simple_one_for_one + + other -> + other + end + + intensity = Keyword.get(options, :max_restarts, 3) + period = Keyword.get(options, :max_seconds, 5) + auto_shutdown = Keyword.get(options, :auto_shutdown, :never) + + flags = %{ + strategy: strategy, + intensity: intensity, + period: period, + auto_shutdown: auto_shutdown + } + + {:ok, {flags, Enum.map(children, &init_child/1)}} + end + + defp init_child(module) when is_atom(module) do + init_child({module, []}) + end + + defp init_child({module, arg}) when is_atom(module) do + try do + module.child_spec(arg) + rescue + e in UndefinedFunctionError -> + case __STACKTRACE__ do + [{^module, :child_spec, [^arg], _} | _] -> + raise ArgumentError, child_spec_error(module) + + stack -> + reraise e, stack + end + end + end + + defp init_child(map) when is_map(map) do + map + end + + defp init_child({_, _, _, _, _, _} = tuple) do + tuple + end + + defp init_child(other) do + raise ArgumentError, """ + supervisors expect each child to be one of the following: + + * a module + * a {module, arg} tuple + * a child specification as a map with at least the :id and :start fields + * or a tuple with 6 elements generated by Supervisor.Spec (deprecated) + + Got: #{inspect(other)} + """ + end + + defp child_spec_error(module) do + if Code.ensure_loaded?(module) do + """ + The module #{inspect(module)} was given as a child to a supervisor + but it does not implement child_spec/1. + + If you own the given module, please define a child_spec/1 function + that receives an argument and returns a child specification as a map. + For example: + + def child_spec(opts) do + %{ + id: __MODULE__, + start: {__MODULE__, :start_link, [opts]}, + type: :worker, + restart: :permanent, + shutdown: 500 + } + end + + Note that "use Agent", "use GenServer" and so on automatically define + this function for you. + + However, if you don't own the given module and it doesn't implement + child_spec/1, instead of passing the module name directly as a supervisor + child, you will have to pass a child specification as a map: + + %{ + id: #{inspect(module)}, + start: {#{inspect(module)}, :start_link, [arg1, arg2]} + } + + See the Supervisor documentation for more information + """ + else + "The module #{inspect(module)} was given as a child to a supervisor but it does not exist" + end + end + + @doc """ + Builds and overrides a child specification. + + Similar to `start_link/2` and `init/2`, it expects a module, `{module, arg}`, + or a [child specification](`t:child_spec/0`). + + If a two-element tuple in the shape of `{module, arg}` is given, + the child specification is retrieved by calling `module.child_spec(arg)`. + + If a module is given, the child specification is retrieved by calling + `module.child_spec([])`. + + After the child specification is retrieved, the fields on `overrides` + are directly applied to the child spec. If `overrides` has keys that + do not map to any child specification field, an error is raised. + + See the "Child specification" section in the module documentation + for all of the available keys for overriding. + + ## Examples + + This function is often used to set an `:id` option when + the same module needs to be started multiple times in the + supervision tree: + + Supervisor.child_spec({Agent, fn -> :ok end}, id: {Agent, 1}) + #=> %{id: {Agent, 1}, + #=> start: {Agent, :start_link, [fn -> :ok end]}} + + """ + @spec child_spec(child_spec() | module_spec(), child_spec_overrides()) :: child_spec() + def child_spec(module_or_map, overrides) + + def child_spec({_, _, _, _, _, _} = tuple, _overrides) do + raise ArgumentError, + "old tuple-based child specification #{inspect(tuple)} " <> + "is not supported in Supervisor.child_spec/2" + end + + def child_spec(module_or_map, overrides) do + Enum.reduce(overrides, init_child(module_or_map), fn + {key, value}, acc + when key in [:id, :start, :restart, :shutdown, :type, :modules, :significant] -> + Map.put(acc, key, value) + + {key, _value}, _acc -> + raise ArgumentError, "unknown key #{inspect(key)} in child specification override" + end) end @doc """ - Starts a supervisor module with the given `arg`. + Starts a module-based supervisor process with the given `module` and `init_arg`. - To start the supervisor, the `init/1` callback will be invoked - in the given module. The `init/1` callback must return a - supervision specification which can be created with the help - of `Supervisor.Spec` module. + To start the supervisor, the `c:init/1` callback will be invoked in the given + `module`, with `init_arg` as its argument. The `c:init/1` callback must return a + supervisor specification which can be created with the help of the `init/2` + function. - If the `init/1` callback returns `:ignore`, this function returns + If the `c:init/1` callback returns `:ignore`, this function returns `:ignore` as well and the supervisor terminates with reason `:normal`. If it fails or returns an incorrect value, this function returns `{:error, term}` where `term` is a term with information about the error, and the supervisor terminates with reason `term`. The `:name` option can also be given in order to register a supervisor - name, the supported values are described under the `Name Registration` + name, the supported values are described in the "Name registration" section in the `GenServer` module docs. - - Other failure conditions are specified in `start_link/2` docs. """ - @spec start_link(module, term, options) :: on_start - def start_link(module, arg, options \\ []) when is_list(options) do + + # It is important to keep the two-arity spec because it is a catch-all + # to start_link(children, options). + @spec start_link(module, term) :: on_start + @spec start_link(module, term, [option]) :: on_start + def start_link(module, init_arg, options \\ []) when is_list(options) do case Keyword.get(options, :name) do nil -> - :supervisor.start_link(module, arg) + :supervisor.start_link(module, init_arg) + atom when is_atom(atom) -> - :supervisor.start_link({:local, atom}, module, arg) - other when is_tuple(other) -> - :supervisor.start_link(other, module, arg) + :supervisor.start_link({:local, atom}, module, init_arg) + + {:global, _term} = tuple -> + :supervisor.start_link(tuple, module, init_arg) + + {:via, via_module, _term} = tuple when is_atom(via_module) -> + :supervisor.start_link(tuple, module, init_arg) + + other -> + raise ArgumentError, """ + expected :name option to be one of the following: + + * nil + * atom + * {:global, term} + * {:via, module, term} + + Got: #{inspect(other)} + """ end end @doc """ - Dynamically adds and starts a child specification to the supervisor. + Adds a child specification to `supervisor` and starts that child. - `child_spec` should be a valid child specification (unless the supervisor - is a `:simple_one_for_one` supervisor, see below). The child process will + `child_spec` should be a valid child specification. The child process will be started as defined in the child specification. - In the case of `:simple_one_for_one`, the child specification defined in - the supervisor will be used and instead of a `child_spec`, an arbitrary list - of terms is expected. The child process will then be started by appending - the given list to the existing function arguments in the child specification. - - If there already exists a child specification with the specified id, - `child_spec` is discarded and the function returns an error with `:already_started` - or `:already_present` if the corresponding child process is running or not. + If a child specification with the specified ID already exists, `child_spec` is + discarded and this function returns an error with `:already_started` or + `:already_present` if the corresponding child process is running or not, + respectively. + + If the child process start function returns `{:ok, child}` or `{:ok, child, + info}`, then child specification and PID are added to the supervisor and + this function returns the same value. + + If the child process start function returns `:ignore`, the child specification + is added to the supervisor, the PID is set to `:undefined` and this function + returns `{:ok, :undefined}`. + + If the child process start function returns an error tuple or an erroneous + value, or if it fails, the child specification is discarded and this function + returns `{:error, error}` where `error` is a term containing information about + the error and child specification. + + > #### Order Among Children {: .tip} + > + > The child specification is **appended** to the children of `supervisor`. + > This guarantees that semantics of things such as the `:rest_for_one` strategy + > are preserved correctly. + """ + @spec start_child( + supervisor, + child_spec | module_spec | (old_erlang_child_spec :: :supervisor.child_spec()) + ) :: on_start_child + def start_child(supervisor, {_, _, _, _, _, _} = child_spec) do + call(supervisor, {:start_child, child_spec}) + end - If the child process start function returns `{:ok, child}` or `{:ok, child, info}`, - the child specification and pid is added to the supervisor and the function returns - the same value. + def start_child(supervisor, args) when is_list(args) do + IO.warn_once( + {__MODULE__, :start_child}, + fn -> + "Supervisor.start_child/2 with a list of args is deprecated, please use DynamicSupervisor instead" + end, + _stacktrace_drop_levels = 2 + ) - If the child process start function returns `:ignore, the child specification is - added to the supervisor, the pid is set to undefined and the function returns - `{:ok, :undefined}`. + call(supervisor, {:start_child, args}) + end - If the child process start function returns an error tuple or an erroneous value, - or if it fails, the child specification is discarded and the function returns - `{:error, error}` where `error` is a term containing information about the error - and child specification. - """ - @spec start_child(supervisor, Supervisor.Spec.spec | [term]) :: on_start_child - defdelegate start_child(supervisor, child_spec_or_args), to: :supervisor + def start_child(supervisor, child_spec) do + call(supervisor, {:start_child, Supervisor.child_spec(child_spec, [])}) + end @doc """ - Terminates the given pid or child id. + Terminates the given child identified by `child_id`. - If the supervisor is not a `simple_one_for_one`, the child id is expected - and the process, if there is one, is terminated; the child specification is + The process is terminated, if there's one. The child specification is kept unless the child is temporary. - In case of a `simple_one_for_one` supervisor, a pid is expected. If the child - specification identifier is given instead of a `pid`, the function will - return `{:error, :simple_one_for_one}`. - - A non-temporary child process may later be restarted by the supervisor. The child - process can also be restarted explicitly by calling `restart_child/2`. Use - `delete_child/2` to remove the child specification. + A non-temporary child process may later be restarted by the supervisor. + The child process can also be restarted explicitly by calling `restart_child/2`. + Use `delete_child/2` to remove the child specification. - If successful, the function returns `:ok`. If there is no child specification or - pid, the function returns `{:error, :not_found}`. + If successful, this function returns `:ok`. If there is no child + specification for the given child ID, this function returns + `{:error, :not_found}`. """ - @spec terminate_child(supervisor, pid | Supervisor.Spec.child_id) :: :ok | {:error, error} - when error: :not_found | :simple_one_for_one - defdelegate terminate_child(supervisor, pid_or_child_id), to: :supervisor + @spec terminate_child(supervisor, term()) :: :ok | {:error, :not_found} + def terminate_child(supervisor, child_id) + + def terminate_child(supervisor, pid) when is_pid(pid) do + IO.warn( + "Supervisor.terminate_child/2 with a PID is deprecated, please use DynamicSupervisor instead" + ) + + call(supervisor, {:terminate_child, pid}) + end + + def terminate_child(supervisor, child_id) do + call(supervisor, {:terminate_child, child_id}) + end @doc """ Deletes the child specification identified by `child_id`. - The corresponding child process must not be running, use `terminate_child/2` - to terminate it. - - If successful, the function returns `:ok`. This function may error with an - appropriate error tuple if the `child_id` is not found, or if the current - process is running or being restarted. + The corresponding child process must not be running; use `terminate_child/2` + to terminate it if it's running. - This operation is not supported by `simple_one_for_one` supervisors. + If successful, this function returns `:ok`. This function may return an error + with an appropriate error tuple if the `child_id` is not found, or if the + current process is running or being restarted. """ - @spec delete_child(supervisor, Supervisor.Spec.child_id) :: :ok | {:error, error} - when error: :not_found | :simple_one_for_one | :running | :restarting - defdelegate delete_child(supervisor, child_id), to: :supervisor + @spec delete_child(supervisor, term()) :: :ok | {:error, error} + when error: :not_found | :running | :restarting + def delete_child(supervisor, child_id) do + call(supervisor, {:delete_child, child_id}) + end @doc """ Restarts a child process identified by `child_id`. @@ -317,54 +1088,55 @@ defmodule Supervisor do Note that for temporary children, the child specification is automatically deleted when the child terminates, and thus it is not possible to restart such children. - If the child process start function returns `{:ok, child}` or - `{:ok, child, info}`, the pid is added to the supervisor and the function returns - the same value. + If the child process start function returns `{:ok, child}` or `{:ok, child, info}`, + the PID is added to the supervisor and this function returns the same value. - If the child process start function returns `:ignore`, the pid remains set to - `:undefined` and the function returns `{:ok, :undefined}`. + If the child process start function returns `:ignore`, the PID remains set to + `:undefined` and this function returns `{:ok, :undefined}`. - This function may error with an appropriate error tuple if the `child_id` is not - found, or if the current process is running or being restarted. + This function may return an error with an appropriate error tuple if the + `child_id` is not found, or if the current process is running or being + restarted. If the child process start function returns an error tuple or an erroneous value, - or if it fails, the function returns `{:error, error}`. - - This operation is not supported by `simple_one_for_one` supervisors. + or if it fails, this function returns `{:error, error}`. """ - @spec restart_child(supervisor, Supervisor.Spec.child_id) :: - {:ok, child} | {:ok, child, term} | {:error, error} - when error: :not_found | :simple_one_for_one | :running | :restarting | term - defdelegate restart_child(supervisor, child_id), to: :supervisor + @spec restart_child(supervisor, term()) :: {:ok, child} | {:ok, child, term} | {:error, error} + when error: :not_found | :running | :restarting | term + def restart_child(supervisor, child_id) do + call(supervisor, {:restart_child, child_id}) + end @doc """ - Returns a list with information about all children. + Returns a list with information about all children of the given supervisor. Note that calling this function when supervising a large number of children - under low memory conditions can cause an out of memory exception. + under low memory conditions can bring the system down due to an out of memory + error. + + This function returns a list of `{id, child, type, modules}` tuples, where: - This function returns a list of tuples containing: + * `id` - as defined in the child specification - * `id` - as defined in the child specification or `:undefined` in the case - of a `simple_one_for_one` supervisor + * `child` - the PID of the corresponding child process, `:restarting` if the + process is about to be restarted, or `:undefined` if there is no such + process - * `child` - the pid of the corresponding child process, the atom - `:restarting` if the process is about to be restarted, or `:undefined` if - there is no such process + * `type` - `:worker` or `:supervisor`, as specified by the child specification - * `type` - `:worker` or `:supervisor` as defined in the child specification + * `modules` - as specified by the child specification - * `modules` – as defined in the child specification """ - @spec which_children(supervisor) :: - [{Supervisor.Spec.child_id | :undefined, - child | :restarting, - Supervisor.Spec.worker, - Supervisor.Spec.modules}] - defdelegate which_children(supervisor), to: :supervisor + @spec which_children(supervisor) :: [ + # inlining module() | :dynamic here because :supervisor.modules() is not exported + {term() | :undefined, child | :restarting, :worker | :supervisor, [module()] | :dynamic} + ] + def which_children(supervisor) do + call(supervisor, :which_children) + end @doc """ - Returns a map containing count values for the supervisor. + Returns a map containing count values for the given supervisor. The map contains the following keys: @@ -373,17 +1145,41 @@ defmodule Supervisor do * `:active` - the count of all actively running child processes managed by this supervisor - * `:supervisors` - the count of all supervisors whether or not the child - process is still alive + * `:supervisors` - the count of all supervisors whether or not these + child supervisors are still alive - * `:workers` - the count of all workers, whether or not the child process - is still alive + * `:workers` - the count of all workers, whether or not these child workers + are still alive """ - @spec count_children(supervisor) :: - [specs: non_neg_integer, active: non_neg_integer, - supervisors: non_neg_integer, workers: non_neg_integer] + @spec count_children(supervisor) :: %{ + specs: non_neg_integer, + active: non_neg_integer, + supervisors: non_neg_integer, + workers: non_neg_integer + } def count_children(supervisor) do - :supervisor.count_children(supervisor) |> :maps.from_list + call(supervisor, :count_children) |> :maps.from_list() + end + + @doc """ + Synchronously stops the given supervisor with the given `reason`. + + It returns `:ok` if the supervisor terminates with the given + reason. If it terminates with another reason, the call exits. + + This function keeps OTP semantics regarding error reporting. + If the reason is any other than `:normal`, `:shutdown` or + `{:shutdown, _}`, an error report is logged. + """ + @spec stop(supervisor, reason :: term, timeout) :: :ok + def stop(supervisor, reason \\ :normal, timeout \\ :infinity) do + GenServer.stop(supervisor, reason, timeout) + end + + @compile {:inline, call: 2} + + defp call(supervisor, req) do + GenServer.call(supervisor, req, :infinity) end end diff --git a/lib/elixir/lib/supervisor/default.ex b/lib/elixir/lib/supervisor/default.ex index 863bce65907..10bac507a7a 100644 --- a/lib/elixir/lib/supervisor/default.ex +++ b/lib/elixir/lib/supervisor/default.ex @@ -1,13 +1,11 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + defmodule Supervisor.Default do @moduledoc false - @behaviour :supervisor - - @doc """ - Supevisor callback that simply returns the given args. - This is the supervisor used by `Supervisor.start_link/2`. - """ def init(args) do args end -end \ No newline at end of file +end diff --git a/lib/elixir/lib/supervisor/spec.ex b/lib/elixir/lib/supervisor/spec.ex index df9f2625b22..454d01877ab 100644 --- a/lib/elixir/lib/supervisor/spec.ex +++ b/lib/elixir/lib/supervisor/spec.ex @@ -1,11 +1,21 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + defmodule Supervisor.Spec do @moduledoc """ - Convenience functions for defining a supervision specification. + Outdated functions for building child specifications. + + The functions in this module are deprecated and they do not work + with the module-based child specs introduced in Elixir v1.5. + Please see the `Supervisor` documentation instead. + + Convenience functions for defining supervisor specifications. ## Example - By using the functions in this module one can define a supervisor - and start it with `Supervisor.start_link/2`: + By using the functions in this module one can specify the children + to be used under a supervisor, started with `Supervisor.start_link/2`: import Supervisor.Spec @@ -16,7 +26,7 @@ defmodule Supervisor.Spec do Supervisor.start_link(children, strategy: :one_for_one) - In many situations, it may be handy to define supervisors backed + Sometimes, it may be handy to define supervisors backed by a module: defmodule MySupervisor do @@ -35,44 +45,37 @@ defmodule Supervisor.Spec do end end - Notice in this case we don't have to explicitly import - `Supervisor.Spec` as `use Supervisor` automatically does so. - - Explicit supervisors as above are required when there is a need to: - - 1. Partialy change the supervision tree during hot-code swaps. - - 2. Define supervisors inside other supervisors. - - 3. Perform actions inside the supervision `init/1` callback. - - For example, you may want to start an ETS table that is linked to - the supervisor (i.e. if the supervision tree needs to be restarted, - the ETS table must be restarted too). + Note that in this case we don't have to explicitly import + `Supervisor.Spec` since `use Supervisor` automatically does so. + Defining a module-based supervisor can be useful, for example, + to perform initialization tasks in the `c:Supervisor.init/1` callback. ## Supervisor and worker options - In the example above, we have defined workers and supervisors - and each accepts the following options: + In the example above, we defined specs for workers and supervisors. + These specs (both for workers as well as supervisors) accept the + following options: * `:id` - a name used to identify the child specification internally by the supervisor; defaults to the given module - name + name for the child worker/supervisor * `:function` - the function to invoke on the child to start it - * `:restart` - defines when the child process should restart + * `:restart` - an atom that defines when a terminated child process should + be restarted (see the "Restart values" section below) - * `:shutdown` - defines how a child process should be terminated + * `:shutdown` - an atom that defines how a child process should be + terminated (see the "Shutdown values" section below) * `:modules` - it should be a list with one element `[module]`, where module is the name of the callback module only if the child process is a `Supervisor` or `GenServer`; if the child - process is a `GenEvent`, modules should be `:dynamic` + process is a `GenEvent`, `:modules` should be `:dynamic` - ### Restart values + ### Restart values (:restart) - The following restart values are supported: + The following restart values are supported in the `:restart` option: * `:permanent` - the child process is always restarted @@ -80,27 +83,36 @@ defmodule Supervisor.Spec do when the supervisor's strategy is `:rest_for_one` or `:one_for_all`) * `:transient` - the child process is restarted only if it - terminates abnormally, i.e. with another exit reason than + terminates abnormally, i.e., with an exit reason other than `:normal`, `:shutdown` or `{:shutdown, term}` - ### Shutdown values + Note that supervisor that reached maximum restart intensity will exit with `:shutdown` reason. + In this case the supervisor will only restart if its child specification was defined with + the `:restart` option set to `:permanent` (the default). + + ### Shutdown values (`:shutdown`) - The following shutdown values are supported: + The following shutdown values are supported in the `:shutdown` option: * `:brutal_kill` - the child process is unconditionally terminated - using `exit(child, :kill)`. + using `Process.exit(child, :kill)` - * `:infinity` - if the child process is a supervisor, it is a mechanism - to give the subtree enough time to shutdown. It can also be used with - workers with care. + * `:infinity` - if the child process is a supervisor, this is a mechanism + to give the subtree enough time to shut down; it can also be used with + workers with care + + * a non-negative integer - the amount of time in milliseconds + that the supervisor tells the child process to terminate by calling + `Process.exit(child, :shutdown)` and then waits for an exit signal back. + If no exit signal is received within the specified time, + the child process is unconditionally terminated + using `Process.exit(child, :kill)` - * Finally, it can also be any integer meaning that the supervisor tells - the child process to terminate by calling `Process.exit(child, :shutdown)` - and then waits for an exit signal back. If no exit signal is received - within the specified time (in miliseconds), the child process is - unconditionally terminated using `Process.exit(child, :kill)`. """ + @moduledoc deprecated: + "Use the new child specifications outlined in the Supervisor module instead" + @typedoc "Supported strategies" @type strategy :: :simple_one_for_one | :one_for_one | :one_for_all | :rest_for_one @@ -108,7 +120,7 @@ defmodule Supervisor.Spec do @type restart :: :permanent | :transient | :temporary @typedoc "Supported shutdown values" - @type shutdown :: :brutal_kill | :infinity | non_neg_integer + @type shutdown :: timeout | :brutal_kill @typedoc "Supported worker values" @type worker :: :worker | :supervisor @@ -116,26 +128,24 @@ defmodule Supervisor.Spec do @typedoc "Supported module values" @type modules :: :dynamic | [module] - @typedoc "Supported id values" + @typedoc "Supported ID values" @type child_id :: term @typedoc "The supervisor specification" - @type spec :: {child_id, - start_fun :: {module, atom, [term]}, - restart, - shutdown, - worker, - modules} + @type spec :: + {child_id, start_fun :: {module, atom, [term]}, restart, shutdown, worker, modules} @doc """ - Receives a list of children (workers or supervisors) to - supervise and a set of options. + Receives a list of `children` (workers or supervisors) to + supervise and a set of `options`. - Returns a tuple containing the supervisor specification. + Returns a tuple containing the supervisor specification. This tuple can be + used as the return value of the `c:Supervisor.init/1` callback when implementing a + module-based supervisor. ## Examples - supervise children, strategy: :one_for_one + supervise(children, strategy: :one_for_one) ## Options @@ -144,36 +154,52 @@ defmodule Supervisor.Spec do `:simple_one_for_one`. You can learn more about strategies in the `Supervisor` module docs. - * `:max_restarts` - the maximum amount of restarts allowed in - a time frame. Defaults to 5. + * `:max_restarts` - the maximum number of restarts allowed in + a time frame. Defaults to `3`. * `:max_seconds` - the time frame in which `:max_restarts` applies. - Defaults to 5. + Defaults to `5`. - The `:strategy` option is required and by default maximum 5 restarts - are allowed within 5 seconds. Please check the `Supervisor` module for - a complete description of the available strategies. + The `:strategy` option is required and by default a maximum of 3 restarts is + allowed within 5 seconds. Check the `Supervisor` module for a detailed + description of the available strategies. """ - @spec supervise([spec], strategy: strategy, - max_restarts: non_neg_integer, - max_seconds: non_neg_integer) :: {:ok, tuple} + @spec supervise( + [spec], + strategy: strategy, + max_restarts: non_neg_integer, + max_seconds: pos_integer + ) :: {:ok, tuple} + @deprecated "Use the new child specifications outlined in the Supervisor module instead" def supervise(children, options) do - unless strategy = options[:strategy] do + if !(strategy = options[:strategy]) do raise ArgumentError, "expected :strategy option to be given" end - maxR = Keyword.get(options, :max_restarts, 5) + maxR = Keyword.get(options, :max_restarts, 3) maxS = Keyword.get(options, :max_seconds, 5) - assert_unique_ids(Enum.map(children, &elem(&1, 0))) + assert_unique_ids(Enum.map(children, &get_id/1)) {:ok, {{strategy, maxR, maxS}, children}} end - defp assert_unique_ids([id|rest]) do + defp get_id({id, _, _, _, _, _}) do + id + end + + defp get_id(other) do + raise ArgumentError, + "invalid tuple specification given to supervise/2. If you are trying to use " <> + "the map child specification that is part of the Elixir v1.5, use Supervisor.init/2 " <> + "instead of Supervisor.Spec.supervise/2. See the Supervisor module for more information. " <> + "Got: #{inspect(other)}" + end + + defp assert_unique_ids([id | rest]) do if id in rest do raise ArgumentError, - "duplicated id #{inspect id} found in the supervisor specification, " <> - "please explicitly pass the :id option when defining this worker/supervisor" + "duplicate ID #{inspect(id)} found in the supervisor specification, " <> + "please explicitly pass the :id option when defining this worker/supervisor" else assert_unique_ids(rest) end @@ -187,22 +213,32 @@ defmodule Supervisor.Spec do Defines the given `module` as a worker which will be started with the given arguments. - worker ExUnit.Runner, [], restart: :permanent + worker(ExUnit.Runner, [], restart: :permanent) By default, the function `start_link` is invoked on the given module. Overall, the default values for the options are: - [id: module, - function: :start_link, - restart: :permanent, - shutdown: 5000, - modules: [module]] + [ + id: module, + function: :start_link, + restart: :permanent, + shutdown: 5000, + modules: [module] + ] - Check `Supervisor.Spec` module docs for more information on - the options. + See the "Supervisor and worker options" section in the `Supervisor.Spec` module for more + information on the available options. """ - @spec worker(module, [term], [restart: restart, shutdown: shutdown, - id: term, function: atom, modules: modules]) :: spec + @spec worker( + module, + [term], + restart: restart, + shutdown: shutdown, + id: term, + function: atom, + modules: modules + ) :: spec + @deprecated "Use the new child specifications outlined in the Supervisor module instead" def worker(module, args, options \\ []) do child(:worker, module, args, options) end @@ -211,38 +247,47 @@ defmodule Supervisor.Spec do Defines the given `module` as a supervisor which will be started with the given arguments. - supervisor ExUnit.Runner, [], restart: :permanent + supervisor(module, [], restart: :permanent) By default, the function `start_link` is invoked on the given module. Overall, the default values for the options are: - [id: module, - function: :start_link, - restart: :permanent, - shutdown: :infinity, - modules: [module]] + [ + id: module, + function: :start_link, + restart: :permanent, + shutdown: :infinity, + modules: [module] + ] - Check `Supervisor.Spec` module docs for more information on - the options. + See the "Supervisor and worker options" section in the `Supervisor.Spec` module for more + information on the available options. """ - @spec supervisor(module, [term], [restart: restart, shutdown: shutdown, - id: term, function: atom, modules: modules]) :: spec + @spec supervisor( + module, + [term], + restart: restart, + shutdown: shutdown, + id: term, + function: atom, + modules: modules + ) :: spec + @deprecated "Use the new child specifications outlined in the Supervisor module instead" def supervisor(module, args, options \\ []) do options = Keyword.put_new(options, :shutdown, :infinity) child(:supervisor, module, args, options) end defp child(type, module, args, options) do - id = Keyword.get(options, :id, module) - modules = Keyword.get(options, :modules, modules(module)) + id = Keyword.get(options, :id, module) + modules = Keyword.get(options, :modules, modules(module)) function = Keyword.get(options, :function, :start_link) - restart = Keyword.get(options, :restart, :permanent) + restart = Keyword.get(options, :restart, :permanent) shutdown = Keyword.get(options, :shutdown, 5000) - {id, {module, function, args}, - restart, shutdown, type, modules} + {id, {module, function, args}, restart, shutdown, type, modules} end defp modules(GenEvent), do: :dynamic - defp modules(module), do: [module] + defp modules(module), do: [module] end diff --git a/lib/elixir/lib/system.ex b/lib/elixir/lib/system.ex index 6b35d7abf32..36094065872 100644 --- a/lib/elixir/lib/system.ex +++ b/lib/elixir/lib/system.ex @@ -1,50 +1,228 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + defmodule System do @moduledoc """ - The System module provides access to variables used or - maintained by the VM and to functions that interact directly + The `System` module provides functions that interact directly with the VM or the host system. + + ## Time + + The `System` module also provides functions that work with time, + returning different times kept by the system with support for + different time units. + + One of the complexities in relying on system times is that they + may be adjusted. For example, when you enter and leave daylight + saving time, the system clock will be adjusted, often adding + or removing one hour. We call such changes "time warps". In + order to understand how such changes may be harmful, imagine + the following code: + + ## DO NOT DO THIS + prev = System.os_time() + # ... execute some code ... + next = System.os_time() + diff = next - prev + + If, while the code is executing, the system clock changes, + some code that executed in 1 second may be reported as taking + over 1 hour! To address such concerns, the VM provides a + monotonic time via `System.monotonic_time/0` which never + decreases and does not leap: + + ## DO THIS + prev = System.monotonic_time() + # ... execute some code ... + next = System.monotonic_time() + diff = next - prev + + Generally speaking, the VM provides three time measurements: + + * `os_time/0` - the time reported by the operating system (OS). This time may be + adjusted forwards or backwards in time with no limitation; + + * `system_time/0` - the VM view of the `os_time/0`. The system time and operating + system time may not match in case of time warps although the VM works towards + aligning them. This time is not monotonic (i.e., it may decrease) + as its behavior is configured [by the VM time warp + mode](https://www.erlang.org/doc/apps/erts/time_correction.html#Time_Warp_Modes); + + * `monotonic_time/0` - a monotonically increasing time provided + by the Erlang VM. This is not strictly monotonically increasing. Multiple + sequential calls of the function may return the same value. + + The time functions in this module work in the `:native` unit + (unless specified otherwise), which is operating system dependent. Most of + the time, all calculations are done in the `:native` unit, to + avoid loss of precision, with `convert_time_unit/3` being + invoked at the end to convert to a specific time unit like + `:millisecond` or `:microsecond`. See the `t:time_unit/0` type for + more information. + + For a more complete rundown on the VM support for different + times, see the [chapter on time and time + correction](https://www.erlang.org/doc/apps/erts/time_correction.html) + in the Erlang docs. """ - defp strip_re(iodata, pattern) do - :re.replace(iodata, pattern, "", [return: :binary]) + defmodule EnvError do + @moduledoc """ + An exception raised when a system environment variable is not set. + + For example, see `System.fetch_env!/1`. + """ + + defexception [:env] + + @impl true + def message(%{env: env}) do + "could not fetch environment variable #{inspect(env)} because it is not set" + end + end + + @typedoc """ + The time unit to be passed to functions like `monotonic_time/1` and others. + + The `:second`, `:millisecond`, `:microsecond` and `:nanosecond` time + units controls the return value of the functions that accept a time unit. + + A time unit can also be a strictly positive integer. In this case, it + represents the "parts per second": the time will be returned in `1 / + parts_per_second` seconds. For example, using the `:millisecond` time unit + is equivalent to using `1000` as the time unit (as the time will be returned + in 1/1000 seconds - milliseconds). + """ + @type time_unit :: + :second + | :millisecond + | :microsecond + | :nanosecond + | pos_integer + + @type signal :: + :sigabrt + | :sigalrm + | :sigchld + | :sighup + | :sigquit + | :sigstop + | :sigterm + | :sigtstp + | :sigusr1 + | :sigusr2 + + @type cmd_opts :: [ + into: Collectable.t(), + lines: pos_integer(), + cd: Path.t(), + env: [{binary(), binary() | nil}], + arg0: binary(), + stderr_to_stdout: boolean(), + use_stdio: boolean(), + parallelism: boolean() + ] + + @type shell_opts :: [ + into: Collectable.t(), + lines: pos_integer(), + cd: Path.t(), + env: [{binary(), binary() | nil}], + stderr_to_stdout: boolean(), + use_stdio: boolean(), + parallelism: boolean(), + close_stdin: boolean() + ] + + @vm_signals [:sigquit, :sigterm, :sigusr1] + @os_signals [:sighup, :sigabrt, :sigalrm, :sigusr2, :sigchld, :sigstop, :sigtstp] + @signals @vm_signals ++ @os_signals + + @base_dir :filename.join(__DIR__, "../../..") + @version_file :filename.join(@base_dir, "VERSION") + + defp strip(iodata) do + :re.replace(iodata, "^[\s\r\n\t]+|[\s\r\n\t]+$", "", [:global, return: :binary]) end defp read_stripped(path) do case :file.read_file(path) do {:ok, binary} -> - strip_re(binary, "^\s+|\s+$") - _ -> "" + strip(binary) + + _ -> + "" end end - # Read and strip the version from the `VERSION` file. + # Read and strip the version from the VERSION file. defmacrop get_version do - case read_stripped(:filename.join(__DIR__, "../../../VERSION")) do - "" -> raise RuntimeError, message: "could not read the version number from VERSION" + case read_stripped(@version_file) do + "" -> raise "could not read the version number from VERSION" data -> data end end - # Tries to run `git describe --always --tags`. In the case of success returns - # the most recent tag. If that is not available, tries to read the commit hash - # from .git/HEAD. If that fails, returns an empty string. - defmacrop get_describe do - dirpath = :filename.join(__DIR__, "../../../.git") - case :file.read_file_info(dirpath) do - {:ok, _} -> - if :os.find_executable('git') do - data = :os.cmd('git describe --always --tags') - strip_re(data, "\n") - else - read_stripped(:filename.join(".git", "HEAD")) - end - _ -> "" - end + # Returns OTP version that Elixir was compiled with. + defmacrop get_otp_release do + :erlang.list_to_binary(:erlang.system_info(:otp_release)) + end + + # Tries to run "git rev-parse --short=7 HEAD". In the case of success returns + # the short revision hash. If that fails, returns an empty string. + defmacrop get_revision do + null = + case :os.type() do + {:win32, _} -> ~c"NUL" + _ -> ~c"/dev/null" + end + + ~c"git rev-parse --short=7 HEAD 2> " + |> Kernel.++(null) + |> :os.cmd() + |> strip() end + defp revision, do: get_revision() + # Get the date at compilation time. + # Follows https://reproducible-builds.org/specs/source-date-epoch/ defmacrop get_date do - IO.iodata_to_binary :httpd_util.rfc1123_date + unix_epoch = + if source_date_epoch = :os.getenv(~c"SOURCE_DATE_EPOCH") do + try do + List.to_integer(source_date_epoch) + rescue + _ -> nil + end + end + + unix_epoch = unix_epoch || :os.system_time(:second) + + {{year, month, day}, {hour, minute, second}} = + :calendar.gregorian_seconds_to_datetime(unix_epoch + 62_167_219_200) + + "~4..0b-~2..0b-~2..0bT~2..0b:~2..0b:~2..0bZ" + |> :io_lib.format([year, month, day, hour, minute, second]) + |> :erlang.iolist_to_binary() + end + + @doc """ + Returns the endianness. + """ + @spec endianness() :: :little | :big + def endianness do + :erlang.system_info(:endian) + end + + @doc """ + Returns the endianness the system was compiled with. + """ + @endianness :erlang.system_info(:endian) + @spec compiled_endianness() :: :little | :big + def compiled_endianness do + @endianness end @doc """ @@ -52,38 +230,105 @@ defmodule System do Returns Elixir's version as binary. """ - @spec version() :: String.t - def version, do: get_version + @spec version() :: String.t() + def version, do: get_version() @doc """ Elixir build information. - Returns a keyword list with Elixir version, git tag info and compilation date. + Returns a map with the Elixir version, the Erlang/OTP release it was compiled + with, a short Git revision hash and the date and time it was built. + + Every value in the map is a string, and these are: + + * `:build` - the Elixir version, short Git revision hash and + Erlang/OTP release it was compiled with + * `:date` - a string representation of the ISO8601 date and time it was built + * `:otp_release` - OTP release it was compiled with + * `:revision` - short Git revision hash. If Git was not available at building + time, it is set to `""` + * `:version` - the Elixir version + + One should not rely on the specific formats returned by each of those fields. + Instead one should use specialized functions, such as `version/0` to retrieve + the Elixir version and `otp_release/0` to retrieve the Erlang/OTP release. + + ## Examples + + iex> System.build_info() + %{ + build: "1.9.0-dev (772a00a0c) (compiled with Erlang/OTP 21)", + date: "2018-12-24T01:09:21Z", + otp_release: "21", + revision: "772a00a0c", + version: "1.9.0-dev" + } + """ - @spec build_info() :: map + @spec build_info() :: %{ + build: String.t(), + date: String.t(), + revision: String.t(), + version: String.t(), + otp_release: String.t() + } def build_info do - %{version: version, tag: get_describe, date: get_date} + %{ + build: build(), + date: get_date(), + revision: revision(), + version: version(), + otp_release: get_otp_release() + } + end + + # Returns a string of the build info + defp build do + {:ok, v} = Version.parse(version()) + + revision_string = if v.pre != [] and revision() != "", do: " (#{revision()})", else: "" + otp_version_string = " (compiled with Erlang/OTP #{get_otp_release()})" + + version() <> revision_string <> otp_version_string end @doc """ - List command line arguments. + Lists command line arguments. Returns the list of command line arguments passed to the program. """ - @spec argv() :: [String.t] + @spec argv() :: [String.t()] def argv do - :elixir_code_server.call :argv + :elixir_config.get(:argv) end @doc """ - Modify command line arguments. + Modifies command line arguments. Changes the list of command line arguments. Use it with caution, as it destroys any previous argv information. """ - @spec argv([String.t]) :: :ok + @spec argv([String.t()]) :: :ok def argv(args) do - :elixir_code_server.cast({:argv, args}) + :elixir_config.put(:argv, args) + end + + @doc """ + Marks if the system should halt or not at the end of ARGV processing. + """ + @doc since: "1.9.0" + @spec no_halt(boolean) :: :ok + def no_halt(boolean) when is_boolean(boolean) do + :elixir_config.put(:no_halt, boolean) + end + + @doc """ + Checks if the system will halt or not at the end of ARGV processing. + """ + @doc since: "1.9.0" + @spec no_halt() :: boolean + def no_halt() do + :elixir_config.get(:no_halt) end @doc """ @@ -92,9 +337,11 @@ defmodule System do Returns the current working directory or `nil` if one is not available. """ + @deprecated "Use File.cwd/0 instead" + @spec cwd() :: String.t() | nil def cwd do - case :file.get_cwd do - {:ok, base} -> IO.chardata_to_string(base) + case File.cwd() do + {:ok, cwd} -> cwd _ -> nil end end @@ -104,21 +351,32 @@ defmodule System do Returns the current working directory or raises `RuntimeError`. """ + @deprecated "Use File.cwd!/0 instead" + @spec cwd!() :: String.t() def cwd! do - cwd || - raise RuntimeError, message: "could not get a current working directory, the current location is not accessible" + case File.cwd() do + {:ok, cwd} -> + cwd + + _ -> + raise "could not get a current working directory, the current location is not accessible" + end end @doc """ User home directory. Returns the user home directory (platform independent). - Returns `nil` if no user home is set. """ + @spec user_home() :: String.t() | nil def user_home do - case :os.type() do - {:win32, _} -> get_windows_home - _ -> get_unix_home + case :init.get_argument(:home) do + {:ok, [[home] | _]} -> + encoding = :file.native_name_encoding() + :unicode.characters_to_binary(home, encoding, encoding) + + _ -> + nil end end @@ -128,23 +386,9 @@ defmodule System do Same as `user_home/0` but raises `RuntimeError` instead of returning `nil` if no user home is set. """ + @spec user_home!() :: String.t() def user_home! do - user_home || - raise RuntimeError, message: "could not find the user home, please set the HOME environment variable" - end - - defp get_unix_home do - get_env("HOME") - end - - defp get_windows_home do - :filename.absname( - get_env("USERPROFILE") || ( - hd = get_env("HOMEDRIVE") - hp = get_env("HOMEPATH") - hd && hp && hd <> hp - ) - ) + user_home() || raise "could not find the user home, please set the HOME environment variable" end @doc ~S""" @@ -156,17 +400,22 @@ defmodule System do 1. the directory named by the TMPDIR environment variable 2. the directory named by the TEMP environment variable 3. the directory named by the TMP environment variable - 4. `C:\TMP` on Windows or `/tmp` on Unix + 4. `C:\TMP` on Windows or `/tmp` on Unix-like operating systems 5. as a last resort, the current working directory Returns `nil` if none of the above are writable. """ + @spec tmp_dir() :: String.t() | nil def tmp_dir do - write_env_tmp_dir('TMPDIR') || - write_env_tmp_dir('TEMP') || - write_env_tmp_dir('TMP') || - write_tmp_dir('/tmp') || - ((cwd = cwd()) && write_tmp_dir(cwd)) + write_env_tmp_dir(~c"TMPDIR") || write_env_tmp_dir(~c"TEMP") || write_env_tmp_dir(~c"TMP") || + write_tmp_dir(~c"/tmp") || write_cwd_tmp_dir() + end + + defp write_cwd_tmp_dir do + case File.cwd() do + {:ok, cwd} -> write_tmp_dir(cwd) + _ -> nil + end end @doc """ @@ -175,16 +424,16 @@ defmodule System do Same as `tmp_dir/0` but raises `RuntimeError` instead of returning `nil` if no temp dir is set. """ + @spec tmp_dir!() :: String.t() def tmp_dir! do - tmp_dir || - raise RuntimeError, message: "could not get a writable temporary directory, " <> - "please set the TMPDIR environment variable" + tmp_dir() || + raise "could not get a writable temporary directory, please set the TMPDIR environment variable" end defp write_env_tmp_dir(env) do case :os.getenv(env) do false -> nil - tmp -> write_tmp_dir(tmp) + tmp -> write_tmp_dir(tmp) end end @@ -194,135 +443,353 @@ defmodule System do case {stat.type, stat.access} do {:directory, access} when access in [:read_write, :write] -> IO.chardata_to_string(dir) + _ -> nil end - {:error, _} -> nil + + {:error, _} -> + nil end end @doc """ - Register a program exit handler function. + Registers a program exit handler function. + + Registers a function that will be invoked at the end of an Elixir script. + A script is typically started via the command line via the `elixir` and + `mix` executables. - Registers a function that will be invoked - at the end of program execution. Useful for - invoking a hook in "script" mode. + The handler always executes in a different process from the one it was + registered in. As a consequence, any resources managed by the calling process + (ETS tables, open files, and others) won't be available by the time the handler + function is invoked. - The function must receive the exit status code - as an argument. + The function must receive the exit status code as an argument. + + If the VM terminates programmatically, via `System.stop/1`, `System.halt/1`, + or exit signals, the `at_exit/1` callbacks are not guaranteed to be executed. """ + @spec at_exit((non_neg_integer -> any)) :: :ok def at_exit(fun) when is_function(fun, 1) do - :elixir_code_server.cast {:at_exit, fun} + :elixir_config.update(:at_exit, &[fun | &1]) + :ok + end + + defmodule SignalHandler do + @moduledoc false + @behaviour :gen_event + + @impl true + def init({event, fun}) do + {:ok, {event, fun}} + end + + @impl true + def handle_call(_message, state) do + {:ok, :ok, state} + end + + @impl true + def handle_event(signal, {event, fun}) do + if signal == event, do: :ok = fun.() + {:ok, {event, fun}} + end + + @impl true + def handle_info(_, {event, fun}) do + {:ok, {event, fun}} + end end @doc """ - Execute a system command. + Traps the given `signal` to execute the `fun`. + + > #### Avoid setting traps in libraries {: .warning} + > + > Trapping signals may have strong implications + > on how a system shuts down and behaves in production and + > therefore it is extremely discouraged for libraries to + > set their own traps. Instead, they should redirect users + > to configure them themselves. The only cases where it is + > acceptable for libraries to set their own traps is when + > using Elixir in script mode, such as in `.exs` files and + > via Mix tasks. + + An optional `id` that uniquely identifies the function + can be given, otherwise a unique one is automatically + generated. If a previously registered `id` is given, + this function returns an error tuple. The `id` can be + used to remove a registered signal by calling + `untrap_signal/2`. + + The given `fun` receives no arguments and it must return + `:ok`. + + It returns `{:ok, id}` in case of success, + `{:error, :already_registered}` in case the id has already + been registered for the given signal, or `{:error, :not_sup}` + in case trapping exists is not supported by the current OS. + + The first time a signal is trapped, it will override the + default behavior from the operating system. If the same + signal is trapped multiple times, subsequent functions + given to `trap_signal` will execute *first*. In other + words, you can consider each function is prepended to + the signal handler. + + By default, the Erlang VM register traps to the three + signals: + + * `:sigstop` - gracefully shuts down the VM with `stop/0` + * `:sigquit` - halts the VM via `halt/0` + * `:sigusr1` - halts the VM via status code of 1 + + Therefore, if you add traps to the signals above, the + default behavior above will be executed after all user + signals. + + ## Implementation notes + + All signals run from a single process. Therefore, blocking the + `fun` will block subsequent traps. It is also not possible to add + or remove traps from within a trap itself. + + Internally, this functionality is built on top of `:os.set_signal/2`. + When you register a trap, Elixir automatically sets it to `:handle` + and it reverts it back to `:default` once all traps are removed + (except for `:sigquit`, `:sigterm`, and `:sigusr1` which are always + handled). If you or a library call `:os.set_signal/2` directly, + it may disable Elixir traps (or Elixir may override your configuration). + """ + @doc since: "1.12.0" + @spec trap_signal(signal, (-> :ok)) :: {:ok, reference()} | {:error, :not_sup} + @spec trap_signal(signal, id, (-> :ok)) :: + {:ok, id} | {:error, :already_registered} | {:error, :not_sup} + when id: term() + def trap_signal(signal, id \\ make_ref(), fun) + when signal in @signals and is_function(fun, 0) do + :elixir_config.serial(fn -> + gen_id = {signal, id} + + if {SignalHandler, gen_id} in signal_handlers() do + {:error, :already_registered} + else + try do + :os.set_signal(signal, :handle) + rescue + _ -> {:error, :not_sup} + else + :ok -> + :ok = + :gen_event.add_handler(:erl_signal_server, {SignalHandler, gen_id}, {signal, fun}) - Executes `command` in a command shell of the target OS, - captures the standard output of the command and returns - the result as a binary. + {:ok, id} + end + end + end) + end - If `command` is a char list, a char list is returned. - Otherwise a string, correctly encoded in UTF-8, is expected. + @doc """ + Removes a previously registered `signal` with `id`. """ - @spec cmd(String.t) :: String.t - @spec cmd(char_list) :: char_list + @doc since: "1.12.0" + @spec untrap_signal(signal, id) :: :ok | {:error, :not_found} when id: term + def untrap_signal(signal, id) when signal in @signals do + :elixir_config.serial(fn -> + gen_id = {signal, id} + + case :gen_event.delete_handler(:erl_signal_server, {SignalHandler, gen_id}, :delete) do + :ok -> + if not trapping?(signal) do + :os.set_signal(signal, :default) + end + + :ok + + {:error, :module_not_found} -> + {:error, :not_found} + end + end) + end - def cmd(command) when is_list(command) do - :os.cmd(command) + defp trapping?(signal) do + signal in @vm_signals or + Enum.any?(signal_handlers(), &match?({_, {^signal, _}}, &1)) end - def cmd(command) when is_binary(command) do - List.to_string :os.cmd(String.to_char_list(command)) + defp signal_handlers do + :gen_event.which_handlers(:erl_signal_server) end @doc """ - Locate an executable on the system. + Locates an executable on the system. This function looks up an executable program given - its name using the environment variable PATH on Unix - and Windows. It also considers the proper executable - extension for each OS, so for Windows it will try to + its name using the environment variable PATH on Windows and Unix-like + operating systems. It also considers the proper executable + extension for each operating system, so for Windows it will try to lookup files with `.com`, `.cmd` or similar extensions. - - If `program` is a char list, a char list is returned. - Returns a binary otherwise. """ @spec find_executable(binary) :: binary | nil - @spec find_executable(char_list) :: char_list | nil - - def find_executable(program) when is_list(program) do - :os.find_executable(program) || nil - end - def find_executable(program) when is_binary(program) do - case :os.find_executable(String.to_char_list(program)) do + assert_no_null_byte!(program, "System.find_executable/1") + + case :os.find_executable(String.to_charlist(program)) do false -> nil other -> List.to_string(other) end end @doc """ - System environment variables. + Returns all system environment variables. - Returns a list of all environment variables. Each variable is given as a - `{name, value}` tuple where both `name` and `value` are strings. + The returned value is a map containing name-value pairs. + Variable names and their values are strings. """ - @spec get_env() :: %{String.t => String.t} + @spec get_env() :: %{optional(String.t()) => String.t()} def get_env do - Enum.into(:os.getenv, %{}, fn var -> - var = IO.chardata_to_string var - [k, v] = String.split var, "=", parts: 2 - {k, v} + Map.new(:os.env(), fn {k, v} -> + {IO.chardata_to_string(k), IO.chardata_to_string(v)} end) end @doc """ - Environment variable value. + Returns the value of the given environment variable. + + The returned value of the environment variable + `varname` is a string. If the environment variable + is not set, returns the string specified in `default` or + `nil` if none is specified. + + ## Examples + + iex> System.get_env("PORT") + "4000" + + iex> System.get_env("NOT_SET") + nil + + iex> System.get_env("NOT_SET", "4001") + "4001" - Returns the value of the environment variable - `varname` as a binary, or `nil` if the environment - variable is undefined. """ - @spec get_env(binary) :: binary | nil - def get_env(varname) when is_binary(varname) do - case :os.getenv(String.to_char_list(varname)) do - false -> nil + @doc since: "1.9.0" + @spec get_env(String.t(), String.t()) :: String.t() + @spec get_env(String.t(), nil) :: String.t() | nil + def get_env(varname, default \\ nil) + when is_binary(varname) and + (is_binary(default) or is_nil(default)) do + case :os.getenv(String.to_charlist(varname)) do + false -> default other -> List.to_string(other) end end + @doc """ + Returns the value of the given environment variable or `:error` if not found. + + If the environment variable `varname` is set, then `{:ok, value}` is returned + where `value` is a string. If `varname` is not set, `:error` is returned. + + ## Examples + + iex> System.fetch_env("PORT") + {:ok, "4000"} + + iex> System.fetch_env("NOT_SET") + :error + + """ + @doc since: "1.9.0" + @spec fetch_env(String.t()) :: {:ok, String.t()} | :error + def fetch_env(varname) when is_binary(varname) do + case :os.getenv(String.to_charlist(varname)) do + false -> :error + other -> {:ok, List.to_string(other)} + end + end + + @doc """ + Returns the value of the given environment variable or raises if not found. + + Same as `get_env/1` but raises instead of returning `nil` when the variable is + not set. + + ## Examples + + iex> System.fetch_env!("PORT") + "4000" + + iex> System.fetch_env!("NOT_SET") + ** (System.EnvError) could not fetch environment variable "NOT_SET" because it is not set + + """ + @doc since: "1.9.0" + @spec fetch_env!(String.t()) :: String.t() + def fetch_env!(varname) when is_binary(varname) do + get_env(varname) || raise(EnvError, env: varname) + end + @doc """ Erlang VM process identifier. Returns the process identifier of the current Erlang emulator in the format most commonly used by the operating system environment. - See http://www.erlang.org/doc/man/os.html#getpid-0 for more info. + For more information, see `:os.getpid/0`. """ + @deprecated "Use System.pid/0 instead" @spec get_pid() :: binary - def get_pid, do: IO.iodata_to_binary(:os.getpid) + def get_pid, do: IO.iodata_to_binary(:os.getpid()) @doc """ - Set an environment variable value. + Sets an environment variable value. Sets a new `value` for the environment variable `varname`. """ @spec put_env(binary, binary) :: :ok def put_env(varname, value) when is_binary(varname) and is_binary(value) do - :os.putenv String.to_char_list(varname), String.to_char_list(value) - :ok + case :binary.match(varname, "=") do + {_, _} -> + raise ArgumentError, + "cannot execute System.put_env/2 for key with \"=\", got: #{inspect(varname)}" + + :nomatch -> + :os.putenv(String.to_charlist(varname), String.to_charlist(value)) + :ok + end end @doc """ - Set multiple environment variables. + Sets multiple environment variables. Sets a new value for each environment variable corresponding - to each key in `dict`. + to each `{key, value}` pair in `enum`. Keys and non-nil values + are automatically converted to charlists. `nil` values erase + the given keys. + + Overall, this is a convenience wrapper around `put_env/2` and + `delete_env/2` with support for different key and value formats. """ - @spec put_env(Dict.t) :: :ok - def put_env(dict) do - Enum.each dict, fn {key, val} -> put_env key, val end + @spec put_env(Enumerable.t()) :: :ok + def put_env(enum) do + Enum.each(enum, fn + {key, nil} -> + :os.unsetenv(to_charlist(key)) + + {key, val} -> + key = to_charlist(key) + + case :string.find(key, "=") do + :nomatch -> + :os.putenv(key, to_charlist(val)) + + _ -> + raise ArgumentError, + "cannot execute System.put_env/1 for key with \"=\", got: #{inspect(key)}" + end + end) end @doc """ @@ -330,30 +797,30 @@ defmodule System do Removes the variable `varname` from the environment. """ - @spec delete_env(String.t) :: :ok + @spec delete_env(String.t()) :: :ok def delete_env(varname) do - :os.unsetenv(String.to_char_list(varname)) + :os.unsetenv(String.to_charlist(varname)) :ok end @doc """ - Last exception stacktrace. - - Note that the Erlang VM (and therefore this function) does not - return the current stacktrace but rather the stacktrace of the - latest exception. + Deprecated mechanism to retrieve the last exception stacktrace. - Inlined by the compiler into `:erlang.get_stacktrace/0`. + It always return an empty list. """ + @deprecated "Use __STACKTRACE__ instead" def stacktrace do - :erlang.get_stacktrace + [] end @doc """ - Halt the Erlang runtime system. + Immediately halts the Erlang runtime system. - Halts the Erlang runtime system where the argument `status` must be a - non-negative integer, the atom `:abort` or a binary. + Terminates the Erlang runtime system without properly shutting down + applications and ports. Please see `stop/1` for a careful shutdown of the + system. + + `status` must be a non-negative integer, the atom `:abort` or a binary. * If an integer, the runtime system exits with the integer value which is returned to the operating system. @@ -361,13 +828,13 @@ defmodule System do * If `:abort`, the runtime system aborts producing a core dump, if that is enabled in the operating system. - * If a string, an erlang crash dump is produced with status as slogan, + * If a string, an Erlang crash dump is produced with status as slogan, and then the runtime system exits with status code 1. Note that on many platforms, only the status codes 0-255 are supported by the operating system. - For more information, check: http://www.erlang.org/doc/man/erlang.html#halt-1 + For more information, see `:erlang.halt/1`. ## Examples @@ -385,6 +852,630 @@ defmodule System do end def halt(status) when is_binary(status) do - :erlang.halt(String.to_char_list(status)) + :erlang.halt(String.to_charlist(status)) + end + + @doc """ + Returns the operating system PID for the current Erlang runtime system instance. + + Returns a string containing the (usually) numerical identifier for a process. + On Unix-like operating systems, this is typically the return value of the `getpid()` system call. + On Windows, the process ID as returned by the `GetCurrentProcessId()` system + call is used. + + ## Examples + + System.pid() + + """ + @doc since: "1.9.0" + @spec pid :: String.t() + def pid do + List.to_string(:os.getpid()) + end + + @doc """ + Restarts all applications in the Erlang runtime system. + + All applications are taken down smoothly, all code is unloaded, and all ports + are closed before the system starts all applications once again. + + ## Examples + + System.restart() + + """ + @doc since: "1.9.0" + @spec restart :: :ok + defdelegate restart(), to: :init + + @doc """ + Asynchronously and carefully stops the Erlang runtime system. + + All applications are taken down smoothly, all code is unloaded, and all ports + are closed before the system terminates by calling `halt/1`. + + `status` must be a non-negative integer or a binary. + + * If an integer, the runtime system exits with the integer value which is + returned to the operating system. On many platforms, only the status codes + 0-255 are supported by the operating system. + + * If a binary, an Erlang crash dump is produced with status as slogan, and + then the runtime system exits with status code 1. + + Note this function is asynchronous and the current process will continue + executing after this function is invoked. In case you want to block the + current process until the system effectively shuts down, you can invoke + `Process.sleep(:infinity)`. + + ## Examples + + System.stop(0) + System.stop(1) + + """ + @doc since: "1.5.0" + @spec stop(non_neg_integer | binary) :: :ok + def stop(status \\ 0) + + def stop(status) when is_integer(status) do + at_exit(fn _ -> Process.sleep(:infinity) end) + :init.stop(status) + end + + def stop(status) when is_binary(status) do + at_exit(fn _ -> Process.sleep(:infinity) end) + :init.stop(String.to_charlist(status)) + end + + @doc ~S""" + Executes the given `command` in the OS shell. + + It uses `sh` for Unix-like systems and `cmd` for Windows. + + > #### Watch out {: .warning} + > + > Use this function with care. In particular, **never + > pass untrusted user input to this function**, as the user would be + > able to perform "command injection attacks" by executing any code + > directly on the machine. Generally speaking, prefer to use `cmd/3` + > over this function. + + ## Examples + + iex> System.shell("echo hello") + {"hello\n", 0} + + If you want to stream the output to Standard IO as it arrives: + + iex> System.shell("echo hello", into: IO.stream()) + hello + {%IO.Stream{}, 0} + + ## Options + + It accepts the same options as `cmd/3` (except for `arg0`). + It also accepts the following exclusive options: + + * `:close_stdin` (since v1.14.1) - if the stdin should be closed + on Unix systems, forcing any command that waits on stdin to + immediately terminate. Defaults to `false`. + """ + @doc since: "1.12.0" + @spec shell(binary, shell_opts) :: {Collectable.t(), exit_status :: non_neg_integer} + def shell(command, opts \\ []) when is_binary(command) do + command |> String.trim() |> do_shell(opts) + end + + defp do_shell("", _opts), do: {"", 0} + + defp do_shell(command, opts) do + assert_no_null_byte!(command, "System.shell/2") + {close_stdin?, opts} = Keyword.pop(opts, :close_stdin, false) + + # Finding shell command logic from :os.cmd in OTP + # https://github.com/erlang/otp/blob/8deb96fb1d017307e22d2ab88968b9ef9f1b71d0/lib/kernel/src/os.erl#L184 + case :os.type() do + {:unix, _} -> + shell_path = :os.find_executable(~c"sh") || :erlang.error(:enoent, [command, opts]) + close_stdin = if close_stdin?, do: " + command = String.to_charlist(command) + + command = + case {System.get_env("COMSPEC"), osname} do + {nil, :windows} -> ~c"command.com /s /c " ++ command + {nil, _} -> ~c"cmd /s /c " ++ command + {cmd, _} -> ~c"#{cmd} /s /c " ++ command + end + + do_cmd({:spawn, command}, [], opts) + end + end + + @doc ~S""" + Executes the given `command` with `args`. + + `command` is expected to be an executable available in PATH + unless an absolute path is given. + + `args` must be a list of binaries which the executable will receive + as its arguments as is. This means that: + + * environment variables will not be interpolated + * wildcard expansion will not happen (unless `Path.wildcard/2` is used + explicitly) + * arguments do not need to be escaped or quoted for shell safety + + This function returns a tuple containing the collected result + and the command exit status. + + Internally, this function uses a `Port` for interacting with the + outside world. However, if you plan to run a long-running program, + ports guarantee stdin/stdout devices will be closed but it does not + automatically terminate the program. The documentation for the + `Port` module describes this problem and possible solutions under + the "Orphan operating system processes" section. + + > #### Windows argument splitting and untrusted arguments {: .warning} + > + > On Unix systems, arguments are passed to a new operating system + > process as an array of strings but on Windows it is up to the child + > process to parse them and some Windows programs may apply their own + > rules, which are inconsistent with the standard C runtime `argv` parsing + > + > This is particularly troublesome when invoking `.bat` or `.com` files + > as these run implicitly through `cmd.exe`, whose argument parsing is + > vulnerable to malicious input and can be used to run arbitrary shell + > commands. + > + > Therefore, if you are running on Windows and you execute batch + > files or `.com` applications, you must not pass untrusted input as + > arguments to the program. You may avoid accidentally executing them + > by explicitly passing the extension of the program you want to run, + > such as `.exe`, and double check the program is indeed not a batch + > file or `.com` application. + + ## Options + + * `:into` - injects the result into the given collectable, defaults to `""` + + * `:lines` - (since v1.15.0) reads the output by lines instead of in bytes. It expects a + number of maximum bytes to buffer internally (1024 is a reasonable default). + The collectable will be called with each finished line (regardless of buffer + size) and without the EOL character + + * `:cd` - the directory to run the command in + + * `:env` - an enumerable of tuples containing environment key-value as + binary. The child process inherits all environment variables from its + parent process, the Elixir application, except those overwritten or + cleared using this option. Specify a value of `nil` to clear (unset) an + environment variable, which is useful for preventing credentials passed + to the application from leaking into child processes + + * `:arg0` - sets the command arg0 + + * `:stderr_to_stdout` - redirects stderr to stdout when `true`, no effect + if `use_stdio` is `false`. + + * `:use_stdio` - `true` by default, setting it to false allows direct + interaction with the terminal from the callee + + * `:parallelism` - when `true`, the VM will schedule port tasks to improve + parallelism in the system. If set to `false`, the VM will try to perform + commands immediately, improving latency at the expense of parallelism. + The default is `false`, and can be set on system startup by passing the + [`+spp`](https://www.erlang.org/doc/man/erl.html#+spp) flag to `--erl`. + Use `:erlang.system_info(:port_parallelism)` to check if enabled. + + ## Error reasons + + If invalid arguments are given, `ArgumentError` is raised by + `System.cmd/3`. `System.cmd/3` also expects a strict set of + options and will raise if unknown or invalid options are given. + + Furthermore, `System.cmd/3` may fail with one of the POSIX reasons + detailed below: + + * `:system_limit` - all available ports in the Erlang emulator are in use + + * `:enomem` - there was not enough memory to create the port + + * `:eagain` - there are no more available operating system processes + + * `:enametoolong` - the external command given was too long + + * `:emfile` - there are no more available file descriptors + (for the operating system process that the Erlang emulator runs in) + + * `:enfile` - the file table is full (for the entire operating system) + + * `:eacces` - the command does not point to an executable file + + * `:enoent` - the command does not point to an existing file + + ## Shell commands + + If you desire to execute a trusted command inside a shell, with pipes, + redirecting and so on, please check `shell/2`. + + ## Examples + + iex> System.cmd("echo", ["hello"]) + {"hello\n", 0} + + iex> System.cmd("echo", ["hello"], env: [{"MIX_ENV", "test"}]) + {"hello\n", 0} + + If you want to stream the output to Standard IO as it arrives: + + iex> System.cmd("echo", ["hello"], into: IO.stream()) + hello + {%IO.Stream{}, 0} + + If you want to read lines: + + iex> System.cmd("echo", ["hello\nworld"], into: [], lines: 1024) + {["hello", "world"], 0} + + """ + @spec cmd(binary, [binary], cmd_opts) :: {Collectable.t(), exit_status :: non_neg_integer} + def cmd(command, args, opts \\ []) when is_binary(command) and is_list(args) do + assert_no_null_byte!(command, "System.cmd/3") + + if not Enum.all?(args, &is_binary/1) do + raise ArgumentError, "all arguments for System.cmd/3 must be binaries" + end + + cmd = String.to_charlist(command) + + cmd = + if Path.type(cmd) == :absolute do + cmd + else + :os.find_executable(cmd) || :erlang.error(:enoent, [command, args, opts]) + end + + do_cmd({:spawn_executable, cmd}, [args: args], opts) + end + + defp do_cmd(port_init, base_opts, opts) do + {use_stdio?, opts} = Keyword.pop(opts, :use_stdio, true) + + {into, line, opts} = + cmd_opts(opts, [:exit_status, :binary, :hide] ++ base_opts, "", false, use_stdio?) + + {initial, fun} = Collectable.into(into) + + try do + case line do + true -> do_port_line(Port.open(port_init, opts), initial, fun, []) + false -> do_port_byte(Port.open(port_init, opts), initial, fun) + end + catch + kind, reason -> + fun.(initial, :halt) + :erlang.raise(kind, reason, __STACKTRACE__) + else + {acc, status} -> {fun.(acc, :done), status} + end + end + + defp do_port_byte(port, acc, fun) do + receive do + {^port, {:data, data}} -> + do_port_byte(port, fun.(acc, {:cont, data}), fun) + + {^port, {:exit_status, status}} -> + {acc, status} + end + end + + defp do_port_line(port, acc, fun, buffer) do + receive do + {^port, {:data, {:noeol, data}}} -> + do_port_line(port, acc, fun, [data | buffer]) + + {^port, {:data, {:eol, data}}} -> + data = [data | buffer] |> Enum.reverse() |> IO.iodata_to_binary() + do_port_line(port, fun.(acc, {:cont, data}), fun, []) + + {^port, {:exit_status, status}} -> + # Data may arrive after exit status on line mode + receive do + {^port, {:data, {_, data}}} -> + data = [data | buffer] |> Enum.reverse() |> IO.iodata_to_binary() + {fun.(acc, {:cont, data}), status} + after + 0 -> {acc, status} + end + end + end + + defp cmd_opts([{:into, any} | t], opts, _into, line, stdio?), + do: cmd_opts(t, opts, any, line, stdio?) + + defp cmd_opts([{:cd, bin} | t], opts, into, line, stdio?) when is_binary(bin), + do: cmd_opts(t, [{:cd, bin} | opts], into, line, stdio?) + + defp cmd_opts([{:arg0, bin} | t], opts, into, line, stdio?) when is_binary(bin), + do: cmd_opts(t, [{:arg0, bin} | opts], into, line, stdio?) + + defp cmd_opts([{:stderr_to_stdout, true} | t], opts, into, line, true), + do: cmd_opts(t, [:stderr_to_stdout | opts], into, line, true) + + defp cmd_opts([{:stderr_to_stdout, true} | _], _opts, _into, _line, false), + do: raise(ArgumentError, "cannot use \"stderr_to_stdout: true\" and \"use_stdio: false\"") + + defp cmd_opts([{:stderr_to_stdout, false} | t], opts, into, line, stdio?), + do: cmd_opts(t, opts, into, line, stdio?) + + defp cmd_opts([{:parallelism, bool} | t], opts, into, line, stdio?) when is_boolean(bool), + do: cmd_opts(t, [{:parallelism, bool} | opts], into, line, stdio?) + + defp cmd_opts([{:env, enum} | t], opts, into, line, stdio?), + do: cmd_opts(t, [{:env, validate_env(enum)} | opts], into, line, stdio?) + + defp cmd_opts([{:lines, max_line_length} | t], opts, into, _line, stdio?) + when is_integer(max_line_length) and max_line_length > 0, + do: cmd_opts(t, [{:line, max_line_length} | opts], into, true, stdio?) + + defp cmd_opts([{key, val} | _], _opts, _into, _line, _stdio?), + do: raise(ArgumentError, "invalid option #{inspect(key)} with value #{inspect(val)}") + + defp cmd_opts([], opts, into, line, stdio?) do + opt = if stdio?, do: :use_stdio, else: :nouse_stdio + {into, line, [opt | opts]} + end + + defp validate_env(enum) do + Enum.map(enum, fn + {k, nil} -> + {String.to_charlist(k), false} + + {k, v} -> + {String.to_charlist(k), String.to_charlist(v)} + + other -> + raise ArgumentError, "invalid environment key-value #{inspect(other)}" + end) + end + + @doc """ + Returns the current monotonic time in the `:native` time unit. + + This time is monotonically increasing and starts in an unspecified + point in time. This is not strictly monotonically increasing. Multiple + sequential calls of the function may return the same value. + + Inlined by the compiler. + """ + @spec monotonic_time() :: integer + def monotonic_time do + :erlang.monotonic_time() + end + + @doc """ + Returns the current monotonic time in the given time unit. + + This time is monotonically increasing and starts in an unspecified + point in time. + """ + @spec monotonic_time(time_unit | :native) :: integer + def monotonic_time(unit) do + :erlang.monotonic_time(normalize_time_unit(unit)) + end + + @doc """ + Returns the current system time in the `:native` time unit. + + It is the VM view of the `os_time/0`. They may not match in + case of time warps although the VM works towards aligning + them. This time is not monotonic. + + Inlined by the compiler. + """ + @spec system_time() :: integer + def system_time do + :erlang.system_time() + end + + @doc """ + Returns the current system time in the given time unit. + + It is the VM view of the `os_time/0`. They may not match in + case of time warps although the VM works towards aligning + them. This time is not monotonic. + """ + @spec system_time(time_unit | :native) :: integer + def system_time(unit) do + :erlang.system_time(normalize_time_unit(unit)) + end + + @doc """ + Converts `time` from time unit `from_unit` to time unit `to_unit`. + + The result is rounded via the floor function. + + `convert_time_unit/3` accepts an additional time unit (other than the + ones in the `t:time_unit/0` type) called `:native`. `:native` is the time + unit used by the Erlang runtime system. It's determined when the runtime + starts and stays the same until the runtime is stopped, but could differ + the next time the runtime is started on the same machine. For this reason, + you should use this function to convert `:native` time units to a predictable + unit before you display them to humans. + + To determine how many seconds the `:native` unit represents in your current + runtime, you can call this function to convert 1 second to the `:native` + time unit: `System.convert_time_unit(1, :second, :native)`. + """ + @spec convert_time_unit(integer, time_unit | :native, time_unit | :native) :: integer + def convert_time_unit(time, from_unit, to_unit) do + :erlang.convert_time_unit(time, normalize_time_unit(from_unit), normalize_time_unit(to_unit)) + end + + @doc """ + Returns the current time offset between the Erlang VM monotonic + time and the Erlang VM system time. + + The result is returned in the `:native` time unit. + + See `time_offset/1` for more information. + + Inlined by the compiler. + """ + @spec time_offset() :: integer + def time_offset do + :erlang.time_offset() + end + + @doc """ + Returns the current time offset between the Erlang VM monotonic + time and the Erlang VM system time. + + The result is returned in the given time unit `unit`. The returned + offset, added to an Erlang monotonic time (for instance, one obtained with + `monotonic_time/1`), gives the Erlang system time that corresponds + to that monotonic time. + """ + @spec time_offset(time_unit | :native) :: integer + def time_offset(unit) do + :erlang.time_offset(normalize_time_unit(unit)) + end + + @doc """ + Returns the current operating system (OS) time. + + The result is returned in the `:native` time unit. + + This time may be adjusted forwards or backwards in time + with no limitation and is not monotonic. + + Inlined by the compiler. + """ + @spec os_time() :: integer + @doc since: "1.3.0" + def os_time do + :os.system_time() + end + + @doc """ + Returns the current operating system (OS) time in the given time `unit`. + + This time may be adjusted forwards or backwards in time + with no limitation and is not monotonic. + """ + @spec os_time(time_unit | :native) :: integer + @doc since: "1.3.0" + def os_time(unit) do + :os.system_time(normalize_time_unit(unit)) + end + + @doc """ + Returns the Erlang/OTP release number. + """ + @spec otp_release :: String.t() + @doc since: "1.3.0" + def otp_release do + :erlang.list_to_binary(:erlang.system_info(:otp_release)) + end + + @doc """ + Returns the number of schedulers in the VM. + """ + @spec schedulers :: pos_integer + @doc since: "1.3.0" + def schedulers do + :erlang.system_info(:schedulers) + end + + @doc """ + Returns the number of schedulers online in the VM. + """ + @spec schedulers_online :: pos_integer + @doc since: "1.3.0" + def schedulers_online do + :erlang.system_info(:schedulers_online) + end + + @doc """ + Generates and returns an integer that is unique in the current runtime + instance. + + "Unique" means that this function, called with the same list of `modifiers`, + will never return the same integer more than once on the current runtime + instance. + + If `modifiers` is `[]`, then a unique integer (that can be positive or negative) is returned. + Other modifiers can be passed to change the properties of the returned integer: + + * `:positive` - the returned integer is guaranteed to be positive. + * `:monotonic` - the returned integer is monotonically increasing. This + means that, on the same runtime instance (but even on different + processes), integers returned using the `:monotonic` modifier will always + be strictly less than integers returned by successive calls with the + `:monotonic` modifier. + + All modifiers listed above can be combined; repeated modifiers in `modifiers` + will be ignored. + + Inlined by the compiler. + """ + @spec unique_integer([:positive | :monotonic]) :: integer + def unique_integer(modifiers \\ []) do + :erlang.unique_integer(modifiers) + end + + defp assert_no_null_byte!(binary, operation) do + case :binary.match(binary, "\0") do + {_, _} -> + raise ArgumentError, + "cannot execute #{operation} for program with null byte, got: #{inspect(binary)}" + + :nomatch -> + binary + end + end + + defp normalize_time_unit(:native), do: :native + + defp normalize_time_unit(:second), do: :second + defp normalize_time_unit(:millisecond), do: :millisecond + defp normalize_time_unit(:microsecond), do: :microsecond + defp normalize_time_unit(:nanosecond), do: :nanosecond + + defp normalize_time_unit(:seconds), do: warn(:seconds, :second) + defp normalize_time_unit(:milliseconds), do: warn(:milliseconds, :millisecond) + defp normalize_time_unit(:microseconds), do: warn(:microseconds, :microsecond) + defp normalize_time_unit(:nanoseconds), do: warn(:nanoseconds, :nanosecond) + + defp normalize_time_unit(:milli_seconds), do: warn(:milli_seconds, :millisecond) + defp normalize_time_unit(:micro_seconds), do: warn(:micro_seconds, :microsecond) + defp normalize_time_unit(:nano_seconds), do: warn(:nano_seconds, :nanosecond) + + defp normalize_time_unit(unit) when is_integer(unit) and unit > 0, do: unit + + defp normalize_time_unit(other) do + raise ArgumentError, + "unsupported time unit. Expected :second, :millisecond, " <> + ":microsecond, :nanosecond, or a positive integer, " <> "got #{inspect(other)}" + end + + defp warn(unit, replacement_unit) do + IO.warn_once( + {__MODULE__, unit}, + fn -> + "deprecated time unit: #{inspect(unit)}. A time unit should be " <> + ":second, :millisecond, :microsecond, :nanosecond, or a positive integer" + end, + _stacktrace_drop_levels = 4 + ) + + replacement_unit end end diff --git a/lib/elixir/lib/task.ex b/lib/elixir/lib/task.ex index 0ba6df0cf89..43d7e2e66c8 100644 --- a/lib/elixir/lib/task.ex +++ b/lib/elixir/lib/task.ex @@ -1,220 +1,1499 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + defmodule Task do @moduledoc """ - Conveniences for spawning and awaiting for tasks. + Conveniences for spawning and awaiting tasks. Tasks are processes meant to execute one particular - action throughout their life-cycle, often with little or no + action throughout their lifetime, often with little or no communication with other processes. The most common use case - for tasks is to compute a value asynchronously: + for tasks is to convert sequential code into concurrent code + by computing a value asynchronously: task = Task.async(fn -> do_some_work() end) - res = do_some_other_work() + res = do_some_other_work() res + Task.await(task) - Tasks spawned with `async` can be awaited on by its caller - process (and only its caller) as shown in the example above. + Tasks spawned with `async` can be awaited on by their caller + process (and only their caller) as shown in the example above. They are implemented by spawning a process that sends a message to the caller once the given computation is performed. - Besides `async/1` and `await/2`, tasks can also be - started as part of supervision trees and dynamically spawned - in remote nodes. We will explore all three scenarios next. + Compared to plain processes, started with `spawn/1`, tasks + include monitoring metadata and logging in case of errors. + + Besides `async/1` and `await/2`, tasks can also be + started as part of a supervision tree and dynamically spawned + on remote nodes. We will explore these scenarios next. ## async and await - The most common way to spawn a task is with `Task.async/1`. A new - process will be created, linked and monitored by the caller. Once - the task action finishes, a message will be sent to the caller - with the result. + One of the common uses of tasks is to convert sequential code + into concurrent code with `Task.async/1` while keeping its semantics. + When invoked, a new process will be created, linked and monitored + by the caller. Once the task action finishes, a message will be sent + to the caller with the result. - `Task.await/2` is used to read the message sent by the task. On - `await`, Elixir will also setup a monitor to verify if the process - exited for any abnormal reason (or in case exits are being - trapped by the caller). + `Task.await/2` is used to read the message sent by the task. - ## Supervised tasks + There are two important things to consider when using `async`: - It is also possible to spawn a task inside a supervision tree - with `start_link/1` and `start_link/3`: + 1. If you are using async tasks, you **must await** a reply + as they are *always* sent. If you are not expecting a reply, + consider using `Task.start_link/1` as detailed below. - Task.start_link(fn -> IO.puts "ok" end) + 2. Async tasks link the caller and the spawned process. This + means that, if the caller crashes, the task will crash + too and vice-versa. This is on purpose: if the process + meant to receive the result no longer exists, there is + no purpose in completing the computation. If this is not + desired, you will want to use supervised tasks, described + in a subsequent section. - Such tasks can be mounted in your supervision tree as: + ## Tasks are processes - import Supervisor.Spec + Tasks are processes and so data will need to be completely copied + to them. Take the following code as an example: - children = [ - worker(Task, [fn -> IO.puts "ok" end]) - ] + large_data = fetch_large_data() + task = Task.async(fn -> do_some_work(large_data) end) + res = do_some_other_work() + res + Task.await(task) + + The code above copies over all of `large_data`, which can be + resource intensive depending on the size of the data. + There are two ways to address this. - Since these tasks are supervised and not directly linked to - the caller, they cannot be awaited on. Note `start_link/1`, - unlike `async/1`, returns `{:ok, pid}` (which is - the result expected by supervision trees). + First, if you need to access only part of `large_data`, + consider extracting it before the task: - ## Supervision trees + large_data = fetch_large_data() + subset_data = large_data.some_field + task = Task.async(fn -> do_some_work(subset_data) end) - The `Task.Supervisor` module allows developers to start supervisors - that dynamically supervise tasks: + Alternatively, if you can move the data loading altogether + to the task, it may be even better: + + task = Task.async(fn -> + large_data = fetch_large_data() + do_some_work(large_data) + end) + + ## Dynamically supervised tasks + + The `Task.Supervisor` module allows developers to dynamically + create multiple supervised tasks. + + A short example is: {:ok, pid} = Task.Supervisor.start_link() - Task.Supervisor.async(pid, fn -> do_work() end) - `Task.Supervisor` also makes it possible to spawn tasks in remote nodes as - long as the supervisor is registered locally or globally: + task = + Task.Supervisor.async(pid, fn -> + # Do something + end) - # In the remote node - Task.Supervisor.start_link(name: :tasks_sup) + Task.await(task) - # In the client - Task.Supervisor.async({:tasks_sup, :remote@local}, fn -> do_work() end) + However, in the majority of cases, you want to add the task supervisor + to your supervision tree: - `Task.Supervisor` is more often started in your supervision tree as: + Supervisor.start_link([ + {Task.Supervisor, name: MyApp.TaskSupervisor} + ], strategy: :one_for_one) - import Supervisor.Spec + And now you can use async/await by passing the name of + the supervisor instead of the pid: - children = [ - supervisor(Task.Supervisor, [[name: :tasks_sup]]) - ] + Task.Supervisor.async(MyApp.TaskSupervisor, fn -> + # Do something + end) + |> Task.await() + + We encourage developers to rely on supervised tasks as much as possible. + Supervised tasks improve the visibility of how many tasks are running + at a given moment and enable a variety of patterns that give you + explicit control on how to handle the results, errors, and timeouts. + Here is a summary: + + * Using `Task.Supervisor.start_child/2` allows you to start a fire-and-forget + task when you don't care about its results or if it completes successfully or not. + + * Using `Task.Supervisor.async/2` + `Task.await/2` allows you to execute + tasks concurrently and retrieve its result. If the task fails, + the caller will also fail. + + * Using `Task.Supervisor.async_nolink/2` + `Task.yield/2` + `Task.shutdown/2` + allows you to execute tasks concurrently and retrieve their results + or the reason they failed within a given time frame. If the task fails, + the caller won't fail. You will receive the error reason either on + `yield` or `shutdown`. + + Furthermore, the supervisor guarantees all tasks terminate within a + configurable shutdown period when your application shuts down. See the + `Task.Supervisor` module for details on the supported operations. + + ### Distributed tasks + + With `Task.Supervisor`, it is easy to dynamically start tasks across nodes: + + # First on the remote node named :remote@local + Task.Supervisor.start_link(name: MyApp.DistSupervisor) + + # Then on the local client node + supervisor = {MyApp.DistSupervisor, :remote@local} + Task.Supervisor.async(supervisor, MyMod, :my_fun, [arg1, arg2, arg3]) + + Note that, as above, when working with distributed tasks, one should use the + `Task.Supervisor.async/5` function that expects explicit module, function, + and arguments, instead of `Task.Supervisor.async/3` that works with anonymous + functions. That's because anonymous functions expect the same module version + to exist on all involved nodes. Check the `Agent` module documentation for + more information on distributed processes as the limitations described there + apply to the whole ecosystem. + + ## Statically supervised tasks + + The `Task` module implements the `child_spec/1` function, which + allows it to be started directly under a regular `Supervisor` - + instead of a `Task.Supervisor` - by passing a tuple with a function + to run: + + Supervisor.start_link([ + {Task, fn -> :some_work end} + ], strategy: :one_for_one) + + This is often useful when you need to execute some steps while + setting up your supervision tree. For example: to warm up caches, + log the initialization status, and such. + + If you don't want to put the Task code directly under the `Supervisor`, + you can wrap the `Task` in its own module, similar to how you would + do with a `GenServer` or an `Agent`: + + defmodule MyTask do + use Task + + def start_link(arg) do + Task.start_link(__MODULE__, :run, [arg]) + end + + def run(arg) do + # ... + end + end + + And then passing it to the supervisor: + + Supervisor.start_link([ + {MyTask, arg} + ], strategy: :one_for_one) + + Since these tasks are supervised and not directly linked to the caller, + they cannot be awaited on. By default, the functions `Task.start/1` + and `Task.start_link/1` are for fire-and-forget tasks, where you don't + care about the results or if it completes successfully or not. + + > #### `use Task` {: .info} + > + > When you `use Task`, the `Task` module will define a + > `child_spec/1` function, so your module can be used + > as a child in a supervision tree. + + `use Task` defines a `child_spec/1` function, allowing the + defined module to be put under a supervision tree. The generated + `child_spec/1` can be customized with the following options: + + * `:id` - the child specification identifier, defaults to the current module + * `:restart` - when the child should be restarted, defaults to `:temporary` + * `:shutdown` - how to shut down the child, either immediately or by giving it time to shut down + + Opposite to `GenServer`, `Agent` and `Supervisor`, a Task has + a default `:restart` of `:temporary`. This means the task will + not be restarted even if it crashes. If you desire the task to + be restarted for non-successful exits, do: + + use Task, restart: :transient + + If you want the task to always be restarted: + + use Task, restart: :permanent + + See the "Child specification" section in the `Supervisor` module + for more detailed information. The `@doc` annotation immediately + preceding `use Task` will be attached to the generated `child_spec/1` + function. + + ## Ancestor and Caller Tracking + + Whenever you start a new process, Elixir annotates the process with the parent + through the `$ancestors` key in the process dictionary. This is often used to + track the hierarchy inside a supervision tree. + + For example, we recommend developers to always start tasks under a supervisor. + This provides more visibility and allows you to control how those tasks are + terminated when a node shuts down. That might look something like + `Task.Supervisor.start_child(MySupervisor, task_function)`. This means + that, although your code is the one invoking the task, the actual ancestor of + the task is the supervisor, as the supervisor is the one effectively starting it. + + To track the relationship between your code and the task, we use the `$callers` + key in the process dictionary. Therefore, assuming the `Task.Supervisor` call + above, we have: + + [your code] -- calls --> [supervisor] ---- spawns --> [task] + + Which means we store the following relationships: + + [your code] [supervisor] <-- ancestor -- [task] + ^ | + |--------------------- caller ---------------------| - Check `Task.Supervisor` for other operations supported by the Task supervisor. + The list of callers of the current process can be retrieved from the Process + dictionary with `Process.get(:"$callers")`. This will return either `nil` or + a list `[pid_n, ..., pid2, pid1]` with at least one entry where `pid_n` is + the PID that called the current process, `pid2` called `pid_n`, and `pid2` was + called by `pid1`. + + If a task crashes, the callers field is included as part of the log message + metadata under the `:callers` key. """ @doc """ The Task struct. - It contains two fields: + It contains these fields: + + * `:mfa` - a three-element tuple containing the module, function name, + and arity invoked to start the task in `async/1` and `async/3` + + * `:owner` - the PID of the process that started the task + + * `:pid` - the PID of the task process; `nil` if there is no process + specifically assigned for the task + + * `:ref` - an opaque term used as the task monitor reference + + """ + @enforce_keys [:mfa, :owner, :pid, :ref] + defstruct @enforce_keys + + @typedoc """ + The Task type. + + See [`%Task{}`](`__struct__/0`) for information about each field of the structure. + """ + @type t :: %__MODULE__{ + mfa: mfa(), + owner: pid(), + pid: pid() | nil, + ref: ref() + } + + @typedoc """ + The task opaque reference. + """ + @opaque ref :: reference() + + @typedoc """ + Options given to `async_stream` functions. + """ + @typedoc since: "1.17.0" + @type async_stream_option :: + {:max_concurrency, pos_integer()} + | {:ordered, boolean()} + | {:timeout, timeout()} + | {:on_timeout, :exit | :kill_task} + | {:zip_input_on_exit, boolean()} + + defguardp is_timeout(timeout) + when timeout == :infinity or (is_integer(timeout) and timeout >= 0) - * `:pid` - the process reference of the task process; it may be a pid - or a tuple containing the process and node names + @doc """ + Returns a specification to start a task under a supervisor. - * `:ref` - the task monitor reference + `arg` is passed as the argument to `Task.start_link/1` in the `:start` field + of the spec. + For more information, see the `Supervisor` module, + the `Supervisor.child_spec/2` function and the `t:Supervisor.child_spec/0` type. """ - defstruct pid: nil, ref: nil + @doc since: "1.5.0" + @spec child_spec(term) :: Supervisor.child_spec() + def child_spec(arg) do + %{ + id: Task, + start: {Task, :start_link, [arg]}, + restart: :temporary + } + end + + @doc false + defmacro __using__(opts) do + quote location: :keep, bind_quoted: [opts: opts] do + if not Module.has_attribute?(__MODULE__, :doc) do + @doc """ + Returns a specification to start this module under a supervisor. + + `arg` is passed as the argument to `Task.start_link/1` in the `:start` field + of the spec. + + For more information, see the `Supervisor` module, + the `Supervisor.child_spec/2` function and the `t:Supervisor.child_spec/0` type. + """ + end + + def child_spec(arg) do + default = %{ + id: __MODULE__, + start: {__MODULE__, :start_link, [arg]}, + restart: :temporary + } + + Supervisor.child_spec(default, unquote(Macro.escape(opts))) + end + + defoverridable child_spec: 1 + end + end @doc """ - Starts a task as part of a supervision tree. + Starts a task as part of a supervision tree with the given `fun`. + + `fun` must be a zero-arity anonymous function. + + This is used to start a statically supervised task under a supervision tree. """ - @spec start_link(fun) :: {:ok, pid} - def start_link(fun) do + @spec start_link((-> any)) :: {:ok, pid} + def start_link(fun) when is_function(fun, 0) do start_link(:erlang, :apply, [fun, []]) end @doc """ - Starts a task as part of a supervision tree. + Starts a task as part of a supervision tree with the given + `module`, `function`, and `args`. + + This is used to start a statically supervised task under a supervision tree. """ @spec start_link(module, atom, [term]) :: {:ok, pid} - def start_link(mod, fun, args) do - Task.Supervised.start_link(get_info(self), {mod, fun, args}) + def start_link(module, function, args) + when is_atom(module) and is_atom(function) and is_list(args) do + mfa = {module, function, args} + Task.Supervised.start_link(get_owner(self()), get_callers(self()), mfa) + end + + @doc """ + Starts a task. + + `fun` must be a zero-arity anonymous function. + + This should only used when the task is used for side-effects + (like I/O) and you have no interest on its results nor if it + completes successfully. + + If the current node is shutdown, the node will terminate even + if the task was not completed. For this reason, we recommend + to use `Task.Supervisor.start_child/2` instead, which allows + you to control the shutdown time via the `:shutdown` option. + """ + @spec start((-> any)) :: {:ok, pid} + def start(fun) when is_function(fun, 0) do + start(:erlang, :apply, [fun, []]) + end + + @doc """ + Starts a task. + + This should only used when the task is used for side-effects + (like I/O) and you have no interest on its results nor if it + completes successfully. + + If the current node is shutdown, the node will terminate even + if the task was not completed. For this reason, we recommend + to use `Task.Supervisor.start_child/2` instead, which allows + you to control the shutdown time via the `:shutdown` option. + """ + @spec start(module, atom, [term]) :: {:ok, pid} + def start(module, function_name, args) + when is_atom(module) and is_atom(function_name) and is_list(args) do + mfa = {module, function_name, args} + Task.Supervised.start(get_owner(self()), get_callers(self()), mfa) end @doc """ - Starts a task that can be awaited on. + Starts a task that must be awaited on. + + `fun` must be a zero-arity anonymous function. This function + spawns a process that is linked to and monitored by the caller + process. A `Task` struct is returned containing the relevant + information. + + If you start an `async`, you **must await**. This is either done + by calling `Task.await/2` or `Task.yield/2` followed by + `Task.shutdown/2` on the returned task. Alternatively, if you + spawn a task inside a `GenServer`, then the `GenServer` will + automatically await for you and call `c:GenServer.handle_info/2` + with the task response and associated `:DOWN` message. + + Read the `Task` module documentation for more information about + the general usage of async tasks. + + ## Linking This function spawns a process that is linked to and monitored - by the caller process. A `Task` struct is returned containing - the relevant information. + by the caller process. The linking part is important because it + aborts the task if the parent process dies. It also guarantees + the code before async/await has the same properties after you + add the async call. For example, imagine you have this: + + x = heavy_function() + y = some_function() + x + y + + Now you want to make the `heavy_function()` async: + + x = Task.async(&heavy_function/0) + y = some_function() + Task.await(x) + y - ## Task's message format + As before, if `heavy_function/0` fails, the whole computation will + fail, including the caller process. If you don't want the task + to fail then you must change the `heavy_fun/0` code in the + same way you would achieve it if you didn't have the async call. + For example, to either return `{:ok, val} | :error` results or, + in more extreme cases, by using `try/rescue`. In other words, + an asynchronous task should be thought of as an extension of the + caller process rather than a mechanism to isolate it from all errors. - The reply sent by the task will be in the format `{ref, msg}`, - where `ref` is the monitoring reference held by the task. + If you don't want to link the caller to the task, then you + must use a supervised task with `Task.Supervisor` and call + `Task.Supervisor.async_nolink/2`. + + In any case, avoid any of the following: + + * Setting `:trap_exit` to `true` - trapping exits should be + used only in special circumstances as it would make your + process immune to not only exits from the task but from + any other processes. + + Moreover, even when trapping exits, calling `await` will + still exit if the task has terminated without sending its + result back. + + * Unlinking the task process started with `async`/`await`. + If you unlink the processes and the task does not belong + to any supervisor, you may leave dangling tasks in case + the caller process dies. + + ## Metadata + + The task created with this function stores `:erlang.apply/2` in + its `:mfa` metadata field, which is used internally to apply + the anonymous function. Use `async/3` if you want another function + to be used as metadata. """ - @spec async(fun) :: t - def async(fun) do + @spec async((-> any)) :: t + def async(fun) when is_function(fun, 0) do async(:erlang, :apply, [fun, []]) end @doc """ - Starts a task that can be awaited on. + Starts a task that must be awaited on. - Similar to `async/1`, but the task is specified by the given - module, function and arguments. + Similar to `async/1` except the function to be started is + specified by the given `module`, `function_name`, and `args`. + The `module`, `function_name`, and its arity are stored as + a tuple in the `:mfa` field for reflection purposes. """ @spec async(module, atom, [term]) :: t - def async(mod, fun, args) do - mfa = {mod, fun, args} - pid = :proc_lib.spawn_link(Task.Supervised, :async, [self, get_info(self), mfa]) - ref = Process.monitor(pid) - send(pid, {self(), ref}) - %Task{pid: pid, ref: ref} + def async(module, function_name, args) + when is_atom(module) and is_atom(function_name) and is_list(args) do + mfargs = {module, function_name, args} + owner = self() + # No need to monitor because the processes are linked + {:ok, pid} = Task.Supervised.start_link(get_owner(owner), :nomonitor) + + alias = build_alias(pid) + send(pid, {owner, alias, alias, get_callers(owner), mfargs}) + %Task{pid: pid, ref: alias, owner: owner, mfa: {module, function_name, length(args)}} + end + + @doc """ + Starts a task that immediately completes with the given `result`. + + Unlike `async/1`, this task does not spawn a linked process. It can + be awaited or yielded like any other task. + + ## Usage + + In some cases, it is useful to create a "completed" task that represents + a task that has already run and generated a result. For example, when + processing data you may be able to determine that certain inputs are + invalid before dispatching them for further processing: + + def process(data) do + tasks = + for entry <- data do + if invalid_input?(entry) do + Task.completed({:error, :invalid_input}) + else + Task.async(fn -> further_process(entry) end) + end + end + + Task.await_many(tasks) + end + + In many cases, `Task.completed/1` may be avoided in favor of returning the + result directly. You should generally only require this variant when working + with mixed asynchrony, when a group of inputs will be handled partially + synchronously and partially asynchronously. + """ + @doc since: "1.13.0" + @spec completed(any) :: t + def completed(result) do + ref = make_ref() + owner = self() + + # "complete" the task immediately + send(owner, {ref, result}) + + %Task{pid: nil, ref: ref, owner: owner, mfa: {Task, :completed, 1}} end - defp get_info(self) do - {node(), - case Process.info(self, :registered_name) do - {:registered_name, []} -> self() - {:registered_name, name} -> name - end} + @doc """ + Returns a stream where the given function (`module` and `function_name`) + is mapped concurrently on each element in `enumerable`. + + Each element of `enumerable` will be prepended to the given `args` and + processed by its own task. Those tasks will be linked to an intermediate + process that is then linked to the caller process. This means a failure + in a task terminates the caller process and a failure in the caller + process terminates all tasks. + + When streamed, each task will emit `{:ok, value}` upon successful + completion or `{:exit, reason}` if the caller is trapping exits. + It's possible to have `{:exit, {element, reason}}` for exits + using the `:zip_input_on_exit` option. The order of results depends + on the value of the `:ordered` option. + + The level of concurrency and the time tasks are allowed to run can + be controlled via options (see the "Options" section below). + + Consider using `Task.Supervisor.async_stream/6` to start tasks + under a supervisor. If you find yourself trapping exits to ensure + errors in the tasks do not terminate the caller process, consider + using `Task.Supervisor.async_stream_nolink/6` to start tasks that + are not linked to the caller process. + + ## Options + + * `:max_concurrency` - sets the maximum number of tasks to run + at the same time. Defaults to `System.schedulers_online/0`. + + * `:ordered` - whether the results should be returned in the same order + as the input stream. When the output is ordered, Elixir may need to + buffer results to emit them in the original order. Setting this option + to false disables the need to buffer at the cost of removing ordering. + This is also useful when you're using the tasks only for the side effects. + Note that regardless of what `:ordered` is set to, the tasks will + process asynchronously. If you need to process elements in order, + consider using `Enum.map/2` or `Enum.each/2` instead. Defaults to `true`. + + * `:timeout` - the maximum amount of time (in milliseconds or `:infinity`) + each task is allowed to execute for. Defaults to `5000`. + + * `:on_timeout` - what to do when a task times out. The possible + values are: + * `:exit` (default) - the caller (the process that spawned the tasks) exits. + * `:kill_task` - the task that timed out is killed. The value + emitted for that task is `{:exit, :timeout}`. + + * `:zip_input_on_exit` - (since v1.14.0) adds the original + input to `:exit` tuples. The value emitted for that task is + `{:exit, {input, reason}}`, where `input` is the collection element + that caused an exit during processing. Defaults to `false`. + + ## Example + + Let's build a stream and then enumerate it: + + stream = Task.async_stream(collection, Mod, :expensive_fun, []) + Enum.to_list(stream) + + The concurrency can be increased or decreased using the `:max_concurrency` + option. For example, if the tasks are IO heavy, the value can be increased: + + max_concurrency = System.schedulers_online() * 2 + stream = Task.async_stream(collection, Mod, :expensive_fun, [], max_concurrency: max_concurrency) + Enum.to_list(stream) + + If you do not care about the results of the computation, you can run + the stream with `Stream.run/1`. Also set `ordered: false`, as you don't + care about the order of the results either: + + stream = Task.async_stream(collection, Mod, :expensive_fun, [], ordered: false) + Stream.run(stream) + + ## First async tasks to complete + + You can also use `async_stream/3` to execute M tasks and find the N tasks + to complete. For example: + + [ + &heavy_call_1/0, + &heavy_call_2/0, + &heavy_call_3/0 + ] + |> Task.async_stream(fn fun -> fun.() end, ordered: false, max_concurrency: 3) + |> Stream.filter(&match?({:ok, _}, &1)) + |> Enum.take(2) + + In the example above, we are executing three tasks and waiting for the + first 2 to complete. We use `Stream.filter/2` to restrict ourselves only + to successfully completed tasks, and then use `Enum.take/2` to retrieve + N items. Note it is important to set both `ordered: false` and + `max_concurrency: M`, where M is the number of tasks, to make sure all + calls execute concurrently. + + ### Attention: unbound async + take + + If you want to potentially process a high number of items and keep only + part of the results, you may end-up processing more items than desired. + Let's see an example: + + 1..100 + |> Task.async_stream(fn i -> + Process.sleep(100) + IO.puts(to_string(i)) + end) + |> Enum.take(10) + + Running the example above in a machine with 8 cores will process 16 items, + even though you want only 10 elements, since `async_stream/3` process items + concurrently. That's because it will process 8 elements at once. Then all 8 + elements complete at roughly the same time, causing 8 elements to be kicked + off for processing. Out of these extra 8, only 2 will be used, and the rest + will be terminated. + + Depending on the problem, you can filter or limit the number of elements + upfront: + + 1..100 + |> Stream.take(10) + |> Task.async_stream(fn i -> + Process.sleep(100) + IO.puts(to_string(i)) + end) + |> Enum.to_list() + + In other cases, you likely want to tweak `:max_concurrency` to limit how + many elements may be over processed at the cost of reducing concurrency. + You can also set the number of elements to take to be a multiple of + `:max_concurrency`. For instance, setting `max_concurrency: 5` in the + example above. + """ + @doc since: "1.4.0" + @spec async_stream(Enumerable.t(), module, atom, [term], [async_stream_option]) :: + Enumerable.t() + def async_stream(enumerable, module, function_name, args, options \\ []) + when is_atom(module) and is_atom(function_name) and is_list(args) do + build_stream(enumerable, {module, function_name, args}, options) end @doc """ - Awaits for a task reply. + Returns a stream that runs the given function `fun` concurrently + on each element in `enumerable`. + + Works the same as `async_stream/5` but with an anonymous function instead of a + module-function-arguments tuple. `fun` must be a one-arity anonymous function. + + Each `enumerable` element is passed as argument to the given function `fun` and + processed by its own task. The tasks will be linked to the caller process, similarly + to `async/1`. + + ## Example + + Count the code points in each string asynchronously, then add the counts together using reduce. + + iex> strings = ["long string", "longer string", "there are many of these"] + iex> stream = Task.async_stream(strings, fn text -> text |> String.codepoints() |> Enum.count() end) + iex> Enum.sum_by(stream, fn {:ok, num} -> num end) + 47 + + See `async_stream/5` for discussion, options, and more examples. + """ + @doc since: "1.4.0" + @spec async_stream(Enumerable.t(), (term -> term), [async_stream_option]) :: Enumerable.t() + def async_stream(enumerable, fun, options \\ []) + when is_function(fun, 1) and is_list(options) do + build_stream(enumerable, fun, options) + end + + defp build_stream(enumerable, fun, options) do + options = Task.Supervised.validate_stream_options(options) + + fn acc, acc_fun -> + owner = get_owner(self()) + + Task.Supervised.stream(enumerable, acc, acc_fun, get_callers(self()), fun, options, fn -> + # No need to monitor because the processes are linked + {:ok, pid} = Task.Supervised.start_link(owner, :nomonitor) + {:ok, :link, pid} + end) + end + end + + # Returns a tuple with the node where this is executed and either the + # registered name of the given PID or the PID of where this is executed. Used + # when exiting from tasks to print out from where the task was started. + defp get_owner(pid) do + self_or_name = + case Process.info(pid, :registered_name) do + {:registered_name, name} when is_atom(name) -> name + _ -> pid + end + + {node(), self_or_name, pid} + end + + defp get_callers(owner) do + case :erlang.get(:"$callers") do + [_ | _] = list -> [owner | list] + _ -> [owner] + end + end + + @doc ~S""" + Awaits a task reply and returns it. + + In case the task process dies, the caller process will exit with the same + reason as the task. + + A timeout, in milliseconds or `:infinity`, can be given with a default value + of `5000`. If the timeout is exceeded, then the caller process will exit. + If the task process is linked to the caller process which is the case when + a task is started with `async`, then the task process will also exit. If the + task process is trapping exits or not linked to the caller process, then it + will continue to run. + + This function assumes the task's monitor is still active or the monitor's + `:DOWN` message is in the message queue. If it has been demonitored, or the + message already received, this function will wait for the duration of the + timeout awaiting the message. + + This function can only be called once for any given task. If you want + to be able to check multiple times if a long-running task has finished + its computation, use `yield/2` instead. + + ## Examples + + iex> task = Task.async(fn -> 1 + 1 end) + iex> Task.await(task) + 2 + + ## Compatibility with OTP behaviours + + It is not recommended to `await` a long-running task inside an OTP + behaviour such as `GenServer`. Instead, you should match on the message + coming from a task inside your `c:GenServer.handle_info/2` callback. + + A GenServer will receive two messages on `handle_info/2`: + + * `{ref, result}` - the reply message where `ref` is the monitor + reference returned by the `task.ref` and `result` is the task + result + + * `{:DOWN, ref, :process, pid, reason}` - since all tasks are also + monitored, you will also receive the `:DOWN` message delivered by + `Process.monitor/1`. If you receive the `:DOWN` message without a + a reply, it means the task crashed + + Another consideration to have in mind is that tasks started by `Task.async/1` + are always linked to their callers and you may not want the GenServer to + crash if the task crashes. Therefore, it is preferable to instead use + `Task.Supervisor.async_nolink/3` inside OTP behaviours. For completeness, here + is an example of a GenServer that start tasks and handles their results: + + defmodule GenServerTaskExample do + use GenServer + + def start_link(opts) do + GenServer.start_link(__MODULE__, :ok, opts) + end + + def init(_opts) do + # We will keep all running tasks in a map + {:ok, %{tasks: %{}}} + end + + # Imagine we invoke a task from the GenServer to access a URL... + def handle_call(:some_message, _from, state) do + url = ... + task = Task.Supervisor.async_nolink(MyApp.TaskSupervisor, fn -> fetch_url(url) end) + + # After we start the task, we store its reference and the url it is fetching + state = put_in(state.tasks[task.ref], url) + + {:reply, :ok, state} + end + + # If the task succeeds... + def handle_info({ref, result}, state) do + # The task succeed so we can demonitor its reference + Process.demonitor(ref, [:flush]) + + {url, state} = pop_in(state.tasks[ref]) + IO.puts("Got #{inspect(result)} for URL #{inspect url}") + {:noreply, state} + end + + # If the task fails... + def handle_info({:DOWN, ref, _, _, reason}, state) do + {url, state} = pop_in(state.tasks[ref]) + IO.puts("URL #{inspect url} failed with reason #{inspect(reason)}") + {:noreply, state} + end + end + + With the server defined, you will want to start the task supervisor + above and the GenServer in your supervision tree: + + children = [ + {Task.Supervisor, name: MyApp.TaskSupervisor}, + {GenServerTaskExample, name: MyApp.GenServerTaskExample} + ] + + Supervisor.start_link(children, strategy: :one_for_one) - A timeout, in milliseconds, can be given with default value - of `5000`. In case the task process dies, this function will - exit with the same reason as the task. """ - @spec await(t, timeout) :: term | no_return - def await(%Task{ref: ref}=task, timeout \\ 5000) do + @spec await(t, timeout) :: term + def await(%Task{ref: ref, owner: owner} = task, timeout \\ 5000) when is_timeout(timeout) do + if owner != self() do + raise ArgumentError, invalid_owner_error(task) + end + + await_receive(ref, task, timeout) + end + + defp await_receive(ref, task, timeout) do receive do {^ref, reply} -> - Process.demonitor(ref, [:flush]) + demonitor(ref) reply - {:DOWN, ^ref, _, _, :noconnection} -> - mfa = {__MODULE__, :await, [task, timeout]} - exit({{:nodedown, node(task.pid)}, mfa}) - {:DOWN, ^ref, _, _, reason} -> - exit({reason, {__MODULE__, :await, [task, timeout]}}) + + {:DOWN, ^ref, _, proc, reason} -> + exit({reason(reason, proc), {__MODULE__, :await, [task, timeout]}}) after timeout -> - Process.demonitor(ref, [:flush]) + demonitor(ref) exit({:timeout, {__MODULE__, :await, [task, timeout]}}) end end @doc """ - Receives a group of tasks and a message and finds - a task that matches the given message. + Ignores an existing task. - This function returns a tuple with the task and the - returned value in case the message matches a task that - exited with success, it raises in case the found task - failed or `nil` if no task was found. + This means the task will continue running, but it will be unlinked + and you can no longer yield, await or shut it down. - This function is useful in situations where multiple - tasks are spawned and their results are collected - later on. For example, a `GenServer` can spawn tasks, - store the tasks in a list and later use `Task.find/2` - to see if incoming messages are from any of the tasks. + Returns `{:ok, reply}` if the reply is received before ignoring the task, + `{:exit, reason}` if the task died before ignoring it, otherwise `nil`. + + Important: avoid using [`Task.async/1,3`](`async/1`) and then immediately ignoring + the task. If you want to start tasks you don't care about their + results, use `Task.Supervisor.start_child/2` instead. """ - @spec find([t], any) :: {term, t} | nil | no_return - def find(tasks, msg) + @doc since: "1.13.0" + @spec ignore(t) :: {:ok, term} | {:exit, term} | nil + def ignore(%Task{ref: ref, pid: pid, owner: owner} = task) do + if owner != self() do + raise ArgumentError, invalid_owner_error(task) + end - def find(tasks, {ref, reply}) when is_reference(ref) do - Enum.find_value tasks, fn - %Task{ref: task_ref} = t when ref == task_ref -> - Process.demonitor(ref, [:flush]) - {reply, t} - %Task{} -> + ignore_receive(ref, pid, task) + end + + defp ignore_receive(ref, pid, task) do + receive do + {^ref, reply} -> + pid && Process.unlink(pid) + demonitor(ref) + {:ok, reply} + + {:DOWN, ^ref, _, proc, :noconnection} -> + exit({reason(:noconnection, proc), {__MODULE__, :ignore, [task]}}) + + {:DOWN, ^ref, _, _, reason} -> + {:exit, reason} + after + 0 -> + pid && Process.unlink(pid) + demonitor(ref) nil end end - def find(tasks, {:DOWN, ref, _, _, reason} = msg) when is_reference(ref) do - find = fn(%Task{ref: task_ref}) -> task_ref == ref end - case Enum.find(tasks, find) do - %Task{pid: pid} when reason == :noconnection -> - exit({{:nodedown, node(pid)}, {__MODULE__, :find, [tasks, msg]}}) + @doc """ + Awaits replies from multiple tasks and returns them. + + This function receives a list of tasks and waits for their replies in the + given time interval. It returns a list of the results, in the same order as + the tasks supplied in the `tasks` input argument. + + If any of the task processes dies, the caller process will exit with the same + reason as that task. + + A timeout, in milliseconds or `:infinity`, can be given with a default value + of `5000`. If the timeout is exceeded, then the caller process will exit. + Any task processes that are linked to the caller process (which is the case + when a task is started with `async`) will also exit. Any task processes that + are trapping exits or not linked to the caller process will continue to run. + + This function assumes the tasks' monitors are still active or the monitor's + `:DOWN` message is in the message queue. If any tasks have been demonitored, + or the message already received, this function will wait for the duration of + the timeout. + + This function can only be called once for any given task. If you want to be + able to check multiple times if a long-running task has finished its + computation, use `yield_many/2` instead. + + ## Compatibility with OTP behaviours + + It is not recommended to `await` long-running tasks inside an OTP behaviour + such as `GenServer`. See `await/2` for more information. + + ## Examples + + iex> tasks = [ + ...> Task.async(fn -> 1 + 1 end), + ...> Task.async(fn -> 2 + 3 end) + ...> ] + iex> Task.await_many(tasks) + [2, 5] + + """ + @doc since: "1.11.0" + @spec await_many([t], timeout) :: [term] + def await_many(tasks, timeout \\ 5000) when is_timeout(timeout) do + awaiting = + Map.new(tasks, fn %Task{ref: ref, owner: owner} = task -> + if owner != self() do + raise ArgumentError, invalid_owner_error(task) + end + + {ref, true} + end) + + timeout_ref = make_ref() + + timer_ref = + if timeout != :infinity do + Process.send_after(self(), timeout_ref, timeout) + end + + try do + await_many(tasks, timeout, awaiting, %{}, timeout_ref) + after + timer_ref && Process.cancel_timer(timer_ref) + receive do: (^timeout_ref -> :ok), after: (0 -> :ok) + end + end + + defp await_many(tasks, _timeout, awaiting, replies, _timeout_ref) + when map_size(awaiting) == 0 do + for %{ref: ref} <- tasks, do: Map.fetch!(replies, ref) + end + + defp await_many(tasks, timeout, awaiting, replies, timeout_ref) do + receive do + ^timeout_ref -> + demonitor_pending_tasks(awaiting) + exit({:timeout, {__MODULE__, :await_many, [tasks, timeout]}}) + + {:DOWN, ref, _, proc, reason} when is_map_key(awaiting, ref) -> + demonitor_pending_tasks(awaiting) + exit({reason(reason, proc), {__MODULE__, :await_many, [tasks, timeout]}}) + + {ref, reply} when is_map_key(awaiting, ref) -> + demonitor(ref) + + await_many( + tasks, + timeout, + Map.delete(awaiting, ref), + Map.put(replies, ref, reply), + timeout_ref + ) + end + end + + defp demonitor_pending_tasks(awaiting) do + Enum.each(awaiting, fn {ref, _} -> + demonitor(ref) + end) + end + + @doc false + @deprecated "Pattern match directly on the message instead" + def find(tasks, {ref, reply}) when is_reference(ref) do + Enum.find_value(tasks, fn + %Task{ref: ^ref} = task -> + demonitor(ref) + {reply, task} + %Task{} -> - exit({reason, {__MODULE__, :find, [tasks, msg]}}) - nil -> nil + end) + end + + def find(tasks, {:DOWN, ref, _, proc, reason} = msg) when is_reference(ref) do + find = fn %Task{ref: task_ref} -> task_ref == ref end + + if Enum.find(tasks, find) do + exit({reason(reason, proc), {__MODULE__, :find, [tasks, msg]}}) end end def find(_tasks, _msg) do nil end + + @doc ~S""" + Temporarily blocks the caller process waiting for a task reply. + + Returns `{:ok, reply}` if the reply is received, `nil` if + no reply has arrived, or `{:exit, reason}` if the task has already + exited. Keep in mind that normally a task failure also causes + the process owning the task to exit. Therefore this function can + return `{:exit, reason}` if at least one of the conditions below apply: + + * the task process exited with the reason `:normal` + * the task isn't linked to the caller (the task was started + with `Task.Supervisor.async_nolink/2` or `Task.Supervisor.async_nolink/4`) + * the caller is trapping exits + + A timeout, in milliseconds or `:infinity`, can be given with a default value + of `5000`. If the time runs out before a message from the task is received, + this function will return `nil` and the monitor will remain active. Therefore + `yield/2` can be called multiple times on the same task. + + This function assumes the task's monitor is still active or the + monitor's `:DOWN` message is in the message queue. If it has been + demonitored or the message already received, this function will wait + for the duration of the timeout awaiting the message. + + If you intend to shut the task down if it has not responded within `timeout` + milliseconds, you should chain this together with `shutdown/1`, like so: + + case Task.yield(task, timeout) || Task.shutdown(task) do + {:ok, result} -> + result + + nil -> + Logger.warning("Failed to get a result in #{timeout}ms") + nil + end + + If you intend to check on the task but leave it running after the timeout, + you can chain this together with `ignore/1`, like so: + + case Task.yield(task, timeout) || Task.ignore(task) do + {:ok, result} -> + result + + nil -> + Logger.warning("Failed to get a result in #{timeout}ms") + nil + end + + That ensures that if the task completes after the `timeout` but before `shutdown/1` + has been called, you will still get the result, since `shutdown/1` is designed to + handle this case and return the result. + """ + @spec yield(t, timeout) :: {:ok, term} | {:exit, term} | nil + def yield(%Task{ref: ref, owner: owner} = task, timeout \\ 5000) when is_timeout(timeout) do + if owner != self() do + raise ArgumentError, invalid_owner_error(task) + end + + yield_receive(ref, task, timeout) + end + + defp yield_receive(ref, task, timeout) do + receive do + {^ref, reply} -> + demonitor(ref) + {:ok, reply} + + {:DOWN, ^ref, _, proc, :noconnection} -> + exit({reason(:noconnection, proc), {__MODULE__, :yield, [task, timeout]}}) + + {:DOWN, ^ref, _, _, reason} -> + {:exit, reason} + after + timeout -> + nil + end + end + + @doc """ + Yields to multiple tasks in the given time interval. + + This function receives a list of tasks and waits for their + replies in the given time interval. It returns a list + of two-element tuples, with the task as the first element + and the yielded result as the second. The tasks in the returned + list will be in the same order as the tasks supplied in the `tasks` + input argument. + + Similarly to `yield/2`, each task's result will be + + * `{:ok, term}` if the task has successfully reported its + result back in the given time interval + * `{:exit, reason}` if the task has died + * `nil` if the task keeps running, either because a limit + has been reached or past the timeout + + Check `yield/2` for more information. + + ## Example + + `Task.yield_many/2` allows developers to spawn multiple tasks + and retrieve the results received in a given time frame. + If we combine it with `Task.shutdown/2` (or `Task.ignore/1`), + it allows us to gather those results and cancel (or ignore) + the tasks that have not replied in time. + + Let's see an example. + + tasks = + for i <- 1..10 do + Task.async(fn -> + Process.sleep(i * 1000) + i + end) + end + + tasks_with_results = Task.yield_many(tasks, timeout: 5000) + + results = + Enum.map(tasks_with_results, fn {task, res} -> + # Shut down the tasks that did not reply nor exit + res || Task.shutdown(task, :brutal_kill) + end) + + # Here we are matching only on {:ok, value} and + # ignoring {:exit, _} (crashed tasks) and `nil` (no replies) + for {:ok, value} <- results do + IO.inspect(value) + end + + In the example above, we create tasks that sleep from 1 + up to 10 seconds and return the number of seconds they slept for. + If you execute the code all at once, you should see 1 up to 5 + printed, as those were the tasks that have replied in the + given time. All other tasks will have been shut down using + the `Task.shutdown/2` call. + + As a convenience, you can achieve a similar behavior to above + by specifying the `:on_timeout` option to be `:kill_task` (or + `:ignore`). See `Task.await_many/2` if you would rather exit + the caller process on timeout. + + ## Options + + The second argument is either a timeout or options, which defaults + to this: + + * `:limit` - the maximum amount of tasks to wait for. + If the limit is reached before the timeout, this function + returns immediately without triggering the `:on_timeout` behaviour + + * `:timeout` - the maximum amount of time (in milliseconds or `:infinity`) + each task is allowed to execute for. Defaults to `5000`. + + * `:on_timeout` - what to do when a task times out. The possible + values are: + * `:nothing` - do nothing (default). The tasks can still be + awaited on, yielded on, ignored, or shut down later. + * `:ignore` - the results of the task will be ignored. + * `:kill_task` - the task that timed out is killed. + """ + @spec yield_many([t], timeout) :: [{t, {:ok, term} | {:exit, term} | nil}] + @spec yield_many([t], + limit: pos_integer(), + timeout: timeout, + on_timeout: :nothing | :ignore | :kill_task + ) :: + [{t, {:ok, term} | {:exit, term} | nil}] + def yield_many(tasks, opts \\ []) + + def yield_many(tasks, timeout) when is_timeout(timeout) do + yield_many(tasks, timeout: timeout) + end + + def yield_many(tasks, opts) when is_list(opts) do + refs = + Map.new(tasks, fn %Task{ref: ref, owner: owner} = task -> + if owner != self() do + raise ArgumentError, invalid_owner_error(task) + end + + {ref, nil} + end) + + on_timeout = Keyword.get(opts, :on_timeout, :nothing) + timeout = Keyword.get(opts, :timeout, 5_000) + limit = Keyword.get(opts, :limit, map_size(refs)) + timeout_ref = make_ref() + + timer_ref = + if timeout != :infinity do + Process.send_after(self(), timeout_ref, timeout) + end + + try do + yield_many(limit, refs, timeout_ref, timer_ref) + catch + {:noconnection, reason} -> + exit({reason, {__MODULE__, :yield_many, [tasks, timeout]}}) + else + {timed_out?, refs} -> + for task <- tasks do + value = + with nil <- Map.fetch!(refs, task.ref) do + case on_timeout do + _ when not timed_out? -> nil + :nothing -> nil + :kill_task -> shutdown(task, :brutal_kill) + :ignore -> ignore(task) + end + end + + {task, value} + end + end + end + + defp yield_many(0, refs, timeout_ref, timer_ref) do + timer_ref && Process.cancel_timer(timer_ref) + receive do: (^timeout_ref -> :ok), after: (0 -> :ok) + {false, refs} + end + + defp yield_many(limit, refs, timeout_ref, timer_ref) do + receive do + {ref, reply} when is_map_key(refs, ref) -> + demonitor(ref) + yield_many(limit - 1, %{refs | ref => {:ok, reply}}, timeout_ref, timer_ref) + + {:DOWN, ref, _, proc, reason} when is_map_key(refs, ref) -> + if reason == :noconnection do + throw({:noconnection, reason(:noconnection, proc)}) + else + yield_many(limit - 1, %{refs | ref => {:exit, reason}}, timeout_ref, timer_ref) + end + + ^timeout_ref -> + {true, refs} + end + end + + @doc """ + Unlinks and shuts down the task, and then checks for a reply. + + Returns `{:ok, reply}` if the reply is received while shutting down the task, + `{:exit, reason}` if the task died, otherwise `nil`. Once shut down, + you can no longer await or yield it. + + The second argument is either a timeout or `:brutal_kill`. In case + of a timeout, a `:shutdown` exit signal is sent to the task process + and if it does not exit within the timeout, it is killed. With `:brutal_kill` + the task is killed straight away. In case the task terminates abnormally + (possibly killed by another process), this function will exit with the same reason. + + It is not required to call this function when terminating the caller, unless + exiting with reason `:normal` or if the task is trapping exits. If the caller is + exiting with a reason other than `:normal` and the task is not trapping exits, the + caller's exit signal will stop the task. The caller can exit with reason + `:shutdown` to shut down all of its linked processes, including tasks, that + are not trapping exits without generating any log messages. + + If there is no process linked to the task, such as tasks started by + `Task.completed/1`, we check for a response or error accordingly, but without + shutting a process down. + + If a task's monitor has already been demonitored or received and there is not + a response waiting in the message queue this function will return + `{:exit, :noproc}` as the result or exit reason can not be determined. + """ + @spec shutdown(t, timeout | :brutal_kill) :: {:ok, term} | {:exit, term} | nil + def shutdown(task, shutdown \\ 5000) + + def shutdown(%Task{pid: nil} = task, _) do + ignore(task) + end + + def shutdown(%Task{owner: owner} = task, _) when owner != self() do + raise ArgumentError, invalid_owner_error(task) + end + + def shutdown(%Task{pid: pid, ref: ref} = task, :brutal_kill) do + mon = build_monitor(pid) + shutdown_send(pid, :kill) + + case shutdown_receive(ref, mon, task, :brutal_kill, :infinity) do + {:down, proc, :noconnection} -> + exit({reason(:noconnection, proc), {__MODULE__, :shutdown, [task, :brutal_kill]}}) + + {:down, _, reason} -> + {:exit, reason} + + result -> + result + end + end + + def shutdown(%Task{pid: pid, ref: ref} = task, timeout) when is_timeout(timeout) do + mon = build_monitor(pid) + shutdown_send(pid, :shutdown) + + case shutdown_receive(ref, mon, task, :shutdown, timeout) do + {:down, proc, :noconnection} -> + exit({reason(:noconnection, proc), {__MODULE__, :shutdown, [task, timeout]}}) + + {:down, _, reason} -> + {:exit, reason} + + result -> + result + end + end + + # Spawn a process to ensure task gets exit signal + # if process dies from exit signal between unlink and exit. + defp shutdown_send(pid, reason) do + caller = self() + ref = make_ref() + enforcer = spawn(fn -> shutdown_send(pid, reason, caller, ref) end) + Process.unlink(pid) + Process.exit(pid, reason) + send(enforcer, {:done, ref}) + :ok + end + + defp shutdown_send(pid, reason, caller, ref) do + mon = Process.monitor(caller) + + receive do + {:done, ^ref} -> :ok + {:DOWN, ^mon, _, _, _} -> Process.exit(pid, reason) + end + end + + defp shutdown_receive(ref, mon, task, type, timeout) do + receive do + {:DOWN, ^mon, _, _, :shutdown} when type in [:shutdown, :timeout_kill] -> + demonitor(ref) + flush_reply(ref) + + {:DOWN, ^mon, _, _, :killed} when type == :brutal_kill -> + demonitor(ref) + flush_reply(ref) + + {:DOWN, ^mon, _, proc, :noproc} -> + reason = flush_noproc(ref, proc, type) + flush_reply(ref) || reason + + {:DOWN, ^mon, _, proc, reason} -> + demonitor(ref) + flush_reply(ref) || {:down, proc, reason} + after + timeout -> + Process.exit(task.pid, :kill) + shutdown_receive(ref, mon, task, :timeout_kill, :infinity) + end + end + + defp flush_reply(ref) do + receive do + {^ref, reply} -> {:ok, reply} + after + 0 -> nil + end + end + + defp flush_noproc(ref, proc, type) do + receive do + {:DOWN, ^ref, _, _, :shutdown} when type in [:shutdown, :timeout_kill] -> + nil + + {:DOWN, ^ref, _, _, :killed} when type == :brutal_kill -> + nil + + {:DOWN, ^ref, _, _, reason} -> + {:down, proc, reason} + after + 0 -> + demonitor(ref) + {:down, proc, :noproc} + end + end + + # exported only to avoid dialyzer opaqueness check in internal Task modules + @doc false + @spec __alias__(pid()) :: Task.ref() + def __alias__(pid) do + build_alias(pid) + end + + ## Optimizations + + defp build_monitor(pid) do + :erlang.monitor(:process, pid) + end + + defp build_alias(pid) do + :erlang.monitor(:process, pid, alias: :demonitor) + end + + @doc false + # This instructs the Erlang compiler to apply selective + # receive optimizations to several functions in this module. + # This function is never invoked directly, it is only here + # for compiler optimization purposes. + # + # To verify which functions have been optimized, run the + # following command after Elixir is compiled from the project + # root: + # + # ERL_COMPILER_OPTIONS=recv_opt_info elixir lib/elixir/lib/task.ex + # + def __recv_opt_info__(pid, task) do + await_receive(build_alias(pid), task, :infinity) + shutdown_receive(build_alias(pid), build_monitor(pid), task, :shutdown, :infinity) + yield_receive(build_alias(pid), task, :infinity) + ignore_receive(build_alias(pid), pid, task) + end + + ## Helpers + + defp demonitor(ref) when is_reference(ref) do + Process.demonitor(ref, [:flush]) + :ok + end + + defp reason(:noconnection, proc), do: {:nodedown, monitor_node(proc)} + defp reason(reason, _), do: reason + + defp monitor_node(pid) when is_pid(pid), do: node(pid) + defp monitor_node({_, node}), do: node + + defp invalid_owner_error(task) do + "task #{inspect(task)} must be queried from the owner but was queried from #{inspect(self())}" + end end diff --git a/lib/elixir/lib/task/supervised.ex b/lib/elixir/lib/task/supervised.ex index 921e8432e8c..54149a45fe6 100644 --- a/lib/elixir/lib/task/supervised.ex +++ b/lib/elixir/lib/task/supervised.ex @@ -1,28 +1,49 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + defmodule Task.Supervised do @moduledoc false + @ref_timeout 5000 + + def start(owner, callers, fun) do + {:ok, spawn(__MODULE__, :noreply, [owner, get_ancestors(), callers, fun])} + end - def start_link(info, fun) do - {:ok, :proc_lib.spawn_link(__MODULE__, :noreply, [info, fun])} + def start_link(owner, callers, fun) do + {:ok, spawn_link(__MODULE__, :noreply, [owner, get_ancestors(), callers, fun])} end - def start_link(caller, info, fun) do - :proc_lib.start_link(__MODULE__, :reply, [caller, info, fun]) + def start_link(owner, monitor) do + {:ok, spawn_link(__MODULE__, :reply, [owner, get_ancestors(), monitor])} end - def async(caller, info, mfa) do - initial_call(mfa) - ref = receive do: ({^caller, ref} -> ref) - send caller, {ref, do_apply(info, mfa)} + def reply({_, _, owner_pid} = owner, ancestors, monitor) do + put_ancestors(ancestors) + + case monitor do + :monitor -> + mref = Process.monitor(owner_pid) + reply(owner, owner_pid, mref, @ref_timeout) + + :nomonitor -> + reply(owner, owner_pid, nil, :infinity) + end end - def reply(caller, info, mfa) do - initial_call(mfa) - :erlang.link(caller) - :proc_lib.init_ack({:ok, self()}) + defp reply(owner, owner_pid, mref, timeout) do + receive do + {^owner_pid, ref, reply_to, callers, mfa} -> + put_initial_call(mfa) + put_callers(callers) + _ = is_reference(mref) && Process.demonitor(mref, [:flush]) + send(reply_to, {ref, invoke_mfa(owner, mfa)}) - ref = - # There is a race condition on this operation when working accross - # node that manifests if a `Task.Supervisor.async/1` call is made + {:DOWN, ^mref, _, _, reason} -> + exit({:shutdown, reason}) + after + # There is a race condition on this operation when working across + # node that manifests if a "Task.Supervisor.async/2" call is made # while the supervisor is busy spawning previous tasks. # # Imagine the following workflow: @@ -33,75 +54,634 @@ defmodule Task.Supervised do # 4. The calling process has not exited and so does not send its monitor reference # 5. The spawned task waits forever for the monitor reference so it can begin # - # We have solved this by specifying a timeout of 5000 seconds. - # Given no work is done in the client in between the task start and + # We have solved this by specifying a timeout of 5000 milliseconds. + # Given no work is done in the client between the task start and # sending the reference, 5000 should be enough to not raise false # negatives unless the nodes are indeed not available. - receive do - {^caller, ref} -> ref - after - 5000 -> exit(:timeout) - end + # + # The same situation could occur with "Task.Supervisor.async_nolink/2", + # except a monitor is used instead of a link. + timeout -> + exit(:timeout) + end + end + + def noreply(owner, ancestors, callers, mfa) do + put_initial_call(mfa) + put_ancestors(ancestors) + put_callers(callers) + invoke_mfa(owner, mfa) + end + + defp get_ancestors() do + case :erlang.get(:"$ancestors") do + ancestors when is_list(ancestors) -> [self() | ancestors] + _ -> [self()] + end + end - send caller, {ref, do_apply(info, mfa)} + defp put_ancestors(ancestors) do + Process.put(:"$ancestors", ancestors) end - def noreply(info, mfa) do - initial_call(mfa) - do_apply(info, mfa) + defp put_callers(callers) do + Process.put(:"$callers", callers) end - defp initial_call(mfa) do + defp put_initial_call(mfa) do Process.put(:"$initial_call", get_initial_call(mfa)) end defp get_initial_call({:erlang, :apply, [fun, []]}) when is_function(fun, 0) do - {:module, module} = :erlang.fun_info(fun, :module) - {:name, name} = :erlang.fun_info(fun, :name) - {module, name, 0} + :erlang.fun_info_mfa(fun) end defp get_initial_call({mod, fun, args}) do {mod, fun, length(args)} end - defp do_apply(info, {module, fun, args} = mfa) do + defp invoke_mfa(owner, {module, fun, args} = mfa) do try do apply(module, fun, args) catch - :error, value -> - exit(info, mfa, {value, System.stacktrace()}) - :throw, value -> - exit(info, mfa, {{:nocatch, value}, System.stacktrace()}) - :exit, value -> - exit(info, mfa, value) + :exit, value + when value == :normal + when value == :shutdown + when tuple_size(value) == 2 and elem(value, 0) == :shutdown -> + :erlang.raise(:exit, value, __STACKTRACE__) + + kind, value -> + {fun, args} = get_running(mfa) + + :logger.error( + %{ + label: {Task.Supervisor, :terminating}, + report: %{ + name: self(), + starter: get_from(owner), + function: fun, + args: args, + reason: {log_value(kind, value), __STACKTRACE__}, + # TODO use Process.get_label/0 when we require Erlang/OTP 27+ + process_label: Process.get(:"$process_label", :undefined) + } + }, + %{ + domain: [:otp, :elixir], + error_logger: %{tag: :error_msg}, + report_cb: &__MODULE__.format_report/1, + callers: Process.get(:"$callers") + } + ) + + :erlang.raise(:exit, exit_reason(kind, value, __STACKTRACE__), __STACKTRACE__) end end - defp exit(_info, _mfa, reason) - when reason == :normal - when reason == :shutdown - when tuple_size(reason) == 2 and elem(reason, 0) == :shutdown do - exit(reason) + defp exit_reason(:error, reason, stacktrace), do: {reason, stacktrace} + defp exit_reason(:exit, reason, _stacktrace), do: reason + defp exit_reason(:throw, reason, stacktrace), do: {{:nocatch, reason}, stacktrace} + + defp log_value(:throw, value), do: {:nocatch, value} + defp log_value(_, value), do: value + + @doc false + def format_report(%{ + label: {Task.Supervisor, :terminating}, + report: %{ + name: name, + starter: starter, + function: fun, + args: args, + reason: reason, + process_label: process_label + } + }) do + message = + ~c"** Started from ~p~n" ++ + ~c"** When function == ~p~n" ++ + ~c"** arguments == ~p~n" ++ ~c"** Reason for termination == ~n" ++ ~c"** ~p~n" + + terms = [starter, fun, args, get_reason(reason)] + + {message, terms} = + case process_label do + :undefined -> {message, terms} + _ -> {~c"** Process Label == ~p~n" ++ message, [process_label | terms]} + end + + message = + ~c"** Task ~p terminating~n" ++ message + + {message, [name | terms]} + end + + defp get_from({node, pid_or_name, _pid}) when node == node(), do: pid_or_name + defp get_from({node, name, _pid}) when is_atom(name), do: {node, name} + defp get_from({_node, _name, pid}), do: pid + + defp get_running({:erlang, :apply, [fun, []]}) when is_function(fun, 0), do: {fun, []} + defp get_running({mod, fun, args}), do: {Function.capture(mod, fun, length(args)), args} + + defp get_reason({:undef, [{mod, fun, args, _info} | _] = stacktrace} = reason) + when is_atom(mod) and is_atom(fun) do + cond do + not Code.loaded?(mod) -> + {:"module could not be loaded", stacktrace} + + is_list(args) and not function_exported?(mod, fun, length(args)) -> + {:"function not exported", stacktrace} + + is_integer(args) and not function_exported?(mod, fun, args) -> + {:"function not exported", stacktrace} + + true -> + reason + end + end + + defp get_reason(reason) do + reason + end + + ## Stream + + def validate_stream_options(options) do + max_concurrency = Keyword.get_lazy(options, :max_concurrency, &System.schedulers_online/0) + on_timeout = Keyword.get(options, :on_timeout, :exit) + timeout = Keyword.get(options, :timeout, 5000) + ordered = Keyword.get(options, :ordered, true) + zip_input_on_exit = Keyword.get(options, :zip_input_on_exit, false) + + if not (is_integer(max_concurrency) and max_concurrency > 0) do + raise ArgumentError, ":max_concurrency must be an integer greater than zero" + end + + if on_timeout not in [:exit, :kill_task] do + raise ArgumentError, ":on_timeout must be either :exit or :kill_task" + end + + if not ((is_integer(timeout) and timeout >= 0) or timeout == :infinity) do + raise ArgumentError, ":timeout must be either a positive integer or :infinity" + end + + %{ + max_concurrency: max_concurrency, + on_timeout: on_timeout, + timeout: timeout, + ordered: ordered, + zip_input_on_exit: zip_input_on_exit + } + end + + def stream(enumerable, acc, reducer, callers, mfa, options, spawn) when is_map(options) do + next = &Enumerable.reduce(enumerable, &1, fn x, acc -> {:suspend, [x | acc]} end) + parent = self() + + {:trap_exit, trap_exit?} = Process.info(self(), :trap_exit) + + # Start a process responsible for spawning processes and translating "down" + # messages. This process will trap exits if the current process is trapping + # exit, or it won't trap exits otherwise. + spawn_opts = [:link, :monitor] + + {monitor_pid, monitor_ref} = + Process.spawn( + fn -> stream_monitor(parent, spawn, trap_exit?, options.timeout) end, + spawn_opts + ) + + # Now that we have the pid of the "monitor" process and the reference of the + # monitor we use to monitor such process, we can inform the monitor process + # about our reference to it. + send(monitor_pid, {parent, monitor_ref}) + + config = + Map.merge( + options, + %{ + reducer: reducer, + monitor_pid: monitor_pid, + monitor_ref: monitor_ref, + callers: callers, + mfa: mfa + } + ) + + stream_reduce( + acc, + options.max_concurrency, + _spawned = 0, + _delivered = 0, + _waiting = %{}, + next, + config + ) + end + + defp stream_reduce({:halt, acc}, _max, _spawned, _delivered, _waiting, next, config) do + stream_close(config) + is_function(next) && next.({:halt, []}) + {:halted, acc} + end + + defp stream_reduce({:suspend, acc}, max, spawned, delivered, waiting, next, config) do + continuation = &stream_reduce(&1, max, spawned, delivered, waiting, next, config) + {:suspended, acc, continuation} + end + + # All spawned, all delivered, next is :done. + defp stream_reduce({:cont, acc}, _max, spawned, delivered, _waiting, next, config) + when spawned == delivered and next == :done do + stream_close(config) + {:done, acc} + end + + # No more tasks to spawn because max == 0 or next is :done. We wait for task + # responses or tasks going down. + defp stream_reduce({:cont, acc}, max, spawned, delivered, waiting, next, config) + when max == 0 + when next == :done do + %{ + monitor_pid: monitor_pid, + monitor_ref: monitor_ref, + timeout: timeout, + on_timeout: on_timeout, + zip_input_on_exit: zip_input_on_exit?, + ordered: ordered? + } = config + + receive do + # The task at position "position" replied with "value". We put the + # response in the "waiting" map and do nothing, since we'll only act on + # this response when the replying task dies (we'll see this in the :down + # message). + {{^monitor_ref, position}, reply} -> + %{^position => {pid, :running, _element}} = waiting + waiting = Map.put(waiting, position, {pid, {:ok, reply}}) + stream_reduce({:cont, acc}, max, spawned, delivered, waiting, next, config) + + # The task at position "position" died for some reason. We check if it + # replied already (then the death is peaceful) or if it's still running + # (then the reply from this task will be {:exit, reason}). This message is + # sent to us by the monitor process, not by the dying task directly. + {kind, {^monitor_ref, position}, reason} + when kind in [:down, :timed_out] -> + result = + case waiting do + # If the task replied, we don't care whether it went down for timeout + # or for normal reasons. + %{^position => {_, {:ok, _} = ok}} -> + ok + + # If the task exited by itself before replying, we emit {:exit, reason}. + %{^position => {_, :running, element}} + when kind == :down -> + if zip_input_on_exit?, do: {:exit, {element, reason}}, else: {:exit, reason} + + # If the task timed out before replying, we either exit (on_timeout: :exit) + # or emit {:exit, :timeout} (on_timeout: :kill_task) (note the task is already + # dead at this point). + %{^position => {_, :running, element}} + when kind == :timed_out -> + if on_timeout == :exit do + stream_cleanup_inbox(monitor_pid, monitor_ref) + exit({:timeout, {__MODULE__, :stream, [timeout]}}) + else + if zip_input_on_exit?, do: {:exit, {element, :timeout}}, else: {:exit, :timeout} + end + end + + if ordered? do + waiting = Map.put(waiting, position, {:done, result}) + stream_deliver({:cont, acc}, max + 1, spawned, delivered, waiting, next, config) + else + pair = deliver_now(result, acc, next, config) + waiting = Map.delete(waiting, position) + stream_reduce(pair, max + 1, spawned, delivered + 1, waiting, next, config) + end + + # The monitor process died. We just cleanup the messages from the monitor + # process and exit. + {:DOWN, ^monitor_ref, _, _, reason} -> + stream_cleanup_inbox(monitor_pid, monitor_ref) + exit({reason, {__MODULE__, :stream, [timeout]}}) + end + end + + defp stream_reduce({:cont, acc}, max, spawned, delivered, waiting, next, config) do + try do + next.({:cont, []}) + catch + kind, reason -> + stream_close(config) + :erlang.raise(kind, reason, __STACKTRACE__) + else + {:suspended, [value], next} -> + waiting = stream_spawn(value, spawned, waiting, config) + stream_reduce({:cont, acc}, max - 1, spawned + 1, delivered, waiting, next, config) + + {_, [value]} -> + waiting = stream_spawn(value, spawned, waiting, config) + stream_reduce({:cont, acc}, max - 1, spawned + 1, delivered, waiting, :done, config) + + {_, []} -> + stream_reduce({:cont, acc}, max, spawned, delivered, waiting, :done, config) + end + end + + defp deliver_now(reply, acc, next, config) do + %{reducer: reducer} = config + + try do + reducer.(reply, acc) + catch + kind, reason -> + is_function(next) && next.({:halt, []}) + stream_close(config) + :erlang.raise(kind, reason, __STACKTRACE__) + end + end + + defp stream_deliver({:suspend, acc}, max, spawned, delivered, waiting, next, config) do + continuation = &stream_deliver(&1, max, spawned, delivered, waiting, next, config) + {:suspended, acc, continuation} + end + + defp stream_deliver({:halt, acc}, max, spawned, delivered, waiting, next, config) do + stream_reduce({:halt, acc}, max, spawned, delivered, waiting, next, config) + end + + defp stream_deliver({:cont, acc}, max, spawned, delivered, waiting, next, config) do + %{reducer: reducer} = config + + case waiting do + %{^delivered => {:done, reply}} -> + try do + reducer.(reply, acc) + catch + kind, reason -> + is_function(next) && next.({:halt, []}) + stream_close(config) + :erlang.raise(kind, reason, __STACKTRACE__) + else + pair -> + waiting = Map.delete(waiting, delivered) + stream_deliver(pair, max, spawned, delivered + 1, waiting, next, config) + end + + %{} -> + stream_reduce({:cont, acc}, max, spawned, delivered, waiting, next, config) + end end - defp exit(info, mfa, reason) do - {fun, args} = get_running(mfa) + defp stream_close(%{monitor_pid: monitor_pid, monitor_ref: monitor_ref, timeout: timeout}) do + send(monitor_pid, {:stop, monitor_ref}) - :error_logger.format( - "** Task ~p terminating~n" <> - "** Started from ~p~n" <> - "** When function == ~p~n" <> - "** arguments == ~p~n" <> - "** Reason for termination == ~n" <> - "** ~p~n", [self, get_from(info), fun, args, reason]) + receive do + {:DOWN, ^monitor_ref, _, _, :normal} -> + stream_cleanup_inbox(monitor_pid, monitor_ref) + :ok + + {:DOWN, ^monitor_ref, _, _, reason} -> + stream_cleanup_inbox(monitor_pid, monitor_ref) + exit({reason, {__MODULE__, :stream, [timeout]}}) + end + end + + defp stream_cleanup_inbox(monitor_pid, monitor_ref) do + receive do + {:EXIT, ^monitor_pid, _} -> stream_cleanup_inbox(monitor_ref) + after + 0 -> stream_cleanup_inbox(monitor_ref) + end + end + + defp stream_cleanup_inbox(monitor_ref) do + receive do + {{^monitor_ref, _}, _} -> + stream_cleanup_inbox(monitor_ref) + + {kind, {^monitor_ref, _}, _} when kind in [:down, :timed_out] -> + stream_cleanup_inbox(monitor_ref) + after + 0 -> + :ok + end + end + + # This function spawns a task for the given "value", and puts the pid of this + # new task in the map of "waiting" tasks, which is returned. + defp stream_spawn(value, spawned, waiting, config) do + %{ + monitor_pid: monitor_pid, + monitor_ref: monitor_ref, + timeout: timeout, + callers: callers, + mfa: mfa, + zip_input_on_exit: zip_input_on_exit? + } = config + + send(monitor_pid, {:spawn, spawned}) + + receive do + {:spawned, {^monitor_ref, ^spawned}, pid} -> + mfa_with_value = normalize_mfa_with_arg(mfa, value) + send(pid, {self(), {monitor_ref, spawned}, self(), callers, mfa_with_value}) + stored_value = if zip_input_on_exit?, do: value, else: nil + Map.put(waiting, spawned, {pid, :running, stored_value}) + + {:max_children, ^monitor_ref} -> + stream_close(config) + + raise """ + reached the maximum number of tasks for this task supervisor. The maximum number \ + of tasks that are allowed to run at the same time under this supervisor can be \ + configured with the :max_children option passed to Task.Supervisor.start_link/1. When \ + using async_stream or async_stream_nolink, make sure to configure :max_concurrency to \ + be lower or equal to :max_children and pay attention to whether other tasks are also \ + spawned under the same task supervisor.\ + """ + + {:DOWN, ^monitor_ref, _, ^monitor_pid, reason} -> + stream_cleanup_inbox(monitor_pid, monitor_ref) + exit({reason, {__MODULE__, :stream, [timeout]}}) + end + end + + defp stream_monitor(parent_pid, spawn, trap_exit?, timeout) do + Process.flag(:trap_exit, trap_exit?) + parent_ref = Process.monitor(parent_pid) + + # Let's wait for the parent process to tell this process the monitor ref + # it's using to monitor this process. If the parent process dies while this + # process waits, this process dies with the same reason. + receive do + {^parent_pid, monitor_ref} -> + config = %{ + parent_pid: parent_pid, + parent_ref: parent_ref, + spawn: spawn, + monitor_ref: monitor_ref, + timeout: timeout + } + + stream_monitor_loop(_running_tasks = %{}, config) + + {:DOWN, ^parent_ref, _, _, reason} -> + exit(reason) + end + end + + defp stream_monitor_loop(running_tasks, config) do + %{ + spawn: spawn, + parent_pid: parent_pid, + monitor_ref: monitor_ref, + timeout: timeout + } = config + + receive do + # The parent process is telling us to spawn a new task to process + # "value". We spawn it and notify the parent about its pid. + {:spawn, position} -> + case spawn.() do + {:ok, type, pid} -> + ref = Process.monitor(pid) + + # Schedule a timeout message to ourselves, unless the timeout was set to :infinity + timer_ref = + case timeout do + :infinity -> nil + timeout -> Process.send_after(self(), {:timeout, {monitor_ref, ref}}, timeout) + end + + send(parent_pid, {:spawned, {monitor_ref, position}, pid}) + + running_tasks = + Map.put(running_tasks, ref, %{ + position: position, + type: type, + pid: pid, + timer_ref: timer_ref, + timed_out?: false + }) + + stream_monitor_loop(running_tasks, config) + + {:error, :max_children} -> + send(parent_pid, {:max_children, monitor_ref}) + stream_waiting_for_stop_loop(running_tasks, config) + end + + # One of the spawned processes went down. We inform the parent process of + # this and keep going. + {:DOWN, ref, _, _, reason} when is_map_key(running_tasks, ref) -> + {task, running_tasks} = Map.pop(running_tasks, ref) + %{position: position, timer_ref: timer_ref, timed_out?: timed_out?} = task + + if timer_ref != nil do + :ok = Process.cancel_timer(timer_ref, async: true, info: false) + end + + message_kind = if(timed_out?, do: :timed_out, else: :down) + send(parent_pid, {message_kind, {monitor_ref, position}, reason}) + stream_monitor_loop(running_tasks, config) + + # One of the spawned processes timed out. We kill that process here + # regardless of the value of :on_timeout. We then send a message to the + # parent process informing it that a task timed out, and the parent + # process decides what to do. + {:timeout, {^monitor_ref, ref}} -> + running_tasks = + case running_tasks do + %{^ref => %{pid: pid, timed_out?: false} = task_info} -> + unlink_and_kill(pid) + Map.put(running_tasks, ref, %{task_info | timed_out?: true}) + + _other -> + running_tasks + end + + stream_monitor_loop(running_tasks, config) + + {:EXIT, _, _} -> + stream_monitor_loop(running_tasks, config) + + other -> + handle_stop_or_parent_down(other, running_tasks, config) + stream_monitor_loop(running_tasks, config) + end + end + + defp stream_waiting_for_stop_loop(running_tasks, config) do + receive do + message -> + handle_stop_or_parent_down(message, running_tasks, config) + stream_waiting_for_stop_loop(running_tasks, config) + end + end + + # The parent process is telling us to stop because the stream is being + # closed. In this case, we forcibly kill all spawned processes and then + # exit gracefully ourselves. + defp handle_stop_or_parent_down( + {:stop, monitor_ref}, + running_tasks, + %{monitor_ref: monitor_ref} + ) do + Process.flag(:trap_exit, true) + + for {_ref, %{pid: pid}} <- running_tasks, do: Process.exit(pid, :kill) + + for {ref, _task} <- running_tasks do + receive do + {:DOWN, ^ref, _, _, _} -> :ok + end + end + + exit(:normal) + end + + # The parent process went down with a given reason. We kill all the + # spawned processes (that are also linked) with the same reason, and then + # exit ourselves with the same reason. + defp handle_stop_or_parent_down( + {:DOWN, parent_ref, _, _, reason}, + running_tasks, + %{parent_ref: parent_ref} + ) do + for {_ref, %{type: :link, pid: pid}} <- running_tasks do + Process.exit(pid, reason) + end exit(reason) end - defp get_from({node, pid_or_name}) when node == node(), do: pid_or_name - defp get_from(other), do: other + # We ignore all other messages. + defp handle_stop_or_parent_down(_other, _running_tasks, _config) do + :ok + end - defp get_running({:erlang, :apply, [fun, []]}) when is_function(fun, 0), do: {fun, []} - defp get_running({mod, fun, args}), do: {:erlang.make_fun(mod, fun, length(args)), args} + defp unlink_and_kill(pid) do + caller = self() + ref = make_ref() + + enforcer = + spawn(fn -> + mon = Process.monitor(caller) + + receive do + {:done, ^ref} -> :ok + {:DOWN, ^mon, _, _, _} -> Process.exit(pid, :kill) + end + end) + + Process.unlink(pid) + Process.exit(pid, :kill) + send(enforcer, {:done, ref}) + end + + defp normalize_mfa_with_arg({mod, fun, args}, arg), do: {mod, fun, [arg | args]} + defp normalize_mfa_with_arg(fun, arg), do: {:erlang, :apply, [fun, [arg]]} end diff --git a/lib/elixir/lib/task/supervisor.ex b/lib/elixir/lib/task/supervisor.ex index 8366ba7c3bc..0164e9b8111 100644 --- a/lib/elixir/lib/task/supervisor.ex +++ b/lib/elixir/lib/task/supervisor.ex @@ -1,112 +1,656 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + defmodule Task.Supervisor do @moduledoc """ - A tasks supervisor. + A task supervisor. This module defines a supervisor which can be used to dynamically - supervise tasks. Behind the scenes, this module is implemented as a - `:simple_one_for_one` supervisor where the workers are temporary - (i.e. they are not restarted after they die). + supervise tasks. + + A task supervisor is started with no children, often under a + supervisor and a name: + + children = [ + {Task.Supervisor, name: MyApp.TaskSupervisor} + ] + + Supervisor.start_link(children, strategy: :one_for_one) + + The options given in the child specification are documented in `start_link/1`. + + Once started, you can start tasks directly under the supervisor, for example: + + task = Task.Supervisor.async(MyApp.TaskSupervisor, fn -> + :do_some_work + end) + + See the `Task` module for more examples. + + ## Scalability and partitioning + + The `Task.Supervisor` is a single process responsible for starting + other processes. In some applications, the `Task.Supervisor` may + become a bottleneck. To address this, you can start multiple instances + of the `Task.Supervisor` and then pick a random instance to start + the task on. - The functions in this module allow tasks to be spawned and awaited - from a supervisor, similar to the functions defined in the `Task` module. + Instead of: - ## Name Registration + children = [ + {Task.Supervisor, name: MyApp.TaskSupervisor} + ] + + and: + + Task.Supervisor.async(MyApp.TaskSupervisor, fn -> :do_some_work end) + + You can do this: + + children = [ + {PartitionSupervisor, + child_spec: Task.Supervisor, + name: MyApp.TaskSupervisors} + ] + + and then: + + Task.Supervisor.async( + {:via, PartitionSupervisor, {MyApp.TaskSupervisors, self()}}, + fn -> :do_some_work end + ) + + In the code above, we start a partition supervisor that will by default + start a dynamic supervisor for each core in your machine. Then, instead + of calling the `Task.Supervisor` by name, you call it through the + partition supervisor using the `{:via, PartitionSupervisor, {name, key}}` + format, where `name` is the name of the partition supervisor and `key` + is the routing key. We picked `self()` as the routing key, which means + each process will be assigned one of the existing task supervisors. + Read the `PartitionSupervisor` docs for more information. + + ## Name registration A `Task.Supervisor` is bound to the same name registration rules as a - `GenServer`. Read more about it in the `GenServer` docs. + `GenServer`. Read more about them in the `GenServer` docs. """ + @typedoc "Option values used by `start_link`" + @type option :: + GenServer.option() + | DynamicSupervisor.init_option() + + @typedoc """ + Options given to `async_stream` and `async_stream_nolink` functions. + """ + @typedoc since: "1.17.0" + @type async_stream_option :: Task.async_stream_option() | {:shutdown, Supervisor.shutdown()} + + @typedoc """ + Options for `async/3`, `async/5`, `async_nolink/3`, and `async_nolink/5` functions. + """ + @type async_opts :: [ + shutdown: :brutal_kill | timeout() + ] + + @type start_child_opts :: [ + restart: :temporary | :transient | :permanent, + shutdown: :brutal_kill | timeout() + ] + + @doc false + def child_spec(opts) when is_list(opts) do + id = + case Keyword.get(opts, :name, Task.Supervisor) do + name when is_atom(name) -> name + {:global, name} -> name + {:via, _module, name} -> name + end + + %{ + id: id, + start: {Task.Supervisor, :start_link, [opts]}, + type: :supervisor + } + end + @doc """ Starts a new supervisor. - The supported options are: + ## Examples + + A task supervisor is typically started under a supervision tree using + the tuple format: + + {Task.Supervisor, name: MyApp.TaskSupervisor} + + You can also start it by calling `start_link/1` directly: - * `:name` - used to register a supervisor name, the supported values are - described under the `Name Registration` section in the `GenServer` module - docs; + Task.Supervisor.start_link(name: MyApp.TaskSupervisor) - * `:shutdown` - `:brutal_kill` if the tasks must be killed directly on shutdown - or an integer indicating the timeout value, defaults to 5000 milliseconds; + But this is recommended only for scripting and should be avoided in + production code. Generally speaking, processes should always be started + inside supervision trees. + + ## Options + + * `:name` - used to register a supervisor name, the supported values are + described under the `Name Registration` section in the `GenServer` module + docs; + + * `:max_restarts`, `:max_seconds`, and `:max_children` - as specified in + `DynamicSupervisor`; + + This function could also receive `:restart` and `:shutdown` as options + but those two options have been deprecated and it is now preferred to + give them directly to `start_child`. """ - @spec start_link(Supervisor.options) :: Supervisor.on_start - def start_link(opts \\ []) do - import Supervisor.Spec - {shutdown, opts} = Keyword.pop(opts, :shutdown, 5000) - children = [worker(Task.Supervised, [], restart: :temporary, shutdown: shutdown)] - Supervisor.start_link(children, [strategy: :simple_one_for_one] ++ opts) + @spec start_link([option]) :: Supervisor.on_start() + def start_link(options \\ []) do + {restart, options} = Keyword.pop(options, :restart) + {shutdown, options} = Keyword.pop(options, :shutdown) + + if restart || shutdown do + IO.warn( + ":restart and :shutdown options in Task.Supervisor.start_link/1 " <> + "are deprecated. Please pass those options on start_child/3 instead" + ) + end + + keys = [:max_children, :max_seconds, :max_restarts] + {sup_opts, start_opts} = Keyword.split(options, keys) + restart_and_shutdown = {restart || :temporary, shutdown || 5000} + DynamicSupervisor.start_link(__MODULE__, {restart_and_shutdown, sup_opts}, start_opts) + end + + @doc false + def init({{_restart, _shutdown} = arg, options}) do + Process.put(__MODULE__, arg) + DynamicSupervisor.init([strategy: :one_for_one] ++ options) end @doc """ Starts a task that can be awaited on. - The `supervisor` must be a reference as defined in `Task.Supervisor`. - For more information on tasks, check the `Task` module. + The `supervisor` must be a reference as defined in `Supervisor`. + The task will still be linked to the caller, see `Task.async/1` for + more information and `async_nolink/3` for a non-linked variant. + + Raises an error if `supervisor` has reached the maximum number of + children. + + ## Options + + * `:shutdown` - `:brutal_kill` if the tasks must be killed directly on shutdown + or an integer indicating the timeout value, defaults to 5000 milliseconds. + The tasks must trap exits for the timeout to have an effect. + + """ + @spec async(Supervisor.supervisor(), (-> any), async_opts) :: Task.t() + def async(supervisor, fun, options \\ []) do + async(supervisor, :erlang, :apply, [fun, []], options) + end + + @doc """ + Starts a task that can be awaited on. + + The `supervisor` must be a reference as defined in `Supervisor`. + The task will still be linked to the caller, see `Task.async/1` for + more information and `async_nolink/3` for a non-linked variant. + + Raises an error if `supervisor` has reached the maximum number of + children. + + ## Options + + * `:shutdown` - `:brutal_kill` if the tasks must be killed directly on shutdown + or an integer indicating the timeout value, defaults to 5000 milliseconds. + The tasks must trap exits for the timeout to have an effect. + + """ + @spec async(Supervisor.supervisor(), module, atom, [term], async_opts) :: Task.t() + def async(supervisor, module, fun, args, options \\ []) do + async(supervisor, :link, module, fun, args, options) + end + + @doc """ + Starts a task that can be awaited on. + + The `supervisor` must be a reference as defined in `Supervisor`. + The task won't be linked to the caller, see `Task.async/1` for + more information. + + Raises an error if `supervisor` has reached the maximum number of + children. + + Note this function requires the task supervisor to have `:temporary` + as the `:restart` option (the default), as `async_nolink/3` keeps a + direct reference to the task which is lost if the task is restarted. + + ## Options + + * `:shutdown` - `:brutal_kill` if the tasks must be killed directly on shutdown + or an integer indicating the timeout value, defaults to 5000 milliseconds. + The tasks must trap exits for the timeout to have an effect. + + ## Compatibility with OTP behaviours + + If you create a task using `async_nolink` inside an OTP behaviour + like `GenServer`, you should match on the message coming from the + task inside your `c:GenServer.handle_info/2` callback. + + The reply sent by the task will be in the format `{ref, result}`, + where `ref` is the monitor reference held by the task struct + and `result` is the return value of the task function. + + Keep in mind that, regardless of how the task created with `async_nolink` + terminates, the caller's process will always receive a `:DOWN` message + with the same `ref` value that is held by the task struct. If the task + terminates normally, the reason in the `:DOWN` message will be `:normal`. + + ## Examples + + Typically, you use `async_nolink/3` when there is a reasonable expectation that + the task may fail, and you don't want it to take down the caller. Let's see an + example where a `GenServer` is meant to run a single task and track its status: + + defmodule MyApp.Server do + use GenServer + + # ... + + def start_task do + GenServer.call(__MODULE__, :start_task) + end + + # In this case the task is already running, so we just return :ok. + def handle_call(:start_task, _from, %{ref: ref} = state) when is_reference(ref) do + {:reply, :ok, state} + end + + # The task is not running yet, so let's start it. + def handle_call(:start_task, _from, %{ref: nil} = state) do + task = + Task.Supervisor.async_nolink(MyApp.TaskSupervisor, fn -> + ... + end) + + # We return :ok and the server will continue running + {:reply, :ok, %{state | ref: task.ref}} + end + + # The task completed successfully + def handle_info({ref, answer}, %{ref: ref} = state) do + # We don't care about the DOWN message now, so let's demonitor and flush it + Process.demonitor(ref, [:flush]) + # Do something with the result and then return + {:noreply, %{state | ref: nil}} + end + + # The task failed + def handle_info({:DOWN, ref, :process, _pid, _reason}, %{ref: ref} = state) do + # Log and possibly restart the task... + {:noreply, %{state | ref: nil}} + end + end + """ - @spec async(Supervisor.supervisor, fun) :: Task.t - def async(supervisor, fun) do - async(supervisor, :erlang, :apply, [fun, []]) + @spec async_nolink(Supervisor.supervisor(), (-> any), async_opts) :: Task.t() + def async_nolink(supervisor, fun, options \\ []) do + async_nolink(supervisor, :erlang, :apply, [fun, []], options) end @doc """ Starts a task that can be awaited on. - The `supervisor` must be a reference as defined in `Task.Supervisor`. - For more information on tasks, check the `Task` module. + The `supervisor` must be a reference as defined in `Supervisor`. + The task won't be linked to the caller, see `Task.async/1` for + more information. + + Raises an error if `supervisor` has reached the maximum number of + children. + + Note this function requires the task supervisor to have `:temporary` + as the `:restart` option (the default), as `async_nolink/5` keeps a + direct reference to the task which is lost if the task is restarted. """ - @spec async(Supervisor.supervisor, module, atom, [term]) :: Task.t - def async(supervisor, module, fun, args) do - args = [self, get_info(self), {module, fun, args}] - {:ok, pid} = Supervisor.start_child(supervisor, args) - ref = Process.monitor(pid) - send pid, {self(), ref} - %Task{pid: pid, ref: ref} + @spec async_nolink(Supervisor.supervisor(), module, atom, [term], async_opts) :: Task.t() + def async_nolink(supervisor, module, fun, args, options \\ []) do + async(supervisor, :nolink, module, fun, args, options) + end + + @doc """ + Returns a stream where the given function (`module` and `function`) + is mapped concurrently on each element in `enumerable`. + + Each element will be prepended to the given `args` and processed by its + own task. The tasks will be spawned under the given `supervisor` and + linked to the caller process, similarly to `async/5`. + + When streamed, each task will emit `{:ok, value}` upon successful + completion or `{:exit, reason}` if the caller is trapping exits. + The order of results depends on the value of the `:ordered` option. + + The level of concurrency and the time tasks are allowed to run can + be controlled via options (see the "Options" section below). + + If you find yourself trapping exits to handle exits inside + the async stream, consider using `async_stream_nolink/6` to start tasks + that are not linked to the calling process. + + ## Options + + * `:max_concurrency` - sets the maximum number of tasks to run + at the same time. Defaults to `System.schedulers_online/0`. + + * `:ordered` - whether the results should be returned in the same order + as the input stream. This option is useful when you have large + streams and don't want to buffer results before they are delivered. + This is also useful when you're using the tasks for side effects. + Defaults to `true`. + + * `:timeout` - the maximum amount of time to wait (in milliseconds) + without receiving a task reply (across all running tasks). + Defaults to `5000`. + + * `:on_timeout` - what do to when a task times out. The possible + values are: + * `:exit` (default) - the process that spawned the tasks exits. + * `:kill_task` - the task that timed out is killed. The value + emitted for that task is `{:exit, :timeout}`. + + * `:zip_input_on_exit` - (since v1.14.0) adds the original + input to `:exit` tuples. The value emitted for that task is + `{:exit, {input, reason}}`, where `input` is the collection element + that caused an exited during processing. Defaults to `false`. + + * `:shutdown` - `:brutal_kill` if the tasks must be killed directly on shutdown + or an integer indicating the timeout value. Defaults to `5000` milliseconds. + The tasks must trap exits for the timeout to have an effect. + + ## Examples + + Let's build a stream and then enumerate it: + + stream = Task.Supervisor.async_stream(MySupervisor, collection, Mod, :expensive_fun, []) + Enum.to_list(stream) + + """ + @doc since: "1.4.0" + @spec async_stream( + Supervisor.supervisor(), + Enumerable.t(), + module, + atom, + [term], + [async_stream_option] + ) :: Enumerable.t() + def async_stream(supervisor, enumerable, module, function, args, options \\ []) + when is_atom(module) and is_atom(function) and is_list(args) do + build_stream(supervisor, :link, enumerable, {module, function, args}, options) + end + + @doc """ + Returns a stream that runs the given function `fun` concurrently + on each element in `enumerable`. + + Each element in `enumerable` is passed as argument to the given function `fun` + and processed by its own task. The tasks will be spawned under the given + `supervisor` and linked to the caller process, similarly to `async/3`. + + See `async_stream/6` for discussion, options, and examples. + """ + @doc since: "1.4.0" + @spec async_stream( + Supervisor.supervisor(), + Enumerable.t(), + (term -> term), + [async_stream_option] + ) :: Enumerable.t() + def async_stream(supervisor, enumerable, fun, options \\ []) when is_function(fun, 1) do + build_stream(supervisor, :link, enumerable, fun, options) + end + + @doc """ + Returns a stream where the given function (`module` and `function`) + is mapped concurrently on each element in `enumerable`. + + Each element in `enumerable` will be prepended to the given `args` and processed + by its own task. The tasks will be spawned under the given `supervisor` and + will not be linked to the caller process, similarly to `async_nolink/5`. + + See `async_stream/6` for discussion, options, and examples. + """ + @doc since: "1.4.0" + @spec async_stream_nolink( + Supervisor.supervisor(), + Enumerable.t(), + module, + atom, + [term], + [async_stream_option] + ) :: Enumerable.t() + def async_stream_nolink(supervisor, enumerable, module, function, args, options \\ []) + when is_atom(module) and is_atom(function) and is_list(args) do + build_stream(supervisor, :nolink, enumerable, {module, function, args}, options) + end + + @doc ~S""" + Returns a stream that runs the given `function` concurrently on each + element in `enumerable`. + + Each element in `enumerable` is passed as argument to the given function `fun` + and processed by its own task. The tasks will be spawned under the given + `supervisor` and will not be linked to the caller process, similarly + to `async_nolink/3`. + + See `async_stream/6` for discussion and examples. + + ## Error handling and cleanup + + Even if tasks are not linked to the caller, there is no risk of leaving dangling tasks + running after the stream halts. + + Consider the following example: + + Task.Supervisor.async_stream_nolink(MySupervisor, collection, fun, on_timeout: :kill_task, ordered: false) + |> Enum.each(fn + {:ok, _} -> :ok + {:exit, reason} -> raise "Task exited: #{Exception.format_exit(reason)}" + end) + + If one task raises or times out: + + 1. the second clause gets called + 2. an exception is raised + 3. the stream halts + 4. all ongoing tasks will be shut down + + Here is another example: + + Task.Supervisor.async_stream_nolink(MySupervisor, collection, fun, on_timeout: :kill_task, ordered: false) + |> Stream.filter(&match?({:ok, _}, &1)) + |> Enum.take(3) + + This will return the three first tasks to succeed, ignoring timeouts and errors, and shut down + every ongoing task. + + Just running the stream with `Stream.run/1` on the other hand would ignore errors and process the whole stream. + + """ + @doc since: "1.4.0" + @spec async_stream_nolink( + Supervisor.supervisor(), + Enumerable.t(), + (term -> term), + [async_stream_option] + ) :: Enumerable.t() + def async_stream_nolink(supervisor, enumerable, fun, options \\ []) when is_function(fun, 1) do + build_stream(supervisor, :nolink, enumerable, fun, options) end @doc """ Terminates the child with the given `pid`. """ - @spec terminate_child(Supervisor.supervisor, pid) :: :ok + @spec terminate_child(Supervisor.supervisor(), pid) :: :ok | {:error, :not_found} def terminate_child(supervisor, pid) when is_pid(pid) do - :supervisor.terminate_child(supervisor, pid) + DynamicSupervisor.terminate_child(supervisor, pid) end @doc """ - Returns all children pids. + Returns all children PIDs except those that are restarting. + + Note that calling this function when supervising a large number + of children under low memory conditions can bring the system down due to an + out of memory error. """ - @spec children(Supervisor.supervisor) :: [pid] + @spec children(Supervisor.supervisor()) :: [pid] def children(supervisor) do - :supervisor.which_children(supervisor) |> Enum.map(&elem(&1, 1)) + for {_, pid, _, _} <- DynamicSupervisor.which_children(supervisor), is_pid(pid), do: pid end @doc """ - Starts a task as child of the given `supervisor`. + Starts a task as a child of the given `supervisor`. + + Task.Supervisor.start_child(MyTaskSupervisor, fn -> + IO.puts("I am running in a task") + end) Note that the spawned process is not linked to the caller, but only to the supervisor. This command is useful in case the - task needs to perform side-effects (like I/O) and does not need - to report back to the caller. + task needs to perform side-effects (like I/O) and you have no + interest in its results nor if it completes successfully. + + ## Options + + * `:restart` - the restart strategy, may be `:temporary` (the default), + `:transient` or `:permanent`. `:temporary` means the task is never + restarted, `:transient` means it is restarted if the exit is not + `:normal`, `:shutdown` or `{:shutdown, reason}`. A `:permanent` restart + strategy means it is always restarted. + + * `:shutdown` - `:brutal_kill` if the task must be killed directly on shutdown + or an integer indicating the timeout value, defaults to 5000 milliseconds. + The task must trap exits for the timeout to have an effect. + """ - @spec start_child(Supervisor.supervisor, fun) :: {:ok, pid} - def start_child(supervisor, fun) do - start_child(supervisor, :erlang, :apply, [fun, []]) + @spec start_child(Supervisor.supervisor(), (-> any), start_child_opts) :: + DynamicSupervisor.on_start_child() + def start_child(supervisor, fun, options \\ []) do + restart = options[:restart] + shutdown = options[:shutdown] + args = [get_owner(self()), get_callers(self()), {:erlang, :apply, [fun, []]}] + start_child_with_spec(supervisor, args, restart, shutdown) end @doc """ - Starts a task as child of the given `supervisor`. + Starts a task as a child of the given `supervisor`. - Similar to `start_child/2` except the task is specified + Similar to `start_child/3` except the task is specified by the given `module`, `fun` and `args`. """ - @spec start_child(Supervisor.supervisor, module, atom, [term]) :: {:ok, pid} - def start_child(supervisor, module, fun, args) do - Supervisor.start_child(supervisor, [get_info(self), {module, fun, args}]) + @spec start_child(Supervisor.supervisor(), module, atom, [term], start_child_opts) :: + DynamicSupervisor.on_start_child() + def start_child(supervisor, module, fun, args, options \\ []) + when is_atom(fun) and is_list(args) do + restart = options[:restart] + shutdown = options[:shutdown] + mfa = {module, fun, args} + owner = get_owner(self()) + callers = get_callers(self()) + + if restart == :temporary or restart == nil do + start_child_maybe_temporary(supervisor, owner, callers, restart, shutdown, mfa) + else + start_child_with_spec(supervisor, [owner, callers, mfa], restart, shutdown) + end + end + + defp start_child_maybe_temporary(supervisor, owner, callers, restart, shutdown, mfa) do + case start_child_with_spec(supervisor, [owner, :monitor], restart, shutdown) do + # TODO: This only exists because we need to support reading restart/shutdown + # from two different places. Remove this, the init function and the associated + # clause in DynamicSupervisor on Elixir v2.0 + {:restart, restart} -> + start_child_with_spec(supervisor, [owner, callers, mfa], restart, shutdown) + + {:ok, pid} -> + # We mimic async but there is nothing to reply to + alias = make_ref() + send(pid, {self(), alias, alias, callers, mfa}) + {:ok, pid} + + {:error, _} = error -> + error + end + end + + defp start_child_with_spec(supervisor, args, restart, shutdown) do + GenServer.call(supervisor, {:start_task, args, restart, shutdown}, :infinity) + end + + defp get_owner(pid) do + self_or_name = + case Process.info(pid, :registered_name) do + {:registered_name, name} when is_atom(name) -> name + _ -> pid + end + + {node(), self_or_name, pid} end - defp get_info(self) do - {node(), - case Process.info(self, :registered_name) do - {:registered_name, []} -> self() - {:registered_name, name} -> name - end} + defp get_callers(owner) do + case :erlang.get(:"$callers") do + [_ | _] = list -> [owner | list] + _ -> [owner] + end + end + + defp async(supervisor, link_type, module, fun, args, options) do + owner = self() + shutdown = options[:shutdown] + + case start_child_with_spec(supervisor, [get_owner(owner), :monitor], :temporary, shutdown) do + {:ok, pid} -> + if link_type == :link, do: Process.link(pid) + alias = Task.__alias__(pid) + send(pid, {owner, alias, alias, get_callers(owner), {module, fun, args}}) + %Task{pid: pid, ref: alias, owner: owner, mfa: {module, fun, length(args)}} + + {:error, :max_children} -> + raise """ + reached the maximum number of tasks for this task supervisor. The maximum number \ + of tasks that are allowed to run at the same time under this supervisor can be \ + configured with the :max_children option passed to Task.Supervisor.start_link/1\ + """ + end + end + + defp build_stream(supervisor, link_type, enumerable, fun, options) do + shutdown = Keyword.get(options, :shutdown, 5000) + + if not ((is_integer(shutdown) and shutdown >= 0) or shutdown == :brutal_kill) do + raise ArgumentError, ":shutdown must be either a positive integer or :brutal_kill" + end + + options = Task.Supervised.validate_stream_options(options) + + fn acc, acc_fun -> + owner = get_owner(self()) + + Task.Supervised.stream(enumerable, acc, acc_fun, get_callers(self()), fun, options, fn -> + args = [owner, :monitor] + + case start_child_with_spec(supervisor, args, :temporary, shutdown) do + {:ok, pid} -> + if link_type == :link, do: Process.link(pid) + {:ok, link_type, pid} + + {:error, :max_children} -> + {:error, :max_children} + end + end) + end end end diff --git a/lib/elixir/lib/tuple.ex b/lib/elixir/lib/tuple.ex index b20256aa547..f2da15d6c40 100644 --- a/lib/elixir/lib/tuple.ex +++ b/lib/elixir/lib/tuple.ex @@ -1,12 +1,58 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + defmodule Tuple do @moduledoc """ Functions for working with tuples. + + Please note the following functions for tuples are found in `Kernel`: + + * `elem/2` - accesses a tuple by index + * `put_elem/3` - inserts a value into a tuple by index + * `tuple_size/1` - gets the number of elements in a tuple + + Tuples are intended as fixed-size containers for multiple elements. + To manipulate a collection of elements, use a list instead. `Enum` + functions do not work on tuples. + + Tuples are denoted with curly braces: + + iex> {} + {} + iex> {1, :two, "three"} + {1, :two, "three"} + + A tuple may contain elements of different types, which are stored + contiguously in memory. Accessing any element takes constant time, + but modifying a tuple, which produces a shallow copy, takes linear time. + Tuples are good for reading data while lists are better for traversals. + + Tuples are typically used either when a function has multiple return values + or for error handling. `File.read/1` returns `{:ok, contents}` if reading + the given file is successful, or else `{:error, reason}` such as when + the file does not exist. + + The functions in this module that add and remove elements from tuples are + rarely used in practice, as they typically imply tuples are being used as + collections. To append to a tuple, it is preferable to extract the elements + from the old tuple with pattern matching, and then create a new tuple: + + tuple = {:ok, :example} + + # Avoid + result = Tuple.insert_at(tuple, 2, %{}) + + # Prefer + {:ok, atom} = tuple + result = {:ok, atom, %{}} + """ @doc """ Creates a new tuple. - Creates a tuple of size `size` containing the + Creates a tuple of `size` containing the given `data` at every position. Inlined by the compiler. @@ -18,16 +64,16 @@ defmodule Tuple do """ @spec duplicate(term, non_neg_integer) :: tuple - def duplicate(data, size) do + def duplicate(data, size) when is_integer(size) and size >= 0 do :erlang.make_tuple(size, data) end @doc """ Inserts an element into a tuple. - Inserts `value` into `tuple` at the given zero-based `index`. - Raises an `ArgumentError` if `index` is greater than the - length of `tuple`. + Inserts `value` into `tuple` at the given `index`. + Raises an `ArgumentError` if `index` is negative or greater than the + length of `tuple`. Index is zero-based. Inlined by the compiler. @@ -36,19 +82,27 @@ defmodule Tuple do iex> tuple = {:bar, :baz} iex> Tuple.insert_at(tuple, 0, :foo) {:foo, :bar, :baz} + iex> Tuple.insert_at(tuple, 2, :bong) + {:bar, :baz, :bong} """ @spec insert_at(tuple, non_neg_integer, term) :: tuple - def insert_at(tuple, index, term) do - :erlang.insert_element(index + 1, tuple, term) + def insert_at(tuple, index, value) when is_integer(index) and index >= 0 do + :erlang.insert_element(index + 1, tuple, value) + end + + @doc false + @deprecated "Use insert_at instead" + def append(tuple, value) do + :erlang.append_element(tuple, value) end @doc """ Removes an element from a tuple. - Deletes the element at the zero-based `index` from `tuple`. - Raises an `ArgumentError` if `index` is greater than - or equal to the length of `tuple`. + Deletes the element at the given `index` from `tuple`. + Raises an `ArgumentError` if `index` is negative or greater than + or equal to the length of `tuple`. Index is zero-based. Inlined by the compiler. @@ -60,14 +114,61 @@ defmodule Tuple do """ @spec delete_at(tuple, non_neg_integer) :: tuple - def delete_at(tuple, index) do + def delete_at(tuple, index) when is_integer(index) and index >= 0 do :erlang.delete_element(index + 1, tuple) end + @doc """ + Computes a sum of tuple elements. + + ## Examples + + iex> Tuple.sum({255, 255}) + 510 + iex> Tuple.sum({255, 0.0}) + 255.0 + iex> Tuple.sum({}) + 0 + """ + @doc since: "1.12.0" + @spec sum(tuple) :: number() + def sum(tuple), do: sum(tuple, tuple_size(tuple)) + + defp sum(_tuple, 0), do: 0 + defp sum(tuple, index), do: :erlang.element(index, tuple) + sum(tuple, index - 1) + + @doc """ + Computes a product of tuple elements. + + ## Examples + + iex> Tuple.product({255, 255}) + 65025 + iex> Tuple.product({255, 1.0}) + 255.0 + iex> Tuple.product({}) + 1 + """ + @doc since: "1.12.0" + @spec product(tuple) :: number() + def product(tuple), do: product(tuple, tuple_size(tuple)) + + defp product(_tuple, 0), do: 1 + defp product(tuple, index), do: :erlang.element(index, tuple) * product(tuple, index - 1) + @doc """ Converts a tuple to a list. + Returns a new list with all the tuple elements. + Inlined by the compiler. + + ## Examples + + iex> tuple = {:foo, :bar, :baz} + iex> Tuple.to_list(tuple) + [:foo, :bar, :baz] + """ @spec to_list(tuple) :: list def to_list(tuple) do diff --git a/lib/elixir/lib/uri.ex b/lib/elixir/lib/uri.ex index 9b918874d63..3561cc68892 100644 --- a/lib/elixir/lib/uri.ex +++ b/lib/elixir/lib/uri.ex @@ -1,39 +1,77 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + defmodule URI do @moduledoc """ - Utilities for working with and creating URIs. + Utilities for working with URIs. + + This module provides functions for working with URIs (for example, parsing + URIs or encoding query strings). The functions in this module are implemented + according to [RFC 3986](https://tools.ietf.org/html/rfc3986) and it also + provides additional functionality for handling "application/x-www-form-urlencoded" + segments. + + Additionally, the Erlang [`:uri_string` module](`:uri_string`) provides additional + functionality such as RFC 3986 compliant URI normalization. """ - defstruct scheme: nil, path: nil, query: nil, - fragment: nil, authority: nil, - userinfo: nil, host: nil, port: nil + @doc """ + The URI struct. - import Bitwise + The fields are defined to match the following URI representation + (with field names between brackets): - @ports %{ - "ftp" => 21, - "http" => 80, - "https" => 443, - "ldap" => 389, - "sftp" => 22, - "tftp" => 69, - } + [scheme]://[userinfo]@[host]:[port][path]?[query]#[fragment] - Enum.each @ports, fn {scheme, port} -> - def normalize_scheme(unquote(scheme)), do: unquote(scheme) - def default_port(unquote(scheme)), do: unquote(port) - end - @doc """ - Normalizes the scheme according to the spec by downcasing it. + Note the `authority` field is deprecated. `parse/1` will still + populate it for backwards compatibility but you should generally + avoid setting or getting it. """ - def normalize_scheme(nil), do: nil - def normalize_scheme(scheme), do: String.downcase(scheme) + @derive {Inspect, optional: [:authority]} + defstruct [:scheme, :authority, :userinfo, :host, :port, :path, :query, :fragment] + + @type t :: %__MODULE__{ + scheme: nil | binary, + authority: authority, + userinfo: nil | binary, + host: nil | binary, + port: nil | :inet.port_number(), + path: nil | binary, + query: nil | binary, + fragment: nil | binary + } + + @typedoc deprecated: "The authority field is deprecated" + @opaque authority :: nil | binary + + defmodule Error do + @moduledoc """ + An exception raised when an error occurs when a `URI` is invalid. + + For example, see `URI.new!/1`. + """ + + defexception [:action, :reason, :part] + + @doc false + def message(%Error{action: action, reason: reason, part: part}) do + "cannot #{action} due to reason #{reason}: #{inspect(part)}" + end + end + + import Bitwise + + @reserved_characters ~c":/?#[]@!$&'()*+,;=" + @formatted_reserved_characters Enum.map_join(@reserved_characters, ", ", &<>) @doc """ - Returns the default port for a given scheme. + Returns the default port for a given `scheme`. - If the scheme is unknown to URI, returns `nil`. - Any scheme may be registered via `default_port/2`. + If the scheme is unknown to the `URI` module, this function returns + `nil`. The default port for any scheme can be configured globally + via `default_port/2`. ## Examples @@ -44,48 +82,111 @@ defmodule URI do nil """ + @spec default_port(binary) :: nil | non_neg_integer def default_port(scheme) when is_binary(scheme) do - {:ok, dict} = Application.fetch_env(:elixir, :uri) - Map.get(dict, scheme) + :elixir_config.get({:uri, scheme}, nil) end @doc """ - Registers a scheme with a default port. + Registers the default `port` for the given `scheme`. + + After this function is called, `port` will be returned by + `default_port/1` for the given scheme `scheme`. Note that this function + changes the default port for the given `scheme` *globally*, meaning for + every application. It is recommended for this function to be invoked in your - application start callback in case you want to register + application's start callback in case you want to register new URIs. """ - def default_port(scheme, port) when is_binary(scheme) and port > 0 do - {:ok, dict} = Application.fetch_env(:elixir, :uri) - Application.put_env(:elixir, :uri, Map.put(dict, scheme, port), persistent: true) + @spec default_port(binary, non_neg_integer) :: :ok + def default_port(scheme, port) when is_binary(scheme) and is_integer(port) and port >= 0 do + :elixir_config.put({:uri, scheme}, port) end @doc """ - Encodes an enumerable into a query string. + Encodes `enumerable` into a query string using `encoding`. - Takes an enumerable (containing a sequence of two-item tuples) - and returns a string of the form "key1=value1&key2=value2..." where - keys and values are URL encoded as per `encode/1`. + Takes an enumerable that enumerates as a list of two-element + tuples (for instance, a map or a keyword list) and returns a string + in the form of `key1=value1&key2=value2...`. Keys and values can be any term that implements the `String.Chars` - protocol, except lists which are explicitly forbidden. + protocol with the exception of lists, which are explicitly forbidden. + + You can specify one of the following `encoding` strategies: + + * `:www_form` - (default, since v1.12.0) keys and values are URL encoded as + per `encode_www_form/1`. This is the format typically used by browsers on + query strings and form data. It encodes " " as "+". + + * `:rfc3986` - (since v1.12.0) the same as `:www_form` except it encodes + " " as "%20" according [RFC 3986](https://tools.ietf.org/html/rfc3986). + This is the best option if you are encoding in a non-browser situation, + since encoding spaces as "+" can be ambiguous to URI parsers. This can + inadvertently lead to spaces being interpreted as literal plus signs. + + Encoding defaults to `:www_form` for backward compatibility. ## Examples - iex> hd = %{"foo" => 1, "bar" => 2} - iex> URI.encode_query(hd) + iex> query = %{"foo" => 1, "bar" => 2} + iex> URI.encode_query(query) "bar=2&foo=1" + iex> query = %{"key" => "value with spaces"} + iex> URI.encode_query(query) + "key=value+with+spaces" + + iex> query = %{"key" => "value with spaces"} + iex> URI.encode_query(query, :rfc3986) + "key=value%20with%20spaces" + + iex> URI.encode_query(%{key: [:a, :list]}) + ** (ArgumentError) encode_query/2 values cannot be lists, got: [:a, :list] + """ - def encode_query(l), do: Enum.map_join(l, "&", &pair/1) + @spec encode_query(Enumerable.t(), :rfc3986 | :www_form) :: binary + def encode_query(enumerable, encoding \\ :www_form) do + Enum.map_join(enumerable, "&", &encode_kv_pair(&1, encoding)) + end + + defp encode_kv_pair({key, _}, _encoding) when is_list(key) do + raise ArgumentError, "encode_query/2 keys cannot be lists, got: #{inspect(key)}" + end + + defp encode_kv_pair({_, value}, _encoding) when is_list(value) do + raise ArgumentError, "encode_query/2 values cannot be lists, got: #{inspect(value)}" + end + + defp encode_kv_pair({key, value}, :rfc3986) do + encode(Kernel.to_string(key), &char_unreserved?/1) <> + "=" <> encode(Kernel.to_string(value), &char_unreserved?/1) + end + + defp encode_kv_pair({key, value}, :www_form) do + encode_www_form(Kernel.to_string(key)) <> "=" <> encode_www_form(Kernel.to_string(value)) + end @doc """ - Decodes a query string into a dictionary (by default uses a map). + Decodes `query` into a map. + + Given a query string in the form of `key1=value1&key2=value2...`, this + function inserts each key-value pair in the query string as one entry in the + given `map`. Keys and values in the resulting map will be binaries. Keys and + values will be percent-unescaped. + + You can specify one of the following `encoding` options: + + * `:www_form` - (default, since v1.12.0) keys and values are decoded as per + `decode_www_form/1`. This is the format typically used by browsers on + query strings and form data. It decodes "+" as " ". - Given a query string of the form "key1=value1&key2=value2...", produces a - map with one entry for each key-value pair. Each key and value will be a - binary. Keys and values will be percent-unescaped. + * `:rfc3986` - (since v1.12.0) keys and values are decoded as per + `decode/1`. The result is the same as `:www_form` except for leaving "+" + as is in line with [RFC 3986](https://tools.ietf.org/html/rfc3986). + + Encoding defaults to `:www_form` for backward compatibility. Use `query_decoder/1` if you want to iterate over each value manually. @@ -94,107 +195,227 @@ defmodule URI do iex> URI.decode_query("foo=1&bar=2") %{"bar" => "2", "foo" => "1"} + iex> URI.decode_query("percent=oh+yes%21", %{"starting" => "map"}) + %{"percent" => "oh yes!", "starting" => "map"} + + iex> URI.decode_query("percent=oh+yes%21", %{}, :rfc3986) + %{"percent" => "oh+yes!"} + """ - def decode_query(q, dict \\ %{}) when is_binary(q) do - Enum.reduce query_decoder(q), dict, fn({k, v}, acc) -> Dict.put(acc, k, v) end + @spec decode_query(binary, %{optional(binary) => binary}, :rfc3986 | :www_form) :: %{ + optional(binary) => binary + } + def decode_query(query, map \\ %{}, encoding \\ :www_form) + + def decode_query(query, %_{} = dict, encoding) when is_binary(query) do + IO.warn( + "URI.decode_query/3 expects the second argument to be a map, other usage is deprecated" + ) + + decode_query_into_dict(query, dict, encoding) + end + + def decode_query(query, map, encoding) when is_binary(query) and is_map(map) do + decode_query_into_map(query, map, encoding) + end + + def decode_query(query, dict, encoding) when is_binary(query) do + IO.warn( + "URI.decode_query/3 expects the second argument to be a map, other usage is deprecated" + ) + + decode_query_into_dict(query, dict, encoding) + end + + defp decode_query_into_map(query, map, encoding) do + case decode_next_query_pair(query, encoding) do + nil -> + map + + {{key, value}, rest} -> + decode_query_into_map(rest, Map.put(map, key, value), encoding) + end + end + + defp decode_query_into_dict(query, dict, encoding) do + case decode_next_query_pair(query, encoding) do + nil -> + dict + + {{key, value}, rest} -> + # Avoid warnings about Dict being deprecated + dict_module = String.to_atom("Dict") + decode_query_into_dict(rest, dict_module.put(dict, key, value), encoding) + end end @doc """ - Returns an iterator function over the query string that decodes - the query string in steps. + Returns a stream of two-element tuples representing key-value pairs in the + given `query`. + + Key and value in each tuple will be binaries and will be percent-unescaped. + + You can specify one of the following `encoding` options: + + * `:www_form` - (default, since v1.12.0) keys and values are decoded as per + `decode_www_form/1`. This is the format typically used by browsers on + query strings and form data. It decodes "+" as " ". + + * `:rfc3986` - (since v1.12.0) keys and values are decoded as per + `decode/1`. The result is the same as `:www_form` except for leaving "+" + as is in line with [RFC 3986](https://tools.ietf.org/html/rfc3986). + + Encoding defaults to `:www_form` for backward compatibility. ## Examples - iex> URI.query_decoder("foo=1&bar=2") |> Enum.map &(&1) + iex> URI.query_decoder("foo=1&bar=2") |> Enum.to_list() [{"foo", "1"}, {"bar", "2"}] + iex> URI.query_decoder("food=bread%26butter&drinks=tap%20water+please") |> Enum.to_list() + [{"food", "bread&butter"}, {"drinks", "tap water please"}] + + iex> URI.query_decoder("food=bread%26butter&drinks=tap%20water+please", :rfc3986) |> Enum.to_list() + [{"food", "bread&butter"}, {"drinks", "tap water+please"}] + """ - def query_decoder(q) when is_binary(q) do - Stream.unfold(q, &do_decoder/1) + @spec query_decoder(binary, :rfc3986 | :www_form) :: Enumerable.t() + def query_decoder(query, encoding \\ :www_form) when is_binary(query) do + Stream.unfold(query, &decode_next_query_pair(&1, encoding)) end - defp do_decoder("") do + defp decode_next_query_pair("", _encoding) do nil end - defp do_decoder(q) do - {first, next} = - case :binary.split(q, "&") do - [first, rest] -> {first, rest} - [first] -> {first, ""} + defp decode_next_query_pair(query, encoding) do + {undecoded_next_pair, rest} = + case :binary.split(query, "&") do + [next_pair, rest] -> {next_pair, rest} + [next_pair] -> {next_pair, ""} end - current = - case :binary.split(first, "=") do + next_pair = + case :binary.split(undecoded_next_pair, "=") do [key, value] -> - {decode_www_form(key), decode_www_form(value)} + {decode_with_encoding(key, encoding), decode_with_encoding(value, encoding)} + [key] -> - {decode_www_form(key), nil} + {decode_with_encoding(key, encoding), ""} end - {current, next} + {next_pair, rest} end - defp pair({k, _}) when is_list(k) do - raise ArgumentError, "encode_query/1 keys cannot be lists, got: #{inspect k}" + defp decode_with_encoding(string, :www_form) do + decode_www_form(string) end - defp pair({_, v}) when is_list(v) do - raise ArgumentError, "encode_query/1 values cannot be lists, got: #{inspect v}" + defp decode_with_encoding(string, :rfc3986) do + decode(string) end - defp pair({k, v}) do - encode_www_form(to_string(k)) <> - "=" <> encode_www_form(to_string(v)) - end + @doc ~s""" + Checks if `character` is a reserved one in a URI. - @doc """ - Checks if the character is a "reserved" character in a URI. + As specified in [RFC 3986, section 2.2](https://tools.ietf.org/html/rfc3986#section-2.2), + the following characters are reserved: #{@formatted_reserved_characters} + + ## Examples + + iex> URI.char_reserved?(?+) + true - Reserved characters are specified in RFC3986, section 2.2. """ - def char_reserved?(c) do - c in ':/?#[]@!$&\'()*+,;=' + @spec char_reserved?(byte) :: boolean + def char_reserved?(character) do + character in @reserved_characters end @doc """ - Checks if the character is a "unreserved" character in a URI. + Checks if `character` is an unreserved one in a URI. + + As specified in [RFC 3986, section 2.3](https://tools.ietf.org/html/rfc3986#section-2.3), + the following characters are unreserved: + + * Alphanumeric characters: `A-Z`, `a-z`, `0-9` + * `~`, `_`, `-`, `.` + + ## Examples + + iex> URI.char_unreserved?(?_) + true - Unreserved characters are specified in RFC3986, section 2.3. """ - def char_unreserved?(c) do - c in ?0..?9 or - c in ?a..?z or - c in ?A..?Z or - c in '~_-.' + @spec char_unreserved?(byte) :: boolean + def char_unreserved?(character) do + character in ?0..?9 or character in ?a..?z or character in ?A..?Z or character in ~c"~_-." end @doc """ - Checks if the character is allowed unescaped in a URI. + Checks if `character` is allowed unescaped in a URI. This is the default used by `URI.encode/2` where both - reserved and unreserved characters are kept unescaped. + [reserved](`char_reserved?/1`) and [unreserved characters](`char_unreserved?/1`) + are kept unescaped. + + ## Examples + + iex> URI.char_unescaped?(?{) + false + """ - def char_unescaped?(c) do - char_reserved?(c) or char_unreserved?(c) + @spec char_unescaped?(byte) :: boolean + def char_unescaped?(character) do + char_reserved?(character) or char_unreserved?(character) end @doc """ - Percent-escape a URI. - Accepts `predicate` function as an argument to specify if char can be left as is. + Percent-encodes all characters that require escaping in `string`. - ## Example + The optional `predicate` argument specifies a function used to detect whether + a byte in the `string` should be escaped: + + * if the function returns a truthy value, the byte should be kept as-is. + * if the function returns a falsy value, the byte should be escaped. + + The `predicate` argument can use some built-in functions: + + * `URI.char_unescaped?/1` (default) - reserved characters (such as `:` + and `/`) or unreserved (such as letters and numbers) are kept as-is. + It's typically used to encode the whole URI. + * `URI.char_unreserved?/1` - unreserved characters (such as letters and + numbers) are kept as-is. It's typically used to encode components in + a URI, such as query or fragment. + * `URI.char_reserved?/1` - Reserved characters (such as `:` and `/`) are + kept as-is. + + And, you can also use custom functions. + + See `encode_www_form/1` if you are interested in encoding `string` as + "x-www-form-urlencoded". + + ## Examples iex> URI.encode("ftp://s-ite.tld/?value=put it+й") "ftp://s-ite.tld/?value=put%20it+%D0%B9" + iex> URI.encode("a string", &(&1 != ?i)) + "a str%69ng" + """ - def encode(str, predicate \\ &char_unescaped?/1) when is_binary(str) do - for <>, into: "", do: percent(c, predicate) + @spec encode(binary, (byte -> as_boolean(term))) :: binary + def encode(string, predicate \\ &char_unescaped?/1) + when is_binary(string) and is_function(predicate, 1) do + for <>, into: "", do: percent(byte, predicate) end @doc """ - Encode a string as "x-www-urlencoded". + Encodes `string` as "x-www-form-urlencoded". + + Note "x-www-form-urlencoded" is not specified as part of + RFC 3986. However, it is a commonly used format to encode + query strings and form data by browsers. ## Example @@ -202,44 +423,47 @@ defmodule URI do "put%3A+it%2B%D0%B9" """ - def encode_www_form(str) when is_binary(str) do - for <>, into: "" do - case percent(c, &char_unreserved?/1) do + @spec encode_www_form(binary) :: binary + def encode_www_form(string) when is_binary(string) do + for <>, into: "" do + case percent(byte, &char_unreserved?/1) do "%20" -> "+" - pct -> pct + percent -> percent end end end - defp percent(c, predicate) do - if predicate.(c) do - <> + defp percent(char, predicate) do + if predicate.(char) do + <> else - "%" <> hex(bsr(c, 4)) <> hex(band(c, 15)) + <<"%", hex(bsr(char, 4)), hex(band(char, 15))>> end end - defp hex(n) when n <= 9, do: <> - defp hex(n), do: <> + defp hex(n) when n <= 9, do: n + ?0 + defp hex(n), do: n + ?A - 10 @doc """ - Percent-unescape a URI. + Percent-unescapes a URI. ## Examples - iex> URI.decode("http%3A%2F%2Felixir-lang.org") - "http://elixir-lang.org" + iex> URI.decode("https%3A%2F%2Felixir-lang.org") + "https://elixir-lang.org" """ + @spec decode(binary) :: binary def decode(uri) do - unpercent(uri) - catch - :malformed_uri -> - raise ArgumentError, "malformed URI #{inspect uri}" + unpercent(uri, "", false) end @doc """ - Decode a string as "x-www-urlencoded". + Decodes `string` as "x-www-form-urlencoded". + + Note "x-www-form-urlencoded" is not specified as part of + RFC 3986. However, it is a commonly used format to encode + query strings and form data by browsers. ## Examples @@ -247,116 +471,623 @@ defmodule URI do " Enum.map_join(" ", &unpercent/1) - catch - :malformed_uri -> - raise ArgumentError, "malformed URI #{inspect str}" + @spec decode_www_form(binary) :: binary + def decode_www_form(string) when is_binary(string) do + unpercent(string, "", true) + end + + defp unpercent(<>, acc, spaces = true) do + unpercent(tail, <>, spaces) end - defp unpercent(<>) do - <> <> unpercent(tail) + defp unpercent(<>, acc, spaces) do + with <> <- tail, + dec1 when is_integer(dec1) <- hex_to_dec(hex1), + dec2 when is_integer(dec2) <- hex_to_dec(hex2) do + unpercent(tail, <>, spaces) + else + _ -> unpercent(tail, <>, spaces) + end end - defp unpercent(<>), do: throw(:malformed_uri) - defp unpercent(<>), do: throw(:malformed_uri) - defp unpercent(<>) do - <> <> unpercent(tail) + defp unpercent(<>, acc, spaces) do + unpercent(tail, <>, spaces) end - defp unpercent(<<>>), do: <<>> + defp unpercent(<<>>, acc, _spaces), do: acc + @compile {:inline, hex_to_dec: 1} defp hex_to_dec(n) when n in ?A..?F, do: n - ?A + 10 defp hex_to_dec(n) when n in ?a..?f, do: n - ?a + 10 defp hex_to_dec(n) when n in ?0..?9, do: n - ?0 - defp hex_to_dec(_n), do: throw(:malformed_uri) + defp hex_to_dec(_n), do: nil @doc """ - Parses a URI into components. + Creates a new URI struct by parsing and validating a string or from an existing URI. - URIs have portions that are handled specially for the particular - scheme of the URI. For example, http and https have different - default ports. Such values can be accessed and registered via - `URI.default_port/1` and `URI.default_port/2`. + If a `%URI{}` struct is given, it returns `{:ok, uri}` as is. If a string is + given, it will parse and validate it. If the string is valid, it returns + `{:ok, uri}`, otherwise it returns `{:error, part}` with the invalid part + of the URI. For parsing URIs without further validation, see `parse/1`. - ## Examples + This function can parse both absolute and relative URLs. You can check + if a URI is absolute or relative by checking if the `scheme` field is + `nil` or not. + + When a URI is given without a port, the value returned by `URI.default_port/1` + for the URI's scheme is used for the `:port` field. The scheme is also + normalized to lowercase. - iex> URI.parse("http://elixir-lang.org/") - %URI{scheme: "http", path: "/", query: nil, fragment: nil, - authority: "elixir-lang.org", userinfo: nil, - host: "elixir-lang.org", port: 80} + ## Examples + iex> URI.new("https://elixir-lang.org/") + {:ok, %URI{ + fragment: nil, + host: "elixir-lang.org", + path: "/", + port: 443, + query: nil, + scheme: "https", + userinfo: nil + }} + + iex> URI.new("//elixir-lang.org/") + {:ok, %URI{ + fragment: nil, + host: "elixir-lang.org", + path: "/", + port: nil, + query: nil, + scheme: nil, + userinfo: nil + }} + + iex> URI.new("/foo/bar") + {:ok, %URI{ + fragment: nil, + host: nil, + path: "/foo/bar", + port: nil, + query: nil, + scheme: nil, + userinfo: nil + }} + + iex> URI.new("foo/bar") + {:ok, %URI{ + fragment: nil, + host: nil, + path: "foo/bar", + port: nil, + query: nil, + scheme: nil, + userinfo: nil + }} + + iex> URI.new("//[fe80::]/") + {:ok, %URI{ + fragment: nil, + host: "fe80::", + path: "/", + port: nil, + query: nil, + scheme: nil, + userinfo: nil + }} + + iex> URI.new("https:?query") + {:ok, %URI{ + fragment: nil, + host: nil, + path: nil, + port: 443, + query: "query", + scheme: "https", + userinfo: nil + }} + + iex> URI.new("/invalid_greater_than_in_path/>") + {:error, ">"} + + Giving an existing URI simply returns it wrapped in a tuple: + + iex> {:ok, uri} = URI.new("https://elixir-lang.org/") + iex> URI.new(uri) + {:ok, %URI{ + fragment: nil, + host: "elixir-lang.org", + path: "/", + port: 443, + query: nil, + scheme: "https", + userinfo: nil + }} """ - def parse(%URI{} = uri), do: uri + @doc since: "1.13.0" + @spec new(t() | String.t()) :: {:ok, t()} | {:error, String.t()} + def new(%URI{} = uri), do: {:ok, uri} + + def new(binary) when is_binary(binary) do + case :uri_string.parse(binary) do + %{} = map -> {:ok, uri_from_map(map)} + {:error, :invalid_uri, term} -> {:error, Kernel.to_string(term)} + end + end - def parse(s) when is_binary(s) do - # From http://tools.ietf.org/html/rfc3986#appendix-B - regex = ~r/^(([^:\/?#]+):)?(\/\/([^\/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?/ - parts = nillify(Regex.run(regex, s)) + @doc """ + Similar to `new/1` but raises `URI.Error` if an invalid string is given. + + ## Examples - destructure [_, _, scheme, _, authority, path, _, query, _, fragment], parts - {userinfo, host, port} = split_authority(authority) + iex> URI.new!("https://elixir-lang.org/") + %URI{ + fragment: nil, + host: "elixir-lang.org", + path: "/", + port: 443, + query: nil, + scheme: "https", + userinfo: nil + } + + iex> URI.new!("/invalid_greater_than_in_path/>") + ** (URI.Error) cannot parse due to reason invalid_uri: ">" + + Giving an existing URI simply returns it: + + iex> uri = URI.new!("https://elixir-lang.org/") + iex> URI.new!(uri) + %URI{ + fragment: nil, + host: "elixir-lang.org", + path: "/", + port: 443, + query: nil, + scheme: "https", + userinfo: nil + } + """ + @doc since: "1.13.0" + @spec new!(t() | String.t()) :: t() + def new!(%URI{} = uri), do: uri - if authority do - authority = "" + def new!(binary) when is_binary(binary) do + case :uri_string.parse(binary) do + %{} = map -> + uri_from_map(map) - if userinfo, do: authority = authority <> userinfo <> "@" - if host, do: authority = authority <> host - if port, do: authority = authority <> ":" <> Integer.to_string(port) + {:error, reason, part} -> + raise Error, action: :parse, reason: reason, part: Kernel.to_string(part) end + end + + defp uri_from_map(%{path: ""} = map), do: uri_from_map(%{map | path: nil}) - scheme = normalize_scheme(scheme) + defp uri_from_map(map) do + uri = Map.merge(%URI{}, map) - if nil?(port) and not nil?(scheme) do - port = default_port(scheme) + case map do + %{scheme: scheme} -> + scheme = String.downcase(scheme, :ascii) + + case map do + %{port: port} when is_integer(port) -> + %{uri | scheme: scheme} + + %{} -> + %{uri | scheme: scheme, port: default_port(scheme)} + end + + %{port: :undefined} -> + %{uri | port: nil} + + %{} -> + uri end + end + + @doc """ + Parses a URI into its components, without further validation. + + This function can parse both absolute and relative URLs. You can check + if a URI is absolute or relative by checking if the `scheme` field is + nil or not. Furthermore, this function expects both absolute and + relative URIs to be well-formed and does not perform any validation. + See the "Examples" section below. Use `new/1` if you want to validate + the URI fields after parsing. + + When a URI is given without a port, the value returned by `URI.default_port/1` + for the URI's scheme is used for the `:port` field. The scheme is also + normalized to lowercase. + + If a `%URI{}` struct is given to this function, this function returns it + unmodified. + + > #### `:authority` field {: .info} + > + > This function sets the deprecated field `:authority` for backwards-compatibility reasons. + + ## Examples + + iex> URI.parse("https://elixir-lang.org/") + %URI{ + authority: "elixir-lang.org", + fragment: nil, + host: "elixir-lang.org", + path: "/", + port: 443, + query: nil, + scheme: "https", + userinfo: nil + } + + iex> URI.parse("//elixir-lang.org/") + %URI{ + authority: "elixir-lang.org", + fragment: nil, + host: "elixir-lang.org", + path: "/", + port: nil, + query: nil, + scheme: nil, + userinfo: nil + } + + iex> URI.parse("/foo/bar") + %URI{ + fragment: nil, + host: nil, + path: "/foo/bar", + port: nil, + query: nil, + scheme: nil, + userinfo: nil + } + + iex> URI.parse("foo/bar") + %URI{ + fragment: nil, + host: nil, + path: "foo/bar", + port: nil, + query: nil, + scheme: nil, + userinfo: nil + } + + In contrast to `URI.new/1`, this function will parse poorly-formed + URIs, for example: + + iex> URI.parse("/invalid_greater_than_in_path/>") + %URI{ + fragment: nil, + host: nil, + path: "/invalid_greater_than_in_path/>", + port: nil, + query: nil, + scheme: nil, + userinfo: nil + } + + Another example is a URI with brackets in query strings. It is accepted + by `parse/1`, it is commonly accepted by browsers, but it will be refused + by `new/1`: + + iex> URI.parse("/?foo[bar]=baz") + %URI{ + fragment: nil, + host: nil, + path: "/", + port: nil, + query: "foo[bar]=baz", + scheme: nil, + userinfo: nil + } + + """ + @spec parse(t | binary) :: t + def parse(%URI{} = uri), do: uri + + def parse(string) when is_binary(string) do + # From https://tools.ietf.org/html/rfc3986#appendix-B + # Parts: 12 3 4 5 6 7 8 9 + regex = ~r{^(([a-z][a-z0-9\+\-\.]*):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?}i + + parts = Regex.run(regex, string) + + destructure [ + _full, + # 1 + _scheme_with_colon, + # 2 + scheme, + # 3 + authority_with_slashes, + # 4 + _authority, + # 5 + path, + # 6 + query_with_question_mark, + # 7 + _query, + # 8 + _fragment_with_hash, + # 9 + fragment + ], + parts + + path = nilify(path) + scheme = nilify(scheme) + query = nilify_query(query_with_question_mark) + {authority, userinfo, host, port} = split_authority(authority_with_slashes) + + scheme = scheme && String.downcase(scheme) + port = port || (scheme && default_port(scheme)) %URI{ - scheme: scheme, path: path, query: query, - fragment: fragment, authority: authority, - userinfo: userinfo, host: host, port: port + scheme: scheme, + path: path, + query: query, + fragment: fragment, + authority: authority, + userinfo: userinfo, + host: host, + port: port } end + defp nilify_query("?" <> query), do: query + defp nilify_query(_other), do: nil + # Split an authority into its userinfo, host and port parts. - defp split_authority(s) do - s = s || "" - components = Regex.run ~r/(^(.*)@)?(\[[a-zA-Z0-9:.]*\]|[^:]*)(:(\d*))?/, s + # + # Note that the host field is returned *without* [] even if, according to + # RFC3986 grammar, a native IPv6 address requires them. + defp split_authority("") do + {nil, nil, nil, nil} + end + + defp split_authority("//") do + {"", nil, "", nil} + end - destructure [_, _, userinfo, host, _, port], nillify(components) - port = if port, do: String.to_integer(port) - host = if host, do: host |> String.lstrip(?[) |> String.rstrip(?]) + defp split_authority("//" <> authority) do + regex = ~r/(^(.*)@)?(\[[a-zA-Z0-9:.]*\]|[^:]*)(:(\d*))?/ + components = Regex.run(regex, authority) - {userinfo, host, port} + destructure [_, _, userinfo, host, _, port], components + userinfo = nilify(userinfo) + host = if nilify(host), do: host |> String.trim_leading("[") |> String.trim_trailing("]") + port = if nilify(port), do: String.to_integer(port) + + {authority, userinfo, host, port} end # Regex.run returns empty strings sometimes. We want # to replace those with nil for consistency. - defp nillify(l) do - for s <- l do - if byte_size(s) > 0, do: s, else: nil + defp nilify(""), do: nil + defp nilify(other), do: other + + @doc """ + Returns the string representation of the given [URI struct](`t:t/0`). + + ## Examples + + iex> uri = URI.parse("http://google.com") + iex> URI.to_string(uri) + "http://google.com" + + iex> uri = URI.parse("foo://bar.baz") + iex> URI.to_string(uri) + "foo://bar.baz" + + """ + @spec to_string(t) :: binary + defdelegate to_string(uri), to: String.Chars.URI + + @doc ~S""" + Merges two URIs. + + This function merges two URIs as per + [RFC 3986, section 5.2](https://tools.ietf.org/html/rfc3986#section-5.2). + + ## Examples + + iex> URI.merge(URI.parse("http://google.com"), "/query") |> to_string() + "http://google.com/query" + + iex> URI.merge("http://example.com", "http://google.com") |> to_string() + "http://google.com" + + """ + @spec merge(t | binary, t | binary) :: t + def merge(uri, rel) + + def merge(%URI{scheme: nil}, _rel) do + raise ArgumentError, "you must merge onto an absolute URI" + end + + def merge(_base, %URI{scheme: rel_scheme} = rel) when rel_scheme != nil do + %{rel | path: remove_dot_segments_from_path(rel.path)} + end + + def merge(%URI{} = base, %URI{host: host} = rel) when host != nil do + %{rel | scheme: base.scheme, path: remove_dot_segments_from_path(rel.path)} + end + + def merge(%URI{} = base, %URI{path: nil} = rel) do + %{base | query: rel.query || base.query, fragment: rel.fragment} + end + + def merge(%URI{host: nil, path: nil} = base, %URI{} = rel) do + %{ + base + | path: remove_dot_segments_from_path(rel.path), + query: rel.query, + fragment: rel.fragment + } + end + + def merge(%URI{} = base, %URI{} = rel) do + new_path = merge_paths(base.path, rel.path) + %{base | path: new_path, query: rel.query, fragment: rel.fragment} + end + + def merge(base, rel) do + merge(parse(base), parse(rel)) + end + + defp merge_paths(nil, rel_path), do: merge_paths("/", rel_path) + defp merge_paths(_, "/" <> _ = rel_path), do: remove_dot_segments_from_path(rel_path) + + defp merge_paths(base_path, rel_path) do + (path_to_segments(base_path) ++ [:+] ++ path_to_segments(rel_path)) + |> remove_dot_segments([]) + |> join_reversed_segments() + end + + defp remove_dot_segments_from_path(nil), do: nil + + defp remove_dot_segments_from_path(path) do + path_to_segments(path) + |> remove_dot_segments([]) + |> join_reversed_segments() + end + + defp path_to_segments(path) do + case String.split(path, "/") do + ["" | tail] -> [:/ | tail] + segments -> segments end end + + defp remove_dot_segments([], acc), do: acc + defp remove_dot_segments([:/ | tail], acc), do: remove_dot_segments(tail, [:/ | acc]) + defp remove_dot_segments([_, :+ | tail], acc), do: remove_dot_segments(tail, acc) + defp remove_dot_segments(["."], acc), do: remove_dot_segments([], ["" | acc]) + defp remove_dot_segments(["." | tail], acc), do: remove_dot_segments(tail, acc) + defp remove_dot_segments([".." | tail], [:/]), do: remove_dot_segments(tail, [:/]) + defp remove_dot_segments([".."], [_ | acc]), do: remove_dot_segments([], ["" | acc]) + defp remove_dot_segments([".." | tail], [_ | acc]), do: remove_dot_segments(tail, acc) + defp remove_dot_segments([head | tail], acc), do: remove_dot_segments(tail, [head | acc]) + + defp join_reversed_segments([:/]), do: "/" + + defp join_reversed_segments(segments) do + case Enum.reverse(segments) do + [:/ | tail] -> ["" | tail] + list -> list + end + |> Enum.join("/") + end + + @doc """ + Appends `query` to the given `uri`. + + The given `query` is not automatically encoded, use `encode/2` or `encode_www_form/1`. + + ## Examples + + iex> URI.append_query(URI.parse("http://example.com/"), "x=1") |> URI.to_string() + "http://example.com/?x=1" + + iex> URI.append_query(URI.parse("http://example.com/?x=1"), "y=2") |> URI.to_string() + "http://example.com/?x=1&y=2" + + iex> URI.append_query(URI.parse("http://example.com/?x=1"), "x=2") |> URI.to_string() + "http://example.com/?x=1&x=2" + """ + @doc since: "1.14.0" + @spec append_query(t(), binary()) :: t() + def append_query(%URI{} = uri, query) when is_binary(query) and uri.query in [nil, ""] do + %{uri | query: query} + end + + def append_query(%URI{} = uri, query) when is_binary(query) do + if String.ends_with?(uri.query, "&") do + %{uri | query: uri.query <> query} + else + %{uri | query: uri.query <> "&" <> query} + end + end + + @doc """ + Appends `path` to the given `uri`. + + Path must start with `/` and cannot contain additional URL components like + fragments or query strings. This function further assumes the path is valid and + it does not contain a query string or fragment parts. + + ## Examples + + iex> URI.append_path(URI.parse("http://example.com/foo/?x=1"), "/my-path") |> URI.to_string() + "http://example.com/foo/my-path?x=1" + + iex> URI.append_path(URI.parse("http://example.com"), "my-path") + ** (ArgumentError) path must start with "/", got: "my-path" + + """ + @doc since: "1.15.0" + @spec append_path(t(), String.t()) :: t() + def append_path(%URI{}, "//" <> _ = path) do + raise ArgumentError, ~s|path cannot start with "//", got: #{inspect(path)}| + end + + def append_path(%URI{path: path} = uri, "/" <> rest = all) do + cond do + path == nil -> %{uri | path: all} + path != "" and :binary.last(path) == ?/ -> %{uri | path: path <> rest} + true -> %{uri | path: path <> all} + end + end + + def append_path(%URI{}, path) when is_binary(path) do + raise ArgumentError, ~s|path must start with "/", got: #{inspect(path)}| + end end defimpl String.Chars, for: URI do - def to_string(uri) do - scheme = uri.scheme + def to_string(%{host: host, path: path} = uri) + when host != nil and is_binary(path) and + path != "" and binary_part(path, 0, 1) != "/" do + raise ArgumentError, + ":path in URI must be empty or an absolute path if URL has a :host, got: #{inspect(uri)}" + end - if scheme && (port = URI.default_port(scheme)) do - if uri.port == port, do: uri = %{uri | port: nil} - end + def to_string(%{scheme: scheme, port: port, path: path, query: query, fragment: fragment} = uri) do + uri = + case scheme && URI.default_port(scheme) do + ^port -> %{uri | port: nil} + _ -> uri + end - result = "" + # Based on https://tools.ietf.org/html/rfc3986#section-5.3 + authority = extract_authority(uri) - if uri.scheme, do: result = result <> uri.scheme <> "://" - if uri.userinfo, do: result = result <> uri.userinfo <> "@" - if uri.host, do: result = result <> uri.host - if uri.port, do: result = result <> ":" <> Integer.to_string(uri.port) - if uri.path, do: result = result <> uri.path - if uri.query, do: result = result <> "?" <> uri.query - if uri.fragment, do: result = result <> "#" <> uri.fragment + IO.iodata_to_binary([ + if(scheme, do: [scheme, ?:], else: []), + if(authority, do: ["//" | authority], else: []), + if(path, do: path, else: []), + if(query, do: ["?" | query], else: []), + if(fragment, do: ["#" | fragment], else: []) + ]) + end + + defp extract_authority(%{host: nil, authority: authority}) do + authority + end - result + defp extract_authority(%{host: host, userinfo: userinfo, port: port}) do + # According to the grammar at + # https://tools.ietf.org/html/rfc3986#appendix-A, a "host" can have a colon + # in it only if it's an IPv6 or "IPvFuture" address, so if there's a colon + # in the host we can safely surround it with []. + [ + if(userinfo, do: [userinfo | "@"], else: []), + if(String.contains?(host, ":"), do: ["[", host | "]"], else: host), + if(port, do: [":" | Integer.to_string(port)], else: []) + ] end end diff --git a/lib/elixir/lib/version.ex b/lib/elixir/lib/version.ex index 19289de4c8f..2749aed0225 100644 --- a/lib/elixir/lib/version.ex +++ b/lib/elixir/lib/version.ex @@ -1,3 +1,7 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + defmodule Version do @moduledoc ~S""" Functions for parsing and matching versions against requirements. @@ -5,35 +9,34 @@ defmodule Version do A version is a string in a specific format or a `Version` generated after parsing via `Version.parse/1`. - `Version` parsing and requirements follow - [SemVer 2.0 schema](http://semver.org/). + Although Elixir projects are not required to follow SemVer, + they must follow the format outlined on [SemVer 2.0 schema](https://semver.org/). ## Versions - In a nutshell, a version is given by three numbers: + In a nutshell, a version is represented by three numbers: MAJOR.MINOR.PATCH - Pre-releases are supported by appending `-[0-9A-Za-z-\.]`: + Pre-releases are supported by optionally appending a hyphen and a series of + period-separated identifiers immediately following the patch version. + Identifiers consist of only ASCII alphanumeric characters and hyphens (`[0-9A-Za-z-]`): "1.0.0-alpha.3" - Build information can be added by appending `+[0-9A-Za-z-\.]`: - - "1.0.0-alpha.3+20130417140000" - - ## Struct + Build information can be added by appending a plus sign and a series of + dot-separated identifiers immediately following the patch or pre-release version. + Identifiers consist of only ASCII alphanumeric characters and hyphens (`[0-9A-Za-z-]`): - The version is represented by the Version struct and it has its - fields named according to Semver: `:major`, `:minor`, `:patch`, - `:pre` and `:build`. + "1.0.0-alpha.3+20130417140000.amd64" ## Requirements Requirements allow you to specify which versions of a given - dependency you are willing to work against. It supports common - operators like `>=`, `<=`, `>`, `==` and friends that - work as one would expect: + dependency you are willing to work against. Requirements support the common + comparison operators such as `>`, `>=`, `<`, `<=`, and `==` that work as one + would expect, and additionally the special operator `~>` described in detail + further below. # Only version 2.0.0 "== 2.0.0" @@ -41,6 +44,11 @@ defmodule Version do # Anything later than 2.0.0 "> 2.0.0" + You can skip the operator, which is equivalent to `==`: + + # Only version 2.0.0 + "2.0.0" + Requirements also support `and` and `or` for complex conditions: # 2.0.0 and later until 2.1.0 @@ -51,99 +59,313 @@ defmodule Version do "~> 2.0.0" + `~>` will never include pre-release versions of its upper bound, + regardless of the usage of the `:allow_pre` option, or whether the operand + is a pre-release version. It can also be used to set an upper bound on only the major + version part. See the table below for `~>` requirements and + their corresponding translations. + + `~>` | Translation + :------------- | :--------------------- + `~> 2.0.0` | `>= 2.0.0 and < 2.1.0` + `~> 2.1.2` | `>= 2.1.2 and < 2.2.0` + `~> 2.1.3-dev` | `>= 2.1.3-dev and < 2.2.0` + `~> 2.0` | `>= 2.0.0 and < 3.0.0` + `~> 2.1` | `>= 2.1.0 and < 3.0.0` + + The requirement operand after the `~>` is allowed to omit the patch version, + allowing us to express `~> 2.1` or `~> 2.1-dev`, something that wouldn't be allowed + when using the common comparison operators. + + When the `:allow_pre` option is set `false` in `Version.match?/3`, the requirement + will not match a pre-release version unless the operand is a pre-release version. + The default is to always allow pre-releases but note that in + Hex `:allow_pre` is set to `false`. See the table below for examples. + + Requirement | Version | `:allow_pre` | Matches + :------------- | :---------- | :---------------- | :------ + `~> 2.0` | `2.1.0` | `true` or `false` | `true` + `~> 2.0` | `3.0.0` | `true` or `false` | `false` + `~> 2.0.0` | `2.0.5` | `true` or `false` | `true` + `~> 2.0.0` | `2.1.0` | `true` or `false` | `false` + `~> 2.1.2` | `2.1.6-dev` | `true` | `true` + `~> 2.1.2` | `2.1.6-dev` | `false` | `false` + `~> 2.1-dev` | `2.2.0-dev` | `true` or `false` | `true` + `~> 2.1.2-dev` | `2.1.6-dev` | `true` or `false` | `true` + `>= 2.1.0` | `2.2.0-dev` | `true` | `true` + `>= 2.1.0` | `2.2.0-dev` | `false` | `false` + `>= 2.1.0-dev` | `2.2.6-dev` | `true` or `false` | `true` + """ import Kernel, except: [match?: 2] - defstruct [:major, :minor, :patch, :pre, :build] - @type version :: String.t | t - @type requirement :: String.t | Version.Requirement.t - @type matchable :: {major :: String.t | non_neg_integer, - minor :: non_neg_integer | nil, - patch :: non_neg_integer | nil, - pre :: [String.t]} + @doc """ + The Version struct. + + It contains the fields `:major`, `:minor`, `:patch`, `:pre`, and + `:build` according to SemVer 2.0, where `:pre` is a list. + + You can read those fields but you should not create a new `Version` + directly via the struct syntax. Instead use the functions in this + module. + """ + @enforce_keys [:major, :minor, :patch] + @derive {Inspect, optional: [:pre, :build]} + defstruct [:major, :minor, :patch, pre: [], build: nil] + + @type version :: String.t() | t + @type requirement :: String.t() | Version.Requirement.t() + @type major :: non_neg_integer + @type minor :: non_neg_integer + @type patch :: non_neg_integer + @type pre :: [String.t() | non_neg_integer] + @type build :: String.t() | nil + @type t :: %__MODULE__{major: major, minor: minor, patch: patch, pre: pre, build: build} + + @type match_opts :: [allow_pre: boolean()] defmodule Requirement do - @moduledoc false - defstruct [:source, :matchspec] + @moduledoc """ + A struct that holds version requirement information. + + The struct fields are private and should not be accessed. + + See the "Requirements" section in the `Version` module + for more information. + """ + + defstruct [:source, :lexed] + + @opaque t :: %__MODULE__{ + source: String.t(), + lexed: [atom | matchable] + } + + @typep matchable :: + {Version.major(), Version.minor(), Version.patch(), Version.pre(), Version.build()} + + @compile inline: [compare: 2] + + @doc false + @spec new(String.t(), [atom | matchable]) :: t + def new(source, lexed) do + %__MODULE__{source: source, lexed: lexed} + end + + @doc false + @spec compile_requirement(t) :: t + def compile_requirement(%Requirement{} = requirement) do + requirement + end + + @doc false + @spec match?(t, tuple) :: boolean + def match?(%__MODULE__{lexed: [operator, req | rest]}, version) do + match_lexed?(rest, version, match_op?(operator, req, version)) + end + + defp match_lexed?([:and, operator, req | rest], version, acc), + do: match_lexed?(rest, version, acc and match_op?(operator, req, version)) + + defp match_lexed?([:or, operator, req | rest], version, acc), + do: acc or match_lexed?(rest, version, match_op?(operator, req, version)) + + defp match_lexed?([], _version, acc), + do: acc + + defp match_op?(:==, req, version) do + compare(version, req) == :eq + end + + defp match_op?(:!=, req, version) do + compare(version, req) != :eq + end + + defp match_op?(:~>, {major, minor, nil, req_pre, _}, {_, _, _, pre, allow_pre} = version) do + compare(version, {major, minor, 0, req_pre, nil}) in [:eq, :gt] and + compare(version, {major + 1, 0, 0, [0], nil}) == :lt and + (allow_pre or req_pre != [] or pre == []) + end + + defp match_op?(:~>, {major, minor, _, req_pre, _} = req, {_, _, _, pre, allow_pre} = version) do + compare(version, req) in [:eq, :gt] and + compare(version, {major, minor + 1, 0, [0], nil}) == :lt and + (allow_pre or req_pre != [] or pre == []) + end + + defp match_op?(:>, {_, _, _, req_pre, _} = req, {_, _, _, pre, allow_pre} = version) do + compare(version, req) == :gt and (allow_pre or req_pre != [] or pre == []) + end + + defp match_op?(:>=, {_, _, _, req_pre, _} = req, {_, _, _, pre, allow_pre} = version) do + compare(version, req) in [:eq, :gt] and (allow_pre or req_pre != [] or pre == []) + end + + defp match_op?(:<, req, version) do + compare(version, req) == :lt + end + + defp match_op?(:<=, req, version) do + compare(version, req) in [:eq, :lt] + end + + defp compare({major1, minor1, patch1, pre1, _}, {major2, minor2, patch2, pre2, _}) do + cond do + major1 > major2 -> :gt + major1 < major2 -> :lt + minor1 > minor2 -> :gt + minor1 < minor2 -> :lt + patch1 > patch2 -> :gt + patch1 < patch2 -> :lt + pre1 == [] and pre2 != [] -> :gt + pre1 != [] and pre2 == [] -> :lt + pre1 > pre2 -> :gt + pre1 < pre2 -> :lt + true -> :eq + end + end end defmodule InvalidRequirementError do - defexception [:message] + @moduledoc """ + An exception raised when a version requirement is invalid. + + For example, see `Version.parse_requirement!/1`. + """ + + defexception [:requirement] + + @impl true + def exception(requirement) when is_binary(requirement) do + %__MODULE__{requirement: requirement} + end + + @impl true + def message(%{requirement: requirement}) do + "invalid requirement: #{inspect(requirement)}" + end end defmodule InvalidVersionError do - defexception [:message] + @moduledoc """ + An exception raised when a version is invalid. + + For example, see `Version.parse!/1`. + """ + + defexception [:version] + + @impl true + def exception(version) when is_binary(version) do + %__MODULE__{version: version} + end + + @impl true + def message(%{version: version}) do + "invalid version: #{inspect(version)}" + end end @doc """ - Check if the given version matches the specification. + Checks if the given version matches the specification. Returns `true` if `version` satisfies `requirement`, `false` otherwise. Raises a `Version.InvalidRequirementError` exception if `requirement` is not - parseable, or `Version.InvalidVersionError` if `version` is not parseable. + parsable, or a `Version.InvalidVersionError` exception if `version` is not parsable. If given an already parsed version and requirement this function won't raise. + ## Options + + * `:allow_pre` (boolean) - when `false`, pre-release versions will not match + unless the operand is a pre-release version. Defaults to `true`. + For examples, please refer to the table above under the "Requirements" section. + ## Examples - iex> Version.match?("2.0.0", ">1.0.0") + iex> Version.match?("2.0.0", "> 1.0.0") + true + + iex> Version.match?("2.0.0", "== 1.0.0") + false + + iex> Version.match?("2.1.6-dev", "~> 2.1.2") true - iex> Version.match?("2.0.0", "==1.0.0") + iex> Version.match?("2.1.6-dev", "~> 2.1.2", allow_pre: false) false - iex> Version.match?("foo", "==1.0.0") - ** (Version.InvalidVersionError) foo + iex> Version.match?("foo", "== 1.0.0") + ** (Version.InvalidVersionError) invalid version: "foo" - iex> Version.match?("2.0.0", "== ==1.0.0") - ** (Version.InvalidRequirementError) == ==1.0.0 + iex> Version.match?("2.0.0", "== == 1.0.0") + ** (Version.InvalidRequirementError) invalid requirement: "== == 1.0.0" """ - @spec match?(version, requirement) :: boolean - def match?(vsn, req) when is_binary(req) do - case parse_requirement(req) do - {:ok, req} -> - match?(vsn, req) - :error -> - raise InvalidRequirementError, message: req - end + @spec match?(version, requirement, match_opts) :: boolean + def match?(version, requirement, opts \\ []) + + def match?(version, requirement, opts) when is_binary(requirement) do + match?(version, parse_requirement!(requirement), opts) end - def match?(version, %Requirement{matchspec: spec}) do - {:ok, result} = :ets.test_ms(to_matchable(version), spec) - result != false + def match?(version, requirement, opts) do + allow_pre = Keyword.get(opts, :allow_pre, true) + matchable_pattern = to_matchable(version, allow_pre) + + Requirement.match?(requirement, matchable_pattern) end @doc """ - Compares two versions. Returns `:gt` if first version is greater than - the second and `:lt` for vice versa. If the two versions are equal `:eq` - is returned + Compares two versions. + + Returns `:gt` if the first version is greater than the second one, and `:lt` + for vice versa. If the two versions are equal, `:eq` is returned. - Raises a `Version.InvalidVersionError` exception if `version` is not parseable. - If given an already parsed version this function won't raise. + Pre-releases are strictly less than their corresponding release versions. + + Patch segments are compared lexicographically if they are alphanumeric, and + numerically otherwise. + + Build segments are ignored: if two versions differ only in their build segment + they are considered to be equal. + + Raises a `Version.InvalidVersionError` exception if any of the two given + versions are not parsable. If given an already parsed version this function + won't raise. ## Examples iex> Version.compare("2.0.1-alpha1", "2.0.0") :gt + iex> Version.compare("1.0.0-beta", "1.0.0-rc1") + :lt + + iex> Version.compare("1.0.0-10", "1.0.0-2") + :gt + iex> Version.compare("2.0.1+build0", "2.0.1") :eq iex> Version.compare("invalid", "2.0.1") - ** (Version.InvalidVersionError) invalid + ** (Version.InvalidVersionError) invalid version: "invalid" """ @spec compare(version, version) :: :gt | :eq | :lt - def compare(vsn1, vsn2) do - do_compare(to_matchable(vsn1), to_matchable(vsn2)) + def compare(version1, version2) do + do_compare(to_matchable(version1, true), to_matchable(version2, true)) end - defp do_compare({major1, minor1, patch1, pre1}, {major2, minor2, patch2, pre2}) do + defp do_compare({major1, minor1, patch1, pre1, _}, {major2, minor2, patch2, pre2, _}) do cond do - {major1, minor1, patch1} > {major2, minor2, patch2} -> :gt - {major1, minor1, patch1} < {major2, minor2, patch2} -> :lt + major1 > major2 -> :gt + major1 < major2 -> :lt + minor1 > minor2 -> :gt + minor1 < minor2 -> :lt + patch1 > patch2 -> :gt + patch1 < patch2 -> :lt pre1 == [] and pre2 != [] -> :gt pre1 != [] and pre2 == [] -> :lt pre1 > pre2 -> :gt @@ -153,355 +375,318 @@ defmodule Version do end @doc """ - Parse a version string into a `Version`. + Parses a version string into a `Version` struct. ## Examples - iex> Version.parse("2.0.1-alpha1") |> elem(1) - #Version<2.0.1-alpha1> + iex> Version.parse("2.0.1-alpha1") + {:ok, %Version{major: 2, minor: 0, patch: 1, pre: ["alpha1"]}} iex> Version.parse("2.0-alpha1") :error """ - @spec parse(String.t) :: {:ok, t} | :error + @spec parse(String.t()) :: {:ok, t} | :error def parse(string) when is_binary(string) do case Version.Parser.parse_version(string) do - {:ok, {major, minor, patch, pre}} -> - vsn = %Version{major: major, minor: minor, patch: patch, - pre: pre, build: get_build(string)} - {:ok, vsn} - :error -> - :error + {:ok, {major, minor, patch, pre, build_parts}} -> + build = if build_parts == [], do: nil, else: Enum.join(build_parts, ".") + version = %Version{major: major, minor: minor, patch: patch, pre: pre, build: build} + {:ok, version} + + :error -> + :error + end + end + + @doc """ + Parses a version string into a `Version`. + + If `string` is an invalid version, a `Version.InvalidVersionError` is raised. + + ## Examples + + iex> Version.parse!("2.0.1-alpha1") + %Version{major: 2, minor: 0, patch: 1, pre: ["alpha1"]} + + iex> Version.parse!("2.0-alpha1") + ** (Version.InvalidVersionError) invalid version: "2.0-alpha1" + + """ + @spec parse!(String.t()) :: t + def parse!(string) when is_binary(string) do + case parse(string) do + {:ok, version} -> version + :error -> raise InvalidVersionError, string end end @doc """ - Parse a version requirement string into a `Version.Requirement`. + Parses a version requirement string into a `Version.Requirement` struct. ## Examples - iex> Version.parse_requirement("== 2.0.1") |> elem(1) - #Version.Requirement<== 2.0.1> + iex> {:ok, requirement} = Version.parse_requirement("== 2.0.1") + iex> requirement + Version.parse_requirement!("== 2.0.1") iex> Version.parse_requirement("== == 2.0.1") :error """ - @spec parse_requirement(String.t) :: {:ok, Requirement.t} | :error + @spec parse_requirement(String.t()) :: {:ok, Requirement.t()} | :error def parse_requirement(string) when is_binary(string) do case Version.Parser.parse_requirement(string) do - {:ok, spec} -> - {:ok, %Requirement{source: string, matchspec: spec}} - :error -> - :error + {:ok, lexed} -> {:ok, Requirement.new(string, lexed)} + :error -> :error end end - defp to_matchable(%Version{major: major, minor: minor, patch: patch, pre: pre}) do - {major, minor, patch, pre} - end + @doc """ + Parses a version requirement string into a `Version.Requirement` struct. - defp to_matchable(string) do - case Version.Parser.parse_version(string) do - {:ok, vsn} -> vsn - :error -> raise InvalidVersionError, message: string - end - end + If `string` is an invalid requirement, a `Version.InvalidRequirementError` is raised. - defp get_build(string) do - case Regex.run(~r/\+([^\s]+)$/, string) do - nil -> - nil + ## Examples - [_, build] -> - build - end - end + iex> Version.parse_requirement!("== 2.0.1") + Version.parse_requirement!("== 2.0.1") - defmodule Parser.DSL do - @moduledoc false + iex> Version.parse_requirement!("== == 2.0.1") + ** (Version.InvalidRequirementError) invalid requirement: "== == 2.0.1" - defmacro deflexer(match, do: body) when is_binary(match) do - quote do - def lexer(unquote(match) <> rest, acc) do - lexer(rest, [unquote(body) | acc]) - end - end + """ + @doc since: "1.8.0" + @spec parse_requirement!(String.t()) :: Requirement.t() + def parse_requirement!(string) when is_binary(string) do + case parse_requirement(string) do + {:ok, requirement} -> requirement + :error -> raise InvalidRequirementError, string end + end - defmacro deflexer(acc, do: body) do - quote do - def lexer("", unquote(acc)) do - unquote(body) - end - end - end + @doc """ + Compiles a requirement to an internal representation that may optimize matching. - defmacro deflexer(char, acc, do: body) do - quote do - def lexer(<< unquote(char) :: utf8, rest :: binary >>, unquote(acc)) do - unquote(char) = << unquote(char) :: utf8 >> + The internal representation is opaque. + """ + @spec compile_requirement(Requirement.t()) :: Requirement.t() + defdelegate compile_requirement(requirement), to: Requirement - lexer(rest, unquote(body)) - end - end - end + defp to_matchable(%Version{major: major, minor: minor, patch: patch, pre: pre}, allow_pre?) do + {major, minor, patch, pre, allow_pre?} end - defmodule Parser do - @moduledoc false - import Parser.DSL - - deflexer ">=", do: :'>=' - deflexer "<=", do: :'<=' - deflexer "~>", do: :'~>' - deflexer ">", do: :'>' - deflexer "<", do: :'<' - deflexer "==", do: :'==' - deflexer "!=", do: :'!=' - deflexer "!", do: :'!=' - deflexer " or ", do: :'||' - deflexer " and ", do: :'&&' - deflexer " ", do: :' ' + defp to_matchable(string, allow_pre?) do + case Version.Parser.parse_version(string) do + {:ok, {major, minor, patch, pre, _build_parts}} -> + {major, minor, patch, pre, allow_pre?} - deflexer x, [] do - [x, :'=='] + :error -> + raise InvalidVersionError, string end + end - deflexer x, [h | acc] do - cond do - is_binary h -> - [h <> x | acc] + @doc """ + Converts the given version to a string. - h in [:'||', :'&&'] -> - [x, :'==', h | acc] + ## Examples - true -> - [x, h | acc] - end - end + iex> Version.to_string(%Version{major: 1, minor: 2, patch: 3}) + "1.2.3" + iex> Version.to_string(Version.parse!("1.14.0-rc.0+build0")) + "1.14.0-rc.0+build0" + """ + @doc since: "1.14.0" + @spec to_string(Version.t()) :: String.t() + def to_string(%Version{} = version) do + pre = pre_to_string(version.pre) + build = if build = version.build, do: "+#{build}" + "#{version.major}.#{version.minor}.#{version.patch}#{pre}#{build}" + end - deflexer acc do - Enum.filter(Enum.reverse(acc), &(&1 != :' ')) - end + defp pre_to_string([]) do + "" + end - @version_regex ~r/^ - (\d+) # major - (?:\.(\d+))? # minor - (?:\.(\d+))? # patch - (?:\-([\d\w\.\-]+))? # pre - (?:\+([\d\w\-]+))? # build - $/x + defp pre_to_string(pre) do + "-" <> + Enum.map_join(pre, ".", fn + int when is_integer(int) -> Integer.to_string(int) + string when is_binary(string) -> string + end) + end - @spec parse_requirement(String.t) :: {:ok, Version.Requirement.t} | :error - def parse_requirement(source) do - lexed = lexer(source, []) - to_matchspec(lexed) - end + defmodule Parser do + @moduledoc false - defp nillify(""), do: nil - defp nillify(o), do: o + operators = [ + {">=", :>=}, + {"<=", :<=}, + {"~>", :~>}, + {">", :>}, + {"<", :<}, + {"==", :==}, + {" or ", :or}, + {" and ", :and} + ] - @spec parse_version(String.t) :: {:ok, Version.matchable} | :error - def parse_version(string, approximate? \\ false) when is_binary(string) do - if parsed = Regex.run(@version_regex, string) do - destructure [_, major, minor, patch, pre], parsed - patch = nillify(patch) - pre = nillify(pre) - - if nil?(minor) or (nil?(patch) and not approximate?) do - :error - else - major = String.to_integer(major) - minor = String.to_integer(minor) - patch = patch && String.to_integer(patch) - - case parse_pre(pre) do - {:ok, pre} -> - {:ok, {major, minor, patch, pre}} - :error -> - :error - end - end - else - :error - end + def lexer(string) do + lexer(string, "", []) end - defp parse_pre(nil), do: {:ok, []} - defp parse_pre(pre), do: parse_pre(String.split(pre, "."), []) - - defp parse_pre([piece|t], acc) do - cond do - piece =~ ~r/^(0|[1-9][0-9]*)$/ -> - parse_pre(t, [String.to_integer(piece)|acc]) - piece =~ ~r/^[0-9]*$/ -> - :error - true -> - parse_pre(t, [piece|acc]) + for {string_op, atom_op} <- operators do + defp lexer(unquote(string_op) <> rest, buffer, acc) do + lexer(rest, "", [unquote(atom_op) | maybe_prepend_buffer(buffer, acc)]) end end - defp parse_pre([], acc) do - {:ok, Enum.reverse(acc)} + defp lexer("!=" <> rest, buffer, acc) do + IO.warn("!= inside Version requirements is deprecated, use ~> or >= instead") + lexer(rest, "", [:!= | maybe_prepend_buffer(buffer, acc)]) end - defp valid_requirement?([]), do: false - defp valid_requirement?([a | next]), do: valid_requirement?(a, next) - - # it must finish with a version - defp valid_requirement?(a, []) when is_binary(a) do - true + defp lexer("!" <> rest, buffer, acc) do + IO.warn("! inside Version requirements is deprecated, use ~> or >= instead") + lexer(rest, "", [:!= | maybe_prepend_buffer(buffer, acc)]) end - # or | and - defp valid_requirement?(a, [b | next]) when is_atom(a) and is_atom(b) and a in [:'||', :'&&'] do - valid_requirement?(b, next) + defp lexer(" " <> rest, buffer, acc) do + lexer(rest, "", maybe_prepend_buffer(buffer, acc)) end - # or | and - defp valid_requirement?(a, [b | next]) when is_binary(a) and is_atom(b) and b in [:'||', :'&&'] do - valid_requirement?(b, next) + defp lexer(<>, buffer, acc) do + lexer(rest, <>, acc) end - # or | and - defp valid_requirement?(a, [b | next]) when is_atom(a) and is_binary(b) and a in [:'||', :'&&'] do - valid_requirement?(b, next) + defp lexer(<<>>, buffer, acc) do + maybe_prepend_buffer(buffer, acc) end - # - defp valid_requirement?(a, [b | next]) when is_atom(a) and is_binary(b) do - valid_requirement?(b, next) - end + defp maybe_prepend_buffer("", acc), do: acc - defp valid_requirement?(_, _) do - false - end + defp maybe_prepend_buffer(buffer, [head | _] = acc) + when is_atom(head) and head not in [:and, :or], + do: [buffer | acc] - defp approximate_upper(version) do - case version do - {major, _minor, nil, _} -> - {major + 1, 0, 0, [0]} + defp maybe_prepend_buffer(buffer, acc), + do: [buffer, :== | acc] - {major, minor, _patch, _} -> - {major, minor + 1, 0, [0]} + defp revert_lexed([version, op, cond | rest], acc) + when is_binary(version) and is_atom(op) and cond in [:or, :and] do + with {:ok, version} <- validate_requirement(op, version) do + revert_lexed(rest, [cond, op, version | acc]) end end - defp to_matchspec(lexed) do - if valid_requirement?(lexed) do - first = to_condition(lexed) - rest = Enum.drop(lexed, 2) - {:ok, [{{:'$1', :'$2', :'$3', :'$4'}, [to_condition(first, rest)], [:'$_']}]} - else - :error + defp revert_lexed([version, op], acc) when is_binary(version) and is_atom(op) do + with {:ok, version} <- validate_requirement(op, version) do + {:ok, [op, version | acc]} end - catch - :invalid_matchspec -> :error end - defp to_condition([:'==', version | _]) do - version = parse_condition(version) - {:'==', :'$_', {:const, version}} - end + defp revert_lexed(_rest, _acc), do: :error - defp to_condition([:'!=', version | _]) do - version = parse_condition(version) - {:'/=', :'$_', {:const, version}} + defp validate_requirement(op, version) do + case parse_version(version, true) do + {:ok, version} when op == :~> -> {:ok, version} + {:ok, {_, _, patch, _, _} = version} when is_integer(patch) -> {:ok, version} + _ -> :error + end end - defp to_condition([:'~>', version | _]) do - from = parse_condition(version, true) - to = approximate_upper(from) + @spec parse_requirement(String.t()) :: {:ok, term} | :error + def parse_requirement(source) do + revert_lexed(lexer(source), []) + end - {:andalso, to_condition([:'>=', matchable_to_string(from)]), - to_condition([:'<', matchable_to_string(to)])} + def parse_version(string, approximate? \\ false) when is_binary(string) do + destructure [version_with_pre, build], String.split(string, "+", parts: 2) + destructure [version, pre], String.split(version_with_pre, "-", parts: 2) + destructure [major, minor, patch, next], String.split(version, ".") + + with nil <- next, + {:ok, major} <- require_digits(major), + {:ok, minor} <- require_digits(minor), + {:ok, patch} <- maybe_patch(patch, approximate?), + {:ok, pre_parts} <- optional_dot_separated(pre), + {:ok, pre_parts} <- convert_parts_to_integer(pre_parts, []), + {:ok, build_parts} <- optional_dot_separated(build) do + {:ok, {major, minor, patch, pre_parts, build_parts}} + else + _other -> :error + end end - defp to_condition([:'>', version | _]) do - {major, minor, patch, pre} = parse_condition(version) + defp require_digits(nil), do: :error - {:orelse, {:'>', {{:'$1', :'$2', :'$3'}}, - {:const, {major, minor, patch}}}, - {:andalso, {:'==', {{:'$1', :'$2', :'$3'}}, - {:const, {major, minor, patch}}}, - {:orelse, {:andalso, {:'==', {:length, :'$4'}, 0}, - {:'/=', length(pre), 0}}, - {:andalso, {:'/=', length(pre), 0}, - {:orelse, {:'>', {:length, :'$4'}, length(pre)}, - {:andalso, {:'==', {:length, :'$4'}, length(pre)}, - {:'>', :'$4', {:const, pre}}}}}}}} + defp require_digits(string) do + if leading_zero?(string), do: :error, else: parse_digits(string, "") end - defp to_condition([:'>=', version | _]) do - matchable = parse_condition(version) + defp leading_zero?(<>), do: true + defp leading_zero?(_), do: false - {:orelse, {:'==', :'$_', {:const, matchable}}, - to_condition([:'>', version])} - end + defp parse_digits(<>, acc) when char in ?0..?9, + do: parse_digits(rest, <>) - defp to_condition([:'<', version | _]) do - {major, minor, patch, pre} = parse_condition(version) + defp parse_digits(<<>>, acc) when byte_size(acc) > 0, do: {:ok, String.to_integer(acc)} + defp parse_digits(_, _acc), do: :error - {:orelse, {:'<', {{:'$1', :'$2', :'$3'}}, - {:const, {major, minor, patch}}}, - {:andalso, {:'==', {{:'$1', :'$2', :'$3'}}, - {:const, {major, minor, patch}}}, - {:orelse, {:andalso, {:'/=', {:length, :'$4'}, 0}, - {:'==', length(pre), 0}}, - {:andalso, {:'/=', {:length, :'$4'}, 0}, - {:orelse, {:'<', {:length, :'$4'}, length(pre)}, - {:andalso, {:'==', {:length, :'$4'}, length(pre)}, - {:'<', :'$4', {:const, pre}}}}}}}} - end + defp maybe_patch(patch, approximate?) + defp maybe_patch(nil, true), do: {:ok, nil} + defp maybe_patch(patch, _), do: require_digits(patch) + + defp optional_dot_separated(nil), do: {:ok, []} - defp to_condition([:'<=', version | _]) do - matchable = parse_condition(version) + defp optional_dot_separated(string) do + parts = String.split(string, ".") - {:orelse, {:'==', :'$_', {:const, matchable}}, - to_condition([:'<', version])} + if Enum.all?(parts, &(&1 != "" and valid_identifier?(&1))) do + {:ok, parts} + else + :error + end end - defp to_condition(current, []) do - current + defp convert_parts_to_integer([part | rest], acc) do + case parse_digits(part, "") do + {:ok, integer} -> + if leading_zero?(part) do + :error + else + convert_parts_to_integer(rest, [integer | acc]) + end + + :error -> + convert_parts_to_integer(rest, [part | acc]) + end end - defp to_condition(current, [:'&&', operator, version | rest]) do - to_condition({:andalso, current, to_condition([operator, version])}, rest) + defp convert_parts_to_integer([], acc) do + {:ok, Enum.reverse(acc)} end - defp to_condition(current, [:'||', operator, version | rest]) do - to_condition({:orelse, current, to_condition([operator, version])}, rest) + defp valid_identifier?(<>) + when char in ?0..?9 + when char in ?a..?z + when char in ?A..?Z + when char == ?- do + valid_identifier?(rest) end - defp parse_condition(version, approximate? \\ false) do - case parse_version(version, approximate?) do - {:ok, version} -> version - :error -> throw :invalid_matchspec - end + defp valid_identifier?(<<>>) do + true end - defp matchable_to_string({major, minor, patch, pre}) do - patch = if patch, do: "#{patch}", else: "0" - pre = if pre != [], do: "-#{Enum.join(pre, ".")}" - "#{major}.#{minor}.#{patch}#{pre}" + defp valid_identifier?(_other) do + false end end end defimpl String.Chars, for: Version do - def to_string(version) do - pre = unless Enum.empty?(pre = version.pre), do: "-#{pre}" - build = if build = version.build, do: "+#{build}" - "#{version.major}.#{version.minor}.#{version.patch}#{pre}#{build}" - end -end - -defimpl Inspect, for: Version do - def inspect(self, _opts) do - "#Version<" <> to_string(self) <> ">" - end + defdelegate to_string(version), to: Version end defimpl String.Chars, for: Version.Requirement do @@ -511,7 +696,9 @@ defimpl String.Chars, for: Version.Requirement do end defimpl Inspect, for: Version.Requirement do - def inspect(%Version.Requirement{source: source}, _opts) do - "#Version.Requirement<" <> source <> ">" + def inspect(%Version.Requirement{source: source}, opts) do + colorized = Inspect.Algebra.color_doc("\"" <> source <> "\"", :string, opts) + + Inspect.Algebra.concat(["Version.parse_requirement!(", colorized, ")"]) end end diff --git a/lib/elixir/mix.exs b/lib/elixir/mix.exs index 4b39a746602..e94a0c0140b 100644 --- a/lib/elixir/mix.exs +++ b/lib/elixir/mix.exs @@ -1,12 +1,15 @@ -defmodule Elixir.Mixfile do +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +defmodule Elixir.MixProject do use Mix.Project def project do - [app: :elixir, - version: System.version, - build_per_environment: false, - escript_embed_elixir: false, - escript_main_module: :elixir, - escript_emu_args: "%%! -noshell\n"] + [ + app: :elixir, + version: System.version(), + build_per_environment: false + ] end end diff --git a/lib/elixir/pages/anti-patterns/code-anti-patterns.md b/lib/elixir/pages/anti-patterns/code-anti-patterns.md new file mode 100644 index 00000000000..d4bfebc0fed --- /dev/null +++ b/lib/elixir/pages/anti-patterns/code-anti-patterns.md @@ -0,0 +1,636 @@ + + +# Code-related anti-patterns + +This document outlines potential anti-patterns related to your code and particular Elixir idioms and features. + +## Comments overuse + +#### Problem + +When you overuse comments or comment self-explanatory code, it can have the effect of making code *less readable*. + +#### Example + +```elixir +# Returns the Unix timestamp of 5 minutes from the current time +defp unix_five_min_from_now do + # Get the current time + now = DateTime.utc_now() + + # Convert it to a Unix timestamp + unix_now = DateTime.to_unix(now, :second) + + # Add five minutes in seconds + unix_now + (60 * 5) +end +``` + +#### Refactoring + +Prefer clear and self-explanatory function names, module names, and variable names when possible. In the example above, the function name explains well what the function does, so you likely won't need the comment before it. The code also explains the operations well through variable names and clear function calls. + +You could refactor the code above like this: + +```elixir +@five_min_in_seconds 60 * 5 + +defp unix_five_min_from_now do + now = DateTime.utc_now() + unix_now = DateTime.to_unix(now, :second) + unix_now + @five_min_in_seconds +end +``` + +We removed the unnecessary comments. We also added a `@five_min_in_seconds` module attribute, which serves the additional purpose of giving a name to the "magic" number `60 * 5`, making the code clearer and more expressive. + +#### Additional remarks + +Elixir makes a clear distinction between **documentation** and code comments. The language has built-in first-class support for documentation through `@doc`, `@moduledoc`, and more. See the ["Writing documentation"](../getting-started/writing-documentation.md) guide for more information. + +## Complex `else` clauses in `with` + +#### Problem + +This anti-pattern refers to `with` expressions that flatten all its error clauses into a single complex `else` block. This situation is harmful to the code readability and maintainability because it's difficult to know from which clause the error value came. + +#### Example + +An example of this anti-pattern, as shown below, is a function `open_decoded_file/1` that reads a Base64-encoded string content from a file and returns a decoded binary string. This function uses a `with` expression that needs to handle two possible errors, all of which are concentrated in a single complex `else` block. + +```elixir +def open_decoded_file(path) do + with {:ok, encoded} <- File.read(path), + {:ok, decoded} <- Base.decode64(encoded) do + {:ok, String.trim(decoded)} + else + {:error, _} -> {:error, :badfile} + :error -> {:error, :badencoding} + end +end +``` + +In the code above, it is unclear how each pattern on the left side of `<-` relates to their error at the end. The more patterns in a `with`, the less clear the code gets, and the more likely it is that unrelated failures will overlap each other. + +#### Refactoring + +In this situation, instead of concentrating all error handling within a single complex `else` block, it is better to normalize the return types in specific private functions. In this way, `with` can focus on the success case and the errors are normalized closer to where they happen, leading to better organized and maintainable code. + +```elixir +def open_decoded_file(path) do + with {:ok, encoded} <- file_read(path), + {:ok, decoded} <- base_decode64(encoded) do + {:ok, String.trim(decoded)} + end +end + +defp file_read(path) do + case File.read(path) do + {:ok, contents} -> {:ok, contents} + {:error, _} -> {:error, :badfile} + end +end + +defp base_decode64(contents) do + case Base.decode64(contents) do + {:ok, decoded} -> {:ok, decoded} + :error -> {:error, :badencoding} + end +end +``` + +## Complex extractions in clauses + +#### Problem + +When we use multi-clause functions, it is possible to extract values in the clauses for further usage and for pattern matching/guard checking. This extraction itself does not represent an anti-pattern, but when you have *extractions made across several clauses and several arguments of the same function*, it becomes hard to know which extracted parts are used for pattern/guards and what is used only inside the function body. This anti-pattern is related to [Unrelated multi-clause function](design-anti-patterns.md#unrelated-multi-clause-function), but with implications of its own. It impairs the code readability in a different way. + +#### Example + +The multi-clause function `drive/1` is extracting fields of an `%User{}` struct for usage in the clause expression (`age`) and for usage in the function body (`name`): + +```elixir +def drive(%User{name: name, age: age}) when age >= 18 do + "#{name} can drive" +end + +def drive(%User{name: name, age: age}) when age < 18 do + "#{name} cannot drive" +end +``` + +While the example above is small and does not constitute an anti-pattern, it is an example of mixed extraction and pattern matching. A situation where `drive/1` was more complex, having many more clauses, arguments, and extractions, would make it hard to know at a glance which variables are used for pattern/guards and which ones are not. + +#### Refactoring + +As shown below, a possible solution to this anti-pattern is to extract only pattern/guard related variables in the signature once you have many arguments or multiple clauses: + +```elixir +def drive(%User{age: age} = user) when age >= 18 do + %User{name: name} = user + "#{name} can drive" +end + +def drive(%User{age: age} = user) when age < 18 do + %User{name: name} = user + "#{name} cannot drive" +end +``` + +## Dynamic atom creation + +#### Problem + +An `Atom` is an Elixir basic type whose value is its own name. Atoms are often useful to identify resources or express the state, or result, of an operation. Creating atoms dynamically is not an anti-pattern by itself. However, atoms are not garbage collected by the Erlang Virtual Machine, so values of this type live in memory during a software's entire execution lifetime. The Erlang VM limits the number of atoms that can exist in an application by default to *1 048 576*, which is more than enough to cover all atoms defined in a program, but attempts to serve as an early limit for applications which are "leaking atoms" through dynamic creation. + +For these reasons, creating atoms dynamically can be considered an anti-pattern when the developer has no control over how many atoms will be created during the software execution. This unpredictable scenario can expose the software to unexpected behavior caused by excessive memory usage, or even by reaching the maximum number of *atoms* possible. + +#### Example + +Picture yourself implementing code that converts string values into atoms. These strings could have been received from an external system, either as part of a request into our application, or as part of a response to your application. This dynamic and unpredictable scenario poses a security risk, as these uncontrolled conversions can potentially trigger out-of-memory errors. + +```elixir +defmodule MyRequestHandler do + def parse(%{"status" => status, "message" => message} = _payload) do + %{status: String.to_atom(status), message: message} + end +end +``` + +```elixir +iex> MyRequestHandler.parse(%{"status" => "ok", "message" => "all good"}) +%{status: :ok, message: "all good"} +``` + +When we use the `String.to_atom/1` function to dynamically create an atom, it essentially gains potential access to create arbitrary atoms in our system, causing us to lose control over adhering to the limits established by the BEAM. This issue could be exploited by someone to create enough atoms to shut down a system. + +#### Refactoring + +To eliminate this anti-pattern, developers must either perform explicit conversions by mapping strings to atoms or replace the use of `String.to_atom/1` with `String.to_existing_atom/1`. An explicit conversion could be done as follows: + +```elixir +defmodule MyRequestHandler do + def parse(%{"status" => status, "message" => message} = _payload) do + %{status: convert_status(status), message: message} + end + + defp convert_status("ok"), do: :ok + defp convert_status("error"), do: :error + defp convert_status("redirect"), do: :redirect +end +``` + +```elixir +iex> MyRequestHandler.parse(%{"status" => "status_not_seen_anywhere", "message" => "all good"}) +** (FunctionClauseError) no function clause matching in MyRequestHandler.convert_status/1 +``` + +By explicitly listing all supported statuses, you guarantee only a limited number of conversions may happen. Passing an invalid status will lead to a function clause error. + +An alternative is to use `String.to_existing_atom/1`, which will only convert a string to atom if the atom already exists in the system: + +```elixir +defmodule MyRequestHandler do + def parse(%{"status" => status, "message" => message} = _payload) do + %{status: String.to_existing_atom(status), message: message} + end +end +``` + +```elixir +iex> MyRequestHandler.parse(%{"status" => "status_not_seen_anywhere", "message" => "all good"}) +** (ArgumentError) errors were found at the given arguments: + + * 1st argument: not an already existing atom +``` + +In such cases, passing an unknown status will raise as long as the status was not defined anywhere as an atom in the system. However, assuming `status` can be either `:ok`, `:error`, or `:redirect`, how can you guarantee those atoms exist? You must ensure those atoms exist somewhere **in the same module** where `String.to_existing_atom/1` is called. For example, if you had this code: + +```elixir +defmodule MyRequestHandler do + def parse(%{"status" => status, "message" => message} = _payload) do + %{status: String.to_existing_atom(status), message: message} + end + + def handle(%{status: status}) do + case status do + :ok -> ... + :error -> ... + :redirect -> ... + end + end +end +``` + +All valid statuses are defined as atoms within the same module, and that's enough. If you want to be explicit, you could also have a function that lists them: + +```elixir +def valid_statuses do + [:ok, :error, :redirect] +end +``` + +However, keep in mind using a module attribute or defining the atoms in the module body, outside of a function, are not sufficient, as the module body is only executed during compilation and it is not necessarily part of the compiled module loaded at runtime. + +## Long parameter list + +#### Problem + +In a functional language like Elixir, functions tend to explicitly receive all inputs and return all relevant outputs, instead of relying on mutations or side-effects. As functions grow in complexity, the amount of arguments (parameters) they need to work with may grow, to a point where the function's interface becomes confusing and prone to errors during use. + +#### Example + +In the following example, the `loan/6` functions takes too many arguments, causing its interface to be confusing and potentially leading developers to introduce errors during calls to this function. + +```elixir +defmodule Library do + # Too many parameters that can be grouped! + def loan(user_name, email, password, user_alias, book_title, book_ed) do + ... + end +end +``` + +#### Refactoring + +To address this anti-pattern, related arguments can be grouped using key-value data structures, such as maps, structs, or even keyword lists in the case of optional arguments. This effectively reduces the number of arguments and the key-value data structures adds clarity to the caller. + +For this particular example, the arguments to `loan/6` can be grouped into two different maps, thereby reducing its arity to `loan/2`: + +```elixir +defmodule Library do + def loan(%{name: name, email: email, password: password, alias: alias} = user, %{title: title, ed: ed} = book) do + ... + end +end +``` + +In some cases, the function with too many arguments may be a private function, which gives us more flexibility over how to separate the function arguments. One possible suggestion for such scenarios is to split the arguments in two maps (or tuples): one map keeps the data that may change, and the other keeps the data that won't change (read-only). This gives us a mechanical option to refactor the code. + +Other times, a function may legitimately take half a dozen or more completely unrelated arguments. This may suggest the function is trying to do too much and would be better broken into multiple functions, each responsible for a smaller piece of the overall responsibility. + +## Namespace trespassing + +#### Problem + +This anti-pattern manifests when a package author or a library defines modules outside of its "namespace". A library should use its name as a "prefix" for all of its modules. For example, a package named `:my_lib` should define all of its modules within the `MyLib` namespace, such as `MyLib.User`, `MyLib.SubModule`, `MyLib.Application`, and `MyLib` itself. + +This is important because the Erlang VM can only load one instance of a module at a time. So if there are multiple libraries that define the same module, then they are incompatible with each other due to this limitation. By always using the library name as a prefix, it avoids module name clashes due to the unique prefix. + +#### Example + +This problem commonly manifests when writing an extension of another library. For example, imagine you are writing a package that adds authentication to [Plug](https://github.com/elixir-plug/plug) called `:plug_auth`. You must avoid defining modules within the `Plug` namespace: + +```elixir +defmodule Plug.Auth do + # ... +end +``` + +Even if `Plug` does not currently define a `Plug.Auth` module, it may add such a module in the future, which would ultimately conflict with `plug_auth`'s definition. + +#### Refactoring + +Given the package is named `:plug_auth`, it must define modules inside the `PlugAuth` namespace: + +```elixir +defmodule PlugAuth do + # ... +end +``` + +#### Additional remarks + +There are few known exceptions to this anti-pattern: + + * [Protocol implementations](`Kernel.defimpl/2`) are, by design, defined under the protocol namespace + + * In some scenarios, the namespace owner may allow exceptions to this rule. For example, in Elixir itself, you defined [custom Mix tasks](`Mix.Task`) by placing them under the `Mix.Tasks` namespace, such as `Mix.Tasks.PlugAuth` + + * If you are the maintainer for both `plug` and `plug_auth`, then you may allow `plug_auth` to define modules with the `Plug` namespace, such as `Plug.Auth`. However, you are responsible for avoiding or managing any conflicts that may arise in the future + +## Non-assertive map access + +#### Problem + +In Elixir, it is possible to access values from `Map`s, which are key-value data structures, either statically or dynamically. + +When a key is expected to exist in a map, it must be accessed using the `map.key` notation, making it clear to developers (and the compiler) that the key must exist. If the key does not exist, an exception is raised (and in some cases also compiler warnings). This is also known as the static notation, as the key is known at the time of writing the code. + +When a key is optional, the `map[:key]` notation must be used instead. This way, if the informed key does not exist, `nil` is returned. This is the dynamic notation, as it also supports dynamic key access, such as `map[some_var]`. + +When you use `map[:key]` to access a key that always exists in the map, you are making the code less clear for developers and for the compiler, as they now need to work with the assumption the key may not be there. This mismatch may also make it harder to track certain bugs. If the key is unexpectedly missing, you will have a `nil` value propagate through the system, instead of raising on map access. + +##### Table: Comparison of map access notations + +| Access notation | Key exists | Key doesn't exist | Use case | +| --------------- | ---------- | ----------------- | -------- | +| `map.key` | Returns the value | Raises `KeyError` | Structs and maps with known atom keys | +| `map[:key]` | Returns the value | Returns `nil` | Any `Access`-based data structure, optional keys | + +#### Example + +The function `plot/1` tries to draw a graphic to represent the position of a point in a Cartesian plane. This function receives a parameter of `Map` type with the point attributes, which can be a point of a 2D or 3D Cartesian coordinate system. This function uses dynamic access to retrieve values for the map keys: + +```elixir +defmodule Graphics do + def plot(point) do + # Some other code... + {point[:x], point[:y], point[:z]} + end +end +``` + +```elixir +iex> point_2d = %{x: 2, y: 3} +%{x: 2, y: 3} +iex> point_3d = %{x: 5, y: 6, z: 7} +%{x: 5, y: 6, z: 7} +iex> Graphics.plot(point_2d) +{2, 3, nil} +iex> Graphics.plot(point_3d) +{5, 6, 7} +``` + +Given we want to plot both 2D and 3D points, the behavior above is expected. But what happens if we forget to pass a point with either `:x` or `:y`? + +```elixir +iex> bad_point = %{y: 3, z: 4} +%{y: 3, z: 4} +iex> Graphics.plot(bad_point) +{nil, 3, 4} +``` + +The behavior above is unexpected because our function should not work with points without a `:x` key. This leads to subtle bugs, as we may now pass `nil` to another function, instead of raising early on, as shown next: + +```iex +iex> point_without_x = %{y: 10} +%{y: 10} +iex> {x, y, _} = Graphics.plot(point_without_x) +{nil, 10, nil} +iex> distance_from_origin = :math.sqrt(x * x + y * y) +** (ArithmeticError) bad argument in arithmetic expression + :erlang.*(nil, nil) +``` + +The error above occurs later in the code because `nil` (from missing `:x`) is invalid for arithmetic operations, making it harder to identify the original issue. + +#### Refactoring + +To remove this anti-pattern, we must use the dynamic `map[:key]` syntax and the static `map.key` notation according to our requirements. We expect `:x` and `:y` to always exist, but not `:z`. The next code illustrates the refactoring of `plot/1`, removing this anti-pattern: + +```elixir +defmodule Graphics do + def plot(point) do + # Some other code... + {point.x, point.y, point[:z]} + end +end +``` + +```elixir +iex> Graphics.plot(point_2d) +{2, 3, nil} +iex> Graphics.plot(bad_point) +** (KeyError) key :x not found in: %{y: 3, z: 4} + graphic.ex:4: Graphics.plot/1 +``` + +This is beneficial because: + +1. It makes your expectations clear to others reading the code +2. It fails fast when required data is missing +3. It allows the compiler to provide warnings when accessing non-existent fields, particularly in compile-time structures like structs + +Overall, the usage of `map.key` and `map[:key]` encode important information about your data structure, allowing developers to be clear about their intent. The `Access` module documentation also provides useful reference on this topic. You can also consider the `Map` module when working with maps of any keys, which contains functions for fetching keys (with or without default values), updating and removing keys, traversals, and more. + +An alternative to refactor this anti-pattern is to use pattern matching, defining explicit clauses for 2D vs 3D points: + +```elixir +defmodule Graphics do + # 3d + def plot(%{x: x, y: y, z: z}) do + # Some other code... + {x, y, z} + end + + # 2d + def plot(%{x: x, y: y}) do + # Some other code... + {x, y} + end +end +``` + +Pattern-matching is specially useful when matching over multiple keys as well as on the values themselves at once. In the example above, the code will not only extract the values but also verify that the required keys exist. If we try to call `plot/1` with a map that doesn't have the required keys, we'll get a `FunctionClauseError`: + +```elixir +iex> incomplete_point = %{x: 5} +%{x: 5} +iex> Graphics.plot(incomplete_point) +** (FunctionClauseError) no function clause matching in Graphics.plot/1 + + The following arguments were given to Graphics.plot/1: + + # 1 + %{x: 5} +``` + +Another option is to use structs. By default, structs only support static access to its fields. In such scenarios, you may consider defining structs for both 2D and 3D points: + +```elixir +defmodule Point2D do + @enforce_keys [:x, :y] + defstruct [x: nil, y: nil] +end +``` + +Generally speaking, structs are useful when sharing data structures across modules, at the cost of adding a compile time dependency between these modules. If module `A` uses a struct defined in module `B`, `A` must be recompiled if the fields in the struct `B` change. + +In summary, Elixir provides several ways to access map values, each with different behaviors: + +1. **Static access** (`map.key`): Fails fast when keys are missing, ideal for structs and maps with known atom keys +2. **Dynamic access** (`map[:key]`): Works with any `Access` data structure, suitable for optional fields, returns nil for missing keys +3. **Pattern matching**: Provides a powerful way to both extract values and ensure required map/struct keys exist in one operation + +Choosing the right approach depends if the keys are known upfront or not. Static access and pattern matching are mostly equivalent (although pattern matching allows you to match on multiple keys at once, including matching on the struct name). + +#### Additional remarks + +This anti-pattern was formerly known as [Accessing non-existent map/struct fields](https://github.com/lucasvegi/Elixir-Code-Smells#accessing-non-existent-mapstruct-fields). + +## Non-assertive pattern matching + +#### Problem + +Overall, Elixir systems are composed of many supervised processes, so the effects of an error are localized to a single process, and don't propagate to the entire application. A supervisor detects the failing process, reports it, and possibly restarts it. This anti-pattern arises when developers write defensive or imprecise code, capable of returning incorrect values which were not planned for, instead of programming in an assertive style through pattern matching and guards. + +#### Example + +The function `get_value/2` tries to extract a value from a specific key of a URL query string. As it is not implemented using pattern matching, `get_value/2` always returns a value, regardless of the format of the URL query string passed as a parameter in the call. Sometimes the returned value will be valid. However, if a URL query string with an unexpected format is used in the call, `get_value/2` will extract incorrect values from it: + +```elixir +defmodule Extract do + def get_value(string, desired_key) do + parts = String.split(string, "&") + + Enum.find_value(parts, fn pair -> + key_value = String.split(pair, "=") + Enum.at(key_value, 0) == desired_key && Enum.at(key_value, 1) + end) + end +end +``` + +```elixir +# URL query string with the planned format - OK! +iex> Extract.get_value("name=Lucas&university=UFMG&lab=ASERG", "lab") +"ASERG" +iex> Extract.get_value("name=Lucas&university=UFMG&lab=ASERG", "university") +"UFMG" +# Unplanned URL query string format - Unplanned value extraction! +iex> Extract.get_value("name=Lucas&university=institution=UFMG&lab=ASERG", "university") +"institution" # <= why not "institution=UFMG"? or only "UFMG"? +``` + +#### Refactoring + +To remove this anti-pattern, `get_value/2` can be refactored through the use of pattern matching. So, if an unexpected URL query string format is used, the function will crash instead of returning an invalid value. This behavior, shown below, allows clients to decide how to handle these errors and doesn't give a false impression that the code is working correctly when unexpected values are extracted: + +```elixir +defmodule Extract do + def get_value(string, desired_key) do + parts = String.split(string, "&") + + Enum.find_value(parts, fn pair -> + [key, value] = String.split(pair, "=") # <= pattern matching + key == desired_key && value + end) + end +end +``` + +```elixir +# URL query string with the planned format - OK! +iex> Extract.get_value("name=Lucas&university=UFMG&lab=ASERG", "name") +"Lucas" +# Unplanned URL query string format - Crash explaining the problem to the client! +iex> Extract.get_value("name=Lucas&university=institution=UFMG&lab=ASERG", "university") +** (MatchError) no match of right hand side value: ["university", "institution", "UFMG"] + extract.ex:7: anonymous fn/2 in Extract.get_value/2 # <= left hand: [key, value] pair +iex> Extract.get_value("name=Lucas&university&lab=ASERG", "university") +** (MatchError) no match of right hand side value: ["university"] + extract.ex:7: anonymous fn/2 in Extract.get_value/2 # <= left hand: [key, value] pair +``` + +Elixir and pattern matching promote an assertive style of programming where you handle the known cases. Once an unexpected scenario arises, you can decide to address it accordingly based on practical examples, or conclude the scenario is indeed invalid and the exception is the desired choice. + +`case/2` is another important construct in Elixir that help us write assertive code, by matching on specific patterns. For example, if a function returns `{:ok, ...}` or `{:error, ...}`, prefer to explicitly match on both patterns: + +```elixir +case some_function(arg) do + {:ok, value} -> # ... + {:error, _} -> # ... +end +``` + +In particular, avoid matching solely on `_`, as shown below: + +```elixir +case some_function(arg) do + {:ok, value} -> # ... + _ -> # ... +end +``` + + Matching on `_` is less clear in intent and it may hide bugs if `some_function/1` adds new return values in the future. + +#### Additional remarks + +This anti-pattern was formerly known as [Speculative assumptions](https://github.com/lucasvegi/Elixir-Code-Smells#speculative-assumptions). + +## Non-assertive truthiness + +#### Problem + +Elixir provides the concept of truthiness: `nil` and `false` are considered "falsy" and all other values are "truthy". Many constructs in the language, such as `&&/2`, `||/2`, and `!/1` handle truthy and falsy values. Using those operators is not an anti-pattern. However, using those operators when all operands are expected to be booleans, may be an anti-pattern. + +#### Example + +The simplest scenario where this anti-pattern manifests is in conditionals, such as: + +```elixir +if is_binary(name) && is_integer(age) do + # ... +else + # ... +end +``` + +Given both operands of `&&/2` are booleans, the code is more generic than necessary, and potentially unclear. + +#### Refactoring + +To remove this anti-pattern, we can replace `&&/2`, `||/2`, and `!/1` by `and/2`, `or/2`, and `not/1` respectively. These operators assert at least their first argument is a boolean: + +```elixir +if is_binary(name) and is_integer(age) do + # ... +else + # ... +end +``` + +This technique may be particularly important when working with Erlang code. Erlang does not have the concept of truthiness. It never returns `nil`, instead its functions may return `:error` or `:undefined` in places an Elixir developer would return `nil`. Therefore, to avoid accidentally interpreting `:undefined` or `:error` as a truthy value, you may prefer to use `and/2`, `or/2`, and `not/1` exclusively when interfacing with Erlang APIs. + +## Structs with 32 fields or more + +#### Problem + +Structs in Elixir are implemented as compile-time maps, which have a predefined amount of fields. When structs have 32 or more fields, their internal representation in the Erlang Virtual Machines changes, potentially leading to bloating and higher memory usage. + +#### Example + +Any struct with 32 or more fields will be problematic: + +```elixir +defmodule MyExample do + defstruct [ + :field1, + :field2, + ..., + :field35 + ] +end +``` + +The Erlang VM has two internal representations for maps: a flat map and a hash map. A flat map is represented internally as two tuples: one tuple containing the keys and another tuple holding the values. Whenever you update a flat map, the tuple keys are shared, reducing the amount of memory used by the update. A hash map has a more complex structure, which is efficient for a large amount of keys, but it does not share the key space. + +Maps of up to 32 keys are represented as flat maps. All others are hash map. Structs *are* maps (with a metadata field called `__struct__`) and so any struct with fewer than 32 fields is represented as a flat map. This allows us to optimize several struct operations, as we never add or remove fields to structs, we simply update them. + +Furthermore, structs of the same name "instantiated" in the same module will share the same "tuple keys" at compilation times, as long as they have fewer than 32 fields. For example, in the following code: + +```elixir +defmodule Example do + def users do + [%User{name: "John"}, %User{name: "Meg"}, ...] + end +end +``` + +All user structs will point to the same tuple keys at compile-time, also reducing the memory cost of instantiating structs with `%MyStruct{...}` notation. This optimization is also not available if the struct has 32 keys or more. + +#### Refactoring + +Removing this anti-pattern, in a nutshell, requires ensuring your struct has fewer than 32 fields. There are a few techniques you could apply: + + * If the struct has "optional" fields, for example, fields which are initialized with nil, you could nest all optional fields into other field, called `:metadata`, `:optionals`, or similar. This could lead to benefits such as being able to use pattern matching to check if a field exists or not, instead of relying on `nil` values + + * You could nest structs, by storing structs within other fields. Fields that are rarely read or written to are good candidates to be moved to a nested struct + + * You could nest fields as tuples. For example, if two fields are always read or updated together, they could be moved to a tuple (or another composite data structure) + +The challenge is to balance the changes above with API ergonomics, in particular, when fields may be frequently read and written to. diff --git a/lib/elixir/pages/anti-patterns/design-anti-patterns.md b/lib/elixir/pages/anti-patterns/design-anti-patterns.md new file mode 100644 index 00000000000..6e0d6dfa30e --- /dev/null +++ b/lib/elixir/pages/anti-patterns/design-anti-patterns.md @@ -0,0 +1,486 @@ + + +# Design-related anti-patterns + +This document outlines potential anti-patterns related to your modules, functions, and the role they play within a codebase. + +## Alternative return types + +#### Problem + +This anti-pattern refers to functions that receive options (typically as a *keyword list* parameter) that drastically change their return type. Because options are optional and sometimes set dynamically, if they also change the return type, it may be hard to understand what the function actually returns. + +#### Example + +An example of this anti-pattern, as shown below, is when a function has many alternative return types, depending on the options received as a parameter. + +```elixir +defmodule AlternativeInteger do + @spec parse(String.t(), keyword()) :: integer() | {integer(), String.t()} | :error + def parse(string, options \\ []) when is_list(options) do + if Keyword.get(options, :discard_rest, false) do + case Integer.parse(string) do + {int, _rest} -> int + :error -> :error + end + else + Integer.parse(string) + end + end +end +``` + +```elixir +iex> AlternativeInteger.parse("13") +{13, ""} +iex> AlternativeInteger.parse("13", discard_rest: false) +{13, ""} +iex> AlternativeInteger.parse("13", discard_rest: true) +13 +``` + +#### Refactoring + +To refactor this anti-pattern, as shown next, add a specific function for each return type (for example, `parse_discard_rest/1`), no longer delegating this to options passed as arguments. + +```elixir +defmodule AlternativeInteger do + @spec parse(String.t()) :: {integer(), String.t()} | :error + def parse(string) do + Integer.parse(string) + end + + @spec parse_discard_rest(String.t()) :: integer() | :error + def parse_discard_rest(string) do + case Integer.parse(string) do + {int, _rest} -> int + :error -> :error + end + end +end +``` + +```elixir +iex> AlternativeInteger.parse("13") +{13, ""} +iex> AlternativeInteger.parse_discard_rest("13") +13 +``` + +## Boolean obsession + +#### Problem + +This anti-pattern happens when booleans are used instead of atoms to encode information. The usage of booleans themselves is not an anti-pattern, but whenever multiple booleans are used with overlapping states, replacing the booleans by atoms (or composite data types such as *tuples*) may lead to clearer code. + +This is a special case of [*Primitive obsession*](#primitive-obsession), specific to boolean values. + +#### Example + +An example of this anti-pattern is a function that receives two or more options, such as `editor: true` and `admin: true`, to configure its behavior in overlapping ways. In the code below, the `:editor` option has no effect if `:admin` is set, meaning that the `:admin` option has higher priority than `:editor`, and they are ultimately related. + +```elixir +defmodule MyApp do + def process(invoice, options \\ []) do + cond do + options[:admin] -> # Is an admin + options[:editor] -> # Is an editor + true -> # Is none + end + end +end +``` + +#### Refactoring + +Instead of using multiple options, the code above could be refactored to receive a single option, called `:role`, that can be either `:admin`, `:editor`, or `:default`: + +```elixir +defmodule MyApp do + def process(invoice, options \\ []) do + case Keyword.get(options, :role, :default) do + :admin -> # Is an admin + :editor -> # Is an editor + :default -> # Is none + end + end +end +``` + +This anti-pattern may also happen in our own data structures. For example, we may define a `User` struct with two boolean fields, `:editor` and `:admin`, while a single field named `:role` may be preferred. + +Finally, it is worth noting that using atoms may be preferred even when we have a single boolean argument/option. For example, consider an invoice which may be set as approved/unapproved. One option is to provide a function that expects a boolean: + +```elixir +MyApp.update(invoice, approved: true) +``` + +However, using atoms may read better and make it simpler to add further states (such as pending) in the future: + +```elixir +MyApp.update(invoice, status: :approved) +``` + +Remember booleans are internally represented as atoms. Therefore there is no performance penalty in one approach over the other. + +## Exceptions for control-flow + +#### Problem + +This anti-pattern refers to code that uses `Exception`s for control flow. Exception handling itself does not represent an anti-pattern, but developers must prefer to use `case` and pattern matching to change the flow of their code, instead of `try/rescue`. In turn, library authors should provide developers with APIs to handle errors without relying on exception handling. When developers have no freedom to decide if an error is exceptional or not, this is considered an anti-pattern. + +#### Example + +An example of this anti-pattern, as shown below, is using `try/rescue` to deal with file operations: + +```elixir +defmodule MyModule do + def print_file(file) do + try do + IO.puts(File.read!(file)) + rescue + e -> IO.puts(:stderr, Exception.message(e)) + end + end +end +``` + +```elixir +iex> MyModule.print_file("valid_file") +This is a valid file! +:ok +iex> MyModule.print_file("invalid_file") +could not read file "invalid_file": no such file or directory +:ok +``` + +#### Refactoring + +To refactor this anti-pattern, as shown next, use `File.read/1`, which returns tuples instead of raising when a file cannot be read: + +```elixir +defmodule MyModule do + def print_file(file) do + case File.read(file) do + {:ok, binary} -> IO.puts(binary) + {:error, reason} -> IO.puts(:stderr, "could not read file #{file}: #{reason}") + end + end +end +``` + +This is only possible because the `File` module provides APIs for reading files with tuples as results (`File.read/1`), as well as a version that raises an exception (`File.read!/1`). The bang (exclamation point) is effectively part of [Elixir's naming conventions](naming-conventions.md#trailing-bang-foo). + +Library authors are encouraged to follow the same practices. In practice, the bang variant is implemented on top of the non-raising version of the code. For example, `File.read!/1` is implemented as: + +```elixir +def read!(path) do + case read(path) do + {:ok, binary} -> + binary + + {:error, reason} -> + raise File.Error, reason: reason, action: "read file", path: IO.chardata_to_string(path) + end +end +``` + +A common practice followed by the community is to make the non-raising version return `{:ok, result}` or `{:error, Exception.t}`. For example, an HTTP client may return `{:ok, %HTTP.Response{}}` on success cases and `{:error, %HTTP.Error{}}` for failures, where `HTTP.Error` is [implemented as an exception](`Kernel.defexception/1`). This makes it convenient for anyone to raise an exception by simply calling `Kernel.raise/1`. + +#### Additional remarks + +This anti-pattern is of special importance to library authors and whenever writing functions that will be invoked by other developers and third-party code. Nevertheless, there are still scenarios where developers can afford to raise exceptions directly, for example: + + * invalid arguments: it is expected that functions will raise for invalid arguments, as those are structural error and not semantic errors. For example, `File.read(123)` will always raise, because `123` is never a valid filename + + * during tests, scripts, etc: those are common scenarios where you want your code to fail as soon as possible in case of errors. Using `!` functions, such as `File.read!/1`, allows you to do so quickly and with clear error messages + + * some frameworks, such as [Phoenix](https://phoenixframework.org), allow developers to raise exceptions in their code and uses a protocol to convert these exceptions into semantic HTTP responses + +This anti-pattern was formerly known as [Using exceptions for control-flow](https://github.com/lucasvegi/Elixir-Code-Smells#using-exceptions-for-control-flow). + +## Primitive obsession + +#### Problem + +This anti-pattern happens when Elixir basic types (for example, *integer*, *float*, and *string*) are excessively used to carry structured information, rather than creating specific composite data types (for example, *tuples*, *maps*, and *structs*) that can better represent a domain. + +#### Example + +An example of this anti-pattern is the use of a single *string* to represent an `Address`. An `Address` is a more complex structure than a simple basic (aka, primitive) value. + +```elixir +defmodule MyApp do + def extract_postal_code(address) when is_binary(address) do + # Extract postal code with address... + end + + def fill_in_country(address) when is_binary(address) do + # Fill in missing country... + end +end +``` + +While you may receive the `address` as a string from a database, web request, or a third-party, if you find yourself frequently manipulating or extracting information from the string, it is a good indicator you should convert the address into structured data: + +Another example of this anti-pattern is using floating numbers to model money and currency, when [richer data structures should be preferred](https://hexdocs.pm/ex_money/). + +#### Refactoring + +Possible solutions to this anti-pattern is to use maps or structs to model our address. The example below creates an `Address` struct, better representing this domain through a composite type. Additionally, we introduce a `parse/1` function, that converts the string into an `Address`, which will simplify the logic of remaining functions. With this modification, we can extract each field of this composite type individually when needed. + +```elixir +defmodule Address do + defstruct [:street, :city, :state, :postal_code, :country] +end +``` + +```elixir +defmodule MyApp do + def parse(address) when is_binary(address) do + # Returns %Address{} + end + + def extract_postal_code(%Address{} = address) do + # Extract postal code with address... + end + + def fill_in_country(%Address{} = address) do + # Fill in missing country... + end +end +``` + +## Unrelated multi-clause function + +#### Problem + +Using multi-clause functions is a powerful Elixir feature. However, some developers may abuse this feature to group *unrelated* functionality, which is an anti-pattern. + +#### Example + +A frequent example of this usage of multi-clause functions occurs when developers mix unrelated business logic into the same function definition, in a way that the behavior of each clause becomes completely distinct from the others. Such functions often have too broad specifications, making it difficult for other developers to understand and maintain them. + +Some developers may use documentation mechanisms such as `@doc` annotations to compensate for poor code readability, however the documentation itself may end-up full of conditionals to describe how the function behaves for each different argument combination. This is a good indicator that the clauses are ultimately unrelated. + +```elixir +@doc """ +Updates a struct. + +If given a product, it will... + +If given an animal, it will... +""" +def update(%Product{count: count, material: material}) do + # ... +end + +def update(%Animal{count: count, skin: skin}) do + # ... +end +``` + +If updating an animal is completely different from updating a product and requires a different set of rules, it may be worth splitting those over different functions or even different modules. + +#### Refactoring + +As shown below, a possible solution to this anti-pattern is to break the business rules that are mixed up in a single unrelated multi-clause function in simple functions. Each function can have a specific name and `@doc`, describing its behavior and parameters received. While this refactoring sounds simple, it can impact the function's callers, so be careful! + +```elixir +@doc """ +Updates a product. + +It will... +""" +def update_product(%Product{count: count, material: material}) do + # ... +end + +@doc """ +Updates an animal. + +It will... +""" +def update_animal(%Animal{count: count, skin: skin}) do + # ... +end +``` + +These functions may still be implemented with multiple clauses, as long as the clauses group related functionality. For example, `update_product` could be in practice implemented as follows: + +```elixir +def update_product(%Product{count: 0}) do + # ... +end + +def update_product(%Product{material: material}) + when material in ["metal", "glass"] do + # ... +end + +def update_product(%Product{material: material}) + when material not in ["metal", "glass"] do + # ... +end +``` + +You can see this pattern in practice within Elixir itself. The `+/2` operator can add `Integer`s and `Float`s together, but not `String`s, which instead use the `<>/2` operator. In this sense, it is reasonable to handle integers and floats in the same operation, but strings are unrelated enough to deserve their own function. + +You will also find examples in Elixir of functions that work with any struct, which would seemingly be an occurrence of this anti-pattern, such as `struct/2`: + +```elixir +iex> struct(URI.parse("/foo/bar"), path: "/bar/baz") +%URI{ + scheme: nil, + userinfo: nil, + host: nil, + port: nil, + path: "/bar/baz", + query: nil, + fragment: nil +} +``` + +The difference here is that the `struct/2` function behaves precisely the same for any struct given, therefore there is no question of how the function handles different inputs. If the behavior is clear and consistent for all inputs, then the anti-pattern does not take place. + +## Using application configuration for libraries + +#### Problem + +The [*application environment*](https://hexdocs.pm/elixir/Application.html#module-the-application-environment) can be used to parameterize global values that can be used in an Elixir system. This mechanism can be very useful and therefore is not considered an anti-pattern by itself. However, library authors should avoid using the application environment to configure their library. The reason is exactly that the application environment is a **global** state, so there can only be a single value for each key in the environment for an application. This makes it impossible for multiple applications depending on the same library to configure the same aspect of the library in different ways. + +#### Example + +The `DashSplitter` module represents a library that configures the behavior of its functions through the global application environment. These configurations are concentrated in the *config/config.exs* file, shown below: + +```elixir +import Config + +config :app_config, + parts: 3 + +import_config "#{config_env()}.exs" +``` + +One of the functions implemented by the `DashSplitter` library is `split/1`. This function aims to separate a string received via a parameter into a certain number of parts. The character used as a separator in `split/1` is always `"-"` and the number of parts the string is split into is defined globally by the application environment. This value is retrieved by the `split/1` function by calling `Application.fetch_env!/2`, as shown next: + +```elixir +defmodule DashSplitter do + def split(string) when is_binary(string) do + parts = Application.fetch_env!(:app_config, :parts) # <= retrieve parameterized value + String.split(string, "-", parts: parts) # <= parts: 3 + end +end +``` + +Due to this parameterized value used by the `DashSplitter` library, all applications dependent on it can only use the `split/1` function with identical behavior about the number of parts generated by string separation. Currently, this value is equal to 3, as we can see in the use examples shown below: + +```elixir +iex> DashSplitter.split("Lucas-Francisco-Vegi") +["Lucas", "Francisco", "Vegi"] +iex> DashSplitter.split("Lucas-Francisco-da-Matta-Vegi") +["Lucas", "Francisco", "da-Matta-Vegi"] +``` + +#### Refactoring + +To remove this anti-pattern, this type of configuration should be performed using a parameter passed to the function. The code shown below performs the refactoring of the `split/1` function by accepting [keyword lists](`Keyword`) as a new optional parameter. With this new parameter, it is possible to modify the default behavior of the function at the time of its call, allowing multiple different ways of using `split/2` within the same application: + +```elixir +defmodule DashSplitter do + def split(string, opts \\ []) when is_binary(string) and is_list(opts) do + parts = Keyword.get(opts, :parts, 2) # <= default config of parts == 2 + String.split(string, "-", parts: parts) + end +end +``` + +```elixir +iex> DashSplitter.split("Lucas-Francisco-da-Matta-Vegi", [parts: 5]) +["Lucas", "Francisco", "da", "Matta", "Vegi"] +iex> DashSplitter.split("Lucas-Francisco-da-Matta-Vegi") #<= default config is used! +["Lucas", "Francisco-da-Matta-Vegi"] +``` + +Of course, not all uses of the application environment by libraries are incorrect. One example is using configuration to replace a component (or dependency) of a library by another that must behave the exact same. Consider a library that needs to parse CSV files. The library author may pick one package to use as default parser but allow its users to swap to different implementations via the application environment. At the end of the day, choosing a different CSV parser should not change the outcome, and library authors can even enforce this by [defining behaviours](../references/typespecs.md#behaviours) with the exact semantics they expect. + +#### Additional remarks: Supervision trees + +In practice, libraries may require additional configuration beyond keyword lists. For example, if a library needs to start a supervision tree, how can the user of said library customize its supervision tree? Given the supervision tree itself is global (as it belongs to the library), library authors may be tempted to use the application configuration once more. + +One solution is for the library to provide its own child specification, instead of starting the supervision tree itself. This allows the user to start all necessary processes under its own supervision tree, potentially passing custom configuration options during initialization. + +You can see this pattern in practice in projects like [Nx](https://github.com/elixir-nx/nx) and [DNS Cluster](https://github.com/phoenixframework/dns_cluster). These libraries require that you list processes under your own supervision tree: + +```elixir +children = [ + {DNSCluster, query: "my.subdomain"} +] +``` + +In such cases, if the users of `DNSCluster` need to configure DNSCluster per environment, they can be the ones reading from the application environment, without the library forcing them to: + +```elixir +children = [ + {DNSCluster, query: Application.get_env(:my_app, :dns_cluster_query) || :ignore} +] +``` + +Some libraries, such as [Ecto](https://github.com/elixir-ecto/ecto), allow you to pass your application name as an option (called `:otp_app` or similar) and then automatically read the environment from *your* application. While this addresses the issue with the application environment being global, as they read from each individual application, it comes at the cost of some indirection, compared to the example above where users explicitly read their application environment from their own code, whenever desired. + +#### Additional remarks: Compile-time configuration + +A similar discussion entails compile-time configuration. What if a library author requires some configuration to be provided at compilation time? + +Once again, instead of forcing users of your library to provide compile-time configuration, you may want to allow users of your library to generate the code themselves. That's the approach taken by libraries such as [Ecto](https://github.com/elixir-ecto/ecto): + +```elixir +defmodule MyApp.Repo do + use Ecto.Repo, adapter: Ecto.Adapters.Postgres +end +``` + +Instead of forcing developers to share a single repository, Ecto allows its users to define as many repositories as they want. Given the `:adapter` configuration is required at compile-time, it is a required value on `use Ecto.Repo`. If developers want to configure the adapter per environment, then it is their choice: + +```elixir +defmodule MyApp.Repo do + use Ecto.Repo, adapter: Application.compile_env(:my_app, :repo_adapter) +end +``` + +On the other hand, [code generation comes with its own anti-patterns](macro-anti-patterns.md), and must be considered carefully. That's to say: while using the application environment for libraries is discouraged, especially compile-time configuration, in some cases they may be the best option. For example, consider a library needs to parse CSV or JSON files to generate code based on data files. In such cases, it is best to provide reasonable defaults and make them customizable via the application environment, instead of asking each user of your library to generate the exact same code. + +#### Additional remarks: Mix tasks + +For Mix tasks and related tools, it may be necessary to provide per-project configuration. For example, imagine you have a `:linter` project, which supports setting the output file and the verbosity level. You may choose to configure it through application environment: + +```elixir +config :linter, + output_file: "/path/to/output.json", + verbosity: 3 +``` + +However, `Mix` allows tasks to read per-project configuration via `Mix.Project.config/0`. In this case, you can configure the `:linter` directly in the `mix.exs` file: + +```elixir +def project do + [ + app: :my_app, + version: "1.0.0", + linter: [ + output_file: "/path/to/output.json", + verbosity: 3 + ], + ... + ] +end +``` + +Additionally, if a Mix task is available, you can also accept these options as command line arguments (see `OptionParser`): + +```bash +mix linter --output-file /path/to/output.json --verbosity 3 +``` diff --git a/lib/elixir/pages/anti-patterns/macro-anti-patterns.md b/lib/elixir/pages/anti-patterns/macro-anti-patterns.md new file mode 100644 index 00000000000..9ccb413642e --- /dev/null +++ b/lib/elixir/pages/anti-patterns/macro-anti-patterns.md @@ -0,0 +1,397 @@ + + +# Meta-programming anti-patterns + +This document outlines potential anti-patterns related to meta-programming. + +## Compile-time dependencies + +#### Problem + +This anti-pattern is related to dependencies between files in Elixir. Because macros are used at compile-time, the use of any macro in Elixir adds a compile-time dependency to the module that defines the macro. + +However, when macros are used in the body of a module, the arguments to the macro themselves may become compile-time dependencies. These dependencies may lead to dependency graphs where changing a single file causes several files to be recompiled. + +#### Example + +Let's take the [`Plug`](https://github.com/elixir-plug/plug) library as an example. The `Plug` project allows you to specify several modules, also known as plugs, which will be invoked whenever there is a request. As a user of `Plug`, you would use it as follows: + +```elixir +defmodule MyApp do + use Plug.Builder + + plug MyApp.Authentication +end +``` + +And imagine `Plug` has the following definitions of the macros above (simplified): + +```elixir +defmodule Plug.Builder do + defmacro __using__(_opts) do + quote do + Module.register_attribute(__MODULE__, :plugs, accumulate: true) + @before_compile Plug.Builder + end + end + + defmacro plug(mod) do + quote do + @plugs unquote(mod) + end + end + + ... +end +``` + +The implementation accumulates all modules inside the `@plugs` module attribute. Right before the module is compiled, `Plug.Builder` will reads all modules stored in `@plugs` and compile them into a function, like this: + +```elixir +def call(conn, _opts) do + MyApp.Authentication.call(conn) +end +``` + +The trouble with the code above is that, because the `plug MyApp.Authentication` was invoked at compile-time, the module `MyApp.Authentication` is now a compile-time dependency of `MyApp`, even though `MyApp.Authentication` is never used at compile-time. If `MyApp.Authentication` depends on other modules, even at runtime, this can now lead to a large recompilation graph in case of changes. + +#### Refactoring + +To address this anti-pattern, a macro can expand literals within the context they are meant to be used, as follows: + +```elixir + defmacro plug(mod) do + mod = Macro.expand_literals(mod, %{__CALLER__ | function: {:call, 2}}) + + quote do + @plugs unquote(mod) + end + end +``` + +In the example above, since `mod` is used only within the `call/2` function, we prematurely expand module reference as if it was inside the `call/2` function. Now `MyApp.Authentication` is only a runtime dependency of `MyApp`, no longer a compile-time one. + +Note, however, the above must only be done if your macros do not attempt to invoke any function, access any struct, or any other metadata of the module at compile-time. If you interact with the module given to a macro anywhere outside of definition of a function, then you effectively have a compile-time dependency. And, even though you generally want to avoid them, it is not always possible. + +In actual projects, developers may use `mix xref trace path/to/file.ex` to execute a file and have it print information about which modules it depends on, and if those modules are compile-time, runtime, or export dependencies. See `mix xref` for more information. + +## Large code generation + +#### Problem + +This anti-pattern is related to macros that generate too much code. When a macro generates a large amount of code, it impacts how the compiler and/or the runtime work. The reason for this is that Elixir may have to expand, compile, and execute the code multiple times, which will make compilation slower and the resulting compiled artifacts larger. + +#### Example + +Imagine you are defining a router for a web application, where you could have macros like `get/2`. On every invocation of the macro (which could be hundreds), the code inside `get/2` will be expanded and compiled, which can generate a large volume of code overall. + +```elixir +defmodule Routes do + defmacro get(route, handler) do + quote do + route = unquote(route) + handler = unquote(handler) + + if not is_binary(route) do + raise ArgumentError, "route must be a binary" + end + + if not is_atom(handler) do + raise ArgumentError, "handler must be a module" + end + + @store_route_for_compilation {route, handler} + end + end +end +``` + +#### Refactoring + +To remove this anti-pattern, the developer should simplify the macro, delegating part of its work to other functions. As shown below, by encapsulating the code inside `quote/1` inside the function `__define__/3` instead, we reduce the code that is expanded and compiled on every invocation of the macro, and instead we dispatch to a function to do the bulk of the work. + +```elixir +defmodule Routes do + defmacro get(route, handler) do + quote do + Routes.__define__(__MODULE__, unquote(route), unquote(handler)) + end + end + + def __define__(module, route, handler) do + if not is_binary(route) do + raise ArgumentError, "route must be a binary" + end + + if not is_atom(handler) do + raise ArgumentError, "handler must be a module" + end + + Module.put_attribute(module, :store_route_for_compilation, {route, handler}) + end +end +``` + +## Unnecessary macros + +#### Problem + +*Macros* are powerful meta-programming mechanisms that can be used in Elixir to extend the language. While using macros is not an anti-pattern in itself, this meta-programming mechanism should only be used when absolutely necessary. Whenever a macro is used, but it would have been possible to solve the same problem using functions or other existing Elixir structures, the code becomes unnecessarily more complex and less readable. Because macros are more difficult to implement and reason about, their indiscriminate use can compromise the evolution of a system, reducing its maintainability. + +#### Example + +The `MyMath` module implements the `sum/2` macro to perform the sum of two numbers received as parameters. While this code has no syntax errors and can be executed correctly to get the desired result, it is unnecessarily more complex. By implementing this functionality as a macro rather than a conventional function, the code became less clear: + +```elixir +defmodule MyMath do + defmacro sum(v1, v2) do + quote do + unquote(v1) + unquote(v2) + end + end +end +``` + +```elixir +iex> require MyMath +MyMath +iex> MyMath.sum(3, 5) +8 +iex> MyMath.sum(3 + 1, 5 + 6) +15 +``` + +#### Refactoring + +To remove this anti-pattern, the developer must replace the unnecessary macro with structures that are simpler to write and understand, such as named functions. The code shown below is the result of the refactoring of the previous example. Basically, the `sum/2` macro has been transformed into a conventional named function. Note that the `require/2` call is no longer needed: + +```elixir +defmodule MyMath do + def sum(v1, v2) do # <= The macro became a named function + v1 + v2 + end +end +``` + +```elixir +iex> MyMath.sum(3, 5) +8 +iex> MyMath.sum(3+1, 5+6) +15 +``` + +## `use` instead of `import` + +#### Problem + +Elixir has mechanisms such as `import/1`, `alias/1`, and `use/1` to establish dependencies between modules. Code implemented with these mechanisms does not characterize a smell by itself. However, while the `import/1` and `alias/1` directives have lexical scope and only facilitate a module calling functions of another, the `use/1` directive has a *broader scope*, which can be problematic. + +The `use/1` directive allows a module to inject any type of code into another, including propagating dependencies. In this way, using the `use/1` directive makes code harder to read, because to understand exactly what will happen when it references a module, it is necessary to have knowledge of the internal details of the referenced module. + +#### Example + +The code shown below is an example of this anti-pattern. It defines three modules -- `ModuleA`, `Library`, and `ClientApp`. `ClientApp` is reusing code from the `Library` via the `use/1` directive, but is unaware of its internal details. This makes it harder for the author of `ClientApp` to visualize which modules and functionality are now available within its module. To make matters worse, `Library` also imports `ModuleA`, which defines a `foo/0` function that conflicts with a local function defined in `ClientApp`: + +```elixir +defmodule ModuleA do + def foo do + "From Module A" + end +end +``` + +```elixir +defmodule Library do + defmacro __using__(_opts) do + quote do + import Library + import ModuleA # <= propagating dependencies! + end + end + + def from_lib do + "From Library" + end +end +``` + +```elixir +defmodule ClientApp do + use Library + + def foo do + "Local function from client app" + end + + def from_client_app do + from_lib() <> " - " <> foo() + end +end +``` + +When we try to compile `ClientApp`, Elixir detects the conflict and throws the following error: + +```text +error: imported ModuleA.foo/0 conflicts with local function + └ client_app.ex:4: +``` + +#### Refactoring + +To remove this anti-pattern, we recommend library authors avoid providing `__using__/1` callbacks whenever it can be replaced by `alias/1` or `import/1` directives. In the following code, we assume `use Library` is no longer available and `ClientApp` was refactored in this way, and with that, the code is clearer and the conflict as previously shown no longer exists: + +```elixir +defmodule ClientApp do + import Library + + def foo do + "Local function from client app" + end + + def from_client_app do + from_lib() <> " - " <> foo() + end +end +``` + +```elixir +iex> ClientApp.from_client_app() +"From Library - Local function from client app" +``` + +#### Additional remarks + +In situations where you need to do more than importing and aliasing modules, providing `use MyModule` may be necessary, as it provides a common extension point within the Elixir ecosystem. + +Therefore, to provide guidance and clarity, we recommend library authors to include an admonition block in their `@moduledoc` that explains how `use MyModule` impacts the developer's code. As an example, the `GenServer` documentation outlines: + +> #### `use GenServer` {: .info} +> +> When you `use GenServer`, the `GenServer` module will +> set `@behaviour GenServer` and define a `child_spec/1` +> function, so your module can be used as a child +> in a supervision tree. + +Think of this summary as a ["Nutrition facts label"](https://en.wikipedia.org/wiki/Nutrition_facts_label) for code generation. Make sure to only list changes made to the public API of the module. For example, if `use Library` sets an internal attribute called `@_some_module_info` and this attribute is never meant to be public, avoid documenting it in the nutrition facts. + +For convenience, the markup notation to generate the admonition block above is this: + +```markdown +> #### `use GenServer` {: .info} +> +> When you `use GenServer`, the `GenServer` module will +> set `@behaviour GenServer` and define a `child_spec/1` +> function, so your module can be used as a child +> in a supervision tree. +``` + +## Untracked compile-time dependencies + +#### Problem + +This anti-pattern is the opposite of ["Compile-time dependencies"](#compile-time-dependencies) and it happens when a compile-time dependency is accidentally bypassed, making the Elixir compiler unable to track dependencies and recompile files correctly. This happens when building aliases (in other words, module names) dynamically, either within a module or within a macro. + +#### Example + +For example, imagine you invoke a module at compile-time, you could write it as such: + +```elixir +defmodule MyModule do + SomeOtherModule.example() +end +``` + +In this case, Elixir knows `MyModule` is invoked `SomeOtherModule.example/0` outside of a function, and therefore at compile-time. + +Elixir can also track module names even during dynamic calls: + +```elixir +defmodule MyModule do + mods = [OtherModule.Foo, OtherModule.Bar] + + for mod <- mods do + mod.example() + end +end +``` + +In the previous example, even though Elixir does not know which modules the function `example/0` was invoked on, it knows the modules `OtherModule.Foo` and `OtherModule.Bar` are referred outside of a function and therefore they become compile-time dependencies. If any of them change, Elixir will recompile `MyModule` itself. + +However, you should not programmatically generate the module names themselves, as that would make it impossible for Elixir to track them. More precisely, do not do this: + +```elixir +defmodule MyModule do + parts = [:Foo, :Bar] + + for part <- parts do + Module.concat(OtherModule, part).example() + end +end +``` + +In this case, because the whole module was generated, Elixir sees a dependency only to `OtherModule`, never to `OtherModule.Foo` and `OtherModule.Bar`, potentially leading to inconsistencies when recompiling projects. + +A similar bug can happen when abusing the property that aliases are simply atoms, defining the atoms directly. In the case below, Elixir never sees the aliases, leading to untracked compile-time dependencies: + +```elixir +defmodule MyModule do + mods = [:"Elixir.OtherModule.Foo", :"Elixir.OtherModule.Bar"] + + for mod <- mods do + mod.example() + end +end +``` + +#### Refactoring + +To address this anti-pattern, you should avoid defining module names programmatically. For example, if you need to dispatch to multiple modules, do so by using full module names. + +Instead of: + +```elixir +defmodule MyModule do + parts = [:Foo, :Bar] + + for part <- parts do + Module.concat(OtherModule, part).example() + end +end +``` + +Do: + +```elixir +defmodule MyModule do + mods = [OtherModule.Foo, OtherModule.Bar] + + for mod <- mods do + mod.example() + end +end +``` + +If you really need to define modules dynamically, you can do so via meta-programming, building the whole module name at compile-time: + +```elixir +defmodule MyMacro do + defmacro call_examples(parts) do + for part <- parts do + quote do + # This builds OtherModule.Foo at compile-time + OtherModule.unquote(part).example() + end + end + end +end + +defmodule MyModule do + import MyMacro + call_examples [:Foo, :Bar] +end +``` + +In actual projects, developers may use `mix xref trace path/to/file.ex` to execute a file and have it print information about which modules it depends on, and if those modules are compile-time, runtime, or export dependencies. This can help you debug if the dependencies are being properly tracked in relation to external modules. See `mix xref` for more information. diff --git a/lib/elixir/pages/anti-patterns/process-anti-patterns.md b/lib/elixir/pages/anti-patterns/process-anti-patterns.md new file mode 100644 index 00000000000..07e27eb268b --- /dev/null +++ b/lib/elixir/pages/anti-patterns/process-anti-patterns.md @@ -0,0 +1,329 @@ + + +# Process-related anti-patterns + +This document outlines potential anti-patterns related to processes and process-based abstractions. + +## Code organization by process + +#### Problem + +This anti-pattern refers to code that is unnecessarily organized by processes. A process itself does not represent an anti-pattern, but it should only be used to model runtime properties (such as concurrency, access to shared resources, error isolation, etc). When you use a process for code organization, it can create bottlenecks in the system. + +#### Example + +An example of this anti-pattern, as shown below, is a module that implements arithmetic operations (like `add` and `subtract`) by means of a `GenServer` process. If the number of calls to this single process grows, this code organization can compromise the system performance, therefore becoming a bottleneck. + +```elixir +defmodule Calculator do + @moduledoc """ + Calculator that performs basic arithmetic operations. + + This code is unnecessarily organized in a GenServer process. + """ + + use GenServer + + def add(a, b, pid) do + GenServer.call(pid, {:add, a, b}) + end + + def subtract(a, b, pid) do + GenServer.call(pid, {:subtract, a, b}) + end + + @impl GenServer + def init(init_arg) do + {:ok, init_arg} + end + + @impl GenServer + def handle_call({:add, a, b}, _from, state) do + {:reply, a + b, state} + end + + def handle_call({:subtract, a, b}, _from, state) do + {:reply, a - b, state} + end +end +``` + +```elixir +iex> {:ok, pid} = GenServer.start_link(Calculator, :init) +{:ok, #PID<0.132.0>} +iex> Calculator.add(1, 5, pid) +6 +iex> Calculator.subtract(2, 3, pid) +-1 +``` + +#### Refactoring + +In Elixir, as shown next, code organization must be done only through modules and functions. Whenever possible, a library should not impose specific behavior (such as parallelization) on its users. It is better to delegate this behavioral decision to the developers of clients, thus increasing the potential for code reuse of a library. + +```elixir +defmodule Calculator do + def add(a, b) do + a + b + end + + def subtract(a, b) do + a - b + end +end +``` + +```elixir +iex> Calculator.add(1, 5) +6 +iex> Calculator.subtract(2, 3) +-1 +``` + +## Scattered process interfaces + +#### Problem + +In Elixir, the use of an `Agent`, a `GenServer`, or any other process abstraction is not an anti-pattern in itself. However, when the responsibility for direct interaction with a process is spread throughout the entire system, it can become problematic. This bad practice can increase the difficulty of code maintenance and make the code more prone to bugs. + +#### Example + +The following code seeks to illustrate this anti-pattern. The responsibility for interacting directly with the `Agent` is spread across four different modules (`A`, `B`, `C`, and `D`). + +```elixir +defmodule A do + def update(process) do + # Some other code... + Agent.update(process, fn _list -> 123 end) + end +end +``` + +```elixir +defmodule B do + def update(process) do + # Some other code... + Agent.update(process, fn content -> %{a: content} end) + end +end +``` + +```elixir +defmodule C do + def update(process) do + # Some other code... + Agent.update(process, fn content -> [:atom_value | content] end) + end +end +``` + +```elixir +defmodule D do + def get(process) do + # Some other code... + Agent.get(process, fn content -> content end) + end +end +``` + +This spreading of responsibility can generate duplicated code and make code maintenance more difficult. Also, due to the lack of control over the format of the shared data, complex composed data can be shared. This freedom to use any format of data is dangerous and can induce developers to introduce bugs. + +```elixir +# start an agent with initial state of an empty list +iex> {:ok, agent} = Agent.start_link(fn -> [] end) +{:ok, #PID<0.135.0>} + +# many data formats (for example, List, Map, Integer, Atom) are +# combined through direct access spread across the entire system +iex> A.update(agent) +iex> B.update(agent) +iex> C.update(agent) + +# state of shared information +iex> D.get(agent) +[:atom_value, %{a: 123}] +``` + +For a `GenServer` and other behaviours, this anti-pattern will manifest when scattering calls to `GenServer.call/3` and `GenServer.cast/2` throughout multiple modules, instead of encapsulating all the interaction with the `GenServer` in a single place. + +#### Refactoring + +Instead of spreading direct access to a process abstraction, such as `Agent`, over many places in the code, it is better to refactor this code by centralizing the responsibility for interacting with a process in a single module. This refactoring improves maintainability by removing duplicated code; it also allows you to limit the accepted format for shared data, reducing bug-proneness. As shown below, the module `Foo.Bucket` is centralizing the responsibility for interacting with the `Agent`. Any other place in the code that needs to access shared data must now delegate this action to `Foo.Bucket`. Also, `Foo.Bucket` now only allows data to be shared in `Map` format. + +```elixir +defmodule Foo.Bucket do + use Agent + + def start_link(_opts) do + Agent.start_link(fn -> %{} end) + end + + def get(bucket, key) do + Agent.get(bucket, &Map.get(&1, key)) + end + + def put(bucket, key, value) do + Agent.update(bucket, &Map.put(&1, key, value)) + end +end +``` + +The following are examples of how to delegate access to shared data (provided by an `Agent`) to `Foo.Bucket`. + +```elixir +# start an agent through `Foo.Bucket` +iex> {:ok, bucket} = Foo.Bucket.start_link(%{}) +{:ok, #PID<0.114.0>} + +# add shared values to the keys `milk` and `beer` +iex> Foo.Bucket.put(bucket, "milk", 3) +iex> Foo.Bucket.put(bucket, "beer", 7) + +# access shared data of specific keys +iex> Foo.Bucket.get(bucket, "beer") +7 +iex> Foo.Bucket.get(bucket, "milk") +3 +``` + +#### Additional remarks + +This anti-pattern was formerly known as [Agent obsession](https://github.com/lucasvegi/Elixir-Code-Smells/tree/main#agent-obsession). + +## Sending unnecessary data + +#### Problem + +Sending a message to a process can be an expensive operation if the message is big enough. That's because that message will be fully copied to the receiving process, which may be CPU and memory intensive. This is due to Erlang's "share nothing" architecture, where each process has its own memory, which simplifies and speeds up garbage collection. + +This is more obvious when using `send/2`, `GenServer.call/3`, or the initial data in `GenServer.start_link/3`. Notably this also happens when using `spawn/1`, `Task.async/1`, `Task.async_stream/3`, and so on. It is more subtle here as the anonymous function passed to these functions captures the variables it references, and all captured variables will be copied over. By doing this, you can accidentally send way more data to a process than you actually need. + +#### Example + +Imagine you were to implement some simple reporting of IP addresses that made requests against your application. You want to do this asynchronously and not block processing, so you decide to use `spawn/1`. It may seem like a good idea to hand over the whole connection because we might need more data later. However passing the connection results in copying a lot of unnecessary data like the request body, params, etc. + +```elixir +# log_request_ip send the ip to some external service +spawn(fn -> log_request_ip(conn) end) +``` + +This problem also occurs when accessing only the relevant parts: + +```elixir +spawn(fn -> log_request_ip(conn.remote_ip) end) +``` + +This will still copy over all of `conn`, because the `conn` variable is being captured inside the spawned function. The function then extracts the `remote_ip` field, but only after the whole `conn` has been copied over. + +`send/2` and the `GenServer` APIs also rely on message passing. In the example below, the `conn` is once again copied to the underlying `GenServer`: + +```elixir +GenServer.cast(pid, {:report_ip_address, conn}) +``` + +#### Refactoring + +This anti-pattern has many potential remedies: + + * Limit the data you send to the absolute necessary minimum instead of sending an entire struct. For example, don't send an entire `conn` struct if all you need is a couple of fields. + + * If the only process that needs data is the one you are sending to, consider making the process fetch that data instead of passing it. + + * Some abstractions, such as [`:persistent_term`](`:persistent_term`), allows you to share data between processes, as long as such data changes infrequently. + +In our case, limiting the input data is a reasonable strategy. If all we need *right now* is the IP address, then let's only work with that and make sure we're only passing the IP address into the closure, like so: + +```elixir +ip_address = conn.remote_ip +spawn(fn -> log_request_ip(ip_address) end) +``` + +Or in the `GenServer` case: + +```elixir +GenServer.cast(pid, {:report_ip_address, conn.remote_ip}) +``` + +## Unsupervised processes + +#### Problem + +In Elixir, creating a process outside a supervision tree is not an anti-pattern in itself. However, when you spawn many long-running processes outside of supervision trees, this can make visibility and monitoring of these processes difficult, preventing developers from fully controlling their lifecycle. + +#### Example + +The following code example seeks to illustrate a library responsible for maintaining a numerical `Counter` through a `Agent` process *outside a supervision tree*. + +```elixir +defmodule Counter do + @moduledoc """ + Global counter implemented as an Agent. + """ + + use Agent + + @doc "Starts a counter process." + def start_link(opts \\ []) do + initial_state = Keyword.get(opts, :initial_value, 0) + name = Keyword.get(opts, :name, __MODULE__) + Agent.start_link(fn -> initial_state end, name: name) + end + + @doc "Gets the current value of the given counter." + def get(name \\ __MODULE__) do + Agent.get(name, fn state -> state end) + end + + @doc "Bumps the value of the given counter." + def bump(name \\ __MODULE__, value) do + Agent.get_and_update(fn state -> {state, value + state} end) + end +end +``` + +While it is possible to start the process outside of a supervision tree: + +```elixir +iex> Counter.start_link() +{:ok, #PID<0.115.0>} +iex> Counter.bump(13) +0 +iex> Counter.get() +13 +``` + +Such processes are harder to observe and control their lifecycle. For example, if you have other processes that depend on the `Counter` above, you will need ad-hoc mechanisms to make sure they are initialized in order. Furthermore, when your application is shutting down, there is no guarantee when they are terminated. + +#### Refactoring + +To ensure that clients of a library have full control over their systems, regardless of the number of processes used and the lifetime of each one, all processes must be started inside a supervision tree. As shown below, this code uses a `Supervisor` as a supervision tree. + +```elixir +defmodule SupervisedProcess.Application do + use Application + + @impl true + def start(_type, _args) do + children = [ + # With the default values for counter and name + Counter, + # With custom values for counter, name, and a custom ID + Supervisor.child_spec( + {Counter, name: :other_counter, initial_value: 15}, + id: :other_counter + ) + ] + + Supervisor.start_link(children, strategy: :one_for_one, name: App.Supervisor) + end +end +``` + +Besides having a deterministic order in which processes are started, supervision trees also guarantee they are terminated in reverse order, allowing you to perform any necessary clean up during shut down. Furthermore, supervision strategies allows us to configure exactly how process should act in case of unexpected failures. + +Finally, applications and supervision trees can be introspected through applications like the [Phoenix.LiveDashboard](http://github.com/phoenixframework/phoenix_live_dashboard) and [Erlang's built-in observer](https://www.erlang.org/doc/apps/observer/observer_ug): + +Observer GUI screenshot diff --git a/lib/elixir/pages/anti-patterns/what-anti-patterns.md b/lib/elixir/pages/anti-patterns/what-anti-patterns.md new file mode 100644 index 00000000000..b22678a5c61 --- /dev/null +++ b/lib/elixir/pages/anti-patterns/what-anti-patterns.md @@ -0,0 +1,48 @@ + + +# What are anti-patterns? + +Anti-patterns describe common mistakes or indicators of problems in code. +They are also known as "code smells". + +The goal of these guides is to document potential anti-patterns found in Elixir software +and teach developers how to identify them and their pitfalls. If an existing piece +of code matches an anti-pattern, it does not mean your code must be rewritten. +Sometimes, even if a snippet matches a potential anti-pattern and its limitations, +it may be the best approach to the problem at hand. No codebase is free of anti-patterns +and one should not aim to remove all of them. + +The anti-patterns in these guides are broken into 4 main categories: + + * **Code-related anti-patterns:** related to your code and particular + language idioms and features; + + * **Design-related anti-patterns:** related to your modules, functions, + and the role they play within a codebase; + + * **Process-related anti-patterns:** related to processes and process-based + abstractions; + + * **Meta-programming anti-patterns:** related to meta-programming. + +Each anti-pattern is documented using the following structure: + + * **Name:** Unique identifier of the anti-pattern. This name is important to facilitate + communication between developers; + + * **Problem:** How the anti-pattern can harm code quality and what impacts this can have + for developers; + + * **Example:** Code and textual descriptions to illustrate the occurrence of the anti-pattern; + + * **Refactoring:** Ways to change your code to improve its qualities. Examples of refactored + code are presented to illustrate these changes. + +An additional section with "Additional Remarks" may be provided. Those may include known scenarios where the anti-pattern does not apply. + +The initial catalog of anti-patterns was proposed by Lucas Vegi and Marco Tulio Valente, from [ASERG/DCC/UFMG](http://aserg.labsoft.dcc.ufmg.br/). For more info, see [Understanding Code Smells in Elixir Functional Language](https://github.com/lucasvegi/Elixir-Code-Smells/blob/main/etc/2023-emse-code-smells-elixir.pdf) and [the associated code repository](https://github.com/lucasvegi/Elixir-Code-Smells). + +Additionally, the Security Working Group of the [Erlang Ecosystem Foundation](https://security.erlef.org) publishes [documents with security resources and best-practices of both Erlang and Elixir, including detailed guides for web applications](https://security.erlef.org). diff --git a/lib/elixir/pages/cheatsheets/enum-cheat.cheatmd b/lib/elixir/pages/cheatsheets/enum-cheat.cheatmd new file mode 100644 index 00000000000..1dfaa6cd8ce --- /dev/null +++ b/lib/elixir/pages/cheatsheets/enum-cheat.cheatmd @@ -0,0 +1,1058 @@ + + +# Enum cheatsheet + +A quick reference into the `Enum` module, a module for working with collections (known as enumerables). Most of the examples below use the following data structure: + +```elixir +cart = [ + %{fruit: "apple", count: 3}, + %{fruit: "banana", count: 1}, + %{fruit: "orange", count: 6} +] +``` + +Some examples use the [`string =~ part`](`=~/2`) operator, which checks the string on the left contains the part on the right. + +## Predicates +{: .col-2} + +### [`any?(enum, fun)`](`Enum.any?/2`) + +```elixir +iex> Enum.any?(cart, & &1.fruit == "orange") +true +iex> Enum.any?(cart, & &1.fruit == "pear") +false +``` + +`any?` with an empty collection is always false: + +```elixir +iex> Enum.any?([], & &1.fruit == "orange") +false +``` + +### [`all?(enum, fun)`](`Enum.all?/2`) + +```elixir +iex> Enum.all?(cart, & &1.count > 0) +true +iex> Enum.all?(cart, & &1.count > 1) +false +``` + +`all?` with an empty collection is always true: + +```elixir +iex> Enum.all?([], & &1.count > 0) +true +``` + +### [`member?(enum, value)`](`Enum.member?/2`) + +```elixir +iex> Enum.member?(cart, %{fruit: "apple", count: 3}) +true +iex> Enum.member?(cart, :something_else) +false +``` + +`item in enum` is equivalent to `Enum.member?(enum, item)`: + +```elixir +iex> %{fruit: "apple", count: 3} in cart +true +iex> :something_else in cart +false +``` + +### [`empty?(enum)`](`Enum.empty?/1`) + +```elixir +iex> Enum.empty?(cart) +false +iex> Enum.empty?([]) +true +``` + +## Filtering +{: .col-2} + +### [`filter(enum, fun)`](`Enum.filter/2`) + +```elixir +iex> Enum.filter(cart, &(&1.fruit =~ "o")) +[%{fruit: "orange", count: 6}] +iex> Enum.filter(cart, &(&1.fruit =~ "e")) +[ + %{fruit: "apple", count: 3}, + %{fruit: "orange", count: 6} +] +``` + +### [`reject(enum, fun)`](`Enum.reject/2`) + +```elixir +iex> Enum.reject(cart, &(&1.fruit =~ "o")) +[ + %{fruit: "apple", count: 3}, + %{fruit: "banana", count: 1} +] +``` + +### [`flat_map(enum, fun)`](`Enum.flat_map/2`) + +This function (also listed [below](#concatenating-flattening)) can +be used to transform and filter in one pass, returning empty lists +to exclude results: + +```elixir +iex> Enum.flat_map(cart, fn item -> +...> if item.count > 1, do: [item.fruit], else: [] +...> end) +["apple", "orange"] +``` + +### [`Comprehension`](`for/1`) + +Filtering can also be done with comprehensions: + +```elixir +iex> for item <- cart, item.fruit =~ "e" do +...> item +...> end +[ + %{fruit: "apple", count: 3}, + %{fruit: "orange", count: 6} +] +``` + +Pattern-matching in comprehensions acts as a filter as well: + +```elixir +iex> for %{count: 1, fruit: fruit} <- cart do +...> fruit +...> end +["banana"] +``` + +## Mapping +{: .col-2} + +### [`map(enum, fun)`](`Enum.map/2`) + +```elixir +iex> Enum.map(cart, & &1.fruit) +["apple", "banana", "orange"] +iex> Enum.map(cart, fn item -> +...> %{item | count: item.count + 10} +...> end) +[ + %{fruit: "apple", count: 13}, + %{fruit: "banana", count: 11}, + %{fruit: "orange", count: 16} +] +``` + +### [`map_every(enum, nth, fun)`](`Enum.map_every/3`) + +```elixir +iex> Enum.map_every(cart, 2, fn item -> +...> %{item | count: item.count + 10} +...> end) +[ + %{fruit: "apple", count: 13}, + %{fruit: "banana", count: 1}, + %{fruit: "orange", count: 16} +] +``` + +### [`Comprehension`](`for/1`) + +Mapping can also be done with comprehensions: + +```elixir +iex> for item <- cart do +...> item.fruit +...> end +["apple", "banana", "orange"] +``` + +You can also filter and map at once: + +```elixir +iex> for item <- cart, item.fruit =~ "e" do +...> item.fruit +...> end +["apple", "orange"] +``` + +## Side-effects +{: .col-2} + +### [`each(enum, fun)`](`Enum.each/2`) + +```elixir +iex> Enum.each(cart, &IO.puts(&1.fruit)) +apple +banana +orange +:ok +``` + +`Enum.each/2` is used exclusively for side-effects. + +## Accumulating +{: .col-2} + +### [`reduce(enum, acc, fun)`](`Enum.reduce/3`) + +```elixir +iex> Enum.reduce(cart, 0, fn item, acc -> +...> item.count + acc +...> end) +10 +``` + +### [`map_reduce(enum, acc, fun)`](`Enum.map_reduce/3`) + +```elixir +iex> Enum.map_reduce(cart, 0, fn item, acc -> +...> {item.fruit, item.count + acc} +...> end) +{["apple", "banana", "orange"], 10} +``` + +### [`scan(enum, acc, fun)`](`Enum.scan/3`) + +```elixir +iex> Enum.scan(cart, 0, fn item, acc -> +...> item.count + acc +...> end) +[3, 4, 10] +``` + +### [`reduce_while(enum, acc, fun)`](`Enum.reduce_while/3`) + +```elixir +iex> Enum.reduce_while(cart, 0, fn item, acc -> +...> if item.fruit == "orange" do +...> {:halt, acc} +...> else +...> {:cont, item.count + acc} +...> end +...> end) +4 +``` + +### [`Comprehension`](`for/1`) + +Reducing can also be done with comprehensions: + +```elixir +iex> for item <- cart, reduce: 0 do +...> acc -> item.count + acc +...> end +10 +``` + +You can also filter and reduce at once: + +```elixir +iex> for item <- cart, item.fruit =~ "e", reduce: 0 do +...> acc -> item.count + acc +...> end +9 +``` + +## Aggregations +{: .col-2} + +### [`count(enum)`](`Enum.count/1`) + +```elixir +iex> Enum.count(cart) +3 +``` + +See `Enum.count_until/2` to count until a limit. + +### [`frequencies(enum)`](`Enum.frequencies/1`) + +```elixir +iex> Enum.frequencies(["apple", "banana", "orange", "apple"]) +%{"apple" => 2, "banana" => 1, "orange" => 1} +``` + +### [`frequencies_by(enum, key_fun)`](`Enum.frequencies_by/2`) + +Frequencies of the last letter of the fruit: + +```elixir +iex> Enum.frequencies_by(cart, &String.last(&1.fruit)) +%{"a" => 1, "e" => 2} +``` + +### [`count(enum, fun)`](`Enum.count/2`) + +```elixir +iex> Enum.count(cart, &(&1.fruit =~ "e")) +2 +iex> Enum.count(cart, &(&1.fruit =~ "y")) +0 +``` + +See `Enum.count_until/3` to count until a limit with a function. + +### [`sum(enum)`](`Enum.sum/1`) + +```elixir +iex> cart |> Enum.map(& &1.count) |> Enum.sum() +10 +``` + +Note: this should typically be done in one pass using `Enum.sum_by/2`. + +### [`sum_by(enum, mapper)`](`Enum.sum_by/2`) + +```elixir +iex> Enum.sum_by(cart, & &1.count) +10 +``` + +### [`product(enum)`](`Enum.product/1`) + +```elixir +iex> cart |> Enum.map(& &1.count) |> Enum.product() +18 +``` + +Note: this should typically be done in one pass using `Enum.product_by/2`. + +### [`product_by(enum, mapper)`](`Enum.product_by/2`) + +```elixir +iex> Enum.product_by(cart, & &1.count) +18 +``` + +## Sorting +{: .col-2} + +### [`sort(enum, sorter \\ :asc)`](`Enum.sort/2`) + +```elixir +iex> cart |> Enum.map(& &1.fruit) |> Enum.sort() +["apple", "banana", "orange"] +iex> cart |> Enum.map(& &1.fruit) |> Enum.sort(:desc) +["orange", "banana", "apple"] +``` + +When sorting structs, use `Enum.sort/2` with a module as sorter. + +### [`sort_by(enum, mapper, sorter \\ :asc)`](`Enum.sort_by/2`) + +```elixir +iex> Enum.sort_by(cart, & &1.count) +[ + %{fruit: "banana", count: 1}, + %{fruit: "apple", count: 3}, + %{fruit: "orange", count: 6} +] +iex> Enum.sort_by(cart, & &1.count, :desc) +[ + %{fruit: "orange", count: 6}, + %{fruit: "apple", count: 3}, + %{fruit: "banana", count: 1} +] +``` + +When the sorted by value is a struct, use `Enum.sort_by/3` with a module as sorter. + +### [`min(enum)`](`Enum.min/1`) + +```elixir +iex> cart |> Enum.map(& &1.count) |> Enum.min() +1 +``` + +When comparing structs, use `Enum.min/2` with a module as sorter. + +### [`min_by(enum, mapper)`](`Enum.min_by/2`) + +```elixir +iex> Enum.min_by(cart, & &1.count) +%{fruit: "banana", count: 1} +``` + +When comparing structs, use `Enum.min_by/3` with a module as sorter. + +### [`max(enum)`](`Enum.max/1`) + +```elixir +iex> cart |> Enum.map(& &1.count) |> Enum.max() +6 +``` + +When comparing structs, use `Enum.max/2` with a module as sorter. + +### [`max_by(enum, mapper)`](`Enum.max_by/2`) + +```elixir +iex> Enum.max_by(cart, & &1.count) +%{fruit: "orange", count: 6} +``` + +When comparing structs, use `Enum.max_by/3` with a module as sorter. + +## Concatenating & flattening +{: .col-2} + +### [`concat(enums)`](`Enum.concat/1`) + +```elixir +iex> Enum.concat([[1, 2, 3], [4, 5, 6], [7, 8, 9]]) +[1, 2, 3, 4, 5, 6, 7, 8, 9] +``` + +### [`concat(left, right)`](`Enum.concat/2`) + +```elixir +iex> Enum.concat([1, 2, 3], [4, 5, 6]) +[1, 2, 3, 4, 5, 6] +``` + +### [`flat_map(enum, fun)`](`Enum.flat_map/2`) + +```elixir +iex> Enum.flat_map(cart, fn item -> +...> List.duplicate(item.fruit, item.count) +...> end) +["apple", "apple", "apple", "banana", "orange", + "orange", "orange", "orange", "orange", "orange"] +``` + +### [`flat_map_reduce(enum, acc, fun)`](`Enum.flat_map_reduce/3`) + +```elixir +iex> Enum.flat_map_reduce(cart, 0, fn item, acc -> +...> list = List.duplicate(item.fruit, item.count) +...> acc = acc + item.count +...> {list, acc} +...> end) +{["apple", "apple", "apple", "banana", "orange", + "orange", "orange", "orange", "orange", "orange"], 10} +``` + +### [`Comprehension`](`for/1`) + +Flattening can also be done with comprehensions: + +```elixir +iex> for item <- cart, +...> fruit <- List.duplicate(item.fruit, item.count) do +...> fruit +...> end +["apple", "apple", "apple", "banana", "orange", + "orange", "orange", "orange", "orange", "orange"] +``` + +## Conversion +{: .col-2} + +### [`into(enum, collectable)`](`Enum.into/2`) + +```elixir +iex> pairs = [{"apple", 3}, {"banana", 1}, {"orange", 6}] +iex> Enum.into(pairs, %{}) +%{"apple" => 3, "banana" => 1, "orange" => 6} +``` + +### [`into(enum, collectable, transform)`](`Enum.into/3`) + +```elixir +iex> Enum.into(cart, %{}, fn item -> +...> {item.fruit, item.count} +...> end) +%{"apple" => 3, "banana" => 1, "orange" => 6} +``` + +### [`to_list(enum)`](`Enum.to_list/1`) + +```elixir +iex> Enum.to_list(1..5) +[1, 2, 3, 4, 5] +``` + +### [`Comprehension`](`for/1`) + +Conversion can also be done with comprehensions: + +```elixir +iex> for item <- cart, into: %{} do +...> {item.fruit, item.count} +...> end +%{"apple" => 3, "banana" => 1, "orange" => 6} +``` + +## Duplicates & uniques +{: .col-2} + +### [`dedup(enum)`](`Enum.dedup/1`) + +`dedup` only removes contiguous duplicates: + +```elixir +iex> Enum.dedup([1, 2, 2, 3, 3, 3, 1, 2, 3]) +[1, 2, 3, 1, 2, 3] +``` + +### [`dedup_by(enum, fun)`](`Enum.dedup_by/2`) + +Remove contiguous entries given a property: + +```elixir +iex> Enum.dedup_by(cart, & &1.fruit =~ "a") +[%{fruit: "apple", count: 3}] +iex> Enum.dedup_by(cart, & &1.count < 5) +[ + %{fruit: "apple", count: 3}, + %{fruit: "orange", count: 6} +] +``` + +### [`uniq(enum)`](`Enum.uniq/1`) + +`uniq` applies to the whole collection: + +```elixir +iex> Enum.uniq([1, 2, 2, 3, 3, 3, 1, 2, 3]) +[1, 2, 3] +``` + +Comprehensions also support the `uniq: true` option. + +### [`uniq_by(enum, fun)`](`Enum.uniq_by/2`) + +Get entries which are unique by the last letter of the fruit: + +```elixir +iex> Enum.uniq_by(cart, &String.last(&1.fruit)) +[ + %{fruit: "apple", count: 3}, + %{fruit: "banana", count: 1} +] +``` + +## Indexing +{: .col-2} + +### [`at(enum, index, default \\ nil)`](`Enum.at/2`) + +```elixir +iex> Enum.at(cart, 0) +%{fruit: "apple", count: 3} +iex> Enum.at(cart, 10) +nil +iex> Enum.at(cart, 10, :none) +:none +``` + +Accessing a list by index in a loop is discouraged. + +### [`fetch(enum, index)`](`Enum.fetch/2`) + +```elixir +iex> Enum.fetch(cart, 0) +{:ok, %{fruit: "apple", count: 3}} +iex> Enum.fetch(cart, 10) +:error +``` + +### [`fetch!(enum, index)`](`Enum.fetch!/2`) + +```elixir +iex> Enum.fetch!(cart, 0) +%{fruit: "apple", count: 3} +iex> Enum.fetch!(cart, 10) +** (Enum.OutOfBoundsError) out of bounds error +``` + +### [`with_index(enum)`](`Enum.with_index/1`) + +```elixir +iex> Enum.with_index(cart) +[ + {%{fruit: "apple", count: 3}, 0}, + {%{fruit: "banana", count: 1}, 1}, + {%{fruit: "orange", count: 6}, 2} +] +``` + +### [`with_index(enum, fun)`](`Enum.with_index/2`) + +```elixir +iex> Enum.with_index(cart, fn item, index -> +...> {item.fruit, index} +...> end) +[ + {"apple", 0}, + {"banana", 1}, + {"orange", 2} +] +``` + +## Finding +{: .col-2} + +### [`find(enum, default \\ nil, fun)`](`Enum.find/2`) + +```elixir +iex> Enum.find(cart, &(&1.fruit =~ "o")) +%{fruit: "orange", count: 6} +iex> Enum.find(cart, &(&1.fruit =~ "y")) +nil +iex> Enum.find(cart, :none, &(&1.fruit =~ "y")) +:none +``` + +### [`find_index(enum, fun)`](`Enum.find_index/2`) + +```elixir +iex> Enum.find_index(cart, &(&1.fruit =~ "o")) +2 +iex> Enum.find_index(cart, &(&1.fruit =~ "y")) +nil +``` + +### [`find_value(enum, default \\ nil, fun)`](`Enum.find_value/2`) + +```elixir +iex> Enum.find_value(cart, fn item -> +...> if item.count == 1, do: item.fruit, else: nil +...> end) +"banana" +iex> Enum.find_value(cart, :none, fn item -> +...> if item.count == 100, do: item.fruit, else: nil +...> end) +:none +``` + +## Grouping +{: .col-2} + +### [`group_by(enum, key_fun)`](`Enum.group_by/2`) + +Group by the last letter of the fruit: + +```elixir +iex> Enum.group_by(cart, &String.last(&1.fruit)) +%{ + "a" => [%{fruit: "banana", count: 1}], + "e" => [ + %{fruit: "apple", count: 3}, + %{fruit: "orange", count: 6} + ] +} +``` + +### [`group_by(enum, key_fun, value_fun)`](`Enum.group_by/3`) + +Group by the last letter of the fruit with custom value: + +```elixir +iex> Enum.group_by(cart, &String.last(&1.fruit), & &1.fruit) +%{ + "a" => ["banana"], + "e" => ["apple", "orange"] +} +``` + +## Joining & interspersing +{: .col-2} + +### [`join(enum, joiner \\ "")`](`Enum.join/2`) + +```elixir +iex> Enum.join(["apple", "banana", "orange"], ", ") +"apple, banana, orange" +``` + +### [`map_join(enum, joiner \\ "", mapper)`](`Enum.map_join/3`) + +```elixir +iex> Enum.map_join(cart, ", ", & &1.fruit) +"apple, banana, orange" +``` + +### [`intersperse(enum, separator \\ "")`](`Enum.intersperse/2`) + +```elixir +iex> Enum.intersperse(["apple", "banana", "orange"], ", ") +["apple", ", ", "banana", ", ", "orange"] +``` + +### [`map_intersperse(enum, separator \\ "", mapper)`](`Enum.map_intersperse/3`) + +```elixir +iex> Enum.map_intersperse(cart, ", ", & &1.fruit) +["apple", ", ", "banana", ", ", "orange"] +``` + +## Slicing +{: .col-2} + +### [`slice(enum, index_range)`](`Enum.slice/2`) + +```elixir +iex> Enum.slice(cart, 0..1) +[ + %{fruit: "apple", count: 3}, + %{fruit: "banana", count: 1} +] +``` + +Negative ranges count from the back: + +```elixir +iex> Enum.slice(cart, -2..-1) +[ + %{fruit: "banana", count: 1}, + %{fruit: "orange", count: 6} +] +``` + +### [`slice(enum, start_index, amount)`](`Enum.slice/3`) + +```elixir +iex> Enum.slice(cart, 1, 2) +[ + %{fruit: "banana", count: 1}, + %{fruit: "orange", count: 6} +] +``` + +### [`slide(enum, range_or_single_index, insertion_index)`](`Enum.slide/3`) + +```elixir +fruits = ["apple", "banana", "grape", "orange", "pear"] +iex> Enum.slide(fruits, 2, 0) +["grape", "apple", "banana", "orange", "pear"] +iex> Enum.slide(fruits, 2, 4) +["apple", "banana", "orange", "pear", "grape"] +iex> Enum.slide(fruits, 1..3, 0) +["banana", "grape", "orange", "apple", "pear"] +iex> Enum.slide(fruits, 1..3, 4) +["apple", "pear", "banana", "grape", "orange"] +``` + +## Reversing +{: .col-2} + +### [`reverse(enum)`](`Enum.reverse/1`) + +```elixir +iex> Enum.reverse(cart) +[ + %{fruit: "orange", count: 6}, + %{fruit: "banana", count: 1}, + %{fruit: "apple", count: 3} +] +``` + +### [`reverse(enum, tail)`](`Enum.reverse/2`) + +```elixir +iex> Enum.reverse(cart, [:this_will_be, :the_tail]) +[ + %{fruit: "orange", count: 6}, + %{fruit: "banana", count: 1}, + %{fruit: "apple", count: 3}, + :this_will_be, + :the_tail +] +``` + +### [`reverse_slice(enum, start_index, count)`](`Enum.reverse_slice/3`) + +```elixir +iex> Enum.reverse_slice(cart, 1, 2) +[ + %{fruit: "apple", count: 3}, + %{fruit: "orange", count: 6}, + %{fruit: "banana", count: 1} +] +``` + +## Splitting +{: .col-2} + +### [`split(enum, amount)`](`Enum.split/2`) + +```elixir +iex> Enum.split(cart, 1) +{[%{fruit: "apple", count: 3}], + [ + %{fruit: "banana", count: 1}, + %{fruit: "orange", count: 6} + ]} +``` + +Negative indexes count from the back: + +```elixir +iex> Enum.split(cart, -1) +{[ + %{fruit: "apple", count: 3}, + %{fruit: "banana", count: 1} + ], + [%{fruit: "orange", count: 6}]} +``` + +### [`split_while(enum, fun)`](`Enum.split_while/2`) + +Stops splitting as soon as it is false: + +```elixir +iex> Enum.split_while(cart, &(&1.fruit =~ "e")) +{[%{fruit: "apple", count: 3}], + [ + %{fruit: "banana", count: 1}, + %{fruit: "orange", count: 6} + ]} +``` + +### [`split_with(enum, fun)`](`Enum.split_with/2`) + +Splits the whole collection: + +```elixir +iex> Enum.split_with(cart, &(&1.fruit =~ "e")) +{[ + %{fruit: "apple", count: 3}, + %{fruit: "orange", count: 6} + ], + [%{fruit: "banana", count: 1}]} +``` + +## Splitting (drop and take) +{: .col-2} + +### [`drop(enum, amount)`](`Enum.drop/2`) + +```elixir +iex> Enum.drop(cart, 1) +[ + %{fruit: "banana", count: 1}, + %{fruit: "orange", count: 6} +] +``` + +Negative indexes count from the back: + +```elixir +iex> Enum.drop(cart, -1) +[ + %{fruit: "apple", count: 3}, + %{fruit: "banana", count: 1} +] +``` + +### [`drop_every(enum, nth)`](`Enum.drop_every/2`) + +```elixir +iex> Enum.drop_every(cart, 2) +[%{fruit: "banana", count: 1}] +``` + +### [`drop_while(enum, fun)`](`Enum.drop_while/2`) + +```elixir +iex> Enum.drop_while(cart, &(&1.fruit =~ "e")) +[ + %{fruit: "banana", count: 1}, + %{fruit: "orange", count: 6} +] +``` + +### [`take(enum, amount)`](`Enum.take/2`) + +```elixir +iex> Enum.take(cart, 1) +[%{fruit: "apple", count: 3}] +``` + +Negative indexes count from the back: + +```elixir +iex> Enum.take(cart, -1) +[%{fruit: "orange", count: 6}] +``` + +### [`take_every(enum, nth)`](`Enum.take_every/2`) + +```elixir +iex> Enum.take_every(cart, 2) +[ + %{fruit: "apple", count: 3}, + %{fruit: "orange", count: 6} +] +``` + +### [`take_while(enum, fun)`](`Enum.take_while/2`) + +```elixir +iex> Enum.take_while(cart, &(&1.fruit =~ "e")) +[%{fruit: "apple", count: 3}] +``` + +## Random +{: .col-2} + +### [`random(enum)`](`Enum.random/1`) + +Results will vary on every call: + +```elixir +iex> Enum.random(cart) +%{fruit: "orange", count: 6} +``` + +### [`take_random(enum, count)`](`Enum.take_random/2`) + +Results will vary on every call: + +```elixir +iex> Enum.take_random(cart, 2) +[ + %{fruit: "orange", count: 6}, + %{fruit: "apple", count: 3} +] +``` + +### [`shuffle(enum)`](`Enum.shuffle/1`) + +Results will vary on every call: + +```elixir +iex> Enum.shuffle(cart) +[ + %{fruit: "orange", count: 6}, + %{fruit: "apple", count: 3}, + %{fruit: "banana", count: 1} +] +``` + +## Chunking +{: .col-2} + +### [`chunk_by(enum, fun)`](`Enum.chunk_by/2`) + +```elixir +iex> Enum.chunk_by(cart, &String.length(&1.fruit)) +[ + [%{fruit: "apple", count: 3}], + [ + %{fruit: "banana", count: 1}, + %{fruit: "orange", count: 6} + ] +] +``` + +### [`chunk_every(enum, count)`](`Enum.chunk_every/2`) + +```elixir +iex> Enum.chunk_every(cart, 2) +[ + [ + %{fruit: "apple", count: 3}, + %{fruit: "banana", count: 1} + ], + [%{fruit: "orange", count: 6}] +] +``` + +### [`chunk_every(enum, count, step, leftover \\ [])`](`Enum.chunk_every/2`) + +```elixir +iex> Enum.chunk_every(cart, 2, 2, [:elements, :to_complete]) +[ + [ + %{fruit: "apple", count: 3}, + %{fruit: "banana", count: 1} + ], + [ + %{fruit: "orange", count: 6}, + :elements + ] +] +iex> Enum.chunk_every(cart, 2, 1, :discard) +[ + [ + %{fruit: "apple", count: 3}, + %{fruit: "banana", count: 1} + ], + [ + %{fruit: "banana", count: 1}, + %{fruit: "orange", count: 6} + ] +] +``` + +See `Enum.chunk_while/4` for custom chunking. + +## Zipping +{: .col-2} + +### [`zip(enum1, enum2)`](`Enum.zip/2`) + +```elixir +iex> fruits = ["apple", "banana", "orange"] +iex> counts = [3, 1, 6] +iex> Enum.zip(fruits, counts) +[{"apple", 3}, {"banana", 1}, {"orange", 6}] +``` + +See `Enum.zip/1` for zipping many collections at once. + +### [`zip_with(enum1, enum2, fun)`](`Enum.zip_with/2`) + +```elixir +iex> fruits = ["apple", "banana", "orange"] +iex> counts = [3, 1, 6] +iex> Enum.zip_with(fruits, counts, fn fruit, count -> +...> %{fruit: fruit, count: count} +...> end) +[ + %{fruit: "apple", count: 3}, + %{fruit: "banana", count: 1}, + %{fruit: "orange", count: 6} +] +``` + +See `Enum.zip_with/2` for zipping many collections at once. + +### [`zip_reduce(left, right, acc, fun)`](`Enum.zip_reduce/4`) + +```elixir +iex> fruits = ["apple", "banana", "orange"] +iex> counts = [3, 1, 6] +iex> Enum.zip_reduce(fruits, counts, 0, fn fruit, count, acc -> +...> price = if fruit =~ "e", do: count * 2, else: count +...> acc + price +...> end) +19 +``` + +See `Enum.zip_reduce/3` for zipping many collections at once. + +### [`unzip(list)`](`Enum.unzip/1`) + +```elixir +iex> cart |> Enum.map(&{&1.fruit, &1.count}) |> Enum.unzip() +{["apple", "banana", "orange"], [3, 1, 6]} +``` diff --git a/lib/elixir/pages/cheatsheets/types-cheat.cheatmd b/lib/elixir/pages/cheatsheets/types-cheat.cheatmd new file mode 100644 index 00000000000..0b60cd1808b --- /dev/null +++ b/lib/elixir/pages/cheatsheets/types-cheat.cheatmd @@ -0,0 +1,188 @@ + + +# Set-theoretic types cheatsheet + +## Set operators + +#### Union + +```elixir +type1 or type2 +``` + +#### Intersection + +```elixir +type1 and type2 +``` + +#### Difference + +```elixir +type1 and not type2 +``` + +#### Negation + +```elixir +not type +``` + +## Data types + +### Indivisible types + +```elixir +binary() +empty_list() +integer() +float() +pid() +port() +reference() +``` + +### Atoms + +#### All atoms + +```elixir +atom() +``` + +#### Individual atoms + +```elixir +:ok +:error +SomeModule +``` + +### Functions + +#### All functions + +```elixir +function() +``` + +#### `n`-arity functions + +```elixir +(-> :ok) +(integer() -> boolean()) +(binary(), binary() -> binary()) +``` + +#### Multiple clauses + +```elixir +(integer() -> binary()) and (binary() -> atom()) +``` + +### Maps + +#### All maps + +```elixir +map() +``` + +#### Empty map + +```elixir +empty_map() +``` + +#### Maps with atom keys + +```elixir +# Only has the keys name and age +%{name: binary(), age: integer()} + +# Has the name key and age is optional +%{name: binary(), age: if_set(integer())} + +# Has the keys name and age and may have other keys (open map) +%{..., name: binary(), age: integer()} + +# Has the key name, may have other keys, but age is not set +%{..., name: binary(), age: not_set()} +``` + +#### Maps with domain keys (domain keys are always treated as optional) + +```elixir +# Has atom and binary keys +%{atom() => binary(), binary() => binary()} + +# Has atom and binary keys and may have other keys (open map) +%{..., atom() => binary(), binary() => binary()} +``` + +#### Maps with mixed keys + +```elixir +# Has atom keys with binary values but a `:root` key of type integer +%{atom() => binary(), root: integer()} + +# Has atom keys with binary values but a `:root` key of type integer, and may have other keys +%{..., atom() => binary(), root: integer()} +``` + +#### Domain keys are `atom()`, `binary()`, `integer()`, `float()`, `fun()`, `list()`, `map()`, `pid()`, `port()`, `reference()`, `tuple()` + +### Non-empty lists + +#### Proper lists + +```elixir +non_empty_list(elem_type) +``` + +#### Improper lists (as long as `tail_type` does not include lists) + +```elixir +non_empty_list(elem_type, tail_type) +``` + +### Tuples + +#### All tuples + +```elixir +tuple() +``` + +#### n-element tuples + +```elixir +{:ok, binary()} +{:error, binary(), term()} +{pid(), reference()} +``` + +#### At least n-element tuples + +``` +{binary(), binary(), ...} +``` + +## Additional types for convenience + +#### Booleans + +```elixir +boolean() = true or false +``` + +#### Lists + +```elixir +list() = empty_list() or non_empty_list(term()) +list(a) = empty_list() or non_empty_list(a) +list(a, b) = empty_list() or non_empty_list(a, b) +``` \ No newline at end of file diff --git a/lib/elixir/pages/getting-started/alias-require-and-import.md b/lib/elixir/pages/getting-started/alias-require-and-import.md new file mode 100644 index 00000000000..31d1564960e --- /dev/null +++ b/lib/elixir/pages/getting-started/alias-require-and-import.md @@ -0,0 +1,167 @@ + + +# alias, require, import, and use + +In order to facilitate software reuse, Elixir provides three directives (`alias`, `require`, and `import`) plus a macro called `use` summarized below: + +```elixir +# Alias the module so it can be called as Bar instead of Foo.Bar +alias Foo.Bar, as: Bar + +# Require the module in order to use its macros +require Foo + +# Import functions from Foo so they can be called without the `Foo.` prefix +import Foo + +# Invokes the custom code defined in Foo as an extension point +use Foo +``` + +We are going to explore them in detail now. Keep in mind the first three are called directives because they have *lexical scope*, while `use` is a common extension point that allows the used module to inject code. + +## alias + +`alias` allows you to set up aliases for any given module name. + +Imagine a module uses a specialized list implemented in `Math.List`. The `alias` directive allows referring to `Math.List` just as `List` within the module definition: + +```elixir +defmodule Stats do + alias Math.List, as: List + # In the remaining module definition List expands to Math.List. +end +``` + +The original `List` can still be accessed within `Stats` by the fully-qualified name `Elixir.List`. + +> All modules defined in Elixir are defined inside the main `Elixir` namespace, such as `Elixir.String`. However, for convenience, you can omit "Elixir." when referencing them. + +Aliases are frequently used to define shortcuts. In fact, calling `alias` without an `:as` option sets the alias automatically to the last part of the module name, for example: + +```elixir +alias Math.List +``` + +Is the same as: + +```elixir +alias Math.List, as: List +``` + +Note that `alias` is *lexically scoped*, which allows you to set aliases inside specific functions: + +```elixir +defmodule Math do + def plus(a, b) do + alias Math.List + # ... + end + + def minus(a, b) do + # ... + end +end +``` + +In the example above, since we are invoking `alias` inside the function `plus/2`, the alias will be valid only inside the function `plus/2`. `minus/2` won't be affected at all. + +## require + +Elixir provides macros as a mechanism for meta-programming (writing code that generates code). Macros are expanded at compile time. + +Public functions in modules are globally available, but in order to use macros, you need to opt-in by requiring the module they are defined in. + +```elixir +iex> Integer.is_odd(3) +** (UndefinedFunctionError) function Integer.is_odd/1 is undefined or private. However, there is a macro with the same name and arity. Be sure to require Integer if you intend to invoke this macro + (elixir) Integer.is_odd(3) +iex> require Integer +Integer +iex> Integer.is_odd(3) +true +``` + +In Elixir, `Integer.is_odd/1` is defined as a macro so that it can be used as a guard. This means that, in order to invoke `Integer.is_odd/1`, we need to first require the `Integer` module. + +Note that like the `alias` directive, `require` is also lexically scoped. We will talk more about macros in a later chapter. + +## import + +We use `import` whenever we want to access functions or macros from other modules without using the fully-qualified name. Note we can only import public functions, as private functions are never accessible externally. + +For example, if we want to use the `duplicate/2` function from the `List` module several times, we can import it: + +```elixir +iex> import List, only: [duplicate: 2] +List +iex> duplicate(:ok, 3) +[:ok, :ok, :ok] +``` + +We imported only the function `duplicate` (with arity 2) from `List`. Although `:only` is optional, its usage is recommended in order to avoid importing all the functions of a given module inside the current scope. `:except` could also be given as an option in order to import everything in a module except a list of functions. + +Note that `import` is *lexically scoped* too. This means that we can import specific macros or functions inside function definitions: + +```elixir +defmodule Math do + def some_function do + import List, only: [duplicate: 2] + duplicate(:ok, 10) + end +end +``` + +In the example above, the imported `List.duplicate/2` is only visible within that specific function. `duplicate/2` won't be available in any other function in that module (or any other module for that matter). + +While `import`s can be useful for frameworks and libraries to build abstractions, developers should generally prefer `alias` to `import` on their own codebases, as aliases make the origin of the function being invoked clearer. + +## use + +The `use` macro is frequently used as an extension point. This means that, when you `use` a module `FooBar`, you allow that module to inject *any* code in the current module, such as importing itself or other modules, defining new functions, setting a module state, etc. + +For example, in order to write tests using the ExUnit framework, a developer should use the `ExUnit.Case` module: + +```elixir +defmodule AssertionTest do + use ExUnit.Case, async: true + + test "always pass" do + assert true + end +end +``` + +Behind the scenes, `use` requires the given module and then calls the `__using__/1` callback on it allowing the module to inject some code into the current context. Some modules (for example, the above `ExUnit.Case`, but also `Supervisor` and `GenServer`) use this mechanism to populate your module with some basic behaviour, which your module is intended to override or complete. + +Generally speaking, the following module: + +```elixir +defmodule Example do + use Feature, option: :value +end +``` + +is compiled into + +```elixir +defmodule Example do + require Feature + Feature.__using__(option: :value) +end +``` + +Since `use` allows any code to run, we can't really know the side-effects of using a module without reading its documentation. Therefore use this function with care and only if strictly required. Don't use `use` where an `import` or `alias` would do. + +## Multi alias/import/require/use + +It is possible to `alias`, `import`, `require`, or `use` multiple modules at once. This is particularly useful once we start nesting modules, which is very common when building Elixir applications. For example, imagine you have an application where all modules are nested under `MyApp`, you can alias the modules `MyApp.Foo`, `MyApp.Bar` and `MyApp.Baz` at once as follows: + +```elixir +alias MyApp.{Foo, Bar, Baz} +``` + +With this, we have finished our tour of Elixir modules. diff --git a/lib/elixir/pages/getting-started/anonymous-functions.md b/lib/elixir/pages/getting-started/anonymous-functions.md new file mode 100644 index 00000000000..3192b4b7ee5 --- /dev/null +++ b/lib/elixir/pages/getting-started/anonymous-functions.md @@ -0,0 +1,186 @@ + + +# Anonymous functions + +Anonymous functions allow us to store and pass executable code around as if it was an integer or a string. Let's learn more. + +## Identifying functions and documentation + +Before we move on to discuss anonymous functions, let's talk about how Elixir identifies named functions – the functions defined in [modules](modules-and-functions.md). + +Functions in Elixir are identified by both their name and their arity. The arity of a function describes the number of arguments that the function takes. From this point on we will use both the function name and its arity to describe functions throughout the documentation. `trunc/1` identifies the function which is named `trunc` and takes `1` argument, whereas `trunc/2` identifies a different (nonexistent) function with the same name but with an arity of `2`. + +We can also use this syntax to access documentation. The Elixir shell defines the [`h`](`IEx.Helpers.h/1`) function, which you can use to access documentation for any function. For example, typing `h trunc/1` is going to print the documentation for the `trunc/1` function: + +```elixir +iex> h trunc/1 + def trunc(number) + +Returns the integer part of number. +``` + +`h trunc/1` works because it is defined in the `Kernel` module. All functions in the `Kernel` module are automatically imported into our namespace. Most often you will also include the module name when looking up the documentation for a given function: + +```elixir +iex> h Kernel.trunc/1 + def trunc(number) + +Returns the integer part of number. +``` + +You can use the module+function identifiers to lookup documentation for anything, including operators (try `h Kernel.+/2`). Invoking [`h`](`IEx.Helpers.h/1`) without arguments displays the documentation for `IEx.Helpers`, which is where `h` and other functionalities are defined. + +## Defining anonymous functions + +Anonymous functions in Elixir are delimited by the keywords `fn` and `end`: + +```elixir +iex> add = fn a, b -> a + b end +#Function<12.71889879/2 in :erl_eval.expr/5> +``` + +In the example above, we defined an anonymous function that receives two arguments, `a` and `b`, and returns the result of `a + b`. The arguments are always on the left-hand side of `->` and the code to be executed on the right-hand side. The anonymous function is stored in the variable `add`. You can see it returns a value represented by `#Function<...>`. While its representation is opaque, the `:erl_eval.expr` bit tells us the function was defined in the shell (during evaluation). + +We can invoke anonymous functions by passing arguments to it, using a dot (`.`) between the variable and the opening parenthesis: + +```elixir +iex> add.(1, 2) +3 +``` + +The dot makes it clear when you are calling an anonymous function, stored in the variable `add`, opposed to a function named `add/2`. For example, if you have an anonymous function stored in the variable `is_atom`, there is no ambiguity between `is_atom.(:foo)` and `is_atom(:foo)`. If both used the same `is_atom(:foo)` syntax, the only way to know the actual behavior of `is_atom(:foo)` would be by scanning all code thus far for a possible definition of the `is_atom` variable. This scanning hurts maintainability as it requires developers to track additional context in their head when reading and writing code. + +Anonymous functions in Elixir are also identified by the number of arguments they receive. We can check if a value is a function using `is_function/1` and also check its arity by using `is_function/2`: + +```elixir +iex> is_function(add) +true +# check if add is a function that expects exactly 2 arguments +iex> is_function(add, 2) +true +# check if add is a function that expects exactly 1 argument +iex> is_function(add, 1) +false +``` + +## Closures + +Anonymous functions can also access variables that are in scope when the function is defined. This is typically referred to as closures, as they close over their scope. Let's define a new anonymous function that uses the `add` anonymous function we have previously defined: + +```elixir +iex> double = fn a -> add.(a, a) end +#Function<6.71889879/1 in :erl_eval.expr/5> +iex> double.(2) +4 +``` + +A variable assigned inside a function does not affect its surrounding environment: + +```elixir +iex> x = 42 +42 +iex> (fn -> x = 0 end).() +0 +iex> x +42 +``` + +## Clauses and guards + +Similar to `case/2`, we can pattern match on the arguments of anonymous functions as well as define multiple clauses and guards: + +```elixir +iex> f = fn +...> x, y when x > 0 -> x + y +...> x, y -> x * y +...> end +#Function<12.71889879/2 in :erl_eval.expr/5> +iex> f.(1, 3) +4 +iex> f.(-1, 3) +-3 +``` + +The number of arguments in each anonymous function clause needs to be the same, otherwise an error is raised. + +```elixir +iex> f2 = fn +...> x, y when x > 0 -> x + y +...> x, y, z -> x * y + z +...> end +** (CompileError) iex:1: cannot mix clauses with different arities in anonymous functions +``` + +## The capture operator + +Throughout this guide, we have been using the notation `name/arity` to refer to functions. It happens that this notation can actually be used to capture an existing function into a data-type we can pass around, similar to how anonymous functions behave. + +```elixir +iex> fun = &is_atom/1 +&:erlang.is_atom/1 +iex> is_function(fun) +true +iex> fun.(:hello) +true +iex> fun.(123) +false +``` + +As you can see, once a function is captured, we can pass it as argument or invoke it using the anonymous function notation. The returned value above also hints we can capture functions defined in modules: + +```elixir +iex> fun = &String.length/1 +&String.length/1 +iex> fun.("hello") +5 +``` + +Since operators are functions in Elixir, you can also capture operators: + +```elixir +iex> add = &+/2 +&:erlang.+/2 +iex> add.(1, 2) +3 +``` + +The capture syntax can also be used as a shortcut for creating functions that wrap existing functions. For example, imagine you want to create an anonymous function that checks if a given function has arity 2. You could write it as: + +```elixir +iex> is_arity_2 = fn fun -> is_function(fun, 2) end +#Function<8.71889879/1 in :erl_eval.expr/5> +iex> is_arity_2.(add) +true +``` + +But using the capture syntax, you can write it as: + +```elixir +iex> is_arity_2 = &is_function(&1, 2) +#Function<8.71889879/1 in :erl_eval.expr/5> +iex> is_arity_2.(add) +true +``` + +The `&1` represents the first argument passed into the function. Therefore both `is_arity_2` anonymous functions defined above are equivalent. + +Once again, given operators are function calls, the capture syntax shorthand also works with operators, or even string interpolation: + +```elixir +iex> fun = &(&1 + 1) +#Function<6.71889879/1 in :erl_eval.expr/5> +iex> fun.(1) +2 + +iex> fun2 = &"Good #{&1}" +#Function<6.127694169/1 in :erl_eval.expr/5> +iex> fun2.("morning") +"Good morning" +``` + +`&(&1 + 1)` above is exactly the same as `fn x -> x + 1 end`. You can read more about the capture operator `&` in [its documentation](`&/1`). + +Next let's revisit some of the data-types we learned in the past and dig deeper into how they work. diff --git a/lib/elixir/pages/getting-started/basic-types.md b/lib/elixir/pages/getting-started/basic-types.md new file mode 100644 index 00000000000..b446b3bd65e --- /dev/null +++ b/lib/elixir/pages/getting-started/basic-types.md @@ -0,0 +1,313 @@ + + +# Basic types + +In this chapter we will learn more about Elixir basic types: integers, floats, booleans, atoms, and strings. Other data types, such as lists and tuples, will be explored in the next chapter. + +```elixir +iex> 1 # integer +iex> 0x1F # integer +iex> 1.0 # float +iex> true # boolean +iex> :atom # atom / symbol +iex> "elixir" # string +iex> [1, 2, 3] # list +iex> {1, 2, 3} # tuple +``` + +## Basic arithmetic + +Open up `iex` and type the following expressions: + +```elixir +iex> 1 + 2 +3 +iex> 5 * 5 +25 +iex> 10 / 2 +5.0 +``` + +Notice that `10 / 2` returned a float `5.0` instead of an integer `5`. This is expected. In Elixir, the operator [`/`](`//2`) always returns a float. If you want to do integer division or get the division remainder, you can invoke the [`div`](`div/2`) and [`rem`](`rem/2`) functions: + +```elixir +iex> div(10, 2) +5 +iex> div 10, 2 +5 +iex> rem 10, 3 +1 +``` + +Notice that Elixir allows you to drop the parentheses when invoking functions that expect one or more arguments. This feature gives a cleaner syntax when writing declarations and control-flow constructs. However, Elixir developers generally prefer to use parentheses. + +Elixir also supports shortcut notations for entering binary, octal, and hexadecimal numbers: + +```elixir +iex> 0b1010 +10 +iex> 0o777 +511 +iex> 0x1F +31 +``` + +Float numbers require a dot followed by at least one digit and also support `e` for scientific notation: + +```elixir +iex> 1.0 +1.0 +iex> 1.0e-10 +1.0e-10 +``` + +Floats in Elixir are 64-bit precision. + +You can invoke the [`round`](`round/1`) function to get the closest integer to a given float, or the [`trunc`](`trunc/1`) function to get the integer part of a float. + +```elixir +iex> round(3.58) +4 +iex> trunc(3.58) +3 +``` + +Finally, we work with different data types, we will learn Elixir provides several predicate functions to check for the type of a value. For example, [`is_integer`](`is_integer/1`) can be used to check if a value is an integer or not: + +```elixir +iex> is_integer(1) +true +iex> is_integer(2.0) +false +``` + +You can also use [`is_float`](`is_float/1`) or [`is_number`](`is_number/1`) to check, respectively, if an argument is a float, or either an integer or float. + +## Booleans and `nil` + +Elixir supports `true` and `false` as booleans: + +```elixir +iex> true +true +iex> true == false +false +``` + +Elixir also provides three boolean operators: [`or`](`or/2`), [`and`](`and/2`), and [`not`](`not/1`). These operators are strict in the sense that they expect something that evaluates to a boolean (`true` or `false`) as their first argument: + +```elixir +iex> true and true +true +iex> false or is_boolean(true) +true +``` + +Providing a non-boolean will raise an exception: + +```elixir +iex> 1 and true +** (BadBooleanError) expected a boolean on left-side of "and", got: 1 +``` + +`or` and `and` are short-circuit operators. They only execute the right side if the left side is not enough to determine the result: + +```elixir +iex> false and raise("This error will never be raised") +false +iex> true or raise("This error will never be raised") +true +``` + +Elixir also provides the concept of `nil`, to indicate the absence of a value, and a set of logical operators that also manipulate `nil`: `||/2`, `&&/2`, and `!/1`. For these operators, `false` and `nil` are considered "falsy", all other values are considered "truthy": + +```elixir +# or +iex> 1 || true +1 +iex> false || 11 +11 + +# and +iex> nil && 13 +nil +iex> true && 17 +17 + +# not +iex> !true +false +iex> !1 +false +iex> !nil +true +``` + +Similarly, values like `0` and `""`, which some other programming languages consider to be "falsy", are also "truthy" in Elixir. + +As a rule of thumb, use `and`, `or` and `not` when you are expecting booleans. If any of the arguments are non-boolean, use `&&`, `||` and `!`. + +## Atoms + +An atom is a constant whose value is its own name. Some other languages call these symbols. They are often useful to enumerate over distinct values, such as: + +```elixir +iex> :apple +:apple +iex> :orange +:orange +iex> :watermelon +:watermelon +``` + +Atoms are equal if their names are equal. + +```elixir +iex> :apple == :apple +true +iex> :apple == :orange +false +``` + +Often they are used to express the state of an operation, by using values such as `:ok` and `:error`. + +The booleans `true` and `false` are also atoms: + +```elixir +iex> true == :true +true +iex> is_atom(false) +true +iex> is_boolean(:false) +true +``` + +Elixir allows you to skip the leading `:` for the atoms `false`, `true` and `nil`. + +## Strings + +Strings in Elixir are delimited by double quotes, and they are encoded in UTF-8: + +```elixir +iex> "hellö" +"hellö" +``` + +> Note: if you are running on Windows, there is a chance your terminal does not use UTF-8 by default. You can change the encoding of your current session by running `chcp 65001` before entering IEx. + +You can concatenate two strings with the [`<>`](`<>/2`) operator: + +```elixir +iex> "hello " <> "world!" +"hello world!" +``` + +Elixir also supports string interpolation: + +```elixir +iex> string = "world" +iex> "hello #{string}!" +"hello world!" +``` + +String concatenation requires both sides to be strings but interpolation supports any data type that may be converted to a string: + +```elixir +iex> number = 42 +iex> "i am #{number} years old!" +"i am 42 years old!" +``` + +Strings can have line breaks in them. You can introduce them using escape sequences: + +```elixir +iex> "hello +...> world" +"hello\nworld" +iex> "hello\nworld" +"hello\nworld" +``` + +You can print a string using the [`IO.puts`](`IO.puts/1`) function from the `IO` module: + +```elixir +iex> IO.puts("hello\nworld") +hello +world +:ok +``` + +Notice that the [`IO.puts`](`IO.puts/1`) function returns the atom `:ok` after printing. + +Strings in Elixir are represented internally by contiguous sequences of bytes known as binaries: + +```elixir +iex> is_binary("hellö") +true +``` + +We can also get the number of bytes in a string: + +```elixir +iex> byte_size("hellö") +6 +``` + +Notice that the number of bytes in that string is 6, even though it has 5 graphemes. That's because the grapheme "ö" takes 2 bytes to be represented in UTF-8. We can get the actual length of the string, based on the number of graphemes, by using the [`String.length`](`String.length/1`) function: + +```elixir +iex> String.length("hellö") +5 +``` + +The `String` module contains a bunch of functions that operate on strings as defined in the Unicode standard: + +```elixir +iex> String.upcase("hellö") +"HELLÖ" +``` + +## Structural comparison + +Elixir also provides [`==`](`==/2`), [`!=`](`!=/2`), [`<=`](`<=/2`), [`>=`](`>=/2`), [`<`](``](`>/2`) as comparison operators. We can compare numbers: + +```elixir +iex> 1 == 1 +true +iex> 1 != 2 +true +iex> 1 < 2 +true +``` + +But also atoms, strings, booleans, etc: + +```elixir +iex> "foo" == "foo" +true +iex> "foo" == "bar" +false +``` + +Integers and floats compare the same if they have the same value: + +```elixir +iex> 1 == 1.0 +true +iex> 1 == 2.0 +false +``` + +However, you can use the strict comparison operator [`===`](`===/2`) and [`!==`](`!==/2`) if you want to distinguish between integers and floats: + +```elixir +iex> 1 === 1.0 +false +``` + +The comparison operators in Elixir can compare across any data type. We say these operators perform _structural comparison_. For more information, you can read our documentation on [Structural vs Semantic comparisons](`Kernel#module-structural-comparison`). + +Elixir also provides data-types for expressing collections, such as lists and tuples, which we learn next. When we talk about concurrency and fault-tolerance via processes, we will also discuss ports, pids, and references, but that will come on later chapters. Let's move forward. diff --git a/lib/elixir/pages/getting-started/binaries-strings-and-charlists.md b/lib/elixir/pages/getting-started/binaries-strings-and-charlists.md new file mode 100644 index 00000000000..c6ab4919973 --- /dev/null +++ b/lib/elixir/pages/getting-started/binaries-strings-and-charlists.md @@ -0,0 +1,309 @@ + + +# Binaries, strings, and charlists + +In ["Basic types"](basic-types.md), we learned a bit about strings and we used the `is_binary/1` function for checks: + +```elixir +iex> string = "hello" +"hello" +iex> is_binary(string) +true +``` + +In this chapter, we will gain clarity on what exactly binaries are and how they relate to strings. We will also learn about charlists, `~c"like this"`, which are often used for interoperability with Erlang. + +Although strings are one of the most common data types in computer languages, they are subtly complex and are often misunderstood. To understand strings in Elixir, let's first discuss [Unicode](https://en.wikipedia.org/wiki/Unicode) and character encodings, specifically the [UTF-8](https://en.wikipedia.org/wiki/UTF-8) encoding. + +## Unicode and Code Points + +In order to facilitate meaningful communication between computers across multiple languages, a standard is required so that the ones and zeros on one machine mean the same thing when they are transmitted to another. The [Unicode Standard](https://unicode.org/standard/standard.html) acts as an official registry of virtually all the characters we know: this includes characters from classical and historical texts, emoji, and formatting and control characters as well. + +Unicode organizes all of the characters in its repertoire into code charts, and each character is given a unique numerical index. This numerical index is known as a [Code Point](https://en.wikipedia.org/wiki/Code_point). + +In Elixir you can use a `?` in front of a character literal to reveal its code point: + +```elixir +iex> ?a +97 +iex> ?ł +322 +``` + +Note that most Unicode code charts will refer to a code point by its hexadecimal (hex) representation, e.g. `97` translates to `0061` in hex, and we can represent any Unicode character in an Elixir string by using the `\uXXXX` notation and the hex representation of its code point number: + +```elixir +iex> "\u0061" == "a" +true +iex> 0x0061 = 97 = ?a +97 +``` + +The hex representation will also help you look up information about a code point, e.g. [https://codepoints.net/U+0061](https://codepoints.net/U+0061) has a data sheet all about the lower case `a`, a.k.a. code point 97. + +## UTF-8 and Encodings + +Now that we understand what the Unicode standard is and what code points are, we can finally talk about encodings. Whereas the code point is **what** we store, an encoding deals with **how** we store it: encoding is an implementation. In other words, we need a mechanism to convert the code point numbers into bytes so they can be stored in memory, written to disk, etc. + +Elixir uses UTF-8 to encode its strings, which means that code points are encoded as a series of 8-bit bytes. UTF-8 is a **variable width** character encoding that uses one to four bytes to store each code point. It is capable of encoding all valid Unicode code points. Let's see an example: + +```elixir +iex> string = "héllo" +"héllo" +iex> String.length(string) +5 +iex> byte_size(string) +6 +``` + +Although the string above has 5 characters, it uses 6 bytes, as two bytes are used to represent the character `é`. + +> Note: if you are running on Windows, there is a chance your terminal does not use UTF-8 by default. You can change the encoding of your current session by running `chcp 65001` before entering `iex` (`iex.bat`). + +Besides defining characters, UTF-8 also provides a notion of graphemes. Graphemes may consist of multiple characters that are often perceived as one. For example, the [woman firefighter emoji](https://emojipedia.org/woman-firefighter/) is represented as the combination of three characters: the woman emoji (👩), a hidden zero-width joiner, and the fire engine emoji (🚒): + +```elixir +iex> String.codepoints("👩‍🚒") +["👩", "‍", "🚒"] +iex> String.graphemes("👩‍🚒") +["👩‍🚒"] +``` + +However, Elixir is smart enough to know they are seen as a single character, and therefore the length is still one: + +```elixir +iex> String.length("👩‍🚒") +1 +``` + +> Note: if you can't see the emoji above in your terminal, you need to make sure your terminal supports emoji and that you are using a font that can render them. + +Although these rules may sound complicated, UTF-8 encoded documents are everywhere. This page itself is encoded in UTF-8. The encoding information is given to your browser which then knows how to render all of the bytes, characters, and graphemes accordingly. + +If you want to see the exact bytes that a string would be stored in a file, a common trick is to concatenate the null byte `<<0>>` to it: + +```elixir +iex> "hełło" <> <<0>> +<<104, 101, 197, 130, 197, 130, 111, 0>> +``` + +Alternatively, you can view a string's binary representation by using `IO.inspect/2`: + +```elixir +iex> IO.inspect("hełło", binaries: :as_binaries) +<<104, 101, 197, 130, 197, 130, 111>> +``` + +We are getting a little bit ahead of ourselves. Let's talk about bitstrings to learn about what exactly the `<<>>` constructor means. + +## Bitstrings + +Although we have covered code points and UTF-8 encoding, we still need to go a bit deeper into how exactly we store the encoded bytes, and this is where we introduce the **bitstring**. A bitstring is a fundamental data type in Elixir, denoted with the [`<<>>`](`<<>>/1`) syntax. **A bitstring is a contiguous sequence of bits in memory.** + +By default, 8 bits (i.e. 1 byte) is used to store each number in a bitstring, but you can manually specify the number of bits via a `::n` modifier to denote the size in `n` bits, or you can use the more verbose declaration `::size(n)`: + +```elixir +iex> <<42>> == <<42::8>> +true +iex> <<3::4>> +<<3::size(4)>> +``` + +For example, the decimal number `3` when represented with 4 bits in base 2 would be `0011`, which is equivalent to the values `0`, `0`, `1`, `1`, each stored using 1 bit: + +```elixir +iex> <<0::1, 0::1, 1::1, 1::1>> == <<3::4>> +true +``` + +Any value that exceeds what can be stored by the number of bits provisioned is truncated: + +```elixir +iex> <<1>> == <<257>> +true +``` + +Here, 257 in base 2 would be represented as `100000001`, but since we have reserved only 8 bits for its representation (by default), the left-most bit is ignored and the value becomes truncated to `00000001`, or simply `1` in decimal. + +A complete reference for the bitstring constructor can be found in [`<<>>`](`<<>>/1`)'s documentation. + +## Binaries + +**A binary is a bitstring where the number of bits is divisible by 8.** That means that every binary is a bitstring, but not every bitstring is a binary. We can use the `is_bitstring/1` and `is_binary/1` functions to demonstrate this. + +```elixir +iex> is_bitstring(<<3::4>>) +true +iex> is_binary(<<3::4>>) +false +iex> is_bitstring(<<0, 255, 42>>) +true +iex> is_binary(<<0, 255, 42>>) +true +iex> is_binary(<<42::16>>) +true +``` + +We can pattern match on binaries / bitstrings: + +```elixir +iex> <<0, 1, x>> = <<0, 1, 2>> +<<0, 1, 2>> +iex> x +2 +iex> <<0, 1, x>> = <<0, 1, 2, 3>> +** (MatchError) no match of right hand side value: <<0, 1, 2, 3>> +``` + +Note that unless you explicitly use `::` modifiers, each entry in the binary pattern is expected to match a single byte (exactly 8 bits). If we want to match on a binary of unknown size, we can use the `binary` modifier at the end of the pattern: + +```elixir +iex> <<0, 1, x::binary>> = <<0, 1, 2, 3>> +<<0, 1, 2, 3>> +iex> x +<<2, 3>> +``` + +There are a couple other modifiers that can be useful when doing pattern matches on binaries. The `binary-size(n)` modifier will match `n` bytes in a binary: + +```elixir +iex> <> = <<0, 1, 2, 3>> +<<0, 1, 2, 3>> +iex> head +<<0, 1>> +iex> rest +<<2, 3>> +``` + +**A string is a UTF-8 encoded binary**, where the code point for each character is encoded using 1 to 4 bytes. Thus every string is a binary, but due to the UTF-8 standard encoding rules, not every binary is a valid string. + +```elixir +iex> is_binary("hello") +true +iex> is_binary(<<239, 191, 19>>) +true +iex> String.valid?(<<239, 191, 19>>) +false +``` + +The string concatenation operator [`<>`](`<>/2`) is actually a binary concatenation operator: + +```elixir +iex> "a" <> "ha" +"aha" +iex> <<0, 1>> <> <<2, 3>> +<<0, 1, 2, 3>> +``` + +Given that strings are binaries, we can also pattern match on strings: + +```elixir +iex> <> = "banana" +"banana" +iex> head == ?b +true +iex> rest +"anana" +``` + +However, remember that binary pattern matching works on *bytes*, so matching on the string like "über" with multibyte characters won't match on the *character*, it will match on the *first byte of that character*: + +```elixir +iex> "ü" <> <<0>> +<<195, 188, 0>> +iex> <> = "über" +"über" +iex> x == ?ü +false +iex> rest +<<188, 98, 101, 114>> +``` + +Above, `x` matched on only the first byte of the multibyte `ü` character. + +Therefore, when pattern matching on strings, it is important to use the `utf8` modifier: + +```elixir +iex> <> = "über" +"über" +iex> x == ?ü +true +iex> rest +"ber" +``` + +## Charlists + +Our tour of our bitstrings, binaries, and strings is nearly complete, but we have one more data type to explain: the charlist. + +**A charlist is a list of integers where all the integers are valid code points.** In practice, you will not come across them often, only in specific scenarios such as interfacing with older Erlang libraries that do not accept binaries as arguments. + +```elixir +iex> ~c"hello" +~c"hello" +iex> [?h, ?e, ?l, ?l, ?o] +~c"hello" +``` + +The [`~c`](`Kernel.sigil_c/2`) sigil (we'll cover sigils later in the ["Sigils"](sigils.md) chapter) indicates the fact that we are dealing with a charlist and not a regular string. + +Instead of containing bytes, a charlist contains integer code points. However, the list is only printed as a sigil if all code points are within the ASCII range: + +```elixir +iex> ~c"hełło" +[104, 101, 322, 322, 111] +iex> is_list(~c"hełło") +true +``` + +This is done to ease interoperability with Erlang, even though it may lead to some surprising behavior. For example, if you are storing a list of integers that happen to range between 0 and 127, by default IEx will interpret this as a charlist and it will display the corresponding ASCII characters. + +```elixir +iex> heartbeats_per_minute = [99, 97, 116] +~c"cat" +``` + +You can always force charlists to be printed in their list representation by calling the `inspect/2` function: + +```elixir +iex> inspect(heartbeats_per_minute, charlists: :as_list) +"[99, 97, 116]" +``` + +Furthermore, you can convert a charlist to a string and back by using the `to_string/1` and `to_charlist/1`: + +```elixir +iex> to_charlist("hełło") +[104, 101, 322, 322, 111] +iex> to_string(~c"hełło") +"hełło" +iex> to_string(:hello) +"hello" +iex> to_string(1) +"1" +``` + +The functions above are polymorphic, in other words, they accept many shapes: not only do they convert charlists to strings (and vice-versa), they can also convert integers, atoms, and so on. + +String (binary) concatenation uses the [`<>`](`<>/2`) operator but charlists, being lists, use the list concatenation operator [`++`](`++/2`): + +```elixir +iex> ~c"this " <> ~c"fails" +** (ArgumentError) expected binary argument in <> operator but got: ~c"this " + (elixir) lib/kernel.ex:1821: Kernel.wrap_concatenation/3 + (elixir) lib/kernel.ex:1808: Kernel.extract_concatenations/2 + (elixir) expanding macro: Kernel.<>/2 + iex:1: (file) +iex> ~c"this " ++ ~c"works" +~c"this works" +iex> "he" ++ "llo" +** (ArgumentError) argument error + :erlang.++("he", "llo") +iex> "he" <> "llo" +"hello" +``` + +With binaries, strings, and charlists out of the way, it is time to talk about key-value data structures. diff --git a/lib/elixir/pages/getting-started/case-cond-and-if.md b/lib/elixir/pages/getting-started/case-cond-and-if.md new file mode 100644 index 00000000000..9c354105e5c --- /dev/null +++ b/lib/elixir/pages/getting-started/case-cond-and-if.md @@ -0,0 +1,186 @@ + + +# case, cond, and if + +In this chapter, we will learn about the [`case`](`case/2`), [`cond`](`cond/1`), and [`if`](`if/2`) control flow structures. + +## case + +[`case`](`case/2`) allows us to compare a value against many patterns until we find a matching one: + +```elixir +iex> case {1, 2, 3} do +...> {4, 5, 6} -> +...> "This clause won't match" +...> {1, x, 3} -> +...> "This clause will match and bind x to 2 in this clause" +...> _ -> +...> "This clause would match any value" +...> end +"This clause will match and bind x to 2 in this clause" +``` + +If you want to pattern match against an existing variable, you need to use the [`^`](`^/1`) operator: + +```elixir +iex> x = 1 +1 +iex> case 10 do +...> ^x -> "Won't match" +...> _ -> "Will match" +...> end +"Will match" +``` + +Clauses also allow extra conditions to be specified via guards: + +```elixir +iex> case {1, 2, 3} do +...> {1, x, 3} when x > 0 -> +...> "Will match" +...> _ -> +...> "Would match, if guard condition were not satisfied" +...> end +"Will match" +``` + +The first clause above will only match when `x` is positive. + +Keep in mind errors in guards do not leak but simply make the guard fail: + +```elixir +iex> hd(1) +** (ArgumentError) argument error +iex> case 1 do +...> x when hd(x) -> "Won't match" +...> x -> "Got #{x}" +...> end +"Got 1" +``` + +If none of the clauses match, an error is raised: + +```elixir +iex> case :ok do +...> :error -> "Won't match" +...> end +** (CaseClauseError) no case clause matching: :ok +``` + +The documentation for the `Kernel` module lists all available guards in its sidebar. You can also consult the complete [Patterns and Guards](../references/patterns-and-guards.md#guards) reference for in-depth documentation. + +## if + +[`case`](`case/2`) builds on pattern matching and guards to destructure and match on certain conditions. However, patterns and guards are limited only to certain expressions which are optimized by the compiler. In many situations, you need to write conditions that go beyond what can be expressed with [`case`](`case/2`). For those, [`if`](`if/2`) is a useful alternative: + +```elixir +iex> if true do +...> "This works!" +...> end +"This works!" +iex> if false do +...> "This will never be seen" +...> end +nil +``` + +If the condition given to [`if`](`if/2`) returns `false` or `nil`, the body given between `do`-`end` is not executed and instead it returns `nil`. + +[`if`](`if/2`) also supports `else` blocks: + +```elixir +iex> if nil do +...> "This won't be seen" +...> else +...> "This will" +...> end +"This will" +``` + +### Expressions + +Some programming languages make a distinction about expressions (code that returns a value) and statements (code that returns no value). In Elixir, there are only expressions, no statements. Everything you write in Elixir language returns some value. + +This property allows variables to be scoped to individual blocks of code such as [`if`](`if/2`), [`case`](`case/2`), where declarations or changes are only visible inside the block. A change can't leak to outer blocks, which makes code easier to follow and understand. For example: + +```elixir +iex> x = 1 +1 +iex> if true do +...> x = x + 1 +...> end +2 +iex> x +1 +``` + +You see the return value of the [`if`](`if/2`) expression as the resulting `2` here. To retain changes made within the [`if`](`if/2`) expression on the outer block you need to assign the returned value to a variable in the outer block. + +```elixir +iex> x = 1 +1 +iex> x = +...> if true do +...> x + 1 +...> else +...> x +...> end +2 +``` + +With all expressions returning a value there's also no need for alternative constructs, such as ternary operators posing as an alternative to [`if`](`if/2`). Elixir does include an inline notation for [`if`](`if/2`) and, as we will [learn later](keywords-and-maps.md#do-blocks-and-keywords), it is a syntactic variation on `if`'s arguments. + +> #### `if` is a macro {: .info} +> +> An interesting note regarding [`if`](`if/2`) is that it is implemented as a macro in the language: it isn't a special language construct as it would be in many languages. You can check the documentation and its source for more information. + +If you find yourself nesting several [`if`](`if/2`) blocks, you may want to consider using [`cond`](`cond/1`) instead. Let's check it out. + +## cond + +We have used `case` to find a matching clause from many patterns. We have used `if` to check for a single condition. If you need to check across several conditions and find the first one that does not evaluate to `nil` or `false`, [`cond`](`cond/1`) is a useful construct: + +```elixir +iex> cond do +...> 2 + 2 == 5 -> +...> "This will not be true" +...> 2 * 2 == 3 -> +...> "Nor this" +...> 1 + 1 == 2 -> +...> "But this will" +...> end +"But this will" +``` + +This is equivalent to `else if` clauses in many imperative languages - although used less frequently in Elixir. + +If all of the conditions return `nil` or `false`, an error (`CondClauseError`) is raised. For this reason, it may be necessary to add a final condition, equal to `true`, which will always match: + +```elixir +iex> cond do +...> 2 + 2 == 5 -> +...> "This is never true" +...> 2 * 2 == 3 -> +...> "Nor this" +...> true -> +...> "This is always true (equivalent to else)" +...> end +"This is always true (equivalent to else)" +``` + +Similar to [`if`](`if/2`), [`cond`](`cond/1`) considers any value besides `nil` and `false` to be true: + +```elixir +iex> cond do +...> hd([1, 2, 3]) -> +...> "1 is considered as true" +...> end +"1 is considered as true" +``` + +## Summing up + +We have concluded the introduction to the most fundamental control-flow constructs in Elixir. Generally speaking, Elixir developers prefer pattern matching and guards, using [`case`](`case/2`) and function definitions (which we will explore in future chapters), as they are succinct and precise. When your logic cannot be outlined within patterns and guards, you may consider [`if`](`if/2`), falling back to [`cond`](`cond/1`) when there are several conditions to check. diff --git a/lib/elixir/pages/getting-started/comprehensions.md b/lib/elixir/pages/getting-started/comprehensions.md new file mode 100644 index 00000000000..ff775ee14f3 --- /dev/null +++ b/lib/elixir/pages/getting-started/comprehensions.md @@ -0,0 +1,115 @@ + + +# Comprehensions + +In Elixir, it is common to loop over an `Enumerable`, often filtering out some results and mapping values into another list. Comprehensions are syntactic sugar for such constructs: they group those common tasks into the `for` special form. + +For example, we can map a list of integers into their squared values: + +```elixir +iex> for n <- [1, 2, 3, 4], do: n * n +[1, 4, 9, 16] +``` + +A comprehension is made of three parts: generators, filters, and collectables. + +## Generators and filters + +In the expression above, `n <- [1, 2, 3, 4]` is the **generator**. It is literally generating values to be used in the comprehension. Any enumerable can be passed on the right-hand side of the generator expression: + +```elixir +iex> for n <- 1..4, do: n * n +[1, 4, 9, 16] +``` + +Generator expressions also support pattern matching on their left-hand side; all non-matching patterns are *ignored*. Imagine that, instead of a range, we have a keyword list where the key is the atom `:good` or `:bad` and we only want to compute the square of the `:good` values: + +```elixir +iex> values = [good: 1, good: 2, bad: 3, good: 4] +iex> for {:good, n} <- values, do: n * n +[1, 4, 16] +``` + +Alternatively to pattern matching, filters can be used to select some particular elements. For example, we can select the multiples of 3 and discard all others: + +```elixir +iex> for n <- 0..5, rem(n, 3) == 0, do: n * n +[0, 9] +``` + +Comprehensions discard all elements for which the filter expression returns `false` or `nil`; all other values are selected. + +Comprehensions generally provide a much more concise representation than using the equivalent functions from the `Enum` and `Stream` modules. Furthermore, comprehensions also allow multiple generators and filters to be given. Here is an example that receives a list of directories and gets the size of each file in those directories: + +```elixir +dirs = ["/home/mikey", "/home/james"] + +for dir <- dirs, + file <- File.ls!(dir), + path = Path.join(dir, file), + File.regular?(path) do + File.stat!(path).size +end +``` + +Multiple generators can also be used to calculate the Cartesian product of two lists: + +```elixir +iex> for i <- [:a, :b, :c], j <- [1, 2], do: {i, j} +[a: 1, a: 2, b: 1, b: 2, c: 1, c: 2] +``` + +Finally, keep in mind that variable assignments inside the comprehension, be it in generators, filters or inside the block, are not reflected outside of the comprehension. + +## Bitstring generators + +Bitstring generators are also supported and are very useful when you need to comprehend over bitstring streams. The example below receives a list of pixels from a binary with their respective red, green and blue values and converts them into tuples of three elements each: + +```elixir +iex> pixels = <<213, 45, 132, 64, 76, 32, 76, 0, 0, 234, 32, 15>> +iex> for <>, do: {r, g, b} +[{213, 45, 132}, {64, 76, 32}, {76, 0, 0}, {234, 32, 15}] +``` + +A bitstring generator can be mixed with "regular" enumerable generators, and supports filters as well. + +## The `:into` option + +In the examples above, all the comprehensions returned lists as their result. However, the result of a comprehension can be inserted into different data structures by passing the `:into` option to the comprehension. + +For example, a bitstring generator can be used with the `:into` option in order to easily remove all spaces in a string: + +```elixir +iex> for <>, c != ?\s, into: "", do: <> +"helloworld" +``` + +Sets, maps, and other dictionaries can also be given to the `:into` option. In general, `:into` accepts any structure that implements the `Collectable` protocol. + +A common use case of `:into` can be transforming values in a map: + +```elixir +iex> for {key, val} <- %{"a" => 1, "b" => 2}, into: %{}, do: {key, val * val} +%{"a" => 1, "b" => 4} +``` + +Let's make another example using streams. Since the `IO` module provides streams (that are both `Enumerable`s and `Collectable`s), an echo terminal that echoes back the upcased version of whatever is typed can be implemented using comprehensions: + +```elixir +iex> stream = IO.stream(:stdio, :line) +iex> for line <- stream, into: stream do +...> String.upcase(line) <> "\n" +...> end +``` + +Now type any string into the terminal and you will see that the same value will be printed in upper-case. Unfortunately, this example also got your IEx shell stuck in the comprehension, so you will need to hit `Ctrl+C` twice to get out of it. :) + +## Other options + +Comprehensions support other options, such as `:reduce` and `:uniq`. Here are additional resources to learn more about comprehensions: + + * [`for` official reference in Elixir documentation](`for/1`) + * [Mitchell Hanberg's comprehensive guide to Elixir's comprehensions](https://www.mitchellhanberg.com/the-comprehensive-guide-to-elixirs-for-comprehension/) diff --git a/lib/elixir/pages/getting-started/debugging.md b/lib/elixir/pages/getting-started/debugging.md new file mode 100644 index 00000000000..fbb32c65f9f --- /dev/null +++ b/lib/elixir/pages/getting-started/debugging.md @@ -0,0 +1,193 @@ + + +# Debugging + +There are a number of ways to debug code in Elixir. In this chapter we will cover some of the more common ways of doing so. + +## IO.inspect/2 + +What makes `IO.inspect(item, opts \\ [])` really useful in debugging is that it returns the `item` argument passed to it without affecting the behavior of the original code. Let's see an example. + +```elixir +(1..10) +|> IO.inspect() +|> Enum.map(fn x -> x * 2 end) +|> IO.inspect() +|> Enum.sum() +|> IO.inspect() +``` + +Prints: + +```elixir +1..10 +[2, 4, 6, 8, 10, 12, 14, 16, 18, 20] +110 +``` + +As you can see `IO.inspect/2` makes it possible to "spy" on values almost anywhere in your code without altering the result, making it very helpful inside of a pipeline like in the above case. + +`IO.inspect/2` also provides the ability to decorate the output with a `label` option. The label will be printed before the inspected `item`: + +```elixir +[1, 2, 3] +|> IO.inspect(label: "before") +|> Enum.map(&(&1 * 2)) +|> IO.inspect(label: "after") +|> Enum.sum +``` + +Prints: + +```elixir +before: [1, 2, 3] +after: [2, 4, 6] +``` + +It is also very common to use `IO.inspect/2` with `binding/0`, which returns all variable names and their values: + +```elixir +def some_function(a, b, c) do + IO.inspect(binding()) + ... +end +``` + +When `some_function/3` is invoked with `:foo`, `"bar"`, `:baz` it prints: + +```elixir +[a: :foo, b: "bar", c: :baz] +``` + +See `IO.inspect/2` and `Inspect.Opts` respectively to learn more about the function and read about all supported options. + +## dbg/2 + +Elixir v1.14 introduced `dbg/2`. `dbg` is similar to `IO.inspect/2` but specifically tailored for debugging. It prints the value passed to it and returns it (just like `IO.inspect/2`), but it also prints the code and location. + +```elixir +# In my_file.exs +feature = %{name: :dbg, inspiration: "Rust"} +dbg(feature) +dbg(Map.put(feature, :in_version, "1.14.0")) +``` + +The code above prints this: + +```text +[my_file.exs:2: (file)] +feature #=> %{inspiration: "Rust", name: :dbg} +[my_file.exs:3: (file)] +Map.put(feature, :in_version, "1.14.0") #=> %{in_version: "1.14.0", inspiration: "Rust", name: :dbg} +``` + +When talking about `IO.inspect/2`, we mentioned its usefulness when placed between steps of `|>` pipelines. `dbg` does it better: it understands Elixir code, so it will print values at _every step of the pipeline_. + +```elixir +# In dbg_pipes.exs +__ENV__.file +|> String.split("/", trim: true) +|> List.last() +|> File.exists?() +|> dbg() +``` + +This code prints: + +```text +[dbg_pipes.exs:5: (file)] +__ENV__.file #=> "/home/myuser/dbg_pipes.exs" +|> String.split("/", trim: true) #=> ["home", "myuser", "dbg_pipes.exs"] +|> List.last() #=> "dbg_pipes.exs" +|> File.exists?() #=> true +``` + +While `dbg` provides conveniences around Elixir constructs, you will need `IEx` if you want to execute code and set breakpoints while debugging. + +## Pry + +When using `IEx`, you may pass `--dbg pry` as an option to "stop" the code execution where the `dbg` call is: + +```console +$ iex --dbg pry +``` + +Or to debug inside of a project: + +```console +$ iex --dbg pry -S mix +``` + +Now any call to `dbg` will ask if you want to pry the existing code. If you accept, you'll be able to access all variables, as well as imports and aliases from the code, directly from IEx. This is called "prying". While the pry session is running, the code execution stops, until `continue` (or `c`) or `next` (or `n`) are called. Remember you can always run `iex` in the context of a project with `iex -S mix TASK`. + + + +## Breakpoints + +`dbg` calls require us to change the code we intend to debug and has limited stepping functionality. Luckily IEx also provides a `IEx.break!/2` function which allows you to set and manage breakpoints on any Elixir code without modifying its source: + + + +Similar to `dbg`, once a breakpoint is reached, code execution stops until `continue` (or `c`) or `next` (or `n`) are invoked. Breakpoints can navigate line-by-line by default, however, they do not have access to aliases and imports when breakpoints are set on compiled modules. + +The `mix test` task direct integration with breakpoints via the `-b`/`--breakpoints` flag. When the flag is used, a breakpoint is set at the beginning of every test that will run: + + + +Here are some commands you can use in practice: + +```console +# Debug all failed tests +$ iex -S mix test --breakpoints --failed +# Debug the test at the given file:line +$ iex -S mix test -b path/to/file:line +``` + +## Observer + +For debugging complex systems, jumping at the code is not enough. It is necessary to have an understanding of the whole virtual machine, processes, applications, as well as set up tracing mechanisms. Luckily this can be achieved in Erlang with `:observer`. In your application: + +```elixir +$ iex +iex> :observer.start() +``` + +> #### Missing dependencies {: .warning} +> +> When running `iex` inside a project with `iex -S mix`, `observer` won't be available as a dependency. To do so, you will need to call the following functions before: +> +> ```elixir +> iex> Mix.ensure_application!(:wx) # Not necessary on Erlang/OTP 27+ +> iex> Mix.ensure_application!(:runtime_tools) # Not necessary on Erlang/OTP 27+ +> iex> Mix.ensure_application!(:observer) +> iex> :observer.start() +> ``` +> +> If any of the calls above fail, here is what may have happened: some package managers default to installing a minimized Erlang without WX bindings for GUI support. In some package managers, you may be able to replace the headless Erlang with a more complete package (look for packages named `erlang` vs `erlang-nox` on Debian/Ubuntu/Arch). In others managers, you may need to install a separate `erlang-wx` (or similarly named) package. + +The above will open another Graphical User Interface that provides many panes to fully understand and navigate the runtime and your project. + +We explore the Observer in the context of an actual project [in the Dynamic Supervisor chapter of the Mix & OTP guide](../mix-and-otp/dynamic-supervisor.md). This is one of the debugging techniques [the Phoenix framework used to achieve 2 million connections on a single machine](https://phoenixframework.org/blog/the-road-to-2-million-websocket-connections). + +If you are using the Phoenix web framework, it ships with the [Phoenix LiveDashboard](https://github.com/phoenixframework/phoenix_live_dashboard), a web dashboard for production nodes which provides similar features to Observer. + +Finally, remember you can also get a mini-overview of the runtime info by calling `runtime_info/0` directly in IEx. + +## Other tools and community + +We have just scratched the surface of what the Erlang VM has to offer, for example: + + * Alongside the observer application, Erlang also includes a [`:crashdump_viewer`](`:crashdump_viewer`) to view crash dumps + + * Integration with OS level tracers, such as [Linux Trace Toolkit](https://www.erlang.org/doc/apps/runtime_tools/lttng), [DTRACE](https://www.erlang.org/doc/apps/runtime_tools/dtrace), and [SystemTap](https://www.erlang.org/doc/apps/runtime_tools/systemtap) + + * [Microstate accounting](`:msacc`) measures how much time the runtime spends in several low-level tasks in a short time interval + + * Mix ships with many tasks under the `profile` namespace, such as `mix profile.cprof` and `mix profile.fprof` + + * For more advanced use cases, we recommend the excellent [Erlang in Anger](https://www.erlang-in-anger.com/), which is available as a free ebook + +Happy debugging! diff --git a/lib/elixir/pages/getting-started/enumerable-and-streams.md b/lib/elixir/pages/getting-started/enumerable-and-streams.md new file mode 100644 index 00000000000..b544b973ddb --- /dev/null +++ b/lib/elixir/pages/getting-started/enumerable-and-streams.md @@ -0,0 +1,113 @@ + + +# Enumerables and Streams + +While Elixir allows us to write recursive code, most operations we perform on collections is done with the help of the `Enum` and `Stream` modules. Let's learn how. + +## Enumerables + +Elixir provides the concept of enumerables and the `Enum` module to work with them. We have already learned two enumerables: lists and maps. + +```elixir +iex> Enum.map([1, 2, 3], fn x -> x * 2 end) +[2, 4, 6] +iex> Enum.map(%{1 => 2, 3 => 4}, fn {k, v} -> k * v end) +[2, 12] +``` + +The `Enum` module provides a huge range of functions to transform, sort, group, filter and retrieve items from enumerables. It is one of the modules developers use frequently in their Elixir code. For a general overview of all functions in the `Enum` module, see [the `Enum` cheatsheet](enum-cheat.cheatmd). + +Elixir also provides ranges (see `Range`), which are also enumerable: + +```elixir +iex> Enum.map(1..3, fn x -> x * 2 end) +[2, 4, 6] +iex> Enum.reduce(1..3, 0, &+/2) +6 +``` + +The functions in the `Enum` module are limited to, as the name says, enumerating values in data structures. For specific operations, like inserting and updating particular elements, you may need to reach for modules specific to the data type. For example, if you want to insert an element at a given position in a list, you should use the `List.insert_at/3` function, as it would make little sense to insert a value into, for example, a range. + +We say the functions in the `Enum` module are polymorphic because they can work with diverse data types. In particular, the functions in the `Enum` module can work with any data type that implements the `Enumerable` protocol. We are going to discuss Protocols in a later chapter, for now we are going to move on to a specific kind of enumerable called a stream. + +## Eager vs Lazy + +All the functions in the `Enum` module are eager. Many functions expect an enumerable and return a list back: + +```elixir +iex> odd? = fn x -> rem(x, 2) != 0 end +#Function<6.80484245/1 in :erl_eval.expr/5> +iex> Enum.filter(1..3, odd?) +[1, 3] +``` + +This means that when performing multiple operations with `Enum`, each operation is going to generate an intermediate list until we reach the result: + +```elixir +iex> 1..100_000 |> Enum.map(&(&1 * 3)) |> Enum.filter(odd?) |> Enum.sum() +7500000000 +``` + +The example above has a pipeline of operations. We start with a range and then multiply each element in the range by 3. This first operation will now create and return a list with `100_000` items. Then we keep all odd elements from the list, generating a new list, now with `50_000` items, and then we sum all entries. + +## The pipe operator + +The `|>` symbol used in the snippet above is the **pipe operator**: it takes the output from the expression on its left side and passes it as the first argument to the function call on its right side. Its purpose is to highlight the data being transformed by a series of functions. To see how it can make the code cleaner, have a look at the example above rewritten without using the `|>` operator: + +```elixir +iex> Enum.sum(Enum.filter(Enum.map(1..100_000, &(&1 * 3)), odd?)) +7500000000 +``` + +Find more about the pipe operator [by reading its documentation](`|>/2`). + +## Streams + +As an alternative to `Enum`, Elixir provides the `Stream` module which supports lazy operations: + +```elixir +iex> 1..100_000 |> Stream.map(&(&1 * 3)) |> Stream.filter(odd?) |> Enum.sum() +7500000000 +``` + +Streams are lazy, composable enumerables. + +In the example above, `1..100_000 |> Stream.map(&(&1 * 3))` returns a data type, an actual stream, that represents the `map` computation over the range `1..100_000`: + +```elixir +iex> 1..100_000 |> Stream.map(&(&1 * 3)) +#Stream<[enum: 1..100000, funs: [#Function<34.16982430/1 in Stream.map/2>]]> +``` + +Furthermore, they are composable because we can pipe many stream operations: + +```elixir +iex> 1..100_000 |> Stream.map(&(&1 * 3)) |> Stream.filter(odd?) +#Stream<[enum: 1..100000, funs: [...]]> +``` + +Instead of generating intermediate lists, streams build a series of computations that are invoked only when we pass the underlying stream to the `Enum` module. Streams are useful when working with large, *possibly infinite*, collections. + +Many functions in the `Stream` module accept any enumerable as an argument and return a stream as a result. It also provides functions for creating streams. For example, `Stream.cycle/1` can be used to create a stream that cycles a given enumerable infinitely. Be careful to not call a function like `Enum.map/2` on such streams, as they would cycle forever: + +```elixir +iex> stream = Stream.cycle([1, 2, 3]) +#Function<15.16982430/2 in Stream.unfold/2> +iex> Enum.take(stream, 10) +[1, 2, 3, 1, 2, 3, 1, 2, 3, 1] +``` + +Another interesting function is `Stream.resource/3` which can be used to wrap around resources, guaranteeing they are opened right before enumeration and closed afterwards, even in the case of failures. For example, `File.stream!/1` builds on top of `Stream.resource/3` to stream files: + +```elixir +iex> "path/to/file" |> File.stream!() |> Enum.take(10) +``` + +The example above will fetch the first 10 lines of the file you have selected. This means streams can be very useful for handling large files or even slow resources like network resources. + +The `Enum` and `Stream` modules provide a wide range of functions, but you don't have to know all of them by heart. Familiarize yourself with `Enum.map/2`, `Enum.reduce/3` and other functions with either `map` or `reduce` in their names, and you will naturally build an intuition around the most important use cases. You may also focus on the `Enum` module first and only move to `Stream` for the particular scenarios where laziness is required, to either deal with slow resources or large, possibly infinite, collections. + +Next, we'll look at a feature central to Elixir, Processes, which allows us to write concurrent, parallel and distributed programs in an easy and understandable way. diff --git a/lib/elixir/pages/getting-started/erlang-libraries.md b/lib/elixir/pages/getting-started/erlang-libraries.md new file mode 100644 index 00000000000..08c6db79d6d --- /dev/null +++ b/lib/elixir/pages/getting-started/erlang-libraries.md @@ -0,0 +1,193 @@ + + +# Erlang libraries + +Elixir provides excellent interoperability with Erlang libraries. In fact, Elixir discourages simply wrapping Erlang libraries in favor of directly interfacing with Erlang code. In this section, we will present some of the most common and useful Erlang functionality that is not found in Elixir. + +Erlang modules have a different naming convention than in Elixir and start in lowercase. In both cases, module names are atoms and we invoke functions by dispatching to the module name: + +```elixir +iex> is_atom(String) +true +iex> String.first("hello") +"h" +iex> is_atom(:binary) +true +iex> :binary.first("hello") +104 +``` + +As you grow more proficient in Elixir, you may want to explore the Erlang [STDLIB Reference Manual](http://www.erlang.org/doc/apps/stdlib/index.html) in more detail. + +## The binary module + +The built-in Elixir String module handles binaries that are UTF-8 encoded. [The `:binary` module](`:binary`) is useful when you are dealing with binary data that is not necessarily UTF-8 encoded. + +```elixir +iex> String.to_charlist("Ø") +[216] +iex> :binary.bin_to_list("Ø") +[195, 152] +``` + +The above example shows the difference; the `String` module returns Unicode codepoints, while `:binary` deals with raw data bytes. + +## Formatted text output + +Elixir does not contain a function similar to `printf` found in C and other languages. Luckily, the Erlang standard library functions `:io.format/2` and `:io_lib.format/2` may be used. The first formats to terminal output, while the second formats to an iolist. The format specifiers differ from `printf`, [refer to the Erlang documentation for details](`:io.format/2`). + +```elixir +iex> :io.format("Pi is approximately given by:~10.3f~n", [:math.pi]) +Pi is approximately given by: 3.142 +:ok +iex> to_string(:io_lib.format("Pi is approximately given by:~10.3f~n", [:math.pi])) +"Pi is approximately given by: 3.142\n" +``` + +## The crypto module + +[The `:crypto` module](`:crypto`) contains hashing functions, digital signatures, encryption and more: + +```elixir +iex> Base.encode16(:crypto.hash(:sha256, "Elixir")) +"3315715A7A3AD57428298676C5AE465DADA38D951BDFAC9348A8A31E9C7401CB" +``` + +The `:crypto` module is part of the `:crypto` application that ships with Erlang. This means you must list the `:crypto` application as an additional application in your project configuration. To do this, edit your `mix.exs` file to include: + +```elixir +def application do + [extra_applications: [:crypto]] +end +``` + +Any module that is not part of the `:kernel` or `:stdlib` Erlang applications must have their application explicitly listed in your `mix.exs`. You can find the application name of any Erlang module in the Erlang documentation, immediately below the Erlang logo in the sidebar. + +## The digraph module + +The [`:digraph`](`:digraph`) and [`:digraph_utils`](`:digraph_utils`) modules contain functions for dealing with directed graphs built of vertices and edges. After constructing the graph, the algorithms in there will help find, for instance, the shortest path between two vertices, or loops in the graph. + +Given three vertices, find the shortest path from the first to the last. + +```elixir +iex> digraph = :digraph.new() +iex> coords = [{0.0, 0.0}, {1.0, 0.0}, {1.0, 1.0}] +iex> [v0, v1, v2] = (for c <- coords, do: :digraph.add_vertex(digraph, c)) +iex> :digraph.add_edge(digraph, v0, v1) +iex> :digraph.add_edge(digraph, v1, v2) +iex> :digraph.get_short_path(digraph, v0, v2) +[{0.0, 0.0}, {1.0, 0.0}, {1.0, 1.0}] +``` + +Note that the functions in `:digraph` alter the graph structure in-place, this +is possible because they are implemented as ETS tables, explained next. + +## Erlang Term Storage (ETS) + +The modules [`:ets`](`:ets`) and [`:dets`](`:dets`) handle storage of large data structures in memory or on disk respectively. + +ETS lets you create a table containing tuples. By default, ETS tables are protected, which means only the owner process may write to the table but any other process can read. ETS has some functionality to allow a table to be used as a simple database, a key-value store or as a cache mechanism. + +The functions in the `ets` module will modify the state of the table as a side-effect. + +```elixir +iex> table = :ets.new(:ets_test, []) +# Store as tuples with {name, population} +iex> :ets.insert(table, {"China", 1_374_000_000}) +iex> :ets.insert(table, {"India", 1_284_000_000}) +iex> :ets.insert(table, {"USA", 322_000_000}) +iex> :ets.i(table) +<1 > {<<"India">>,1284000000} +<2 > {<<"USA">>,322000000} +<3 > {<<"China">>,1374000000} +``` + +## The math module + +The [`:math`](`:math`) module contains common mathematical operations covering trigonometry, exponential, and logarithmic functions. + +```elixir +iex> angle_45_deg = :math.pi() * 45.0 / 180.0 +iex> :math.sin(angle_45_deg) +0.7071067811865475 +iex> :math.exp(55.0) +7.694785265142018e23 +iex> :math.log(7.694785265142018e23) +55.0 +``` + +## The queue module + +The [`:queue`](`:queue`) module provides a data structure that implements (double-ended) FIFO (first-in first-out) queues efficiently: + +```elixir +iex> q = :queue.new +iex> q = :queue.in("A", q) +iex> q = :queue.in("B", q) +iex> {value, q} = :queue.out(q) +iex> value +{:value, "A"} +iex> {value, q} = :queue.out(q) +iex> value +{:value, "B"} +iex> {value, q} = :queue.out(q) +iex> value +:empty +``` + +## The rand module + +The [`:rand`](`:rand`) has functions for returning random values and setting the random seed. + +```elixir +iex> :rand.uniform() +0.8175669086010815 +iex> _ = :rand.seed(:exs1024, {123, 123_534, 345_345}) +iex> :rand.uniform() +0.5820506340260994 +iex> :rand.uniform(6) +6 +``` + +## The zip and zlib modules + +The [`:zip`](`:zip`) module lets you read and write ZIP files to and from disk or memory, as well as extracting file information. + +This code counts the number of files in a ZIP file: + +```elixir +iex> :zip.foldl(fn _, _, _, acc -> acc + 1 end, 0, :binary.bin_to_list("file.zip")) +{:ok, 633} +``` + +The [`:zlib`](`:zlib`) module deals with data compression in zlib format, as found in the `gzip` command line utility found in Unix systems. + +```elixir +iex> song = " +...> Mary had a little lamb, +...> His fleece was white as snow, +...> And everywhere that Mary went, +...> The lamb was sure to go." +iex> compressed = :zlib.compress(song) +iex> byte_size(song) +110 +iex> byte_size(compressed) +99 +iex> :zlib.uncompress(compressed) +"\nMary had a little lamb,\nHis fleece was white as snow,\nAnd everywhere that Mary went,\nThe lamb was sure to go." +``` + +## Learning Erlang + +If you want to get deeper into Erlang, here's a list of online resources that cover Erlang's fundamentals and its more advanced features: + + * This [Erlang Syntax: A Crash Course](https://elixir-lang.org/crash-course.html) provides a concise intro to Erlang's syntax. Each code snippet is accompanied by equivalent code in Elixir. This is an opportunity for you to not only get some exposure to Erlang's syntax but also review what you learned about Elixir. + + * Erlang's official website has a short [tutorial](https://www.erlang.org/course). There is a chapter with pictures briefly describing Erlang's primitives for [concurrent programming](https://www.erlang.org/course/concurrent_programming.html). + + * [Learn You Some Erlang for Great Good!](http://learnyousomeerlang.com/) is an excellent introduction to Erlang, its design principles, standard library, best practices, and much more. Once you have read through the crash course mentioned above, you'll be able to safely skip the first couple of chapters in the book that mostly deal with the syntax. When you reach [The Hitchhiker's Guide to Concurrency](http://learnyousomeerlang.com/the-hitchhikers-guide-to-concurrency) chapter, that's where the real fun starts. + +Our last step is to take a look at existing Elixir (and Erlang) libraries you might use while debugging. diff --git a/lib/elixir/pages/getting-started/introduction.md b/lib/elixir/pages/getting-started/introduction.md new file mode 100644 index 00000000000..c642addcdce --- /dev/null +++ b/lib/elixir/pages/getting-started/introduction.md @@ -0,0 +1,60 @@ + + +# Introduction + +Welcome! + +This guide will teach you about Elixir fundamentals - the language syntax, how to define modules, the common data structures in the language, and more. This chapter will focus on ensuring that Elixir is installed and that you can successfully run Elixir's Interactive Shell, called IEx. + +Let's get started. + +## Installation + +If you haven't yet installed Elixir, visit our [installation page](https://elixir-lang.org/install.html). Once you are done, you can run `elixir --version` to get the current Elixir version. The requirements for this guide are: + + * Elixir 1.15.0 onwards + * Erlang/OTP 26 onwards + +If you are looking for other resources for learning Elixir, you can also consult the [learning page](https://elixir-lang.org/learning.html) of the official website. + +## Interactive mode + +When you install Elixir, you will have three new command line executables: `iex`, `elixir` and `elixirc`. + +For now, let's start by running `iex` (or `iex.bat` if you are on Windows PowerShell, where `iex` is a PowerShell command) which stands for Interactive Elixir. In interactive mode, we can type any Elixir expression and get its result. Let's warm up with some basic expressions. + +Open up `iex` and type the following expressions: + +```elixir +Erlang/OTP 26 [64-bit] [smp:2:2] [...] + +Interactive Elixir - press Ctrl+C to exit +iex(1)> 40 + 2 +42 +iex(2)> "hello" <> " world" +"hello world" +``` + +Please note that some details like version numbers may differ a bit in your session, that's not important. By executing the code above, you should evaluate expressions and see their results. To exit `iex` press `Ctrl+C` twice. + +It seems we are ready to go! We will use the interactive shell quite a lot in the next chapters to get a bit more familiar with the language constructs and basic types, starting in the next chapter. + +## Running scripts + +After getting familiar with the basics of the language you may want to try writing simple programs. This can be accomplished by putting the following Elixir code into a file: + +```elixir +IO.puts("Hello world from Elixir") +``` + +Save it as `simple.exs` and execute it with `elixir`: + +```console +$ elixir simple.exs +Hello world from Elixir +``` + +`iex` and `elixir` are all we need to learn the main language concepts. There is a separate guide named ["Mix and OTP guide"](../mix-and-otp/introduction-to-mix.md) that explores how to actually create, manage, and test full-blown Elixir projects. For now, let's move on to learn the basic data types in the language. diff --git a/lib/elixir/pages/getting-started/io-and-the-file-system.md b/lib/elixir/pages/getting-started/io-and-the-file-system.md new file mode 100644 index 00000000000..3ffff5d8375 --- /dev/null +++ b/lib/elixir/pages/getting-started/io-and-the-file-system.md @@ -0,0 +1,208 @@ + + +# IO and the file system + +This chapter introduces the input/output mechanisms, file-system-related tasks, and related modules such as `IO`, `File`, and `Path`. The IO system provides a great opportunity to shed some light on some philosophies and curiosities of Elixir and the Erlang VM. + +## The `IO` module + +The `IO` module is the main mechanism in Elixir for reading and writing to standard input/output (`:stdio`), standard error (`:stderr`), files, and other IO devices. Usage of the module is pretty straightforward: + +```elixir +iex> IO.puts("hello world") +hello world +:ok +iex> IO.gets("yes or no? ") +yes or no? yes +"yes\n" +``` + +By default, functions in the `IO` module read from the standard input and write to the standard output. We can change that by passing, for example, `:stderr` as an argument (in order to write to the standard error device): + +```elixir +iex> IO.puts(:stderr, "hello world") +hello world +:ok +``` + +## The `File` module + +The `File` module contains functions that allow us to open files as IO devices. By default, files are opened in binary mode, which requires developers to use the specific `IO.binread/2` and `IO.binwrite/2` functions from the `IO` module: + +> #### Potential data loss warning {: .warning} +> +> The following code opens a file for writing. If an existing file is available at the given path, its contents will be deleted. + +```elixir +iex> {:ok, file} = File.open("path/to/file/hello", [:write]) +{:ok, #PID<0.47.0>} +iex> IO.binwrite(file, "world") +:ok +iex> File.close(file) +:ok +iex> File.read("path/to/file/hello") +{:ok, "world"} +``` + +The file could be opened with the `:append` option, instead of `:write`, to preserve its contents. You may also pass the `:utf8` option, which tells the `File` module to interpret the bytes read from the file as UTF-8-encoded bytes. + +Besides functions for opening, reading and writing files, the `File` module has many functions to work with the file system. Those functions are named after their UNIX equivalents. For example, `File.rm/1` can be used to remove files, `File.mkdir/1` to create directories, `File.mkdir_p/1` to create directories and all their parent chain. There are even `File.cp_r/2` and `File.rm_rf/1` to respectively copy and remove files and directories recursively (i.e., copying and removing the contents of the directories too). + +You will also notice that functions in the `File` module have two variants: one "regular" variant and another variant with a trailing bang (`!`). For example, when we read the `"hello"` file in the example above, we use `File.read/1`. Alternatively, we can use `File.read!/1`: + +```elixir +iex> File.read("path/to/file/hello") +{:ok, "world"} +iex> File.read!("path/to/file/hello") +"world" +iex> File.read("path/to/file/unknown") +{:error, :enoent} +iex> File.read!("path/to/file/unknown") +** (File.Error) could not read file "path/to/file/unknown": no such file or directory +``` + +Notice that the version with `!` returns the contents of the file instead of a tuple, and if anything goes wrong the function raises an error. + +The version without `!` is preferred when you want to handle different outcomes using pattern matching: + +```elixir +case File.read("path/to/file/hello") do + {:ok, body} -> # do something with the `body` + {:error, reason} -> # handle the error caused by `reason` +end +``` + +However, if you expect the file to be there, the bang variation is more useful as it raises a meaningful error message. Avoid writing: + +```elixir +{:ok, body} = File.read("path/to/file/unknown") +``` + +as, in case of an error, `File.read/1` will return `{:error, reason}` and the pattern matching will fail. You will still get the desired result (a raised error), but the message will be about the pattern which doesn't match (thus being cryptic in respect to what the error actually is about). + +Therefore, if you don't want to handle the error outcomes, prefer to use the functions ending with an exclamation mark, such as `File.read!/1`. + +## The `Path` module + +The majority of the functions in the `File` module expect paths as arguments. Most commonly, those paths will be regular binaries. The `Path` module provides facilities for working with such paths: + +```elixir +iex> Path.join("foo", "bar") +"foo/bar" +iex> Path.expand("~/hello") +"/Users/jose/hello" +``` + +Using functions from the `Path` module as opposed to directly manipulating strings is preferred since the `Path` module takes care of different operating systems transparently. Finally, keep in mind that Elixir will automatically convert slashes (`/`) into backslashes (`\`) on Windows when performing file operations. + +With this, we have covered the main modules that Elixir provides for dealing with IO and interacting with the file system. In the next section, we will peek a bit under the covers and learn how the IO system is implemented in the VM. + +## Processes + +You may have noticed that `File.open/2` returns a tuple like `{:ok, pid}`: + +```elixir +iex> {:ok, file} = File.open("hello") +{:ok, #PID<0.47.0>} +``` + +This happens because the `IO` module actually works with processes (see [the previous chapter](processes.md)). Given a file is a process, when you write to a file that has been closed, you are actually sending a message to a process which has been terminated: + +```elixir +iex> File.close(file) +:ok +iex> IO.write(file, "is anybody out there") +** (ErlangError) Erlang error: :terminated: + + * 1st argument: the device has terminated + + (stdlib 5.0) io.erl:94: :io.put_chars(#PID<0.114.0>, "is anybody out there") + iex:4: (file) +``` + +Let's see in more detail what happens when you request `IO.write(pid, binary)`. The `IO` module sends a message to the process identified by `pid` with the desired operation. A small ad-hoc process can help us see it: + +```elixir +iex> pid = spawn(fn -> +...> receive do +...> msg -> IO.inspect(msg) +...> end +...> end) +#PID<0.57.0> +iex> IO.write(pid, "hello") +{:io_request, #PID<0.41.0>, #Reference<0.0.8.91>, + {:put_chars, :unicode, "hello"}} +** (ErlangError) erlang error: :terminated +``` + +After `IO.write/2`, we can see the request sent by the `IO` module printed out (a four-elements tuple). Soon after that, we see that it fails since the `IO` module expected some kind of result, which we did not supply. + +By modeling IO devices with processes, the Erlang VM allows us to even read and write to files across nodes. Neat! + +## `iodata` and `chardata` + +In all of the examples above, we used binaries when writing to files. However, most of the IO functions in Elixir also accept either "iodata" or "chardata". + +One of the main reasons for using "iodata" and "chardata" is for performance. For example, +imagine you need to greet someone in your application: + +```elixir +name = "Mary" +IO.puts("Hello " <> name <> "!") +``` + +Given strings in Elixir are immutable, as most data structures, the example above will copy the string "Mary" into the new "Hello Mary!" string. While this is unlikely to matter for the short string as above, copying can be quite expensive for large strings! For this reason, the IO functions in Elixir allow you to pass instead a list of strings: + +```elixir +name = "Mary" +IO.puts(["Hello ", name, "!"]) +``` + +In the example above, there is no copying. Instead we create a list that contains the original name. We call such lists either "iodata" or "chardata" and we will learn the precise difference between them soon. + +Those lists are very useful because it can actually simplify the processing strings in several scenarios. For example, imagine you have a list of values, such as `["apple", "banana", "lemon"]` that you want to write to disk separated by commas. How can you achieve this? + +One option is to use `Enum.join/2` and convert the values to a string: + +```elixir +iex> Enum.join(["apple", "banana", "lemon"], ",") +"apple,banana,lemon" +``` + +The above returns a new string by copying each value into the new string. However, with the knowledge in this section, we know that we can pass a list of strings to the IO/File functions. So instead we can do: + +```elixir +iex> Enum.intersperse(["apple", "banana", "lemon"], ",") +["apple", ",", "banana", ",", "lemon"] +``` + +"iodata" and "chardata" do not only contain strings, but they may contain arbitrary nested lists of strings too: + +```elixir +iex> IO.puts(["apple", [",", "banana", [",", "lemon"]]]) +``` + +"iodata" and "chardata" may also contain integers. For example, we could print our comma separated list of values by using `?,` as separator, which is the integer representing a comma (`44`): + +```elixir +iex> IO.puts(["apple", ?,, "banana", ?,, "lemon"]) +``` + +The difference between "iodata" and "chardata" is precisely what said integer represents. For iodata, the integers represent bytes. For chardata, the integers represent Unicode codepoints. For ASCII characters, the byte representation is the same as the codepoint representation, so it fits both classifications. However, the default IO device works with chardata, which means we can do: + +```elixir +iex> IO.puts([?O, ?l, ?á, ?\s, "Mary", ?!]) +``` + +Charlists, such as `~c"hello world"`, are lists of integers, and therefore are chardata. + +We packed a lot into this small section, so let's break it down: + + * iodata and chardata are lists of binaries and integers. Those binaries and integers can be arbitrarily nested inside lists. Their goal is to give flexibility and performance when working with IO devices and files; + + * the choice between iodata and chardata depends on the encoding of the IO device. If the file is opened without encoding, the file expects iodata, and the functions in the `IO` module starting with `bin*` must be used. The default IO device (`:stdio`) and files opened with `:utf8` encoding expect chardata and work with the remaining functions in the `IO` module; + +This finishes our tour of IO devices and IO related functionality. We have learned about three Elixir modules - `IO`, `File`, and `Path` - as well as how the VM uses processes for the underlying IO mechanisms and how to use `chardata` and `iodata` for IO operations. diff --git a/lib/elixir/pages/getting-started/keywords-and-maps.md b/lib/elixir/pages/getting-started/keywords-and-maps.md new file mode 100644 index 00000000000..656e84c96ca --- /dev/null +++ b/lib/elixir/pages/getting-started/keywords-and-maps.md @@ -0,0 +1,286 @@ + + +# Keyword lists and maps + +Now let's talk about associative data structures. Associative data structures are able to associate a key to a certain value. Different languages call these different names like dictionaries, hashes, associative arrays, etc. + +In Elixir, we have two main associative data structures: keyword lists and maps. + +## Keyword lists + +Keyword lists are a data-structure used to pass options to functions. Let's see a scenario where they may be useful. + +Imagine you want to split a string of numbers. Initially, we can invoke `String.split/2` passing two strings as arguments: + +```elixir +iex> String.split("1 2 3 4", " ") +["1", "2", "3", "4"] +``` + +What if you only want to split at most 2 times? The `String.split/3` function allows the `parts` option to be set to the maximum number of entries in the result: + +```elixir +iex> String.split("1 2 3 4", " ", [parts: 3]) +["1", "2", "3 4"] +``` + +As you can see, we got 3 parts, the last one containing the remaining of the input without splitting it. + +Now imagine that some of the inputs you must split on contains additional spaces between the numbers: + +```elixir +iex> String.split("1 2 3 4", " ", [parts: 3]) +["1", "", "2 3 4"] +``` + +As you can see, the additional spaces lead to empty entries in the output. Luckily, we can also set the `trim` option to `true` to remove them: + +```elixir +iex> String.split("1 2 3 4", " ", [parts: 3, trim: true]) +["1", "2", " 3 4"] +``` + +Once again we got 3 parts, with the last one containing the leftovers. + +`[parts: 3]` and `[parts: 3, trim: true]` are keyword lists. When a keyword list is the last argument of a function, we can skip the brackets and write: + +```elixir +iex> String.split("1 2 3 4", " ", parts: 3, trim: true) +["1", "2", " 3 4"] +``` + +As shown in the example above, keyword lists are mostly used as optional arguments to functions. + +As the name implies, keyword lists are simply lists. In particular, they are lists consisting of 2-item tuples where the first element (the key) is an atom and the second element can be any value. Both representations are the same: + +```elixir +iex> [{:parts, 3}, {:trim, true}] == [parts: 3, trim: true] +true +``` + +Keyword lists are important because they have three special characteristics: + + * Keys must be atoms. + * Keys are ordered, as specified by the developer. + * Keys can be given more than once. + +For example, we use the fact that keys can be repeated when [importing functions](../getting-started/alias-require-and-import.md) in Elixir: + +```elixir +iex> import String, only: [split: 1, split: 2] +String +iex> split("hello world") +["hello", "world"] +``` + +In the example above, we imported both `split/1` and `split/2` from the `String` module, allowing us to invoke them without typing the module name. We used a keyword list to list the functions to import. + +Since keyword lists are lists, we can use all operations available to lists. For example, we can use `++` to add new values to a keyword list: + +```elixir +iex> list = [a: 1, b: 2] +[a: 1, b: 2] +iex> list ++ [c: 3] +[a: 1, b: 2, c: 3] +iex> [a: 0] ++ list +[a: 0, a: 1, b: 2] +``` + +You can read the value of a keyword list using the brackets syntax, which will return the value of the first matching key. This is also known as the access syntax, as it is defined by the `Access` module: + +```elixir +iex> list[:a] +1 +iex> list[:b] +2 +``` + +Although we can pattern match on keyword lists, it is not done in practice since pattern matching on lists requires the number of items and their order to match: + +```elixir +iex> [a: a] = [a: 1] +[a: 1] +iex> a +1 +iex> [a: a] = [a: 1, b: 2] +** (MatchError) no match of right hand side value: [a: 1, b: 2] +iex> [b: b, a: a] = [a: 1, b: 2] +** (MatchError) no match of right hand side value: [a: 1, b: 2] +``` + +Furthermore, given keyword lists are often used as optional arguments, they are used in situations where not all keys may be present, which would make it impossible to match on them. In a nutshell, do not pattern match on keyword lists. + +In order to manipulate keyword lists, Elixir provides the `Keyword` module. Remember, though, keyword lists are simply lists, and as such they provide the same linear performance characteristics: the longer the list, the longer it will take to find a key, to count the number of items, and so on. If you need to store a large amount of keys in a key-value data structure, Elixir offers maps, which we will soon learn. + +### `do`-blocks and keywords + +As we have seen, keywords are mostly used in the language to pass optional values. In fact, we have used keywords in earlier chapters. Let's look at the `if/2` macro: + +```elixir +iex> if true do +...> "This will be seen" +...> else +...> "This won't" +...> end +"This will be seen" +``` + +In the example above, the `do` and `else` blocks make up a keyword list. They are nothing more than a syntax convenience on top of keyword lists. We can rewrite the above to: + +```elixir +iex> if(true, do: "This will be seen", else: "This won't") +"This will be seen" +``` + +Pay close attention to both syntaxes. The second example uses keyword lists, exactly as in the `String.split/3` example, so we separate each key-value pair with commas and each key is followed by `:`. In the `do`-blocks, we use bare words, such as `do`, `else`, and `end`, and separate them by a newline. They are useful precisely when writing blocks of code. Most of the time, you will use the block syntax, but it is good to know they are equivalent. + +The fact the block syntax is equivalent to keywords means we only need few data structures to represent the language, keeping it simple overall. We will come back to this topic when discussing [optional syntax](optional-syntax.md) and [meta-programming](../meta-programming/quote-and-unquote.md). + +With this out of the way, let's talk about maps. + +## Maps as key-value pairs + +Whenever you need to store key-value pairs, maps are the "go to" data structure in Elixir. A map is created using the `%{}` syntax: + +```elixir +iex> map = %{:a => 1, 2 => :b} +%{2 => :b, :a => 1} +iex> map[:a] +1 +iex> map[2] +:b +iex> map[:c] +nil +``` + +Compared to keyword lists, we can already see two differences: + + * Maps allow any value as a key. + * Maps have their own internal ordering, which is not guaranteed to be the same across different maps, even if they have the same keys + +In contrast to keyword lists, maps are very useful with pattern matching. When a map is used in a pattern, it will always match on a subset of the given value: + +```elixir +iex> %{} = %{:a => 1, 2 => :b} +%{2 => :b, :a => 1} +iex> %{:a => a} = %{:a => 1, 2 => :b} +%{2 => :b, :a => 1} +iex> a +1 +iex> %{:c => c} = %{:a => 1, 2 => :b} +** (MatchError) no match of right hand side value: %{2 => :b, :a => 1} +``` + +As shown above, a map matches as long as the keys in the pattern exist in the given map. Therefore, an empty map matches all maps. + +The `Map` module provides a very similar API to the `Keyword` module with convenience functions to add, remove, and update maps keys: + +```elixir +iex> Map.get(%{:a => 1, 2 => :b}, :a) +1 +iex> Map.put(%{:a => 1, 2 => :b}, :c, 3) +%{2 => :b, :a => 1, :c => 3} +iex> Map.to_list(%{:a => 1, 2 => :b}) +[{2, :b}, {:a, 1}] +``` + +## Maps of predefined keys + +In the previous section, we have used maps as a key-value data structure where keys can be added or removed at any time. However, it is also common to create maps with a predefined set of keys. Their values may be updated, but new keys are never added nor removed. This is useful when we know the shape of the data we are working with and, if we get a different key, it likely means a mistake was done elsewhere. In such cases, the keys are most often atoms: + +```elixir +iex> map = %{:name => "John", :age => 23} +%{name: "John", age: 23} +``` + +As you can see from the printed result above, Elixir also allows you to write maps of atom keys using the same `key: value` syntax as keyword lists: + +```elixir +iex> map = %{name: "John", age: 23} +%{name: "John", age: 23} +``` + +When a key is an atom, we can also access them using the `map.key` syntax: + +```elixir +iex> map.name +"John" +iex> map.agee +** (KeyError) key :agee not found in: %{name: "John", age: 23} +``` + +There is also syntax for updating keys, which also raises if the key has not yet been defined: + +```elixir +iex> %{map | name: "Mary"} +%{name: "Mary", age: 23} +iex> %{map | agee: 27} +** (KeyError) key :agee not found in: %{name: "John", age: 23} +``` + +These operations have one large benefit in that they raise if the key does not exist in the map and the compiler may even detect and warn when possible. This makes them useful to get quick feedback and spot bugs and typos early on. This is also the syntax used to power another Elixir feature called "Structs", which we will learn later on. + +Elixir developers typically prefer to use the `map.key` syntax and pattern matching instead of the functions in the `Map` module when working with maps because they lead to an assertive style of programming. [This blog post by José Valim](https://dashbit.co/blog/writing-assertive-code-with-elixir) provides insight and examples on how you get more concise and faster software by writing assertive code in Elixir. + +In a further chapter you'll learn about ["Structs"](structs.md), which further enforce the idea of a map with predefined keys. + +## Nested data structures + +Often we will have maps inside maps, or even keywords lists inside maps, and so forth. Elixir provides conveniences for manipulating nested data structures via the `get_in/1`, `put_in/2`, `update_in/2`, and other macros giving the same conveniences you would find in imperative languages while keeping the immutable properties of the language. + +Imagine you have the following structure: + +```elixir +iex> users = [ + john: %{name: "John", age: 27, languages: ["Erlang", "Ruby", "Elixir"]}, + mary: %{name: "Mary", age: 29, languages: ["Elixir", "F#", "Clojure"]} +] +[ + john: %{age: 27, languages: ["Erlang", "Ruby", "Elixir"], name: "John"}, + mary: %{age: 29, languages: ["Elixir", "F#", "Clojure"], name: "Mary"} +] +``` + +We have a keyword list of users where each value is a map containing the name, age and a list of programming languages each user likes. If we wanted to access the age for john, we could write: + +```elixir +iex> users[:john].age +27 +``` + +It happens we can also use this same syntax for updating the value: + +```elixir +iex> users = put_in(users[:john].age, 31) +[ + john: %{age: 31, languages: ["Erlang", "Ruby", "Elixir"], name: "John"}, + mary: %{age: 29, languages: ["Elixir", "F#", "Clojure"], name: "Mary"} +] +``` + +The `update_in/2` macro is similar but allows us to pass a function that controls how the value changes. For example, let's remove "Clojure" from Mary's list of languages: + +```elixir +iex> users = update_in(users[:mary].languages, fn languages -> List.delete(languages, "Clojure") end) +[ + john: %{age: 31, languages: ["Erlang", "Ruby", "Elixir"], name: "John"}, + mary: %{age: 29, languages: ["Elixir", "F#"], name: "Mary"} +] +``` + +## Summary + +There are two different data structures for working with key-value stores in Elixir. Alongside the `Access` module and pattern matching, they provide a rich set of tools for manipulating complex, potentially nested, data structures. + +As we conclude this chapter, remember that you should: + + * Use keyword lists for passing optional values to functions + + * Use maps for general key-value data structures + + * Use maps when working with data that has a predefined set of keys + +Now let's talk about modules and functions. diff --git a/lib/elixir/pages/getting-started/lists-and-tuples.md b/lib/elixir/pages/getting-started/lists-and-tuples.md new file mode 100644 index 00000000000..9a7e06a124b --- /dev/null +++ b/lib/elixir/pages/getting-started/lists-and-tuples.md @@ -0,0 +1,197 @@ + + +# Lists and tuples + +In this chapter we will learn two of the most used collection data-types in Elixir: lists and tuples. + +## (Linked) Lists + +Elixir uses square brackets to specify a list of values. Values can be of any type: + +```elixir +iex> [1, 2, true, 3] +[1, 2, true, 3] +iex> length([1, 2, 3]) +3 +``` + +Two lists can be concatenated or subtracted using the [`++`](`++/2`) and [`--`](`--/2`) operators respectively: + +```elixir +iex> [1, 2, 3] ++ [4, 5, 6] +[1, 2, 3, 4, 5, 6] +iex> [1, true, 2, false, 3, true] -- [true, false] +[1, 2, 3, true] +``` + +List operators never modify the existing list. Concatenating to or removing elements from a list returns a new list. We say that Elixir data structures are *immutable*. One advantage of immutability is that it leads to clearer code. You can freely pass the data around with the guarantee no one will mutate it in memory - only transform it. + +Throughout the tutorial, we will talk a lot about the head and tail of a list. The head is the first element of a list and the tail is the remainder of the list. They can be retrieved with the functions [`hd`](`hd/1`) and [`tl`](`tl/1`). Let's assign a list to a variable and retrieve its head and tail: + +```elixir +iex> list = [1, 2, 3] +iex> hd(list) +1 +iex> tl(list) +[2, 3] +``` + +Getting the head or the tail of an empty list throws an error: + +```elixir +iex> hd([]) +** (ArgumentError) argument error +``` + +Sometimes you will create a list and it will return a quoted value preceded by `~c`. For example: + +```elixir +iex> [11, 12, 13] +~c"\v\f\r" +iex> [104, 101, 108, 108, 111] +~c"hello" +``` + +When Elixir sees a list of printable ASCII numbers, Elixir will print that as a charlist (literally a list of characters). Charlists are quite common when interfacing with existing Erlang code. Whenever you see a value in IEx and you are not quite sure what it is, you can use [`i`](`IEx.Helpers.i/1`) to retrieve information about it: + +```elixir +iex> i ~c"hello" +Term + i ~c"hello" +Data type + List +Description + ... +Raw representation + [104, 101, 108, 108, 111] +Reference modules + List +Implemented protocols + ... +``` + +We will talk more about charlists in the ["Binaries, strings, and charlists"](binaries-strings-and-charlists.md) chapter. + +> #### Single-quoted strings {: .info} +> +> In Elixir, you can also use `'hello'` to build charlists, but this notation has been soft-deprecated in Elixir v1.15 and will emit warnings in future versions. Prefer to write `~c"hello"` instead. + +## Tuples + +Elixir uses curly brackets to define tuples. Like lists, tuples can hold any value: + +```elixir +iex> {:ok, "hello"} +{:ok, "hello"} +iex> tuple_size({:ok, "hello"}) +2 +``` + +Tuples store elements contiguously in memory. This means accessing a tuple element by index or getting the tuple size is a fast operation. Indexes start from zero: + +```elixir +iex> tuple = {:ok, "hello"} +{:ok, "hello"} +iex> elem(tuple, 1) +"hello" +iex> tuple_size(tuple) +2 +``` + +It is also possible to put an element at a particular index in a tuple with [`put_elem`](`put_elem/3`): + +```elixir +iex> tuple = {:ok, "hello"} +{:ok, "hello"} +iex> put_elem(tuple, 1, "world") +{:ok, "world"} +iex> tuple +{:ok, "hello"} +``` + +Notice that [`put_elem`](`put_elem/3`) returned a new tuple. The original tuple stored in the `tuple` variable was not modified. Like lists, tuples are also immutable. Every operation on a tuple returns a new tuple, it never changes the given one. + +## Lists or tuples? + +What is the difference between lists and tuples? + +Lists are stored in memory as linked lists, meaning that each element in a list holds its value and points to the following element until the end of the list is reached. This means accessing the length of a list is a linear operation: we need to traverse the whole list in order to figure out its size. + +Similarly, the performance of list concatenation depends on the length of the left-hand list: + +```elixir +iex> list = [1, 2, 3] +[1, 2, 3] + +# This is fast as we only need to traverse `[0]` to prepend to `list` +iex> [0] ++ list +[0, 1, 2, 3] + +# This is slow as we need to traverse `list` to append 4 +iex> list ++ [4] +[1, 2, 3, 4] +``` + +Tuples, on the other hand, are stored contiguously in memory. This means getting the tuple size or accessing an element by index is fast. On the other hand, updating or adding elements to tuples is expensive because it requires creating a new tuple in memory: + +```elixir +iex> tuple = {:a, :b, :c, :d} +{:a, :b, :c, :d} +iex> put_elem(tuple, 2, :e) +{:a, :b, :e, :d} +``` + +Note, however, the elements themselves are not copied. When you update a tuple, all entries are shared between the old and the new tuple, except for the entry that has been replaced. This rule applies to most data structures in Elixir. This reduces the amount of memory allocation the language needs to perform and is only possible thanks to the immutable semantics of the language. + +Those performance characteristics dictate the usage of those data structures. In a nutshell, lists are used when the number of elements returned may vary. Tuples have a fixed size. Let's see two examples from the `String` module: + +```elixir +iex> String.split("hello world") +["hello", "world"] +iex> String.split("hello beautiful world") +["hello", "beautiful", "world"] +``` + +The [`String.split`](`String.split/1`) function breaks a string into a list of strings on every whitespace character. Since the amount of elements returned depends on the input, we use a list. + +On the other hand, [`String.split_at`](`String.split_at/2`) splits a string in two parts at a given position. Since it always returns two entries, regardless of the input size, it returns tuples: + +```elixir +iex> String.split_at("hello world", 3) +{"hel", "lo world"} +iex> String.split_at("hello world", -4) +{"hello w", "orld"} +``` + +It is also very common to use tuples and atoms to create "tagged tuples", which is a handy return value when an operation may succeed or fail. For example, [`File.read`](`File.read/1`) reads the contents of a file at a given path, which may or may not exist. It returns tagged tuples: + +```elixir +iex> File.read("path/to/existing/file") +{:ok, "... contents ..."} +iex> File.read("path/to/unknown/file") +{:error, :enoent} +``` + +If the path given to [`File.read`](`File.read/1`) exists, it returns a tuple with the atom `:ok` as the first element and the file contents as the second. Otherwise, it returns a tuple with `:error` and the error description. As we will soon learn, Elixir allows us to *pattern match* on tagged tuples and effortlessly handle both success and failure cases. + +Given Elixir consistently follows those rules, the choice between lists and tuples get clearer as you learn and use the language. Elixir often guides you to do the right thing. For example, there is an [`elem`](`elem/2`) function to access a tuple item: + +```elixir +iex> tuple = {:ok, "hello"} +{:ok, "hello"} +iex> elem(tuple, 1) +"hello" +``` + +However, given you often don't know the number of elements in a list, there is no built-in equivalent for accessing arbitrary entries in a lists, apart from its head. + +## Size or length? + +When counting the elements in a data structure, Elixir also abides by a simple rule: the function is named `size` if the operation is in constant time (the value is pre-calculated) or `length` if the operation is linear (calculating the length gets slower as the input grows). As a mnemonic, both "length" and "linear" start with "l". + +For example, we have used 4 counting functions so far: [`byte_size`](`byte_size/1`) (for the number of bytes in a string), [`tuple_size`](`tuple_size/1`) (for tuple size), [`length`](`length/1`) (for list length) and [`String.length`](`String.length/1`) (for the number of graphemes in a string). We use [`byte_size`](`byte_size/1`) to get the number of bytes in a string, which is a cheap operation. Retrieving the number of Unicode graphemes, on the other hand, uses [`String.length`](`String.length/1`), and may be expensive as it relies on a traversal of the entire string. + +Now that we are familiar with the basic data-types in the language, let's learn important constructs for writing code, before we discuss more complex data structures. diff --git a/lib/elixir/pages/getting-started/module-attributes.md b/lib/elixir/pages/getting-started/module-attributes.md new file mode 100644 index 00000000000..e1d824b8231 --- /dev/null +++ b/lib/elixir/pages/getting-started/module-attributes.md @@ -0,0 +1,205 @@ + + +# Module attributes + +Module attributes in Elixir serve three purposes: + +1. as module and function annotations +2. as temporary module storage to be used during compilation +3. as compile-time constants + +Let's check these examples. + +## As annotations + +Elixir brings the concept of module attributes from Erlang. For example: + +```elixir +defmodule MyServer do + @moduledoc "My server code." +end +``` + +In the example above, we are defining the module documentation by using the module attribute syntax. Elixir has a handful of reserved attributes. Here are a few of them, the most commonly used ones: + + * `@moduledoc` — provides documentation for the current module. + * `@doc` — provides documentation for the function or macro that follows the attribute. + * `@spec` — provides a typespec for the function that follows the attribute. + * `@behaviour` — (notice the British spelling) used for specifying an OTP or user-defined behaviour. + +`@moduledoc` and `@doc` are by far the most used attributes, and we expect you to use them a lot. Elixir treats documentation as first-class and provides many functions to access documentation. We will cover them [in their own chapter](writing-documentation.md). + +Documentation is only accessible from compiled modules. So in order to give it a try, let's once again define the `Math` module, but this time within a file named `math.ex`: + +```elixir +defmodule Math do + @moduledoc """ + Provides math-related functions. + + ## Examples + + iex> Math.sum(1, 2) + 3 + + """ + + @doc """ + Calculates the sum of two numbers. + """ + def sum(a, b), do: a + b +end +``` + +Elixir promotes the use of Markdown with heredocs to write readable documentation. Heredocs are multi-line strings, they start and end with triple double-quotes, keeping the formatting of the inner text. + +Now let's compile it. Start `iex` and then invoke [the `c/2` helper](`IEx.Helpers.c/2`): + +```elixir +iex> c("math.ex", ".") +[Math] +``` + +And now we can access them: + +```elixir +iex> h Math # Docs for module Math +... +iex> h Math.sum # Docs for the sum function +... +``` + +When we compiled the module, you may have noticed Elixir created a `Elixir.Math.beam` file. That's the bytecode for the module and that's where the documentation is stored. + +In our day to day, Elixir developers use the `Mix` build tool to compile code and projects like [ExDoc](https://github.com/elixir-lang/ex_doc) to generate HTML and EPUB pages from the documentation. + +Take a look at the docs for `Module` for a complete list of supported attributes. + +## As temporary storage + +So far, we have seen how to define attributes, but how can we read them? Let's see an example: + +```elixir +defmodule MyServer do + @service URI.parse("https://example.com") + IO.inspect(@service) +end +``` + +> #### Newlines {: .warning} +> +> Do not add a newline between the attribute and its value, otherwise Elixir will assume you are reading the value, rather than setting it. + +Trying to access an attribute that was not defined will print a warning: + +```elixir +defmodule MyServer do + @unknown +end +warning: undefined module attribute @unknown, please remove access to @unknown or explicitly set it before access +``` + +Attributes can also be read inside functions: + +```elixir +defmodule MyApp.Status do + @service URI.parse("https://example.com") + def status(email) do + SomeHttpClient.get(@service) + end +end +``` + +The module attribute is defined at compilation time and its *return value*, not the function call itself, is what will be substituted in for the attribute. So the above will effectively compile to this: + +```elixir +defmodule MyApp.Status do + def status(email) do + SomeHttpClient.get(%URI{ + authority: "example.com", + host: "example.com", + port: 443, + scheme: "https" + }) + end +end +``` + +This can be useful for pre-computing values and then injecting its results into the module. This is what we mean by temporary storage: after the module is compiled, the module attribute is discarded, except for the functions that have read the attribute. Note you cannot invoke functions defined in the same module as part of the attribute itself, as those functions have not yet been defined. + +Every time we read an attribute inside a function, Elixir takes a snapshot of its current value. Therefore if you read the same attribute multiple times inside multiple functions, you end-up increasing compilation times as Elixir now has to compile every snapshot. Generally speaking, you want to avoid reading the same attribute multiple times and instead move it to function. For example, instead of this: + +```elixir +def some_function, do: do_something_with(@example) +def another_function, do: do_something_else_with(@example) +``` + +Prefer this: + +```elixir +def some_function, do: do_something_with(example()) +def another_function, do: do_something_else_with(example()) +defp example, do: @example +``` + +## As compile-time constants + +Module attributes may also be useful as compile-time constants. Generally speaking, functions themselves are sufficient for the role of constants in a codebase. For example, instead of defining: + +```elixir +@hours_in_a_day 24 +``` + +You should prefer: + +```elixir +defp hours_in_a_day(), do: 24 +``` + +You may even define a public function if it needs to be shared across modules. It is common in many projects to have a module called `MyApp.Constants` that defines all constants used throughout the codebase. + +You can even have composite data structures as constants, as long as they are made exclusively of other data types (no function calls, no operators, and no other expressions). For example, you may specify a system configuration constant as follows: + +```elixir +defp system_config(), do: %{timezone: "Etc/UTC", locale: "pt-BR"} +``` + +Given data structures in Elixir are immutable, only a single instance of the data structure above is allocated and shared across all functions calls, as long as it doesn't have any executable expression. + +The use case for module attributes arise when you need to do some work at compile-time and then inject its results inside a function. A common scenario is module attributes inside patterns and guards (as an alternative to `defguard/1`), since they only support a limited set of expressions: + +```elixir +# Inside pattern +@default_timezone "Etc/UTC" +def shift(@default_timezone), do: ... + +# Inside guards +@time_periods [:am, :pm] +def shift(time, period) when period in @time_periods, do: ... +``` + +Module attributes as constants and as temporary storage are most often used together: the module attribute is used to compute and store an expensive value, and then exposed as constant from that module. + +## Going further + +Libraries and frameworks can leverage module attributes to provide custom annotations. To see an example, look no further than Elixir's unit test framework called `ExUnit`. `ExUnit` uses module attributes for multiple different purposes: + +```elixir +defmodule MyTest do + use ExUnit.Case, async: true + + @tag :external + @tag os: :unix + test "contacts external service" do + # ... + end +end +``` + +In the example above, `ExUnit` stores the value of `async: true` in a module attribute to change how the module is compiled. Tags also work as annotations and they can be supplied multiple times, thanks to Elixir's ability to [accumulate attributes](`Module.register_attribute/3`). Then you can use tags to setup and filter tests, such as avoiding executing Unix specific tests while running your test suite on Windows. + +To fully understand how `ExUnit` works, we'd need macros, so we will revisit this pattern in the Meta-programming guide and learn how to use module attributes as storage for custom annotations. + +In the next chapters, we'll explore structs and protocols before moving to exception handling and other constructs like sigils and comprehensions. diff --git a/lib/elixir/pages/getting-started/modules-and-functions.md b/lib/elixir/pages/getting-started/modules-and-functions.md new file mode 100644 index 00000000000..93785c8e418 --- /dev/null +++ b/lib/elixir/pages/getting-started/modules-and-functions.md @@ -0,0 +1,244 @@ + + +# Modules and functions + +In Elixir we group several functions into modules. We've already used many different modules in the previous chapters, such as the `String` module: + +```elixir +iex> String.length("hello") +5 +``` + +In order to create our own modules in Elixir, we use the [`defmodule`](`defmodule/2`) macro. The first letter of a module name (an alias, as described further down) must be in uppercase. We use the [`def`](`def/2`) macro to define functions in that module. The first letter of every function must be in lowercase (or underscore): + +```elixir +iex> defmodule Math do +...> def sum(a, b) do +...> a + b +...> end +...> end + +iex> Math.sum(1, 2) +3 +``` + +In this chapter we will define our own modules, with different levels of complexity. As our examples get longer in size, it can be tricky to type them all in the shell, so we will resort more frequently to scripting. + +## Scripting + +Elixir has two file extensions `.ex` (Elixir) and `.exs` (Elixir scripts). Elixir treats both files exactly the same way, the only difference is in intention. `.ex` files are meant to be compiled while `.exs` files are used for scripting. + +Let's create a file named `math.exs`: + +```elixir +defmodule Math do + def sum(a, b) do + a + b + end +end + +IO.puts Math.sum(1, 2) +``` + +And execute it as: + +```console +$ elixir math.exs +``` + +You can also load the file within `iex` by running: + +```console +$ iex math.exs +``` + +And then have direct access to the `Math` module. + +## Function definition + +Inside a module, we can define functions with `def/2` and private functions with `defp/2`. A function defined with `def/2` can be invoked from other modules while a private function can only be invoked locally. + +```elixir +defmodule Math do + def sum(a, b) do + do_sum(a, b) + end + + defp do_sum(a, b) do + a + b + end +end + +IO.puts Math.sum(1, 2) #=> 3 +IO.puts Math.do_sum(1, 2) #=> ** (UndefinedFunctionError) +``` + +Function declarations also support guards and multiple clauses. If a function has several clauses, Elixir will try each clause until it finds one that matches. Here is an implementation of a function that checks if the given number is zero or not: + +```elixir +defmodule Math do + def zero?(0) do + true + end + + def zero?(x) when is_integer(x) do + false + end +end + +IO.puts Math.zero?(0) #=> true +IO.puts Math.zero?(1) #=> false +IO.puts Math.zero?([1, 2, 3]) #=> ** (FunctionClauseError) +IO.puts Math.zero?(0.0) #=> ** (FunctionClauseError) +``` + +The trailing question mark in `zero?` means that this function returns a boolean. To learn more about the naming conventions for modules, function names, variables and more in Elixir, see [Naming Conventions](../references/naming-conventions.md). + +Giving an argument that does not match any of the clauses raises an error. + +Similar to constructs like `if`, function definitions support both `do:` and `do`-block syntax, as [we learned in the previous chapter](keywords-and-maps.md#do-blocks-and-keywords). For example, we can edit `math.exs` to look like this: + +```elixir +defmodule Math do + def zero?(0), do: true + def zero?(x) when is_integer(x), do: false +end +``` + +And it will provide the same behavior. You may use `do:` for one-liners but always use `do`-blocks for functions spanning multiple lines. If you prefer to be consistent, you can use `do`-blocks throughout your codebase. + +## Default arguments + +Function definitions in Elixir also support default arguments: + +```elixir +defmodule Concat do + def join(a, b, sep \\ " ") do + a <> sep <> b + end +end + +IO.puts(Concat.join("Hello", "world")) #=> Hello world +IO.puts(Concat.join("Hello", "world", "_")) #=> Hello_world +``` + +Any expression is allowed to serve as a default value, but it won't be evaluated during the function definition. Every time the function is invoked and any of its default values have to be used, the expression for that default value will be evaluated: + +```elixir +defmodule DefaultTest do + def dowork(x \\ "hello") do + x + end +end +``` + +```elixir +iex> DefaultTest.dowork() +"hello" +iex> DefaultTest.dowork(123) +123 +iex> DefaultTest.dowork() +"hello" +``` + +If a function with default values has multiple clauses, it is required to create a function head (a function definition without a body) for declaring defaults: + +```elixir +defmodule Concat do + # A function head declaring defaults + def join(a, b, sep \\ " ") + + def join(a, b, _sep) when b == "" do + a + end + + def join(a, b, sep) do + a <> sep <> b + end +end + +IO.puts(Concat.join("Hello", "")) #=> Hello +IO.puts(Concat.join("Hello", "world")) #=> Hello world +IO.puts(Concat.join("Hello", "world", "_")) #=> Hello_world +``` + +When a variable is not used by a function or a clause, we add a leading underscore (`_`) to its name to signal this intent. This rule is also covered in our [Naming Conventions](../references/naming-conventions.md#underscore-_foo) document. + +## Understanding Aliases + +An alias in Elixir is a capitalized identifier (like `String`, `Keyword`, etc) which is converted to an atom during compilation. For instance, the `String` alias translates by default to the atom `:"Elixir.String"`: + +```elixir +iex> is_atom(String) +true +iex> to_string(String) +"Elixir.String" +iex> :"Elixir.String" == String +true +``` + +By using the `alias/2` directive, we are changing the atom the alias expands to. + +Aliases expand to atoms because in the Erlang Virtual Machine (and consequently Elixir) modules are always represented by atoms. By namespacing +those atoms elixir modules avoid conflicting with existing erlang modules. + +```elixir +iex> List.flatten([1, [2], 3]) +[1, 2, 3] +iex> :"Elixir.List".flatten([1, [2], 3]) +[1, 2, 3] +``` + +That's the mechanism we use to call Erlang modules: + +```elixir +iex> :lists.flatten([1, [2], 3]) +[1, 2, 3] +``` + +## Module nesting + +Now that we have talked about aliases, we can talk about nesting and how it works in Elixir. Consider the following example: + +```elixir +defmodule Foo do + defmodule Bar do + end +end +``` + +The example above will define two modules: `Foo` and `Foo.Bar`. The second can be accessed as `Bar` inside `Foo` as long as they are in the same lexical scope. + +If, later, the `Bar` module is moved outside the `Foo` module definition, it must be referenced by its full name (`Foo.Bar`) or an alias must be set using the `alias` directive discussed above. + +**Note**: in Elixir, you don't have to define the `Foo` module before being able to define the `Foo.Bar` module, as they are effectively independent. The above could also be written as: + +```elixir +defmodule Foo.Bar do +end + +defmodule Foo do + alias Foo.Bar + # Can still access it as `Bar` +end +``` + +Aliasing a nested module does not bring parent modules into scope. Consider the following example: + +```elixir +defmodule Foo do + defmodule Bar do + defmodule Baz do + end + end +end + +alias Foo.Bar.Baz +# The module `Foo.Bar.Baz` is now available as `Baz` +# However, the module `Foo.Bar` is *not* available as `Bar` +``` + +As we will see in later chapters, aliases also play a crucial role in macros, to guarantee they are hygienic. diff --git a/lib/elixir/pages/getting-started/optional-syntax.md b/lib/elixir/pages/getting-started/optional-syntax.md new file mode 100644 index 00000000000..c00d1d7da38 --- /dev/null +++ b/lib/elixir/pages/getting-started/optional-syntax.md @@ -0,0 +1,99 @@ + + +# Optional syntax sheet + +In the previous chapters, we learned that the Elixir syntax allows developers to omit delimiters in a few occasions to make code more readable. For example, we learned that parentheses are optional: + +```elixir +iex> length([1, 2, 3]) == length [1, 2, 3] +true +``` + +and that `do`-`end` blocks are equivalent to keyword lists: + +```elixir +# do-end blocks +iex> if true do +...> :this +...> else +...> :that +...> end +:this + +# keyword lists +iex> if true, do: :this, else: :that +:this +``` + +Keyword lists use Elixir's regular notation for separating arguments, where we separate each key-value pair with commas, and each key is followed by `:`. In the `do`-blocks, we get rid of the colons, the commas, and separate each keyword by a newline. They are useful exactly because they remove the verbosity when writing blocks of code. Most of the time, we use the block syntax, but it is good to know they are equivalent. + +Those conveniences, which we call here "optional syntax", allow the language syntax core to be small, without sacrificing the readability and expressiveness of your code. In this brief chapter, we will review the four rules provided by the language, using a short snippet as playground. + +## Walk-through + +Take the following code: + +```elixir +if variable? do + Call.this() +else + Call.that() +end +``` + +Now let's remove the conveniences one by one: + +1. `do`-`end` blocks are equivalent to keywords: + + ```elixir + if variable?, do: Call.this(), else: Call.that() + ``` + +2. Keyword lists as last argument do not require square brackets, but let's add them: + + ```elixir + if variable?, [do: Call.this(), else: Call.that()] + ``` + +3. Keyword lists are the same as lists of two-element tuples: + + ```elixir + if variable?, [{:do, Call.this()}, {:else, Call.that()}] + ``` + +4. Finally, parentheses are optional on function calls, but let's add them: + + ```elixir + if(variable?, [{:do, Call.this()}, {:else, Call.that()}]) + ``` + +That's it! Those four rules outline the optional syntax available in Elixir. + +To understand why these rules matter, we can briefly compare Elixir with many other programming languages. Most programming languages have several keywords for defining methods, functions, conditionals, loops, and so forth. Each of those keywords have their own syntax rules attached to them. + +However, in Elixir, none of these language features require special "keywords", instead they all build from this small set of rules. The other benefit is that developers can also extend the language in a way that is consistent with the language itself, since the constructs for designing and extending the language are the same. We further explore this topic in [the "Meta-programming" guide](../meta-programming/quote-and-unquote.md). + +At the end of the day, those rules are what enables us to write: + +```elixir +defmodule Math do + def add(a, b) do + a + b + end +end +``` + +instead of: + +```elixir +defmodule(Math, [ + {:do, def(add(a, b), [{:do, a + b}])} +]) +``` + +Whenever you have any questions, this quick walk-through has you covered. + +Finally, if you are concerned about when to apply these rules, it's worth noting that the Elixir formatter handles those concerns for you. Most Elixir developers use the `mix format` task to format their codebases according to a well-defined set of rules defined by the Elixir team and the community. For instance, `mix format` will always add parentheses to function calls unless explicitly configured not to do so. This helps to maintain consistency across all codebases within organizations and the wider community. diff --git a/lib/elixir/pages/getting-started/pattern-matching.md b/lib/elixir/pages/getting-started/pattern-matching.md new file mode 100644 index 00000000000..c819d73db85 --- /dev/null +++ b/lib/elixir/pages/getting-started/pattern-matching.md @@ -0,0 +1,203 @@ + + +# Pattern matching + +In this chapter, we will learn why the [`=`](`=/2`) operator in Elixir is called the match operator and how to use it to pattern match inside data structures. We will learn about the pin operator [`^`](`^/1`) used to access previously bound values. + +## The match operator + +We have used the [`=`](`=/2`) operator a couple times to assign variables in Elixir: + +```elixir +iex> x = 1 +1 +iex> x +1 +``` + +In Elixir, the [`=`](`=/2`) operator is actually called *the match operator*. Let's see why: + +```elixir +iex> x = 1 +1 +iex> 1 = x +1 +iex> 2 = x +** (MatchError) no match of right hand side value: 1 +``` + +Notice that `1 = x` is a valid expression, and it matched because both the left and right side are equal to `1`. When the sides do not match, a `MatchError` is raised. + +A variable can only be assigned on the left side of [`=`](`=/2`): + +```elixir +iex> 1 = unknown +** (CompileError) iex:1: undefined variable "unknown" +``` + +## Pattern matching + +The match operator is not only used to match against simple values, but it is also useful for destructuring more complex data types. For example, we can pattern match on tuples: + +```elixir +iex> {a, b, c} = {:hello, "world", 42} +{:hello, "world", 42} +iex> a +:hello +iex> b +"world" +``` + +A pattern match error will occur if the sides can't be matched, for example if the tuples have different sizes: + +```elixir +iex> {a, b, c} = {:hello, "world"} +** (MatchError) no match of right hand side value: {:hello, "world"} +``` + +And also when comparing different types, for example if matching a tuple on the left side with a list on the right side: + +```elixir +iex> {a, b, c} = [:hello, "world", 42] +** (MatchError) no match of right hand side value: [:hello, "world", 42] +``` + +More interestingly, we can match on specific values. The example below asserts that the left side will only match the right side when the right side is a tuple that starts with the atom `:ok`: + +```elixir +iex> {:ok, result} = {:ok, 13} +{:ok, 13} +iex> result +13 + +iex> {:ok, result} = {:error, :oops} +** (MatchError) no match of right hand side value: {:error, :oops} +``` + +We can pattern match on lists: + +```elixir +iex> [a, b, c] = [1, 2, 3] +[1, 2, 3] +iex> a +1 +``` + +A list also supports matching on its own head and tail: + +```elixir +iex> [head | tail] = [1, 2, 3] +[1, 2, 3] +iex> head +1 +iex> tail +[2, 3] +``` + +Similar to the [`hd`](`hd/1`) and [`tl`](`tl/1`) functions, we can't match an empty list with a head and tail pattern: + +```elixir +iex> [head | tail] = [] +** (MatchError) no match of right hand side value: [] +``` + +The `[head | tail]` format is not only used on pattern matching but also for prepending items to a list: + +```elixir +iex> list = [1, 2, 3] +[1, 2, 3] +iex> [0 | list] +[0, 1, 2, 3] +``` + +In some cases, you don't care about a particular value in a pattern. It is a common practice to bind those values to the underscore, `_`. For example, if only the head of the list matters to us, we can assign the tail to underscore: + +```elixir +iex> [head | _] = [1, 2, 3] +[1, 2, 3] +iex> head +1 +``` + +The variable `_` is special in that it can never be read from. Trying to read from it gives a compile error: + +```elixir +iex> _ +** (CompileError) iex:1: invalid use of _. "_" represents a value to be ignored in a pattern and cannot be used in expressions +``` + +If a variable is mentioned more than once in a pattern, all references must bind to the same value: + +```elixir +iex> {x, x} = {1, 1} +{1, 1} +iex> {x, x} = {1, 2} +** (MatchError) no match of right hand side value: {1, 2} +``` + +Although pattern matching allows us to build powerful constructs, its usage is limited. For instance, you cannot make function calls on the left side of a match. The following example is invalid: + +```elixir +iex> length([1, [2], 3]) = 3 +** (CompileError) iex:1: cannot invoke remote function :erlang.length/1 inside match +``` + +Pattern matching allows developers to easily destructure data types such as tuples and lists. As we will see in the following chapters, it is one of the foundations of recursion in Elixir and applies to other types as well, like maps and binaries. + +## The pin operator + +Variables in Elixir can be rebound: + +```elixir +iex> x = 1 +1 +iex> x = 2 +2 +``` + +However, there are times when we don't want variables to be rebound. + +Use the pin operator [`^`](`^/1`) when you want to pattern match against a variable's *existing value* rather than rebinding the variable. + +```elixir +iex> x = 1 +1 +iex> ^x = 2 +** (MatchError) no match of right hand side value: 2 +``` + +Because we have pinned `x` when it was bound to the value of `1`, it is equivalent to the following: + +```elixir +iex> 1 = 2 +** (MatchError) no match of right hand side value: 2 +``` + +Notice that we even see the exact same error message. + +We can use the pin operator inside other pattern matches, such as tuples or lists: + +```elixir +iex> x = 1 +1 +iex> [^x, 2, 3] = [1, 2, 3] +[1, 2, 3] +iex> {y, ^x} = {2, 1} +{2, 1} +iex> y +2 +iex> {y, ^x} = {2, 2} +** (MatchError) no match of right hand side value: {2, 2} +``` + +Because `x` was bound to the value of `1` when it was pinned, this last example could have been written as: + +```elixir +iex> {y, 1} = {2, 2} +** (MatchError) no match of right hand side value: {2, 2} +``` + +This finishes our introduction to pattern matching. As we will see in the next chapter, pattern matching is very common in many language constructs and they can be further augmented with guards. diff --git a/lib/elixir/pages/getting-started/processes.md b/lib/elixir/pages/getting-started/processes.md new file mode 100644 index 00000000000..e0c7b326515 --- /dev/null +++ b/lib/elixir/pages/getting-started/processes.md @@ -0,0 +1,240 @@ + + +# Processes + +In Elixir, all code runs inside processes. Processes are isolated from each other, run concurrent to one another and communicate via message passing. Processes are not only the basis for concurrency in Elixir, but they also provide the means for building distributed and fault-tolerant programs. + +Elixir's processes should not be confused with operating system processes. Processes in Elixir are extremely lightweight in terms of memory and CPU (even compared to threads as used in many other programming languages). Because of this, it is not uncommon to have tens or even hundreds of thousands of processes running simultaneously. + +In this chapter, we will learn about the basic constructs for spawning new processes, as well as sending and receiving messages between processes. + +## Spawning processes + +The basic mechanism for spawning new processes is the auto-imported `spawn/1` function: + +```elixir +iex> spawn(fn -> 1 + 2 end) +#PID<0.43.0> +``` + +`spawn/1` takes a function which it will execute in another process. + +Notice `spawn/1` returns a PID (process identifier). At this point, the process you spawned is very likely dead. The spawned process will execute the given function and exit after the function is done: + +```elixir +iex> pid = spawn(fn -> 1 + 2 end) +#PID<0.44.0> +iex> Process.alive?(pid) +false +``` + +> Note: you will likely get different process identifiers than the ones we are showing in our snippets. + +We can retrieve the PID of the current process by calling `self/0`: + +```elixir +iex> self() +#PID<0.41.0> +iex> Process.alive?(self()) +true +``` + +Processes get much more interesting when we are able to send and receive messages. + +## Sending and receiving messages + +We can send messages to a process with `send/2` and receive them with `receive/1`: + +```elixir +iex> send(self(), {:hello, "world"}) +{:hello, "world"} +iex> receive do +...> {:hello, msg} -> msg +...> {:world, _msg} -> "won't match" +...> end +"world" +``` + +When a message is sent to a process, the message is stored in the process mailbox. The `receive/1` block goes through the current process mailbox searching for a message that matches any of the given patterns. `receive/1` supports guards and many clauses, exactly as `case/2`. + +The process that sends the message does not block on `send/2`, it puts the message in the recipient's mailbox and continues. In particular, a process can send messages to itself. + +If there is no message in the mailbox matching any of the patterns, the current process will wait until a matching message arrives. A timeout can also be specified: + +```elixir +iex> receive do +...> {:hello, msg} -> msg +...> after +...> 1_000 -> "nothing after 1s" +...> end +"nothing after 1s" +``` + +A timeout of 0 can be given when you already expect the message to be in the mailbox. + +Let's put it all together and send messages between processes: + +```elixir +iex> parent = self() +#PID<0.41.0> +iex> spawn(fn -> send(parent, {:hello, self()}) end) +#PID<0.48.0> +iex> receive do +...> {:hello, pid} -> "Got hello from #{inspect pid}" +...> end +"Got hello from #PID<0.48.0>" +``` + +The `inspect/1` function is used to convert a data structure's internal representation into a string, typically for printing. Notice that when the `receive` block gets executed the sender process we have spawned may already be dead, as its only instruction was to send a message. + +While in the shell, you may find the helper `flush/0` quite useful. It flushes and prints all the messages in the mailbox. + +```elixir +iex> send(self(), :hello) +:hello +iex> flush() +:hello +:ok +``` + +## Links + +The majority of times we spawn processes in Elixir, we spawn them as linked processes. Before we show an example with `spawn_link/1`, let's see what happens when a process started with `spawn/1` fails: + +```elixir +iex> spawn(fn -> raise "oops" end) +#PID<0.58.0> + +[error] Process #PID<0.58.00> raised an exception +** (RuntimeError) oops + (stdlib) erl_eval.erl:668: :erl_eval.do_apply/6 +``` + +It merely logged an error but the parent process is still running. That's because processes are isolated. If we want the failure in one process to propagate to another one, we should link them. This can be done with `spawn_link/1`: + +```elixir +iex> self() +#PID<0.41.0> +iex> spawn_link(fn -> raise "oops" end) + +** (EXIT from #PID<0.41.0>) evaluator process exited with reason: an exception was raised: + ** (RuntimeError) oops + (stdlib) erl_eval.erl:668: :erl_eval.do_apply/6 + +[error] Process #PID<0.289.0> raised an exception +** (RuntimeError) oops + (stdlib) erl_eval.erl:668: :erl_eval.do_apply/6 +``` + +Because processes are linked, we now see a message saying the parent process, which is the shell process, has received an EXIT signal from another process causing the shell to terminate. IEx detects this situation and starts a new shell session. + +Linking can also be done manually by calling `Process.link/1`. We recommend that you take a look at the `Process` module for other functionality provided by processes. + +Processes and links play an important role when building fault-tolerant systems. Elixir processes are isolated and don't share anything by default. Therefore, a failure in a process will never crash or corrupt the state of another process. Links, however, allow processes to establish a relationship in case of failure. We often link our processes to supervisors which will detect when a process dies and start a new process in its place. + +While other languages would require us to catch/handle exceptions, in Elixir we are actually fine with letting processes fail because we expect supervisors to properly restart our systems. "Failing fast" (sometimes referred as "let it crash") is a common philosophy when writing Elixir software! + +`spawn/1` and `spawn_link/1` are the basic primitives for creating processes in Elixir. Although we have used them exclusively so far, most of the time we are going to use abstractions that build on top of them. Let's see the most common one, called tasks. + +## Tasks + +Tasks build on top of the spawn functions to provide better error reports and introspection: + +```elixir +iex> Task.start(fn -> raise "oops" end) +{:ok, #PID<0.55.0>} + +15:22:33.046 [error] Task #PID<0.55.0> started from #PID<0.53.0> terminating +** (RuntimeError) oops + (stdlib) erl_eval.erl:668: :erl_eval.do_apply/6 + (elixir) lib/task/supervised.ex:85: Task.Supervised.do_apply/2 + (stdlib) proc_lib.erl:247: :proc_lib.init_p_do_apply/3 +Function: #Function<20.99386804/0 in :erl_eval.expr/5> + Args: [] +``` + +Instead of `spawn/1` and `spawn_link/1`, we use `Task.start/1` and `Task.start_link/1` which return `{:ok, pid}` rather than just the PID. This is what enables tasks to be used in supervision trees. Furthermore, `Task` provides convenience functions, like `Task.async/1` and `Task.await/1`, and functionality to ease distribution. + +We will explore tasks and other abstractions around processes in the ["Mix and OTP guide"](../mix-and-otp/introduction-to-mix.md). + +## State + +We haven't talked about state so far. If you are building an application that requires state, for example, to keep your application configuration, or you need to parse a file and keep it in memory, where would you store it? + +Processes are the most common answer to this question. We can write processes that loop infinitely, maintain state, and send and receive messages. As an example, let's write a module that starts new processes that work as a key-value store in a file named `kv.exs`: + +```elixir +defmodule KV do + def start_link do + Task.start_link(fn -> loop(%{}) end) + end + + defp loop(map) do + receive do + {:get, key, caller} -> + send(caller, Map.get(map, key)) + loop(map) + {:put, key, value} -> + loop(Map.put(map, key, value)) + end + end +end +``` + +Note that the `start_link` function starts a new process that runs the `loop/1` function, starting with an empty map. The `loop/1` (private) function then waits for messages and performs the appropriate action for each message. We made `loop/1` private by using `defp` instead of `def`. In the case of a `:get` message, it sends a message back to the caller and calls `loop/1` again, to wait for a new message. While the `:put` message actually invokes `loop/1` with a new version of the map, with the given `key` and `value` stored. + +Let's give it a try by running `iex kv.exs`: + +```elixir +iex> {:ok, pid} = KV.start_link() +{:ok, #PID<0.62.0>} +iex> send(pid, {:get, :hello, self()}) +{:get, :hello, #PID<0.41.0>} +iex> flush() +nil +:ok +``` + +At first, the process map has no keys, so sending a `:get` message and then flushing the current process inbox returns `nil`. Let's send a `:put` message and try it again: + +```elixir +iex> send(pid, {:put, :hello, :world}) +{:put, :hello, :world} +iex> send(pid, {:get, :hello, self()}) +{:get, :hello, #PID<0.41.0>} +iex> flush() +:world +:ok +``` + +Notice how the process is keeping a state and we can get and update this state by sending the process messages. In fact, any process that knows the `pid` above will be able to send it messages and manipulate the state. + +It is also possible to register the `pid`, giving it a name, and allowing everyone that knows the name to send it messages: + +```elixir +iex> Process.register(pid, :kv) +true +iex> send(:kv, {:get, :hello, self()}) +{:get, :hello, #PID<0.41.0>} +iex> flush() +:world +:ok +``` + +Using processes to maintain state and name registration are very common patterns in Elixir applications. However, most of the time, we won't implement those patterns manually as above, but by using one of the many abstractions that ship with Elixir. For example, Elixir provides `Agent`s, which are simple abstractions around state. Our code above could be directly written as: + +```elixir +iex> {:ok, pid} = Agent.start_link(fn -> %{} end) +{:ok, #PID<0.72.0>} +iex> Agent.update(pid, fn map -> Map.put(map, :hello, :world) end) +:ok +iex> Agent.get(pid, fn map -> Map.get(map, :hello) end) +:world +``` + +A `:name` option could also be given to `Agent.start_link/2` and it would be automatically registered. Besides agents, Elixir provides an API for building generic servers (called `GenServer`), registries, and more, all powered by processes underneath. Those, along with supervision trees, will be explored with more detail in the ["Mix and OTP guide"](../mix-and-otp/introduction-to-mix.md), which will build a complete Elixir application from start to finish. + +For now, let's move on and explore the world of I/O in Elixir. diff --git a/lib/elixir/pages/getting-started/protocols.md b/lib/elixir/pages/getting-started/protocols.md new file mode 100644 index 00000000000..b644004d524 --- /dev/null +++ b/lib/elixir/pages/getting-started/protocols.md @@ -0,0 +1,261 @@ + + +# Protocols + +Protocols are a mechanism to achieve polymorphism in Elixir where you want the behavior to vary depending on the data type. We are already familiar with one way of solving this type of problem: via pattern matching and guard clauses. Consider a simple utility module that would tell us the type of input variable: + +```elixir +defmodule Utility do + def type(value) when is_binary(value), do: "string" + def type(value) when is_integer(value), do: "integer" + # ... other implementations ... +end +``` + +If the use of this module were confined to your own project, you would be able to keep defining new `type/1` functions for each new data type. However, this code could be problematic if it was shared as a dependency by multiple apps because there would be no easy way to extend its functionality. + +This is where protocols can help us: protocols allow us to extend the original behavior for as many data types as we need. That's because **dispatching on a protocol is available to any data type that has implemented the protocol** and a protocol can be implemented by anyone, at any time. + +Here's how we could write the same `Utility.type/1` functionality as a protocol: + +```elixir +defprotocol Utility do + @spec type(t) :: String.t() + def type(value) +end + +defimpl Utility, for: BitString do + def type(_value), do: "string" +end + +defimpl Utility, for: Integer do + def type(_value), do: "integer" +end +``` + +We define the protocol using `defprotocol/2` - its functions and specs may look similar to interfaces or abstract base classes in other languages. We can add as many implementations as we like using `defimpl/2`. The output is exactly the same as if we had a single module with multiple functions: + +```elixir +iex> Utility.type("foo") +"string" +iex> Utility.type(123) +"integer" +``` + +With protocols, however, we are no longer stuck having to continuously modify the same module to support more and more data types. For example, we could spread the `defimpl` calls above over multiple files and Elixir will dispatch the execution to the appropriate implementation based on the data type. Functions defined in a protocol may have more than one input, but the **dispatching will always be based on the data type of the first input**. + +One of the most common protocols you may encounter is the `String.Chars` protocol: implementing its `to_string/1` function for your custom structs will tell the Elixir kernel how to represent them as strings. We will explore all the built-in protocols later. For now, let's implement our own. + +## Example + +Now that you have seen an example of the type of problem protocols help solve and how they solve them, let's look at a more in-depth example. + +In Elixir, we have two idioms for checking how many items there are in a data structure: `length` and `size`. `length` means the information must be computed. For example, `length(list)` needs to traverse the whole list to calculate its length. On the other hand, `tuple_size(tuple)` and `byte_size(binary)` do not depend on the tuple and binary size as the size information is pre-computed in the data structure. + +Even if we have type-specific functions for getting the size built into Elixir (such as `tuple_size/1`), we could implement a generic `Size` protocol that all data structures for which size is pre-computed would implement. + +The protocol definition would look like this: + +```elixir +defprotocol Size do + @doc "Calculates the size (and not the length!) of a data structure" + def size(data) +end +``` + +The `Size` protocol expects a function called `size` that receives one argument (the data structure we want to know the size of) to be implemented. We can now implement this protocol for the data structures that would have a compliant implementation: + +```elixir +defimpl Size, for: BitString do + def size(string), do: byte_size(string) +end + +defimpl Size, for: Map do + def size(map), do: map_size(map) +end + +defimpl Size, for: Tuple do + def size(tuple), do: tuple_size(tuple) +end +``` + +We didn't implement the `Size` protocol for lists as there is no "size" information pre-computed for lists, and the length of a list has to be computed (with `length/1`). + +Now with the protocol defined and implementations in hand, we can start using it: + +```elixir +iex> Size.size("foo") +3 +iex> Size.size({:ok, "hello"}) +2 +iex> Size.size(%{label: "some label"}) +1 +``` + +Passing a data type that doesn't implement the protocol raises an error: + +```elixir +iex> Size.size([1, 2, 3]) +** (Protocol.UndefinedError) protocol Size not implemented for [1, 2, 3] of type List +``` + +It's possible to implement protocols for all Elixir data types: + + * `Atom` + * `BitString` + * `Float` + * `Function` + * `Integer` + * `List` + * `Map` + * `PID` + * `Port` + * `Reference` + * `Tuple` + +## Protocols and structs + +The power of Elixir's extensibility comes when protocols and structs are used together. + +In the [previous chapter](structs.md), we have learned that although structs are maps, they do not share protocol implementations with maps. For example, `MapSet`s (sets based on maps) are implemented as structs. Let's try to use the `Size` protocol with a `MapSet`: + +```elixir +iex> Size.size(%{}) +0 +iex> set = %MapSet{} = MapSet.new +MapSet.new([]) +iex> Size.size(set) +** (Protocol.UndefinedError) protocol Size not implemented for MapSet.new([]) of type MapSet (a struct) +``` + +Instead of sharing protocol implementation with maps, structs require their own protocol implementation. Since a `MapSet` has its size precomputed and accessible through `MapSet.size/1`, we can define a `Size` implementation for it: + +```elixir +defimpl Size, for: MapSet do + def size(set), do: MapSet.size(set) +end +``` + +If desired, you could come up with your own semantics for the size of your struct. Not only that, you could use structs to build more robust data types, like queues, and implement all relevant protocols, such as `Enumerable` and possibly `Size`, for this data type. + +```elixir +defmodule User do + defstruct [:name, :age] +end + +defimpl Size, for: User do + def size(_user), do: 2 +end +``` + +## Implementing `Any` + +Manually implementing protocols for all types can quickly become repetitive and tedious. In such cases, Elixir provides two options: we can explicitly derive the protocol implementation for our types or automatically implement the protocol for all types. In both cases, we need to implement the protocol for `Any`. + +### Deriving + +Elixir allows us to derive a protocol implementation based on the `Any` implementation. Let's first implement `Any` as follows: + +```elixir +defimpl Size, for: Any do + def size(_), do: 0 +end +``` + +The implementation above is arguably not a reasonable one. For example, it makes no sense to say that the size of a `PID` or an `Integer` is `0`. + +However, should we be fine with the implementation for `Any`, in order to use such implementation we would need to tell our struct to explicitly derive the `Size` protocol: + +```elixir +defmodule OtherUser do + @derive [Size] + defstruct [:name, :age] +end +``` + +When deriving, Elixir will implement the `Size` protocol for `OtherUser` based on the implementation provided for `Any`. + +### Fallback to `Any` + +Another alternative to `@derive` is to explicitly tell the protocol to fallback to `Any` when an implementation cannot be found. This can be achieved by setting `@fallback_to_any` to `true` in the protocol definition: + +```elixir +defprotocol Size do + @fallback_to_any true + def size(data) +end +``` + +As we said in the previous section, the implementation of `Size` for `Any` is not one that can apply to any data type. That's one of the reasons why `@fallback_to_any` is an opt-in behavior. For the majority of protocols, raising an error when a protocol is not implemented is the proper behavior. That said, assuming we have implemented `Any` as in the previous section: + +```elixir +defimpl Size, for: Any do + def size(_), do: 0 +end +``` + +Now all data types (including structs) that have not implemented the `Size` protocol will be considered to have a size of `0`. + +Which technique is best between deriving and falling back to `Any` depends on the use case but, given Elixir developers prefer explicit over implicit, you may see many libraries pushing towards the `@derive` approach. + +## Built-in protocols + +Elixir ships with some built-in protocols. In previous chapters, we have discussed the `Enum` module which provides many functions that work with any data structure that implements the `Enumerable` protocol: + +```elixir +iex> Enum.map([1, 2, 3], fn x -> x * 2 end) +[2, 4, 6] +iex> Enum.reduce(1..3, 0, fn x, acc -> x + acc end) +6 +``` + +Another useful example is the `String.Chars` protocol, which specifies how to convert a data structure to its human representation as a string. It's exposed via the `to_string` function: + +```elixir +iex> to_string(:hello) +"hello" +``` + +Notice that string interpolation in Elixir calls the `to_string` function: + +```elixir +iex> "age: #{25}" +"age: 25" +``` + +The snippet above only works because numbers implement the `String.Chars` protocol. Passing a tuple, for example, will lead to an error: + +```elixir +iex> tuple = {1, 2, 3} +{1, 2, 3} +iex> "tuple: #{tuple}" +** (Protocol.UndefinedError) protocol String.Chars not implemented for {1, 2, 3} of type Tuple +``` + +When there is a need to "print" a more complex data structure, one can use the `inspect` function, based on the `Inspect` protocol: + +```elixir +iex> "tuple: #{inspect(tuple)}" +"tuple: {1, 2, 3}" +``` + +The `Inspect` protocol is the protocol used to transform any data structure into a readable textual representation. This is what tools like IEx use to print results: + +```elixir +iex> {1, 2, 3} +{1, 2, 3} +iex> %User{} +%User{name: "john", age: 27} +``` + +Keep in mind that, by convention, whenever the inspected value starts with `#`, it is representing a data structure in non-valid Elixir syntax. This means the inspect protocol is not reversible as information may be lost along the way: + +```elixir +iex> inspect &(&1+2) +"#Function<6.71889879/1 in :erl_eval.expr/5>" +``` + +There are other protocols in Elixir, but this covers the most common ones. You can learn more about protocols and implementations in the `Protocol` module. diff --git a/lib/elixir/pages/getting-started/recursion.md b/lib/elixir/pages/getting-started/recursion.md new file mode 100644 index 00000000000..d1f0b0f6276 --- /dev/null +++ b/lib/elixir/pages/getting-started/recursion.md @@ -0,0 +1,139 @@ + + +# Recursion + +Elixir does not provide loop constructs. Instead we leverage recursion and high-level functions for working with collections. This chapter will explore the former. + +## Loops through recursion + +Due to immutability, loops in Elixir (as in any functional programming language) are written differently from imperative languages. For example, in an imperative language like C, one would write: + +```c +for(i = 0; i < sizeof(array); i++) { + array[i] = array[i] * 2; +} +``` + +In the example above, we are mutating both the array and the variable `i`. However, data structures in Elixir are immutable. For this reason, functional languages rely on recursion: a function is called recursively until a condition is reached that stops the recursive action from continuing. No data is mutated in this process. Consider the example below that prints a string an arbitrary number of times: + +```elixir +defmodule Recursion do + def print_multiple_times(msg, n) when n > 0 do + IO.puts(msg) + print_multiple_times(msg, n - 1) + end + + def print_multiple_times(_msg, 0) do + :ok + end +end + +Recursion.print_multiple_times("Hello!", 3) +# Hello! +# Hello! +# Hello! +:ok +``` + +Similar to `case`, a function may have many clauses. A particular clause is executed when the arguments passed to the function match the clause's argument patterns and its guards evaluate to `true`. + +When `print_multiple_times/2` is initially called in the example above, the argument `n` is equal to `3`. + +The first clause has a guard which says "use this definition if and only if `n` is more than `0`". Since this is the case, it prints the `msg` and then calls itself passing `n - 1` (`2`) as the second argument. + +Now we execute the same function again, starting from the first clause. Given the second argument, `n`, is still more than 0, we print the message and call ourselves once more, now with the second argument set to `1`. Then we print the message one last time and call `print_multiple_times("Hello!", 0)`, starting from the top once again. + +When the second argument is zero, the guard `n > 0` evaluates to false, and the first function clause won't execute. Elixir then proceeds to try the next function clause, which explicitly matches on the case where `n` is `0`. This clause, also known as the termination clause, ignores the message argument by assigning it to the `_msg` variable and returns the atom `:ok`. + +Finally, if you pass an argument that does not match any clause, Elixir raises a `FunctionClauseError`: + +```elixir +iex> Recursion.print_multiple_times("Hello!", -1) +** (FunctionClauseError) no function clause matching in Recursion.print_multiple_times/2 + + The following arguments were given to Recursion.print_multiple_times/2: + + # 1 + "Hello!" + + # 2 + -1 + + iex:1: Recursion.print_multiple_times/2 +``` + +## Reduce and map algorithms + +Let's now see how we can use the power of recursion to sum a list of numbers: + +```elixir +defmodule Math do + def sum_list([head | tail], accumulator) do + sum_list(tail, head + accumulator) + end + + def sum_list([], accumulator) do + accumulator + end +end + +IO.puts Math.sum_list([1, 2, 3], 0) #=> 6 +``` + +We invoke `sum_list` with the list `[1, 2, 3]` and the initial value `0` as arguments. We will try each clause until we find one that matches according to the pattern matching rules. In this case, the list `[1, 2, 3]` matches against `[head | tail]` which binds `head` to `1` and `tail` to `[2, 3]`; `accumulator` is set to `0`. + +Then, we add the head of the list to the accumulator `head + accumulator` and call `sum_list` again, recursively, passing the tail of the list as its first argument. The tail will once again match `[head | tail]` until the list is empty, as seen below: + +```elixir +sum_list([1, 2, 3], 0) +sum_list([2, 3], 1) +sum_list([3], 3) +sum_list([], 6) +``` + +When the list is empty, it will match the final clause which returns the final result of `6`. + +The process of taking a list and _reducing_ it down to one value is known as a _reduce algorithm_ and is central to functional programming. + +What if we instead want to double all of the values in our list? + +```elixir +defmodule Math do + def double_each([head | tail]) do + [head * 2 | double_each(tail)] + end + + def double_each([]) do + [] + end +end + +Math.double_each([1, 2, 3]) #=> [2, 4, 6] +``` + +Here we have used recursion to traverse a list, doubling each element and returning a new list. The process of taking a list and _mapping_ over it is known as a _map algorithm_. + +Recursion and [tail call](https://en.wikipedia.org/wiki/Tail_call) optimization are an important part of Elixir and are commonly used to create loops. However, when programming in Elixir you will rarely use recursion as above to manipulate lists. + +The `Enum` module, which we're going to see in the next chapter already provides many conveniences for working with lists. For instance, the examples above could be written as: + +```elixir +iex> Enum.reduce([1, 2, 3], 0, fn x, acc -> x + acc end) +6 +iex> Enum.map([1, 2, 3], fn x -> x * 2 end) +[2, 4, 6] +``` + +Or, using the [capture syntax](`Kernel.SpecialForms.&/1`): + +```elixir +iex> Enum.reduce([1, 2, 3], 0, &+/2) +6 +iex> Enum.map([1, 2, 3], &(&1 * 2)) +[2, 4, 6] +``` + +Let's take a deeper look at `Enumerable` and, while we're at it, its lazy counterpart, `Stream`. diff --git a/lib/elixir/pages/getting-started/sigils.md b/lib/elixir/pages/getting-started/sigils.md new file mode 100644 index 00000000000..6ac0d2004a9 --- /dev/null +++ b/lib/elixir/pages/getting-started/sigils.md @@ -0,0 +1,245 @@ + + +# Sigils + +Elixir provides double-quoted strings as well as a concept called charlists, which are defined using the `~c"hello world"` sigil syntax. In this chapter, we will learn more about sigils and how to define our own. + +One of Elixir's goals is extensibility: developers should be able to extend the language to fit any particular domain. Sigils provide the foundation for extending the language with custom textual representations. Sigils start with the tilde (`~`) character which is followed by either a single lower-case letter or one or more upper-case letters, and then a delimiter. Optional modifiers are added after the final delimiter. + +## Regular expressions + +The most common sigil in Elixir is `~r`, which is used to create [regular expressions](https://en.wikipedia.org/wiki/Regular_Expressions): + +```elixir +# A regular expression that matches strings which contain "foo" or "bar": +iex> regex = ~r/foo|bar/ +~r/foo|bar/ +iex> "foo" =~ regex +true +iex> "bat" =~ regex +false +``` + +Elixir provides Perl-compatible regular expressions (regexes), as implemented by the [PCRE](http://www.pcre.org/) library. Regexes also support modifiers. For example, the `i` modifier makes a regular expression case insensitive: + +```elixir +iex> "HELLO" =~ ~r/hello/ +false +iex> "HELLO" =~ ~r/hello/i +true +``` + +Check out the `Regex` module for more information on other modifiers and the supported operations with regular expressions. + +So far, all examples have used `/` to delimit a regular expression. However, sigils support 8 different delimiters: + +```elixir +~r/hello/ +~r|hello| +~r"hello" +~r'hello' +~r(hello) +~r[hello] +~r{hello} +~r +``` + +The reason behind supporting different delimiters is to provide a way to write literals without escaped delimiters. For example, a regular expression with forward slashes like `~r(^https?://)` reads arguably better than `~r/^https?:\/\//`. Similarly, if the regular expression has forward slashes and capturing groups (that use `()`), you may then choose double quotes instead of parentheses. + +## Strings, charlists, and word lists sigils + +Elixir ships with three sigils for building textual data structures. These allow you to choose an appropriate delimiter for your literal text such that you do not have to worry about escaping. + +### Strings + +The `~s` sigil is used to generate strings, like double quotes are. The `~s` sigil is useful when a string contains double quotes: + +```elixir +iex> ~s(this is a string with "double" quotes, not 'single' ones) +"this is a string with \"double\" quotes, not 'single' ones" +``` + +### Charlists + +The `~c` sigil is the regular way to represent charlists. + +```elixir +iex> [?c, ?a, ?t] +~c"cat" +iex> ~c(this is a char list containing "double quotes") +~c"this is a char list containing \"double quotes\"" +``` + +### Word lists + +The `~w` sigil is used to generate lists of words (*words* are just regular strings). Inside the `~w` sigil, words are separated by whitespace. + +```elixir +iex> ~w(foo bar bat) +["foo", "bar", "bat"] +``` + +The `~w` sigil also accepts the `c`, `s` and `a` modifiers (for charlists, strings, and atoms, respectively), which specify the data type of the elements of the resulting list: + +```elixir +iex> ~w(foo bar bat)a +[:foo, :bar, :bat] +``` + +## Interpolation and escaping in textual sigils + +Elixir supports some sigil variants to deal with escaping characters and interpolation. In particular, uppercase-letter textual sigils do not perform interpolation nor escaping. For example, although both `~s` and `~S` will return strings, the former allows escape codes and interpolation while the latter does not: + +```elixir +iex> ~s(String with escape codes \x26 #{"inter" <> "polation"}) +"String with escape codes & interpolation" +iex> ~S(String without escape codes \x26 without #{interpolation}) +"String without escape codes \\x26 without \#{interpolation}" +``` + +The following escape codes can be used in textual sigils: + + * `\\` – single backslash + * `\a` – bell/alert + * `\b` – backspace + * `\d` - delete + * `\e` - escape + * `\f` - form feed + * `\n` – newline + * `\r` – carriage return + * `\s` – space + * `\t` – tab + * `\v` – vertical tab + * `\0` - null byte + * `\xDD` - represents a single byte in hexadecimal (such as `\x13`) + * `\uDDDD` and `\u{D...}` - represents a Unicode codepoint in hexadecimal (such as `\u{1F600}`) + +In addition to those, a double quote inside a double-quoted string needs to be escaped as `\"`, and, analogously, a single quote inside a single-quoted char list needs to be escaped as `\'`. Nevertheless, it is better style to change delimiters as seen above than to escape them. + +Sigils also support heredocs, that is, three double-quotes or single-quotes as separators: + +```elixir +iex> ~s""" +...> this is +...> a heredoc string +...> """ +``` + +The most common use case for heredoc sigils is when writing documentation. For example, writing escape characters in the documentation would soon become error prone because of the need to double-escape some characters: + +```elixir +@doc """ +Converts double-quotes to single-quotes. + +## Examples + + iex> convert("\\\"foo\\\"") + "'foo'" + +""" +def convert(...) +``` + +By using `~S`, this problem can be avoided altogether: + +```elixir +@doc ~S""" +Converts double-quotes to single-quotes. + +## Examples + + iex> convert("\"foo\"") + "'foo'" + +""" +def convert(...) +``` + +## Calendar sigils + +Elixir offers several sigils to deal with various flavors of times and dates. + +### Date + +A [%Date{}](`Date`) struct contains the fields `year`, `month`, `day`, and `calendar`. You can create one using the `~D` sigil: + +```elixir +iex> d = ~D[2019-10-31] +~D[2019-10-31] +iex> d.day +31 +``` + +### Time + +The [%Time{}](`Time`) struct contains the fields `hour`, `minute`, `second`, `microsecond`, and `calendar`. You can create one using the `~T` sigil: + +```elixir +iex> t = ~T[23:00:07.0] +~T[23:00:07.0] +iex> t.second +7 +``` + +### NaiveDateTime + +The [%NaiveDateTime{}](`NaiveDateTime`) struct contains fields from both `Date` and `Time`. You can create one using the `~N` sigil: + +```elixir +iex> ndt = ~N[2019-10-31 23:00:07] +~N[2019-10-31 23:00:07] +``` + +Why is it called naive? Because it does not contain timezone information. Therefore, the given datetime may not exist at all or it may exist twice in certain timezones - for example, when we move the clock back and forward for daylight saving time. + +### UTC DateTime + +A [%DateTime{}](`DateTime`) struct contains the same fields as a `NaiveDateTime` with the addition of fields to track timezones. The `~U` sigil allows developers to create a DateTime in the UTC timezone: + +```elixir +iex> dt = ~U[2019-10-31 19:59:03Z] +~U[2019-10-31 19:59:03Z] +iex> %DateTime{minute: minute, time_zone: time_zone} = dt +~U[2019-10-31 19:59:03Z] +iex> minute +59 +iex> time_zone +"Etc/UTC" +``` + +## Custom sigils + +As hinted at the beginning of this chapter, sigils in Elixir are extensible. In fact, using the sigil `~r/foo/i` is equivalent to calling `sigil_r` with a binary and a char list as the argument: + +```elixir +iex> sigil_r(<<"foo">>, [?i]) +~r"foo"i +``` + +We can access the documentation for the `~r` sigil via `sigil_r`: + +```elixir +iex> h sigil_r +... +``` + +We can also provide our own sigils by implementing functions that follow the `sigil_{character}` pattern. For example, let's implement the `~i` sigil that returns an integer (with the optional `n` modifier to make it negative): + +```elixir +iex> defmodule MySigils do +...> def sigil_i(string, []), do: String.to_integer(string) +...> def sigil_i(string, [?n]), do: -String.to_integer(string) +...> end +iex> import MySigils +iex> ~i(13) +13 +iex> ~i(42)n +-42 +``` + +Custom sigils may be either a single lowercase character, or an uppercase character followed by more uppercase characters and digits. + +Sigils can also be used to do compile-time work with the help of macros. For example, regular expressions in Elixir are compiled into an efficient representation during compilation of the source code, therefore skipping this step at runtime. If you're interested in the subject, you can learn more about macros and check out how sigils are implemented in the `Kernel` module (where the `sigil_*` functions are defined). diff --git a/lib/elixir/pages/getting-started/structs.md b/lib/elixir/pages/getting-started/structs.md new file mode 100644 index 00000000000..ac71fa8503a --- /dev/null +++ b/lib/elixir/pages/getting-started/structs.md @@ -0,0 +1,169 @@ + + +# Structs + +We learned about maps [in earlier chapters](keywords-and-maps.md): + +```elixir +iex> map = %{a: 1, b: 2} +%{a: 1, b: 2} +iex> map[:a] +1 +iex> %{map | a: 3} +%{a: 3, b: 2} +``` + +Structs are extensions built on top of maps that provide compile-time checks and default values. + +## Defining structs + +To define a struct, the `defstruct/1` construct is used: + +```elixir +iex> defmodule User do +...> defstruct name: "John", age: 27 +...> end +``` + +The keyword list used with `defstruct` defines what fields the struct will have along with their default values. Structs take the name of the module they're defined in. In the example above, we defined a struct named `User`. + +We can now create `User` structs by using a syntax similar to the one used to create maps: + +```elixir +iex> %User{} +%User{age: 27, name: "John"} +iex> %User{name: "Jane"} +%User{age: 27, name: "Jane"} +``` + +Structs provide *compile-time* guarantees that only the fields defined through `defstruct` will be allowed to exist in a struct: + +```elixir +iex> %User{oops: :field} +** (KeyError) key :oops not found expanding struct: User.__struct__/1 +``` + +## Accessing and updating structs + +Structs share the same syntax for accessing and updating fields as maps of fixed keys: + +```elixir +iex> john = %User{} +%User{age: 27, name: "John"} +iex> john.name +"John" +iex> jane = %{john | name: "Jane"} +%User{age: 27, name: "Jane"} +iex> %{jane | oops: :field} +** (KeyError) key :oops not found in: %User{age: 27, name: "Jane"} +``` + +When using the update syntax (`|`), Elixir is aware that no new keys will be added to the struct, allowing the maps underneath to share their structure in memory. In the example above, both `john` and `jane` share the same key structure in memory. + +Structs can also be used in pattern matching, both for matching on the value of specific keys as well as for ensuring that the matching value is a struct of the same type as the matched value. + +```elixir +iex> %User{name: name} = john +%User{age: 27, name: "John"} +iex> name +"John" +iex> %User{} = %{} +** (MatchError) no match of right hand side value: %{} +``` + +For more details on creating, updating, and pattern matching structs, see the documentation for `%/2`. + +## Dynamic struct updates + +When you need to update structs with data from keyword lists or maps, use `Kernel.struct!/2`: + +```elixir +iex> john = %User{name: "John", age: 27} +%User{age: 27, name: "John"} +iex> updates = [name: "Jane", age: 30] +[name: "Jane", age: 30] +iex> struct!(john, updates) +%User{age: 30, name: "Jane"} +``` + +`struct!/2` will raise an error if you try to set invalid fields: + +```elixir +iex> struct!(john, invalid: "field") +** (KeyError) key :invalid not found in: %User{age: 27, name: "John"} +``` + +Use the map update syntax (`%{john | name: "Jane"}`) when you know the exact fields at compile time. Always use `struct!/2` instead of `Map` functions to preserve struct integrity. + +## Structs are bare maps underneath + +Structs are simply maps with a "special" field named `__struct__` that holds the name of the struct: + +```elixir +iex> is_map(john) +true +iex> john.__struct__ +User +``` + +However, structs do not inherit any of the protocols that maps do. For example, you can neither enumerate nor access a struct: + +```elixir +iex> john = %User{} +%User{age: 27, name: "John"} +iex> john[:name] +** (UndefinedFunctionError) function User.fetch/2 is undefined (User does not implement the Access behaviour) + User.fetch(%User{age: 27, name: "John"}, :name) +iex> Enum.each(john, fn {field, value} -> IO.puts(value) end) +** (Protocol.UndefinedError) protocol Enumerable not implemented for %User{age: 27, name: "John"} of type User (a struct) +``` + +Structs alongside protocols provide one of the most important features for Elixir developers: data polymorphism. That's what we will explore in the next chapter. + +## Default values and required keys + +If you don't specify a default key value when defining a struct, `nil` will be assumed: + +```elixir +iex> defmodule Product do +...> defstruct [:name] +...> end +iex> %Product{} +%Product{name: nil} +``` + +You can define a structure combining both fields with explicit default values, and implicit `nil` values. In this case you must first specify the fields which implicitly default to `nil`: + +```elixir +iex> defmodule User do +...> defstruct [:email, name: "John", age: 27] +...> end +iex> %User{} +%User{age: 27, email: nil, name: "John"} +``` + +Doing it in reverse order will raise a syntax error: + +```elixir +iex> defmodule User do +...> defstruct [name: "John", age: 27, :email] +...> end +** (SyntaxError) iex:107: unexpected expression after keyword list. Keyword lists must always come last in lists and maps. +``` + +You can also enforce that certain keys have to be specified when creating the struct via the `@enforce_keys` module attribute: + +```elixir +iex> defmodule Car do +...> @enforce_keys [:make] +...> defstruct [:model, :make] +...> end +iex> %Car{} +** (ArgumentError) the following keys must also be given when building struct Car: [:make] + expanding struct: Car.__struct__/1 +``` + +Enforcing keys provides a simple compile-time guarantee to aid developers when building structs. It is not enforced on updates and it does not provide any sort of value-validation. diff --git a/lib/elixir/pages/getting-started/try-catch-and-rescue.md b/lib/elixir/pages/getting-started/try-catch-and-rescue.md new file mode 100644 index 00000000000..09b3c636aad --- /dev/null +++ b/lib/elixir/pages/getting-started/try-catch-and-rescue.md @@ -0,0 +1,306 @@ + + +# try, catch, and rescue + +Elixir has three error mechanisms: errors, throws, and exits. In this chapter, we will explore each of them and include remarks about when each should be used. + +## Errors + +Errors (or *exceptions*) are used when exceptional things happen in the code. A sample error can be retrieved by trying to add a number to an atom: + +```elixir +iex> :foo + 1 +** (ArithmeticError) bad argument in arithmetic expression + :erlang.+(:foo, 1) +``` + +A runtime error can be raised any time by using `raise/1`: + +```elixir +iex> raise "oops" +** (RuntimeError) oops +``` + +Other errors can be raised with `raise/2` passing the error name and a list of keyword arguments: + +```elixir +iex> raise ArgumentError, message: "invalid argument foo" +** (ArgumentError) invalid argument foo +``` + +You can also define your own errors by creating a module and using the `defexception/1` construct inside it. This way, you'll create an error with the same name as the module it's defined in. The most common case is to define a custom exception with a message field: + +```elixir +iex> defmodule MyError do +iex> defexception message: "default message" +iex> end +iex> raise MyError +** (MyError) default message +iex> raise MyError, message: "custom message" +** (MyError) custom message +``` + +Errors can be **rescued** using the `try/rescue` construct: + +```elixir +iex> try do +...> raise "oops" +...> rescue +...> e in RuntimeError -> e +...> end +%RuntimeError{message: "oops"} +``` + +The example above rescues the runtime error and returns the exception itself, which is then printed in the `iex` session. + +If you don't have any use for the exception, you don't have to pass a variable to `rescue`: + +```elixir +iex> try do +...> raise "oops" +...> rescue +...> RuntimeError -> "Error!" +...> end +"Error!" +``` + +In practice, Elixir developers rarely use the `try/rescue` construct. For example, many languages would force you to rescue an error when a file cannot be opened successfully. Elixir instead provides a `File.read/1` function which returns a tuple containing information about whether the file was opened successfully: + +```elixir +iex> File.read("hello") +{:error, :enoent} +iex> File.write("hello", "world") +:ok +iex> File.read("hello") +{:ok, "world"} +``` + +There is no `try/rescue` here. In case you want to handle multiple outcomes of opening a file, you can use pattern matching using the `case` construct: + +```elixir +iex> case File.read("hello") do +...> {:ok, body} -> IO.puts("Success: #{body}") +...> {:error, reason} -> IO.puts("Error: #{reason}") +...> end +``` + +For the cases where you do expect a file to exist (and the lack of that file is truly an *error*) you may use `File.read!/1`: + +```elixir +iex> File.read!("unknown") +** (File.Error) could not read file "unknown": no such file or directory + (elixir) lib/file.ex:272: File.read!/1 +``` + +At the end of the day, it's up to your application to decide if an error while opening a file is exceptional or not. That's why Elixir doesn't impose exceptions on `File.read/1` and many other functions. Instead, it leaves it up to the developer to choose the best way to proceed. + +Many functions in the standard library follow the pattern of having a counterpart that raises an exception instead of returning tuples to match against. The convention is to create a function (`foo`) which returns `{:ok, result}` or `{:error, reason}` tuples and another function (`foo!`, same name but with a trailing `!`) that takes the same arguments as `foo` but which raises an exception if there's an error. `foo!` should return the result (not wrapped in a tuple) if everything goes fine. The `File` module is a good example of this convention. + +### Fail fast / Let it crash + +One saying that is common in the Erlang community, as well as Elixir's, is "fail fast" / "let it crash". The idea behind let it crash is that, in case something *unexpected* happens, it is best to let the exception happen, without rescuing it. + +It is important to emphasize the word *unexpected*. For example, imagine you are building a script to process files. Your script receives filenames as inputs. It is expected that users may make mistakes and provide unknown filenames. In this scenario, while you could use `File.read!/1` to read files and let it crash in case of invalid filenames, it probably makes more sense to use `File.read/1` and provide users of your script with a clear and precise feedback of what went wrong. + +Other times, you may fully expect a certain file to exist, and in case it does not, it means something terribly wrong has happened elsewhere. In such cases, `File.read!/1` is all you need. + +The second approach also works because, as discussed in the [Processes](processes.md) chapter, all Elixir code runs inside processes that are isolated and don't share anything by default. Therefore, an unhandled exception in a process will never crash or corrupt the state of another process. This allows us to define supervisor processes, which are meant to observe when a process terminates unexpectedly, and start a new one in its place. + +At the end of the day, "fail fast" / "let it crash" is a way of saying that, when *something unexpected* happens, it is best to start from scratch within a new process, freshly started by a supervisor, rather than blindly trying to rescue all possible error cases without the full context of when and how they can happen. + +### Reraise + +While we generally avoid using `try/rescue` in Elixir, one situation where we may want to use such constructs is for observability/monitoring. Imagine you want to log that something went wrong, you could do: + +```elixir +try do + ... some code ... +rescue + e -> + Logger.error(Exception.format(:error, e, __STACKTRACE__)) + reraise e, __STACKTRACE__ +end +``` + +In the example above, we rescued the exception, logged it, and then re-raised it. We use the `__STACKTRACE__` construct both when formatting the exception and when re-raising. This ensures we reraise the exception as is, without changing value or its origin. + +Generally speaking, we take errors in Elixir literally: they are reserved for unexpected and/or exceptional situations, never for controlling the flow of our code. In case you actually need flow control constructs, *throws* should be used. That's what we are going to see next. + +## Throws + +In Elixir, a value can be thrown and later be caught. `throw` and `catch` are reserved for situations where it is not possible to retrieve a value unless by using `throw` and `catch`. + +Those situations are quite uncommon in practice except when interfacing with libraries that do not provide a proper API. For example, let's imagine the `Enum` module did not provide any API for finding a value and that we needed to find the first multiple of 13 in a list of numbers: + +```elixir +iex> try do +...> Enum.each(-50..50, fn x -> +...> if rem(x, 13) == 0, do: throw(x) +...> end) +...> "Got nothing" +...> catch +...> x -> "Got #{x}" +...> end +"Got -39" +``` + +Since `Enum` *does* provide a proper API, in practice `Enum.find/2` is the way to go: + +```elixir +iex> Enum.find(-50..50, &(rem(&1, 13) == 0)) +-39 +``` + +## Exits + +All Elixir code runs inside processes that communicate with each other. When a process dies of "natural causes" (e.g., unhandled exceptions), it sends an `exit` signal. A process can also die by explicitly sending an `exit` signal: + +```elixir +iex> spawn_link(fn -> exit(1) end) +** (EXIT from #PID<0.56.0>) shell process exited with reason: 1 +``` + +In the example above, the linked process died by sending an `exit` signal with a value of 1. The Elixir shell automatically handles those messages and prints them to the terminal. + +`exit` can also be "caught" using `try/catch`: + +```elixir +iex> try do +...> exit("I am exiting") +...> catch +...> :exit, _ -> "not really" +...> end +"not really" +``` + +`catch` can also be used within a function body without a matching `try`. + +```elixir +defmodule Example do + def matched_catch do + exit(:timeout) + catch + :exit, :timeout -> + {:error, :timeout} + end + + def mismatched_catch do + exit(:timeout) + catch + # Since no clause matches, this catch will have no effect + :exit, :explosion -> + {:error, :explosion} + end +end +``` + +However, using `try/catch` is already uncommon and using it to catch exits is even rarer. + +`exit` signals are an important part of the fault tolerant system provided by the Erlang VM. Processes usually run under supervision trees which are themselves processes that listen to `exit` signals from the supervised processes. Once an `exit` signal is received, the supervision strategy kicks in and the supervised process is restarted. + +It is exactly this supervision system that makes constructs like `try/catch` and `try/rescue` so uncommon in Elixir. Instead of rescuing an error, we'd rather "fail fast" since the supervision tree will guarantee our application will go back to a known initial state after the error. + +## After + +Sometimes it's necessary to ensure that a resource is cleaned up after some action that could potentially raise an error. The `try/after` construct allows you to do that. For example, we can open a file and use an `after` clause to close it—even if something goes wrong: + +```elixir +iex> {:ok, file} = File.open("sample", [:utf8, :write]) +iex> try do +...> IO.write(file, "olá") +...> raise "oops, something went wrong" +...> after +...> File.close(file) +...> end +** (RuntimeError) oops, something went wrong +``` + +The `after` clause will be executed regardless of whether or not the tried block succeeds. Note, however, that if a linked process exits, +this process will exit and the `after` clause will not get run. Thus `after` provides only a soft guarantee. Luckily, files in Elixir are also linked to the current processes and therefore they will always get closed if the current process crashes, independent of the +`after` clause. You will find the same to be true for other resources like ETS tables, sockets, ports and more. + +Sometimes you may want to wrap the entire body of a function in a `try` construct, often to guarantee some code will be executed afterwards. In such cases, Elixir allows you to omit the `try` line: + +```elixir +iex> defmodule RunAfter do +...> def without_even_trying do +...> raise "oops" +...> after +...> IO.puts("cleaning up!") +...> end +...> end +iex> RunAfter.without_even_trying +cleaning up! +** (RuntimeError) oops +``` + +Elixir will automatically wrap the function body in a `try` whenever one of `after`, `rescue` or `catch` is specified. The `after` block handles side effects and does not change the return value from the clauses above it. + +## Else + +If an `else` block is present, it will match on the results of the `try` block whenever the `try` block finishes without a throw or an error. + +```elixir +iex> x = 2 +2 +iex> try do +...> 1 / x +...> rescue +...> ArithmeticError -> +...> :infinity +...> else +...> y when y < 1 and y > -1 -> +...> :small +...> _ -> +...> :large +...> end +:small +``` + +Exceptions in the `else` block are not caught. If no pattern inside the `else` block matches, an exception will be raised; this exception is not caught by the current `try/catch/rescue/after` block. + +## Variables scope + +Similar to `case`, `cond`, `if` and other constructs in Elixir, variables defined inside `try/catch/rescue/after` blocks do not leak to the outer context. In other words, this code is invalid: + +```elixir +iex> try do +...> raise "fail" +...> what_happened = :did_not_raise +...> rescue +...> _ -> what_happened = :rescued +...> end +iex> what_happened +** (CompileError) undefined variable "what_happened" +``` + +Instead, you should return the value of the `try` expression: + +```elixir +iex> what_happened = +...> try do +...> raise "fail" +...> :did_not_raise +...> rescue +...> _ -> :rescued +...> end +iex> what_happened +:rescued +``` + +Furthermore, variables defined in the do-block of `try` are not available inside `rescue/after/else` either. This is because the `try` block may fail at any moment and therefore the variables may have never been bound in the first place. So this also isn't valid: + +```elixir +iex> try do +...> raise "fail" +...> another_what_happened = :did_not_raise +...> rescue +...> _ -> another_what_happened +...> end +** (CompileError) undefined variable "another_what_happened" +``` + +This finishes our introduction on `try`, `catch`, and `rescue`. You will find they are used less frequently in Elixir than in other languages. Next we will talk about a very important subject to Elixir developers: writing documentation. diff --git a/lib/elixir/pages/getting-started/writing-documentation.md b/lib/elixir/pages/getting-started/writing-documentation.md new file mode 100644 index 00000000000..2f47c44dab0 --- /dev/null +++ b/lib/elixir/pages/getting-started/writing-documentation.md @@ -0,0 +1,181 @@ + + +# Writing documentation + +Elixir treats documentation as a first-class citizen. Documentation must be easy to write and easy to read. In this guide you will learn how to write documentation in Elixir, covering constructs like module attributes, style practices, and doctests. + +## Markdown + +Elixir documentation is written using Markdown. There are plenty of guides on Markdown online, we recommend the one from GitHub as a getting started point: + + * [Basic writing and formatting syntax](https://help.github.com/articles/basic-writing-and-formatting-syntax/) + +## Module Attributes + +Documentation in Elixir is usually attached to module attributes. Let's see an example: + +```elixir +defmodule MyApp.Hello do + @moduledoc """ + This is the Hello module. + """ + @moduledoc since: "1.0.0" + + @doc """ + Says hello to the given `name`. + + Returns `:ok`. + + ## Examples + + iex> MyApp.Hello.world(:john) + :ok + + """ + @doc since: "1.3.0" + def world(name) do + IO.puts("hello #{name}") + end +end +``` + +The `@moduledoc` attribute is used to add documentation to the module. `@doc` is used before a function to provide documentation for it. Besides the attributes above, `@typedoc` can also be used to attach documentation to types defined as part of typespecs. + +## Function arguments + +When documenting a function, argument names are inferred by the compiler. For example: + +```elixir +def size(%{size: size}) do + size +end +``` + +The compiler will infer this argument as `map`. Sometimes the inference will be suboptimal, especially if the function contains multiple clauses with the argument matching on different values each time. You can specify the proper names for documentation by declaring only the function head at any moment before the implementation: + +```elixir +def size(map_with_size) + +def size(%{size: size}) do + size +end +``` + +## Documentation metadata + +Elixir allows developers to attach arbitrary metadata to the documentation. This is done by passing a keyword list to the relevant attribute (such as `@moduledoc`, `@typedoc`, and `@doc`). + +Metadata can have any key. Documentation tools often use metadata to provide more data to readers and to enrich the user experience. The following keys already have a predefined meaning used by tooling: + +### `:deprecated` + +Another common metadata is `:deprecated`, which emits a warning in the documentation, explaining that its usage is discouraged: + +```elixir +@doc deprecated: "Use Foo.bar/2 instead" +``` + +Note that the `:deprecated` key does not warn when a developer invokes the functions. If you want the code to also emit a warning, you can use the `@deprecated` attribute: + +```elixir +@deprecated "Use Foo.bar/2 instead" +``` + +### `:group` + +The group a function, callback or type belongs to. This is used in `iex` for autocompleting and also to automatically by [ExDoc](https://github.com/elixir-lang/ex_doc/) to group items in the sidebar: + +```elixir +@doc group: "Query" +def all(query) + +@doc group: "Schema" +def insert(schema) +``` + +### `:since` + +It annotates in which version that particular module, function, type, or callback was added: + +```elixir +@doc since: "1.3.0" +def world(name) do + IO.puts("hello #{name}") +end +``` + +## Recommendations + +When writing documentation: + + * Keep the first paragraph of the documentation concise and simple, typically one-line. Tools like [ExDoc](https://github.com/elixir-lang/ex_doc/) use the first line to generate a summary. + + * Reference modules by their full name. Markdown uses backticks (`` ` ``) to quote code. Elixir builds on top of that to automatically generate links when module or function names are referenced. For this reason, always use full module names. If you have a module called `MyApp.Hello`, always reference it as `` `MyApp.Hello` `` and never as `` `Hello` ``. + + * Reference functions by name and arity if they are local, as in `` `world/1` ``, or by module, name and arity if pointing to an external module: `` `MyApp.Hello.world/1` ``. + + * Reference a `@callback` by prepending `c:`, as in `` `c:world/1` ``. + + * Reference a `@type` by prepending `t:`, as in `` `t:values/0` ``. + + * Start new sections with second level Markdown headers `##`. First level headers are reserved for module and function names. + + * Place documentation before the first clause of multi-clause functions. Documentation is always per function and arity and not per clause. + + * Use the `:since` key in the documentation metadata to annotate whenever new functions or modules are added to your API. + +## Doctests + +We advise developers to include examples in their documentation, often under their own `## Examples` heading. To ensure examples do not get out of date, Elixir's test framework (ExUnit) provides a feature called doctests that allows developers to test the examples in their documentation. Doctests work by parsing out code samples starting with `iex>` from the documentation. You can read more about them at `ExUnit.DocTest`. + +## Documentation != Code comments + +Elixir treats documentation and code comments as different concepts. Documentation is an explicit contract between you and users of your Application Programming Interface (API), be they third-party developers, co-workers, or your future self. Modules and functions must always be documented if they are part of your API. + +Code comments are aimed at developers reading the code. They are useful for marking improvements, leaving notes (for example, why you had to resort to a workaround due to a bug in a library), and so forth. They are tied to the source code: you can completely rewrite a function and remove all existing code comments, and it will continue to behave the same, with no change to either its behavior or its documentation. + +Because private functions cannot be accessed externally, Elixir will warn if a private function has a `@doc` attribute and will discard its content. However, you can add code comments to private functions, as with any other piece of code, and we recommend developers to do so whenever they believe it will add relevant information to the readers and maintainers of such code. + +In summary, documentation is a contract with users of your API, who may not necessarily have access to the source code, whereas code comments are for those who interact directly with the source. You can learn and express different guarantees about your software by separating those two concepts. + +## Hiding internal modules and functions + +Besides the modules and functions libraries provide as part of their public interface, libraries may also implement important functionality that is not part of their API. While these modules and functions can be accessed, they are meant to be internal to the library and thus should not have documentation for end users. + +Conveniently, Elixir allows developers to hide modules and functions from the documentation, by setting `@doc false` to hide a particular function, or `@moduledoc false` to hide the whole module. If a module is hidden, you may even document the functions in the module, but the module itself won't be listed in the documentation: + +```elixir +defmodule MyApp.Hidden do + @moduledoc false + + @doc """ + This function won't be listed in docs. + """ + def function_that_wont_be_listed_in_docs do + # ... + end +end +``` + +In case you don't want to hide a whole module, you can hide functions individually: + +```elixir +defmodule MyApp.Sample do + @doc false + def add(a, b), do: a + b +end +``` + +However, keep in mind `@moduledoc false` or `@doc false` do not make a function private. The function above can still be invoked as `MyApp.Sample.add(1, 2)`. Not only that, if `MyApp.Sample` is imported, the `add/2` function will also be imported into the caller. For those reasons, be cautious when adding `@doc false` to functions, instead use one of these two options: + + * Move the undocumented function to a module with `@moduledoc false`, like `MyApp.Hidden`, ensuring the function won't be accidentally exposed or imported. Remember that you can use `@moduledoc false` to hide a whole module and still document each function with `@doc`. Tools will still ignore the module. + + * Start the function name with one or two underscores, for example, `__add__/2`. Functions starting with underscore are automatically treated as hidden, although you can also be explicit and add `@doc false`. The compiler does not import functions with leading underscores and they hint to anyone reading the code of their intended private usage. + +## `Code.fetch_docs/1` + +Elixir stores documentation inside pre-defined chunks in the bytecode. Documentation is not loaded into memory when modules are loaded, instead, it can be read from the bytecode in disk using the `Code.fetch_docs/1` function. The downside is that modules defined in-memory, like the ones defined in IEx, cannot have their documentation accessed as they do not write their bytecode to disk. diff --git a/lib/elixir/pages/images/kv-observer.png b/lib/elixir/pages/images/kv-observer.png new file mode 100644 index 00000000000..7527d7e5c80 Binary files /dev/null and b/lib/elixir/pages/images/kv-observer.png differ diff --git a/lib/elixir/pages/images/logo.png b/lib/elixir/pages/images/logo.png new file mode 100644 index 00000000000..fcb34f09d60 Binary files /dev/null and b/lib/elixir/pages/images/logo.png differ diff --git a/lib/elixir/pages/meta-programming/domain-specific-languages.md b/lib/elixir/pages/meta-programming/domain-specific-languages.md new file mode 100644 index 00000000000..6fc91f77809 --- /dev/null +++ b/lib/elixir/pages/meta-programming/domain-specific-languages.md @@ -0,0 +1,203 @@ + + +# Domain-Specific Languages (DSLs) + +[Domain-specific Languages (DSLs)](https://en.wikipedia.org/wiki/Domain-specific_language) are languages tailored to a specific application domain. You don't need macros in order to have a DSL: every data structure and every function you define in your module is part of your domain-specific language. + +For example, imagine we want to implement a `Validator` module which provides a data validation domain-specific language. We could implement it using data structures, functions, or macros. Let's see what those different DSLs would look like: + +```elixir +# 1. Data structures +import Validator +validate user, name: [length: 1..100], email: [matches: ~r/@/] + +# 2. Functions +import Validator +user +|> validate_length(:name, 1..100) +|> validate_matches(:email, ~r/@/) + +# 3. Macros + modules +defmodule MyValidator do + use Validator + validate_length :name, 1..100 + validate_matches :email, ~r/@/ +end + +MyValidator.validate(user) +``` + +Of all the approaches above, the first is definitely the most flexible. If our domain rules can be encoded with data structures, they are by far the easiest to compose and implement, as Elixir's standard library is filled with functions for manipulating different data types. + +The second approach uses function calls which better suits more complex APIs (for example, if you need to pass many options) and reads nicely in Elixir thanks to the pipe operator. + +The third approach uses macros, and is by far the most complex. It will take more lines of code to implement, it is hard and expensive to test (compared to testing simple functions), and it limits how the user may use the library since all validations need to be defined inside a module. + +To drive the point home, imagine you want to validate a certain attribute only if a given condition is met. We could easily achieve it with the first solution, by manipulating the data structure accordingly, or with the second solution by using conditionals (if/else) before invoking the function. However, it is impossible to do so with the macros approach unless its DSL is augmented. + +In other words: + +```text +data > functions > macros +``` + +That said, there are still cases where using macros and modules to build domain-specific languages is useful. Since we have explored data structures and function definitions in the Getting Started guide, this chapter will explore how to use macros and module attributes to tackle more complex DSLs. + +## Building our own test case + +The goal in this chapter is to build a module named `TestCase` that allows us to write the following: + +```elixir +defmodule MyTest do + use TestCase + + test "arithmetic operations" do + 4 = 2 + 2 + end + + test "list operations" do + [1, 2, 3] = [1, 2] ++ [3] + end +end + +MyTest.run() +``` + +In the example above, by using `TestCase`, we can write tests using the `test` macro, which defines a function named `run` to automatically run all tests for us. Our prototype will rely on the match operator (`=`) as a mechanism to do assertions. + +## The `test` macro + +Let's start by creating a module that defines and imports the `test` macro when used: + +```elixir +defmodule TestCase do + # Callback invoked by `use`. + # + # For now it returns a quoted expression that + # imports the module itself into the user code. + @doc false + defmacro __using__(_opts) do + quote do + import TestCase + end + end + + @doc """ + Defines a test case with the given description. + + ## Examples + + test "arithmetic operations" do + 4 = 2 + 2 + end + + """ + defmacro test(description, do: block) do + function_name = String.to_atom("test " <> description) + quote do + def unquote(function_name)(), do: unquote(block) + end + end +end +``` + +Assuming we defined `TestCase` in a file named `tests.exs`, we can open it up by running `iex tests.exs` and define our first tests: + +```elixir +iex> defmodule MyTest do +...> use TestCase +...> +...> test "hello" do +...> "hello" = "world" +...> end +...> end +``` + +For now, we don't have a mechanism to run tests, but we know that a function named `test hello` was defined behind the scenes. When we invoke it, it should fail: + +```elixir +iex> MyTest."test hello"() +** (MatchError) no match of right hand side value: "world" +``` + +## Storing information with attributes + +In order to finish our `TestCase` implementation, we need to be able to access all defined test cases. One way of doing this is by retrieving the tests at runtime via `__MODULE__.__info__(:functions)`, which returns a list of all functions in a given module. However, considering that we may want to store more information about each test besides the test name, a more flexible approach is required. + +When discussing module attributes in earlier chapters, we mentioned how they can be used as temporary storage. That's exactly the property we will apply in this section. + +In the `__using__/1` implementation, we will initialize a module attribute named `@tests` to an empty list, then store the name of each defined test in this attribute so the tests can be invoked from the `run` function. + +Here is the updated code for the `TestCase` module: + +```elixir +defmodule TestCase do + @doc false + defmacro __using__(_opts) do + quote do + import TestCase + + # Initialize @tests to an empty list + @tests [] + + # Invoke TestCase.__before_compile__/1 before the module is compiled + @before_compile TestCase + end + end + + @doc """ + Defines a test case with the given description. + + ## Examples + + test "arithmetic operations" do + 4 = 2 + 2 + end + + """ + defmacro test(description, do: block) do + function_name = String.to_atom("test " <> description) + quote do + # Prepend the newly defined test to the list of tests + @tests [unquote(function_name) | @tests] + def unquote(function_name)(), do: unquote(block) + end + end + + # This will be invoked right before the target module is compiled + # giving us the perfect opportunity to inject the `run/0` function + @doc false + defmacro __before_compile__(_env) do + quote do + def run do + Enum.each(@tests, fn name -> + IO.puts("Running #{name}") + apply(__MODULE__, name, []) + end) + end + end + end +end +``` + +By starting a new IEx session, we can now define our tests and run them: + +```elixir +iex> defmodule MyTest do +...> use TestCase +...> +...> test "hello" do +...> "hello" = "world" +...> end +...> end +iex> MyTest.run() +Running test hello +** (MatchError) no match of right hand side value: "world" +``` + +Although we have overlooked some details, this is the main idea behind creating domain-specific languages in Elixir via modules and macros. Macros enable us to return quoted expressions that are executed in the caller, which we can then use to transform code and store relevant information in the target module via module attributes. Finally, callbacks such as `@before_compile` allow us to inject code into the module when its definition is complete. + +Besides `@before_compile`, there are other useful module attributes like `@on_definition` and `@after_compile`, which you can read more about in the docs for `Module`. You can also find useful information about macros and the compilation environment in the documentation for the `Macro` and `Macro.Env`. diff --git a/lib/elixir/pages/meta-programming/macros.md b/lib/elixir/pages/meta-programming/macros.md new file mode 100644 index 00000000000..0fbf55db5af --- /dev/null +++ b/lib/elixir/pages/meta-programming/macros.md @@ -0,0 +1,292 @@ + + +# Macros + +Even though Elixir attempts its best to provide a safe environment for macros, most of the responsibility of writing clean code with macros falls on developers. Macros are harder to write than ordinary Elixir functions, and it's considered to be bad style to use them when they're not necessary. Write macros responsibly. + +Elixir already provides mechanisms to write your everyday code in a simple and readable fashion by using its data structures and functions. Macros should only be used as a last resort. Remember that **explicit is better than implicit**. **Clear code is better than concise code.** + +## Our first macro + +Macros in Elixir are defined via `defmacro/2`. + +> For this guide, we will be using files instead of running code samples in IEx. That's because the code samples will span multiple lines of code and typing them all in IEx can be counter-productive. You should be able to run the code samples by saving them into a `macros.exs` file and running it with `elixir macros.exs` or `iex macros.exs`. + +In order to better understand how macros work, let's create a new module where we are going to implement `unless` (which does the opposite of `if/2`), as a macro and as a function: + +```elixir +defmodule Unless do + def fun_unless(clause, do: expression) do + if(!clause, do: expression) + end + + defmacro macro_unless(clause, do: expression) do + quote do + if(!unquote(clause), do: unquote(expression)) + end + end +end +``` + +The function receives the arguments and passes them to `if/2`. However, as we learned in the [previous guide](quote-and-unquote.md), the macro will receive quoted expressions, inject them into the quote, and finally return another quoted expression. + +Let's start `iex` with the module above: + +```console +$ iex macros.exs +``` + +and play with those definitions: + +```elixir +iex> require Unless +iex> Unless.macro_unless(true, do: IO.puts("this should never be printed")) +nil +iex> Unless.fun_unless(true, do: IO.puts("this should never be printed")) +"this should never be printed" +nil +``` + +In our *macro* implementation, the sentence was not printed, although it was printed in our *function* implementation. That's because the arguments to a function call are evaluated before calling the function. However, macros do not evaluate their arguments. Instead, they receive the arguments as quoted expressions which are then transformed into other quoted expressions. In this case, we have rewritten our `unless` macro to become an `if/2` behind the scenes. + +In other words, when invoked as: + +```elixir +Unless.macro_unless(true, do: IO.puts("this should never be printed")) +``` + +Our `macro_unless` macro received the following: + +```elixir +macro_unless(true, [do: {{:., [], [{:__aliases__, [alias: false], [:IO]}, :puts]}, [], ["this should never be printed"]}]) +``` + +and it then returned a quoted expression as follows: + +```elixir +{:if, [], + [{:!, [], [true]}, + [do: {{:., [], + [{:__aliases__, + [], [:IO]}, + :puts]}, [], ["this should never be printed"]}]]} +``` + +We can actually verify that this is the case by using `Macro.expand_once/2`: + +```elixir +iex> expr = quote do: Unless.macro_unless(true, do: IO.puts("this should never be printed")) +iex> res = Macro.expand_once(expr, __ENV__) +iex> IO.puts(Macro.to_string(res)) +if(!true) do + IO.puts("this should never be printed") +end +:ok +``` + +`Macro.expand_once/2` receives a quoted expression and expands it according to the current environment. In this case, it expanded/invoked the `Unless.macro_unless/2` macro and returned its result. We then proceeded to convert the returned quoted expression to a string and print it (we will talk about `__ENV__` later in this chapter). + +That's what macros are all about. They are about receiving quoted expressions and transforming them into something else. +In fact, `if/2` in Elixir is implemented as a macro: + +```elixir +defmacro if(clause, do: expression) do + quote do + case unquote(clause) do + x when x in [false, nil] -> nil + _ -> unquote(expression) + end +end +``` + +Constructs such as `if/2`, `defmacro/2`, `def/2`, `defprotocol/2`, and many others used throughout the Elixir standard library are written in pure Elixir, often as a macro. This means that the constructs being used to build the language can be used by developers to extend the language to the domains they are working on. + +We can define any function and macro we want, including ones that override the built-in definitions provided by Elixir. The only exceptions are Elixir special forms which are not implemented in Elixir and therefore cannot be overridden. The full list of special forms is available in `Kernel.SpecialForms`. + +## Macro hygiene + +Elixir macros have "late resolution". This guarantees that a variable defined inside a quote won't conflict with a variable defined in the context where that macro is expanded. For example: + +```elixir +defmodule Hygiene do + defmacro no_interference do + quote do: a = 1 + end +end + +defmodule HygieneTest do + def go do + require Hygiene + a = 13 + Hygiene.no_interference() + a + end +end + +HygieneTest.go() +# => 13 +``` + +In the example above, even though the macro injects `a = 1`, it does not affect the variable `a` defined by the `go/0` function. If a macro wants to explicitly affect the context, it can use `var!/1`: + +```elixir +defmodule Hygiene do + defmacro interference do + quote do: var!(a) = 1 + end +end + +defmodule HygieneTest do + def go do + require Hygiene + a = 13 + Hygiene.interference() + a + end +end + +HygieneTest.go() +# => 1 +``` + +The code above will work but issue a warning: `variable "a" is unused`. The macro is overriding the original value and the original value is never used. + +Variable hygiene only works because Elixir annotates variables with their **context**. For example, a variable `x` defined on line 3 of a module would be represented as: + +```elixir +{:x, [line: 3], nil} +``` + +However, a quoted variable would be represented as: + +```elixir +defmodule Sample do + def quoted do + quote do: x + end +end + +Sample.quoted() #=> {:x, [line: 3], Sample} +``` + +Notice that the *third element* in the quoted variable is the atom `Sample`, instead of `nil`, which marks the variable as coming from the `Sample` module. Therefore, Elixir considers these two variables as coming from different contexts and handles them accordingly. + +Elixir provides similar mechanisms for imports and aliases too. This guarantees that a macro will behave as specified by its source module rather than conflicting with the target module where the macro is expanded. Hygiene can be bypassed under specific situations by using macros like `var!/2` and `alias!/1`, although one must be careful when using those as they directly change the user environment. + +Sometimes variable names might be dynamically created. In such cases, `Macro.var/2` can be used to define new variables: + +```elixir +defmodule Sample do + defmacro initialize_to_char_count(variables) do + Enum.map(variables, fn name -> + var = Macro.var(name, nil) + length = name |> Atom.to_string() |> String.length() + + quote do + unquote(var) = unquote(length) + end + end) + end + + def run do + initialize_to_char_count([:red, :green, :yellow]) + [red, green, yellow] + end +end + +> Sample.run() #=> [3, 5, 6] +``` + +Take note of the second argument to `Macro.var/2`. This is the **context** being used and will determine hygiene as described in the next section. Check out also `Macro.unique_var/2`, for cases when you need to generate variables with unique names. + +## The environment + +When calling `Macro.expand_once/2` earlier in this chapter, we used the special form `__ENV__/0`. + +`__ENV__/0` returns a `Macro.Env` struct which contains useful information about the compilation environment, including the current module, file, and line, all variables defined in the current scope, as well as imports, requires, and more: + +```elixir +iex> __ENV__.module +nil +iex> __ENV__.file +"iex" +iex> __ENV__.requires +[IEx.Helpers, Kernel, Kernel.Typespec] +iex> require Integer +nil +iex> __ENV__.requires +[IEx.Helpers, Integer, Kernel, Kernel.Typespec] +``` + +Many of the functions in the `Macro` module expect a `Macro.Env` environment. You can read more about these functions in `Macro` and learn more about the compilation environment in the `Macro.Env`. + +## Private macros + +Elixir also supports **private macros** via `defmacrop`. Like private functions, these macros are only available inside the module that defines them, and only at compilation time. + +It is important that a macro is defined before its usage. Failing to define a macro before its invocation will raise an error at runtime, since the macro won't be expanded and will be translated to a function call: + +```elixir +iex> defmodule Sample do +...> def four, do: two() + two() +...> defmacrop two, do: 2 +...> end +** (CompileError) iex:2: function two/0 undefined +``` + +## Write macros responsibly + +Macros are a powerful construct and Elixir provides many mechanisms to ensure they are used responsibly. + + * Macros are **hygienic**: by default, variables defined inside a macro are not going to affect the user code. Furthermore, function calls and aliases available in the macro context are not going to leak into the user context. + + * Macros are **lexical**: it is impossible to inject code or macros globally. In order to use a macro, you need to explicitly `require` or `import` the module that defines the macro. + + * Macros are **explicit**: it is impossible to run a macro without explicitly invoking it. For example, some languages allow developers to completely rewrite functions behind the scenes, often via parse transforms or via some reflection mechanisms. In Elixir, a macro must be explicitly invoked in the caller during compilation time. + + * Macros' language is clear: many languages provide syntax shortcuts for `quote` and `unquote`. In Elixir, we preferred to have them explicitly spelled out, in order to clearly delimit the boundaries of a macro definition and its quoted expressions. + +Even with such guarantees, the developer plays a big role when writing macros responsibly. If you are confident you need to resort to macros, remember that macros are not your API. Keep your macro definitions short, including their quoted contents. For example, instead of writing a macro like this: + +```elixir +defmodule MyModule do + defmacro my_macro(a, b, c) do + quote do + do_this(unquote(a)) + # ... + do_that(unquote(b)) + # ... + and_that(unquote(c)) + end + end +end +``` + +write: + +```elixir +defmodule MyModule do + defmacro my_macro(a, b, c) do + quote do + # Keep what you need to do here to a minimum + # and move everything else to a function + MyModule.do_this_that_and_that(unquote(a), unquote(b), unquote(c)) + end + end + + def do_this_that_and_that(a, b, c) do + do_this(a) + ... + do_that(b) + ... + and_that(c) + end +end +``` + +This makes your code clearer and easier to test and maintain, as you can invoke and test `do_this_that_and_that/3` directly. It also helps you design an actual API for developers that do not want to rely on macros. + +With this guide, we finish our introduction to macros. The next guide is a brief discussion on **DSLs** that shows how we can mix macros and module attributes to annotate and extend modules and functions. diff --git a/lib/elixir/pages/meta-programming/quote-and-unquote.md b/lib/elixir/pages/meta-programming/quote-and-unquote.md new file mode 100644 index 00000000000..127861538c6 --- /dev/null +++ b/lib/elixir/pages/meta-programming/quote-and-unquote.md @@ -0,0 +1,153 @@ + + +# Quote and unquote + +This guide aims to introduce the meta-programming techniques available in Elixir. The ability to represent an Elixir program by its own data structures is at the heart of meta-programming. This chapter starts by exploring those structures and the associated `quote/2` and `unquote/1` constructs, so we can take a look at macros in the next guide, and finally build our own domain specific language. + +## Quoting + +The building block of an Elixir program is a tuple with three elements. For example, the function call `sum(1, 2, 3)` is represented internally as: + +```elixir +{:sum, [], [1, 2, 3]} +``` + +You can get the representation of any expression by using the `quote/2` macro: + +```elixir +iex> quote do: sum(1, 2, 3) +{:sum, [], [1, 2, 3]} +``` + +The first element is the function name, the second is a keyword list containing metadata, and the third is the arguments list. + +Operators are also represented as such tuples: + +```elixir +iex> quote do: 1 + 2 +{:+, [context: Elixir, import: Kernel], [1, 2]} +``` + +Even a map is represented as a call to `%{}`: + +```elixir +iex> quote do: %{1 => 2} +{:%{}, [], [{1, 2}]} +``` + +Variables are represented using such triplets, with the difference that the last element is an atom, instead of a list: + +```elixir +iex> quote do: x +{:x, [], Elixir} +``` + +When quoting more complex expressions, we can see that the code is represented in such tuples, which are often nested inside each other in a structure resembling a tree. Many languages would call such representations an [*Abstract Syntax Tree*](https://en.wikipedia.org/wiki/Abstract_syntax_tree) (AST). Elixir calls them *quoted expressions*: + +```elixir +iex> quote do: sum(1, 2 + 3, 4) +{:sum, [], [1, {:+, [context: Elixir, import: Kernel], [2, 3]}, 4]} +``` + +Sometimes, when working with quoted expressions, it may be useful to get the textual code representation back. This can be done with `Macro.to_string/1`: + +```elixir +iex> Macro.to_string(quote do: sum(1, 2 + 3, 4)) +"sum(1, 2 + 3, 4)" +``` + +In general, the tuples above are structured according to the following format: + +```elixir +{atom | tuple, list, list | atom} +``` + + * The first element is an atom or another tuple in the same representation; + * The second element is a keyword list containing metadata, like numbers and contexts; + * The third element is either a list of arguments for the function call or an atom. When this element is an atom, it means the tuple represents a variable. + +Besides the tuple defined above, there are five Elixir literals that, when quoted, return themselves (and not a tuple). They are: + +```elixir +:sum #=> Atoms +1.0 #=> Numbers +[1, 2] #=> Lists +"strings" #=> Strings +{key, value} #=> Tuples with two elements +``` + +Most Elixir code has a straight-forward translation to its underlying quoted expression. We recommend you try out different code samples and see what the results are. For example, what does `String.upcase("foo")` expand to? We have also learned that `if(true, do: :this, else: :that)` is the same as `if true do :this else :that end`. How does this affirmation hold with quoted expressions? + +## Unquoting + +Quoting is about retrieving the inner representation of some particular chunk of code. However, sometimes it may be necessary to inject some other particular chunk of code inside the representation we want to retrieve. + +For example, imagine you have a variable called `number` which contains the number you want to inject inside a quoted expression. + +```elixir +iex> number = 13 +iex> Macro.to_string(quote do: 11 + number) +"11 + number" +``` + +That's not what we wanted, since the value of the `number` variable has not been injected and `number` has been quoted in the expression. In order to inject the *value* of the `number` variable, `unquote/1` has to be used inside the quoted representation: + +```elixir +iex> number = 13 +iex> Macro.to_string(quote do: 11 + unquote(number)) +"11 + 13" +``` + +`unquote/1` can even be used to inject function names: + +```elixir +iex> fun = :hello +iex> Macro.to_string(quote do: unquote(fun)(:world)) +"hello(:world)" +``` + +In some cases, it may be necessary to inject many values inside a list. For example, imagine you have a list containing `[1, 2, 6]`, and we want to inject `[3, 4, 5]` into it. Using `unquote/1` won't yield the desired result: + +```elixir +iex> inner = [3, 4, 5] +iex> Macro.to_string(quote do: [1, 2, unquote(inner), 6]) +"[1, 2, [3, 4, 5], 6]" +``` + +That's when `unquote_splicing/1` comes in handy: + +```elixir +iex> inner = [3, 4, 5] +iex> Macro.to_string(quote do: [1, 2, unquote_splicing(inner), 6]) +"[1, 2, 3, 4, 5, 6]" +``` + +Unquoting is very useful when working with macros. When writing macros, developers are able to receive code chunks and inject them inside other code chunks, which can be used to transform code or write code that generates code during compilation. + +## Escaping + +As we saw at the beginning of this chapter, only some values are valid quoted expressions in Elixir. For example, a map is not a valid quoted expression. Neither is a tuple with four elements. However, such values *can* be expressed as a quoted expression: + +```elixir +iex> quote do: %{1 => 2} +{:%{}, [], [{1, 2}]} +``` + +In some cases, you may need to inject such *values* into *quoted expressions*. To do that, we need to first escape those values into quoted expressions with the help of `Macro.escape/1`: + +```elixir +iex> map = %{hello: :world} +iex> Macro.escape(map) +{:%{}, [], [hello: :world]} +``` + +Macros receive quoted expressions and must return quoted expressions. However, sometimes during the execution of a macro, you may need to work with values and making a distinction between values and quoted expressions will be required. + +In other words, it is important to make a distinction between a regular Elixir value (like a list, a map, a process, a reference, and so on) and a quoted expression. Some values, such as integers, atoms, and strings, have a quoted expression equal to the value itself. Other values, like maps, need to be explicitly converted. Finally, values like functions and references cannot be converted to a quoted expression at all. + +When working with macros and code that generates code, check out the documentation for the `Macro` module, which contains many functions to work with Elixir's AST. + +In this introduction, we have laid the groundwork to finally write our first macro. You can check that out in the [next guide](macros.md). diff --git a/lib/elixir/pages/mix-and-otp/agents.md b/lib/elixir/pages/mix-and-otp/agents.md new file mode 100644 index 00000000000..e629061ea6c --- /dev/null +++ b/lib/elixir/pages/mix-and-otp/agents.md @@ -0,0 +1,210 @@ + + +# Simple state with agents + +In this chapter, we will learn how to keep and share state between multiple entities. If you have previous programming experience, you may think of globally shared variables, but the model we will learn here is quite different. The next chapters will generalize the concepts introduced here. + +If you have skipped the *Getting Started* guide or read it long ago, be sure to re-read the [Processes](../getting-started/processes.md) chapter. We will use it as a starting point. + +## The trouble with (mutable) state + +Elixir is an immutable language where nothing is shared by default. If we want to share information, this is typically done by sending messages between processes. + +When it comes to processes though, we rarely hand-roll our own, instead we use the abstractions available in Elixir and OTP: + + * `Agent` — Simple wrappers around state. + * `GenServer` — "Generic servers" (processes) that encapsulate state, provide sync and async calls, support code reloading, and more. + * `Task` — Asynchronous units of computation that allow spawning a process and potentially retrieving its result at a later time. + +Here, we will use agents, and create a module named `KV.Bucket`, responsible for storing our key-value entries in a way that allows them to be read and modified by other processes. + +## Agents 101 + +`Agent`s are simple wrappers around state. If all you want from a process is to keep state, agents are a great fit. Let's start a `iex` session inside the project with: + +```console +$ iex -S mix +``` + +And play a bit with agents: + +```elixir +iex> {:ok, agent} = Agent.start_link(fn -> [] end) +{:ok, #PID<0.57.0>} +iex> Agent.update(agent, fn list -> ["eggs" | list] end) +:ok +iex> Agent.get(agent, fn list -> list end) +["eggs"] +iex> Agent.stop(agent) +:ok +``` + +We started an agent with an initial state of an empty list. The `start_link/1` function returned the `:ok` tuple with a process identifier (PID) of the agent. We will use this PID for all further interactions. We then updated the agent's state, adding our new item to the head of the list. The second argument of `Agent.update/3` is a function that takes the agent's current state as input and returns its desired new state. Finally, we retrieved the whole list. The second argument of `Agent.get/3` is a function that takes the state as input and returns the value that `Agent.get/3` itself will return. Once we are done with the agent, we can call `Agent.stop/3` to terminate the agent process. + +The `Agent.update/3` function accepts as a second argument any function that receives one argument and returns a value: + +```elixir +iex> {:ok, agent} = Agent.start_link(fn -> [] end) +{:ok, #PID<0.338.0>} +iex> Agent.update(agent, fn _list -> 123 end) +:ok +iex> Agent.update(agent, fn content -> %{a: content} end) +:ok +iex> Agent.update(agent, fn content -> [12 | [content]] end) +:ok +iex> Agent.update(agent, fn list -> [:nop | list] end) +:ok +iex> Agent.get(agent, fn content -> content end) +[:nop, 12, %{a: 123}] +``` + +As you can see, we can modify the agent state in any way we want. Therefore, we most likely don't want to access the Agent API throughout many different places in our code. Instead, we want to encapsulate all Agent-related functionality in a single module, which we will call `KV.Bucket`. Before we implement it, let's write some tests which will outline the API exposed by our module. + +Create a file at `test/kv/bucket_test.exs` (remember the `.exs` extension) with the following: + +```elixir +defmodule KV.BucketTest do + use ExUnit.Case, async: true + + test "stores values by key" do + {:ok, bucket} = KV.Bucket.start_link([]) + assert KV.Bucket.get(bucket, "milk") == nil + + KV.Bucket.put(bucket, "milk", 3) + assert KV.Bucket.get(bucket, "milk") == 3 + end +end +``` + +`use ExUnit.Case` is responsible for setting up our module for testing and imports many test-related functionality, such as the `test/2` macro. + +Our first test starts a new `KV.Bucket` by calling the `start_link/1` and passing an empty list of options. Then we perform some `get/2` and `put/3` operations on it, asserting the result. + +Also note the `async: true` option passed to `ExUnit.Case`. This option makes the test case run in parallel with other `:async` test cases by using multiple cores in our machine. This is extremely useful to speed up our test suite. However, `:async` must *only* be set if the test case does not rely on or change any global values. For example, if the test requires writing to the file system or access a database, keep it synchronous (omit the `:async` option) to avoid race conditions between tests. + +Async or not, our new test should obviously fail, as none of the functionality is implemented in the module being tested: + +```text +1) test stores values by key (KV.BucketTest) + test/kv/bucket_test.exs:4 + ** (UndefinedFunctionError) function KV.Bucket.start_link/1 is undefined (module KV.Bucket is not available) +``` + +In order to fix the failing test, let's create a file at `lib/kv/bucket.ex` with the contents below. Feel free to give a try at implementing the `KV.Bucket` module yourself using agents before peeking at the implementation below. + +```elixir +defmodule KV.Bucket do + use Agent + + @doc """ + Starts a new bucket. + + All options are forwarded to `Agent.start_link/2`. + """ + def start_link(opts) do + Agent.start_link(fn -> %{} end, opts) + end + + @doc """ + Gets a value from the `bucket` by `key`. + """ + def get(bucket, key) do + Agent.get(bucket, &Map.get(&1, key)) + end + + @doc """ + Puts the `value` for the given `key` in the `bucket`. + """ + def put(bucket, key, value) do + Agent.update(bucket, &Map.put(&1, key, value)) + end +end +``` + +The first step in our implementation is to call `use Agent`. This is a pattern we will see throughout the guides and understand in depth in the next chapter. + +Then we define a `start_link/1` function, which will effectively start the agent. It is a convention to define a `start_link/1` function that always accepts a list of options. We then call `Agent.start_link/2` passing an anonymous function that returns the Agent's initial state and the same list of options we received. + +We are keeping a map inside the agent to store our keys and values. Getting and putting values on the map is done with the Agent API and the capture operator `&`, introduced in [the Getting Started guide](../getting-started/anonymous-functions.md#the-capture-operator). The agent passes its state to the anonymous function via the `&1` argument when `Agent.get/2` and `Agent.update/2` are called. + +Now that the `KV.Bucket` module has been defined, our test should pass! You can try it yourself by running: `mix test`. + +## Naming processes + +When starting `KV.Bucket`, we pass a list of options which we forward to `Agent.start_link/2`. One of the options accepted by `Agent.start_link/2` is a name option which allows us to name a process, so we can interact with it using its name instead of its PID. + +Let's write a test as an example. Back on `KV.BucketTest`, add this: + +```elixir + test "stores values by key on a named process" do + {:ok, _} = KV.Bucket.start_link(name: :shopping_list) + assert KV.Bucket.get(:shopping_list, "milk") == nil + + KV.Bucket.put(:shopping_list, "milk", 3) + assert KV.Bucket.get(:shopping_list, "milk") == 3 + end +``` + +However, keep in mind that names are shared in the current node. If two tests attempt to create two processes named `:shopping_list` at the same time, one would succeed and the other would fail. For this reason, it is a common practice in Elixir to name processes started during tests after the test itself, like this: + +```elixir + test "stores values by key on a named process", config do + {:ok, _} = KV.Bucket.start_link(name: config.test) + assert KV.Bucket.get(config.test, "milk") == nil + + KV.Bucket.put(config.test, "milk", 3) + assert KV.Bucket.get(config.test, "milk") == 3 + end +``` + +The `config` argument, passed after the test name, is the *test context* and it includes configuration and metadata about the current test, which is useful in scenarios like these. + +## Other agent actions + +Besides getting a value and updating the agent state, agents allow us to get a value and update the agent state in one function call via `Agent.get_and_update/2`. Let's implement a `KV.Bucket.delete/2` function that deletes a key from the bucket, returning its current value: + +```elixir +@doc """ +Deletes `key` from `bucket`. + +Returns the current value of `key`, if `key` exists. +""" +def delete(bucket, key) do + Agent.get_and_update(bucket, &Map.pop(&1, key)) +end +``` + +Now it is your turn to write a test for the functionality above! Also, be sure to explore [the documentation for the `Agent` module](`Agent`) to learn more about them. + +## Client/server in agents + +Before we move on to the next chapter, let's discuss the client/server dichotomy in agents. Let's expand the `delete/2` function we have just implemented: + +```elixir +def delete(bucket, key) do + Agent.get_and_update(bucket, fn map -> + Map.pop(map, key) + end) +end +``` + +Everything that is inside the function we passed to the agent happens in the agent process. In this case, since the agent process is the one receiving and responding to our messages, we say the agent process is the server. Everything outside the function is happening in the client. + +This distinction is important. If there are expensive actions to be done, you must consider if it will be better to perform these actions on the client or on the server. For example: + +```elixir +def delete(bucket, key) do + Process.sleep(1000) # puts client to sleep + Agent.get_and_update(bucket, fn map -> + Process.sleep(1000) # puts server to sleep + Map.pop(map, key) + end) +end +``` + +When a long action is performed on the server, all other requests to that particular server will wait until the action is done, which may cause some clients to timeout. + +Some APIs, such as GenServers, make a clearer distinction between client and server, and we will explore them in future chapters. Next let's talk about naming things, applications, and supervisors. diff --git a/lib/elixir/pages/mix-and-otp/config-and-distribution.md b/lib/elixir/pages/mix-and-otp/config-and-distribution.md new file mode 100644 index 00000000000..6ed27bfb771 --- /dev/null +++ b/lib/elixir/pages/mix-and-otp/config-and-distribution.md @@ -0,0 +1,305 @@ + + +# Configuration and distribution + +So far we have hardcoded our applications to run a web server on port 4040. This has been somewhat problematic since we can't, for example, run our development server and tests at the same time. In this chapter, we will learn how to use the application environment for configuration, paving the way for us to enable distribution by running multiple development servers on the same machine (on different ports). + +In this last guide, we will make the routing table for our distributed key-value store configurable, and then finally package the software for production. + +Let's do this. + +## Application environment + +In the chapter [Registries, applications, and supervisors](supervisor-and-application.md), we have learned that our project is backed by an application, which bundles our modules and specifies how your supervision tree starts and shuts down. Each application can also have its own configuration, which in Erlang/OTP (and therefore Elixir) is called "application environment". + +We can use the application environment to configure our own application, as well as others. Let's see the application environment in practice. Create a file `config/runtime.exs` with the following: + +```elixir +import Config + +port = + cond do + port_env = System.get_env("PORT") -> + String.to_integer(port_env) + + config_env() == :test -> + 4040 + + true -> + 4050 + end + +config :kv, :port, port +``` + +The above is attempting to read the "PORT" environment variable and use it as the port if defined. Otherwise, we default to port `4040` for tests and port `4050` for other environments, eliminating the conflict between environments we have seen in the past. Then we store its value under the `:port` key of our `:kv` application. + +Now we just need to read this configuration. Open up `lib/kv.ex` and the `start/2` function to the following: + +```elixir + def start(_type, _args) do + port = Application.fetch_env!(:kv, :port) + + children = [ + {Registry, name: KV, keys: :unique}, + {DynamicSupervisor, name: KV.BucketSupervisor, strategy: :one_for_one}, + {Task.Supervisor, name: KV.ServerSupervisor}, + Supervisor.child_spec({Task, fn -> KV.Server.accept(port) end}, restart: :permanent) + ] + + Supervisor.start_link(children, strategy: :one_for_one) + end +``` + +Run `iex -S mix` and you will see the following message printed: + +```text +[info] Accepting connections on port 4050 +``` + +Run tests, without killing the development server, and you will see it running on port 4040. + +Our change was straight-forward. We used `Application.fetch_env!/2` to read the entry for `port` in `:kv`'s environment. We explicitly used `fetch_env!/2` (instead of `get_env/2` or `fetch_env`) because it will raise if the port was not configured (preventing the app from booting). + +## Compile vs runtime configuration + +Configuration files provide a mechanism for us to configure the environment of any application. Elixir provides two configuration entry points: + + * `config/config.exs` — this file is read at build time, before we compile our application and before we even load our dependencies. This means we can't access the code in our application nor in our dependencies. However, it means we can control how they are compiled + + * `config/runtime.exs` — this file is read after our application and dependencies are compiled and therefore it can configure how our application works at runtime. If you want to read system environment variables (via `System.get_env/1`) or access external configuration, this is the appropriate place to do so + +You can learn more about configuration in the `Config` and `Config.Provider` modules. + +Generally speaking, we use `Application.fetch_env!/2` (and friends) to read runtime configuration. `Application.compile_env/2` is available for reading compile-time configuration. This allows Elixir to track which modules to recompile when the compilation environment changes. + +Now that we can start multiple servers, let's explore distribution. + +## Our first distributed code + +Elixir ships with facilities to connect nodes and exchange information between them. In fact, we use the same concepts of processes, message passing and receiving messages when working in a distributed environment because Elixir processes are *location transparent*. This means that when sending a message, it doesn't matter if the recipient process is on the same node or on another node, the VM will be able to deliver the message in both cases. + +In order to run distributed code, we need to start the VM with a name. The name can be short (when in the same network) or long (requires the full computer address). Let's start a new IEx session: + +```console +$ iex --sname foo +``` + +You can see now the prompt is slightly different and shows the node name followed by the computer name: + + Interactive Elixir - press Ctrl+C to exit (type h() ENTER for help) + iex(foo@jv)1> + +My computer is named `jv`, so I see `foo@jv` in the example above, but you will get a different result. We will use `foo@computer-name` in the following examples and you should update them accordingly when trying out the code. + +Let's define a module named `Hello` in this shell: + +```elixir +iex> defmodule Hello do +...> def world, do: IO.puts("hello world") +...> end +``` + +If you have another computer on the same network with both Erlang and Elixir installed, you can start another shell on it. If you don't, you can start another IEx session in another terminal. In either case, give it the short name of `bar`: + +```console +$ iex --sname bar +``` + +Note that inside this new IEx session, we cannot access `Hello.world/0`: + +```elixir +iex> Hello.world +** (UndefinedFunctionError) function Hello.world/0 is undefined (module Hello is not available) + Hello.world() +``` + +However, we can spawn a new process on `foo@computer-name` from `bar@computer-name`! Let's give it a try (where `@computer-name` is the one you see locally): + +```elixir +iex> Node.spawn_link(:"foo@computer-name", fn -> Hello.world() end) +#PID<9014.59.0> +hello world +``` + +Elixir spawned a process on another node and returned its PID. You can see the PID number no longer starts with zero, showing it belongs to another node. The code then executed on the other node where the `Hello.world/0` function exists and invoked that function. Note that the result of "hello world" was printed on the current node `bar` and not on `foo`. In other words, the message to be printed was sent back from `foo` to `bar`. This happens because the process spawned on the other node (`foo`) knows all the output should be sent back to the original node! + +We can send and receive messages from the PID returned by `Node.spawn_link/2` as usual. Let's try a quick ping-pong example: + +```elixir +iex> pid = Node.spawn_link(:"foo@computer-name", fn -> +...> receive do +...> {:ping, client} -> send(client, :pong) +...> end +...> end) +#PID<9014.59.0> +iex> send(pid, {:ping, self()}) +{:ping, #PID<0.73.0>} +iex> flush() +:pong +:ok +``` + +In other words, we can spawn processes in other nodes, hold onto their PIDs, and then send messages to them as if they were running on the same machine. That's the *location transparency* principle. And because everything we have built so far was built on top of messaging passing, we should be able to adjust our key-value store to become a distributed one with little work. + +## Distributed naming registry with `:global` + +First, let's check that our code is not currently distributed. Start a new node like this: + +```console +$ PORT=4100 iex --sname foo -S mix +``` + +And the other like this: + +```console +$ PORT=4101 iex --sname bar -S mix +``` + +Now, within `foo@computer-name`, do this: + +```elixir +iex> :erpc.call(:"bar@computer-name", KV, :create_bucket, ["shopping"]) +{:ok, #PID<22121.164.0>} +``` + +Instead of using `Node.spawn_link/2`, we used [Erlang's builtin RPC module](`:erpc`) to call the function `create_bucket` in the `KV` module passing a one element list with the string "shopping" as the argument list. We could have used `Node.spawn_link/2`, but `:erpc.call/4` conveniently returns the result of the invocation. + +Still in `foo@computer-name`, let's try to access the bucket: + +```elixir +iex> KV.lookup_bucket("shopping") +nil +``` + +It returns `nil`. However, if you run `KV.lookup_bucket("shopping")` in `bar@computer-name`, it will return the proper bucket. In other words, the nodes can communicate with each other, but buckets spawned in one node are not visible to the other. + +This is because we are using [Elixir's Registry](`Registry`) to name our buckets, which is a **local** process registry. In other words, it is designed for processes running on a single node and not for distribution. + +Luckily, Erlang ships with a distributed registry called [`:global`](`:global`), which is directly supported by the `:name` option by passing a `{:global, name}` tuple. All we need to do is update the `via/1` function in `lib/kv.ex` from this: + +```elixir + defp via(name), do: {:via, Registry, {KV, name}} +``` + +to this: + +```elixir + defp via(name), do: {:global, name} +``` + +Do the change above and restart both `foo@computer-name` and `bar@computer-name`. Now, back on `foo@computer-name`, let's give it another try: + +```elixir +iex> :erpc.call(:"bar@computer-name", KV, :create_bucket, ["shopping"]) +{:ok, #PID<21821.179.0>} +iex> KV.lookup_bucket("shopping") +#PID<21821.179.0> +``` + +And there you go! By simply changing which naming registry we used, we now have a distributed key value store. You can even try using `telnet` to connect to the servers on different ports and validate that changes in one session are visible in the other one. Exciting! + +## Node discovery and dependencies + +There is one essential ingredient to wrap up our distributed key-value store. In order for the `:global` registry to work, we need to make sure the nodes are connected to each other. When we run `:erpc` call passing the node name: + +```elixir +:erpc.call(:"bar@computer-name", KV, :create_bucket, ["shopping"]) +``` + +Elixir automatically connected the nodes together. This is easy to do in an IEx session when both instances are running on the same machine but it requires more work in a production environment, where instances are on different machines which may be started at any time and running on different IP addresses. + +Luckily for us, this is also a well-solved problem. For example, if you are using [the Phoenix web framework](https://phoenixframework.org) in production, it ships with [the `dns_cluster` package](https://github.com/phoenixframework/dns_cluster), which automatically runs DNS queries to find new nodes and connect them. If you are using Kubernetes or cloud providers, [packages like `libcluster`](https://github.com/bitwalker/libcluster) ship with different strategies to discover and connect nodes. + +Installing dependencies in Elixir is simple. Most commonly, we use the [Hex Package Manager](https://hex.pm), by listing the dependency inside the deps function in our `mix.exs` file: + +```elixir +def deps do + [{:dns_cluster, "~> 0.2"}] +end +``` + +This dependency refers to the latest version of `dns_cluster` in the 0.x version series that has been pushed to Hex. This is indicated by the `~>` preceding the version number. For more information on specifying version requirements, see the documentation for the `Version` module. + +Typically, stable releases are pushed to Hex. If you want to depend on an external dependency still in development, Mix is able to manage Git dependencies too: + +```elixir +def deps do + [{:dns_cluster, git: "https://github.com/phoenixframework/dns_cluster.git"}] +end +``` + +You will notice that when you add a dependency to your project, Mix generates a `mix.lock` file that guarantees *repeatable builds*. The lock file must be checked in to your version control system, to guarantee that everyone who uses the project will use the same dependency versions as you. + +Mix provides many tasks for working with dependencies, which can be seen in `mix help`: + +```console +$ mix help +mix deps # Lists dependencies and their status +mix deps.clean # Deletes the given dependencies' files +mix deps.compile # Compiles dependencies +mix deps.get # Gets all out of date dependencies +mix deps.tree # Prints the dependency tree +mix deps.unlock # Unlocks the given dependencies +mix deps.update # Updates the given dependencies +``` + +The most common tasks are `mix deps.get` and `mix deps.update`. Once fetched, dependencies are automatically compiled for you. You can read more about deps by running `mix help deps`. + +To wrap up this chapter, we will build a very simple node discovery mechanism, where the name of the nodes we should connect to are given on boot, using the lessons we learned in this chapter. + +## `Node.connect/1` + +We will change our application to support a "NODES" environment variable with the name of all nodes each instance should connect to. + +Open up `config/runtime.exs` and add this to the bottom: + +```elixir +nodes = + System.get_env("NODES", "") + |> String.split(",", trim: true) + |> Enum.map(&String.to_atom/1) + +config :kv, :nodes, nodes +``` + +We fetch the environment variable, split it on "," while discarding all empty strings, and then convert each entry to an atom, as node names are atoms. + +Now, in your `start/2` callback, we will add this to of the `start/2` function: + +```elixir + def start(_type, _args) do + for node <- Application.fetch_env!(:kv, :nodes) do + Node.connect(node) + end +``` + +Now we can start our nodes as: + +```console +$ NODES="foo@computer-name,bar@computer-name" PORT=4040 iex --sname foo -S mix +$ NODES="foo@computer-name,bar@computer-name" PORT=4041 iex --sname bar -S mix +``` + +And they should connect to each other. Give it a try! + +In an actual production system, there is some additional care we must take. For example, we often use `--name` instead of `--sname` and give fully qualified node names. + +Furthermore, when connecting two instances, we must guarantee they have the same cookie, which is a secret Erlang uses to authorize the connection. When they run on the same machine, they share the same cookie by default, but it must be either explicitly set or shared in other ways when deploying in a cluster. + +We will revisit these topics in the last chapter when we talk about releases. + +## Distributed system trade-offs + +In this chapter, we made our key-value store distributed by using the `:global` naming registry. However, it is important to keep in mind that every distributed system, be it a library or a full-blown database, is designed with a series of trade-offs in mind. + +In particular, `:global` requires consistency across all known nodes whenever a new bucket is created. For example, if your cluster has three nodes, creating a new bucket will require all three nodes to agree on its name. This means if one node is unresponsive, perhaps due to a [network partition](https://en.wikipedia.org/wiki/Network_partition), the node will have to either reconnect or be kicked out before registration succeeds. This also means that, as your cluster grows in size, registration becomes more expensive, although lookups are always cheap and immediate. Within the ecosystem, there are other named registries, which explore different trade-offs, such as [Syn](https://github.com/ostinelli/syn). + +Further complications arise when we consider storage. Today, when our nodes terminate, we lose all data stored in the buckets. In our current design, since we allow each node to store their own buckets, it means we would need to backup each node. And, if we don't want data losses, we would also need to replicate the data. + +For those reasons, it is still very common to use a database (or any storage system) when writing production applications in Elixir, and use Elixir to implement the realtime and collaborative aspects of your applications that extend beyond storage. For example, we can use Elixir to track which clients are connected to the cluster at any given moment or implement a feed where users are notified in realtime whenever items are added or removed from a bucket. + +In fact, that's exactly what we will build in the next chapter. Allowing us to wrap up everything we have learned so far and also talk about one of the essential building blocks in Elixir software: GenServers. diff --git a/lib/elixir/pages/mix-and-otp/docs-tests-and-with.md b/lib/elixir/pages/mix-and-otp/docs-tests-and-with.md new file mode 100644 index 00000000000..eb9d0c62412 --- /dev/null +++ b/lib/elixir/pages/mix-and-otp/docs-tests-and-with.md @@ -0,0 +1,426 @@ + + +# Doctests, patterns, and with + +In this chapter, we will implement the code that parses the commands we described in the first chapter: + +```text +CREATE shopping +OK + +PUT shopping milk 1 +OK + +PUT shopping eggs 3 +OK + +GET shopping milk +1 +OK + +DELETE shopping eggs +OK +``` + +After the parsing is done, we will update our server to dispatch the parsed commands to the relevant buckets. + +## Doctests + +On the language homepage, we mention that Elixir makes documentation a first-class citizen in the language. We have explored this concept many times throughout this guide, be it via `mix help` or by typing `h Enum` or another module in an IEx console. + +In this section, we will implement the parsing functionality, document it and make sure our documentation is up to date with doctests. This helps us provide documentation with accurate code samples. + +Let's create our command parser at `lib/kv/command.ex` and start with the doctest: + +```elixir +defmodule KV.Command do + @doc ~S""" + Parses the given `line` into a command. + + ## Examples + + iex> KV.Command.parse("CREATE shopping\r\n") + {:ok, {:create, "shopping"}} + + """ + def parse(_line) do + :not_implemented + end +end +``` + +Doctests are specified by an indentation of four spaces followed by the `iex>` prompt in a documentation string. If a command spans multiple lines, you can use `...>`, as in IEx. The expected result should start at the next line after `iex>` or `...>` line(s) and is terminated either by a newline or a new `iex>` prefix. + +Also, note that we started the documentation string using `@doc ~S"""`. The `~S` prevents the `\r\n` characters from being converted to a carriage return and line feed until they are evaluated in the test. + +To run our doctests, we'll create a file at `test/kv/command_test.exs` and call `doctest KV.Command` in the test case: + +```elixir +defmodule KV.CommandTest do + use ExUnit.Case, async: true + doctest KV.Command +end +``` + +Run the test suite and the doctest should fail: + +```text + 1) doctest KV.Command.parse/1 (1) (KV.CommandTest) + test/kv/command_test.exs:3 + Doctest failed + doctest: + iex> KV.Command.parse("CREATE shopping\r\n") + {:ok, {:create, "shopping"}} + code: KV.Command.parse "CREATE shopping\r\n" === {:ok, {:create, "shopping"}} + left: :not_implemented + right: {:ok, {:create, "shopping"}} + stacktrace: + lib/kv/command.ex:7: KV.Command (module) +``` + +Excellent! + +Now let's make the doctest pass. Let's implement the `parse/1` function: + +```elixir +def parse(line) do + case String.split(line) do + ["CREATE", bucket] -> {:ok, {:create, bucket}} + end +end +``` + +Our implementation splits the line on whitespace and then matches the command against a list. Using `String.split/1` means our commands will be whitespace-insensitive. Leading and trailing whitespace won't matter, nor will consecutive spaces between words. Let's add some new doctests to test this behavior along with the other commands: + +```elixir + @doc ~S""" + Parses the given `line` into a command. + + ## Examples + + iex> KV.Command.parse "CREATE shopping\r\n" + {:ok, {:create, "shopping"}} + + iex> KV.Command.parse "CREATE shopping \r\n" + {:ok, {:create, "shopping"}} + + iex> KV.Command.parse "PUT shopping milk 1\r\n" + {:ok, {:put, "shopping", "milk", "1"}} + + iex> KV.Command.parse "GET shopping milk\r\n" + {:ok, {:get, "shopping", "milk"}} + + iex> KV.Command.parse "DELETE shopping eggs\r\n" + {:ok, {:delete, "shopping", "eggs"}} + + Unknown commands or commands with the wrong number of + arguments return an error: + + iex> KV.Command.parse "UNKNOWN shopping eggs\r\n" + {:error, :unknown_command} + + iex> KV.Command.parse "GET shopping\r\n" + {:error, :unknown_command} + + """ +``` + +With doctests at hand, it is your turn to make tests pass! Once you're ready, you can compare your work with our solution below: + +```elixir + def parse(line) do + case String.split(line) do + ["CREATE", bucket] -> {:ok, {:create, bucket}} + ["GET", bucket, key] -> {:ok, {:get, bucket, key}} + ["PUT", bucket, key, value] -> {:ok, {:put, bucket, key, value}} + ["DELETE", bucket, key] -> {:ok, {:delete, bucket, key}} + _ -> {:error, :unknown_command} + end + end +``` + +Notice how we were able to elegantly parse the commands without adding a bunch of `if/else` clauses that check the command name and number of arguments! + +Finally, you may have observed that each doctest corresponds to a different test in our suite, which now reports a total of 7 doctests. That is because ExUnit considers the following to define two different doctests: + +```elixir +iex> KV.Command.parse("UNKNOWN shopping eggs\r\n") +{:error, :unknown_command} + +iex> KV.Command.parse("GET shopping\r\n") +{:error, :unknown_command} +``` + +Without new lines, as seen below, ExUnit compiles it into a single doctest: + +```elixir +iex> KV.Command.parse("UNKNOWN shopping eggs\r\n") +{:error, :unknown_command} +iex> KV.Command.parse("GET shopping\r\n") +{:error, :unknown_command} +``` + +As the name says, doctest is documentation first and a test later. Their goal is not to replace tests but to provide up-to-date documentation. You can read more about doctests in the `ExUnit.DocTest` documentation. + +## Using `with` + +As we are now able to parse commands, we can finally start implementing the logic that runs the commands. Let's add a stub definition for this function for now: + +```elixir +defmodule KV.Command do + @doc """ + Runs the given command. + """ + def run(command, socket) do + :gen_tcp.send(socket, "OK\r\n") + :ok + end +end +``` + +Before we implement this function, let's change our server to start using our new `parse/1` and `run/1` functions. Remember, our `read_line/1` function was also crashing when the client closed the socket, so let's take the opportunity to fix it, too. Open up `lib/kv/server.ex` and replace the existing server definition: + +```elixir + defp serve(socket) do + socket + |> read_line() + |> write_line(socket) + + serve(socket) + end + + defp read_line(socket) do + {:ok, data} = :gen_tcp.recv(socket, 0) + data + end + + defp write_line(line, socket) do + :gen_tcp.send(socket, line) + end +``` + +by the following: + +```elixir + defp serve(socket) do + msg = + case read_line(socket) do + {:ok, data} -> + case KV.Command.parse(data) do + {:ok, command} -> + KV.Command.run(command, socket) + + {:error, _} = err -> + err + end + + {:error, _} = err -> + err + end + + write_line(socket, msg) + serve(socket) + end + + defp read_line(socket) do + :gen_tcp.recv(socket, 0) + end + + defp write_line(_socket, :ok) do + :ok + end + + defp write_line(socket, {:error, :unknown_command}) do + # Known error; write to the client + :gen_tcp.send(socket, "UNKNOWN COMMAND\r\n") + end + + defp write_line(_socket, {:error, :closed}) do + # The connection was closed, exit politely + exit(:shutdown) + end + + defp write_line(socket, {:error, error}) do + # Unknown error; write to the client and exit + :gen_tcp.send(socket, "ERROR\r\n") + exit(error) + end +``` + +If we start our server, we can now send commands to it. For now, we will get two different responses: "OK" when the command is known and "UNKNOWN COMMAND" otherwise: + +```console +$ telnet 127.0.0.1 4040 +Trying 127.0.0.1... +Connected to localhost. +Escape character is '^]'. +CREATE shopping +OK +HELLO +UNKNOWN COMMAND +``` + +This means our implementation is going in the correct direction, but it doesn't look very elegant, does it? + +The previous implementation used pipelines which made the logic straightforward to follow. However, now that we need to handle different error codes along the way, our server logic is nested inside many `case` calls. + +Thankfully, Elixir has the `with` construct, which allows you to simplify code like the above, replacing nested `case` calls with a chain of matching clauses. Let's rewrite the `serve/1` function to use `with`: + +```elixir + defp serve(socket) do + msg = + with {:ok, data} <- read_line(socket), + {:ok, command} <- KV.Command.parse(data), + do: KV.Command.run(command, socket) + + write_line(socket, msg) + serve(socket) + end +``` + +Much better! `with` will retrieve the value returned by the right-side of `<-` and match it against the pattern on the left side. If the value matches the pattern, `with` moves on to the next expression. In case there is no match, the non-matching value is returned. + +In other words, we converted each expression given to `case/2` as a step in `with`. As soon as any of the steps return something that does not match `{:ok, x}`, `with` aborts, and returns the non-matching value. + +You can read more about `with/1` in our documentation. + +## Running commands + +The last step is to implement `KV.Command.run/1` to run the parsed commands on top of buckets. Its implementation is shown below: + +```elixir + @doc """ + Runs the given command. + """ + def run(command, socket) + + def run({:create, bucket}, socket) do + KV.create_bucket(bucket) + :gen_tcp.send(socket, "OK\r\n") + :ok + end + + def run({:get, bucket, key}, socket) do + lookup(bucket, fn pid -> + value = KV.Bucket.get(pid, key) + :gen_tcp.send(socket, "#{value}\r\nOK\r\n") + :ok + end) + end + + def run({:put, bucket, key, value}, socket) do + lookup(bucket, fn pid -> + KV.Bucket.put(pid, key, value) + :gen_tcp.send(socket, "OK\r\n") + :ok + end) + end + + def run({:delete, bucket, key}, socket) do + lookup(bucket, fn pid -> + KV.Bucket.delete(pid, key) + :gen_tcp.send(socket, "OK\r\n") + :ok + end) + end + + defp lookup(bucket, callback) do + if bucket = KV.lookup_bucket(bucket) do + callback.(bucket) + else + {:error, :not_found} + end + end +``` + +Each function clause dispatches the appropriate command to the appropriate bucket. + +You might have noticed we have a function head, `def run(command, socket)`, without a body. In the [Modules and Functions](../getting-started/modules-and-functions.md#default-arguments) chapter, we learned that a bodiless function can be used to declare default arguments for a multi-clause function. Here is another use case where we use a function without a body to document what the arguments are. + +We have also defined a private function named `lookup/2` to help with the common functionality of looking up a bucket and returning its `pid` if it exists, `{:error, :not_found}` otherwise. + +By the way, since we are now returning `{:error, :not_found}`, we should amend the `write_line/2` function in `KV.Server` to print such error as well: + +```elixir +defp write_line(socket, {:error, :not_found}) do + :gen_tcp.send(socket, "NOT FOUND\r\n") +end +``` + +Our server functionality is almost complete. Only tests are missing. + +## Integration tests + +`KV.Command.run/1`'s implementation is sending commands directly to the `KV` module, which is using a local registry to name processes. This means if we have two tests sending messages to the same bucket, our tests will conflict with each other (and likely fail). One might think this would be a reason to use mocks and other strategies to keep our tests isolated, but such techniques often make our testing environment too distant from how our code actually runs in production, and you may end-up with bugs lurking. + +Luckily, there is a technique that we have been using throughout this guide that would be equally applicable here: it is ok to rely on the local registry as long as each test uses unique names. Using a combination of the test module and test name is more than enough to guarantee that. + +So let's write integration tests that rely on unique names to exercise the whole stack from the TCP server to the bucket. + +Create a new file at `test/kv/server_test.exs` as shown below: + +```elixir +defmodule KV.ServerTest do + use ExUnit.Case, async: true + + @socket_options [:binary, packet: :line, active: false] + + setup config do + {:ok, socket} = :gen_tcp.connect(~c"localhost", 4040, @socket_options) + test_name = config.test |> Atom.to_string() |> String.replace(" ", "-") + %{socket: socket, name: "#{config.module}-#{test_name}"} + end + + test "server interaction", %{socket: socket, name: name} do + # CREATE + assert send_and_recv(socket, "CREATE #{name}\r\n") == "OK\r\n" + + # PUT + assert send_and_recv(socket, "PUT #{name} eggs 3\r\n") == "OK\r\n" + + # GET + assert send_and_recv(socket, "GET #{name} eggs\r\n") == "3\r\n" + assert send_and_recv(socket, "") == "OK\r\n" + + # DELETE + assert send_and_recv(socket, "DELETE #{name} eggs\r\n") == "OK\r\n" + + # GET + assert send_and_recv(socket, "GET #{name} eggs\r\n") == "\r\n" + assert send_and_recv(socket, "") == "OK\r\n" + end + + test "unknown command", %{socket: socket} do + assert send_and_recv(socket, "WHATEVER\r\n") == + "UNKNOWN COMMAND\r\n" + end + + test "unknown bucket", %{socket: socket} do + assert send_and_recv(socket, "GET whatever eggs\r\n") == + "NOT FOUND\r\n" + end + + defp send_and_recv(socket, command) do + :ok = :gen_tcp.send(socket, command) + {:ok, data} = :gen_tcp.recv(socket, 0, 1000) + data + end +end +``` + +Run `mix test` and the tests should all pass. However, make sure to terminate any `iex -S mix` session you may have running, as currently tests and development environment are running on the same port (4040). We will address it in the next chapter. + +We added three tests, the first one tests most bucket actions, while the other two deal with error cases. Given there is a lot of shared setup across these tests, we used the `setup/2` macro to deal with common boilerplate. The macro receives the same *test context* as tests and starts a client TCP connection per test. It also defines a unique bucket name using the module name and the test name, making sure any space in the test name is replaced by `-` as to not interfere with our command parsing logic. + +Then, in each test, we pattern matched on the *test context*, extracting the socket or name as necessary. This is similar to the code we wrote in `test/kv/bucket_test.exs`: + +```elixir + test "stores values by key on a named process", config do +``` + +Except back then we matched on all config and, this time around, we matched only on the data we needed. + +Let's move to the next chapter. We will finally make our system distributed by adding a tiny bit of configuration and, *spoiler alert*, changing one line of code. diff --git a/lib/elixir/pages/mix-and-otp/dynamic-supervisor.md b/lib/elixir/pages/mix-and-otp/dynamic-supervisor.md new file mode 100644 index 00000000000..d3475fcf84c --- /dev/null +++ b/lib/elixir/pages/mix-and-otp/dynamic-supervisor.md @@ -0,0 +1,233 @@ + + +# Supervising dynamic children + +We have successfully learned how our supervision tree is automatically started (and stopped) as part of our application's life cycle. We can also name our buckets via the `:name` option. We also learned that, in practice, we should always start new processes inside supervisors. Let's apply these insights by ensuring our buckets are named and supervised. + +## Child specs + +Supervisors know how to start processes because they are given "child specifications". In our `lib/kv.ex` file, we defined a list of children with a single child spec: + +```elixir +children = [ + {Registry, name: KV, keys: :unique} +] +``` + +When the child specification is a tuple (as above) or module, then it is equivalent to calling the `child_spec/1` function on said module, which then returns the full specification. The pair above is equivalent to: + +```elixir +iex> Registry.child_spec(name: KV, keys: :unique) +%{ + id: KV, + start: {Registry, :start_link, [[name: KV, keys: :unique]]}, + type: :supervisor +} +``` + +The underlying map returns the `:id` (required), the module-function-args triplet to invoke to start the process (required), the type of the process (optional), among other optional keys. In other words, the `child_spec/1` function allows us to compose and encapsulate specifications in modules. + +Therefore, if we want to supervise `KV.Bucket`, we only need to define a `child_spec/1` function. Luckily for us, whenever we invoke `use Agent` (or `use GenServer` or `use Supervisor` and so forth), an implementation with reasonable defaults is provided. So let's take it for a spin. Back on `iex -S mix`, try this: + +```elixir +iex> KV.Bucket.child_spec([]) +%{id: KV.Bucket, start: {KV.Bucket, :start_link, [[]]}} +iex> KV.Bucket.child_spec([name: :shopping]) +%{id: KV.Bucket, start: {KV.Bucket, :start_link, [[name: :shopping]]}} +``` + +Let's try to start it as part of a supervisor then, using the `{module, options}` format to pass the bucket name (let's also use an atom as the name for convenience): + +```elixir +iex> children = [{KV.Bucket, name: :shopping}] +iex> Supervisor.start_link(children, strategy: :one_for_one) +iex> KV.Bucket.put(:shopping, "milk", 1) +:ok +iex> KV.Bucket.get(:shopping, "milk") +1 +``` + +What happens now if we explicitly kill the bucket process? + +```elixir +# Find the pid for the given name +iex> pid = Process.whereis(:shopping) +#PID<0.48.0> +# Send it a kill exit signal +iex> Process.exit(pid, :kill) +true +# But a new process is alive in its place +iex> Process.whereis(:shopping) +#PID<0.50.0> +``` + +Given our buckets can already be supervised, it is time to hook them into our supervision tree. + +## Dynamic supervisors + +Given our buckets can already be supervised, you may be thinking to start them as part of our application `start/2` callback, such as: + +```elixir +children = [ + {Registry, name: KV, keys: :unique} + {KV.Bucket, name: {:via, Registry, {KV, "shopping"}}} +] +``` + +And while the above would definitely work, it comes with a huge caveat: it only starts a single bucket. In practice, we want the user to be able to create new buckets at any time. In other words, we need to start and supervise processes dynamically. + +While the `Supervisor` module has APIs for starting children after its initialization, it was not designed or optimized for the use case of having potentially millions of children. For this purpose, Elixir instead provides the `DynamicSupervisor` module. Using it is quite similar to `Supervisor` except that, instead of specifying the children during start, you do it afterwards. Let's take it for a spin: + +```elixir +iex> {:ok, sup_pid} = DynamicSupervisor.start_link(strategy: :one_for_one) +iex> DynamicSupervisor.start_child(sup_pid, {KV.Bucket, name: :another_list}) +iex> KV.Bucket.put(:another_list, "milk", 1) +:ok +iex> KV.Bucket.get(:another_list, "milk") +1 +``` + +And it all works as expected. In fact, we can even give names to `DynamicSupervisor` themselves, instead of passing PIDs around and also use it to start buckets named using the registry: + +```elixir +iex> DynamicSupervisor.start_link(strategy: :one_for_one, name: :dyn_sup) +iex> name = {:via, Registry, {KV, "yet_another_list"}} +iex> DynamicSupervisor.start_child(:dyn_sup, {KV.Bucket, name: name}) +iex> KV.Bucket.put(name, "milk", 1) +:ok +iex> KV.Bucket.get(name, "milk") +1 +``` + +Overall, processes can be named and supervised, regardless if they are supervisors, agents, etc, since all of Elixir standard library was designed around those capabilities. + +With all ingredients in place to supervise and name buckets, open up the `lib/kv.ex` module and let's add a new function called `KV.lookup_bucket/1`, which receives a name and either create or returns a bucket for the given name: + +```elixir +defmodule KV do + use Application + + @impl true + def start(_type, _args) do + children = [ + {Registry, name: KV, keys: :unique}, + {DynamicSupervisor, name: KV.BucketSupervisor, strategy: :one_for_one} + ] + + Supervisor.start_link(children, strategy: :one_for_one) + end + + @doc """ + Creates a bucket with the given name. + """ + def create_bucket(name) do + DynamicSupervisor.start_child(KV.BucketSupervisor, {KV.Bucket, name: via(name)}) + end + + @doc """ + Looks up the given bucket. + """ + def lookup_bucket(name) do + GenServer.whereis(via(name)) + end + + defp via(name), do: {:via, Registry, {KV, name}} +end +``` + +The code is relatively simple. First we changed `start/2` to also start a dynamic supervisor named `KV.BucketSupervisor`. Then, when implemented `KV.create_bucket/1` which receives a bucket and starts with using our registry and dynamic supervisor. And we also added `KV.lookup_bucket/1` that receives the same name and attempts to find its PID. + +To make sure it all works as expected, let's write a test. Open up `test/kv_test.exs` and add this: + +```elixir +defmodule KVTest do + use ExUnit.Case, async: true + + test "creates and looks up buckets by any name" do + name = "a unique name that won't be shared" + assert is_nil(KV.lookup_bucket(name)) + + assert {:ok, bucket} = KV.create_bucket(name) + assert KV.lookup_bucket(name) == bucket + + assert KV.create_bucket(name) == {:error, {:already_started, bucket}} + end +end +``` + +The test shows we are creating and locating buckets with any name, making sure we use a unique name to avoid conflicts between tests. + +## The `start_supervised` test helper + +Before we move on, let's do some clean up. + +In `test/kv/bucket_test.exs`, we explicitly invoked `KV.Bucket.start_link/1` to start our buckets. However, we now know that we should avoid calling `start_link/1` directly and instead start processes as part of supervision trees. + +In order to aid testing, `ExUnit` already starts a supervision tree per test and provides the `start_supervised` function to start processes within test-specific supervision tree. One advantage of this approach is that `ExUnit` guarantees any started process is shut down at the end of the test too. Let's rewrite our tests to use it instead: + +```elixir +defmodule KV.BucketTest do + use ExUnit.Case, async: true + + test "stores values by key" do + {:ok, bucket} = start_supervised(KV.Bucket) + assert KV.Bucket.get(bucket, "milk") == nil + + KV.Bucket.put(bucket, "milk", 3) + assert KV.Bucket.get(bucket, "milk") == 3 + end + + test "stores values by key on a named process", config do + {:ok, _} = start_supervised({KV.Bucket, name: config.test}) + assert KV.Bucket.get(config.test, "milk") == nil + + KV.Bucket.put(config.test, "milk", 3) + assert KV.Bucket.get(config.test, "milk") == 3 + end +end +``` + +It is a small change, but our tests are now using all of the relevant best practices. Excellent! + +## Observer + +Now that we have defined our supervision tree, it is a great opportunity to introduce the Observer tool that ships with Erlang. Start your application with `iex -S mix` and key this in: + +```elixir +iex> :observer.start() +``` + +> #### Missing dependencies {: .warning} +> +> When running `iex` inside a project with `iex -S mix`, `observer` won't be available as a dependency. To do so, you will need to call the following functions: +> +> ```elixir +> iex> Mix.ensure_application!(:observer) +> iex> :observer.start() +> ``` +> +> If the call above fails, here is what may have happened: some package managers default to installing a minimized Erlang without WX bindings for GUI support. In some package managers, you may be able to replace the headless Erlang with a more complete package (look for packages named `erlang` vs `erlang-nox` on Debian/Ubuntu/Arch). In others managers, you may need to install a separate `erlang-wx` (or similarly named) package. +> +> There are conversations to improve this experience in future releases. + +A GUI should pop up containing all sorts of information about our system, from general statistics to load charts as well as a list of all running processes and applications. + +In the Applications tab, you will see all applications currently running in your system alongside their supervision tree. You can select the `kv` application to explore it further: + +Observer GUI screenshot + +Not only that, as you create new buckets on the terminal, you should see new processes spawned in the supervision tree shown in Observer: + +```elixir +iex> KV.create_bucket("shopping") +#PID<0.89.0> +``` + +We will leave it up to you to further explore what Observer provides. Note you can double-click any process in the supervision tree to retrieve more information about it, as well as right-click a process to send "a kill signal", a perfect way to emulate failures and see if your supervisor reacts as expected. + +At the end of the day, tools like Observer are one of the reasons you want to always start processes inside supervision trees, even if they are temporary, to ensure they are always reachable and introspectable. + +Now that our buckets are named and supervised, we are ready to start our server and start receiving requests. diff --git a/lib/elixir/pages/mix-and-otp/genservers.md b/lib/elixir/pages/mix-and-otp/genservers.md new file mode 100644 index 00000000000..30fcf2a7ce7 --- /dev/null +++ b/lib/elixir/pages/mix-and-otp/genservers.md @@ -0,0 +1,383 @@ + + +# Client-server with GenServer + +To wrap up our distributed key-value store, we will implement a feature where a client can subscribe to a bucket and receive realtime notifications of any modification happening in the bucket, regardless of where in the cluster the bucket is located. + +We will do by adding a new command, called SUBSCRIBE, to be used like this: + +```text +SUBSCRIBE shopping +milk SET TO 1 +eggs SET TO 10 +milk DELETED +``` + +To make this work, we must change our `KV.Bucket` implementation to track subscriptions and emit broadcasts. However, as we will see, we cannot implement such on top of agents, and we will need to rewrite our bucket implementation to a `GenServer`. + +## Links and monitors + +Processes in Elixir are isolated. When they need to communicate, they do so by sending messages. However, how do you know when a process terminates, either because it has completed or due to a crash? + +We have two options: links and monitors. + +We have used links extensively. Whenever we started a process, we typically did so by using `start_link` or similar. The idea behind links is that, if any of the processes crash, the other will crash due to the link. We talked about them in the [Process chapter of the Getting Started guide](../getting-started/processes.md). Here is a refresher: + +```elixir +iex> self() +#PID<0.115.0> +iex> spawn_link(fn -> :nothing_bad_will_happen end) +#PID<0.116.0> +iex> self() +#PID<0.115.0> +``` + +```elixir +iex> spawn_link(fn -> raise "oops" end) +#PID<0.117.0> + +12:37:33.229 [error] Process #PID<0.117.0> raised an exception +Interactive Elixir (1.18.4) - press Ctrl+C to exit (type h() ENTER for help) +iex> self() +#PID<0.118.0> +``` + +The reason why we links are so pervasive is because when we start a process inside a supervisor, we want our process to crash if the supervisor terminates. On the other hand, we don't want the supervisor to crash when a child terminates, and therefore supervisors trap exits from links by calling `Process.flag(:trap_exit, true)`. + +In other words, links create an intrinsic relationship between the processes. If we simply want to track when a process dies, without tying their exit signals to each other, a better solution is to use monitors. When a monitored process terminates, we receive a message in our inbox, regardless of the reason: + +```elixir +iex> pid = spawn(fn -> Process.sleep(5000) end) +#PID<0.119.0> +iex> Process.monitor(pid) +#Reference<0.1076459149.2159017989.118674> +iex> flush() +:ok +# Wait five seconds +iex> flush() +{:DOWN, #Reference<0.1076459149.2159017989.118674>, :process, #PID<0.119.0>, :normal} +:ok +``` + +Once the process terminates, we receive a "DOWN message", represented in a five-element tuple. The last element is the reason why it crashed (`:normal` means it terminated successfully). + +Monitors will play a very important role in our subscribe feature. When a client subscribes to a bucket, the bucket will store the client PID and send messages to it on every change. However, if the client terminates (for example because it was disconnected), the bucket must remove the client from its list of subscribers (otherwise the list would keep on growing forever as clients connect and disconnect). + +We chose the `Agent` module to implement our `KV.Bucket` and, unfortunately, agents cannot receive messages. So the first step is to rewrite our `KV.Bucket` to a `GenServer`. The `GenServer` module documentation has a good overview on what they are and how to implement them. Give it a read and then we are ready to proceed. + +## GenServer callbacks + +A GenServer is a process that invokes a limited set of functions under specific conditions. When we used an `Agent`, we would keep both the client code and the server code side by side, like this: + +```elixir +def put(bucket, key, value) do + Agent.update(bucket, &Map.put(&1, key, value)) +end +``` + +Let's break that code apart a bit: + +```elixir +def put(bucket, key, value) do + # Here is the client code + Agent.update(bucket, fn state -> + # Here is the server code + Map.put(state, key, value) + end) + # Back to the client code +end +``` + +In the code above, we have a process, which we call "the client" sending a request to an agent, "the server". The request contains an anonymous function, which must be executed by the server. + +In a GenServer, the code above would be two separate functions, roughly like this: + +```elixir +def put(bucket, key, value) do + # Send the server a :put "instruction" + GenServer.call(bucket, {:put, key, value}) +end + +# Server callback + +def handle_call({:put, key, value}, _from, state) do + {:reply, :ok, Map.put(state, key, value)} +end +``` + +Let's go ahead and rewrite `KV.Bucket` at once. Open up `lib/kv/bucket.ex` and replace its contents with this new version: + +```elixir +defmodule KV.Bucket do + use GenServer + + @doc """ + Starts a new bucket. + """ + def start_link(opts) do + GenServer.start_link(__MODULE__, %{}, opts) + end + + @doc """ + Gets a value from the `bucket` by `key`. + """ + def get(bucket, key) do + GenServer.call(bucket, {:get, key}) + end + + @doc """ + Puts the `value` for the given `key` in the `bucket`. + """ + def put(bucket, key, value) do + GenServer.call(bucket, {:put, key, value}) + end + + @doc """ + Deletes `key` from `bucket`. + + Returns the current value of `key`, if `key` exists. + """ + def delete(bucket, key) do + GenServer.call(bucket, {:delete, key}) + end + + ### Callbacks + + @impl true + def init(bucket) do + state = %{ + bucket: bucket + } + + {:ok, state} + end + + @impl true + def handle_call({:get, key}, _from, state) do + value = get_in(state.bucket[key]) + {:reply, value, state} + end + + def handle_call({:put, key, value}, _from, state) do + state = put_in(state.bucket[key], value) + {:reply, :ok, state} + end + + def handle_call({:delete, key}, _from, state) do + {value, state} = pop_in(state.bucket[key]) + {:reply, value, state} + end +end +``` + +The first function is `start_link/1`, which starts a new GenServer passing a list of options. `GenServer.start_link/3`, which takes three arguments: + +1. The module where the server callbacks are implemented, in this case `__MODULE__` (meaning the current module) + +2. The initialization arguments, in this case the empty bucket `%{}` + +3. A list of options which can be used to specify things like the name of the server. Once again, we forward the list of options that we receive on `start_link/1` to `GenServer.start_link/3`, as we did for agents + +Once started, the GenServer will invoke the `init/1` callback, that receives the second argument given to `GenServer.start_link/3` and returns `{:ok, state}`, where state is a new map. We can already notice how the `GenServer` API makes the client/server segregation more apparent. `start_link/3` happens in the client, while `init/1` is the respective callback that runs on the server. + +There are two types of requests you can send to a GenServer: calls and casts. Calls are synchronous and the server **must** send a response back to such requests. While the server computes the response, the client is **waiting**. Casts are asynchronous: the server won't send a response back and therefore the client won't wait for one. Both requests are messages sent to the server, and will be handled in sequence. So far we have only used `GenServer.call/2`, to keep the same semantics as the Agent, but we will give `cast` a try when implementing subscriptions. Given we kept the same behaviour, all tests will still pass. + +Each request must be implemented as a specific callback. For `call/2` requests, we implement a `handle_call/3` callback that receives the `request`, the process from which we received the request (`_from`), and the current server state (`state`). The `handle_call/3` callback returns a tuple in the format `{:reply, reply, updated_state}`. The first element of the tuple, `:reply`, indicates that the server should send a reply back to the client. The second element, `reply`, is what will be sent to the client while the third, `updated_state` is the new server state. + +Another Elixir feature we used in the implementation above are the nested traversal functions: `get_in/1`, `put_in/2`, and `pop_in/1`. Instead of keeping the `bucket` as our GenServer state, we defined a state map with a `bucket` key inside. This will be important as we also need to track subscribers as part of the GenServer state. These new functions make it straight-forward to manipulate data structures nested in other data structures. + +With our GenServer in place, let's work on subscription, starting with the tests. + +## Implementing subscriptions + +Our new test will subscribe to a bucket and then assert that, as operations are performed against the bucket, we receive messages of said events. + +Open up `test/kv/bucket_test.exs` and key this in: + +```elixir + test "subscribes to puts and deletes" do + {:ok, bucket} = start_supervised(KV.Bucket) + KV.Bucket.subscribe(bucket) + + KV.Bucket.put(bucket, "milk", 3) + assert_receive {:put, "milk", 3} + + # Also check it works even from another process + spawn(fn -> KV.Bucket.delete(bucket, "milk") end) + assert_receive {:delete, "milk"} + end +``` + +In order to make the test pass, we need to implement the `KV.Bucket.subscribe/1`. So let's add these three new functions to `KV.Bucket`: + +```elixir + @doc """ + Subscribes the current process to the bucket. + """ + def subscribe(bucket) do + GenServer.cast(bucket, {:subscribe, self()}) + end + + @impl true + def handle_cast({:subscribe, pid}, state) do + Process.monitor(pid) + state = update_in(state.subscribers, &MapSet.put(&1, pid)) + {:noreply, state} + end + + @impl true + def handle_info({:DOWN, _ref, _type, pid, _reason}, state) do + state = update_in(state.subscribers, &MapSet.delete(&1, pid)) + {:noreply, state} + end +``` + +On subscription, we send a `cast/2` request with the current process identifier and implement its `handle_cast/2` callback that receives the `request` and the current server state. We then proceed to monitor the given `pid` and add it to the list of subscribers, which we are implementing using `MapSet`. The `handle_cast/2` callback returns a tuple in the format `{:noreply, updated_state}`. Note that in a real application we would have probably implemented it with a synchronous call, as it provides back pressure, instead of an asynchronous cast. We are doing it this way to illustrate how to implement a cast callback. + +Then, because we have monitored a process, once that process terminates, we will receive a "DOWN message". GenServers handle regular messages using the `handle_info/2` callback, which also typically return `{:noreply, updated_state}`. In this callback, we remove the PID that terminated from our list of subscribers. + +We are almost there. We can see both `handle_cast/2` and `handle_info/2` callbacks assume there is a subscribers key in our state with a `MapSet`. So let's add it by updating the existing `init/1` to the following: + +```elixir + @impl true + def init(bucket) do + state = %{ + bucket: bucket, + subscribers: MapSet.new() + } + + {:ok, state} + end +``` + +And finally let's update the callbacks for `put/3` and `delete/2` to broadcast messages whenever they are invoked, like this: + +```elixir + def handle_call({:put, key, value}, _from, state) do + state = put_in(state.bucket[key], value) + broadcast(state, {:put, key, value}) + {:reply, :ok, state} + end + + def handle_call({:delete, key}, _from, state) do + {value, state} = pop_in(state.bucket[key]) + broadcast(state, {:delete, key}) + {:reply, value, state} + end + + defp broadcast(state, message) do + for pid <- state.subscribers do + send(pid, message) + end + end +``` + +There is no need to modify the callback for `get/2`. And that's it, run the tests again, and our new test should pass! + +## Wiring it all up + +Now that our bucket deals with subscriptions, we need to expose this new functionality in our server. Let's once again start with the test. + +Open up `test/kv/server_test.exs` and add this new test: + +```elixir + test "subscribes to buckets", %{socket: socket, name: name} do + assert send_and_recv(socket, "CREATE #{name}\r\n") == "OK\r\n" + :gen_tcp.send(socket, "SUBSCRIBE #{name}\r\n") + + {:ok, other} = :gen_tcp.connect(~c"localhost", 4040, @socket_options) + + assert send_and_recv(other, "PUT #{name} milk 3\r\n") == "OK\r\n" + assert :gen_tcp.recv(socket, 0, 1000) == {:ok, "milk SET TO 3\r\n"} + + assert send_and_recv(other, "DELETE #{name} milk\r\n") == "OK\r\n" + assert :gen_tcp.recv(socket, 0, 1000) == {:ok, "milk DELETED\r\n"} + end +``` + +The test creates a bucket and subscribes to it. Then it opens up another TCP connection to send commands. For each command sent, we expect the subscribed socket to receive a message. + +To make the test pass, we need to change `KV.Command` to parse the new `SUBSCRIBE` command and then run it. Open up `lib/kv/commands.ex` and then first change the `parse/1` definition to the following: + +```elixir + def parse(line) do + case String.split(line) do + ["SUBSCRIBE", bucket] -> {:ok, {:subscribe, bucket}} + ["CREATE", bucket] -> {:ok, {:create, bucket}} + ["GET", bucket, key] -> {:ok, {:get, bucket, key}} + ["PUT", bucket, key, value] -> {:ok, {:put, bucket, key, value}} + ["DELETE", bucket, key] -> {:ok, {:delete, bucket, key}} + _ -> {:error, :unknown_command} + end + end +``` + +We added a new clause that converts "SUBSCRIBE" into a tuple. Now we need to match on this tuple within `run/1`. We can do so by adding a new clause at the bottom of `run/1`, with the following code: + +```elixir + def run({:subscribe, bucket}, socket) do + lookup(bucket, fn pid -> + KV.Bucket.subscribe(pid) + :inet.setopts(socket, active: true) + receive_messages(socket) + end) + end + + defp receive_messages(socket) do + receive do + {:put, key, value} -> + :gen_tcp.send(socket, "#{key} SET TO #{value}\r\n") + receive_messages(socket) + + {:delete, key} -> + :gen_tcp.send(socket, "#{key} DELETED\r\n") + receive_messages(socket) + + {:tcp_closed, ^socket} -> + {:error, :closed} + + # If we receive any message, including socket writes, we discard them + _ -> + receive_messages(socket) + end + end +``` + +Let's go over it by parts. We use the existing `lookup/2` private function to lookup for a bucket. If one is found, we subscribe the current process to the bucket. Then we call `:inet.setopts(socket, active: true)` (which we will explain soon) and `receive_messages/1`. + +`receive_messages/1` awaits for messages from the bucket and then calls itself again, becoming a loop. We match on `{:put, key, value}` and `{:delete, key}` and write to those events to the socket. We also match on `{:tcp_closed, ^socket}`, which is a message that will be delivered if the TCP socket closes, and use it to abort the loop. We discard any other message. + +At this point you may be wondering: where does `{:tcp_closed, ^socket}` come from? + +So far, when receiving messages from the socket, we used `:gen_tcp.recv/3` to perform calls that will block the current process until content is available. This is known as "passive mode". However, we can also ask `:gen_tcp` to stream messages to the current process inbox as they arrive, which is known as "active mode", which is exactly what we configured when we called `:inet.setopts(socket, active: true)`. Those messages have the shape `{:tcp, socket, data}`. When the socket is in active mode and it is closed, it delivers a `{:tcp_closed, socket}` message. Once we receive this message, we exit the loop, which will exit the connection process. Since the bucket is monitoring the process, it will automatically remove the subscription too. You could verify this in practice by adding a `COUNT SUBSCRIPTIONS` command that returns the number of subscribers for a given bucket. + +In practice, many systems would prefer to call `:inet.setopts(socket, active: :once)` to specify only a single TCP message should be delivered to avoid overflowing message queues. Once the message is received, they call `:inet.setopts/2` again. In our case, we are simply discarding anything that arrives over the socket, so setting `active: true` is equally fine. In all scenarios, the benefit of using active mode is that the process can receive TCP messages as well as messages from other processes at the same time, instead of blocking on `:gen_tcp.recv/3`. + +To wrap it all up, you should give our new feature a try in a distributed setting too. Start two `NODES=... PORT=... iex --sname ... -S mix` instances. In one of them, create a bucket. In the other, subscribe to the same bucket. Once you go back to the first shell, you will see that, even as you send commands to the bucket in one machine, the messages will be streamed to the other one. In other words, our subscription system is also distributed, and all we had to do is to send messages! + +## `call`, `cast` or `info`? + +So far we have used three callbacks: `handle_call/3`, `handle_cast/2` and `handle_info/2`. Here is what we should consider when deciding when to use each: + +1. `handle_call/3` must be used for synchronous requests. This should be the default choice as waiting for the server reply is a useful back-pressure mechanism. + +2. `handle_cast/2` must be used for asynchronous requests, when you don't care about a reply. A cast does not guarantee the server has received the message and, for this reason, should be used sparingly. For example, the `subscribe/1` function we have defined in this chapter should have used `call/2`. We have used `cast/2` for educational purposes. + +3. `handle_info/2` must be used for all other messages a server may receive that are not sent via `GenServer.call/2` or `GenServer.cast/2`, including regular messages sent with `send/2`. The monitoring `:DOWN` messages are an example of this. + +To help developers remember the differences between call, cast and info, the supported return values and more, we have a tiny [GenServer cheat sheet](https://elixir-lang.org/downloads/cheatsheets/gen-server.pdf). + +## Agents or GenServers? + +Before moving forward to the last chapter, you may be wondering: in the future, should you use an `Agent` or a `GenServer`? + +As we saw throughout this guide, agents are straight-forward to get started but they are limited in what they can do. Agents are effectively a subset of GenServers. In fact, agents are implemented on top of GenServers. As well as supervisors, the `Registry` module, and many other features you will find in both Erlang and Elixir. + +In other words, GenServers are the most essential component for building concurrent and fault-tolerant systems in Elixir. They provide a robust and flexible framework for managing state and coordinating interactions between processes. + +For those reasons, many adopt a rule of thumb to never use Agents and jump straight into GenServers instead. On the other hand, others are more than fine with using agents to store a bit of state here and there. Either way, you will be fine! + +This is the last feature we have implemented for our distributed key-value store. In the next chapter, we will learn how to package our application before shipping it to production. diff --git a/lib/elixir/pages/mix-and-otp/introduction-to-mix.md b/lib/elixir/pages/mix-and-otp/introduction-to-mix.md new file mode 100644 index 00000000000..80b2bdd2688 --- /dev/null +++ b/lib/elixir/pages/mix-and-otp/introduction-to-mix.md @@ -0,0 +1,329 @@ + + +# Introduction to Mix + +In this guide, we will build a complete Elixir application, with its own supervision tree, configuration, tests, and more. + +The requirements for this guide are (see `elixir -v`): + + * Elixir 1.18.0 onwards + * Erlang/OTP 27 onwards + +The application works as a distributed key-value store. We are going to organize key-value pairs into buckets and distribute those buckets across multiple nodes. We will also build a simple client that allows us to connect to any of those nodes and send requests such as: + +```text +CREATE shopping +OK + +PUT shopping milk 1 +OK + +PUT shopping eggs 3 +OK + +GET shopping milk +1 +OK + +DELETE shopping eggs +OK +``` + +In order to build our key-value application, we are going to use three main tools: + + * ***OTP*** *(Open Telecom Platform)* is a set of libraries that ships with Erlang. Erlang developers use OTP to build robust, fault-tolerant applications. In this chapter we will explore how many aspects from OTP integrate with Elixir, including supervision trees, event managers and more; + + * ***[Mix](`Mix`)*** is a build tool that ships with Elixir that provides tasks for creating, compiling, testing your application, managing its dependencies and much more; + + * ***[ExUnit](`ExUnit`)*** is a unit-test based framework that ships with Elixir. + +In this chapter, we will create our first project using Mix and explore different features in OTP, Mix, and ExUnit as we go. + +> #### Source code {: .info} +> +> The final code for the application built in this guide is in [this repository](https://github.com/josevalim/kv) and can be used as a reference. + +> #### Is this guide required reading? {: .info} +> +> This guide is not required reading in your Elixir journey. We'll explain. +> +> As an Elixir developer, you will most likely use one of the many existing frameworks when writing your Elixir code. [Phoenix](https://phoenixframework.org) covers web applications, [Ecto](https://github.com/elixir-ecto/ecto) communicates with databases, you can craft embedded software with [Nerves](https://nerves-project.org/), [Nx](https://github.com/elixir-nx) powers machine learning and AI projects, [Membrane](https://membrane.stream/) assembles audio/video processing pipelines, [Broadway](https://elixir-broadway.org/) handles data ingestion and processing, and many more. These frameworks handle the lower level details of concurrency, distribution, and fault-tolerance, so you, as a user, can focus on your own needs and demands. +> +> On the other hand, if you want to learn the foundations these frameworks are built upon, and the abstractions that power the Elixir ecosystem, this guide will give you a tour through several important concepts. + +## Our first project + +When you install Elixir, besides getting the `elixir`, `elixirc`, and `iex` executables, you also get an executable Elixir script named `mix`. + +Let's create our first project by invoking `mix new` from the command line. We'll pass the project path as the argument (`kv`, in this case). By default, the application name and module name will be retrieved from the path. So we tell Mix that our main module should be the all-uppercase `KV`, instead of the default, which would have been `Kv`: + +```console +$ mix new kv --module KV +``` + +Mix will create a directory named `kv` with a few files in it: + +```text +* creating README.md +* creating .formatter.exs +* creating .gitignore +* creating mix.exs +* creating lib +* creating lib/kv.ex +* creating test +* creating test/test_helper.exs +* creating test/kv_test.exs +``` + +Let's take a brief look at those generated files. + +> #### Executables in the `PATH` {: .info} +> +> Mix is an Elixir executable. This means that in order to run `mix`, you need to have both `mix` and `elixir` executables in your [`PATH`](https://en.wikipedia.org/wiki/PATH_(variable)). That's what happens when you install Elixir. + +## Project compilation + +A file named `mix.exs` was generated inside our new project folder (`kv`) and its main responsibility is to configure our project. Let's take a look at it: + +```elixir +defmodule KV.MixProject do + use Mix.Project + + def project do + [ + app: :kv, + version: "0.1.0", + elixir: "~> 1.11", + start_permanent: Mix.env() == :prod, + deps: deps() + ] + end + + # Run "mix help compile.app" to learn about applications + def application do + [ + extra_applications: [:logger] + ] + end + + # Run "mix help deps" to learn about dependencies + defp deps do + [ + # {:dep_from_hexpm, "~> 0.3.0"}, + # {:dep_from_git, git: "https://github.com/elixir-lang/my_dep.git", tag: "0.1.0"}, + ] + end +end +``` + +Our `mix.exs` defines two public functions: `project`, which returns project configuration like the project name and version, and `application`, which is used to generate an application file. + +There is also a private function named `deps`, which is invoked from the `project` function, that defines our project dependencies. Defining `deps` as a separate function is not required, but it helps keep the project configuration tidy. + +Mix also generates a file at `lib/kv.ex` with a module containing exactly one function, called `hello`: + +```elixir +defmodule KV do + @moduledoc """ + Documentation for KV. + """ + + @doc """ + Hello world. + + ## Examples + + iex> KV.hello() + :world + + """ + def hello do + :world + end +end + +``` + +This structure is enough to compile our project: + +```console +$ cd kv +$ mix compile +``` + +Will output: + +```text +Compiling 1 file (.ex) +Generated kv app +``` + +The `lib/kv.ex` file was compiled and an application manifest named `kv.app` was generated. All compilation artifacts are placed inside the `_build` directory using the options defined in the `mix.exs` file. + +Once the project is compiled, you can start a `iex` session inside the project by running the command below. The `-S mix` is necessary to load the project in the interactive shell: + +```console +$ iex -S mix +``` + +We are going to work on this `kv` project, making modifications and trying out the latest changes from a `iex` session. While you may start a new session whenever there are changes to the project source code, you can also recompile the project from within `iex` with the `recompile` helper, like this: + +```elixir +iex> recompile() +Compiling 1 file (.ex) +:ok +iex> recompile() +:noop +``` + +If anything had to be compiled, you see some informative text, and get the `:ok` atom back, otherwise the function is silent, and returns `:noop`. + +## Running tests + +Mix also generated the appropriate structure for running our project tests. Mix projects usually follow the convention of having a `_test.exs` file in the `test` directory for each file in the `lib` directory. For this reason, we can already find a `test/kv_test.exs` corresponding to our `lib/kv.ex` file. It doesn't do much at this point: + +```elixir +defmodule KVTest do + use ExUnit.Case + doctest KV + + test "greets the world" do + assert KV.hello() == :world + end +end +``` + +It is important to note a couple of things: + +1. the test file is an Elixir script file (`.exs`). This is convenient because we don't need to compile test files before running them; + +2. we define a test module named `KVTest`, in which we [`use ExUnit.Case`](`ExUnit.Case`) to inject the testing API; + +3. we use one of the imported macros, `ExUnit.DocTest.doctest/1`, to indicate that the `KV` module contains doctests (we will discuss those in a later chapter); + +4. we use the `ExUnit.Case.test/2` macro to define a simple test; + +Mix also generated a file named `test/test_helper.exs` which is responsible for setting up the test framework: + +```elixir +ExUnit.start() +``` + +This file will be required by Mix every time before we run our tests. We can run tests with: + +```console +$ mix test +Compiled lib/kv.ex +Generated kv app +Running ExUnit with seed: 540224, max_cases: 16 +.. + +Finished in 0.04 seconds +1 doctest, 1 test, 0 failures +``` + +Notice that by running `mix test`, Mix has compiled the source files and generated the application manifest once again. This happens because Mix supports multiple environments, which we will discuss later in this chapter. + +Furthermore, you can see that ExUnit prints a dot for each successful test and automatically randomizes tests too. Let's make the test fail on purpose and see what happens. + +Change the assertion in `test/kv_test.exs` to the following: + +```elixir +assert KV.hello() == :oops +``` + +Now run `mix test` again (notice this time there will be no compilation): + +```text + 1) test greets the world (KVTest) + test/kv_test.exs:5 + Assertion with == failed + code: assert KV.hello() == :oops + left: :world + right: :oops + stacktrace: + test/kv_test.exs:6: (test) + +. + +Finished in 0.05 seconds +1 doctest, 1 test, 1 failure +``` + +For each failure, ExUnit prints a detailed report, containing the test name with the test case, the code that failed and the values for the left side and right side (RHS) of the `==` operator. + +In the second line of the failure, right below the test name, there is the location where the test was defined. If you copy the test location in full, including the file and line number, and append it to `mix test`, Mix will load and run just that particular test: + +```console +$ mix test test/kv_test.exs:5 +``` + +This shortcut will be extremely useful as we build our project, allowing us to quickly iterate by running a single test. + +Finally, the stacktrace relates to the failure itself, giving information about the test and often the place the failure was generated from within the source files. + +## Automatic code formatting + +One of the files generated by `mix new` is the `.formatter.exs`. Elixir ships with a code formatter that is capable of automatically formatting our codebase according to a consistent style. The formatter is triggered with the `mix format` task. The generated `.formatter.exs` file configures which files should be formatted when `mix format` runs. + +To give the formatter a try, change a file in the `lib` or `test` directories to include extra spaces or extra newlines, such as `def hello do`, and then run `mix format`. + +Most editors provide built-in integration with the formatter, allowing a file to be formatted on save or via a chosen keybinding. If you are learning Elixir, editor integration gives you useful and quick feedback when learning the Elixir syntax. + +For companies and teams, we recommend developers to run `mix format --check-formatted` on their continuous integration servers, ensuring all current and future code follows the standard. + +You can learn more about the code formatter by checking [the format task documentation](`mix format`) or by reading [the release announcement for Elixir v1.6](https://elixir-lang.org/blog/2018/01/17/elixir-v1-6-0-released/), the first version to include the formatter. + +## Environments + +Mix provides the concept of "environments". They allow a developer to customize compilation and other options for specific scenarios. By default, Mix understands three environments: + + * `:dev` — the one in which Mix tasks (like `compile`) run by default + * `:test` — used by `mix test` + * `:prod` — the one you will use to run your project in production + +The environment applies only to the current project. As we will see in future chapters, any dependency you add to your project will by default run in the `:prod` environment. + +Customization per environment can be done by accessing the `Mix.env/0` in your `mix.exs` file, which returns the current environment as an atom. That's what we have used in the `:start_permanent` options: + +```elixir +def project do + [ + ..., + start_permanent: Mix.env() == :prod, + ... + ] +end +``` + +When true, the `:start_permanent` option starts your application in permanent mode, which means the Erlang VM will crash if your application's supervision tree shuts down. Notice we don't want this behavior in dev and test because it is useful to keep the VM instance running in those environments for troubleshooting purposes. + +Mix will default to the `:dev` environment, except for the `test` task that will default to the `:test` environment. The environment can be changed via the `MIX_ENV` environment variable: + +```console +$ MIX_ENV=prod mix compile +``` + +Or on Windows: + +```batch +> set "MIX_ENV=prod" && mix compile +``` + +> #### Mix in production {: .warning} +> +> Mix is a **build tool** and, as such, it is not expected to be available in production. Therefore, it is recommended to access `Mix.env/0` only in configuration files and inside `mix.exs`, never in your application code (`lib`). + +## Exploring + +There is much more to Mix, and we will continue to explore it as we build our project. A general overview is available on the [Mix documentation](`Mix`) and you can always invoke the help task to list all available tasks: + +```console +$ mix help +$ mix help compile +``` + +Now let's move forward and add the first modules and functions to our application. diff --git a/lib/elixir/pages/mix-and-otp/releases.md b/lib/elixir/pages/mix-and-otp/releases.md new file mode 100644 index 00000000000..526d1bbb35d --- /dev/null +++ b/lib/elixir/pages/mix-and-otp/releases.md @@ -0,0 +1,170 @@ + + +# Releases + +Now that our application is ready, you may be wondering how we can package our application to run in production. After all, all of our code so far depends on Erlang and Elixir versions that are installed in your current system. To achieve this goal, Elixir provides releases. + +A release is a self-contained directory that consists of your application code, all of its dependencies, plus the whole Erlang Virtual Machine (VM) and runtime. Once a release is assembled, it can be packaged and deployed to a target as long as the target runs on the same operating system (OS) distribution and version as the machine that assembled the release. + +To get started, simply run `mix release` while setting `MIX_ENV=prod`: + +```console +$ MIX_ENV=prod mix release +Compiling 4 files (.ex) +Generated kv app +* assembling kv-0.1.0 on MIX_ENV=prod +* using config/runtime.exs to configure the release at runtime + +Release created at _build/prod/rel/kv + + # To start your system + _build/prod/rel/kv/bin/kv start + +Once the release is running: + + # To connect to it remotely + _build/prod/rel/kv/bin/kv remote + + # To stop it gracefully (you may also send SIGINT/SIGTERM) + _build/prod/rel/kv/bin/kv stop + +To list all commands: + + _build/prod/rel/kv/bin/kv +``` + +Excellent! A release was assembled in `_build/prod/rel/kv`. Everything you need to run your application is inside that directory. In particular, there is a `bin/kv` file which is the entry point to your system. It supports multiple commands, such as: + + * `bin/kv start`, `bin/kv start_iex`, `bin/kv restart`, and `bin/kv stop` — for general management of the release + + * `bin/kv rpc COMMAND` and `bin/kv remote` — for running commands on the running system or to connect to the running system + + * `bin/kv eval COMMAND` — to start a fresh system that runs a single command and then shuts down + + * `bin/kv daemon` and `bin/kv daemon_iex` — to start the system as a daemon on Unix-like systems + + * `bin/kv install` — to install the system as a service on Windows machines + +If you run `bin/kv start_iex` inside the release directory, it will start the system using a short name (`--sname`) equal to the release name, which in this case is `kv`. The next step is to start two instances, on different ports and different names, as we did earlier on. But before we do this, let's talk a bit about the benefits of releases. + +## Why releases? + +Releases allow developers to precompile and package all of their code and the runtime into a single unit. The benefits of releases are: + + * Code preloading. The VM has two mechanisms for loading code: interactive and embedded. By default, it runs in the interactive mode which dynamically loads modules when they are used for the first time. The first time your application calls `Enum.map/2`, the VM will find the `Enum` module and load it. There's a downside. When you start a new server in production, it may need to load many other modules, causing the first requests to have an unusual spike in response time. Releases run in embedded mode, which loads all available modules upfront, guaranteeing your system is ready to handle requests after booting. + + * Configuration and customization. Releases give developers fine grained control over system configuration and the VM flags used to start the system. + + * Self-contained. A release does not require the source code to be included in your production artifacts. All of the code is precompiled and packaged. Releases do not even require Erlang or Elixir on your servers, as they include the Erlang VM and its runtime by default. Furthermore, both Erlang and Elixir standard libraries are stripped to bring only the parts you are actually using. + + * Multiple releases. You can assemble different releases with different configuration per application or even with different applications altogether. + +We have written extensive documentation on releases, so [please check the official documentation for more information](`mix release`). For now, we will continue exploring some of the features outlined above. + +## Configuring releases + +Releases also provide built-in hooks for configuring almost every need of the production system: + + * `config/config.exs` — provides build-time application configuration, which is executed before our application compiles. This file often imports configuration files based on the environment, such as `config/dev.exs` and `config/prod.exs`. + + * `config/runtime.exs` — provides runtime application configuration. It is executed every time the release boots and is further extensible via config providers. + + * `rel/env.sh.eex` and `rel/env.bat.eex` — template files that are copied into every release and executed on every command to set up environment variables, including ones specific to the VM, and the general environment. + + * `rel/vm.args.eex` — a template file that is copied into every release and provides static configuration of the Erlang Virtual Machine and other runtime flags. + +In this case, we already have specified a `config/runtime.exs` that deals with both `PORT` and `NODES` environment variables. Furthermore, while releases don't accept a `--sname` parameter, they do allow us to set the name via the `RELEASE_NODE` env var. Therefore, we can start two copies of the system by jumping into `_build/prod/rel/kv` and typing this (remember to adjust `@computer-name` to your actual computer name): + +```console +$ NODES="foo@computer-name,bar@computer-name" PORT=4040 RELEASE_NODE="foo" bin/kv start_iex +``` + +```console +$ NODES="foo@computer-name,bar@computer-name" PORT=4041 RELEASE_NODE="bar" bin/kv start_iex +``` + +To verify it all worked out, you can type `Node.list` in the IEx section and see if it returns the other node. If it doesn't, you can start diagnosing, first by comparing the node names within each `iex>` prompt and calling `Node.connect/1` directly. With applications running, you can `telnet` into them as usual too. + +While the above is enough to get started, you may want to perform advanced configuration based on the environment you are replying to. Releases provide scripts for that, which are great to automate based on host, network, or cloud settings. + +## Operating System scripts + +Every release contains an environment file, named `env.sh` on Unix-like systems and `env.bat` on Windows machines, that executes before the Elixir system starts. In this file, you can execute any OS-level code, such as invoke other applications, set environment variables and so on. Some of those environment variables can even configure how the release itself runs. + +For instance, releases run using short-names (`--sname`). However, if you want to actually run a distributed key-value store in production, you will need multiple nodes and start the release with the `--name` option. We can achieve this by setting the `RELEASE_DISTRIBUTION` environment variable inside the `env.sh` and `env.bat` files. Mix already has a template for said files which we can customize, so let's ask Mix to copy them to our application: + + $ mix release.init + * creating rel/vm.args.eex + * creating rel/remote.vm.args.eex + * creating rel/env.sh.eex + * creating rel/env.bat.eex + +If you open up `rel/env.sh.eex`, you will see: + +```shell +#!/bin/sh + +# # Sets and enables heart (recommended only in daemon mode) +# case $RELEASE_COMMAND in +# daemon*) +# HEART_COMMAND="$RELEASE_ROOT/bin/$RELEASE_NAME $RELEASE_COMMAND" +# export HEART_COMMAND +# export ELIXIR_ERL_OPTIONS="-heart" +# ;; +# *) +# ;; +# esac + +# # Set the release to load code on demand (interactive) instead of preloading (embedded). +# export RELEASE_MODE=interactive + +# # Set the release to work across nodes. +# # RELEASE_DISTRIBUTION must be "sname" (local), "name" (distributed) or "none". +# export RELEASE_DISTRIBUTION=name +# export RELEASE_NODE=<%= @release.name %> +``` + +The steps necessary to work across nodes is already commented out as an example. You can enable full distribution by setting the `RELEASE_DISTRIBUTION` variable to `name`. + +If you are on Windows, you will have to open up `rel/env.bat.eex`, where you will find this: + +```bat +@echo off +rem Set the release to load code on demand (interactive) instead of preloading (embedded). +rem set RELEASE_MODE=interactive + +rem Set the release to work across nodes. +rem RELEASE_DISTRIBUTION must be "sname" (local), "name" (distributed) or "none". +rem set RELEASE_DISTRIBUTION=name +rem set RELEASE_NODE=<%= @release.name %> +``` + +Once again, set the `RELEASE_DISTRIBUTION` variable to `name` and you are good to go! + +## VM arguments + +The `rel/vm.args.eex` allows you to specify low-level flags that control how the Erlang VM and its runtime operate. You specify entries as if you were specifying arguments in the command line with code comments also supported. Here is the default generated file: + + ## Customize flags given to the VM: https://www.erlang.org/doc/man/erl.html + ## -mode/-name/-sname/-setcookie are configured via env vars, do not set them here + + ## Increase number of concurrent ports/sockets + ##+Q 65536 + + ## Tweak GC to run more often + ##-env ERL_FULLSWEEP_AFTER 10 + +You can see [a complete list of VM arguments and flags in the Erlang documentation](http://www.erlang.org/doc/man/erl.html). + +## Summing up + +Throughout the guide, we have built a very simple distributed key-value store as an opportunity to explore many constructs like generic servers, supervisors, tasks, agents, applications and more. Not only that, we have written tests for the whole application, got familiar with ExUnit, and learned how to use the Mix build tool to accomplish a wide range of tasks. + +If you are looking for a distributed key-value store to use in production, you should definitely look into [Riak](http://riak.com/products/riak-kv/), which also runs in the Erlang VM. In Riak, the buckets are replicated and stored across several nodes to avoid data loss. + +Of course, Elixir can be used for much more than distributed key-value stores. Embedded systems, data-processing and data-ingestion, web applications, audio/video streaming systems, machine learning, and others are many of the different domains Elixir excels at. We hope this guide has prepared you to explore any of those domains or any future domain you may desire to bring Elixir into. + +Happy coding! diff --git a/lib/elixir/pages/mix-and-otp/supervisor-and-application.md b/lib/elixir/pages/mix-and-otp/supervisor-and-application.md new file mode 100644 index 00000000000..acc1fc8ee83 --- /dev/null +++ b/lib/elixir/pages/mix-and-otp/supervisor-and-application.md @@ -0,0 +1,253 @@ + + +# Registries and supervision trees + +In the [previous chapter](agents.md), we used agents to represent our buckets. In the [introduction to mix](introduction-to-mix.md), we specified we would like to name each bucket so we can do the following: + +```text +CREATE shopping +OK + +PUT shopping milk 1 +OK + +GET shopping milk +1 +OK +``` + +In the example session above we interacted with the "shopping" bucket by referencing its name. Therefore, an important feature in our key-value store is to give names to processes. + +We have also learned in the previous chapter we can already name our buckets. For example: + +```elixir +iex> KV.Bucket.start_link(name: :shopping) +{:ok, #PID<0.43.0>} +iex> KV.Bucket.put(:shopping, "milk", 1) +:ok +iex> KV.Bucket.get(:shopping, "milk") +1 +``` + +However, naming dynamic processes with atoms is a terrible idea! If we use atoms, we would need to convert the bucket name (often received from an external client) to atoms, and **we should never convert user input to atoms**. This is because atoms are not garbage collected. Once an atom is created, it is never reclaimed. Generating atoms from user input would mean the user can inject enough different names to exhaust our system memory! + +In practice, it is more likely you will reach the Erlang VM limit for the maximum number of atoms before you run out of memory, which will bring your system down regardless. + +Luckily, Elixir (and Erlang) comes with built-in abstractions for naming processes, called name registries, each with different trade-offs which we will explore throughout these guides. + +## Local, decentralized, and scalable registry + +Elixir ships with a single-node process registry module aptly called `Registry`. Its main feature is that you can use any Elixir value to name a process, not only atoms. Let's take it for a spin in `iex`: + +```elixir +iex> Registry.start_link(name: KV, keys: :unique) +iex> name = {:via, Registry, {KV, "shopping"}} +iex> KV.Bucket.start_link(name: name) +{:ok, #PID<0.43.0>} +iex> KV.Bucket.put(name, "milk", 1) +:ok +iex> KV.Bucket.get(name, "milk") +1 +``` + +As you can see, instead of passing an atom to the `:name` option, we pass a tuple of shape `{:via, registry_module, {registry_name, process_name}}`, and everything just worked. You could have used anything as the `process_name`, even an integer or a map! That's because all of Elixir built-in behaviours, agents, supervisors, tasks, etc, are compatible with naming registries, as long as you pass them using the "via" tuple format. + +Therefore, all we need to do to name our buckets is to start a `Registry`, using `Registry.start_link/1`. But you may be wondering, where exactly should we place that? + +## Understanding applications + +Every Elixir project is an application. Elixir itself is defined in an application named `:elixir`. The `ExUnit.Case` module is part of the `:ex_unit` application. And so forth. + +In fact, we have been working inside an application this entire time. Every time we changed a file and ran `mix compile`, we could see a `Generated kv app` message in the compilation output. + +We can find the generated `.app` file at `_build/dev/lib/kv/ebin/kv.app`. Let's have a look at its contents: + +```erlang +{application,kv, + [{applications,[kernel,stdlib,elixir,logger]}, + {description,"kv"}, + {modules,['Elixir.KV','Elixir.KV.Bucket']}, + {registered,[]}, + {vsn,"0.1.0"}]}. +``` + +This file contains Erlang terms (written using Erlang syntax). Even though we are not familiar with Erlang, it is easy to guess this file holds our application definition. It contains our application `version`, all the modules defined by it, as well as a list of applications we depend on, like Erlang's `kernel`, `elixir` itself, and `logger`. + +> The `logger` application ships as part of Elixir. We stated that our application needs it by specifying it in the `:extra_applications` list in `mix.exs`. See the [official documentation](`Logger`) for more information. + +In a nutshell, an application consists of all the modules defined in the `.app` file, including the `.app` file itself. The application itself is located at the `_build/dev/lib/kv` folder and typically has only two directories: `ebin`, for Elixir artifacts, such as `.beam` and `.app` files, and `priv`, with any other artifact or asset you may need in your application. + +Although Mix generates and maintains the `.app` file for us, we can customize its contents by adding new entries to the `application/0` function inside the `mix.exs` project file. We are going to do our first customization soon. + +### Starting applications + +Each application in our system can be started and stopped. The rules for starting and stopping an application are also defined in the `.app` file. When we invoke `iex -S mix`, Mix compiles our application and then starts it. + +Let's see this in practice. Start a console with `iex -S mix` and try: + +```elixir +iex> Application.start(:kv) +{:error, {:already_started, :kv}} +``` + +Oops, it's already started. Mix starts the current application and all of its dependencies automatically. This is also true for `mix test` and many other Mix commands. + +We can, however, stop our `:kv` application, as well as the `:logger` application: + +```elixir +iex> Application.stop(:kv) +:ok +iex> Application.stop(:logger) +:ok +``` + +And let's try to start our application again: + +```elixir +iex> Application.start(:kv) +{:error, {:not_started, :logger}} +``` + +Now we get an error because an application that `:kv` depends on (`:logger` in this case) isn't started. We need to either start each application manually in the correct order or call `Application.ensure_all_started/1` as follows: + +```elixir +iex> Application.ensure_all_started(:kv) +{:ok, [:logger, :kv]} +``` + +In practice, our tools always start our applications for us, and you don't have to worry about the above, but it is good to know how it all works behind the scenes. + +### The application callback + +Whenever we invoke `iex -S mix`, Mix automatically starts our application by calling `Application.start(:kv)`. But can we customize what happens when our application starts? As a matter of fact, we can! To do so, we define an application callback. + +The first step is to tell our application definition (for example, our `.app` file) which module is going to implement the application callback. Let's do so by opening `mix.exs` and changing `def application` to the following: + +```elixir + def application do + [ + extra_applications: [:logger], + mod: {KV, []} + ] + end +``` + +The `:mod` option specifies the "application callback module", followed by the arguments to be passed on application start. The application callback module can be any module that invokes `use Application`. Since we have specified `KV` as the module callback, let's change the `KV` module defined in `lib/kv.ex` to the following: + +```elixir +defmodule KV do + use Application +end +``` + +Now run `mix test` and you will see a couple things happening. First of all, you will get a compilation warning: + +```text +Compiling 1 file (.ex) + warning: function start/2 required by behaviour Application is not implemented (in module KV) + │ + 1 │ defmodule KV do + │ ~~~~~~~~~~~~~~~ + │ + └─ lib/kv.ex:1: KV (module) +``` + +This warning is telling us that `use Application` actually defines a behaviour, which expects us to implement to a `start/2` function in our `KV` module. + +Then our application does not even boot because the `start/2` function is not actually implemented: + +```text +18:29:39.109 [notice] Application kv exited: exited in: KV.start(:normal, []) + ** (EXIT) an exception was raised: + ** (UndefinedFunctionError) function KV.start/2 is undefined or private +``` + +Implementing the `start/2` callback is relatively straight-forward, all we need to do is to start a supervision tree, and return `{:ok, root_supervisor_pid}`. The `Supervisor.start_link/2` function does precisely that, it only expects a list of children and the supervision strategy. Let's just pass an empty list of children for now: + +```elixir +defmodule KV do + use Application + + # The @impl true annotation says we are implementing a callback + @impl true + def start(_type, _args) do + Supervisor.start_link([], strategy: :one_for_one) + end +end +``` + +Now run `mix test` again and our app should boot but we should see one failure. When we changed the `KV` module, we broke the boilerplate test case which tested the `KV.hello/0` function. You can simply remove that test case and we are back to a green suite. + +We wrote very little code but we did something incredibly powerful. We now have a function, `KV.start/2` that is invoked whenever your application starts. This gives us the perfect place to start our key-value registry. The `Application` module also allows us to define a `stop/1` callback and other functionality. You can check the `Application` and `Supervisor` modules for extensive documentation on their uses. + +Let's finally start our registry. + +## Supervision trees + +Now that we have the `start/2` callback, we can finally go ahead and start our registry. You may be tempted to do it like this: + +```elixir + def start(_type, _args) do + Registry.start_link(name: KV, keys: :unique) + Supervisor.start_link([], strategy: :one_for_one) + end +``` + +However, this would not be a good idea. In Elixir, we typically start processes inside supervision trees. In fact, we rarely use the `start_link` functions to start processes (except at the root of the supervision tree itself). Instead, do this: + +```elixir + def start(_type, _args) do + children = [ + {Registry, name: KV, keys: :unique} + ] + + Supervisor.start_link(children, strategy: :one_for_one) + end +``` + +A supervisor receives one or more child specifications that tell it exactly how to start each child. A child specification is typically represented by a `{module, options}` pair, as shown above, and often as simply the module name. Sometimes, these children are supervisors themselves, giving us supervision trees. + +Let's take it for a spin and see if we can indeed name our buckets using our new registry. Let's make sure to start a new `iex -S mix` (`recompile()` is not enough, as it does not reload your supervision tree) and then: + +```iex +iex> name = {:via, Registry, {KV, "shopping"}} +iex> KV.Bucket.start_link(name: name) +{:ok, #PID<0.43.0>} +iex> KV.Bucket.put(name, "milk", 1) +:ok +iex> KV.Bucket.get(name, "milk") +1 +``` + +Perfect, this time we didn't need to start the registry inside `iex`, as it was started as part of the application itself. + +By starting processes inside supervisors, we gain important properties such as: + + * **Introspection**: for each application, you can fully introspect and visualize each process in its supervision tree, its memory usage, message queue, etc + + * **Resilience**: when a process fails for an unexpected reason, its supervisor controls if and how those processes should be restarted, leading to self-healing systems + + * **Graceful shutdown**: when your application is shutting down, the children of a supervision tree are terminated in the opposite order they were started, leading to graceful shutdowns + +## Projects or applications? + +Mix makes a distinction between projects and applications. Based on the contents of our `mix.exs` file, we would say we have a Mix project that defines the `:kv` application. + +When we say "project" you should think about Mix. Mix is the tool that manages your project. It knows how to compile your project, test your project and more. It also knows how to compile and start the application relevant to your project. + +When we talk about applications, we talk about OTP. Applications are the entities that are started and stopped as a whole by the runtime. You can learn more about applications and how they relate to booting and shutting down of your system as a whole in the documentation for the `Application` module. + +## Summing up + +We learned important concepts in this chapter: + + * Naming registries allow us to find processes in a given machine (or, as we will see in the future, even in a cluster) + + * Applications bundle our modules, its dependencies, and how code starts and stops + + * Processes are started as part of supervisors for introspection and fault-tolerance + +In the next chapter, we will tie it all up by making sure all our buckets are named and supervised. To do so, we will learn a new tool called dynamic supervisors. diff --git a/lib/elixir/pages/mix-and-otp/task-and-gen-tcp.md b/lib/elixir/pages/mix-and-otp/task-and-gen-tcp.md new file mode 100644 index 00000000000..28468361b26 --- /dev/null +++ b/lib/elixir/pages/mix-and-otp/task-and-gen-tcp.md @@ -0,0 +1,308 @@ + + +# Task and gen_tcp + +In this chapter, we are going to learn how to use Erlang's [`:gen_tcp` module](`:gen_tcp`) to serve requests. This provides a great opportunity to explore Elixir's `Task` module. In future chapters, we will expand our server so that it can actually interact with buckets. + +## Echo server + +We will start our TCP server by first implementing an echo server. It will send a response with the text it received in the request. We will slowly improve our server until it is supervised and ready to handle multiple connections. + +A TCP server, in broad strokes, performs the following steps: + + 1. Listens to a port until the port is available and it gets hold of the socket + 2. Waits for a client connection on that port and accepts it + 3. Reads the client request and writes a response back + +Let's implement those steps. Create a new `lib/kv/server.ex` and add the following functions: + +```elixir +defmodule KV.Server do + require Logger + + def accept(port) do + # The options below mean: + # + # 1. `:binary` - receives data as binaries (instead of lists) + # 2. `packet: :line` - receives data line by line + # 3. `active: false` - blocks on `:gen_tcp.recv/2` until data is available + # 4. `reuseaddr: true` - allows us to reuse the address if the listener crashes + # + {:ok, socket} = + :gen_tcp.listen(port, [:binary, packet: :line, active: false, reuseaddr: true]) + Logger.info("Accepting connections on port #{port}") + loop_acceptor(socket) + end + + defp loop_acceptor(socket) do + {:ok, client} = :gen_tcp.accept(socket) + serve(client) + loop_acceptor(socket) + end + + defp serve(socket) do + socket + |> read_line() + |> write_line(socket) + + serve(socket) + end + + defp read_line(socket) do + {:ok, data} = :gen_tcp.recv(socket, 0) + data + end + + defp write_line(line, socket) do + :gen_tcp.send(socket, line) + end +end +``` + +We are going to start our server by calling `KV.Server.accept(4040)`, where 4040 is the port. The first step in `accept/1` is to listen to the port until the socket becomes available and then call `loop_acceptor/1`. `loop_acceptor/1` is a loop accepting client connections. For each accepted connection, we call `serve/1`. + +`serve/1` is another loop that reads a line from the socket and writes those lines back to the socket. Note that the `serve/1` function uses the pipe operator `|>/2` to express this flow of operations. The pipe operator evaluates the left side and passes its result as the first argument to the function on the right side. The example above: + +```elixir +socket |> read_line() |> write_line(socket) +``` + +is equivalent to: + +```elixir +write_line(read_line(socket), socket) +``` + +The `read_line/1` implementation receives data from the socket using `:gen_tcp.recv/2` and `write_line/2` writes to the socket using `:gen_tcp.send/2`. + +Note that `serve/1` is an infinite loop called sequentially inside `loop_acceptor/1`, so the tail call to `loop_acceptor/1` is never reached and could be avoided. However, as we shall see, we will need to execute `serve/1` in a separate process, so we will need that tail call soon. + +This is pretty much all we need to implement our echo server. Let's give it a try! + +Start an IEx session inside the `kv_server` application with `iex -S mix`. Inside IEx, run: + +```elixir +iex> KV.Server.accept(4040) +``` + +The server is now running, and you will even notice the console is blocked. Let's use [a `telnet` client](https://en.wikipedia.org/wiki/Telnet) to access our server. There are clients available on most operating systems, and their command lines are generally similar: + +```console +$ telnet 127.0.0.1 4040 +Trying 127.0.0.1... +Connected to localhost. +Escape character is '^]'. +hello +hello +is it me +is it me +you are looking for? +you are looking for? +``` + +Type "hello", press enter, and you will get "hello" back. Excellent! + +My particular telnet client can be exited by typing `ctrl + ]`, typing `quit`, and pressing ``, but your client may require different steps. + +Once you exit the telnet client, you will likely see an error in the IEx session: + +```text +** (MatchError) no match of right hand side value: {:error, :closed} + (kv) lib/kv/server.ex:45: KV.Server.read_line/1 + (kv) lib/kv/server.ex:37: KV.Server.serve/1 + (kv) lib/kv/server.ex:30: KV.Server.loop_acceptor/1 +``` + +That's because we were expecting data from `:gen_tcp.recv/2` but the client closed the connection. We need to handle such cases better in future revisions of our server. + +For now, there is a more important bug we need to fix: what happens if our TCP acceptor crashes? Since there is no supervision, the server dies and we won't be able to serve more requests, because it won't be restarted. That's why we must move our server to a supervision tree. + +## Tasks + +Whenever you have an existing function and you simply want to execute it when your application starts, the `Task` module is exactly what you need. For example, it has a `Task.start_link/1` function that receives an anonymous function and executes it inside a new process that will be part of a supervision tree. + +Let's give it a try. Open up `lib/kv.ex` and let's add a new child: + +```elixir + def start(_type, _args) do + children = [ + {Registry, name: KV, keys: :unique}, + {DynamicSupervisor, name: KV.BucketSupervisor, strategy: :one_for_one}, + {Task, fn -> KV.Server.accept(4040) end} + ] + + Supervisor.start_link(children, strategy: :one_for_one) + end +``` + +With this change, we are saying that we want to run `KV.Server.accept(4040)` as a task. We are hardcoding the port for now but we will make this a configuration in later chapters. As usual, we've passed a two-element tuple as a child specification, which in turn will invoke `Task.start_link/1`. + +Now that the server is part of the supervision tree, it should start automatically when we run the application. Run `iex -S mix` to boot the app and use the `telnet` client to make sure that everything still works: + +```console +$ telnet 127.0.0.1 4321 +Trying 127.0.0.1... +Connected to localhost. +Escape character is '^]'. +say you +say you +say me +say me +``` + +Yes, it works! However, can it handle more than one client? + +Try to connect two telnet clients at the same time. When you do so, you will notice that the second client doesn't echo: + +```console +$ telnet 127.0.0.1 4321 +Trying 127.0.0.1... +Connected to localhost. +Escape character is '^]'. +hello +hello? +HELLOOOOOO? +``` + +It doesn't seem to work at all. That's because we are serving requests in the same process that are accepting connections. When one client is connected, we can't accept another client. + +## Adding (flawed) concurrency + +In order to make our server handle simultaneous connections, we need to have one process working as an acceptor that spawns other processes to serve requests. One solution would be to change: + +```elixir +defp loop_acceptor(socket) do + {:ok, client} = :gen_tcp.accept(socket) + serve(client) + loop_acceptor(socket) +end +``` + +to also use `Task.start_link/1`: + +```elixir +defp loop_acceptor(socket) do + {:ok, client} = :gen_tcp.accept(socket) + {:ok, pid} = Task.start_link(fn -> serve(client) end) + :ok = :gen_tcp.controlling_process(client, pid) + loop_acceptor(socket) +end +``` + +In the new acceptor loop, we are starting a new task every time there is a new client. Now, if you attempt to connect two clients at the same time, it should work! + +Or does it? For example, what happens when you exit one telnet session? The other session should crash! The reason of this crash is two fold: + +1. We have a bug in our server where we don't expect `:gen_tcp.recv/2` to return an `{:error, :closed}` tuple + +2. Because each server task is linked to the acceptor process, if one task crashes, the acceptor process will also crash, taking down all other tasks and clients + +An important rule of thumb throughout this guide is to always start processes as children of supervisors. The code above is an excellent example of what happens when we don't. If we don't isolate the different parts of our systems, failures can now cascade through our system, as it would happen in other languages. + +To fix this, we could use a `DynamicSupervisor`, but tasks also provide a specialized `Task.Supervisor` which has better ergonomics and is optimized for supervising tasks themselves. Let's give it a try. + +## Adding a task supervisor + +Let's change `start/2` in `lib/kv.ex` once more, to add the task supervisor to our tree: + +```elixir + def start(_type, _args) do + children = [ + {Registry, name: KV, keys: :unique}, + {DynamicSupervisor, name: KV.BucketSupervisor, strategy: :one_for_one}, + {Task.Supervisor, name: KV.ServerSupervisor}, + {Task, fn -> KV.Server.accept(4040) end} + ] + + Supervisor.start_link(children, strategy: :one_for_one) + end +``` + +We'll now start a `Task.Supervisor` process with name `KV.TaskSupervisor`. Keep in mind that the order children are started matters. For example, the acceptor must come last because, if it comes first, it means our application can start accepting requests before the `Task.Supervisor` is running or before we can locate buckets. Shutting down an application will also stop the children in reverse order, guaranteeing a clean termination. + +Now we need to change `loop_acceptor/1` to use `Task.Supervisor` to serve each request: + +```elixir +defp loop_acceptor(socket) do + {:ok, client} = :gen_tcp.accept(socket) + {:ok, pid} = Task.Supervisor.start_child(KV.ServerSupervisor, fn -> serve(client) end) + :ok = :gen_tcp.controlling_process(client, pid) + loop_acceptor(socket) +end +``` + +You might notice that we added a line, `:ok = :gen_tcp.controlling_process(client, pid)`. This makes the child process the "controlling process" of the `client` socket. If we didn't do this, the acceptor would bring down all the clients if it crashed because sockets would be tied to the process that accepted them (which is the default behavior). + +Now start a new server with `iex -S mix` and try to open up many concurrent telnet clients. You will notice that quitting a client does not bring the acceptor down, even though we haven't fixed the bug in `:gen_tcp.recv/2` yet (which we will address in the next chapter). Excellent! + +## Restart strategies + +There is one important topic we haven't explored yet with the necessary depth. What happens when a supervised process crashes? + +In the previous chapter, when we started a bucket and killed it, the supervisor automatically started one in its place: + +```elixir +iex> children = [{KV.Bucket, name: :shopping}] +iex> Supervisor.start_link(children, strategy: :one_for_one) +iex> KV.Bucket.put(:shopping, "milk", 1) +iex> pid = Process.whereis(:shopping) +#PID<0.48.0> +iex> Process.exit(pid, :kill) +true +iex> Process.whereis(:shopping) +#PID<0.50.0> +``` + +What exactly happens when a process terminates is part of its child specification. For `KV.Bucket`, we have this: + +```elixir +iex> KV.Bucket.child_spec([]) +%{id: KV.Bucket, start: {KV.Bucket, :start_link, [[]]}} +``` + +However, for tasks, we have this: + +```elixir +iex> Task.child_spec(fn -> :ok end) +%{ + id: Task, + restart: :temporary, + start: {Task, :start_link, [#Function<43.39164016/0 in :erl_eval.expr/6>]} +} +``` + +Notice that a task says `:restart` is `:temporary`. `KV.Bucket` says nothing, which means it defaults to `:permanent`. `:temporary` means that a process is never restarted, regardless of why it crashed. `:permanent` means a process is always restarted, regardless of the exit reason. There is also `:transient`, which means it won't be restarted as long as it terminates successfully. + +Now we must ask ourselves, are those the correct settings? + +For `KV.Bucket`, using `:permanent` seems logical, as we should not require the user to recreate a bucket they have previously created. Although currently we would lose the bucket data, in an actual system we would add mechanisms to recover it on initialization. However, for tasks, we have used them in two opposing ways in this chapter, which means at least one of them is wrong. + +We use a task to start the acceptor. The acceptor is a critical component of our infrastructure. If it crashes, it means we won't accept further requests, and our server would then be useless as no one can connect to it. On the other hand, we also use `Task.Supervisor` to start tasks that deal with each connection. In this case, restarting may not be useful at all, given the reason we crashed could just as well be a connection issue, and attempting to restart over the same connection would lead to further failures. + +Therefore, we want the acceptor to actually run in `:permanent` mode, while we preserve the `Task.Supervisor` as `:temporary`. Luckily Elixir has an API that allows us to change an existing child specification, which we use below. + +Let's change `start/2` in `lib/kv.ex` once more to the following: + +```elixir + def start(_type, _args) do + children = [ + {Registry, name: KV, keys: :unique}, + {DynamicSupervisor, name: KV.BucketSupervisor, strategy: :one_for_one}, + {Task.Supervisor, name: KV.ServerSupervisor}, + Supervisor.child_spec({Task, fn -> KV.Server.accept(4040) end}, restart: :permanent) + ] + + Supervisor.start_link(children, strategy: :one_for_one) + end +``` + +Now we have an always running acceptor that starts temporary task processes under an always running task supervisor. + +## Leveraging the ecosystem + +In this chapter, we implemented a basic TCP acceptor while exploring concurrency and fault-tolerance. Our acceptor can manage concurrent connections, but it is still not ready for production. Production-ready TCP servers run a pool of acceptors, each with their own supervisor. Elixir's `PartitionSupervisor` might be used to partition and scale the acceptor, but it is out of scope for this guide. In practice, you will use existing packages tailored for this use-case, such as [Ranch](https://github.com/ninenines/ranch) (in Erlang) or [Thousand Island](https://github.com/mtrudel/thousand_island) (in Elixir). + +In the next chapter, we will start parsing the client requests and sending responses, finishing our server. diff --git a/lib/elixir/pages/references/compatibility-and-deprecations.md b/lib/elixir/pages/references/compatibility-and-deprecations.md new file mode 100644 index 00000000000..75a637fe804 --- /dev/null +++ b/lib/elixir/pages/references/compatibility-and-deprecations.md @@ -0,0 +1,246 @@ + + +# Compatibility and deprecations + +Elixir is versioned according to a vMAJOR.MINOR.PATCH schema. + +Elixir is currently at major version v1. A new backwards compatible minor release happens every 6 months. Patch releases are not scheduled and are made whenever there are bug fixes or security patches. + +Elixir applies bug fixes only to the latest minor branch. Security patches are available for the last 5 minor branches: + +Elixir version | Support +:------------- | :----------------------------- +1.20 | Development +1.19 | Bug fixes and security patches +1.18 | Security patches only +1.17 | Security patches only +1.16 | Security patches only +1.15 | Security patches only + +New releases are announced in the read-only [announcements mailing list](https://groups.google.com/group/elixir-lang-ann). All security releases [will be tagged with `[security]`](https://groups.google.com/forum/#!searchin/elixir-lang-ann/%5Bsecurity%5D%7Csort:date). + +There are currently no plans for a major v2 release. + +## Between non-major Elixir versions + +Elixir minor and patch releases are backwards compatible: well-defined behaviors and documented APIs in a given version will continue working on future versions. + +Although we expect the vast majority of programs to remain compatible over time, it is impossible to guarantee that no future change will break any program. Under some unlikely circumstances, we may introduce changes that break existing code: + + * Security: a security issue in the implementation may arise whose resolution requires backwards incompatible changes. We reserve the right to address such security issues. + + * Bugs: if an API has undesired behavior, a program that depends on the buggy behavior may break if the bug is fixed. We reserve the right to fix such bugs. + + * Compiler front-end: improvements may be done to the compiler, introducing new warnings for ambiguous modes and providing more detailed error messages. Those can lead to compilation errors (when running with `--warnings-as-errors`) or tooling failures when asserting on specific error messages (although one should avoid such). We reserve the right to do such improvements. + + * Imports: new functions may be added to the `Kernel` module, which is auto-imported. They may collide with local functions defined in your modules. Collisions can be resolved in a backwards compatible fashion using `import Kernel, except: [...]` with a list of all functions you don't want to be imported from `Kernel`. We reserve the right to do such additions. + +In order to continue evolving the language without introducing breaking changes, Elixir will rely on deprecations to demote certain practices and promote new ones. Our deprecation policy is outlined in the ["Deprecations" section](#deprecations). + +The only exception to the compatibility guarantees above are experimental features, which will be explicitly marked as such, and do not provide any compatibility guarantee until they are stabilized. + +## Between Elixir and Erlang/OTP + +Erlang/OTP versioning is independent from the versioning of Elixir. Erlang releases a new major version yearly. Our goal is to support the last three Erlang major versions by the time Elixir is released. The compatibility table is shown below. + +Elixir version | Supported Erlang/OTP versions +:------------- | :------------------------------- +1.20 | 26 - 28 +1.19 | 26 - 28 +1.18 | 25 - 27 +1.17 | 25 - 27 +1.16 | 24 - 26 +1.15 | 24 - 26 +1.14 | 23 - 25 (and Erlang/OTP 26 from v1.14.5) +1.13 | 22 - 24 (and Erlang/OTP 25 from v1.13.4) +1.12 | 22 - 24 +1.11 | 21 - 23 (and Erlang/OTP 24 from v1.11.4) +1.10 | 21 - 22 (and Erlang/OTP 23 from v1.10.3) +1.9 | 20 - 22 +1.8 | 20 - 22 +1.7 | 19 - 22 +1.6 | 19 - 20 (and Erlang/OTP 21 from v1.6.6) +1.5 | 18 - 20 +1.4 | 18 - 19 (and Erlang/OTP 20 from v1.4.5) +1.3 | 18 - 19 +1.2 | 18 - 18 (and Erlang/OTP 19 from v1.2.6) +1.1 | 17 - 18 +1.0 | 17 - 17 (and Erlang/OTP 18 from v1.0.5) + +Elixir may add compatibility to new Erlang/OTP versions on patch releases, such as support for Erlang/OTP 20 in v1.4.5. Those releases are made for convenience and typically contain the minimum changes for Elixir to run without errors, if any changes are necessary. Only the next minor release, in this case v1.5.0, effectively leverages the new features provided by the latest Erlang/OTP release. + +## Deprecations + +### Policy + +Elixir deprecations happen in 3 steps: + + 1. The feature is soft-deprecated. It means both CHANGELOG and documentation must list the feature as deprecated but no warning is effectively emitted by running the code. There is no requirement to soft-deprecate a feature. + + 2. The feature is effectively deprecated by emitting warnings on usage. This is also known as hard-deprecation. In order to deprecate a feature, the proposed alternative MUST exist for AT LEAST THREE minor versions. For example, `Enum.uniq/2` was soft-deprecated in favor of `Enum.uniq_by/2` in Elixir v1.1. This means a deprecation warning may only be emitted by Elixir v1.4 or later. + + 3. The feature is removed. This can only happen on major releases. This means deprecated features in Elixir v1.x shall only be removed by Elixir v2.x. + +### Table of deprecations + +The first column is the version the feature was hard deprecated. The second column shortly describes the deprecated feature and the third column explains the replacement and from which the version the replacement is available from. + +Version | Deprecated feature | Replaced by (available since) +:-------| :-------------------------------------------------- | :--------------------------------------------------------------- +[v1.20] | `<>` in patterns without `^` | `<>` (v1.15) +[v1.20] | `File.stream!(path, modes, lines_or_bytes)` | `File.stream!(path, lines_or_bytes, modes)` (v1.16) +[v1.20] | `Kernel.ParallelCompiler.async/1` | `Kernel.ParallelCompiler.pmap/2` (v1.16) +[v1.20] | `Logger.*_backend` functions | The `LoggerBackends` module from [`:logger_backends`](https://hex.pm/packages/logger_backends) package +[v1.20] | `Logger.enable/1` and `Logger.disable/1` | `Logger.put_process_level/2` and `Logger.delete_process_level/1` respectively (v1.15) +[v1.19] | CLI configuration in `def project` inside `mix.exs` | Moving it to `def cli` (v1.14) +[v1.19] | Using `,` to separate tasks in `mix do` | Using `+` (v1.14) +[v1.19] | `Logger`'s `:backends` configuration | `Logger`'s `:default_handler` configuration (v1.15) +[v1.19] | Passing a callback to `File.cp/3`, `File.cp!/3`, `File.cp_r/3`, and `File.cp_r!/3` | The `:on_conflict` option (v1.14) +[v1.18] | `<%#` in EEx | `<%!--` (v1.14) or `<% #` (v1.0) +[v1.18] | `EEx.Engine.handle_text/2` callback in EEx | `c:EEx.Engine.handle_text/3` (v1.14) +[v1.18] | Returning a 2-arity function from `Enumerable.slice/1` | Returning a 3-arity function (v1.14) +[v1.18] | Ranges with negative steps in `Range.new/2` | Explicit steps in ranges (v1.11) +[v1.18] | `Tuple.append/2` | `Tuple.insert_at/3` (v1.0) +[v1.18] | `mix cmd --app APP` | `mix do --app APP` (v1.14) +[v1.18] | `List.zip/1` | `Enum.zip/1` (v1.0) +[v1.18] | `Module.eval_quoted/3` | `Code.eval_quoted/3` (v1.0) +[v1.17] | Single-quoted charlists (`'foo'`) | `~c"foo"` (v1.0) +[v1.17] | `left..right` in patterns and guards | `left..right//step` (v1.11) +[v1.17] | `ExUnit.Case.register_test/4` | `register_test/6` (v1.10) +[v1.17] | `:all` in `IO.read/2` and `IO.binread/2` | `:eof` (v1.13) +[v1.16] | `~R/.../` | `~r/.../` (v1.0) +[v1.16] | Ranges with negative steps in `Enum.slice/2` | Explicit steps in ranges (v1.11) +[v1.16] | Ranges with negative steps in `String.slice/2` | Explicit steps in ranges (v1.11) +[v1.15] | `Calendar.ISO.day_of_week/3` | `Calendar.ISO.day_of_week/4` (v1.11) +[v1.15] | `Exception.exception?/1` | `Kernel.is_exception/1` (v1.11) +[v1.15] | `Regex.regex?/1` | `Kernel.is_struct/2` (`Kernel.is_struct(term, Regex)`) (v1.11) +[v1.15] | `Logger.warn/2` | `Logger.warning/2` (v1.11) +[v1.14] | `use Bitwise` | `import Bitwise` (v1.0) +[v1.14] | `~~~/1` | `bnot/2` (v1.0) +[v1.14] | `Application.get_env/3` and similar in module body | `Application.compile_env/3` (v1.10) +[v1.14] | Compiled patterns in `String.starts_with?/2` | Pass a list of strings instead (v1.0) +[v1.14] | `Mix.Tasks.Xref.calls/1` | Compilation tracers (outlined in `Code`) (v1.10) +[v1.14] | `$levelpad` in Logger | *None* +[v1.14] | `<\|>` as a custom operator | Another custom operator (v1.0) +[v1.13] | `!` and `!=` in Version requirements | `~>` or `>=` (v1.0) +[v1.13] | `Mix.Config` | `Config` (v1.9) +[v1.13] | `:strip_beam` config to `mix escript.build` | `:strip_beams` (v1.9) +[v1.13] | `Macro.to_string/2` | `Macro.to_string/1` (v1.0) +[v1.13] | `System.get_pid/0` | `System.pid/0` (v1.9) +[v1.12] | `^^^/2` | `bxor/2` (v1.0) +[v1.12] | `@foo()` to read module attributes | Remove the parenthesis (v1.0) +[v1.12] | `use EEx.Engine` | Explicitly delegate to EEx.Engine instead (v1.0) +[v1.12] | `:xref` compiler in Mix | Nothing (it always runs as part of the compiler now) +[v1.11] | `Mix.Project.compile/2` | `Mix.Task.run("compile", args)` (v1.0) +[v1.11] | `Supervisor.Spec.worker/3` and `Supervisor.Spec.supervisor/3` | The new child specs outlined in `Supervisor` (v1.5) +[v1.11] | `Supervisor.start_child/2` and `Supervisor.terminate_child/2` | `DynamicSupervisor` (v1.6) +[v1.11] | `System.stacktrace/1` | `__STACKTRACE__` in `try/catch/rescue` (v1.7) +[v1.10] | `Code.ensure_compiled?/1` | `Code.ensure_compiled/1` (v1.0) +[v1.10] | `Code.load_file/2` | `Code.require_file/2` (v1.0) or `Code.compile_file/2` (v1.7) +[v1.10] | `Code.loaded_files/0` | `Code.required_files/0` (v1.7) +[v1.10] | `Code.unload_file/1` | `Code.unrequire_files/1` (v1.7) +[v1.10] | Passing non-chardata to `Logger.log/2` | Explicitly convert to string with `to_string/1` (v1.0) +[v1.10] | `:compile_time_purge_level` in `Logger` app environment | `:compile_time_purge_matching` in `Logger` app environment (v1.7) +[v1.10] | `Supervisor.Spec.supervise/2` | The new child specs outlined in `Supervisor` (v1.5) +[v1.10] | `:simple_one_for_one` strategy in `Supervisor` | `DynamicSupervisor` (v1.6) +[v1.10] | `:restart` and `:shutdown` in `Task.Supervisor.start_link/1` | `:restart` and `:shutdown` in `Task.Supervisor.start_child/3` (v1.6) +[v1.9] | Enumerable keys in `Map.drop/2`, `Map.split/2`, and `Map.take/2` | Call `Enum.to_list/1` on the second argument before hand (v1.0) +[v1.9] | `Mix.Project.load_paths/1` | `Mix.Project.compile_path/1` (v1.0) +[v1.9] | Passing `:insert_replaced` to `String.replace/4` | Use `:binary.replace/4` (v1.0) +[v1.8] | Passing a non-empty list to `Collectable.into/1` | `++/2` or `Keyword.merge/2` (v1.0) +[v1.8] | Passing a non-empty list to `:into` in `for/1` | `++/2` or `Keyword.merge/2` (v1.0) +[v1.8] | Passing a non-empty list to `Enum.into/2` | `++/2` or `Keyword.merge/2` (v1.0) +[v1.8] | Time units in its plural form, such as: `:seconds`, `:milliseconds`, and the like | Use the singular form, such as: `:second`, `:millisecond`, and so on (v1.4) +[v1.8] | `Inspect.Algebra.surround/3` | `Inspect.Algebra.concat/2` and `Inspect.Algebra.nest/2` (v1.0) +[v1.8] | `Inspect.Algebra.surround_many/6` | `Inspect.Algebra.container_doc/6` (v1.6) +[v1.9] | `--detached` in `Kernel.CLI` | `--erl "-detached"` (v1.0) +[v1.8] | `Kernel.ParallelCompiler.files/2` | `Kernel.ParallelCompiler.compile/2` (v1.6) +[v1.8] | `Kernel.ParallelCompiler.files_to_path/2` | `Kernel.ParallelCompiler.compile_to_path/2` (v1.6) +[v1.8] | `Kernel.ParallelRequire.files/2` | `Kernel.ParallelCompiler.require/2` (v1.6) +[v1.8] | Returning `{:ok, contents}` or `:error` from `Mix.Compilers.Erlang.compile/6`'s callback | Return `{:ok, contents, warnings}` or `{:error, errors, warnings}` (v1.6) +[v1.8] | `System.cwd/0` and `System.cwd!/0` | `File.cwd/0` and `File.cwd!/0` respectively (v1.0) +[v1.7] | `Code.get_docs/2` | `Code.fetch_docs/1` (v1.7) +[v1.7] | `Enum.chunk/2,3,4` | `Enum.chunk_every/2` and [`Enum.chunk_every/3,4`](`Enum.chunk_every/4`) (v1.5) +[v1.7] | Calling `super/1` in`GenServer` callbacks | Implementing the behaviour explicitly without calling `super/1` (v1.0) +[v1.7] | [`not left in right`](`in/2`) | [`left not in right`](`in/2`) (v1.5) +[v1.7] | `Registry.start_link/3` | `Registry.start_link/1` (v1.5) +[v1.7] | `Stream.chunk/2,3,4` | `Stream.chunk_every/2` and [`Stream.chunk_every/3,4`](`Stream.chunk_every/4`) (v1.5) +[v1.6] | `Enum.partition/2` | `Enum.split_with/2` (v1.4) +[v1.6] | `Macro.unescape_tokens/1,2` | Use `Enum.map/2` to traverse over the arguments (v1.0) +[v1.6] | `Module.add_doc/6` | [`@doc`](`Module`) module attribute (v1.0) +[v1.6] | `Range.range?/1` | Pattern match on [`_.._`](`../2`) (v1.0) +[v1.5] | `()` to mean `nil` | `nil` (v1.0) +[v1.5] | `char_list/0` type | `t:charlist/0` type (v1.3) +[v1.5] | `Atom.to_char_list/1` | `Atom.to_charlist/1` (v1.3) +[v1.5] | `Enum.filter_map/3` | `Enum.filter/2` + `Enum.map/2` or `for/1` comprehensions (v1.0) +[v1.5] | `Float.to_char_list/1` | `Float.to_charlist/1` (v1.3) +[v1.5] | `GenEvent` module | `Supervisor` and `GenServer` (v1.0) +[v1.5] | `<%=` in middle and end expressions in `EEx` | Use `<%` (`<%=` is allowed only in start expressions) (v1.0) +[v1.5] | `:as_char_lists` value in `t:Inspect.Opts.t/0` type | `:as_charlists` value (v1.3) +[v1.5] | `:char_lists` key in `t:Inspect.Opts.t/0` type | `:charlists` key (v1.3) +[v1.5] | `Integer.to_char_list/1,2` | `Integer.to_charlist/1` and `Integer.to_charlist/2` (v1.3) +[v1.5] | `to_char_list/1` | `to_charlist/1` (v1.3) +[v1.5] | `List.Chars.to_char_list/1` | `List.Chars.to_charlist/1` (v1.3) +[v1.5] | `@compile {:parse_transform, _}` in `Module` | *None* +[v1.5] | `Stream.filter_map/3` | `Stream.filter/2` + `Stream.map/2` (v1.0) +[v1.5] | `String.ljust/3` and `String.rjust/3` | Use `String.pad_leading/3` and `String.pad_trailing/3` with a binary padding (v1.3) +[v1.5] | `String.lstrip/1` and `String.rstrip/1` | `String.trim_leading/1` and `String.trim_trailing/1` (v1.3) +[v1.5] | `String.lstrip/2` and `String.rstrip/2` | Use `String.trim_leading/2` and `String.trim_trailing/2` with a binary as second argument (v1.3) +[v1.5] | `String.strip/1` and `String.strip/2` | `String.trim/1` and `String.trim/2` (v1.3) +[v1.5] | `String.to_char_list/1` | `String.to_charlist/1` (v1.3) +[v1.4] | Anonymous functions with no expression after `->` | Use an expression or explicitly return `nil` (v1.0) +[v1.4] | Support for making [private functions](`defp/2`) overridable | Use [public functions](`def/2`) (v1.0) +[v1.4] | Variable used as function call | Use parentheses (v1.0) +[v1.4] | `Access.key/1` | `Access.key/2` (v1.3) +[v1.4] | `Behaviour` module | `@callback` module attribute (v1.0) +[v1.4] | `Enum.uniq/2` | `Enum.uniq_by/2` (v1.2) +[v1.4] | `Float.to_char_list/2` | `:erlang.float_to_list/2` (Erlang/OTP 17) +[v1.4] | `Float.to_string/2` | `:erlang.float_to_binary/2` (Erlang/OTP 17) +[v1.4] | `HashDict` module | `Map` (v1.2) +[v1.4] | `HashSet` module | `MapSet` (v1.1) +[v1.4] | `IEx.Helpers.import_file/2` | `IEx.Helpers.import_file_if_available/1` (v1.3) +[v1.4] | `Mix.Utils.camelize/1` | `Macro.camelize/1` (v1.2) +[v1.4] | `Mix.Utils.underscore/1` | `Macro.underscore/1` (v1.2) +[v1.4] | Multi-letter aliases in `OptionParser` | Use single-letter aliases (v1.0) +[v1.4] | `Set` module | `MapSet` (v1.1) +[v1.4] | `Stream.uniq/2` | `Stream.uniq_by/2` (v1.2) +[v1.3] | `\x{X*}` inside strings/sigils/charlists | `\uXXXX` or `\u{X*}` (v1.1) +[v1.3] | `Dict` module | `Keyword` (v1.0) or `Map` (v1.2) +[v1.3] | `:append_first` option in `defdelegate/2` | Define the function explicitly (v1.0) +[v1.3] | Map/dictionary as 2nd argument in `Enum.group_by/3` | `Enum.reduce/3` (v1.0) +[v1.3] | `Keyword.size/1` | `length/1` (v1.0) +[v1.3] | `Map.size/1` | `map_size/1` (v1.0) +[v1.3] | `/r` option in `Regex` | `/U` (v1.1) +[v1.3] | `Set` behaviour | `MapSet` data structure (v1.1) +[v1.3] | `String.valid_character?/1` | `String.valid?/1` (v1.0) +[v1.3] | `Task.find/2` | Use direct message matching (v1.0) +[v1.3] | Non-map as 2nd argument in `URI.decode_query/2` | Use a map (v1.0) +[v1.2] | `Dict` behaviour | `Map` and `Keyword` (v1.0) +[v1.1] | `?\xHEX` | `0xHEX` (v1.0) +[v1.1] | `Access` protocol | `Access` behaviour (v1.1) +[v1.1] | `as: true \| false` in `alias/2` and `require/2` | *None* + +[v1.1]: https://github.com/elixir-lang/elixir/blob/v1.1/CHANGELOG.md#4-deprecations +[v1.2]: https://github.com/elixir-lang/elixir/blob/v1.2/CHANGELOG.md#changelog-for-elixir-v12 +[v1.3]: https://github.com/elixir-lang/elixir/blob/v1.3/CHANGELOG.md#4-deprecations +[v1.4]: https://github.com/elixir-lang/elixir/blob/v1.4/CHANGELOG.md#4-deprecations +[v1.5]: https://github.com/elixir-lang/elixir/blob/v1.5/CHANGELOG.md#4-deprecations +[v1.6]: https://github.com/elixir-lang/elixir/blob/v1.6/CHANGELOG.md#4-deprecations +[v1.7]: https://github.com/elixir-lang/elixir/blob/v1.7/CHANGELOG.md#4-hard-deprecations +[v1.8]: https://github.com/elixir-lang/elixir/blob/v1.8/CHANGELOG.md#4-hard-deprecations +[v1.9]: https://github.com/elixir-lang/elixir/blob/v1.9/CHANGELOG.md#4-hard-deprecations +[v1.10]: https://github.com/elixir-lang/elixir/blob/v1.10/CHANGELOG.md#4-hard-deprecations +[v1.11]: https://github.com/elixir-lang/elixir/blob/v1.11/CHANGELOG.md#4-hard-deprecations +[v1.12]: https://github.com/elixir-lang/elixir/blob/v1.12/CHANGELOG.md#4-hard-deprecations +[v1.13]: https://github.com/elixir-lang/elixir/blob/v1.13/CHANGELOG.md#4-hard-deprecations +[v1.14]: https://github.com/elixir-lang/elixir/blob/v1.14/CHANGELOG.md#4-hard-deprecations +[v1.15]: https://github.com/elixir-lang/elixir/blob/v1.15/CHANGELOG.md#4-hard-deprecations +[v1.16]: https://github.com/elixir-lang/elixir/blob/v1.16/CHANGELOG.md#4-hard-deprecations +[v1.17]: https://github.com/elixir-lang/elixir/blob/v1.17/CHANGELOG.md#4-hard-deprecations +[v1.18]: https://github.com/elixir-lang/elixir/blob/v1.18/CHANGELOG.md#4-hard-deprecations +[v1.19]: https://github.com/elixir-lang/elixir/blob/v1.19/CHANGELOG.md#4-hard-deprecations +[v1.20]: https://github.com/elixir-lang/elixir/blob/main/CHANGELOG.md#4-hard-deprecations diff --git a/lib/elixir/pages/references/gradual-set-theoretic-types.md b/lib/elixir/pages/references/gradual-set-theoretic-types.md new file mode 100644 index 00000000000..812d3b72352 --- /dev/null +++ b/lib/elixir/pages/references/gradual-set-theoretic-types.md @@ -0,0 +1,221 @@ + + +# Gradual set-theoretic types + +Elixir is in the process of incorporating set-theoretic types into the compiler. This document outlines the current stage of our implementation for this Elixir version. Elixir's type system is: + + * **sound** - the inferred and assigned by the type system align with the behaviour of the program + + * **gradual** - Elixir's type system includes the `dynamic()` type, which can be used when the type of a variable or expression is checked at runtime. In the absence of `dynamic()`, Elixir's type system behaves as a static one + + * **developer friendly** - the types are described, implemented, and composed using basic set operations: unions, intersections, and negation (hence it is a set-theoretic type system) + +The current milestone aims to infer types from existing programs and use them for type checking, enabling the Elixir compiler to find faults and bugs in codebases without requiring changes to existing software. User provided type signatures are planned for future releases. The underlying principles, theory, and roadmap of our work have been outlined in ["The Design Principles of the Elixir Type System" by Giuseppe Castagna, Guillaume Duboc, José Valim](https://arxiv.org/abs/2306.06391). + +## A gentle introduction + +Types in Elixir are written using the type named followed by parentheses, such as `integer()` or `list(integer())`. + +The basic types are: + +```elixir +atom() +binary() +empty_list() +integer() +float() +function() +map() +non_empty_list(elem_type, tail_type) +pid() +port() +reference() +tuple() +``` + +Many of the types above can also be written more precisely. We will discuss their syntax in the next sections, but here are two examples: + + * While `atom()` represents all atoms, the atom `:ok` can also be represented in the type system as `:ok` + + * While `tuple()` represents all tuples, you can specify the type of a two-element tuple where the first element is the atom `:ok` and the second is an integer as `{:ok, integer()}` + +There are also three special types: `none()` (represents an empty set), `term()` (represents all types), `dynamic()` (represents a range of the given types). + +Given the types are set-theoretic, we can compose them using unions (`or`), intersections (`and`), and negations (`not`). For example, to say a function returns either atoms or integers, one could write: `atom() or integer()`. + +Intersections will find the elements in common between the operands. For example, `atom() and integer()`, which in this case is the empty set `none()`. You can combine intersections and negations to perform difference, for example, to say that a function expects all atoms, except `nil` (which is an atom), you could write: `atom() and not nil`. + +You can find a complete reference in the [set-theoretic types cheatsheet](../cheatsheets/types-cheat.cheatmd). + +## The syntax of data types + +In this section we will cover the syntax of all data types. At the moment, developers will interact with those types mostly through compiler warnings and diagnostics. + +### Indivisible types + +These types are indivisibile and have no further representation. They are: `binary()`, `integer()`, `float()`, `pid()`, `port()`, `reference()`. For example, the numbers `1` and `42` are both represented by the type `integer()`. + +### Atoms + +You can represent all atoms as `atom()`. You can also represent each individual atom using their literal syntax. For instance, the atom `:foo` and `:hello_world` are also valid (distinct) types. + +`nil`, `true`, and `false` are also atoms and can be written as is. `boolean()` is a convenience type alias that represents `true or false`. + +### Tuples + +You can represent all tuples as `tuple()`. Tuples may also be written using the curly brackets syntax, such as `{:ok, binary()}`. + +You may use `...` at the end of the tuple to imply the overall size of the tuple is unknown. For example, the following tuple has at least two elements: `{:ok, binary(), ...}`. + +### Lists + +You can represent all _proper_ lists as `list()`, which also includes the empty list. + +You can also specify the type of the list element as argument. For example, `list(integer())` represents the values `[]` and `[1, 2, 3]`, but not `[1, "two", 3]`. + +Internally, Elixir represents the type `list(a)` as the union two distinct types, `empty_list()` and `not_empty_list(a)`. In other words, `list(integer())` is equivalent to `empty_list() or non_empty_list(integer())`. + +#### Improper lists + +While most developers will simply use `list(a)`, the type system can express all different representations of lists in Elixirby passing a second argument to `non_empty_list`, which represents the type of the tail. + +A proper list is one where the tail is the empty list itself. The type `non_empty_list(integer())` is equivalent to `non_empty_list(integer(), empty_list())`. + +If the `tail_type` is anything but a list, then we have an improper list. For example, the value `[1, 2 | 3]` would have the type `non_empty_list(integer(), integer())`. + +If you pass a list type as the tail, then the list type is merged into the element type. For example, `non_empty_list(integer(), list(binary()))` is the same as `non_empty_list(integer() or binary(), empty_list())`. + +### Maps + +You can represent all maps as `map()`. + +Maps may also be written using their literal syntax, such as `%{name: binary(), age: integer()}`, which outlines a map with exactly two keys, `:name` and `:age`, and values of type `binary()` and `integer()` respectively. + +A key may be marked as optional using the `if_set/1` operation on its value type. For example, `%{name: binary(), age: if_set(integer())}` is a map that certainly has the `:name` key but it may have the `:age` key (and if it has such key, its value type is `integer()`). + +We say the maps above are "closed": they only support the keys explicitly defined. We can also mark a map as "open", by including `...` as its first element. + +For example, the type `%{..., name: binary(), age: integer()}` means the keys `:name` and `:age` must exist, with their respective types, but any other key may also be present. In other words, `map()` is the same as `%{...}`. For the empty map, you may write `%{}`, although we recommend using `empty_map()` for clarity. + +#### Domain types + +In the examples above, all map keys were atoms, but we can also use other types as map keys. For example: + +```elixir +# Closed map +%{binary() or atom() => integer()} + +# Open map +%{..., binary() or atom() => integer()} +``` + +Currently, the type system only tracks the top of each individial type as the domain keys. For example, if you say: + +```elixir +%{list(integer()) => integer(), list(binary()) => binary()} +``` + +That's the same as specifying all lists: + +```elixir +%{list() => integer() or binary()} +``` + +The supported domain keys are `atom()`, `binary()`, `integer()`, `float()`, `fun()`, `list()`, `map()`, `pid()`, `port()`, `reference()`, and `tuple()`. + +Furthermore, it is important to note that domain keys are, by definition, optional. Whenever you have a `%{integer() => integer()}`and you try to fetch a key, we must assume the key may not exist (after all, it is not possible to store all integers as map keys as they are infinite). + +#### Mixed keys + +It is also possible to mix domain and atom keys. For example, the following map says that all atom keys are of type `binary()`, except the `:root` key, which has type `integer()`: + +```elixir +# Closed map +%{atom() => binary(), root: integer()} + +# Open map +%{..., atom() => binary(), root: integer()} +``` + +The order of the keys is of increasing precision. `:root` is more precise than `atom()`, therefore it comes later. This mirrors the runtime semantics of maps, where duplicate keys override the value of earlier ones. + +### Functions + +You can represent all functions as `function()`. However, in practice, most functions are represented as arrows. For example, a function that receives an integer and return boolean would be written as `(integer() -> boolean())`. A function that receives two integers and return a string (i.e. a binary) would be written as `(integer(), integer() -> binary())`. + +When representing functions with multiple clauses, which may take different input types, we use intersections. For example, imagine the following function: + +```elixir +def negate(x) when is_integer(x), do: -x +def negate(x) when is_boolean(x), do: not x +``` + +If you give it an integer, it negates it. If you give it a boolean, it negates it. + +We can say this function has the type `(integer() -> integer())` because it is capable of receiving an integer and returning an integer. In this case, `(integer() -> integer())` is a set that represents all functions that can receive an integer and return an integer. Even though this function can receive other arguments and return other values, it is still part of the `(integer() -> integer())` set. + +This function also has the type `(boolean() -> boolean())`, because it also receives booleans and returns booleans. If you pass the function above to another function that expects `(boolean() -> boolean())`, type checking will succeed. Therefore, we can say the overall type of the function is `(integer() -> integer()) and (boolean() -> boolean())`. The intersection means the function belongs to both sets. + +At this point, you may ask, why not a union? As a real-world example, take a t-shirt with green and yellow stripes. We can say the t-shirt belongs to the set of "t-shirts with green color". We can also say the t-shirt belongs to the set of "t-shirts with yellow color". Let's see the difference between unions and intersections: + + * `(t_shirts_with_green() or t_shirts_with_yellow())` - contains t-shirts with either green or yellow, such as green, green and red, green and yellow, but also only yellow, yellow and red, etc. + + * `(t_shirts_with_green() and t_shirts_with_yellow())` - contains t-shirts with both green and yellow (and maybe other colors) + +Since the t-shirt has both colors, we could say it belongs to the union of green and yellow t-shirts, but doing so would not capture the fact it is both green and yellow. Therefore it is more precise to say it belongs to the intersection of both sets. The same way that a function that goes from `(integer() -> integer())` and `(boolean() -> boolean())` is also an intersection. In practice, it is not useful to define the union of two functions in Elixir, so the compiler will point you to the right direction if you specify the wrong one. + +## The `dynamic()` type + +Existing Elixir programs do not have type declarations, but we still want to be able to type check them. This is done with the introduction of the `dynamic()` type. + +When Elixir sees the following function: + +```elixir +def negate(x) when is_integer(x), do: -x +def negate(x) when is_boolean(x), do: not x +``` + +Elixir type checks it as if the function had the type `(dynamic() -> dynamic())`. Then, based on patterns and guards, we can refine the value of the variable `x` to be `dynamic() and integer()` and `dynamic() and boolean()` for each clause respectively. We say `dynamic()` is a gradual type, which leads us to _gradual set-theoretic types_. + +The simplest way to reason about `dynamic()` in Elixir is that it is a range of types. If you have a type `atom() or integer()`, the underlying code needs to work with both `atom() or integer()`. For example, if you call `Integer.to_string(var)`, and `var` has type `atom() or integer()`, the type system will emit a warning, because `Integer.to_string/1` does not accept atoms. + +However, by intersecting a type with `dynamic()`, we make the type gradual and therefore only a subset of the type needs to be valid. For instance, if you call `Integer.to_string(var)`, and `var` has type `dynamic() and (atom() or integer())`, the type system will not emit a warning, because `Integer.to_string/1` works with at least one of the types. For convenience, most programs will write `dynamic(atom() or integer())` instead of the intersection. They are equivalent. + +Compared to other gradually typed languages, the `dynamic()` type in Elixir is quite powerful: it restricts our program to certain types, via intersections, while still emitting warnings once it is certain the code will fail. This makes `dynamic()` an excellent tool for typing existing Elixir code with meaningful warnings. + +If the user provides their own types, and those types are not `dynamic()`, then Elixir's type system behaves as a statically typed one. This brings us to one last property of dynamic types in Elixir: dynamic types are always at the root. For example, when you write a tuple of type `{:ok, dynamic()}`, Elixir will rewrite it to `dynamic({:ok, term()})`. While this has the downside that you cannot make part of a tuple/map/list gradual, only the whole tuple/map/list, it comes with the upside that dynamic is always explicitly at the root, making it harder to accidentally sneak `dynamic()` in a statically typed program. + +## Type inference + +Type inference (or reconstruction) is the ability of a type system automatically deduce, either partially or fully, the type of an expression at compile time. Type inference may occur at different levels. For example, many programming languages can automatically infer the types of variables, also known "local type inference", but not all can infer type signatures of functions. + +Inferring type signatures comes with a series of trade-offs: + + * Speed - type inference algorithms are often more computationally intensive than type checking algorithms. + + * Expressiveness - in any given type system, the constructs that support inference are always a subset of those that can be type-checked. Therefore, if a programming language is restricted to fully reconstructed types, it is less expressive than a solely type checked counterpart. + + * Incremental compilation - type inference complicates incremental compilation. If module A depends on module B, which depends on module C, a change to C may require the type signature in B to be reconstructed, which may then require A to be recomputed (and so on). This dependency chain may require large projects to explicitly add type signatures for stability and compilation efficiency. + + * Cascading errors - when a user accidentally makes type errors or the code has conflicting assumptions, type inference may lead to less clear error messages as the type system tries to reconcile diverging type assumptions across code paths. + +On the other hand, type inference offers the benefit of enabling type checking for functions and codebases without requiring the user to add type annotations. To balance these trade-offs, Elixir aims to provide "module type inference": our goal is to infer the types of functions considering the current module, Elixir's standard library and your dependencies (in the future). Calls to modules within the same project are assumed to be `dynamic()` as to reduce cyclic dependencies and the need for recompilations. Once types are inferred, then the whole project is type checked considering all modules and all types (inferred or otherwise). + +Type inference in Elixir is best-effort: it doesn't guarantee it will find all possible type incompatibilities, only that it may find bugs where all combinations of a type _will_ fail, even in the absence of explicit type annotations. It is meant to be an efficient routine that brings developers some benefits of static typing without requiring any effort from them. + +In the long term, Elixir developers who want typing guarantees must explicitly add type signatures to their functions (see "Roadmap"). Any function with an explicit type signature will be typed checked against the user-provided annotations, as in other statically typed languages, without performing type inference. In summary, type checking will rely on type signatures and only fallback to inferred types when no signature is available. + +## Roadmap + +The current milestone is to implement type inference of existing codebases, as well as type checking of all language constructs, without changes to the Elixir language. At this stage, we want to collect feedback on the quality of error messages and performance, and therefore the type system has no user facing API. Full type inference of patterns was released in Elixir v1.18, and complete inference is expected as part of Elixir v1.20. + +If the results are satisfactory, the next milestone will include a mechanism for defining typed structs. Elixir programs frequently pattern match on structs, which reveals information about the struct fields, but it knows nothing about their respective types. By propagating types from structs and their fields throughout the program, we will increase the type system’s ability to find errors while further straining our type system implementation. Proposals including the required changes to the language surface will be sent to the community once we reach this stage. + +The third milestone is to introduce set-theoretic type signatures for functions. Unfortunately, the existing Erlang Typespecs are not precise enough for set-theoretic types and they will be phased out of the language and have their postprocessing moved into a separate library once this stage concludes. + +## Acknowledgements + +The type system was made possible thanks to a partnership between [CNRS](https://www.cnrs.fr/) and [Remote](https://remote.com/). The development work is currently sponsored by [Fresha](https://www.fresha.com/), and [Tidewave](https://tidewave.ai/). diff --git a/lib/elixir/pages/references/library-guidelines.md b/lib/elixir/pages/references/library-guidelines.md new file mode 100644 index 00000000000..8fdc4457f09 --- /dev/null +++ b/lib/elixir/pages/references/library-guidelines.md @@ -0,0 +1,57 @@ + + +# Library guidelines + +This document outlines general guidelines for those writing and publishing +Elixir libraries meant to be consumed by other developers. + +## Getting started + +You can create a new Elixir library by running the `mix new` command: + + $ mix new my_library + +The project name is given in the `snake_case` convention where all letters are lowercase and words are separate with underscores. This is the same convention used by variables, function names and atoms in Elixir. See the [Naming Conventions](naming-conventions.md) document for more information. + +Every project has a `mix.exs` file, with instructions on how to build, compile, run tests, and so on. Libraries commonly have a `lib` directory, which includes Elixir source code, and a `test` directory. A `src` directory may also exist for Erlang sources. + +The `mix new` command also allows the `--sup` option to scaffold a new project with a supervision tree out of the box. For more information on running your project, see the official [Mix & OTP guide](../mix-and-otp/introduction-to-mix.md) or [Mix documentation](`Mix`). + +## Publishing + +Writing code is only the first of many steps to publish a package. We strongly recommend developers to: + + * Choose a versioning schema. Elixir requires versions to be in the format `MAJOR.MINOR.PATCH` but the meaning of those numbers is up to you. Most projects choose [Semantic Versioning](https://semver.org/). + + * Choose a [license](https://choosealicense.com/). The most common licenses in the Elixir community are the [MIT License](https://choosealicense.com/licenses/mit/) and the [Apache License 2.0](https://choosealicense.com/licenses/apache-2.0/). The latter is also the one used by Elixir itself. + + * Run the [code formatter](`mix format`). The code formatter formats your code according to a consistent style shared by your library and the whole community, making it easier for other developers to understand your code and contribute. + + * Write tests. Elixir ships with a test-framework named [ExUnit](`ExUnit`). The project generated by `mix new` includes sample tests and doctests. + + * Write documentation. The Elixir community is proud of treating documentation as a first-class citizen and making documentation easily accessible. Libraries contribute to the status quo by providing complete API documentation with examples for their modules, types and functions. See the [Writing documentation](../getting-started/writing-documentation.md) chapter of the Getting Started guide for more information. Projects like [ExDoc](https://github.com/elixir-lang/ex_doc) can be used to generate HTML and EPUB documents from the documentation. ExDoc also supports "extra pages", like this one that you are reading. Such pages augment the documentation with tutorials, guides, references, and even cheat-sheets. + + * Follow best practices. The Elixir project documents [a series of anti-patterns](../anti-patterns/what-anti-patterns.md) that you may want to avoid in your code. The [process-related anti-patterns](../anti-patterns/process-anti-patterns.md) and [meta-programming anti-patterns](../anti-patterns/macro-anti-patterns.md) are of special attention to library authors. + +Projects are often made available to other developers [by publishing a Hex package](https://hex.pm/docs/publish). Hex also [supports private packages for organizations](https://hex.pm/pricing). If ExDoc is configured for the Mix project, publishing a package on Hex will also automatically publish the generated documentation to [HexDocs](https://hexdocs.pm). + +## Dependency handling + +When your library is used as a dependency, it runs by default in the `:prod` environment. Therefore, if your library has dependencies that are only useful in development or testing, you want to specify those dependencies with the `:only` option. You can also specify `:optional` dependencies in your library, which are not enforced upon users of your library. In such cases, you should also consider compiling your projects with the `mix compile --no-optional-deps --warnings-as-errors` in your test environments, to ensure your library compiles without warnings even if optional dependencies are missing. See `mix deps` for all available options. + +Keep in mind your library's [lockfile](`Mix.Project#module-configuration`) (usually named `mix.lock`) is _ignored by the host project_. Running `mix deps.get` in the host project attempts to get the latest possible versions of your library’s dependencies, as specified by the requirements in the `deps` section of your `mix.exs`. These versions might be greater than those stored in your `mix.lock` (and hence used in your tests / CI). + +On the other hand, contributors of your library, need a deterministic build, which implies the presence of `mix.lock` in your Version Control System (VCS), such as `git`. + +If you want to validate both scenarios, you should check the `mix.lock` into version control and run two different Continuous Integration (CI) workflows: one that relies on the `mix.lock` for deterministic builds, and another one, that starts with `mix deps.unlock --all` and always compiles your library and runs tests against latest versions of dependencies. The latter one might be even run nightly or otherwise recurrently to stay notified about any possible issue in regard to dependencies updates. + +### Dependency Version Requirements + +When depending on other libraries, the dependency version requirements are ultimately up to you. However, you should consider the effects that an overly strict dependency requirement can have on users of your library. Most dependencies adopt [Semantic Versioning](https://semver.org/), and therefore provide reasonable guarantees about what each release contains. For instance, if you use `{:some_dep, “== 0.2.3”}`, this prevents users from using any other version but the one that you specified, which means that they cannot receive bug fix upgrades to that package. When in doubt, use a dependency in the format of `"~> x.y"`. This prevents the user from using a higher major version of the library, but allows them to upgrade to newer minor and patch versions, which should only include bug fixes and non-breaking improvements. + +The exception to this is pre 1.0 libraries using [Semantic Versioning](https://semver.org/), which provide [no guarantees](https://semver.org/spec/v2.0.0.html#spec-item-4) about what might change from one version to the next. In this scenario, depending on the full patch version, i.e `"~> 0.1.2"` is a better default. + +A common mistake is to use a dependency in the format of `"~> x.y.z"` to express "a version greater than `x.y.z`". For example, if you are depending on `"~> 1.2"`, and the dependency publishes a fix in version `1.2.1` that you need for the next version of your library. If you use `"~> 1.2.1"` to express that dependency, you are preventing users from upgrading to `"1.3.0"` or higher! Instead of `"~> 1.2.1"`, you should use `"~> 1.2 and >= 1.2.1"` as the version requirement. This allows users to use any version less than `2.0`, and greater than `1.2.1`. diff --git a/lib/elixir/pages/references/naming-conventions.md b/lib/elixir/pages/references/naming-conventions.md new file mode 100644 index 00000000000..aac42e7cc51 --- /dev/null +++ b/lib/elixir/pages/references/naming-conventions.md @@ -0,0 +1,147 @@ + + +# Naming conventions + +This document is a reference of the naming conventions in Elixir, from casing to punctuation characters. + +The naming convention is, by definition, a subset of the Elixir syntax. A convention aims to +follow and set best practices for language and the community. If instead you want a complete reference into the Elixir syntax, beyond its conventions, see [the Syntax reference](syntax-reference.md). + +## Casing + +Elixir developers must use `snake_case` when defining variables, function names, module attributes, and the like: + + some_map = %{this_is_a_key: "and a value"} + is_map(some_map) + +Aliases, commonly used as module names, are an exception as they must be capitalized and written in `CamelCase`, like `OptionParser`. For aliases, capital letters are kept in acronyms, like `ExUnit.CaptureIO` or `Mix.SCM`. + +Atoms can be written either in `:snake_case` or `:CamelCase`, although the convention is to use the snake case version throughout Elixir. + +Generally speaking, filenames follow the `snake_case` convention of the module they define. For example, `MyApp` should be defined inside the `my_app.ex` file. However, this is only a convention. At the end of the day any filename can be used as they do not affect the compiled code in any way. + +## Underscore (`_foo`) + +Elixir relies on underscores in different situations. + +For example, a value that is not meant to be used must be assigned to `_` or to a variable starting with underscore: + + iex> {:ok, _contents} = File.read("README.md") + +Function names may also start with an underscore. Such functions are never imported by default: + + iex> defmodule Example do + ...> def _wont_be_imported do + ...> :oops + ...> end + ...> end + + iex> import Example + iex> _wont_be_imported() + ** (CompileError) iex:1: undefined function _wont_be_imported/0 + +Due to this property, Elixir relies on functions starting with underscore to attach compile-time metadata to modules. Such functions are most often in the `__foo__` format. For example, every module in Elixir has an [`__info__/1`](`c:Module.__info__/1`) function: + + iex> String.__info__(:functions) + [at: 2, capitalize: 1, chunk: 2, ...] + +Elixir also includes five special forms that follow the double underscore format: `__CALLER__/0`, `__DIR__/0`, `__ENV__/0`and `__MODULE__/0` retrieve compile-time information about the current environment, while `__STACKTRACE__/0` retrieves the stacktrace for the current exception. + +## Trailing bang (`foo!`) + +A trailing bang (exclamation mark) signifies a function or macro where failure cases raise an exception. They most often exist as a "raising variant" of a function that returns `:ok`/`:error` tuples (or `nil`). + +One example is `File.read/1` and `File.read!/1`. `File.read/1` will return a success or failure tuple, whereas `File.read!/1` will return a plain value or else raise an exception: + + iex> File.read("file.txt") + {:ok, "file contents"} + iex> File.read("no_such_file.txt") + {:error, :enoent} + + iex> File.read!("file.txt") + "file contents" + iex> File.read!("no_such_file.txt") + ** (File.Error) could not read file no_such_file.txt: no such file or directory + +The version without `!` is preferred when you want to handle different outcomes using pattern matching: + + case File.read(file) do + {:ok, body} -> # do something with the `body` + {:error, reason} -> # handle the error caused by `reason` + end + +However, if you expect the outcome to always be successful (for instance, if you expect the file always to exist), the bang variation can be more convenient and will raise a more helpful error message (than a failed pattern match) on failure. + +When thinking about failure cases, we are often thinking about semantic errors related to the operation being performed, such as failing to open a file or trying to fetch key from a map. Errors that come from invalid argument types, or similar, must always raise regardless if the function has a bang or not. In such cases, the exception is often an `ArgumentError` or a detailed `FunctionClauseError`: + + iex(1)> File.read(123) + ** (FunctionClauseError) no function clause matching in IO.chardata_to_string/1 + + The following arguments were given to IO.chardata_to_string/1: + + # 1 + 123 + + Attempted function clauses (showing 2 out of 2): + + def chardata_to_string(string) when is_binary(string) + def chardata_to_string(list) when is_list(list) + +More examples of paired functions: `Base.decode16/2` and `Base.decode16!/2`, `File.cwd/0` and `File.cwd!/0`. In some situations, you may have bang functions without a non-bang counterpart. They also imply the possibility of errors, such as: `Protocol.assert_protocol!/1` and `PartitionSupervisor.resize!/2`. This can be useful if you foresee the possibility of adding a non-raising variant in the future. + +## Trailing question mark (`foo?`) + +Functions that return a boolean are named with a trailing question mark. + +Examples: `Keyword.keyword?/1`, `Mix.debug?/0`, `String.contains?/2` + +However, functions that return booleans and are valid in guards follow another convention, described next. + +## `is_` prefix (`is_foo`) + +Type checks and other boolean checks that are allowed in guard clauses are named with an `is_` prefix. + +Examples: `Integer.is_even/1`, `is_list/1` + +These functions and macros follow the Erlang convention of an `is_` prefix, instead of a trailing question mark, precisely to indicate that they are allowed in guard clauses. Type checks that are not valid in guard clauses do not follow this convention, such as `Keyword.keyword?/1`. + +A trailing question mark should not be used in combination with the `is_` prefix. + +## Special names + +Some names have specific meaning in Elixir. We detail those cases below. + +### length and size + +When you see `size` in a function name, it means the operation runs in constant time (also written as "O(1) time") because the size is stored alongside the data structure. + +Examples: `map_size/1`, `tuple_size/1` + +When you see `length`, the operation runs in linear time ("O(n) time") because the entire data structure has to be traversed. + +Examples: `length/1`, `String.length/1` + +In other words, functions using the word "size" in its name will take the same amount of time whether the data structure is tiny or huge. Conversely, functions having "length" in its name will take more time as the data structure grows in size. + +### get, fetch, fetch! + +When you see the functions `get`, `fetch`, and `fetch!` for key-value data structures, you can expect the following behaviours: + + * `get` returns a default value (which itself defaults to `nil`) if the key is not present, or returns the requested value. + * `fetch` returns `:error` if the key is not present, or returns `{:ok, value}` if it is. + * `fetch!` *raises* if the key is not present, or returns the requested value. + +Examples: `Map.get/2`, `Map.fetch/2`, `Map.fetch!/2`, `Keyword.get/2`, `Keyword.fetch/2`, `Keyword.fetch!/2` + +### compare + +The function `compare/2` should return `:lt` if the first term is less than the second, `:eq` if the two +terms compare as equivalent, or `:gt` if the first term is greater than the second. + +Examples: `DateTime.compare/2` + +Note that this specific convention is important due to the expectations of `Enum.sort/2` diff --git a/lib/elixir/pages/references/operators.md b/lib/elixir/pages/references/operators.md new file mode 100644 index 00000000000..99d61bf4e53 --- /dev/null +++ b/lib/elixir/pages/references/operators.md @@ -0,0 +1,156 @@ + + +# Operators reference + +This document is a complete reference of operators in Elixir, how they are parsed, how they can be defined, and how they can be overridden. + +## General operators + +Elixir provides the following built-in operators: + + * [`+`](`+/1`) and [`-`](`-/1`) - unary positive/negative + * [`+`](`+/2`), [`-`](`-/2`), [`*`](`*/2`), and [`/`](`//2`) - basic arithmetic operations + * [`++`](`++/2`) and [`--`](`--/2`) - list concatenation and subtraction + * [`and`](`and/2`) and [`&&`](`&&/2`) - strict and relaxed boolean "and" + * [`or`](`or/2`) and [`||`](`||/2`) - strict and relaxed boolean "or" + * [`not`](`not/1`) and [`!`](`!/1`) - strict and relaxed boolean "not" + * [`in`](`in/2`) and [`not in`](`in/2`) - membership + * [`@`](`@/1`) - module attribute + * [`..`](`../0`), [`..`](`../2`), and [`..//`](`..///3`) - range creation + * [`<>`](`<>/2`) - binary concatenation + * [`|>`](`|>/2`) - pipeline + * [`=~`](`=~/2`) - text-based match + +Many of those can be used in guards. Consult the [list of allowed guard functions and operators](patterns-and-guards.md#list-of-allowed-functions-and-operators). + +Additionally, there are a few other operators that Elixir parses but doesn't actually use. +See [Custom and overridden operators](#custom-and-overridden-operators) below for a list and for guidelines about their use. + +Some other operators are special forms and cannot be overridden: + + * [`^`](`^/1`) - pin operator + * [`.`](`./2`) - dot operator + * [`=`](`=/2`) - match operator + * [`&`](`&/1`) - capture operator + * [`::`](`::/2`) - type operator + +Finally, these operators appear in the precedence table below but are only meaningful within certain constructs: + + * `=>` - see [`%{}`](`%{}/1`) + * `when` - see [Guards](patterns-and-guards.md#guards) + * `<-` - see [`for`](`for/1`) and [`with`](`with/1`) + * `\\` - see [Default arguments](`Kernel#def/2-default-arguments`) + +## Comparison operators + +Elixir provides the following built-in comparison operators (all of which can be used in guards): + + * [`==`](`==/2`) - equal to + * [`===`](`===/2`) - strictly equal to + * [`!=`](`!=/2`) - not equal to + * [`!==`](`!==/2`) - strictly not equal to + * [`<`](``](`>/2`) - greater-than + * [`<=`](`<=/2`) - less-than or equal to + * [`>=`](`>=/2`) - greater-than or equal to + +The only difference between [`==`](`==/2`) and [`===`](`===/2`) is that [`===`](`===/2`) is strict when it comes to comparing integers and floats: + +```elixir +iex> 1 == 1.0 +true +iex> 1 === 1.0 +false +``` + +[`!=`](`!=/2`) and [`!==`](`!==/2`) act as the negation of [`==`](`==/2`) and [`===`](`===/2`), respectively. + +## Operator precedence and associativity + +The following is a list of all operators that Elixir is capable of parsing, ordered from higher to lower precedence, alongside their associativity: + +Operator | Associativity +---------------------------------------------- | ------------- +`@` | Unary +`.` | Left +`+` `-` `!` `^` `not` | Unary +`**` | Left +`*` `/` | Left +`+` `-` | Left +`++` `--` `+++` `---` `..` `<>` | Right +`//` (valid only inside `..//`) | Right +`in` `not in` | Left +`\|>` `<<<` `>>>` `<<~` `~>>` `<~` `~>` `<~>` | Left +`<` `>` `<=` `>=` | Left +`==` `!=` `=~` `===` `!==` | Left +`&&` `&&&` `and` | Left +`\|\|` `\|\|\|` `or` | Left +`=` | Right +`&`, `...` | Unary +`\|` | Right +`::` | Right +`when` | Right +`<-` `\\` | Left +`=>` (valid only inside `%{}`) | None + +Elixir also has two ternary operators: + +Operator | Associativity +---------------------------------------------- | ------------- +`first..last//step` | Right +`%{map \| key => value, ...}` | None + +> #### Deprecated operator precedence {: .info} +> +> Elixir parses `not left in right` as `not(left in right)` and `!left in right` as `!(left in right)`, which mismatches the precedence table above, but such behaviour is deprecated and emits a warning. Both constructs must be written as `left not in right` instead. In future major versions, the parser will match the table above. + +## Custom and overridden operators + +Elixir is capable of parsing a predefined set of operators. It's not possible to define new operators (as supported by some languages). However, not all operators that Elixir can parse are *used* by Elixir: for example, `+` and `||` are used by Elixir for addition and boolean *or*, but `<~>` is not used (but valid). + +To define an operator, you can use the usual `def*` constructs (`def`, `defp`, `defmacro`, and so on) but with a syntax similar to how the operator is used: + +```elixir +defmodule MyOperators do + # We define ~> to return the maximum of the given two numbers, + # and <~ to return the minimum. + + def a ~> b, do: max(a, b) + def a <~ b, do: min(a, b) +end +``` + +To use the newly defined operators, you **have to** import the module that defines them: + +```elixir +iex> import MyOperators +iex> 1 ~> 2 +2 +iex> 1 <~ 2 +1 +``` + +The following is a table of all the operators that Elixir is capable of parsing, but that are not used by default: + + * `|||` + * `&&&` + * `<<<` + * `>>>` + * `<<~` + * `~>>` + * `<~` + * `~>` + * `<~>` + * `+++` + * `---` + * `...` + +The following operators are used by the `Bitwise` module when imported: [`&&&`](`Bitwise.&&&/2`), [`<<<`](`Bitwise.<<>>`](`Bitwise.>>>/2`), and [`|||`](`Bitwise.|||/2`). See the `Bitwise` documentation for more information. + +Note that the Elixir community generally discourages custom operators. They can be hard to read and even more to understand, as they don't have a descriptive name like functions do. That said, some specific cases or custom domain specific languages (DSLs) may justify these practices. + +It is also possible to replace predefined operators, such as `+`, but doing so is extremely discouraged. diff --git a/lib/elixir/pages/references/patterns-and-guards.md b/lib/elixir/pages/references/patterns-and-guards.md new file mode 100644 index 00000000000..af9dc0866c7 --- /dev/null +++ b/lib/elixir/pages/references/patterns-and-guards.md @@ -0,0 +1,528 @@ + + +# Patterns and guards + +Elixir provides pattern matching, which allows us to assert on the shape or extract values from data structures. Patterns are often augmented with guards, which give developers the ability to perform more complex checks, albeit limited. + +This document provides a complete reference on patterns and guards, their semantics, where they are allowed, and how to extend them. + +## Patterns + +Patterns in Elixir are made of variables, literals, and data structure specific syntax. One of the most used constructs to perform pattern matching is the match operator ([`=`](`=/2`)): + +```iex +iex> x = 1 +1 +iex> 1 = x +1 +``` + +In the example above, `x` starts without a value and has `1` assigned to it. Then, we compare the value of `x` to the literal `1`, which succeeds as they are both `1`. + +Matching `x` against 2 would raise: + +```iex +iex> 2 = x +** (MatchError) no match of right hand side value: 1 +``` + +Patterns are not bidirectional. If you have a variable `y` that was never assigned to (often called an unbound variable) and you write `1 = y`, an error will be raised: + +```iex +iex> 1 = y +** (CompileError) iex:2: undefined variable "y" +``` + +In other words, patterns are allowed only on the left side of `=`. The right side of `=` follows the regular evaluation semantics of the language. + +Now let's cover the pattern matching rules for each construct and then for each relevant data types. + +### Variables + +Variables in patterns are always assigned to: + +```iex +iex> x = 1 +1 +iex> x = 2 +2 +iex> x +2 +``` + +In other words, Elixir supports rebinding. In case you don't want the value of a variable to change, you can use the pin operator (`^`): + +```iex +iex> x = 1 +1 +iex> ^x = 2 +** (MatchError) no match of right hand side value: 2 +``` + +If the same variable appears multiple times in the same pattern, then all of them must be bound to the same value: + +```iex +iex> {x, x} = {1, 1} +{1, 1} +iex> {x, x} = {1, 2} +** (MatchError) no match of right hand side value: {1, 2} +``` + +The underscore variable (`_`) has a special meaning as it can never be bound to any value. It is especially useful when you don't care about certain value in a pattern: + +```iex +iex> {_, integer} = {:not_important, 1} +{:not_important, 1} +iex> integer +1 +iex> _ +** (CompileError) iex:3: invalid use of _ +``` + +A pinned value represents the value itself and not its – even if syntactically equal – pattern. The right hand side is compared to be equal to the pinned value: + +```iex +iex> x = %{} +%{} +iex> {:ok, %{}} = {:ok, %{a: 13}} +{:ok, %{a: 13}} +iex> {:ok, ^x} = {:ok, %{a: 13}} +** (MatchError) no match of right hand side value: {:ok, %{a: 13}} + (stdlib 6.2) erl_eval.erl:667: :erl_eval.expr/6 + iex:2: (file) +``` + +### Literals (numbers and atoms) + +Atoms and numbers (integers and floats) can appear in patterns and they are always represented as is. For example, an atom will only match an atom if they are the same atom: + +```iex +iex> :atom = :atom +:atom +iex> :atom = :another_atom +** (MatchError) no match of right hand side value: :another_atom +``` + +Similar rule applies to numbers. Finally, note that numbers in patterns perform strict comparison. In other words, integers to do not match floats: + +```iex +iex> 1 = 1.0 +** (MatchError) no match of right hand side value: 1.0 +``` + +### Tuples + +Tuples may appear in patterns using the curly brackets syntax (`{}`). A tuple in a pattern will match only tuples of the same size, where each individual tuple element must also match: + +```iex +iex> {:ok, integer} = {:ok, 13} +{:ok, 13} + +# won't match due to different size +iex> {:ok, integer} = {:ok, 11, 13} +** (MatchError) no match of right hand side value: {:ok, 11, 13} + +# won't match due to mismatch on first element +iex> {:ok, binary} = {:error, :enoent} +** (MatchError) no match of right hand side value: {:error, :enoent} +``` + +### Lists + +Lists may appear in patterns using the square brackets syntax (`[]`). A list in a pattern will match only lists of the same size, where each individual list element must also match: + +```iex +iex> [:ok, integer] = [:ok, 13] +[:ok, 13] + +# won't match due to different size +iex> [:ok, integer] = [:ok, 11, 13] +** (MatchError) no match of right hand side value: [:ok, 11, 13] + +# won't match due to mismatch on first element +iex> [:ok, binary] = [:error, :enoent] +** (MatchError) no match of right hand side value: [:error, :enoent] +``` + +Opposite to tuples, lists also allow matching on non-empty lists by using the `[head | tail]` notation, which matches on the `head` and `tail` of a list: + +```iex +iex> [head | tail] = [1, 2, 3] +[1, 2, 3] +iex> head +1 +iex> tail +[2, 3] +``` + +Multiple elements may prefix the `| tail` construct: + +```iex +iex> [first, second | tail] = [1, 2, 3] +[1, 2, 3] +iex> tail +[3] +``` + +Note `[head | tail]` does not match empty lists: + +```elixir +iex> [head | tail] = [] +** (MatchError) no match of right hand side value: [] +``` + +Given charlists are represented as a list of integers, one can also perform prefix matches on charlists using the list concatenation operator ([`++`](`++/2`)): + +```elixir +iex> ~c"hello " ++ world = ~c"hello world" +~c"hello world" +iex> world +~c"world" +``` + +Which is equivalent to matching on `[?h, ?e, ?l, ?l, ?o, ?\s | world]`. Suffix matches (`hello ++ ~c" world"`) are not valid patterns. + +### Maps + +Maps may appear in patterns using the percentage sign followed by the curly brackets syntax (`%{}`). Opposite to lists and tuples, maps perform a subset match. This means a map pattern will match any other map that has at least all of the keys in the pattern. + +Here is an example where all keys match: + +```iex +iex> %{name: name} = %{name: "meg"} +%{name: "meg"} +iex> name +"meg" +``` + +Here is when a subset of the keys match: + +```iex +iex> %{name: name} = %{name: "meg", age: 23} +%{age: 23, name: "meg"} +iex> name +"meg" +``` + +If a key in the pattern is not available in the map, then they won't match: + +```iex +iex> %{name: name, age: age} = %{name: "meg"} +** (MatchError) no match of right hand side value: %{name: "meg"} +``` + +Note that the empty map will match all maps, which is a contrast to tuples and lists, where an empty tuple or an empty list will only match empty tuples and empty lists respectively: + +```iex +iex> %{} = %{name: "meg"} +%{name: "meg"} +``` + +Finally, note map keys in patterns must always be literals or previously bound variables matched with the pin operator. + +### Structs + +Structs may appear in patterns using the percentage sign, the struct module name or a variable followed by the curly brackets syntax (`%{}`). + +Given the following struct: + +```elixir +defmodule User do + defstruct [:name] +end +``` + +Here is an example where all keys match: + +```iex +iex> %User{name: name} = %User{name: "meg"} +%User{name: "meg"} +iex> name +"meg" +``` + +If an unknown key is given, the compiler will raise an error: + +```iex +iex> %User{type: type} = %User{name: "meg"} +** (CompileError) iex: unknown key :type for struct User +``` + +The struct name can be extracted when putting a variable instead of a module name: + +```elixir +iex> %struct_name{} = %User{name: "meg"} +%User{name: "meg"} +iex> struct_name +User +``` + +### Binaries + +Binaries may appear in patterns using the double less-than/greater-than syntax ([`<<>>`](`<<>>/1`)). A binary in a pattern can match multiple segments at the same time, each with different type, size, and unit: + +```iex +iex> <> = <<123, 56>> +"{8" +iex> val +31544 +``` + +See the documentation for [`<<>>`](`<<>>/1`) for a complete definition of pattern matching for binaries. + +Finally, remember that strings in Elixir are UTF-8 encoded binaries. This means that, similar to charlists, prefix matches on strings are also possible with the binary concatenation operator ([`<>`](`<>/2`)): + +```elixir +iex> "hello " <> world = "hello world" +"hello world" +iex> world +"world" +``` + +Suffix matches (`hello <> " world"`) are not valid patterns. + +## Guards + +Guards are a way to augment pattern matching with more complex checks. They are allowed in a predefined set of constructs where pattern matching is allowed, such as function definitions, case clauses, and others. + +Not all expressions are allowed in guard clauses, but only a handful of them. This is a deliberate choice. This way, Elixir (through Erlang) ensures that all guards are predictable (no mutations or other side-effects) and they can be optimized and performed efficiently. + +### List of allowed functions and operators + +You can find the built-in list of guards [in the `Kernel` module](`Kernel#guards`). Here is an overview: + + * comparison operators ([`==`](`==/2`), [`!=`](`!=/2`), [`===`](`===/2`), [`!==`](`!==/2`), + [`<`](``](`>/2`), [`>=`](`>=/2`)), [`max`](`max/2`), [`min`](`min/2`) + * strictly boolean operators ([`and`](`and/2`), [`or`](`or/2`), [`not`](`not/1`)). Note [`&&`](`&&/2`), [`||`](`||/2`), and [`!`](`!/1`) sibling operators are **not allowed** as they're not *strictly* boolean - meaning they don't require arguments to be booleans + * arithmetic unary operators ([`+`](`+/1`), [`-`](`-/1`)) + * arithmetic binary operators ([`+`](`+/2`), [`-`](`-/2`), [`*`](`*/2`), [`/`](`//2`)) + * [`in`](`in/2`) and [`not in`](`in/2`) operators (as long as the right-hand side is a list or a range) + * "type-check" functions (`is_list/1`, `is_number/1`, and the like) + * functions that work on built-in data types (`abs/1`, `hd/1`, `map_size/1`, and others) + * the `map.field` syntax + +The module `Bitwise` also includes a handful of [Erlang bitwise operations as guards](Bitwise.html#guards). + +Macros constructed out of any combination of the above guards are also valid guards - for example, `Integer.is_even/1`. For more information, see the "Custom patterns and guards expressions" section shown below. + +### Why guards + +Let's see an example of a guard used in a function clause: + +```elixir +def empty_map?(map) when map_size(map) == 0, do: true +def empty_map?(map) when is_map(map), do: false +``` + +Guards start with the `when` operator, followed by a guard expression. The clause will be executed if and only if the guard expression returns `true`. Multiple boolean conditions can be combined with the [`and`](`and/2`) and [`or`](`or/2`) operators. + +Writing the `empty_map?/1` function by only using pattern matching would not be possible (as pattern matching on `%{}` would match *any* map, not only the empty ones). + +### Non-passing guards + +A function clause will be executed if and only if its guard expression evaluates to `true`. If any other value is returned, the function clause will be skipped. In particular, guards have no concept of "truthy" or "falsy". + +For example, imagine a function that checks that the head of a list is not `nil`: + +```elixir +def not_nil_head?([head | _]) when head, do: true +def not_nil_head?(_), do: false + +not_nil_head?(["some_value", "another_value"]) +#=> false +``` + +Even though the head of the list is not `nil`, the first clause for `not_nil_head?/1` fails because the expression does not evaluate to `true`, but to `"some_value"`, therefore triggering the second clause which returns `false`. To make the guard behave correctly, you must ensure that the guard evaluates to `true`, like so: + +```elixir +def not_nil_head?([head | _]) when head != nil, do: true +def not_nil_head?(_), do: false + +not_nil_head?(["some_value", "another_value"]) +#=> true +``` + +### Errors in guards + +In guards, when functions would normally raise exceptions, they cause the guard to fail instead. + +For example, the `tuple_size/1` function only works with tuples. If we use it with anything else, an argument error is raised: + +```elixir +iex> tuple_size("hello") +** (ArgumentError) argument error +``` + +However, when used in guards, the corresponding clause will fail to match instead of raising an error: + +```elixir +iex> case "hello" do +...> something when tuple_size(something) == 2 -> +...> :worked +...> _anything_else -> +...> :failed +...> end +:failed +``` + +In many cases, we can take advantage of this. In the code above, we used `tuple_size/1` to both check that the given value is a tuple *and* check its size (instead of using `is_tuple(something) and tuple_size(something) == 2`). + +However, if your guard has multiple conditions, such as checking for tuples or maps, it is best to call type-check functions like `is_tuple/1` before `tuple_size/1`, otherwise the whole guard will fail if a tuple is not given. Alternatively, your function clause can use multiple guards as shown in the following section. + +### Multiple guards in the same clause + +There exists an additional way to simplify a chain of `or` expressions in guards: Elixir supports writing "multiple guards" in the same clause. The following code: + +```elixir +def categorize_number(term) when is_integer(term) or is_float(term) or is_nil(term), + do: :maybe_number +def categorize_number(_other), + do: :something_else +``` + +can be alternatively written as: + +```elixir +def categorize_number(term) + when is_integer(term) + when is_float(term) + when is_nil(term) do + :maybe_number +end + +def categorize_number(_other) do + :something_else +end +``` + +If each guard expression always returns a boolean, the two forms are equivalent. However, recall that if any function call in a guard raises an exception, the entire guard fails. To illustrate this, the following function will not detect empty tuples: + +```elixir +defmodule Check do + # If given a tuple, map_size/1 will raise, and tuple_size/1 will not be evaluated + def empty?(val) when map_size(val) == 0 or tuple_size(val) == 0, do: true + def empty?(_val), do: false +end + +Check.empty?(%{}) +#=> true + +Check.empty?({}) +#=> false # true was expected! +``` + +This could be corrected by ensuring that no exception is raised, either via type checks like `is_map(val) and map_size(val) == 0`, or by using multiple guards, so that if an exception causes one guard to fail, the next one is evaluated. + +```elixir +defmodule Check do + # If given a tuple, map_size/1 will raise, and the second guard will be evaluated + def empty?(val) + when map_size(val) == 0 + when tuple_size(val) == 0, + do: true + + def empty?(_val), do: false +end + +Check.empty?(%{}) +#=> true + +Check.empty?({}) +#=> true +``` + +## Where patterns and guards can be used + +In the examples above, we have used the match operator ([`=`](`=/2`)) and function clauses to showcase patterns and guards respectively. Here is the list of the built-in constructs in Elixir that support patterns and guards. + + * `match?/2`: + + ```elixir + match?({:ok, value} when value > 0, {:ok, 13}) + ``` + + * function clauses: + + ```elixir + def type(term) when is_integer(term), do: :integer + def type(term) when is_float(term), do: :float + ``` + + * [`case`](`case/2`) expressions: + + ```elixir + case x do + 1 -> :one + 2 -> :two + n when is_integer(n) and n > 2 -> :larger_than_two + end + ``` + + * anonymous functions (`fn/1`): + + ```elixir + larger_than_two? = fn + n when is_integer(n) and n > 2 -> true + n when is_integer(n) -> false + end + ``` + + * [`for`](`for/1`) and [`with`](`with/1`) support patterns and guards on the left side of `<-`: + + ```elixir + for x when x >= 0 <- [1, -2, 3, -4], do: x + ``` + + `with` also supports the `else` keyword, which supports patterns matching and guards. + + * [`try`](`try/1`) supports patterns and guards on `catch` and `else` + + * [`receive`](`receive/1`) supports patterns and guards to match on the received messages. + + * custom guards can also be defined with `defguard/1` and `defguardp/1`. A custom guard can only be defined based on existing guards. + +Note that the match operator ([`=`](`=/2`)) does *not* support guards: + +```elixir +{:ok, binary} = File.read("some/file") +``` + +## Custom patterns and guards expressions + +Only the constructs listed in this page are allowed in patterns and guards. However, we can take advantage of macros to write custom patterns guards that can simplify our programs or make them more domain-specific. At the end of the day, what matters is that the *output* of the macros boils down to a combination of the constructs above. + +For example, the `Record` module in Elixir provides a series of macros to be used in patterns and guards that allows tuples to have named fields during compilation. + +For defining your own guards, Elixir even provides conveniences in `defguard` and `defguardp`. Let's look at a quick case study: we want to check whether an argument is an even or an odd integer. With pattern matching this is impossible because there is an infinite number of integers, and therefore we can't pattern match on every single one of them. Therefore we must use guards. We will just focus on checking for even numbers since checking for the odd ones is almost identical. + +Such a guard would look like this: + +```elixir +def my_function(number) when is_integer(number) and rem(number, 2) == 0 do + # do stuff +end +``` + +It would be repetitive to write every time we need this check. Instead, you can use `defguard/1` and `defguardp/1` to create guard macros. Here's an example how: + +```elixir +defmodule MyInteger do + defguard is_even(term) when is_integer(term) and rem(term, 2) == 0 +end +``` + +and then: + +```elixir +import MyInteger, only: [is_even: 1] + +def my_function(number) when is_even(number) do + # do stuff +end +``` + +While it's possible to create custom guards with macros, it's recommended to define them using `defguard/1` and `defguardp/1` which perform additional compile-time checks. diff --git a/lib/elixir/pages/references/syntax-reference.md b/lib/elixir/pages/references/syntax-reference.md new file mode 100644 index 00000000000..c658b2bb2ba --- /dev/null +++ b/lib/elixir/pages/references/syntax-reference.md @@ -0,0 +1,432 @@ + + +# Syntax reference + +Elixir syntax was designed to have a straightforward conversion to an abstract syntax tree (AST). This means the Elixir syntax is mostly uniform with a handful of "syntax sugar" constructs to reduce the noise in common Elixir idioms. + +This document covers all of Elixir syntax constructs as a reference and then discuss their exact AST representation. + +## Reserved words + +These are the reserved words in the Elixir language. They are detailed throughout this guide but summed up here for convenience: + + * `true`, `false`, `nil` - used as atoms + * `when`, `and`, `or`, `not`, `in` - used as operators + * `fn` - used for anonymous function definitions + * `do`, `end`, `catch`, `rescue`, `after`, `else` - used in do-end blocks + +## Data types + +### Numbers + +Integers (`1234`) and floats (`123.4`) in Elixir are represented as a sequence of digits that may be separated by underscore for readability purposes, such as `1_000_000`. Integers never contain a dot (`.`) in their representation. Floats contain a dot and at least one other digit after the dot. Floats also support the scientific notation, such as `123.4e10` or `123.4E10`. + +### Atoms + +Unquoted atoms start with a colon (`:`) which must be immediately followed by a Unicode letter or an underscore. The atom may continue using a sequence of Unicode letters, numbers, underscores, and `@`. Atoms may end in `!` or `?`. Valid unquoted atoms are: `:ok`, `:ISO8601`, and `:integer?`. + +If the colon is immediately followed by a pair of double- or single-quotes surrounding the atom name, the atom is considered quoted. In contrast with an unquoted atom, this one can be made of any Unicode character (not only letters), such as `:'🌢 Elixir'`, `:"++olá++"`, and `:"123"`. + +Quoted and unquoted atoms with the same name are considered equivalent, so `:atom`, `:"atom"`, and `:'atom'` represent the same atom. The only catch is that the compiler will warn when quotes are used in atoms that do not need to be quoted. + +All operators in Elixir are also valid atoms. Valid examples are `:foo`, `:FOO`, `:foo_42`, `:foo@bar`, and `:++`. Invalid examples are `:@foo` (`@` is not allowed at start), `:123` (numbers are not allowed at start), and `:(*)` (not a valid operator). + +`true`, `false`, and `nil` are reserved words that are represented by the atoms `:true`, `:false` and `:nil` respectively. + +To learn more about all Unicode characters allowed in atom, see the [Unicode syntax](unicode-syntax.md) document. + +### Strings + +Single-line strings in Elixir are written between double-quotes, such as `"foo"`. Any double-quote inside the string must be escaped with `\ `. Strings support Unicode characters and are stored as UTF-8 encoded binaries. + +Multi-line strings in Elixir are called heredocs. They are written with three double-quotes, and can have unescaped quotes within them. The resulting string will end with a newline. The indentation of the last `"""` is used to strip indentation from the inner string. For example: + +```elixir +iex> test = """ +...> this +...> is +...> a +...> test +...> """ +" this\n is\n a\n test\n" +iex> test = """ +...> This +...> Is +...> A +...> Test +...> """ +"This\nIs\nA\nTest\n" +``` + +Strings are always represented as themselves in the AST. + +### Charlists + +Charlists are lists of non-negative integers where each integer represents a Unicode code point. + +```elixir +iex(6)> 'abc' === [97, 98, 99] +true +``` + +Charlists are written in single-quotes, such as `'foo'`. Any single-quote inside the string must be escaped with `\ `. +Multi-line charlists are written with three single-quotes (`'''`), the same way multi-line strings are. +However, this syntax is deprecated in favor of the charlist sigil `~c`. + +Charlists are always represented as themselves in the AST. + +For more in-depth information, please read the "Charlists" section in the `List` module. + +### Lists, tuples and binaries + +Data structures such as lists, tuples, and binaries are marked respectively by the delimiters `[...]`, `{...}`, and `<<...>>`. Each element is separated by comma. A trailing comma is also allowed, such as in `[1, 2, 3,]`. + +### Maps and keyword lists + +Maps use the `%{...}` notation and each key-value is given by pairs marked with `=>`, such as `%{"hello" => 1, 2 => "world"}`. + +Both keyword lists (list of two-element tuples where the first element is an atom) and maps with atom keys support a keyword notation where the colon character `:` is moved to the end of the atom. `%{hello: "world"}` is equivalent to `%{:hello => "world"}` and `[foo: :bar]` is equivalent to `[{:foo, :bar}]`. We discuss keywords in later sections. + +### Structs + +Structs built on the map syntax by passing the struct name between `%` and `{`. For example, `%User{...}`. + +## Expressions + +### Variables + +Variables in Elixir must start with an underscore or a Unicode letter that is not in uppercase or titlecase. The variable may continue using a sequence of Unicode letters, numbers, and underscores. Variables may end in `?` or `!`. To learn more about all Unicode characters allowed in variables, see the [Unicode syntax](unicode-syntax.md) document. + +[Elixir's naming conventions](naming-conventions.md) recommend variables to be in `snake_case` format. + +### Non-qualified calls (local calls) + +Non-qualified calls, such as `add(1, 2)`, must start with characters and then follow the same rules as variables, which are optionally followed by parentheses, and then arguments. + +Parentheses are required for zero-arity calls (i.e. calls without arguments), to avoid ambiguity with variables. If parentheses are used, they must immediately follow the function name *without spaces*. For example, `add (1, 2)` is a syntax error, since `(1, 2)` is treated as an invalid block which is attempted to be given as a single argument to `add`. + +[Elixir's naming conventions](naming-conventions.md) recommend calls to be in `snake_case` format. + +### Operators + +As many programming languages, Elixir also support operators as non-qualified calls with their precedence and associativity rules. Constructs such as `=`, `when`, `&` and `@` are simply treated as operators. See [the Operators page](operators.md) for a full reference. + +### Qualified calls (remote calls) + +Qualified calls, such as `Math.add(1, 2)`, must start with characters and then follow the same rules as variables, which are optionally followed by parentheses, and then arguments. Qualified calls also support operators, such as `Kernel.+(1, 2)`. Elixir also allows the function name to be written between double- or single-quotes, allowing any character in between the quotes, such as `Math."++add++"(1, 2)`. + +Similar to non-qualified calls, parentheses have different meaning for zero-arity calls (i.e. calls without arguments). If parentheses are used, such as `mod.fun()`, it means a function call. If parenthesis are skipped, such as `map.field`, it means accessing a field of a map. + +[Elixir's naming conventions](naming-conventions.md) recommend calls to be in `snake_case` format. + +### Aliases + +Aliases are constructs that expand to atoms at compile-time. The alias `String` expands to the atom `:"Elixir.String"`. Aliases must start with an ASCII uppercase character which may be followed by any ASCII letter, number, or underscore. Non-ASCII characters are not supported in aliases. + +Multiple aliases can be joined with `.`, such as `MyApp.String`, and it expands to the atom `:"Elixir.MyApp.String"`. The dot is effectively part of the name but it can also be used for composition. If you define `alias MyApp.Example, as: Example` in your code, then `Example` will always expand to `:"Elixir.MyApp.Example"` and `Example.String` will expand to `:"Elixir.MyApp.Example.String"`. + +[Elixir's naming conventions](naming-conventions.md) recommend aliases to be in `CamelCase` format. + +### Module attributes + +Module attributes are module-specific storage and are written as the composition of the unary operator `@` with variables and local calls. For example, to write to a module attribute named `foo`, use `@foo "value"`, and use `@foo` to read from it. Given module attributes are written using existing constructs, they follow the same rules above defined for operators, variables, and local calls. + +### Blocks + +Blocks are multiple Elixir expressions separated by newlines or semi-colons. A new block may be created at any moment by using parentheses. + +### Left to right arrow + +The left to right arrow (`->`) is used to establish a relationship between left and right, commonly referred as clauses. The left side may have zero, one, or more arguments; the right side is zero, one, or more expressions separated by new line. The `->` may appear one or more times between one of the following terminators: `do`-`end`, `fn`-`end` or `(`-`)`. When `->` is used, only other clauses are allowed between those terminators. Mixing clauses and regular expressions is invalid syntax. + +It is seen on `case` and `cond` constructs between `do` and `end`: + +```elixir +case 1 do + 2 -> 3 + 4 -> 5 +end + +cond do + true -> false +end +``` + +Seen in typespecs between `(` and `)`: + +```elixir +(integer(), boolean() -> integer()) +``` + +It is also used between `fn` and `end` for building anonymous functions: + +```elixir +fn + x, y -> x + y +end +``` + +### Sigils + +Sigils start with `~` and are followed by one lowercase letter or by one or more uppercase letters, immediately followed by one of the following pairs: + + * `(` and `)` + * `{` and `}` + * `[` and `]` + * `<` and `>` + * `"` and `"` + * `'` and `'` + * `|` and `|` + * `/` and `/` + +After closing the pair, zero or more ASCII letters and digits can be given as a modifier. Sigils are expressed as non-qualified calls prefixed with `sigil_` where the first argument is the sigil contents as a string and the second argument is a list of integers as modifiers: + +If the sigil letter is in uppercase, no interpolation is allowed in the sigil, otherwise its contents may be dynamic. Compare the results of the sigils below for more information: + +```elixir +~s/f#{"o"}o/ +~S/f#{"o"}o/ +``` + +Sigils are useful to encode text with their own escaping rules, such as regular expressions, datetimes, and others. + +## The Elixir AST + +Elixir syntax was designed to have a straightforward conversion to an abstract syntax tree (AST). Elixir's AST is a regular Elixir data structure composed of the following elements: + + * atoms - such as `:foo` + * integers - such as `42` + * floats - such as `13.1` + * strings - such as `"hello"` + * lists - such as `[1, 2, 3]` + * tuples with two elements - such as `{"hello", :world}` + * tuples with three elements, representing calls or variables, as explained next + +The building block of Elixir's AST is a call, such as: + +```elixir +sum(1, 2, 3) +``` + +which is represented as a tuple with three elements: + +```elixir +{:sum, meta, [1, 2, 3]} +``` + +the first element is an atom (or another tuple), the second element is a list of two-element tuples with metadata (such as line numbers) and the third is a list of arguments. + +We can retrieve the AST for any Elixir expression by calling `quote`: + +```elixir +quote do + sum() +end +#=> {:sum, [], []} +``` + +Variables are also represented using a tuple with three elements and a combination of lists and atoms, for example: + +```elixir +quote do + sum +end +#=> {:sum, [], Elixir} +``` + +You can see that variables are also represented with a tuple, except the third element is an atom expressing the variable context. + +Over the course of this section, we will explore many Elixir syntax constructs alongside their AST representations. + +### Operators + +Operators are treated as non-qualified calls: + +```elixir +quote do + 1 + 2 +end +#=> {:+, [], [1, 2]} +``` + +Note that `.` is also an operator. Remote calls use the dot in the AST with two arguments, where the second argument is always an atom: + +```elixir +quote do + foo.bar(1, 2, 3) +end +#=> {{:., [], [{:foo, [], Elixir}, :bar]}, [], [1, 2, 3]} +``` + +Calling anonymous functions uses the dot in the AST with a single argument, mirroring the fact the function name is "missing" from right side of the dot: + +```elixir +quote do + foo.(1, 2, 3) +end +#=> {{:., [], [{:foo, [], Elixir}]}, [], [1, 2, 3]} +``` + +### Aliases + +Aliases are represented by an `__aliases__` call with each segment separated by a dot as an argument: + +```elixir +quote do + Foo.Bar.Baz +end +#=> {:__aliases__, [], [:Foo, :Bar, :Baz]} + +quote do + __MODULE__.Bar.Baz +end +#=> {:__aliases__, [], [{:__MODULE__, [], Elixir}, :Bar, :Baz]} +``` + +All arguments, except the first, are guaranteed to be atoms. + +### Data structures + +Remember that lists are literals, so they are represented as themselves in the AST: + +```elixir +quote do + [1, 2, 3] +end +#=> [1, 2, 3] +``` + +Tuples have their own representation, except for two-element tuples, which are represented as themselves: + +```elixir +quote do + {1, 2} +end +#=> {1, 2} + +quote do + {1, 2, 3} +end +#=> {:{}, [], [1, 2, 3]} +``` + +Binaries have a representation similar to tuples, except they are tagged with `:<<>>` instead of `:{}`: + +```elixir +quote do + <<1, 2, 3>> +end +#=> {:<<>>, [], [1, 2, 3]} +``` + +The same applies to maps, where pairs are treated as a list of tuples with two elements: + +```elixir +quote do + %{1 => 2, 3 => 4} +end +#=> {:%{}, [], [{1, 2}, {3, 4}]} +``` + +### Blocks + +Blocks are represented as a `__block__` call with each line as a separate argument: + +```elixir +quote do + 1 + 2 + 3 +end +#=> {:__block__, [], [1, 2, 3]} + +quote do 1; 2; 3; end +#=> {:__block__, [], [1, 2, 3]} +``` + +### Left to right arrow + +The left to right arrow (`->`) is represented similar to operators except that they are always part of a list, its left side represents a list of arguments and the right side is an expression. + +For example, in `case` and `cond`: + +```elixir +quote do + case 1 do + 2 -> 3 + 4 -> 5 + end +end +#=> {:case, [], [1, [do: [{:->, [], [[2], 3]}, {:->, [], [[4], 5]}]]]} + +quote do + cond do + true -> false + end +end +#=> {:cond, [], [[do: [{:->, [], [[true], false]}]]]} +``` + +Between `(` and `)`: + +```elixir +quote do + (1, 2 -> 3 + 4, 5 -> 6) +end +#=> [{:->, [], [[1, 2], 3]}, {:->, [], [[4, 5], 6]}] +``` + +Between `fn` and `end`: + +```elixir +quote do + fn + 1, 2 -> 3 + 4, 5 -> 6 + end +end +#=> {:fn, [], [{:->, [], [[1, 2], 3]}, {:->, [], [[4, 5], 6]}]} +``` + +### Qualified tuples + +Qualified tuples (`foo.{bar, baz}`) are represented by a `{:., [], [expr, :{}]}` call, where the `expr` represents the left hand side of the dot, and the arguments represent the elements inside the curly braces. This is used in Elixir to provide multi aliases: + +```elixir +quote do + Foo.{Bar, Baz} +end +#=> {{:., [], [{:__aliases__, [], [:Foo]}, :{}]}, [], [{:__aliases__, [], [:Bar]}, {:__aliases__, [], [:Baz]}]} +``` + +### `do`-`end` blocks + +Elixir's `do`-`end` blocks are equivalent to keywords as the last argument of a function call, where the block contents are wrapped in parentheses. For example: + +```elixir +if true do + this +else + that +end +``` + +is the same as: + +```elixir +if(true, do: (this), else: (that)) +``` + +While the construct above does not require custom nodes in Elixir's AST, they are restricted only to certain keywords, listed next: + + * `after` + * `catch` + * `else` + * `rescue` + +You will find them in constructs such as `receive`, `try`, and others. You can also find more examples in [the Optional Syntax chapter](../getting-started/optional-syntax.md). diff --git a/lib/elixir/pages/references/typespecs.md b/lib/elixir/pages/references/typespecs.md new file mode 100644 index 00000000000..32ff5ad12eb --- /dev/null +++ b/lib/elixir/pages/references/typespecs.md @@ -0,0 +1,442 @@ + + +# Typespecs reference + +> #### Typespecs are not set-theoretic types {: .warning} +> +> Elixir is in the process of implementing its +> [own type system](./gradual-set-theoretic-types.md) based on set-theoretic types. +> Typespecs, which are described in the following document, are a distinct notation +> for declaring types and specifications based on Erlang. +> Typespecs may be phased out as the set-theoretic type effort moves forward. + +Elixir is a dynamically typed language, and as such, type specifications are never used by the compiler to optimize or modify code. Still, using type specifications is useful because: + + * they provide documentation (for example, tools such as [`ExDoc`](https://hexdocs.pm/ex_doc/) show type specifications in the documentation) + * they're used by tools such as [Dialyzer](`:dialyzer`), that can analyze code with typespecs to find type inconsistencies and possible bugs + +Type specifications (most often referred to as *typespecs*) are defined in different contexts using the following attributes: + + * `@type` + * `@opaque` + * `@typep` + * `@spec` + * `@callback` + * `@macrocallback` + +In addition, you can use `@typedoc` to document a custom `@type` definition. + +See the "User-defined types" and "Defining a specification" sub-sections below for more information on defining types and typespecs. + +## A simple example + + defmodule StringHelpers do + @typedoc "A word from the dictionary" + @type word() :: String.t() + + @spec long_word?(word()) :: boolean() + def long_word?(word) when is_binary(word) do + String.length(word) > 8 + end + end + +In the example above: + + * We declare a new type (`word()`) that is equivalent to the string type (`String.t()`). + + * We describe the type using a `@typedoc`, which will be included in the generated documentation. + + * We specify that the `long_word?/1` function takes an argument of type `word()` and + returns a boolean (`boolean()`), that is, either `true` or `false`. + +## Types and their syntax + +The syntax Elixir provides for type specifications is similar to [the one in Erlang](https://www.erlang.org/doc/reference_manual/typespec.html). Most of the built-in types provided in Erlang (for example, `pid()`) are expressed in the same way: `pid()` (or simply `pid`). Parameterized types (such as `list(integer)`) are supported as well and so are remote types (such as [`Enum.t()`](`t:Enum.t/0`)). Integers and atom literals are allowed as types (for example, `1`, `:atom`, or `false`). All other types are built out of unions of predefined types. Some types can also be declared using their syntactical notation, such as `[type]` for lists, `{type1, type2, ...}` for tuples and `<<_ * _>>` for binaries. + +The notation to represent the union of types is the pipe `|`. For example, the typespec `type :: atom() | pid() | tuple()` creates a type `type` that can be either an `atom`, a `pid`, or a `tuple`. This is usually called a [sum type](https://en.wikipedia.org/wiki/Tagged_union) in other languages + +> #### Differences with set-theoretic types {: .warning} +> +> While they do share some similarities, the types below do not map one-to-one +> to the new types from the set-theoretic type system. +> +> For example, there is no plan to support subsets of the `integer()` type such +> as positive, ranges or literals. +> +> Furthermore, set-theoretic types support the full range of set operations, +> including intersections and negations. + +### Basic types + + type :: + any() # the top type, the set of all terms + | none() # the bottom type, contains no terms + | atom() + | map() # any map + | pid() # process identifier + | port() # port identifier + | reference() + | tuple() # tuple of any size + + ## Numbers + | float() + | integer() + | neg_integer() # ..., -3, -2, -1 + | non_neg_integer() # 0, 1, 2, 3, ... + | pos_integer() # 1, 2, 3, ... + + ## Lists + | list(type) # proper list ([]-terminated) + | nonempty_list(type) # non-empty proper list + | maybe_improper_list(content_type, termination_type) # proper or improper list + | nonempty_improper_list(content_type, termination_type) # improper list + | nonempty_maybe_improper_list(content_type, termination_type) # non-empty proper or improper list + + | Literals # Described in section "Literals" + | BuiltIn # Described in section "Built-in types" + | Remotes # Described in section "Remote types" + | UserDefined # Described in section "User-defined types" + +### Literals + +The following literals are also supported in typespecs: + + type :: ## Atoms + :atom # atoms: :foo, :bar, ... + | true | false | nil # special atom literals + + ## Bitstrings + | <<>> # empty bitstring + | <<_::size>> # size is 0 or a positive integer + | <<_::_*unit>> # unit is an integer from 1 to 256 + | <<_::size, _::_*unit>> + + ## (Anonymous) Functions + | (-> type) # zero-arity, returns type + | (type1, type2 -> type) # two-arity, returns type + | (... -> type) # any arity, returns type + + ## Integers + | 1 # integer + | 1..10 # integer from 1 to 10 + + ## Lists + | [type] # list with any number of type elements + | [] # empty list + | [...] # shorthand for nonempty_list(any()) + | [type, ...] # shorthand for nonempty_list(type) + | [key: value_type] # keyword list with optional key :key of value_type + + ## Maps + | %{} # empty map + | %{key: value_type} # map with required key :key of value_type + | %{key_type => value_type} # map with required pairs of key_type and value_type + | %{required(key_type) => value_type} # map with required pairs of key_type and value_type + | %{optional(key_type) => value_type} # map with optional pairs of key_type and value_type + | %SomeStruct{} # struct with all fields of any type + | %SomeStruct{key: value_type} # struct with required key :key of value_type + + ## Tuples + | {} # empty tuple + | {:ok, type} # two-element tuple with an atom and any type + +### Built-in types + +The following types are also provided by Elixir as shortcuts on top of the basic and literal types described above. + +Built-in type | Defined as +:---------------------- | :--------- +`term()` | `any()` +`arity()` | `0..255` +`as_boolean(t)` | `t` +`binary()` | `<<_::_*8>>` +`nonempty_binary()` | `<<_::8, _::_*8>>` +`bitstring()` | `<<_::_*1>>` +`nonempty_bitstring()` | `<<_::1, _::_*1>>` +`boolean()` | `true` \| `false` +`byte()` | `0..255` +`char()` | `0..0x10FFFF` +`charlist()` | `[char()]` +`nonempty_charlist()` | `[char(), ...]` +`fun()` | `(... -> any)` +`function()` | `fun()` +`identifier()` | `pid()` \| `port()` \| `reference()` +`iodata()` | `iolist()` \| `binary()` +`iolist()` | `maybe_improper_list(byte() \| binary() \| iolist(), binary() \| [])` +`keyword()` | `[{atom(), any()}]` +`keyword(t)` | `[{atom(), t}]` +`list()` | `[any()]` +`nonempty_list()` | `nonempty_list(any())` +`maybe_improper_list()` | `maybe_improper_list(any(), any())` +`nonempty_maybe_improper_list()` | `nonempty_maybe_improper_list(any(), any())` +`mfa()` | `{module(), atom(), arity()}` +`module()` | `atom()` +`no_return()` | `none()` +`node()` | `atom()` +`number()` | `integer()` \| `float()` +`struct()` | `%{:__struct__ => atom(), optional(atom()) => any()}` +`timeout()` | `:infinity` \| `non_neg_integer()` + +`as_boolean(t)` exists to signal users that the given value will be treated as a boolean, where `nil` and `false` will be evaluated as `false` and everything else is `true`. For example, `Enum.filter/2` has the following specification: `filter(t, (element -> as_boolean(term))) :: list`. + +### Remote types + +Any module is also able to define its own types and the modules in Elixir are no exception. For example, the `Range` module defines a `t/0` type that represents a range: this type can be referred to as `t:Range.t/0`. In a similar fashion, a string is `t:String.t/0`, and so on. + +### Maps + +The key types in maps are allowed to overlap, and if they do, the leftmost key takes precedence. +A map value does not belong to this type if it contains a key that is not in the allowed map keys. + +If you want to denote that keys that were not previously defined in the map are allowed, +it is common to end a map type with `optional(any) => any`. + +Note that the syntactic representation of `map()` is `%{optional(any) => any}`, not `%{}`. The notation `%{}` specifies the singleton type for the empty map. + +### Keyword Lists + +Beyond `keyword()` and `keyword(t)`, it can be helpful to compose a spec for an expected keyword list. +For example: + +```elixir +@type option :: {:name, String.t} | {:max, pos_integer} | {:min, pos_integer} +@type options :: [option()] +``` + +This makes it clear that only these options are allowed, none are required, and order does not matter. + +It also allows composition with existing types. +For example: + +```elixir +@type option :: {:my_option, String.t()} | GenServer.option() + +@spec start_link([option()]) :: GenServer.on_start() +def start_link(opts) do + {my_opts, gen_server_opts} = Keyword.split(opts, [:my_option]) + GenServer.start_link(__MODULE__, my_opts, gen_server_opts) +end +``` + +The following spec syntaxes are equivalent: + +```elixir +@type options [{:name, String.t} | {:max, pos_integer} | {:min, pos_integer}] + +@type options [name: String.t, max: pos_integer, min: pos_integer] +``` + +### User-defined types + +The `@type`, `@typep`, and `@opaque` module attributes can be used to define new types: + + @type type_name :: type + @typep type_name :: type + @opaque type_name :: type + +A type defined with `@typep` is private. An opaque type, defined with `@opaque` is a type where the internal structure of the type will not be visible, but the type is still public. + +Types can be parameterized by defining variables as parameters; these variables can then be used to define the type. + + @type dict(key, value) :: [{key, value}] + +## Defining a specification + +A specification for a function can be defined as follows: + + @spec function_name(type1, type2) :: return_type + +Guards can be used to restrict type variables given as arguments to the function. + + @spec function(arg) :: [arg] when arg: atom + +If you want to specify more than one variable, you separate them by a comma. + + @spec function(arg1, arg2) :: {arg1, arg2} when arg1: atom, arg2: integer + +Type variables with no restriction can also be defined using `var`. + + @spec function(arg) :: [arg] when arg: var + +This guard notation only works with `@spec`, `@callback`, and `@macrocallback`. + +You can also name your arguments in a typespec using `arg_name :: arg_type` syntax. This is particularly useful in documentation as a way to differentiate multiple arguments of the same type (or multiple elements of the same type in a type definition): + + @spec days_since_epoch(year :: integer, month :: integer, day :: integer) :: integer + @type color :: {red :: integer, green :: integer, blue :: integer} + +Specifications can be overloaded, just like ordinary functions. + + @spec function(integer) :: atom + @spec function(atom) :: integer + +## Behaviours + +Behaviours in Elixir (and Erlang) are a way to separate and abstract the generic part of a component (which becomes the *behaviour module*) from the specific part (which becomes the *callback module*). + +A behaviour module defines a set of functions and macros (referred to as *callbacks*) that callback modules implementing that behaviour must export. This "interface" identifies the specific part of the component. For example, the `GenServer` behaviour and functions abstract away all the message-passing (sending and receiving) and error reporting that a "server" process will likely want to implement from the specific parts such as the actions that this server process has to perform. + +Say we want to implement a bunch of parsers, each parsing structured data: for example, a JSON parser and a MessagePack parser. Each of these two parsers will *behave* the same way: both will provide a `parse/1` function and an `extensions/0` function. The `parse/1` function will return an Elixir representation of the structured data, while the `extensions/0` function will return a list of file extensions that can be used for each type of data (e.g., `.json` for JSON files). + +We can create a `Parser` behaviour: + +```elixir +defmodule Parser do + @doc """ + Parses a string. + """ + @callback parse(String.t) :: {:ok, term} | {:error, atom} + + @doc """ + Lists all supported file extensions. + """ + @callback extensions() :: [String.t] +end +``` + +As seen in the example above, defining a callback is a matter of defining a specification for that callback, made of: + + * the callback name (`parse` or `extensions` in the example) + * the arguments that the callback must accept (`String.t`) + * the *expected* type of the callback return value + +Modules adopting the `Parser` behaviour will have to implement all the functions defined with the `@callback` attribute. As you can see, `@callback` expects a function name but also a function specification like the ones used with the `@spec` attribute we saw above. + +### Implementing behaviours + +Implementing a behaviour is straightforward: + +```elixir +defmodule JSONParser do + @behaviour Parser + + @impl Parser + def parse(str), do: {:ok, "some json " <> str} # ... parse JSON + + @impl Parser + def extensions, do: [".json"] +end +``` + +```elixir +defmodule CSVParser do + @behaviour Parser + + @impl Parser + def parse(str), do: {:ok, "some csv " <> str} # ... parse CSV + + @impl Parser + def extensions, do: [".csv"] +end +``` + +If a module adopting a given behaviour doesn't implement one of the callbacks required by that behaviour, a compile-time warning will be generated. + +Furthermore, with `@impl` you can also make sure that you are implementing the **correct** callbacks from the given behaviour in an explicit manner. For example, the following parser implements both `parse` and `extensions`. However, thanks to a typo, `BADParser` is implementing `parse/0` instead of `parse/1`. + +```elixir +defmodule BADParser do + @behaviour Parser + + @impl Parser + def parse, do: {:ok, "something bad"} + + @impl Parser + def extensions, do: ["bad"] +end +``` + +This code generates a warning letting you know that you are mistakenly implementing `parse/0` instead of `parse/1`. +You can read more about `@impl` in the [module documentation](`Module#module-impl`). + +### Using behaviours + +Behaviours are useful because you can pass modules around as arguments and you can then *call back* to any of the functions specified in the behaviour. For example, we can have a function that receives a filename, several parsers, and parses the file based on its extension: + +```elixir +@spec parse_path(Path.t(), [module()]) :: {:ok, term} | {:error, atom} +def parse_path(filename, parsers) do + with {:ok, ext} <- parse_extension(filename), + {:ok, parser} <- find_parser(ext, parsers), + {:ok, contents} <- File.read(filename) do + parser.parse(contents) + end +end + +defp parse_extension(filename) do + if ext = Path.extname(filename) do + {:ok, ext} + else + {:error, :no_extension} + end +end + +defp find_parser(ext, parsers) do + if parser = Enum.find(parsers, fn parser -> ext in parser.extensions() end) do + {:ok, parser} + else + {:error, :no_matching_parser} + end +end +``` + +You could also invoke any parser directly: `CSVParser.parse(...)`. + +Note you don't need to define a behaviour in order to dynamically dispatch on a module, but those features often go hand in hand. + +### Optional callbacks + +Optional callbacks are callbacks that callback modules may implement if they want to, but are not required to. Usually, behaviour modules know if they should call those callbacks based on configuration, or they check if the callbacks are defined with `function_exported?/3` or `macro_exported?/3`. + +> ### Unloaded modules {: .warning} +> +> `function_exported?/3` (and `macro_exported?/3`) do *not* load the module in case it is not loaded and Elixir lazily loads modules by default (except on releases). So in practice you will want to invoke `Code.ensure_loaded?/1` before checking if the function/macro is exported. See the documentation for `function_exported?/3` for examples. + +Optional callbacks can be defined through the `@optional_callbacks` module attribute, which has to be a keyword list with function or macro name as key and arity as value. For example: + + defmodule MyBehaviour do + @callback vital_fun() :: any + @callback non_vital_fun() :: any + @macrocallback non_vital_macro(arg :: any) :: Macro.t + @optional_callbacks non_vital_fun: 0, non_vital_macro: 1 + end + +One example of optional callback in Elixir's standard library is `c:GenServer.format_status/1`. + +### Inspecting behaviours + +The `@callback` and `@optional_callbacks` attributes are used to create a `behaviour_info/1` function available on the defining module. This function can be used to retrieve the callbacks and optional callbacks defined by that module. + +For example, for the `MyBehaviour` module defined in "Optional callbacks" above: + + MyBehaviour.behaviour_info(:callbacks) + #=> [vital_fun: 0, "MACRO-non_vital_macro": 2, non_vital_fun: 0] + MyBehaviour.behaviour_info(:optional_callbacks) + #=> ["MACRO-non_vital_macro": 2, non_vital_fun: 0] + +When using `iex`, the `IEx.Helpers.b/1` helper is also available. + +## Pitfalls + +There are some known pitfalls when using typespecs, they are documented next. + +## The `string()` type + +Elixir discourages the use of the `string()` type. The `string()` type refers to Erlang strings, which are known as "charlists" in Elixir. They do not refer to Elixir strings, which are UTF-8 encoded binaries. To avoid confusion, if you attempt to use the type `string()`, Elixir will emit a warning. You should use `charlist()`, `nonempty_charlist()`, `binary()` or `String.t()` accordingly, or any of the several literal representations for these types. + +Note that `String.t()` and `binary()` are equivalent to analysis tools. Although, for those reading the documentation, `String.t()` implies it is a UTF-8 encoded binary. + +## Functions which raise an error + +Typespecs do not need to indicate that a function can raise an error; any function can fail any time if given invalid input. +In the past, the Elixir standard library sometimes used `no_return()` to indicate this, but these usages have been removed. + +The `no_return()` type also should not be used for functions which do return but whose purpose is a "side effect", such as `IO.puts/1`. +In these cases, the expected return type is `:ok`. + +Instead, `no_return()` should be used as the return type for functions which can never return a value. +This includes functions which loop forever calling `receive`, or which exist specifically to raise an error, or which shut down the VM. diff --git a/lib/elixir/pages/references/unicode-syntax.md b/lib/elixir/pages/references/unicode-syntax.md new file mode 100644 index 00000000000..c700590e3c5 --- /dev/null +++ b/lib/elixir/pages/references/unicode-syntax.md @@ -0,0 +1,159 @@ + + +# Unicode syntax + +Elixir supports Unicode throughout the language. This document is a complete reference of how +Elixir supports Unicode in its syntax. + +Strings (`"olá"`) and charlists (`'olá'`) support Unicode since Elixir v1.0. Strings are UTF-8 encoded. Charlists are lists of Unicode code points. In such cases, the contents are kept as written by developers, without any transformation. + +Elixir also supports Unicode in variables, atoms, and calls since Elixir v1.5. The focus of this document is to provide a high-level introduction to how Elixir allows Unicode in its syntax. We also provide technical documentation describing how Elixir complies with the Unicode specification. + +To check the Unicode version of your current Elixir installation, run `String.Unicode.version()`. + +## Introduction + +Elixir allows Unicode characters in its variables, atoms, and calls. However, the Unicode characters must still obey the rules of the language syntax. In particular, variables and calls cannot start with an uppercase letter. From now on, we will refer to those terms as identifiers. + +The characters allowed in identifiers are the ones specified by Unicode. Generally speaking, it is restricted to characters typically used by the writing system of human languages still in activity. In particular, it excludes symbols such as emojis, alternate numeric representations, musical notes, and the like. + +Elixir imposes many restrictions on identifiers for security purposes. For example, the word "josé" can be written in two ways in Unicode: as the combination of the characters `j o s é` and as a combination of the characters `j o s e ́ `, where the accent is its own character. The former is called NFC form and the latter is the NFD form. Elixir normalizes all characters to be the in the NFC form. + +Elixir also disallows mixed-scripts which are not explicitly separated by `_`. For example, it is not possible to name a variable `аdmin`, where `а` is in Cyrillic and the remaining characters are in Latin. Doing so will raise the following error: + +```text +** (SyntaxError) invalid mixed-script identifier found: аdmin + +Mixed-script identifiers are not supported for security reasons. 'аdmin' is made of the following scripts: + + \u0430 а {Cyrillic} + \u0064 d {Latin} + \u006D m {Latin} + \u0069 i {Latin} + \u006E n {Latin} + +Make sure all characters in the identifier resolve to a single script or a highly +restrictive script. See https://hexdocs.pm/elixir/unicode-syntax.html for more information. +``` + +Finally, Elixir will also warn of confusable identifiers in the same file. For example, Elixir will emit a warning if you use both variables `а` (Cyrillic) and `а` (Latin) in your code. + +That's the overall introduction of how Unicode is used in Elixir identifiers. In a nutshell, its goal is to support different writing systems in use today while keeping the Elixir language itself clear and secure. + +For the technical details, see the next sections that cover the technical Unicode requirements. + +## Unicode Standard Annex #31 + +Elixir conforms to the standards outlined in the [Unicode Standard Annex #31: Unicode Identifiers and Syntax](https://unicode.org/reports/tr31/), version 17.0. + +### R1. Default Identifiers + +The general Elixir identifier rule is specified as: + + := * ? + +where `` uses the same categories as the spec but normalizes them to the NFC form (see R4): + +> characters derived from the Unicode General Category of uppercase letters, lowercase letters, titlecase letters, modifier letters, other letters, letter numbers, plus `Other_ID_Start`, minus `Pattern_Syntax` and `Pattern_White_Space` code points +> +> In set notation: `[\p{L}\p{Nl}\p{Other_ID_Start}-\p{Pattern_Syntax}-\p{Pattern_White_Space}]`. + +and `` uses the same categories as the spec but normalizes them to the NFC form (see R4): + +> ID_Start characters, plus characters having the Unicode General Category of nonspacing marks, spacing combining marks, decimal number, connector punctuation, plus `Other_ID_Continue`, minus `Pattern_Syntax` and `Pattern_White_Space` code points. +> +> In set notation: `[\p{ID_Start}\p{Mn}\p{Mc}\p{Nd}\p{Pc}\p{Other_ID_Continue}-\p{Pattern_Syntax}-\p{Pattern_White_Space}]`. + +`` is an addition specific to Elixir that includes only the code points `?` (003F) and `!` (0021). + +The spec also provides a `` set, but Elixir does not include any character on this set. Therefore, the identifier rule has been simplified to consider this. + +Elixir does not allow the use of ZWJ or ZWNJ in identifiers and therefore does not implement R1a. Bidirectional control characters are also not supported. R1b is guaranteed for backwards compatibility purposes. + +#### Atoms + +Unicode atoms in Elixir follow the identifier rule above with the following modifications: + + * `` additionally includes the code point `_` (005F) + * `` additionally includes the code point `@` (0040) + +Note atoms can also be quoted, which allows any characters, such as `:"hello elixir"`. All Elixir operators are also valid atoms, such as `:+`, `:@`, `:|>`, and others. The full description of valid atoms is available in the ["Atoms" section in the syntax reference](syntax-reference.md#atoms). + +#### Variables, local calls, and remote calls + +Variables in Elixir follow the identifier rule above with the following modifications: + + * `` additionally includes the code point `_` (005F) + * `` additionally excludes Lu (letter uppercase) and Lt (letter titlecase) characters + +In set notation: `[\u{005F}\p{Ll}\p{Lm}\p{Lo}\p{Nl}\p{Other_ID_Start}-\p{Pattern_Syntax}-\p{Pattern_White_Space}]`. + +#### Aliases + +Aliases in Elixir only allow ASCII characters, starting in uppercase, and no punctuation characters. + +### R3. Pattern_White_Space and Pattern_Syntax Characters + +Elixir supports only code points `\t` (0009), `\n` (000A), `\r` (000D) and `\s` (0020) as whitespace and therefore does not follow requirement R3. R3 requires a wider variety of whitespace and syntax characters to be supported. + +### R4. Equivalent Normalized Identifiers + +Identifiers in Elixir are case sensitive. + +Elixir normalizes all atoms and variables to NFC form. Quoted-atoms and strings can, however, be in any form and are not verified by the parser. + +In other words, the atom `:josé` can only be written with the code points `006A 006F 0073 00E9` or `006A 006F 0073 0065 0301`, but Elixir will rewrite it to the former (from Elixir 1.14). On the other hand, `:"josé"` may be written as `006A 006F 0073 00E9` or `006A 006F 0073 0065 0301` and its form will be retained, since it is written between quotes. + +Choosing requirement R4 automatically excludes requirements R5, R6, and R7. + +## Unicode Technical Standard #39 + +Elixir conforms to the clauses outlined in the [Unicode Technical Standard #39](https://unicode.org/reports/tr39/) on Security, version 17.0. + +### C1. General Security Profile for Identifiers + +Elixir will not allow tokenization of identifiers with codepoints in `\p{Identifier_Status=Restricted}`, except for the outlined 'Additional normalizations' section below. + +> An implementation following the General Security Profile does not permit any characters in \p{Identifier_Status=Restricted}, ... + +For instance, the 'HANGUL FILLER' (`ㅤ`) character, which is often invisible, is an uncommon codepoint and will trigger a warning. + +### C2. Confusable detection + +Elixir will warn of identifiers that look the same, but aren't. Examples: in `а = a = 1`, the two 'a' characters are Cyrillic and Latin, and could be confused for each other; in `力 = カ = 1`, both are Japanese, but different codepoints, in different scripts of that writing system. Confusable identifiers can lead to hard-to-catch bugs (say, due to copy-pasted code) and can be unsafe, so we will warn of identifiers within a single file that could be confused with each other. + +We use the means described in Section 4, 'Confusable Detection', with one noted modification: + +> Alternatively, it shall declare that it uses a modification, and provide a precise list of character mappings that are added to or removed from the provided ones. + +Elixir will not warn about confusability for identifiers made up exclusively of characters in a-z, A-Z, 0-9, and _. This is because ASCII identifiers have existed for so long that the programming community has had their own means of dealing with confusability between identifiers like `l,1` or `O,0` (for instance, fonts designed for programming usually make it easy to differentiate between those characters). + +### C3. Mixed Script Detection + +Elixir will not allow tokenization of mixed-script identifiers unless it is via chunks separated by an underscore, like `http_сервер`. We use the means described in Section 5.1, Mixed-Script Detection, to determine if script mixing is occurring, with the 'Additional Normalizations' documented in. + +Examples: Elixir allows an identifiers like `幻한`, even though it includes characters from multiple 'scripts', as Han characters may be mixed with Japanese and Korean, according to the rules from UTS 39 5.1. When mixing Latin and Japanese scripts, underscores are necessary, as in `:T_シャツ` (the Japanese word for 't-shirt' with an additional underscore separating the letter T). + +Elixir does not allow code like `if аdmin, do: :ok, else: :err`, where the scriptset for the 'a' character is {Cyrillic} but all other characters have scriptsets of {Latin}. The scriptsets fail to resolve and a descriptive error is shown. + +### C4, C5 (inapplicable) + +'C4 - Restriction Level detection' conformance is not claimed and does not apply to identifiers in code; rather, it applies to classifying the level of safety of a given arbitrary string into one of 5 restriction levels. + +'C5 - Mixed number detection' conformance is inapplicable as Elixir does not support Unicode numbers. + +### Addition Normalizations + +As of Elixir 1.14, some codepoints in `\p{Identifier_Status=Restricted}` are *normalized* to other, unrestricted codepoints. + +This is currently only applied to translate MICRO SIGN (`µ`) to Greek lowercase mu (`μ`). + +The normalization avoids confusability and the mixed-script detection is modified to the extent that the normalized codepoint is given the union of scriptsets from both characters. + + * For instance, in the example of MICRO => MU, MICRO was a 'Common'-script character - the same script given to the '_' underscore codepoint - and thus the normalized character's scriptset will be {Greek, Common}. 'Common' intersects with all non-empty scriptsets, and thus the normalized character can be used in tokens written in any script without causing script mixing. + + * The code points normalized in this fashion are those that are in use in the community, and judged not likely to cause issues with unsafe script mixing. For instance, the MICRO or MU codepoint may be used in an atom or variable dealing with microseconds. diff --git a/lib/elixir/rebar.config b/lib/elixir/rebar.config deleted file mode 100644 index 902a412104c..00000000000 --- a/lib/elixir/rebar.config +++ /dev/null @@ -1,23 +0,0 @@ -{erl_opts, [ - warn_unused_vars, - warn_export_all, - warn_shadow_vars, - warn_unused_import, - warn_unused_function, - warn_bif_clash, - warn_unused_record, - warn_deprecated_function, - warn_obsolete_guard, - strict_validation, - warn_exported_vars, - %% warn_export_vars, - %% warn_missing_spec, - %% warn_untyped_record, - %% warnings_as_errors, - debug_info - ]}. - -{yrl_opts, [ - {report, true}, - {verbose, false} - ]}. diff --git a/lib/elixir/scripts/cover.exs b/lib/elixir/scripts/cover.exs new file mode 100755 index 00000000000..a00973dd8f8 --- /dev/null +++ b/lib/elixir/scripts/cover.exs @@ -0,0 +1,30 @@ +#!bin/elixir + +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team + +Code.require_file("cover_record.exs", __DIR__) +cover_pid = CoverageRecorder.enable_coverage() + +coverdata_inputs = + CoverageRecorder.cover_dir() |> Path.join("ex_unit_*.coverdata") |> Path.wildcard() + +coverdata_output = Path.join(CoverageRecorder.cover_dir(), "combined.coverdata") + +for file <- coverdata_inputs do + :ok = :cover.import(String.to_charlist(file)) +end + +:ok = :cover.export(String.to_charlist(coverdata_output)) + +{:ok, _} = Application.ensure_all_started(:mix) + +# Silence analyse import messages emitted by cover +{:ok, string_io} = StringIO.open("") +Process.group_leader(cover_pid, string_io) + +:ok = + Mix.Tasks.Test.Coverage.generate_cover_results( + output: CoverageRecorder.cover_dir(), + summary: [threshold: 0] + ) diff --git a/lib/elixir/scripts/cover_record.exs b/lib/elixir/scripts/cover_record.exs new file mode 100644 index 00000000000..ffe78f3ed85 --- /dev/null +++ b/lib/elixir/scripts/cover_record.exs @@ -0,0 +1,91 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team + +defmodule CoverageRecorder do + def maybe_record(suite_name) do + if enabled?() do + record(suite_name) + + true + else + false + end + end + + def enable_coverage do + _ = :cover.stop() + {:ok, pid} = :cover.start() + + cover_compile_ebins() + + pid + end + + def cover_dir, do: Path.join(root_dir(), "cover") + + defp enabled? do + case System.fetch_env("COVER") do + {:ok, truthy} when truthy in ~w[1 true yes y] -> + true + + _ -> + false + end + end + + defp root_dir, do: Path.join(__DIR__, "../../..") + defp ebins, do: root_dir() |> Path.join("lib/*/ebin") |> Path.wildcard() + + defp record(suite_name) do + file = Path.join(cover_dir(), "ex_unit_#{suite_name}.coverdata") + + enable_coverage() + + System.at_exit(fn _status -> + File.mkdir_p!(cover_dir()) + + :ok = :cover.export(String.to_charlist(file)) + end) + end + + defp cover_compile_ebins do + relevant_beam_files() + |> Enum.map(&String.to_charlist/1) + |> :cover.compile_beam() + |> Enum.each(fn + {:ok, _module} -> + :ok + + {:error, reason} -> + raise "Failed to cover compile with reason: #{inspect(reason)}" + end) + end + + defp relevant_beam_files do + ebins() + |> Enum.flat_map(fn ebin -> + ebin |> Path.join("*.beam") |> Path.wildcard() + end) + |> Enum.reject(&skip_from_coverage?/1) + end + + @to_skip [ + # Tested via the CLI only + :elixir_sup, + :iex, + Kernel.CLI, + Mix.CLI, + Mix.Compilers.Test, + Mix.Tasks.Test, + Mix.Tasks.Test.Coverage, + + # Bootstrap + :elixir_bootstrap, + Kernel.SpecialForms + ] + + defp skip_from_coverage?(file) do + mod = file |> Path.basename(".beam") |> String.to_atom() + mod in @to_skip or match?({:docs_v1, _, _, _, _, %{deprecated: _}, _}, Code.fetch_docs(mod)) + end +end diff --git a/lib/elixir/scripts/diff.exs b/lib/elixir/scripts/diff.exs new file mode 100644 index 00000000000..5a12794a340 --- /dev/null +++ b/lib/elixir/scripts/diff.exs @@ -0,0 +1,193 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +defmodule Diff do + @moduledoc """ + Utilities for comparing build artifacts. + """ + + @atom_chunks ~w( + atoms + attributes + compile_info + debug_info + exports + labeled_exports + imports + indexed_imports + locals + labeled_locals + )a + + @binary_chunks ~w( + Attr + AtU8 + CInf + Dbgi + Docs + ExCk + ExpT + ImpT + LocT + )c + + @doc """ + Compares the build artifacts of two build directories. + """ + @spec compare_dirs(Path.t(), Path.t()) :: + { + only1_paths :: list(Path.t()), + only2_paths :: list(Path.t()), + diff :: list({Path.t(), diff :: String.t()}) + } + def compare_dirs(dir1, dir2) do + dir1 = Path.expand(dir1) + dir2 = Path.expand(dir2) + + assert_dir!(dir1) + assert_dir!(dir2) + + dir1_paths = relative_paths(dir1) + dir2_paths = relative_paths(dir2) + + only1_paths = dir1_paths -- dir2_paths + only2_paths = dir2_paths -- dir1_paths + common_paths = dir1_paths -- only1_paths + common_files = Enum.reject(common_paths, &File.dir?/1) + + diff = + Enum.flat_map(common_files, fn path -> + file1 = Path.join(dir1, path) + file2 = Path.join(dir2, path) + + case compare_files(file1, file2) do + :eq -> [] + {:diff, diff} -> [{path, diff}] + end + end) + + {only1_paths, only2_paths, diff} + end + + @doc """ + Compares the contents of two files. + + If the files are BEAM files, it performs a more human-friendly + "BEAM-diff". + """ + @spec compare_files(Path.t(), Path.t()) :: :eq | {:diff, diff :: String.t()} + def compare_files(file1, file2) do + content1 = File.read!(file1) + content2 = File.read!(file2) + + if content1 == content2 do + :eq + else + diff = + if String.ends_with?(file1, ".beam") do + beam_diff(file1, content1, file2, content2) + else + file_diff(file1, file2) + end + + {:diff, diff} + end + end + + defp beam_diff(file1, content1, file2, content2) do + chunk_diff(content1, content2, @atom_chunks, &inspect(&1, pretty: true, limit: :infinity)) || + chunk_diff(content1, content2, @binary_chunks, &(&1 |> write_tmp() |> xxd_dump())) || + ( + tmp_file1 = + file1 + |> xxd_dump() + |> write_tmp() + + tmp_file2 = + file2 + |> xxd_dump() + |> write_tmp() + + file_diff(tmp_file1, tmp_file2) + ) + end + + defp chunk_diff(content1, content2, names, formatter) do + with {:ok, {module, chunks1}} <- :beam_lib.chunks(content1, names), + {:ok, {^module, chunks2}} <- :beam_lib.chunks(content2, names), + true <- chunks1 != chunks2 do + if length(chunks1) != length(chunks2) do + """ + Different chunks: + * #{inspect(chunks1)} + * #{inspect(chunks2)} + """ + else + for {{name1, chunk1}, {name2, chunk2}} <- Enum.zip(chunks1, chunks2), + true = name1 == name2, + chunk1 != chunk2 do + tmp_file1 = chunk1 |> formatter.() |> write_tmp() + tmp_file2 = chunk2 |> formatter.() |> write_tmp() + [to_string(name1), ?\n, file_diff(tmp_file1, tmp_file2)] + end + end + else + _ -> nil + end + end + + defp xxd_dump(file) do + {dump, _} = System.cmd("xxd", [file]) + dump + end + + defp file_diff(file1, file2) do + {diff, _} = System.cmd("diff", ["--suppress-common-lines", file1, file2]) + diff + end + + defp relative_paths(dir) do + dir + |> Path.join("**") + |> Path.wildcard() + |> Enum.map(&Path.relative_to(&1, dir)) + end + + defp assert_dir!(dir) do + if not File.dir?(dir) do + raise ArgumentError, "#{inspect(dir)} is not a directory" + end + end + + defp write_tmp(content) do + filename = "tmp-#{System.unique_integer([:positive])}" + File.mkdir_p!("tmp") + File.write!(Path.join("tmp", filename), content) + Path.join("tmp", filename) + end +end + +if :deterministic not in :compile.env_compiler_options() do + IO.puts("Cannot validate if reproducible without setting ERL_COMPILER_OPTIONS=deterministic") + System.halt(1) +end + +case System.argv() do + [dir1, dir2] -> + case Diff.compare_dirs(dir1, dir2) do + {[], [], []} -> + IO.puts("#{inspect(dir1)} and #{inspect(dir2)} are equal") + + {only1, only2, diff} -> + for path <- only1, do: IO.puts("Only in #{dir1}: #{path}") + for path <- only2, do: IO.puts("Only in #{dir2}: #{path}") + for {path, diff} <- diff, do: IO.puts("Diff #{path}:\n#{diff}") + + System.halt(1) + end + + _ -> + IO.puts("Please, provide two directories as arguments") + System.halt(1) +end diff --git a/lib/elixir/scripts/docs_config.exs b/lib/elixir/scripts/docs_config.exs new file mode 100644 index 00000000000..4e9a35254ff --- /dev/null +++ b/lib/elixir/scripts/docs_config.exs @@ -0,0 +1,64 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team + +# Generate docs_config.js for version chooser in ExDoc +[app] = System.argv() +skipped = Version.parse!("1.0.3") +root_dir = Path.expand("../../../", __DIR__) + +git_repo? = + root_dir + |> Path.join(".git") + |> File.dir?() + +versions = + if git_repo? do + {text_tags, 0} = System.cmd("git", ["tag"]) + + for( + "v" <> rest <- String.split(text_tags), + not String.ends_with?(rest, "-latest"), + version = Version.parse!(rest), + Version.compare(version, skipped) == :gt, + do: version + ) + |> Enum.sort({:desc, Version}) + else + IO.warn("skipping version dropdown", []) + + [] + end + +latest = + if git_repo? do + versions + |> Stream.filter(&(&1.pre == [])) + |> Enum.fetch!(0) + |> Version.to_string() + else + System.version() + end + +version_nodes = + for version <- versions do + version_string = Version.to_string(version) + map = %{version: "v#{version_string}", url: "https://hexdocs.pm/#{app}/#{version_string}"} + + if version_string == latest do + Map.put(map, :latest, true) + else + map + end + end + +search_nodes = + for app <- ~w(eex elixir ex_unit iex logger mix)s do + %{name: app, version: latest} + end + +File.mkdir_p!("doc/#{app}") + +File.write!("doc/#{app}/docs_config.js", """ +var versionNodes = #{JSON.encode_to_iodata!(version_nodes)}; +var searchNodes = #{JSON.encode_to_iodata!(search_nodes)}; +""") diff --git a/lib/elixir/scripts/elixir_docs.exs b/lib/elixir/scripts/elixir_docs.exs new file mode 100644 index 00000000000..6caa60a4c46 --- /dev/null +++ b/lib/elixir/scripts/elixir_docs.exs @@ -0,0 +1,245 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +# Returns config for Elixir docs (exclusively) +canonical = System.fetch_env!("CANONICAL") + +[ + search: [ + %{ + name: "Elixir + libraries", + help: "Search Elixir, EEx, ExUnit, IEx, Logger, and Mix", + packages: + Enum.map( + [:elixir, :eex, :ex_unit, :iex, :logger, :mix], + &{&1, String.trim(canonical, "/")} + ) + }, + %{ + name: "Current project", + help: "Search only this project" + } + ], + assets: %{"lib/elixir/pages/images" => "assets"}, + extras: [ + "lib/elixir/pages/getting-started/introduction.md", + "lib/elixir/pages/getting-started/basic-types.md", + "lib/elixir/pages/getting-started/lists-and-tuples.md", + "lib/elixir/pages/getting-started/pattern-matching.md", + "lib/elixir/pages/getting-started/case-cond-and-if.md", + "lib/elixir/pages/getting-started/anonymous-functions.md", + "lib/elixir/pages/getting-started/binaries-strings-and-charlists.md", + "lib/elixir/pages/getting-started/keywords-and-maps.md", + "lib/elixir/pages/getting-started/modules-and-functions.md", + "lib/elixir/pages/getting-started/alias-require-and-import.md", + "lib/elixir/pages/getting-started/module-attributes.md", + "lib/elixir/pages/getting-started/structs.md", + "lib/elixir/pages/getting-started/recursion.md", + "lib/elixir/pages/getting-started/enumerable-and-streams.md", + "lib/elixir/pages/getting-started/comprehensions.md", + "lib/elixir/pages/getting-started/protocols.md", + "lib/elixir/pages/getting-started/sigils.md", + "lib/elixir/pages/getting-started/try-catch-and-rescue.md", + "lib/elixir/pages/getting-started/processes.md", + "lib/elixir/pages/getting-started/io-and-the-file-system.md", + "lib/elixir/pages/getting-started/writing-documentation.md", + "lib/elixir/pages/getting-started/optional-syntax.md", + "lib/elixir/pages/getting-started/erlang-libraries.md", + "lib/elixir/pages/getting-started/debugging.md", + "lib/elixir/pages/cheatsheets/enum-cheat.cheatmd", + "lib/elixir/pages/cheatsheets/types-cheat.cheatmd", + "lib/elixir/pages/anti-patterns/what-anti-patterns.md", + "lib/elixir/pages/anti-patterns/code-anti-patterns.md", + "lib/elixir/pages/anti-patterns/design-anti-patterns.md", + "lib/elixir/pages/anti-patterns/process-anti-patterns.md", + "lib/elixir/pages/anti-patterns/macro-anti-patterns.md", + "lib/elixir/pages/references/compatibility-and-deprecations.md", + "lib/elixir/pages/references/gradual-set-theoretic-types.md", + "lib/elixir/pages/references/library-guidelines.md", + "lib/elixir/pages/references/naming-conventions.md", + "lib/elixir/pages/references/operators.md", + "lib/elixir/pages/references/patterns-and-guards.md", + "lib/elixir/pages/references/syntax-reference.md", + "lib/elixir/pages/references/typespecs.md", + "lib/elixir/pages/references/unicode-syntax.md", + "lib/elixir/pages/mix-and-otp/introduction-to-mix.md", + "lib/elixir/pages/mix-and-otp/agents.md", + "lib/elixir/pages/mix-and-otp/supervisor-and-application.md", + "lib/elixir/pages/mix-and-otp/dynamic-supervisor.md", + "lib/elixir/pages/mix-and-otp/task-and-gen-tcp.md", + "lib/elixir/pages/mix-and-otp/docs-tests-and-with.md", + "lib/elixir/pages/mix-and-otp/config-and-distribution.md", + "lib/elixir/pages/mix-and-otp/genservers.md", + "lib/elixir/pages/mix-and-otp/releases.md", + "lib/elixir/pages/meta-programming/quote-and-unquote.md", + "lib/elixir/pages/meta-programming/macros.md", + "lib/elixir/pages/meta-programming/domain-specific-languages.md", + "CHANGELOG.md" + ], + deps: [ + eex: "https://hexdocs.pm/eex/#{canonical}", + ex_unit: "https://hexdocs.pm/ex_unit/#{canonical}", + iex: "https://hexdocs.pm/iex/#{canonical}", + logger: "https://hexdocs.pm/logger/#{canonical}", + mix: "https://hexdocs.pm/mix/#{canonical}" + ], + groups_for_extras: [ + "Getting started": ~r"pages/getting-started/.*\.md$", + Cheatsheets: ~r"pages/cheatsheets/.*\.cheatmd$", + "Mix & OTP": ~r"pages/mix-and-otp/.*\.md$", + "Anti-patterns": ~r"pages/anti-patterns/.*\.md$", + "Meta-programming": ~r"pages/meta-programming/.*\.md$", + References: ~r"pages/references/.*\.md$" + ], + groups_for_docs: [ + Guards: &(&1[:guard] == true) + ], + skip_undefined_reference_warnings_on: [ + "lib/elixir/pages/references/compatibility-and-deprecations.md" + ], + skip_code_autolink_to: [ + "Enumerable.List", + "Inspect.MapSet" + ], + formatters: ["html", "epub"], + groups_for_modules: [ + # [Kernel, Kernel.SpecialForms], + + "Data Types": [ + Atom, + Base, + Bitwise, + Date, + DateTime, + Duration, + Exception, + Float, + Function, + Integer, + JSON, + Module, + NaiveDateTime, + Record, + Regex, + String, + Time, + Tuple, + URI, + Version, + Version.Requirement + ], + "Collections & Enumerables": [ + Access, + Date.Range, + Enum, + Keyword, + List, + Map, + MapSet, + Range, + Stream + ], + "IO & System": [ + File, + File.Stat, + File.Stream, + IO, + IO.ANSI, + IO.Stream, + OptionParser, + Path, + Port, + StringIO, + System + ], + Calendar: [ + Calendar, + Calendar.ISO, + Calendar.TimeZoneDatabase, + Calendar.UTCOnlyTimeZoneDatabase + ], + "Processes & Applications": [ + Agent, + Application, + Config, + Config.Provider, + Config.Reader, + DynamicSupervisor, + GenServer, + Node, + PartitionSupervisor, + Process, + Registry, + Supervisor, + Task, + Task.Supervisor + ], + Protocols: [ + Collectable, + Enumerable, + JSON.Encoder, + Inspect, + Inspect.Algebra, + Inspect.Opts, + List.Chars, + Protocol, + String.Chars + ], + "Code & Macros": [ + Code, + Code.Fragment, + Kernel.ParallelCompiler, + Macro, + Macro.Env + ] + + ## Automatically detected groups + + # Deprecated: [ + # Behaviour, + # Dict, + # GenEvent, + # HashDict, + # HashSet, + # Set, + # Supervisor.Spec + # ] + ], + before_closing_body_tag: fn + :html -> + """ + + + """ + + _ -> + "" + end +] diff --git a/lib/elixir/scripts/generate_app.escript b/lib/elixir/scripts/generate_app.escript new file mode 100755 index 00000000000..8959fff4c25 --- /dev/null +++ b/lib/elixir/scripts/generate_app.escript @@ -0,0 +1,20 @@ +#!/usr/bin/env escript + +%% SPDX-License-Identifier: Apache-2.0 +%% SPDX-FileCopyrightText: 2021 The Elixir Team +%% SPDX-FileCopyrightText: 2012 Plataformatec + +%% -*- erlang -*- + +main([Version]) -> + Source = "lib/elixir/src/elixir.app.src", + Target = "lib/elixir/ebin/elixir.app", + {ok, [{application, Name, Props0}]} = file:consult(Source), + Ebin = filename:dirname(Target), + Files = filelib:wildcard(filename:join(Ebin, "*.beam")), + Mods = [list_to_atom(filename:basename(F, ".beam")) || F <- Files], + Props1 = lists:keyreplace(modules, 1, Props0, {modules, Mods}), + Props = lists:keyreplace(vsn, 1, Props1, {vsn, Version}), + AppDef = io_lib:format("~tp.~n", [{application, Name, Props}]), + ok = file:write_file(Target, AppDef), + io:format("Generated ~ts app~n", [Name]). diff --git a/lib/elixir/scripts/mix_docs.exs b/lib/elixir/scripts/mix_docs.exs new file mode 100644 index 00000000000..965153537e6 --- /dev/null +++ b/lib/elixir/scripts/mix_docs.exs @@ -0,0 +1,68 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team + +# Returns config for other apps except Elixir +canonical = System.fetch_env!("CANONICAL") + +[ + search: [ + %{ + name: "Elixir + libraries", + help: "Search Elixir, EEx, ExUnit, IEx, Logger, and Mix", + packages: + Enum.map( + [:elixir, :eex, :ex_unit, :iex, :logger, :mix], + &{&1, String.trim(canonical, "/")} + ) + }, + %{ + name: "Current project", + help: "Search only this project" + } + ], + deps: [ + eex: "https://hexdocs.pm/eex/#{canonical}", + elixir: "https://hexdocs.pm/elixir/#{canonical}", + ex_unit: "https://hexdocs.pm/ex_unit/#{canonical}", + iex: "https://hexdocs.pm/iex/#{canonical}", + logger: "https://hexdocs.pm/logger/#{canonical}", + mix: "https://hexdocs.pm/mix/#{canonical}" + ], + formatters: ["html", "epub"], + before_closing_body_tag: fn + :html -> + """ + + + """ + + _ -> + "" + end +] diff --git a/lib/elixir/scripts/windows_installer/.gitignore b/lib/elixir/scripts/windows_installer/.gitignore new file mode 100644 index 00000000000..b5914cb62a6 --- /dev/null +++ b/lib/elixir/scripts/windows_installer/.gitignore @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +tmp/ diff --git a/lib/elixir/scripts/windows_installer/assets/Elixir.ico b/lib/elixir/scripts/windows_installer/assets/Elixir.ico new file mode 100644 index 00000000000..b0ba3ebae48 Binary files /dev/null and b/lib/elixir/scripts/windows_installer/assets/Elixir.ico differ diff --git a/lib/elixir/scripts/windows_installer/build.sh b/lib/elixir/scripts/windows_installer/build.sh new file mode 100755 index 00000000000..eb31f5916fa --- /dev/null +++ b/lib/elixir/scripts/windows_installer/build.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team + +# Usage: +# +# With Elixir archive: +# +# ELIXIR_ZIP=Precompiled.zip OTP_VERSION=25.3.2.2 ./build.sh +set -euo pipefail + +mkdir -p tmp +rm -rf tmp/elixir +unzip -d "tmp/elixir" "${ELIXIR_ZIP}" + +elixir_version=`cat tmp/elixir/VERSION` +otp_release=`erl -noshell -eval 'io:put_chars(erlang:system_info(otp_release)), halt().'` +otp_version=`erl -noshell -eval '{ok, Vsn} = file:read_file(code:root_dir() ++ "/releases/" ++ erlang:system_info(otp_release) ++ "/OTP_VERSION"), io:put_chars(Vsn), halt().'` +elixir_exe=elixir-otp-${otp_release}.exe + +# brew install makensis +# apt install -y nsis +# choco install -y nsis +export PATH="/c/Program Files (x86)/NSIS:${PATH}" +makensis \ + -X"OutFile tmp\\${elixir_exe}" \ + -DOTP_RELEASE="${otp_release}" \ + -DOTP_VERSION=${otp_version} \ + -DELIXIR_DIR=tmp\\elixir \ + -DELIXIR_VERSION=${elixir_version} \ + installer.nsi + +echo "Installer path: tmp/${elixir_exe}" diff --git a/lib/elixir/scripts/windows_installer/installer.nsi b/lib/elixir/scripts/windows_installer/installer.nsi new file mode 100644 index 00000000000..b6f084c4a35 --- /dev/null +++ b/lib/elixir/scripts/windows_installer/installer.nsi @@ -0,0 +1,293 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +!include "MUI2.nsh" +!include "StrFunc.nsh" +${Using:StrFunc} UnStrStr + +Name "Elixir" +ManifestDPIAware true +Unicode True +InstallDir "$PROGRAMFILES64\Elixir" +!define MUI_ICON "assets\Elixir.ico" +!define MUI_UNICON "assets\Elixir.ico" + +; Install Page: Install Erlang/OTP + +Page custom CheckOTPPageShow CheckOTPPageLeave + +var InstalledOTPRelease +var OTPPath + +var Dialog +var NoOTPLabel +var NoOTPLabelCreated +var OTPMismatchLabel +var OTPMismatchLabelCreated +var DownloadOTPLink +var DownloadOTPLinkCreated +var VerifyOTPButton +var VerifyOTPButtonCreated +Function CheckOTPPageShow + !insertmacro MUI_HEADER_TEXT "Checking Erlang/OTP" "" + + nsDialogs::Create 1018 + Pop $Dialog + + ${If} $Dialog == error + Abort + ${EndIf} + + Call VerifyOTP + + nsDialogs::Show +FunctionEnd + +Function VerifyOTP + ${If} $NoOTPLabelCreated == "true" + ShowWindow $NoOTPLabel ${SW_HIDE} + ${EndIf} + + ${If} $OTPMismatchLabelCreated == "true" + ShowWindow $OTPMismatchLabel ${SW_HIDE} + ${EndIf} + + ${If} $DownloadOTPLinkCreated == "true" + ShowWindow $DownloadOTPLink ${SW_HIDE} + ${Else} + StrCpy $DownloadOTPLinkCreated "true" + ${NSD_CreateLink} 0 60u 100% 20u "Download Erlang/OTP ${OTP_RELEASE}" + Pop $DownloadOTPLink + ${NSD_OnClick} $DownloadOTPLink OpenOTPDownloads + ShowWindow $DownloadOTPLink ${SW_HIDE} + ${EndIf} + + ${If} $VerifyOTPButtonCreated == "true" + ShowWindow $VerifyOTPButton ${SW_HIDE} + ${Else} + StrCpy $VerifyOTPButtonCreated "true" + ${NSD_CreateButton} 0 80u 25% 12u "Verify Erlang/OTP" + Pop $VerifyOTPButton + ${NSD_OnClick} $VerifyOTPButton VerifyOTP + ShowWindow $VerifyOTPButton ${SW_HIDE} + ${EndIf} + + StrCpy $0 0 + loop: + EnumRegKey $1 HKLM "SOFTWARE\WOW6432NODE\Ericsson\Erlang" $0 + StrCmp $1 "" done + ReadRegStr $1 HKLM "SOFTWARE\WOW6432NODE\Ericsson\Erlang\$1" "" + StrCpy $OTPPath $1 + IntOp $0 $0 + 1 + goto loop + done: + + ${If} $OTPPath == "" + ${If} $NoOTPLabelCreated != "true" + StrCpy $NoOTPLabelCreated "true" + ${NSD_CreateLabel} 0 0 100% 20u "Couldn't find existing Erlang/OTP installation. Click the link below to download and install it before proceeding." + Pop $NoOTPLabel + ${EndIf} + + ShowWindow $NoOTPLabel ${SW_SHOW} + ShowWindow $DownloadOTPLink ${SW_SHOW} + ShowWindow $VerifyOTPButton ${SW_SHOW} + + ${Else} + nsExec::ExecToStack `$OTPPath\bin\erl.exe -noinput -eval "\ + io:put_chars(erlang:system_info(otp_release)),\ + halt()."` + Pop $0 + Pop $1 + + ${If} $0 == 0 + StrCpy $InstalledOTPRelease $1 + ${If} $InstalledOTPRelease == ${OTP_RELEASE} + ${NSD_CreateLabel} 0 0 100% 60u "Found existing Erlang/OTP $InstalledOTPRelease installation at $OTPPath. Please proceed." + + ${Else} + ${If} $OTPMismatchLabelCreated != "true" + StrCpy $OTPMismatchLabelCreated "true" + ${NSD_CreateLabel} 0 0 100% 60u "Found existing Erlang/OTP $InstalledOTPRelease installation at $OTPPath but this Elixir installer was precompiled for Erlang/OTP ${OTP_RELEASE}. \ + $\r$\n$\r$\nYou can either search for another Elixir installer precompiled for Erlang/OTP $InstalledOTPRelease or download Erlang/OTP ${OTP_RELEASE} and install before proceeding." + Pop $OTPMismatchLabel + ${EndIf} + + ShowWindow $OTPMismatchLabel ${SW_SHOW} + ShowWindow $DownloadOTPLink ${SW_SHOW} + ShowWindow $VerifyOTPButton ${SW_SHOW} + ${EndIf} + ${Else} + SetErrorlevel 5 + MessageBox MB_ICONSTOP "Found existing Erlang/OTP installation at $OTPPath but checking it exited with $0: $1" + ${EndIf} + ${EndIf} +FunctionEnd + +Function OpenOTPDownloads + ExecShell "open" "https://www.erlang.org/downloads/${OTP_RELEASE}" +FunctionEnd + +Function CheckOTPPageLeave +FunctionEnd + +; Install Page: Files + +!insertmacro MUI_PAGE_DIRECTORY +!insertmacro MUI_PAGE_INSTFILES + +; Install Page: Finish + +Page custom FinishPageShow FinishPageLeave + +var AddOTPToPathCheckbox +var AddElixirToPathCheckbox +Function FinishPageShow + !insertmacro MUI_HEADER_TEXT "Finish Setup" "" + + nsDialogs::Create 1018 + Pop $Dialog + + ${If} $Dialog == error + Abort + ${EndIf} + + ; we add to PATH using erlang, so there must be an OTP installed to do so. + ${If} "$OTPPath" != "" + ${NSD_CreateCheckbox} 0 0 195u 10u "&Add $INSTDIR\bin to %PATH%" + Pop $AddElixirToPathCheckbox + SendMessage $AddElixirToPathCheckbox ${BM_SETCHECK} ${BST_CHECKED} 0 + + ${NSD_CreateCheckbox} 0 20u 195u 10u "&Add $OTPPath\bin to %PATH%" + Pop $AddOTPToPathCheckbox + SendMessage $AddOTPToPathCheckbox ${BM_SETCHECK} ${BST_CHECKED} 0 + + ${NSD_CreateLabel} 0 40u 100% 20u "Note: you need to restart your shell for the environment variable changes to take effect." + ${EndIf} + + nsDialogs::Show +FunctionEnd + +var PathsToAdd +Function FinishPageLeave + ${NSD_GetState} $AddOTPToPathCheckbox $0 + ${If} $0 <> ${BST_UNCHECKED} + StrCpy $PathsToAdd ";$OTPPath\bin" + ${EndIf} + + ${NSD_GetState} $AddElixirToPathCheckbox $0 + ${If} $0 <> ${BST_UNCHECKED} + StrCpy $PathsToAdd "$PathsToAdd;$INSTDIR\bin" + ${EndIf} + + ${If} "$PathsToAdd" != "" + nsExec::ExecToStack `"$OTPPath\bin\escript.exe" "$INSTDIR\update_system_path.erl" add "$PathsToAdd"` + Pop $0 + Pop $1 + ${If} $0 != 0 + SetErrorlevel 5 + MessageBox MB_ICONSTOP "adding paths failed with $0: $1" + Quit + ${EndIf} + ${EndIf} +FunctionEnd + +Section "Install Elixir" SectionElixir + SetOutPath "$INSTDIR" + File /r "${ELIXIR_DIR}\" + File "assets\Elixir.ico" + File "update_system_path.erl" + WriteRegStr HKLM "Software\Microsoft\Windows\CurrentVersion\Uninstall\Elixir" "DisplayName" "Elixir" + WriteRegStr HKLM "Software\Microsoft\Windows\CurrentVersion\Uninstall\Elixir" "DisplayVersion" "${ELIXIR_VERSION}" + WriteRegStr HKLM "Software\Microsoft\Windows\CurrentVersion\Uninstall\Elixir" "DisplayIcon" "$INSTDIR\Elixir.ico" + WriteRegStr HKLM "Software\Microsoft\Windows\CurrentVersion\Uninstall\Elixir" "Publisher" "The Elixir Team" + WriteRegStr HKLM "Software\Microsoft\Windows\CurrentVersion\Uninstall\Elixir" "UninstallString" '"$INSTDIR\Uninstall.exe"' + WriteRegDWORD HKLM "Software\Microsoft\Windows\CurrentVersion\Uninstall\Elixir" "NoModify" 1 + WriteRegDWORD HKLM "Software\Microsoft\Windows\CurrentVersion\Uninstall\Elixir" "NoRepair" 1 + + WriteRegStr HKLM "Software\Elixir\Elixir" "InstallRoot" "$INSTDIR" + + WriteUninstaller "Uninstall.exe" +SectionEnd + +; Uninstall Page: Remove from %PATH% + +var RemoveOTPFromPathCheckbox +var RemoveElixirFromPathCheckbox +Function un.FinishPageShow + !insertmacro MUI_HEADER_TEXT "Remove from %PATH%" "" + + StrCpy $0 0 + loop: + EnumRegKey $1 HKLM "SOFTWARE\WOW6432NODE\Ericsson\Erlang" $0 + StrCmp $1 "" done + ReadRegStr $1 HKLM "SOFTWARE\WOW6432NODE\Ericsson\Erlang\$1" "" + StrCpy $OTPPath $1 + IntOp $0 $0 + 1 + goto loop + done: + + nsDialogs::Create 1018 + Pop $Dialog + + ${If} $Dialog == error + Abort + ${EndIf} + + ReadRegStr $0 HKCU "Environment" "Path" + + ${UnStrStr} $1 "$0" "$INSTDIR\bin" + ${If} $1 != "" + ${NSD_CreateCheckbox} 0 0 195u 10u "&Remove $INSTDIR\bin from %PATH%" + Pop $RemoveElixirFromPathCheckbox + SendMessage $RemoveElixirFromPathCheckbox ${BM_SETCHECK} ${BST_CHECKED} 0 + ${EndIf} + + ${UnStrStr} $1 "$0" "$OTPPath\bin" + ${If} $1 != "" + ${NSD_CreateCheckbox} 0 20u 195u 10u "&Remove $OTPPath\bin from %PATH%" + Pop $RemoveOTPFromPathCheckbox + SendMessage $RemoveOTPFromPathCheckbox ${BM_SETCHECK} ${BST_CHECKED} 0 + ${EndIf} + + nsDialogs::Show +FunctionEnd + +var PathsToRemove +Function un.FinishPageLeave + ${NSD_GetState} $RemoveOTPFromPathCheckbox $1 + ${If} $1 <> ${BST_UNCHECKED} + StrCpy $PathsToRemove ";$OTPPath\bin" + ${EndIf} + + ${NSD_GetState} $RemoveElixirFromPathCheckbox $1 + ${If} $1 <> ${BST_UNCHECKED} + StrCpy $PathsToRemove "$PathsToRemove;$INSTDIR\bin" + ${EndIf} + + ${If} "$PathsToRemove" != "" + nsExec::ExecToStack `"$OTPPath\bin\escript.exe" "$INSTDIR\update_system_path.erl" remove "$PathsToRemove"` + Pop $0 + Pop $1 + ${If} $0 != 0 + SetErrorlevel 5 + MessageBox MB_ICONSTOP "removing paths failed with $0: $1" + Quit + ${EndIf} + ${EndIf} +FunctionEnd + +UninstPage custom un.FinishPageShow un.FinishPageLeave + +!insertmacro MUI_UNPAGE_DIRECTORY +!insertmacro MUI_UNPAGE_INSTFILES + +Section "Uninstall" + RMDir /r "$INSTDIR" + DeleteRegKey HKLM "Software\Microsoft\Windows\CurrentVersion\Uninstall\Elixir" + DeleteRegKey HKLM "Software\Elixir\Elixir" + DeleteRegKey /ifempty HKLM "Software\Elixir" +SectionEnd + +!insertmacro MUI_LANGUAGE "English" diff --git a/lib/elixir/scripts/windows_installer/update_system_path.erl b/lib/elixir/scripts/windows_installer/update_system_path.erl new file mode 100644 index 00000000000..568f781bb8b --- /dev/null +++ b/lib/elixir/scripts/windows_installer/update_system_path.erl @@ -0,0 +1,45 @@ +#!/usr/bin/env escript +%% SPDX-License-Identifier: Apache-2.0 +%% SPDX-FileCopyrightText: 2021 The Elixir Team + +%%! -noinput + +%% This file is used by the Elixir installer and uninstaller. +main(["add", ";" ++ PathsToAdd]) -> + {ok, Reg} = win32reg:open([read, write]), + ok = win32reg:change_key(Reg, "\\hkey_current_user\\environment"), + {ok, SystemPath} = win32reg:value(Reg, "path"), + + NewSystemPath = + lists:foldl( + fun(Elem, Acc) -> + Elem ++ ";" ++ + binary_to_list( + iolist_to_binary( + string:replace(Acc, Elem ++ ";", "", all))) + end, + SystemPath, + string:split(PathsToAdd, ";", all) + ), + + ok = win32reg:set_value(Reg, "Path", NewSystemPath), + ok; + +main(["remove", ";" ++ PathsToRemove]) -> + {ok, Reg} = win32reg:open([read, write]), + ok = win32reg:change_key(Reg, "\\hkey_current_user\\environment"), + {ok, SystemPath} = win32reg:value(Reg, "path"), + + NewSystemPath = + lists:foldl( + fun(Elem, Acc) -> + binary_to_list( + iolist_to_binary( + string:replace(Acc, Elem ++ ";", "", all))) + end, + SystemPath, + string:split(PathsToRemove, ";", all) + ), + + ok = win32reg:set_value(Reg, "Path", NewSystemPath), + ok. diff --git a/lib/elixir/src/elixir.app.src b/lib/elixir/src/elixir.app.src new file mode 100644 index 00000000000..b7b5aaca5b8 --- /dev/null +++ b/lib/elixir/src/elixir.app.src @@ -0,0 +1,32 @@ +%% SPDX-License-Identifier: Apache-2.0 +%% SPDX-FileCopyrightText: 2021 The Elixir Team +%% SPDX-FileCopyrightText: 2012 Plataformatec + +{application, elixir, +[{description, "elixir"}, + {vsn, '$will-be-replaced'}, + {modules, '$will-be-replaced'}, + {registered, [elixir_sup, elixir_config, elixir_code_server]}, + {applications, [kernel,stdlib,compiler]}, + {mod, {elixir,[]}}, + {env, [ + {ansi_syntax_colors, [ + {atom, cyan}, + {binary, default_color}, + {boolean, magenta}, + {charlist, yellow}, + {list, default_color}, + {map, default_color}, + {nil, magenta}, + {number, yellow}, + {string, green}, + {tuple, default_color}, + {variable, light_cyan}, + {call, default_color}, + {operator, default_color} + ]}, + {check_endianness, true}, + {dbg_callback, {'Elixir.Macro', dbg, []}}, + {time_zone_database, 'Elixir.Calendar.UTCOnlyTimeZoneDatabase'} + ]} +]}. diff --git a/lib/elixir/src/elixir.erl b/lib/elixir/src/elixir.erl index a7548c56489..352c529dc61 100644 --- a/lib/elixir/src/elixir.erl +++ b/lib/elixir/src/elixir.erl @@ -1,221 +1,524 @@ +%% SPDX-License-Identifier: Apache-2.0 +%% SPDX-FileCopyrightText: 2021 The Elixir Team +%% SPDX-FileCopyrightText: 2012 Plataformatec + %% Main entry point for Elixir functions. All of those functions are %% private to the Elixir compiler and reserved to be used by Elixir only. -module(elixir). -behaviour(application). --export([main/1, start_cli/0, - string_to_quoted/4, 'string_to_quoted!'/4, - env_for_eval/1, env_for_eval/2, quoted_to_erl/2, quoted_to_erl/3, - eval/2, eval/3, eval_forms/3, eval_forms/4, eval_quoted/3]). +-export([start_cli/0, start/0]). +-export([start/2, stop/1, config_change/3]). +-export([ + string_to_tokens/5, tokens_to_quoted/3, 'string_to_quoted!'/5, + env_for_eval/1, quoted_to_erl/2, eval_forms/3, eval_quoted/3, + eval_quoted/4, eval_local_handler/2, eval_external_handler/3, + format_token_error/1 +]). -include("elixir.hrl"). +-define(system, 'Elixir.System'). +-define(elixir_eval_env, {elixir, eval_env}). %% Top level types --export_type([char_list/0, as_boolean/1]). +%% TODO: Remove char_list type on v2.0 +-export_type([charlist/0, char_list/0, nonempty_charlist/0, struct/0, as_boolean/1, keyword/0, keyword/1]). +-type charlist() :: string(). -type char_list() :: string(). +-type nonempty_charlist() :: nonempty_string(). -type as_boolean(T) :: T. +-type keyword() :: [{atom(), any()}]. +-type keyword(T) :: [{atom(), T}]. +-type struct() :: #{'__struct__' := atom(), atom() => any()}. %% OTP Application API --export([start/2, stop/1, config_change/3]). - start(_Type, _Args) -> - %% In case there is a shell, we can't really change its - %% encoding, so we just set binary to true. Otherwise - %% we must set the encoding as the user with no shell - %% has encoding set to latin1. - Opts = - case init:get_argument(noshell) of - {ok, _} -> [binary,{encoding,utf8}]; - error -> [binary] - end, - - io:setopts(standard_io, Opts), - io:setopts(standard_error, [{unicode,true}]), - case file:native_name_encoding() of - latin1 -> - io:format(standard_error, - "warning: the VM is running with native name encoding of latin1 which may cause " - "Elixir to malfunction as it expects utf8. Please ensure your locale is set to UTF-8 " - "(which can be verified by running \"locale\" in your shell)~n", []); + _OTP = parse_otp_release(), + preload_common_modules(), + ok = io:setopts(standard_io, [binary]), + check_file_encoding(file:native_name_encoding()), + + case init:get_argument(elixir_root) of + {ok, [[Root]]} -> + code:add_pathsa([ + Root ++ "/eex/ebin", + Root ++ "/ex_unit/ebin", + Root ++ "/iex/ebin", + Root ++ "/logger/ebin", + Root ++ "/mix/ebin", + Root ++ "/elixir/ebin" + ], cache); _ -> ok end, - elixir_sup:start_link(). + case application:get_env(elixir, check_endianness, true) of + true -> check_endianness(); + false -> ok + end, + + case application:get_env(elixir, ansi_enabled) of + {ok, _} -> + ok; -stop(_S) -> - ok. + undefined -> + application:set_env(elixir, ansi_enabled, prim_tty:isatty(stdout) == true) + end, + + Tokenizer = case code:ensure_loaded('Elixir.String.Tokenizer') of + {module, Mod} -> Mod; + _ -> elixir_tokenizer + end, + + URIConfig = [ + {{uri, <<"ftp">>}, 21}, + {{uri, <<"sftp">>}, 22}, + {{uri, <<"tftp">>}, 69}, + {{uri, <<"http">>}, 80}, + {{uri, <<"https">>}, 443}, + {{uri, <<"ldap">>}, 389}, + {{uri, <<"ws">>}, 80}, + {{uri, <<"wss">>}, 443} + ], + + Config = [ + %% ARGV options + {at_exit, []}, + {argv, []}, + {no_halt, false}, + + %% Compiler options + {docs, true}, + {ignore_already_consolidated, false}, + {ignore_module_conflict, false}, + {infer_signatures, [elixir]}, + {on_undefined_variable, raise}, + {parser_options, [{columns, true}]}, + {debug_info, true}, + {relative_paths, true}, + {no_warn_undefined, []}, + {tracers, []} + | URIConfig + ], + + elixir_config:static(#{bootstrap => false, identifier_tokenizer => Tokenizer}), + Tab = elixir_config:new(Config), + + case elixir_sup:start_link() of + {ok, Sup} -> + {ok, Sup, Tab}; + {error, _Reason} = Error -> + elixir_config:delete(Tab), + Error + end. + +stop(Tab) -> + elixir_config:delete(Tab). config_change(_Changed, _New, _Remove) -> ok. -%% escript entry point +preload_common_modules() -> + %% We attempt to load those modules here so throughout + %% the codebase we can avoid code:ensure_loaded/1 checks. + _ = code:ensure_loaded('Elixir.Kernel'), + _ = code:ensure_loaded('Elixir.Macro.Env'), + ok. -main(Args) -> - application:start(?MODULE), - 'Elixir.Kernel.CLI':main(Args). +parse_otp_release() -> + %% Whenever we change this check, we should also change Makefile. + case string:to_integer(erlang:system_info(otp_release)) of + {Num, _} when Num >= 26 -> + case Num == 28 andalso (code:ensure_loaded(re) == {module, re}) andalso not erlang:function_exported(re, import, 1) of + true -> + io:format(standard_error, + "warning! Erlang/OTP 28.0 detected.~n" + "Regexes will be re-compiled from source at runtime, which will cause degraded performance.~n" + "This can be fixed by using Erlang OTP 28.1+ or 27-.~n" + , []); + false -> + ok + end, + Num; + _ -> + io:format(standard_error, "ERROR! Unsupported Erlang/OTP version, expected Erlang/OTP 26+~n", []), + erlang:halt(1) + end. + +check_endianness() -> + case code:ensure_loaded(?system) of + {module, ?system} -> + Endianness = ?system:endianness(), + case ?system:compiled_endianness() of + Endianness -> + ok; + _ -> + io:format(standard_error, + "warning: Elixir is running in a system with a different endianness than the one its " + "source code was compiled in. Please make sure Elixir and all source files were compiled " + "in a machine with the same endianness as the current one: ~ts~n", [Endianness]) + end; + {error, _} -> + ok + end. + +check_file_encoding(Encoding) -> + case Encoding of + latin1 -> + io:format(standard_error, + "warning: the VM is running with native name encoding of latin1 which may cause " + "Elixir to malfunction as it expects utf8. Please ensure your locale is set to UTF-8 " + "(which can be verified by running \"locale\" in your shell) or set the " + "ELIXIR_ERL_OPTIONS=\"+fnu\" environment variable~n", []); + _ -> + ok + end. %% Boot and process given options. Invoked by Elixir's script. +start() -> + user_drv:start(#{initial_shell => iex:shell()}). + start_cli() -> - application:start(?MODULE), + {ok, _} = application:ensure_all_started(elixir), + + %% We start the Logger so tools that depend on Elixir + %% always have the Logger directly accessible. However + %% Logger is not a dependency of the Elixir application, + %% which means releases that want to use Logger must + %% always list it as part of its applications. + _ = case code:ensure_loaded('Elixir.Logger') of + {module, _} -> application:start(logger); + {error, _} -> ok + end, + 'Elixir.Kernel.CLI':main(init:get_plain_arguments()). %% EVAL HOOKS -env_for_eval(Opts) -> - env_for_eval((elixir_env:new())#{ - local := nil, - requires := elixir_dispatch:default_requires(), - functions := elixir_dispatch:default_functions(), - macros := elixir_dispatch:default_macros() - }, Opts). - -env_for_eval(Env, Opts) -> - Line = case lists:keyfind(line, 1, Opts) of - {line, LineOpt} when is_integer(LineOpt) -> LineOpt; - false -> ?m(Env, line) +env_for_eval(#{lexical_tracker := Pid} = Env) -> + NewEnv = Env#{ + context := nil, + macro_aliases := [], + versioned_vars := #{} + }, + + case is_pid(Pid) of + true -> + case is_process_alive(Pid) of + true -> + NewEnv; + false -> + 'Elixir.IO':warn( + <<"an __ENV__ with outdated compilation information was given to eval, " + "call Macro.Env.prune_compile_info/1 to prune it">> + ), + NewEnv#{lexical_tracker := nil, tracers := []} + end; + false -> + NewEnv#{tracers := []} + end; +env_for_eval(Opts) when is_list(Opts) -> + Env = elixir_env:new(), + Line = elixir_utils:get_line(Opts, Env), + File = elixir_utils:get_file(Opts, Env), + + Module = case lists:keyfind(module, 1, Opts) of + {module, ModuleOpt} when is_atom(ModuleOpt) -> ModuleOpt; + false -> nil end, - File = case lists:keyfind(file, 1, Opts) of - {file, FileOpt} when is_binary(FileOpt) -> FileOpt; - false -> ?m(Env, file) + FA = case lists:keyfind(function, 1, Opts) of + {function, {Function, Arity}} when is_atom(Function), is_integer(Arity) -> {Function, Arity}; + {function, nil} -> nil; + false -> nil end, - Local = case lists:keyfind(delegate_locals_to, 1, Opts) of - {delegate_locals_to, LocalOpt} when is_atom(LocalOpt) -> LocalOpt; - false -> ?m(Env, local) + TempTracers = case lists:keyfind(tracers, 1, Opts) of + {tracers, TracersOpt} when is_list(TracersOpt) -> TracersOpt; + false -> [] end, + %% TODO: Remove the following deprecations in future releases Aliases = case lists:keyfind(aliases, 1, Opts) of - {aliases, AliasesOpt} when is_list(AliasesOpt) -> AliasesOpt; - false -> ?m(Env, aliases) + {aliases, AliasesOpt} when is_list(AliasesOpt) -> + 'Elixir.IO':warn(<<":aliases option in eval is deprecated">>), + AliasesOpt; + false -> + ?key(Env, aliases) end, Requires = case lists:keyfind(requires, 1, Opts) of - {requires, RequiresOpt} when is_list(RequiresOpt) -> ordsets:from_list(RequiresOpt); - false -> ?m(Env, requires) + {requires, RequiresOpt} when is_list(RequiresOpt) -> + 'Elixir.IO':warn(<<":requires option in eval is deprecated">>), + ordsets:from_list(RequiresOpt); + false -> + ?key(Env, requires) end, Functions = case lists:keyfind(functions, 1, Opts) of - {functions, FunctionsOpt} when is_list(FunctionsOpt) -> FunctionsOpt; - false -> ?m(Env, functions) + {functions, FunctionsOpt} when is_list(FunctionsOpt) -> + 'Elixir.IO':warn(<<":functions option in eval is deprecated">>), + FunctionsOpt; + false -> + ?key(Env, functions) end, Macros = case lists:keyfind(macros, 1, Opts) of - {macros, MacrosOpt} when is_list(MacrosOpt) -> MacrosOpt; - false -> ?m(Env, macros) + {macros, MacrosOpt} when is_list(MacrosOpt) -> + 'Elixir.IO':warn(<<":macros option in eval is deprecated">>), + MacrosOpt; + false -> + ?key(Env, macros) end, - Module = case lists:keyfind(module, 1, Opts) of - {module, ModuleOpt} when is_atom(ModuleOpt) -> ModuleOpt; - false -> nil + %% If there is a dead PID or lexical tracker is nil, + %% we assume the tracers also cannot be (re)used. + {LexicalTracker, Tracers} = case lists:keyfind(lexical_tracker, 1, Opts) of + {lexical_tracker, Pid} when is_pid(Pid) -> + 'Elixir.IO':warn(<<":lexical_tracker option in eval is deprecated">>), + case is_process_alive(Pid) of + true -> {Pid, TempTracers}; + false -> {nil, []} + end; + {lexical_tracker, nil} -> + 'Elixir.IO':warn(<<":lexical_tracker option in eval is deprecated">>), + {nil, []}; + false -> + {nil, TempTracers} end, Env#{ - file := File, local := Local, module := Module, - macros := Macros, functions := Functions, + file := File, module := Module, function := FA, tracers := Tracers, + macros := Macros, functions := Functions, lexical_tracker := LexicalTracker, requires := Requires, aliases := Aliases, line := Line }. -%% String evaluation - -eval(String, Binding) -> - eval(String, Binding, []). - -eval(String, Binding, Opts) when is_list(Opts) -> - eval(String, Binding, env_for_eval(Opts)); -eval(String, Binding, #{line := Line, file := File} = E) when - is_list(String), is_list(Binding), is_integer(Line), is_binary(File) -> - Forms = 'string_to_quoted!'(String, Line, File, []), - eval_forms(Forms, Binding, E). - %% Quoted evaluation -eval_quoted(Tree, Binding, Opts) when is_list(Opts) -> - eval_quoted(Tree, Binding, env_for_eval(Opts)); -eval_quoted(Tree, Binding, #{line := Line} = E) -> - eval_forms(elixir_quote:linify(Line, Tree), Binding, E). - -%% Handle forms evaluation. The main difference to -%% to eval_quoted is that it does not linefy the given -%% args. - -eval_forms(Tree, Binding, Opts) when is_list(Opts) -> - eval_forms(Tree, Binding, env_for_eval(Opts)); -eval_forms(Tree, Binding, E) -> - eval_forms(Tree, Binding, E, elixir_env:env_to_scope(E)). -eval_forms(Tree, Binding, Env, Scope) -> - {ParsedBinding, ParsedScope} = elixir_scope:load_binding(Binding, Scope), - ParsedEnv = Env#{vars := [K || {K,_} <- ParsedScope#elixir_scope.vars]}, - {Erl, NewEnv, NewScope} = quoted_to_erl(Tree, ParsedEnv, ParsedScope), +eval_quoted(Tree, Binding, E) -> + eval_quoted(Tree, Binding, E, []). +eval_quoted(Tree, Binding, #{line := Line} = E, Opts) -> + eval_forms(elixir_quote:linify(Line, line, Tree), Binding, E, Opts). + +eval_forms(Tree, Binding, OrigE) -> + eval_forms(Tree, Binding, OrigE, []). +eval_forms(Tree, Binding, OrigE, Opts) -> + Prune = proplists:get_value(prune_binding, Opts, false), + {ExVars, ErlVars, ErlBinding} = elixir_erl_var:load_binding(Binding, Prune), + E = elixir_env:with_vars(OrigE, ExVars), + ExS = elixir_env:env_to_ex(E), + ErlS = elixir_erl_var:from_env(E, ErlVars), + {Erl, NewErlS, NewExS, NewE} = quoted_to_erl(Tree, ErlS, ExS, E), case Erl of - {atom, _, Atom} -> - {Atom, Binding, NewEnv, NewScope}; + {Literal, _, Value} when Literal == atom; Literal == float; Literal == integer -> + if + Prune -> {Value, [], NewE#{versioned_vars := #{}}}; + true -> {Value, Binding, NewE} + end; + _ -> - {value, Value, NewBinding} = erl_eval(Erl, ParsedBinding), - {Value, elixir_scope:dump_binding(NewBinding, NewScope), NewEnv, NewScope} + Exprs = + case Erl of + {block, _, BlockExprs} -> BlockExprs; + _ -> [Erl] + end, + + %% We use remote names so eval works across Elixir versions. + LocalHandler = {value, fun ?MODULE:eval_local_handler/2}, + ExternalHandler = {value, fun ?MODULE:eval_external_handler/3}, + + {value, Value, NewBinding} = + try + %% ?elixir_eval_env is used by the external handler. + %% + %% The reason why we use the process dictionary to pass the environment + %% is because we want to avoid passing closures to erl_eval, as that + %% would effectively tie the eval code to the Elixir version and it is + %% best if it depends solely on Erlang/OTP. + %% + %% The downside is that functions that escape the eval context will no + %% longer have the original environment they came from. + erlang:put(?elixir_eval_env, NewE), + erl_eval:exprs(Exprs, ErlBinding, LocalHandler, ExternalHandler) + after + erlang:erase(?elixir_eval_env) + end, + + PruneBefore = if Prune -> length(Binding); true -> -1 end, + + {DumpedBinding, DumpedVars} = + elixir_erl_var:dump_binding(NewBinding, NewErlS, NewExS, PruneBefore), + + {Value, DumpedBinding, NewE#{versioned_vars := DumpedVars}} end. -erl_eval(Erl, ParsedBinding) -> - % Below must be all one line for locations to be the same when the stacktrace - % needs to be extended to the full stacktrace. - try erl_eval:expr(Erl, ParsedBinding) catch Class:Exception -> erlang:raise(Class, Exception, get_stacktrace()) end. +eval_local_handler(FunName, Args) -> + {current_stacktrace, Stack} = erlang:process_info(self(), current_stacktrace), + Opts = [{module, nil}, {function, FunName}, {arity, length(Args)}, {reason, 'undefined local'}], + Exception = 'Elixir.UndefinedFunctionError':exception(Opts), + erlang:raise(error, Exception, Stack). -get_stacktrace() -> - Stacktrace = erlang:get_stacktrace(), - % eval_eval and eval_bits can call :erlang.raise/3 without the full - % stacktrace. When this occurs re-add the current stacktrace so that no - % stack information is lost. +eval_external_handler(Ann, FunOrModFun, Args) -> try - throw(stack) + case FunOrModFun of + {Mod, Fun} -> apply(Mod, Fun, Args); + Fun -> apply(Fun, Args) + end catch - throw:stack -> - % Ignore stack item for current function. - [_ | CurrentStack] = erlang:get_stacktrace(), - get_stacktrace(Stacktrace, CurrentStack) + Kind:Reason:Stacktrace -> + %% Take everything up to the Elixir module + Pruned = + lists:takewhile(fun + ({elixir,_,_,_}) -> false; + (_) -> true + end, Stacktrace), + + Caller = + lists:dropwhile(fun + ({elixir,_,_,_}) -> false; + (_) -> true + end, Stacktrace), + + %% Now we prune any shared code path from erl_eval + {current_stacktrace, Current} = + erlang:process_info(self(), current_stacktrace), + + %% We need to make sure that we don't generate more + %% frames than supported. So we do our best to drop + %% from the Caller, but if the caller has no frames, + %% we need to drop from Pruned. + {DroppedCaller, ToDrop} = + case Caller of + [] -> {[], true}; + _ -> {lists:droplast(Caller), false} + end, + + Reversed = drop_common(lists:reverse(Current), lists:reverse(Pruned), ToDrop), + + %% Add file+line information at the bottom + Bottom = + case erlang:get(?elixir_eval_env) of + #{file := File} -> + [{elixir_eval, '__FILE__', 1, + [{file, elixir_utils:characters_to_list(File)}, {line, erl_anno:line(Ann)}]}]; + + _ -> + [] + end, + + Custom = lists:reverse(Bottom ++ Reversed, DroppedCaller), + erlang:raise(Kind, Reason, Custom) end. -% The stacktrace did not include the current stack, re-add it. -get_stacktrace([], CurrentStack) -> - CurrentStack; -% The stacktrace includes the current stack. -get_stacktrace(CurrentStack, CurrentStack) -> - CurrentStack; -get_stacktrace([StackItem | Stacktrace], CurrentStack) -> - [StackItem | get_stacktrace(Stacktrace, CurrentStack)]. - -%% Converts a quoted expression to erlang abstract format - -quoted_to_erl(Quoted, Env) -> - quoted_to_erl(Quoted, Env, elixir_env:env_to_scope(Env)). - -quoted_to_erl(Quoted, Env, Scope) -> - {Expanded, NewEnv} = elixir_exp:expand(Quoted, Env), - {Erl, NewScope} = elixir_translator:translate(Expanded, Scope), - {Erl, NewEnv, NewScope}. - -%% Converts a given string (char list) into quote expression - -string_to_quoted(String, StartLine, File, Opts) when is_integer(StartLine), is_binary(File) -> - case elixir_tokenizer:tokenize(String, StartLine, [{file, File}|Opts]) of - {ok, _Line, Tokens} -> - try elixir_parser:parse(Tokens) of - {ok, Forms} -> {ok, Forms}; - {error, {Line, _, [Error, Token]}} -> {error, {Line, to_binary(Error), to_binary(Token)}} - catch - {error, {Line, _, [Error, Token]}} -> {error, {Line, to_binary(Error), to_binary(Token)}} - end; - {error, {Line, Error, Token}, _Rest, _SoFar} -> {error, {Line, to_binary(Error), to_binary(Token)}} +%% We need to check if we have dropped any frames. +%% If we have not dropped frames, then we need to drop one +%% at the end so we can put the elixir_eval frame in. If +%% we have more traces then depth, Erlang would discard +%% the whole stacktrace. +drop_common([H | T1], [H | T2], _ToDrop) -> drop_common(T1, T2, false); +drop_common([_ | T1], T2, ToDrop) -> drop_common(T1, T2, ToDrop); +drop_common([], [{?MODULE, _, _, _} | T2], _ToDrop) -> T2; +drop_common([], [_ | T2], true) -> T2; +drop_common([], T2, _) -> T2. + +%% Converts a quoted expression to Erlang abstract format + +quoted_to_erl(Quoted, E) -> + {_, ErlS} = elixir_erl_var:from_env(E), + ExS = elixir_env:env_to_ex(E), + quoted_to_erl(Quoted, ErlS, ExS, E). + +quoted_to_erl(Quoted, ErlS, ExS, Env) -> + {Expanded, NewExS, NewEnv} = + elixir_expand:expand(Quoted, ExS, Env), + {Erl, NewErlS} = elixir_erl_pass:translate(Expanded, erl_anno:new(?key(Env, line)), ErlS), + {Erl, NewErlS, NewExS, NewEnv}. + +%% Converts a given string (charlist) into quote expression + +string_to_tokens(String, StartLine, StartColumn, File, Opts) when is_integer(StartLine), is_binary(File) -> + case elixir_tokenizer:tokenize(String, StartLine, StartColumn, Opts) of + {ok, _Line, _Column, [], Tokens, []} -> + {ok, lists:reverse(Tokens)}; + {ok, _Line, _Column, Warnings, Tokens, Terminators} -> + (lists:keyfind(emit_warnings, 1, Opts) /= {emit_warnings, false}) andalso + [elixir_errors:erl_warn(L, File, M) || {L, M} <- lists:reverse(Warnings)], + {ok, lists:reverse(Tokens, Terminators)}; + {error, Info, _Rest, _Warnings, _SoFar} -> + {error, format_token_error(Info)} end. -'string_to_quoted!'(String, StartLine, File, Opts) -> - case string_to_quoted(String, StartLine, File, Opts) of +format_token_error({Location, {ErrorPrefix, ErrorSuffix}, Token}) -> + {Location, {to_binary(ErrorPrefix), to_binary(ErrorSuffix)}, to_binary(Token)}; +format_token_error({Location, Error, Token}) -> + {Location, to_binary(Error), to_binary(Token)}. + +tokens_to_quoted(Tokens, WarningFile, Opts) -> + handle_parsing_opts(WarningFile, Opts), + + try elixir_parser:parse(Tokens) of {ok, Forms} -> - Forms; - {error, {Line, Error, Token}} -> - elixir_errors:parse_error(Line, File, Error, Token) + {ok, Forms}; + {error, {Line, _, [{ErrorPrefix, ErrorSuffix}, Token]}} -> + {error, {parser_location(Line), {to_binary(ErrorPrefix), to_binary(ErrorSuffix)}, to_binary(Token)}}; + {error, {Line, _, [Error, Token]}} -> + {error, {parser_location(Line), to_binary(Error), to_binary(Token)}} + after + erase(elixir_parser_warning_file), + erase(elixir_parser_columns), + erase(elixir_token_metadata), + erase(elixir_literal_encoder) + end. + +parser_location({Line, Column, _}) -> + [{line, Line}, {column, Column}]; +parser_location(Meta) -> + Line = + case lists:keyfind(line, 1, Meta) of + {line, L} -> L; + false -> 0 + end, + + case lists:keyfind(column, 1, Meta) of + {column, C} -> [{line, Line}, {column, C}]; + false -> [{line, Line}] + end. + +'string_to_quoted!'(String, StartLine, StartColumn, File, Opts) -> + case string_to_tokens(String, StartLine, StartColumn, File, Opts) of + {ok, Tokens} -> + case tokens_to_quoted(Tokens, File, Opts) of + {ok, Forms} -> + Forms; + {error, {Meta, Error, Token}} -> + Indentation = proplists:get_value(indentation, Opts, 0), + Input = {String, StartLine, StartColumn, Indentation}, + elixir_errors:parse_error(Meta, File, Error, Token, Input) + end; + {error, {Meta, Error, Token}} -> + Indentation = proplists:get_value(indentation, Opts, 0), + Input = {String, StartLine, StartColumn, Indentation}, + elixir_errors:parse_error(Meta, File, Error, Token, Input) end. to_binary(List) when is_list(List) -> elixir_utils:characters_to_binary(List); -to_binary(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8). +to_binary(Atom) when is_atom(Atom) -> atom_to_binary(Atom). + +handle_parsing_opts(File, Opts) -> + WarningFile = + case lists:keyfind(emit_warnings, 1, Opts) of + {emit_warnings, false} -> nil; + _ -> File + end, + LiteralEncoder = + case lists:keyfind(literal_encoder, 1, Opts) of + {literal_encoder, Fun} -> Fun; + false -> false + end, + TokenMetadata = lists:keyfind(token_metadata, 1, Opts) == {token_metadata, true}, + Columns = lists:keyfind(columns, 1, Opts) == {columns, true}, + put(elixir_parser_warning_file, WarningFile), + put(elixir_parser_columns, Columns), + put(elixir_token_metadata, TokenMetadata), + put(elixir_literal_encoder, LiteralEncoder). diff --git a/lib/elixir/src/elixir.hrl b/lib/elixir/src/elixir.hrl new file mode 100644 index 00000000000..eee33351eaf --- /dev/null +++ b/lib/elixir/src/elixir.hrl @@ -0,0 +1,81 @@ +%% SPDX-License-Identifier: Apache-2.0 +%% SPDX-FileCopyrightText: 2021 The Elixir Team +%% SPDX-FileCopyrightText: 2012 Plataformatec + +-define(key(M, K), map_get(K, M)). +-define(ann(Meta), elixir_erl:get_ann(Meta)). +-define(line(Meta), elixir_utils:get_line(Meta)). +-define(generated(Meta), elixir_utils:generated(Meta)). +-define(var_context, ?MODULE). +-define(remote(Ann, Module, Function, Args), {call, Ann, {remote, Ann, {atom, Ann, Module}, {atom, Ann, Function}}, Args}). +-define(tracker, 'Elixir.Kernel.LexicalTracker'). + +-record(elixir_ex, { + %% Stores if __CALLER__ is allowed + caller=false, + %% Stores the variables available before a match. + %% May be one of: + %% + %% * {Read, Cycle :: #{}, Meta :: Counter | {bitsize, Original}} + %% * pin + %% * none. + %% + %% The cycle is used to detect cyclic dependencies between + %% variables in a match. + %% + %% The bitsize is used when dealing with bitstring modifiers, + %% as they allow guards but also support the pin operator. + prematch=none, + %% Stores if __STACKTRACE__ is allowed + stacktrace=false, + %% A map of unused vars and a version counter for vars + unused={#{}, 0}, + %% A list of modules defined in functions (runtime) + runtime_modules=[], + %% A tuple with maps of read and optional write current vars. + %% Read variables is all defined variables. Write variables + %% stores the variables that have been made available (written + %% to) but cannot be currently read. This is used in two occasions: + %% + %% * To store variables graphs inside = in patterns + %% + %% * To store variables defined inside calls. For example, + %% if you write foo(a = 123), the value of `a` cannot be + %% read in the following argument, only after the call + %% + vars={#{}, false} +}). + +-record(elixir_erl, { + %% Can be match, guards or nil + context=nil, + %% Extra information about the context, like pin_guard and map_key + extra=nil, + %% When true, it means caller was invoked + caller=false, + %% Maps of defined variables and their alias + var_names=#{}, + %% Extra guards from args expansion + extra_guards=[], + %% A map counting the variables defined + counter=#{}, + %% A boolean to control if captures should be expanded + expand_captures=false, + %% Holds information about the stacktrace variable + stacktrace=nil +}). + +-record(elixir_tokenizer, { + terminators=[], + unescape=true, + cursor_completion=false, + existing_atoms_only=false, + static_atoms_encoder=nil, + preserve_comments=nil, + identifier_tokenizer=elixir_tokenizer, + ascii_identifiers_only=true, + indentation=0, + column=1, + mismatch_hints=[], + warnings=[] +}). diff --git a/lib/elixir/src/elixir_aliases.erl b/lib/elixir/src/elixir_aliases.erl index 8f06fffc8cb..ea480862bf0 100644 --- a/lib/elixir/src/elixir_aliases.erl +++ b/lib/elixir/src/elixir_aliases.erl @@ -1,101 +1,173 @@ +%% SPDX-License-Identifier: Apache-2.0 +%% SPDX-FileCopyrightText: 2021 The Elixir Team +%% SPDX-FileCopyrightText: 2012 Plataformatec + -module(elixir_aliases). --export([inspect/1, last/1, concat/1, safe_concat/1, format_error/1, - ensure_loaded/3, expand/4, store/7]). +-export([inspect/1, concat/1, safe_concat/1, format_error/1, + ensure_loaded/3, expand/4, expand_or_concat/4, alias/6, require/5]). -include("elixir.hrl"). inspect(Atom) when is_atom(Atom) -> - case elixir_compiler:get_opt(internal) of - true -> atom_to_binary(Atom, utf8); - false -> 'Elixir.Inspect.Atom':inspect(Atom) + case elixir_config:is_bootstrap() of + true -> atom_to_binary(Atom); + false -> 'Elixir.Macro':inspect_atom(literal, Atom) + end. + +require(Meta, Ref, Opts, E, Trace) -> + Trace andalso elixir_env:trace({require, Meta, Ref, Opts}, E), + E#{requires := ordsets:add_element(Ref, ?key(E, requires))}. + +alias(Meta, Ref, IncludeByDefault, Opts, E, Trace) -> + #{aliases := Aliases, macro_aliases := MacroAliases} = E, + + case expand_as(lists:keyfind(as, 1, Opts), IncludeByDefault, Ref) of + {ok, Ref} -> + {ok, false, + E#{aliases := remove_alias(Ref, Aliases), + macro_aliases := remove_macro_alias(Meta, Ref, MacroAliases)}}; + + {ok, New} -> + Trace andalso elixir_env:trace({alias, Meta, Ref, New, Opts}, E), + {ok, New, + E#{aliases := store_alias(New, Ref, Aliases), + macro_aliases := store_macro_alias(Meta, New, Ref, MacroAliases)}}; + + none -> + {ok, false, E}; + + {error, Reason} -> + {error, Reason} end. -%% Store an alias in the given scope -store(_Meta, New, New, _TKV, Aliases, MacroAliases, _Lexical) -> - {Aliases, MacroAliases}; -store(Meta, New, Old, TKV, Aliases, MacroAliases, Lexical) -> - record_warn(Meta, New, TKV, Lexical), - {store_alias(New, Old, Aliases), - store_macro_alias(Meta, New, Old, MacroAliases)}. +expand_as({as, Atom}, _IncludeByDefault, _Ref) when is_atom(Atom), not is_boolean(Atom) -> + case atom_to_list(Atom) of + "Elixir." ++ ([FirstLetter | _] = Rest) when FirstLetter >= $A, FirstLetter =< $Z -> + case string:tokens(Rest, ".") of + [_] -> + {ok, Atom}; + _ -> + {error, {invalid_alias_for_as, nested_alias, Atom}} + end; + _ -> + {error, {invalid_alias_for_as, not_alias, Atom}} + end; +expand_as({as, Other}, _IncludeByDefault, _Ref) -> + {error, {invalid_alias_for_as, not_alias, Other}}; +expand_as(false, true, Ref) -> + case atom_to_list(Ref) of + ("Elixir." ++ [FirstLetter | _]) = List when FirstLetter >= $A, FirstLetter =< $Z -> + Last = last(lists:reverse(List), []), + {ok, list_to_atom("Elixir." ++ Last)}; + _ -> + {error, {invalid_alias_module, Ref}} + end; +expand_as(false, false, _Ref) -> + none. + +last([$. | _], Acc) -> Acc; +last([H | T], Acc) -> last(T, [H | Acc]); +last([], Acc) -> Acc. store_alias(New, Old, Aliases) -> lists:keystore(New, 1, Aliases, {New, Old}). + store_macro_alias(Meta, New, Old, Aliases) -> - case lists:keymember(context, 1, Meta) andalso - lists:keyfind(counter, 1, Meta) of - {counter, Counter} when is_integer(Counter) -> + case lists:keyfind(counter, 1, Meta) of + {counter, Counter} -> lists:keystore(New, 1, Aliases, {New, {Counter, Old}}); - _ -> + false -> Aliases end. -record_warn(Meta, Ref, Opts, Lexical) -> - Warn = - case lists:keyfind(warn, 1, Opts) of - {warn, false} -> false; - {warn, true} -> true; - false -> not lists:keymember(context, 1, Meta) - end, - elixir_lexical:record_alias(Ref, ?line(Meta), Warn, Lexical). +remove_alias(Atom, Aliases) -> + lists:keydelete(Atom, 1, Aliases). + +remove_macro_alias(Meta, Atom, Aliases) -> + case lists:keyfind(counter, 1, Meta) of + {counter, _Counter} -> + lists:keydelete(Atom, 1, Aliases); + false -> + Aliases + end. %% Expand an alias. It returns an atom (meaning that there %% was an expansion) or a list of atoms. -expand({'__aliases__', _Meta, ['Elixir'|_] = List}, _Aliases, _MacroAliases, _LexicalTracker) -> - concat(List); +expand(_Meta, ['Elixir' | _] = List, _E, _Trace) -> + List; + +expand(_Meta, [H | _] = List, _E, _Trace) when not is_atom(H) -> + List; -expand({'__aliases__', Meta, _} = Alias, Aliases, MacroAliases, LexicalTracker) -> +expand(Meta, List, #{aliases := Aliases, macro_aliases := MacroAliases} = E, Trace) -> case lists:keyfind(alias, 1, Meta) of {alias, false} -> - expand(Alias, MacroAliases, LexicalTracker); + expand(Meta, List, MacroAliases, E, Trace); {alias, Atom} when is_atom(Atom) -> Atom; false -> - expand(Alias, Aliases, LexicalTracker) + expand(Meta, List, Aliases, E, Trace) end. -expand({'__aliases__', Meta, [H|T]}, Aliases, LexicalTracker) when is_atom(H) -> +expand(Meta, [H | T], Aliases, E, Trace) -> Lookup = list_to_atom("Elixir." ++ atom_to_list(H)), + Counter = case lists:keyfind(counter, 1, Meta) of {counter, C} -> C; _ -> nil end, + case lookup(Lookup, Aliases, Counter) of - Lookup -> [H|T]; + Lookup -> [H | T]; Atom -> - elixir_lexical:record_alias(Lookup, LexicalTracker), + Trace andalso elixir_env:trace({alias_expansion, Meta, Lookup, Atom}, E), case T of [] -> Atom; - _ -> concat([Atom|T]) + _ -> concat([Atom | T]) end - end; + end. + +%% Expands or concat if possible. -expand({'__aliases__', _Meta, List}, _Aliases, _LexicalTracker) -> - List. +expand_or_concat(Meta, List, E, Trace) -> + case expand(Meta, List, E, Trace) of + [H | T] when is_atom(H) -> concat([H | T]); + AtomOrList -> AtomOrList + end. %% Ensure a module is loaded before its usage. -ensure_loaded(_Meta, 'Elixir.Kernel', _E) -> ok; -ensure_loaded(Meta, Ref, E) -> - try - Ref:module_info(compile) - catch - error:undef -> - Kind = case lists:member(Ref, ?m(E, context_modules)) of - true -> scheduled_module; - false -> unloaded_module - end, - elixir_errors:form_error(Meta, ?m(E, file), ?MODULE, {Kind, Ref}) - end. +%% Skip Kernel verification for bootstrap purposes. +ensure_loaded(_Meta, 'Elixir.Kernel', _E) -> + ok; +ensure_loaded(Meta, Module, #{module := Module} = E) -> + elixir_errors:file_error(Meta, E, ?MODULE, {circular_module, Module}); +ensure_loaded(Meta, Module, E) -> + case code:ensure_loaded(Module) of + {module, Module} -> + ok; -%% Receives an atom and returns the last bit as an alias. + _ -> + case wait_for_module(Module, Meta, E) of + found -> + ok; -last(Atom) -> - Last = last(lists:reverse(atom_to_list(Atom)), []), - list_to_atom("Elixir." ++ Last). + Wait -> + Kind = case lists:member(Module, ?key(E, context_modules)) of + true -> scheduled_module; + false when Wait == deadlock -> deadlock_module; + false -> unloaded_module + end, -last([$.|_], Acc) -> Acc; -last([H|T], Acc) -> last(T, [H|Acc]); -last([], Acc) -> Acc. + elixir_errors:file_error(Meta, E, ?MODULE, {Kind, Module}) + end + end. + +wait_for_module(Module, Meta, E) -> + case erlang:get(elixir_compiler_info) of + undefined -> not_found; + _ -> 'Elixir.Kernel.ErrorHandler':ensure_compiled(Module, module, hard, elixir_utils:get_line(Meta, E)) + end. %% Receives a list of atoms, binaries or lists %% representing modules and concatenates them. @@ -103,20 +175,20 @@ last([], Acc) -> Acc. concat(Args) -> binary_to_atom(do_concat(Args), utf8). safe_concat(Args) -> binary_to_existing_atom(do_concat(Args), utf8). -do_concat([H|T]) when is_atom(H), H /= nil -> - do_concat([atom_to_binary(H, utf8)|T]); -do_concat([<<"Elixir.", _/binary>>=H|T]) -> +do_concat([H | T]) when is_atom(H), H /= nil -> + do_concat([atom_to_binary(H) | T]); +do_concat([<<"Elixir.", _/binary>>=H | T]) -> do_concat(T, H); -do_concat([<<"Elixir">>=H|T]) -> +do_concat([<<"Elixir">>=H | T]) -> do_concat(T, H); do_concat(T) -> do_concat(T, <<"Elixir">>). -do_concat([nil|T], Acc) -> +do_concat([nil | T], Acc) -> do_concat(T, Acc); -do_concat([H|T], Acc) when is_atom(H) -> - do_concat(T, <>); -do_concat([H|T], Acc) when is_binary(H) -> +do_concat([H | T], Acc) when is_atom(H) -> + do_concat(T, <>); +do_concat([H | T], Acc) when is_binary(H) -> do_concat(T, <>); do_concat([], Acc) -> Acc. @@ -127,18 +199,78 @@ to_partial(Arg) when is_binary(Arg) -> Arg. %% Lookup an alias in the current scope. -lookup(Else, Dict, Counter) -> - case lists:keyfind(Else, 1, Dict) of - {Else, {Counter, Value}} -> lookup(Value, Dict, Counter); - {Else, Value} when is_atom(Value) -> lookup(Value, Dict, Counter); +lookup(Else, List, Counter) -> + case lists:keyfind(Else, 1, List) of + {Else, {Counter, Value}} -> Value; + {Else, Value} when is_atom(Value) -> Value; _ -> Else end. %% Errors +format_error({invalid_alias_module, Ref}) -> + io_lib:format("alias cannot be inferred automatically for module: ~ts, please use the :as option. Implicit aliasing is only supported with Elixir modules", + ['Elixir.Macro':to_string(Ref)]); + +format_error({invalid_alias_for_as, Reason, Value}) -> + ExpectedGot = + case Reason of + not_alias -> "expected an alias, got"; + nested_alias -> "expected a simple alias, got nested alias" + end, + io_lib:format("invalid value for option :as, ~ts: ~ts", + [ExpectedGot, 'Elixir.Macro':to_string(Value)]); + format_error({unloaded_module, Module}) -> - io_lib:format("module ~ts is not loaded and could not be found", [elixir_aliases:inspect(Module)]); + io_lib:format("module ~ts is not loaded and could not be found", [inspect(Module)]); + +format_error({deadlock_module, Module}) -> + io_lib:format("module ~ts is not loaded and could not be found. " + "This may be happening because the module you are trying to load " + "directly or indirectly depends on the current module", + [inspect(Module)]); format_error({scheduled_module, Module}) -> - io_lib:format("module ~ts is not loaded but was defined. This happens because you are trying to use a module in the same context it is defined. Try defining the module outside the context that requires it.", - [inspect(Module)]). \ No newline at end of file + io_lib:format( + "module ~ts is not loaded but was defined. This happens when you depend on " + "a module in the same context in which it is defined. For example:\n" + "\n" + " defmodule MyApp do\n" + " defmodule Mod do\n" + " end\n" + "\n" + " use Mod\n" + " end\n" + "\n" + "Try defining the module outside the context that uses it:\n" + "\n" + " defmodule MyApp.Mod do\n" + " end\n" + "\n" + " defmodule MyApp do\n" + " use MyApp.Mod\n" + " end\n" + "\n" + "If the module is defined at the top-level and you are trying to " + "use it at the top-level, this is not supported by Elixir", + [inspect(Module)]); + +format_error({circular_module, Module}) -> + io_lib:format( + "you are trying to use/import/require the module ~ts which is currently being defined.\n" + "\n" + "This may happen if you accidentally override the module you want to use. For example:\n" + "\n" + " defmodule MyApp do\n" + " defmodule Supervisor do\n" + " use Supervisor\n" + " end\n" + " end\n" + "\n" + "In the example above, the new Supervisor conflicts with Elixir's Supervisor. " + "This may be fixed by using the fully qualified name in the definition:\n" + "\n" + " defmodule MyApp.Supervisor do\n" + " use Supervisor\n" + " end\n", + [inspect(Module)]). diff --git a/lib/elixir/src/elixir_bitstring.erl b/lib/elixir/src/elixir_bitstring.erl index fadb4fec843..8e9c6f8e5bb 100644 --- a/lib/elixir/src/elixir_bitstring.erl +++ b/lib/elixir/src/elixir_bitstring.erl @@ -1,223 +1,425 @@ +%% SPDX-License-Identifier: Apache-2.0 +%% SPDX-FileCopyrightText: 2021 The Elixir Team +%% SPDX-FileCopyrightText: 2012 Plataformatec + -module(elixir_bitstring). --export([translate/3, expand/3, has_size/1]). +-export([expand/5, format_error/1, validate_spec/2]). +-import(elixir_errors, [function_error/4]). -include("elixir.hrl"). -%% Expansion +expand_match(Expr, {S, OriginalS}, E) -> + {EExpr, SE, EE} = elixir_expand:expand(Expr, S, E), + {EExpr, {SE, OriginalS}, EE}. -expand(Meta, Args, E) -> - case ?m(E, context) of +expand(Meta, Args, S, E, RequireSize) -> + case ?key(E, context) of match -> - {EArgs, EA} = expand_bitstr(fun elixir_exp:expand/2, Args, [], E), - {{'<<>>', Meta, EArgs}, EA}; + {EArgs, Alignment, {SA, _}, EA} = + expand(Meta, fun expand_match/3, Args, [], {S, S}, E, 0, RequireSize), + + {{'<<>>', [{alignment, Alignment} | Meta], EArgs}, SA, EA}; _ -> - {EArgs, {EC, EV}} = expand_bitstr(fun elixir_exp:expand_arg/2, Args, [], {E, E}), - {{'<<>>', Meta, EArgs}, elixir_env:mergea(EV, EC)} - end. + PairS = {elixir_env:prepare_write(S), S}, -expand_bitstr(_Fun, [], Acc, E) -> - {lists:reverse(Acc), E}; -expand_bitstr(Fun, [{'::',Meta,[Left,Right]}|T], Acc, E) -> - {ELeft, EL} = Fun(Left, E), + {EArgs, Alignment, {SA, _}, EA} = + expand(Meta, fun elixir_expand:expand_arg/3, Args, [], PairS, E, 0, RequireSize), - %% Variables defined outside the binary can be accounted - %% on subparts, however we can't assign new variables. - case E of - {ER, _} -> ok; %% expand_arg, no assigns - _ -> ER = E#{context := nil} %% expand_each, revert assigns + {{'<<>>', [{alignment, Alignment} | Meta], EArgs}, elixir_env:close_write(SA, S), EA} + end. + +expand(_BitstrMeta, _Fun, [], Acc, S, E, Alignment, _RequireSize) -> + {lists:reverse(Acc), Alignment, S, E}; +expand(BitstrMeta, Fun, [{'::', Meta, [Left, Right]} | T], Acc, S, E, Alignment, RequireSize) -> + % We don't want to consider variables added in the Left pattern inside the Right specs + {#elixir_ex{vars=BeforeVars}, _} = S, + + {ELeft, {#elixir_ex{vars=AfterVars} = SL, OriginalS}, EL} = expand_expr(Left, Fun, S, E), + validate_expr(ELeft, Meta, E), + + MatchOrRequireSize = RequireSize or is_match_size(T, EL), + EType = expr_type(ELeft), + ExpectSize = case ELeft of + _ when not MatchOrRequireSize -> optional; + {'^', _, [{_, _, _}]} -> {infer, ELeft}; + _ -> required end, - ERight = expand_bit_info(Meta, Right, ER), - expand_bitstr(Fun, T, [{'::',Meta,[ELeft,ERight]}|Acc], EL); + {ERight, EAlignment, SS, ES} = expand_specs(EType, Meta, Right, SL#elixir_ex{vars=BeforeVars}, OriginalS, EL, ExpectSize), + + EAcc = concat_or_prepend_bitstring(Meta, ELeft, ERight, Acc, ES, MatchOrRequireSize), + expand(BitstrMeta, Fun, T, EAcc, {SS#elixir_ex{vars=AfterVars}, OriginalS}, ES, alignment(Alignment, EAlignment), RequireSize); +expand(BitstrMeta, Fun, [H | T], Acc, S, E, Alignment, RequireSize) -> + Meta = extract_meta(H, BitstrMeta), + {ELeft, {SS, OriginalS}, ES} = expand_expr(H, Fun, S, E), + validate_expr(ELeft, Meta, E), + + MatchOrRequireSize = RequireSize or is_match_size(T, ES), + EType = expr_type(ELeft), + ERight = infer_spec(EType, Meta), + + InferredMeta = [{inferred_bitstring_spec, true} | Meta], + EAcc = concat_or_prepend_bitstring(InferredMeta, ELeft, ERight, Acc, ES, MatchOrRequireSize), + expand(Meta, Fun, T, EAcc, {SS, OriginalS}, ES, Alignment, RequireSize). + +extract_meta({_, Meta, _}, _) -> Meta; +extract_meta(_, Meta) -> Meta. + +%% Variables defined outside the binary can be accounted +%% on subparts; however, we can't assign new variables. +is_match_size([_ | _], #{context := match}) -> true; +is_match_size(_, _) -> false. + +expr_type(Integer) when is_integer(Integer) -> integer; +expr_type(Float) when is_float(Float) -> float; +expr_type(Binary) when is_binary(Binary) -> binary; +expr_type({'<<>>', _, _}) -> bitstring; +expr_type(_) -> default. + +infer_spec(bitstring, Meta) -> {bitstring, Meta, nil}; +infer_spec(binary, Meta) -> {binary, Meta, nil}; +infer_spec(float, Meta) -> {float, Meta, nil}; +infer_spec(integer, Meta) -> {integer, Meta, nil}; +infer_spec(default, Meta) -> {integer, Meta, nil}. + +concat_or_prepend_bitstring(_Meta, {'<<>>', _, []}, _ERight, Acc, _E, _RequireSize) -> + Acc; +concat_or_prepend_bitstring(Meta, {'<<>>', PartsMeta, Parts} = ELeft, ERight, Acc, E, RequireSize) -> + case E of + #{context := match} when RequireSize -> + case lists:last(Parts) of + {'::', SpecMeta, [Bin, {binary, _, nil}]} when not is_binary(Bin) -> + function_error(SpecMeta, E, ?MODULE, unsized_binary); -expand_bitstr(Fun, [H|T], Acc, E) -> - {Expr, ES} = Fun(H, E), - expand_bitstr(Fun, T, [Expr|Acc], ES). + {'::', SpecMeta, [_, {bitstring, _, nil}]} -> + function_error(SpecMeta, E, ?MODULE, unsized_binary); -%% Expand bit info + _ -> + ok + end; + _ -> + ok + end, -expand_bit_info(Meta, Info, E) when is_list(Info) -> - expand_bit_info(Meta, Info, default, [], E); + case ERight of + {binary, _, nil} -> + {alignment, Alignment} = lists:keyfind(alignment, 1, PartsMeta), -expand_bit_info(Meta, Info, E) -> - expand_bit_info(Meta, [Info], E). + if + is_integer(Alignment) -> + (Alignment /= 0) andalso function_error(Meta, E, ?MODULE, {unaligned_binary, ELeft}), + lists:reverse(Parts, Acc); -expand_bit_info(Meta, [{Expr, ExprMeta, Args}|T], Size, Types, E) when is_atom(Expr) -> - ListArgs = if is_atom(Args) -> []; is_list(Args) -> Args end, - case expand_bit_type_or_size(Expr, ListArgs) of - type -> - {EArgs, EE} = elixir_exp:expand_args(ListArgs, E), - expand_bit_info(Meta, T, Size, [{Expr, [], EArgs}|Types], EE); - size -> - case Size of - default -> ok; - _ -> elixir_errors:compile_error(Meta, ?m(E, file), "duplicated size definition in bitstring") - end, - {EArgs, EE} = elixir_exp:expand_args(ListArgs, E), - expand_bit_info(Meta, T, {Expr, [], EArgs}, Types, EE); - none -> - handle_unknown_bit_info(Meta, {Expr, ExprMeta, ListArgs}, T, Size, Types, E) + true -> + [{'::', Meta, [ELeft, ERight]} | Acc] + end; + {bitstring, _, nil} -> + lists:reverse(Parts, Acc) end; +concat_or_prepend_bitstring(Meta, ELeft, ERight, Acc, _E, _RequireSize) -> + [{'::', Meta, [ELeft, ERight]} | Acc]. + +%% Handling of alignment + +alignment(Left, Right) when is_integer(Left), is_integer(Right) -> (Left + Right) rem 8; +alignment(_, _) -> unknown. + +compute_alignment(_, Size, Unit) when is_integer(Size), is_integer(Unit) -> (Size * Unit) rem 8; +compute_alignment(default, Size, Unit) -> compute_alignment(integer, Size, Unit); +compute_alignment(integer, default, Unit) -> compute_alignment(integer, 8, Unit); +compute_alignment(integer, Size, default) -> compute_alignment(integer, Size, 1); +compute_alignment(bitstring, Size, default) -> compute_alignment(bitstring, Size, 1); +compute_alignment(binary, Size, default) -> compute_alignment(binary, Size, 8); +compute_alignment(binary, _, _) -> 0; +compute_alignment(float, _, _) -> 0; +compute_alignment(utf32, _, _) -> 0; +compute_alignment(utf16, _, _) -> 0; +compute_alignment(utf8, _, _) -> 0; +compute_alignment(_, _, _) -> unknown. + +%% Expands the expression of a bitstring, that is, the LHS of :: or +%% an argument of the bitstring (such as "foo" in "<>"). +%% If we are inside a match/guard, we inline interpolations explicitly, +%% otherwise they are inlined by elixir_rewrite.erl. + +expand_expr({{'.', _, [Mod, to_string]}, _, [Arg]} = AST, Fun, S, #{context := Context} = E) + when Context /= nil, (Mod == 'Elixir.Kernel') orelse (Mod == 'Elixir.String.Chars') -> + case Fun(Arg, S, E) of + {EBin, SE, EE} when is_binary(EBin) -> {EBin, SE, EE}; + _ -> Fun(AST, S, E) % Let it raise + end; +expand_expr(Component, Fun, S, E) -> + Fun(Component, S, E). -expand_bit_info(Meta, [Int|T], Size, Types, E) when is_integer(Int) -> - expand_bit_info(Meta, [{size, [], [Int]}|T], Size, Types, E); - -expand_bit_info(Meta, [Expr|_], _Size, _Types, E) -> - elixir_errors:compile_error(Meta, ?m(E, file), - "unknown bitstring specifier ~ts", ['Elixir.Kernel':inspect(Expr)]); - -expand_bit_info(_Meta, [], Size, Types, _) -> - case Size of - default -> lists:reverse(Types); - _ -> [Size|lists:reverse(Types)] - end. +validate_expr(Expr, Meta, #{context := match} = E) -> + case Expr of + {Var, _Meta, Ctx} when is_atom(Var), is_atom(Ctx) -> ok; + {'<<>>', _, _} -> ok; + {'^', _, _} -> ok; + _ when is_number(Expr); is_binary(Expr) -> ok; + _ -> function_error(extract_meta(Expr, Meta), E, ?MODULE, {unknown_match, Expr}) + end; +validate_expr(_Expr, _Meta, _E) -> + ok. + +%% Expands and normalizes types of a bitstring. + +expand_specs(ExprType, Meta, Info, S, OriginalS, E, ExpectSize) -> + Default = + #{size => default, + unit => default, + sign => default, + type => default, + endianness => default}, + {#{size := Size, unit := Unit, type := Type, endianness := Endianness, sign := Sign}, SS, ES} = + expand_each_spec(Meta, unpack_specs(Info, []), Default, S, OriginalS, E), + + MergedType = type(Meta, ExprType, Type, E), + validate_size_required(Meta, ExpectSize, ExprType, MergedType, Size, ES), + SizeAndUnit = size_and_unit(Meta, ExprType, Size, Unit, ES), + Alignment = compute_alignment(MergedType, Size, Unit), + + MaybeInferredSize = case {ExpectSize, MergedType, SizeAndUnit} of + {{infer, PinnedVar}, binary, []} -> [{size, Meta, [{{'.', Meta, [erlang, byte_size]}, Meta, [PinnedVar]}]}]; + {{infer, PinnedVar}, bitstring, []} -> [{size, Meta, [{{'.', Meta, [erlang, bit_size]}, Meta, [PinnedVar]}]}]; + _ -> SizeAndUnit + end, -expand_bit_type_or_size(binary, []) -> type; -expand_bit_type_or_size(integer, []) -> type; -expand_bit_type_or_size(float, []) -> type; -expand_bit_type_or_size(bitstring, []) -> type; -expand_bit_type_or_size(bytes, []) -> type; -expand_bit_type_or_size(bits, []) -> type; -expand_bit_type_or_size(utf8, []) -> type; -expand_bit_type_or_size(utf16, []) -> type; -expand_bit_type_or_size(utf32, []) -> type; -expand_bit_type_or_size(signed, []) -> type; -expand_bit_type_or_size(unsigned, []) -> type; -expand_bit_type_or_size(big, []) -> type; -expand_bit_type_or_size(little, []) -> type; -expand_bit_type_or_size(native, []) -> type; -expand_bit_type_or_size(unit, [_]) -> type; -expand_bit_type_or_size(size, [_]) -> size; -expand_bit_type_or_size(_, _) -> none. - -handle_unknown_bit_info(Meta, {_, ExprMeta, _} = Expr, T, Size, Types, E) -> - case 'Elixir.Macro':expand(Expr, elixir_env:linify({?line(ExprMeta), E})) of - Expr -> - elixir_errors:compile_error(ExprMeta, ?m(E, file), - "unknown bitstring specifier ~ts", ['Elixir.Macro':to_string(Expr)]); - Other -> - List = case is_list(Other) of true -> Other; false -> [Other] end, - expand_bit_info(Meta, List ++ T, Size, Types, E) - end. + [H | T] = build_spec(Meta, Size, Unit, MergedType, Endianness, Sign, MaybeInferredSize, ES), + {lists:foldl(fun(I, Acc) -> {'-', Meta, [Acc, I]} end, H, T), Alignment, SS, ES}. + +type(_, default, default, _) -> + integer; +type(_, ExprType, default, _) -> + ExprType; +type(_, binary, Type, _) when Type == binary; Type == bitstring; Type == utf8; Type == utf16; Type == utf32 -> + Type; +type(_, bitstring, Type, _) when Type == binary; Type == bitstring -> + Type; +type(_, integer, Type, _) when Type == integer; Type == float; Type == utf8; Type == utf16; Type == utf32 -> + Type; +type(_, float, Type, _) when Type == float -> + Type; +type(_, default, Type, _) -> + Type; +type(Meta, Other, Type, E) -> + function_error(Meta, E, ?MODULE, {bittype_mismatch, Type, Other, type}), + Type. + +expand_each_spec(Meta, [{Expr, MetaE, Args} = H | T], Map, S, OriginalS, E) when is_atom(Expr) -> + case validate_spec(Expr, Args) of + {Key, Arg} -> + case Args of + [] -> + elixir_errors:file_warn(Meta, E, ?MODULE, {parens_bittype, Expr}); + _ -> ok + end, + {Value, SE, EE} = expand_spec_arg(Arg, S, OriginalS, E), + validate_spec_arg(Meta, Key, Value, SE, OriginalS, EE), -%% Translation + case maps:get(Key, Map) of + default -> ok; + Value -> ok; + Other -> function_error(Meta, E, ?MODULE, {bittype_mismatch, Value, Other, Key}) + end, -has_size({bin, _, Elements}) -> - not lists:any(fun({bin_element, _Line, _Expr, Size, Types}) -> - (Types /= default) andalso (Size == default) andalso - lists:any(fun(X) -> lists:member(X, Types) end, - [bits, bytes, bitstring, binary]) - end, Elements). + expand_each_spec(Meta, T, maps:put(Key, Value, Map), SE, OriginalS, EE); -translate(Meta, Args, S) -> - case S#elixir_scope.context of - match -> - build_bitstr(fun elixir_translator:translate/2, Args, Meta, S); - _ -> - build_bitstr(fun(X, Acc) -> elixir_translator:translate_arg(X, Acc, S) end, Args, Meta, S) - end. + none -> + HA = case Args of + nil -> + elixir_errors:file_warn(Meta, E, ?MODULE, {unknown_bittype, Expr}), + {Expr, MetaE, []}; + _ -> H + end, -build_bitstr(Fun, Exprs, Meta, S) -> - {Final, FinalS} = build_bitstr_each(Fun, Exprs, Meta, S, []), - {{bin, ?line(Meta), lists:reverse(Final)}, FinalS}. - -build_bitstr_each(_Fun, [], _Meta, S, Acc) -> - {Acc, S}; - -build_bitstr_each(Fun, [{'::',_,[H,V]}|T], Meta, S, Acc) -> - {Size, Types} = extract_bit_info(Meta, V, S#elixir_scope{context=nil}), - build_bitstr_each(Fun, T, Meta, S, Acc, H, Size, Types); - -build_bitstr_each(Fun, [H|T], Meta, S, Acc) -> - build_bitstr_each(Fun, T, Meta, S, Acc, H, default, default). - -build_bitstr_each(Fun, T, Meta, S, Acc, H, default, Types) when is_binary(H) -> - Element = - case types_allow_splice(Types, []) of - true -> - %% See explanation in elixir_utils:elixir_to_erl/1 to know - %% why we can simply convert the binary to a list. - {bin_element, ?line(Meta), {string, 0, binary_to_list(H)}, default, default}; - false -> - case types_require_conversion(Types) of - true -> - {bin_element, ?line(Meta), {string, 0, elixir_utils:characters_to_list(H)}, default, Types}; - false -> - elixir_errors:compile_error(Meta, S#elixir_scope.file, "invalid types for literal string in <<>>. " - "Accepted types are: little, big, utf8, utf16, utf32, bits, bytes, binary, bitstring") - end - end, - - build_bitstr_each(Fun, T, Meta, S, [Element|Acc]); - -build_bitstr_each(_Fun, _T, Meta, S, _Acc, H, _Size, _Types) when is_binary(H) -> - elixir_errors:compile_error(Meta, S#elixir_scope.file, "size is not supported for literal string in <<>>"); - -build_bitstr_each(_Fun, _T, Meta, S, _Acc, H, _Size, _Types) when is_list(H); is_atom(H) -> - elixir_errors:compile_error(Meta, S#elixir_scope.file, "invalid literal ~ts in <<>>", - ['Elixir.Macro':to_string(H)]); - -build_bitstr_each(Fun, T, Meta, S, Acc, H, Size, Types) -> - {Expr, NS} = Fun(H, S), + case 'Elixir.Macro':expand(HA, E#{line := ?line(Meta)}) of + HA -> + function_error(Meta, E, ?MODULE, {undefined_bittype, H}), + expand_each_spec(Meta, T, Map, S, OriginalS, E); - case Expr of - {bin, _, Elements} -> - case (Size == default) andalso types_allow_splice(Types, Elements) of - true -> build_bitstr_each(Fun, T, Meta, NS, lists:reverse(Elements) ++ Acc); - false -> build_bitstr_each(Fun, T, Meta, NS, [{bin_element, ?line(Meta), Expr, Size, Types}|Acc]) + NewTypes -> + expand_each_spec(Meta, unpack_specs(NewTypes, []) ++ T, Map, S, OriginalS, E) + end + end; +expand_each_spec(Meta, [Expr | Tail], Map, S, OriginalS, E) -> + function_error(Meta, E, ?MODULE, {undefined_bittype, Expr}), + expand_each_spec(Meta, Tail, Map, S, OriginalS, E); +expand_each_spec(_Meta, [], Map, S, _OriginalS, E) -> + {Map, S, E}. + +unpack_specs({'-', _, [H, T]}, Acc) -> + unpack_specs(H, unpack_specs(T, Acc)); +unpack_specs({'*', _, [{'_', _, Atom}, Unit]}, Acc) when is_atom(Atom) -> + [{unit, [], [Unit]} | Acc]; +unpack_specs({'*', _, [Size, Unit]}, Acc) -> + [{size, [], [Size]}, {unit, [], [Unit]} | Acc]; +unpack_specs(Size, Acc) when is_integer(Size) -> + [{size, [], [Size]} | Acc]; +unpack_specs({Expr, Meta, Args}, Acc) when is_atom(Expr) -> + ListArgs = if is_atom(Args) -> nil; is_list(Args) -> Args end, + [{Expr, Meta, ListArgs} | Acc]; +unpack_specs(Other, Acc) -> + [Other | Acc]. + +validate_spec(Spec, []) -> validate_spec(Spec, nil); +validate_spec(big, nil) -> {endianness, big}; +validate_spec(little, nil) -> {endianness, little}; +validate_spec(native, nil) -> {endianness, native}; +validate_spec(size, [Size]) -> {size, Size}; +validate_spec(unit, [Unit]) -> {unit, Unit}; +validate_spec(integer, nil) -> {type, integer}; +validate_spec(float, nil) -> {type, float}; +validate_spec(binary, nil) -> {type, binary}; +validate_spec(bytes, nil) -> {type, binary}; +validate_spec(bitstring, nil) -> {type, bitstring}; +validate_spec(bits, nil) -> {type, bitstring}; +validate_spec(utf8, nil) -> {type, utf8}; +validate_spec(utf16, nil) -> {type, utf16}; +validate_spec(utf32, nil) -> {type, utf32}; +validate_spec(signed, nil) -> {sign, signed}; +validate_spec(unsigned, nil) -> {sign, unsigned}; +validate_spec(_, _) -> none. + +expand_spec_arg(Expr, S, _OriginalS, E) when is_atom(Expr); is_integer(Expr) -> + {Expr, S, E}; +expand_spec_arg(Expr, S, OriginalS, #{context := match} = E) -> + %% We can only access variables that are either on prematch or not in original + #elixir_ex{prematch={PreRead, PreCycle, _} = OldPre} = S, + #elixir_ex{vars={OriginalRead, _}} = OriginalS, + NewPre = {PreRead, PreCycle, {bitsize, OriginalRead}}, + {EExpr, SE, EE} = elixir_expand:expand(Expr, S#elixir_ex{prematch=NewPre}, E#{context := guard}), + {EExpr, SE#elixir_ex{prematch=OldPre}, EE#{context := match}}; +expand_spec_arg(Expr, S, OriginalS, E) -> + elixir_expand:expand(Expr, elixir_env:reset_read(S, OriginalS), E). + +validate_spec_arg(Meta, unit, Value, _S, _OriginalS, E) when not is_integer(Value) -> + function_error(Meta, E, ?MODULE, {bad_unit_argument, Value}), + ok; +validate_spec_arg(_Meta, _Key, _Value, _S, _OriginalS, _E) -> + ok. + +validate_size_required(Meta, required, default, Type, default, E) when Type == binary; Type == bitstring -> + function_error(Meta, E, ?MODULE, unsized_binary), + ok; +validate_size_required(_, _, _, _, _, _) -> + ok. + +size_and_unit(Meta, bitstring, Size, Unit, E) when Size /= default; Unit /= default -> + function_error(Meta, E, ?MODULE, bittype_literal_bitstring), + []; +size_and_unit(Meta, binary, Size, Unit, E) when Size /= default; Unit /= default -> + function_error(Meta, E, ?MODULE, bittype_literal_string), + []; +size_and_unit(_Meta, _ExprType, Size, Unit, _E) -> + add_arg(unit, Unit, add_arg(size, Size, [])). + +add_arg(_Key, default, Spec) -> Spec; +add_arg(Key, Arg, Spec) -> [{Key, [], [Arg]} | Spec]. + +build_spec(Meta, Size, Unit, Type, Endianness, Sign, Spec, E) when Type == utf8; Type == utf16; Type == utf32 -> + if + Size /= default; Unit /= default -> + function_error(Meta, E, ?MODULE, bittype_utf); + Sign /= default -> + function_error(Meta, E, ?MODULE, bittype_signed); + true -> + ok + end, + add_spec(Type, add_spec(Endianness, Spec)); + +build_spec(Meta, _Size, Unit, Type, _Endianness, Sign, Spec, E) when Type == binary; Type == bitstring -> + if + Type == bitstring, Unit /= default, Unit /= 1 -> + function_error(Meta, E, ?MODULE, {bittype_mismatch, Unit, 1, unit}); + Sign /= default -> + function_error(Meta, E, ?MODULE, bittype_signed); + true -> + %% Endianness is supported but has no effect, so we just ignore it. + ok + end, + add_spec(Type, Spec); + +build_spec(Meta, Size, Unit, Type, Endianness, Sign, Spec, E) when Type == integer; Type == float -> + NumberSize = number_size(Size, Unit), + if + Type == float, is_integer(NumberSize) -> + case valid_float_size(NumberSize) of + true -> + add_spec(Type, add_spec(Endianness, add_spec(Sign, Spec))); + false -> + function_error(Meta, E, ?MODULE, {bittype_float_size, NumberSize}), + [] end; - _ -> - build_bitstr_each(Fun, T, Meta, NS, [{bin_element, ?line(Meta), Expr, Size, Types}|Acc]) + Size == default, Unit /= default -> + function_error(Meta, E, ?MODULE, bittype_unit), + []; + true -> + add_spec(Type, add_spec(Endianness, add_spec(Sign, Spec))) end. -types_require_conversion([End|T]) when End == little; End == big -> types_require_conversion(T); -types_require_conversion([UTF|T]) when UTF == utf8; UTF == utf16; UTF == utf32 -> types_require_conversion(T); -types_require_conversion([]) -> true; -types_require_conversion(_) -> false. - -types_allow_splice([bytes], Elements) -> is_byte_size(Elements, 0); -types_allow_splice([binary], Elements) -> is_byte_size(Elements, 0); -types_allow_splice([bits], _) -> true; -types_allow_splice([bitstring], _) -> true; -types_allow_splice(default, _) -> true; -types_allow_splice(_, _) -> false. - -is_byte_size([Element|T], Acc) -> - case elem_size(Element) of - {unknown, Unit} when Unit rem 8 == 0 -> is_byte_size(T, Acc); - {unknown, _Unit} -> false; - {Size, Unit} -> is_byte_size(T, Size*Unit + Acc) - end; -is_byte_size([], Size) -> - Size rem 8 == 0. - -elem_size({bin_element, _, _, default, _}) -> {0, 0}; -elem_size({bin_element, _, _, {integer,_,Size}, Types}) -> {Size, unit_size(Types, 1)}; -elem_size({bin_element, _, _, _Size, Types}) -> {unknown, unit_size(Types, 1)}. - -unit_size([binary|T], _) -> unit_size(T, 8); -unit_size([{unit, Size}|_], _) -> Size; -unit_size([_|T], Guess) -> unit_size(T, Guess); -unit_size([], Guess) -> Guess. - -%% Extra bitstring specifiers - -extract_bit_info(Meta, [{size, _, [Arg]}|T], S) -> - case elixir_translator:translate(Arg, S) of - {{Kind, _, _} = Size, _} when Kind == integer; Kind == var -> - {Size, extract_bit_type(Meta, T, S)}; - _ -> - elixir_errors:compile_error(Meta, S#elixir_scope.file, - "size in bitstring expects an integer or a variable as argument, got: ~ts", ['Elixir.Macro':to_string(Arg)]) - end; -extract_bit_info(Meta, T, S) -> - {default, extract_bit_type(Meta, T, S)}. - -extract_bit_type(Meta, [{unit, _, [Arg]}|T], S) when is_integer(Arg) -> - [{unit, Arg}|extract_bit_type(Meta, T, S)]; -extract_bit_type(Meta, [{unit, _, [Arg]}|_], S) -> - elixir_errors:compile_error(Meta, S#elixir_scope.file, - "unit in bitstring expects an integer as argument, got: ~ts", ['Elixir.Macro':to_string(Arg)]); -extract_bit_type(Meta, [{Other, _, []}|T], S) -> - [Other|extract_bit_type(Meta, T, S)]; -extract_bit_type(_Meta, [], _S) -> - []. +number_size(Size, default) when is_integer(Size) -> Size; +number_size(Size, Unit) when is_integer(Size) -> Size * Unit; +number_size(Size, _) -> Size. + +valid_float_size(16) -> true; +valid_float_size(32) -> true; +valid_float_size(64) -> true; +valid_float_size(_) -> false. + +add_spec(default, Spec) -> Spec; +add_spec(Key, Spec) -> [{Key, [], nil} | Spec]. + +format_error({unaligned_binary, Expr}) -> + Message = "expected ~ts to be a binary but its number of bits is not divisible by 8", + io_lib:format(Message, ['Elixir.Macro':to_string(Expr)]); +format_error(unsized_binary) -> + "a binary field without size is only allowed at the end of a binary pattern, " + "at the right side of binary concatenation and never allowed in binary generators. " + "The following examples are invalid:\n\n" + " rest <> \"foo\"\n" + " <>\n\n" + "They are invalid because there is a bits/bitstring component not at the end. " + "However, the \"reverse\" would work:\n\n" + " \"foo\" <> rest\n" + " <<\"foo\", rest::binary>>\n\n"; +format_error(bittype_literal_bitstring) -> + "literal <<>> in bitstring supports only type specifiers, which must be one of: " + "binary or bitstring"; +format_error(bittype_literal_string) -> + "literal string in bitstring supports only endianness and type specifiers, which must be one of: " + "little, big, native, utf8, utf16, utf32, bits, bytes, binary or bitstring"; +format_error(bittype_utf) -> + "size and unit are not supported on utf types"; +format_error(bittype_signed) -> + "signed and unsigned specifiers are supported only on integer and float types"; +format_error(bittype_unit) -> + "integer and float types require a size specifier if the unit specifier is given"; +format_error({bittype_float_size, Other}) -> + io_lib:format("float requires size*unit to be 16, 32, or 64 (default), got: ~p", [Other]); +format_error({undefined_bittype, Expr}) -> + io_lib:format("unknown bitstring specifier: ~ts", ['Elixir.Macro':to_string(Expr)]); +format_error({unknown_bittype, Name}) -> + io_lib:format("bitstring specifier \"~ts\" does not exist and is being expanded to \"~ts()\"," + " please use parentheses to remove the ambiguity.\n" + "You may run \"mix format --migrate\" to fix this warning automatically.", [Name, Name]); +format_error({parens_bittype, Name}) -> + io_lib:format("extra parentheses on a bitstring specifier \"~ts()\" have been deprecated. " + "Please remove the parentheses: \"~ts\".\n" + "You may run \"mix format --migrate\" to fix this warning automatically." + , + [Name, Name]); +format_error({bittype_mismatch, Val1, Val2, Where}) -> + io_lib:format("conflicting ~ts specification for bit field: \"~p\" and \"~p\"", [Where, Val1, Val2]); +format_error({bad_unit_argument, Unit}) -> + io_lib:format("unit in bitstring expects an integer as argument, got: ~ts", + ['Elixir.Macro':to_string(Unit)]); +format_error({unknown_match, Expr}) -> + Message = + "a bitstring only accepts binaries, numbers, and variables inside a match, got: ~ts", + io_lib:format(Message, ['Elixir.Macro':to_string(Expr)]); +format_error({undefined_var_in_spec, Var}) -> + Message = + "undefined variable \"~ts\" in bitstring segment. If the size of the binary is a " + "variable, the variable must be defined prior to its use in the binary/bitstring match " + "itself, or outside the pattern match", + io_lib:format(Message, ['Elixir.Macro':to_string(Var)]). diff --git a/lib/elixir/src/elixir_bootstrap.erl b/lib/elixir/src/elixir_bootstrap.erl index 97cf07fb215..0faa47fec44 100644 --- a/lib/elixir/src/elixir_bootstrap.erl +++ b/lib/elixir/src/elixir_bootstrap.erl @@ -1,5 +1,9 @@ +%% SPDX-License-Identifier: Apache-2.0 +%% SPDX-FileCopyrightText: 2021 The Elixir Team +%% SPDX-FileCopyrightText: 2012 Plataformatec + %% An Erlang module that behaves like an Elixir module -%% used for bootstraping. +%% used for bootstrapping. -module(elixir_bootstrap). -export(['MACRO-def'/2, 'MACRO-def'/3, 'MACRO-defp'/3, 'MACRO-defmodule'/3, 'MACRO-defmacro'/2, 'MACRO-defmacro'/3, 'MACRO-defmacrop'/3, @@ -18,34 +22,46 @@ 'MACRO-defmacro'(Caller, Call, Expr) -> define(Caller, defmacro, Call, Expr). 'MACRO-defmacrop'(Caller, Call, Expr) -> define(Caller, defmacrop, Call, Expr). -'MACRO-defmodule'(_Caller, Alias, [{do,Block}]) -> - {Escaped, _} = elixir_quote:escape(Block, false), - Args = [Alias, Escaped, [], env()], +'MACRO-defmodule'({Line, _S, _E} = _Caller, Alias, [{do, Block}]) -> + Escaped = elixir_quote:escape(Block, escape, false), + Args = [[{line, Line}], Alias, Escaped, [], false, env()], {{'.', [], [elixir_module, compile]}, [], Args}. '__info__'(functions) -> []; '__info__'(macros) -> [{'@', 1}, - {def,1}, - {def,2}, - {defmacro,1}, - {defmacro,2}, - {defmacrop,2}, - {defmodule,2}, - {defp,2}]. - -define({Line,E}, Kind, Call, Expr) -> - {EscapedCall, UC} = elixir_quote:escape(Call, true), - {EscapedExpr, UE} = elixir_quote:escape(Expr, true), - Args = [Line, Kind, not(UC or UE), EscapedCall, EscapedExpr, elixir_locals:cache_env(E)], + {def, 1}, + {def, 2}, + {defmacro, 1}, + {defmacro, 2}, + {defmacrop, 2}, + {defmodule, 2}, + {defp, 2}]. + +define({Line, _S, #{module := Module} = E}, Kind, Call, Expr) -> + UC = elixir_quote:has_unquotes(Call), + UE = elixir_quote:has_unquotes(Expr), + + Store = + case UC or UE of + true -> + elixir_quote:escape({Call, Expr}, escape, true); + + false -> + Key = erlang:unique_integer(), + elixir_module:write_cache(Module, Key, {Call, Expr}), + Key + end, + + Args = [Kind, Store, elixir_module:cache_env(E#{line := Line})], {{'.', [], [elixir_def, store_definition]}, [], Args}. unless_loaded(Fun, Args, Callback) -> - case code:is_loaded(?kernel) of - {_, _} -> apply(?kernel, Fun, Args); - false -> Callback() + case erlang:module_loaded(?kernel) of + true -> apply(?kernel, Fun, Args); + false -> Callback() end. env() -> - {'__ENV__', [], nil}. \ No newline at end of file + {'__ENV__', [], nil}. diff --git a/lib/elixir/src/elixir_clauses.erl b/lib/elixir/src/elixir_clauses.erl index f5b3ef9e688..b252ae8b08a 100644 --- a/lib/elixir/src/elixir_clauses.erl +++ b/lib/elixir/src/elixir_clauses.erl @@ -1,226 +1,615 @@ +%% SPDX-License-Identifier: Apache-2.0 +%% SPDX-FileCopyrightText: 2021 The Elixir Team +%% SPDX-FileCopyrightText: 2012 Plataformatec + %% Handle code related to args, guard and -> matching for case, %% fn, receive and friends. try is handled in elixir_try. -module(elixir_clauses). --export([match/3, clause/7, clauses/4, guards/4, get_pairs/3, get_pairs/4, - extract_splat_guards/1, extract_guards/1]). +-export([parallel_match/4, match/6, clause/6, def/3, head/4, + 'case'/4, 'receive'/4, 'try'/4, 'cond'/4, with/4, + format_error/1]). +-import(elixir_errors, [file_error/4, file_warn/4]). -include("elixir.hrl"). -%% Get pairs from a clause. +%% Deal with parallel matches and loops in variables + +parallel_match(Meta, Expr, S, #{context := match} = E) -> + #elixir_ex{vars={_Read, Write}} = S, + Matches = unpack_match(Expr, Meta, [], E), + + {[{_, EHead} | ETail], EWrites, SM, EM} = + lists:foldl(fun({EMeta, Match}, {AccMatches, AccWrites, SI, EI}) -> + #elixir_ex{vars={Read, _Write}} = SI, + {EMatch, SM, EM} = elixir_expand:expand(Match, SI#elixir_ex{vars={Read, #{}}}, EI), + #elixir_ex{vars={_, EWrite}} = SM, + {[{EMeta, EMatch} | AccMatches], [EWrite | AccWrites], SM, EM} + end, {[], [], S, E}, Matches), + + EMatch = + lists:foldl(fun({EMeta, EMatch}, Acc) -> + {'=', EMeta, [EMatch, Acc]} + end, EHead, ETail), + + #elixir_ex{vars={VRead, _}, prematch={PRead, Cycles, PInfo}} = SM, + {PCycles, PWrites} = store_cycles(EWrites, Cycles, #{}), + VWrite = (Write /= false) andalso elixir_env:merge_vars(Write, PWrites), + {EMatch, SM#elixir_ex{vars={VRead, VWrite}, prematch={PRead, PCycles, PInfo}}, EM}. + +unpack_match({'=', Meta, [{_, VarMeta, _} = Node, Node]}, _Meta, Acc, E) -> + %% TODO: remove this clause on Elixir v1.23 + file_warn(VarMeta, ?key(E, file), ?MODULE, {duplicate_match, Node}), + unpack_match(Node, Meta, Acc, E); +unpack_match({'=', Meta, [Left, Right]}, _Meta, Acc, E) -> + unpack_match(Left, Meta, unpack_match(Right, Meta, Acc, E), E); +unpack_match(Node, Meta, Acc, _E) -> + [{Meta, Node} | Acc]. + +store_cycles([Write | Writes], {Cycles, SkipList}, Acc) -> + %% Compute the variables this parallel pattern depends on + DependsOn = lists:foldl(fun maps:merge/2, Acc, Writes), + + %% For each variable on a sibling, we store it inside the graph (Cycles). + %% The graph will by definition have at least one degree cycles. We need + %% to find variables which depend on each other more than once (tagged as + %% error below) and also all second-degree (or later) cycles. In other + %% words, take this code: + %% + %% {x = y, x = {:ok, y}} = expr() + %% + %% The first parallel match will say we store the following cycle: + %% + %% #{{x,nil} => #{{y,nil} => 1}, {y,nil} => #{{x,nil} => 0}} + %% + %% That's why one degree cycles are allowed. However, once we go + %% over the next parallel pattern, we will have: + %% + %% #{{x,nil} => #{{y,nil} => error}, {y,nil} => #{{x,nil} => error}} + %% + AccCycles = + maps:fold(fun(Pair, _, AccCycles) -> + maps:update_with(Pair, fun(Current) -> + maps:merge_with(fun(_, _, _) -> error end, Current, DependsOn) + end, DependsOn, AccCycles) + end, Cycles, Write), + + %% The SkipList keeps variables that are seen as defined together by other + %% nodes. Those must be skipped on the graph traversal, as they will always + %% contain cycles between them. For example: + %% + %% {{a} = b} = c = expr() + %% + %% In the example above, c sees "a" and "b" as defined together and therefore + %% one should not point to the other when looking for cycles. + AccSkipList = + case map_size(DependsOn) > 1 of + true -> [DependsOn | SkipList]; + false -> SkipList + end, + + store_cycles(Writes, {AccCycles, AccSkipList}, maps:merge(Acc, Write)); +store_cycles([], Cycles, Acc) -> + {Cycles, Acc}. + +validate_cycles({Cycles, SkipList}, Meta, Expr, E) -> + maps:fold(fun(Current, _DependsOn, Seen) -> + recur_cycles(Cycles, Current, root, Seen, SkipList, Meta, Expr, E) + end, #{}, Cycles). + +recur_cycles(Cycles, Current, Source, Seen, SkipList, Meta, Expr, E) -> + case is_map_key(Current, Seen) of + true -> + Seen; -get_pairs(Key, Clauses, As) -> - get_pairs(Key, Clauses, As, false). -get_pairs(Key, Clauses, As, AllowNil) -> - case lists:keyfind(Key, 1, Clauses) of - {Key, Pairs} when is_list(Pairs) -> - [{As, Meta, Left, Right} || {'->', Meta, [Left, Right]} <- Pairs]; - {Key, nil} when AllowNil -> - []; false -> - [] - end. - -%% Translate matches - -match(Fun, Args, #elixir_scope{context=Context, match_vars=MatchVars, - backup_vars=BackupVars, vars=Vars} = S) when Context /= match -> - {Result, NewS} = match(Fun, Args, S#elixir_scope{context=match, - match_vars=ordsets:new(), backup_vars=Vars}), - {Result, NewS#elixir_scope{context=Context, - match_vars=MatchVars, backup_vars=BackupVars}}; -match(Fun, Args, S) -> Fun(Args, S). - -%% Translate clauses with args, guards and expressions - -clause(Line, Fun, Args, Expr, Guards, Return, S) when is_integer(Line) -> - {TArgs, SA} = match(Fun, Args, S#elixir_scope{extra_guards=[]}), - {TExpr, SE} = elixir_translator:translate_block(Expr, Return, SA#elixir_scope{extra_guards=nil}), - - Extra = SA#elixir_scope.extra_guards, - TGuards = guards(Line, Guards, Extra, SA), - {{clause, Line, TArgs, TGuards, unblock(TExpr)}, SE}. - -% Translate/Extract guards from the given expression. - -guards(Line, Guards, Extra, S) -> - SG = S#elixir_scope{context=guard, extra_guards=nil}, - - case Guards of - [] -> case Extra of [] -> []; _ -> [Extra] end; - _ -> [translate_guard(Line, Guard, Extra, SG) || Guard <- Guards] + case maps:get(Current, Cycles) of + #{Current := _} -> + file_error(Meta, E, ?MODULE, {recursive, [Current], Expr}); + + DependsOn -> + %% We traverse over the skip list. For each entry that contains ourselves, + %% we will split the remaining variables in two groups: one which we have + %% direct dependencies and skip once and ones that we always skip. + %% + %% For example, take `foo = {bar = {baz, bat}}`. `baz` and `bat` never depend + %% on each other, so we always skip `baz` and `bat` whenever any of them are + %% seen. However, while `bar` is in the same group as `baz` and `bat` (from the + %% point of view of `foo`), it depends on them, so it only skips once. + {OnceKeys, AlwaysKeys} = + lists:foldl(fun + (#{Current := _} = Skip, Acc) -> + maps:fold(fun(K, _, {AccOnce, AccAlways}) -> + case DependsOn of + #{K := _} -> {[K | AccOnce], AccAlways}; + #{} -> {AccOnce, [K | AccAlways]} + end + end, Acc, Skip); + (_Skip, Acc) -> + Acc + end, {[], []}, SkipList), + + NewSeen = maps:merge(maps:from_keys(AlwaysKeys, false), maps:put(Current, true, Seen)), + + maps:fold(fun + (Key, error, _See) -> + file_error(Meta, E, ?MODULE, {recursive, [Current, Key], Expr}); + + %% Never go back to the node that we came from (as we can always one hop). + (Key, _, AccSeen) when Key =:= Source -> + AccSeen; + + (Key, _, AccSeen) -> + case lists:member(Key, OnceKeys) of + true -> + AccSeen; + + false when map_get(Key, Seen) -> + file_error(Meta, E, ?MODULE, {recursive, [Current | maps:keys(Seen)], Expr}); + + false -> + recur_cycles(Cycles, Key, Current, AccSeen, SkipList, Meta, Expr, E) + end + end, NewSeen, DependsOn) + end end. -translate_guard(Line, Guard, Extra, S) -> - [element(1, elixir_translator:translate(elixir_quote:linify(Line, Guard), S))|Extra]. +%% Match -extract_guards({'when', _, [Left, Right]}) -> {Left, extract_or_guards(Right)}; -extract_guards(Else) -> {Else, []}. +match(Fun, Meta, Expr, AfterS, BeforeS, #{context := nil} = E) -> + #elixir_ex{vars=Current, unused={_, Counter} = Unused} = AfterS, + #elixir_ex{vars={Read, _}, prematch=Prematch} = BeforeS, -extract_or_guards({'when', _, [Left, Right]}) -> [Left|extract_or_guards(Right)]; -extract_or_guards(Term) -> [Term]. - -% Extract guards when multiple left side args are allowed. + CallS = BeforeS#elixir_ex{ + prematch={Read, {#{}, []}, Counter}, + unused=Unused, + vars=Current + }, -extract_splat_guards([{'when', _, [_,_|_] = Args}]) -> - {Left, Right} = elixir_utils:split_last(Args), - {Left, extract_or_guards(Right)}; -extract_splat_guards(Else) -> - {Else, []}. + CallE = E#{context := match}, + {EExpr, SE, EE} = Fun(Expr, CallS, CallE), -% Function for translating macros with match style like case and receive. + #elixir_ex{ + vars=NewCurrent, + unused=NewUnused, + prematch={_, Cycles, _} + } = SE, -clauses(Meta, Clauses, Return, #elixir_scope{export_vars=CV} = S) -> - {TC, TS} = do_clauses(Meta, Clauses, Return, S#elixir_scope{export_vars=[]}), - {TC, TS#elixir_scope{export_vars=elixir_scope:merge_opt_vars(CV, TS#elixir_scope.export_vars)}}. + validate_cycles(Cycles, Meta, {match, Expr}, E), -do_clauses(_Meta, [], _Return, S) -> - {[], S}; + EndS = AfterS#elixir_ex{ + prematch=Prematch, + unused=NewUnused, + vars=NewCurrent + }, -do_clauses(Meta, DecoupledClauses, Return, S) -> - % Transform tree just passing the variables counter forward - % and storing variables defined inside each clause. - Transformer = fun(X, {SAcc, VAcc}) -> - {TX, TS} = each_clause(Meta, X, Return, SAcc), - {TX, {elixir_scope:mergec(S, TS), [TS#elixir_scope.export_vars|VAcc]}} + EndE = EE#{context := ?key(E, context)}, + {EExpr, EndS, EndE}. + +def({Meta, Args, Guards, Body}, S, E) -> + {EArgs, SA, EA} = elixir_expand:expand_args(Args, S#elixir_ex{prematch={#{}, {#{}, []}, 0}}, E#{context := match}), + #elixir_ex{prematch={_, Cycles, _}} = SA, + validate_cycles(Cycles, Meta, {?key(E, function), Args}, E), + {EGuards, SG, EG} = guard(Guards, SA#elixir_ex{prematch=none}, EA#{context := guard}), + {EBody, SB, EB} = elixir_expand:expand(Body, SG, EG#{context := nil}), + elixir_env:check_unused_vars(SB, EB), + {Meta, EArgs, EGuards, EBody}. + +clause(_Meta, _Kind, Fun, {'->', Meta, [Left, Right]}, S, E) -> + {ELeft, SL, EL} = case is_function(Fun, 4) of + true -> Fun(Meta, Left, S, E); + false -> Fun(Left, S, E) + end, + {ERight, SR, ER} = elixir_expand:expand(Right, SL, EL), + {{'->', Meta, [ELeft, ERight]}, SR, ER}; +clause(Meta, Kind, _Fun, _, _, E) -> + file_error(Meta, E, ?MODULE, {bad_or_missing_clauses, Kind}). + +head(Meta, [{'when', WhenMeta, [_ | _] = All}], S, E) -> + {Args, Guard} = elixir_utils:split_last(All), + guarded_head(Meta, WhenMeta, Args, Guard, S, E); +head(Meta, Args, S, E) -> + match(fun elixir_expand:expand_args/3, Meta, Args, S, S, E). + +guarded_head(Meta, WhenMeta, Args, Guard, S, E) -> + {EArgs, SA, EA} = match(fun elixir_expand:expand_args/3, Meta, Args, S, S, E), + {EGuard, SG, EG} = guard(Guard, SA, EA#{context := guard}), + {[{'when', WhenMeta, EArgs ++ [EGuard]}], SG, EG#{context := nil}}. + +guard({'when', Meta, [Left, Right]}, S, E) -> + {ELeft, SL, EL} = guard(Left, S, E), + {ERight, SR, ER} = guard(Right, SL, EL), + {{'when', Meta, [ELeft, ERight]}, SR, ER}; +guard(Guard, S, E) -> + {EGuard, SG, EG} = elixir_expand:expand(Guard, S, E), + warn_zero_length_guard(EGuard, EG), + {EGuard, SG, EG}. + +warn_zero_length_guard({{'.', _, [erlang, Op]}, Meta, + [{{'.', _, [erlang, length]}, _, [Arg]}, 0]}, E) when Op == '=='; Op == '>' -> + Warn = + case Op of + '==' -> {zero_list_length_in_guard, Arg}; + '>' -> {positive_list_length_in_guard, Arg} + end, + file_warn(Meta, ?key(E, file), ?MODULE, Warn); +warn_zero_length_guard({Op, _, [L, R]}, E) when Op == 'or'; Op == 'and' -> + warn_zero_length_guard(L, E), + warn_zero_length_guard(R, E); +warn_zero_length_guard(_, _) -> + ok. + +%% Case + +'case'(Meta, [], _S, E) -> + file_error(Meta, E, elixir_expand, {missing_option, 'case', [do]}); +'case'(Meta, Opts, _S, E) when not is_list(Opts) -> + file_error(Meta, E, elixir_expand, {invalid_args, 'case'}); +'case'(Meta, Opts, S, E) -> + ok = assert_at_most_once('do', Opts, 0, fun(Key) -> + file_error(Meta, E, ?MODULE, {duplicated_clauses, 'case', Key}) + end), + {Case, SA} = lists:mapfoldl(fun(X, SA) -> expand_case(Meta, X, SA, E) end, S, Opts), + {Case, SA, E}. + +expand_case(Meta, {'do', _} = Do, S, E) -> + Fun = expand_head('case', 'do'), + expand_clauses(Meta, 'case', Fun, Do, S, E); +expand_case(Meta, {Key, _}, _S, E) -> + file_error(Meta, E, ?MODULE, {unexpected_option, 'case', Key}). + +%% Cond + +'cond'(Meta, [], _S, E) -> + file_error(Meta, E, elixir_expand, {missing_option, 'cond', [do]}); +'cond'(Meta, Opts, _S, E) when not is_list(Opts) -> + file_error(Meta, E, elixir_expand, {invalid_args, 'cond'}); +'cond'(Meta, Opts, S, E) -> + ok = assert_at_most_once('do', Opts, 0, fun(Key) -> + file_error(Meta, E, ?MODULE, {duplicated_clauses, 'cond', Key}) + end), + {Cond, SA} = lists:mapfoldl(fun(X, SA) -> expand_cond(Meta, X, SA, E) end, S, Opts), + {Cond, SA, E}. + +expand_cond(Meta, {'do', _} = Do, S, E) -> + Fun = expand_one(Meta, 'cond', 'do', fun elixir_expand:expand_args/3), + expand_clauses(Meta, 'cond', Fun, Do, S, E); +expand_cond(Meta, {Key, _}, _S, E) -> + file_error(Meta, E, ?MODULE, {unexpected_option, 'cond', Key}). + +%% Receive + +'receive'(Meta, [], _S, E) -> + file_error(Meta, E, elixir_expand, {missing_option, 'receive', [do, 'after']}); +'receive'(Meta, Opts, _S, E) when not is_list(Opts) -> + file_error(Meta, E, elixir_expand, {invalid_args, 'receive'}); +'receive'(Meta, Opts, S, E) -> + RaiseError = fun(Key) -> + file_error(Meta, E, ?MODULE, {duplicated_clauses, 'receive', Key}) + end, + ok = assert_at_most_once('do', Opts, 0, RaiseError), + ok = assert_at_most_once('after', Opts, 0, RaiseError), + {Receive, SA} = lists:mapfoldl(fun(X, SA) -> expand_receive(Meta, X, SA, E) end, S, Opts), + {Receive, SA, E}. + +expand_receive(_Meta, {'do', {'__block__', _, []}} = Do, S, _E) -> + {Do, S}; +expand_receive(Meta, {'do', _} = Do, S, E) -> + Fun = expand_head('receive', 'do'), + expand_clauses(Meta, 'receive', Fun, Do, S, E); +expand_receive(Meta, {'after', [_]} = After, S, E) -> + Fun = expand_one(Meta, 'receive', 'after', fun elixir_expand:expand_args/3), + expand_clauses(Meta, 'receive', Fun, After, S, E); +expand_receive(Meta, {'after', _}, _S, E) -> + file_error(Meta, E, ?MODULE, multiple_after_clauses_in_receive); +expand_receive(Meta, {Key, _}, _S, E) -> + file_error(Meta, E, ?MODULE, {unexpected_option, 'receive', Key}). + +%% With + +with(Meta, Args, S, E) -> + {Exprs, Opts0} = elixir_utils:split_opts(Args), + S0 = elixir_env:reset_unused_vars(S), + {EExprs, {S1, E1, HasMatch}} = lists:mapfoldl(fun expand_with/2, {S0, E, false}, Exprs), + {EDo, Opts1, S2} = expand_with_do(Meta, Opts0, S, S1, E1), + {EOpts, Opts2, S3} = expand_with_else(Meta, Opts1, S2, E, HasMatch), + + case Opts2 of + [{Key, _} | _] -> + file_error(Meta, E, elixir_clauses, {unexpected_option, with, Key}); + [] -> + ok end, - {TClauses, {TS, ReverseCV}} = - lists:mapfoldl(Transformer, {S, []}, DecoupledClauses), - - % Now get all the variables defined inside each clause - CV = lists:reverse(ReverseCV), - AllVars = lists:foldl(fun elixir_scope:merge_vars/2, [], CV), - - % Create a new scope that contains a list of all variables - % defined inside all the clauses. It returns this new scope and - % a list of tuples where the first element is the variable name, - % the second one is the new pointer to the variable and the third - % is the old pointer. - {FinalVars, FS} = lists:mapfoldl(fun({Key, Val}, Acc) -> - normalize_vars(Key, Val, Acc) - end, TS, AllVars), - - % Expand all clauses by adding a match operation at the end - % that defines variables missing in one clause to the others. - expand_clauses(?line(Meta), TClauses, CV, FinalVars, [], FS). - -expand_clauses(Line, [Clause|T], [ClauseVars|V], FinalVars, Acc, S) -> - case generate_match_vars(FinalVars, ClauseVars, [], []) of - {[], []} -> - expand_clauses(Line, T, V, FinalVars, [Clause|Acc], S); - {Left, Right} -> - MatchExpr = generate_match(Line, Left, Right), - ClauseExprs = element(5, Clause), - [Final|RawClauseExprs] = lists:reverse(ClauseExprs), - - % If the last sentence has a match clause, we need to assign its value - % in the variable list. If not, we insert the variable list before the - % final clause in order to keep it tail call optimized. - {FinalClauseExprs, FS} = case has_match_tuple(Final) of - true -> - case Final of - {match, _, {var, _, UserVarName} = UserVar, _} when UserVarName /= '_' -> - {[UserVar,MatchExpr,Final|RawClauseExprs], S}; - _ -> - {VarName, _, SS} = elixir_scope:build_var('_', S), - StorageVar = {var, Line, VarName}, - StorageExpr = {match, Line, StorageVar, Final}, - {[StorageVar,MatchExpr,StorageExpr|RawClauseExprs], SS} - end; - false -> - {[Final,MatchExpr|RawClauseExprs], S} + {{with, Meta, EExprs ++ [[{do, EDo} | EOpts]]}, S3, E}. + +expand_with({'<-', Meta, [Left, Right]}, {S, E, HasMatch}) -> + {ERight, SR, ER} = elixir_expand:expand(Right, S, E), + SM = elixir_env:reset_read(SR, S), + {[ELeft], SL, EL} = head(Meta, [Left], SM, ER), + NewHasMatch = + case ELeft of + {Var, _, Ctx} when is_atom(Var), is_atom(Ctx) -> HasMatch; + _ -> true + end, + {{'<-', Meta, [ELeft, ERight]}, {SL, EL, NewHasMatch}}; +expand_with(Expr, {S, E, HasMatch}) -> + {EExpr, SE, EE} = elixir_expand:expand(Expr, S, E), + {EExpr, {SE, EE, HasMatch}}. + +expand_with_do(Meta, Opts, S, Acc, E) -> + case lists:keytake(do, 1, Opts) of + {value, {do, Expr}, RestOpts} -> + {EExpr, SAcc, EAcc} = elixir_expand:expand(Expr, Acc, E), + {EExpr, RestOpts, elixir_env:merge_and_check_unused_vars(SAcc, S, EAcc)}; + false -> + file_error(Meta, E, elixir_expand, {missing_option, 'with', [do]}) + end. + +expand_with_else(Meta, Opts, S, E, HasMatch) -> + case lists:keytake('else', 1, Opts) of + {value, Pair, RestOpts} -> + if + HasMatch -> ok; + true -> file_warn(Meta, ?key(E, file), ?MODULE, unmatchable_else_in_with) end, + Fun = expand_head('with', 'else'), + {EPair, SE} = expand_clauses(Meta, 'with', Fun, Pair, S, E), + {[EPair], RestOpts, SE}; + false -> + {[], Opts, S} + end. - FinalClause = setelement(5, Clause, lists:reverse(FinalClauseExprs)), - expand_clauses(Line, T, V, FinalVars, [FinalClause|Acc], FS) +%% Try + +'try'(Meta, [], _S, E) -> + file_error(Meta, E, elixir_expand, {missing_option, 'try', [do]}); +'try'(Meta, [{do, _}], _S, E) -> + file_error(Meta, E, elixir_expand, {missing_option, 'try', ['catch', 'rescue', 'after']}); +'try'(Meta, Opts, _S, E) when not is_list(Opts) -> + file_error(Meta, E, elixir_expand, {invalid_args, 'try'}); +'try'(Meta, Opts, S, E) -> + % TODO: Make this an error on v2.0 + case Opts of + [{do, _}, {'else', _}] -> + file_warn(Meta, ?key(E, file), ?MODULE, {try_with_only_else_clause, origin(Meta, 'try')}); + _ -> + ok + end, + RaiseError = fun(Key) -> + file_error(Meta, E, ?MODULE, {duplicated_clauses, 'try', Key}) + end, + ok = assert_at_most_once('do', Opts, 0, RaiseError), + ok = assert_at_most_once('rescue', Opts, 0, RaiseError), + ok = assert_at_most_once('catch', Opts, 0, RaiseError), + ok = assert_at_most_once('else', Opts, 0, RaiseError), + ok = assert_at_most_once('after', Opts, 0, RaiseError), + ok = warn_catch_before_rescue(Opts, Meta, E, false), + {Try, SA} = lists:mapfoldl(fun(X, SA) -> expand_try(Meta, X, SA, E) end, S, Opts), + {Try, SA, E}. + +expand_try(_Meta, {'do', Expr}, S, E) -> + {EExpr, SE, EE} = elixir_expand:expand(Expr, elixir_env:reset_unused_vars(S), E), + {{'do', EExpr}, elixir_env:merge_and_check_unused_vars(SE, S, EE)}; +expand_try(_Meta, {'after', Expr}, S, E) -> + {EExpr, SE, EE} = elixir_expand:expand(Expr, elixir_env:reset_unused_vars(S), E), + {{'after', EExpr}, elixir_env:merge_and_check_unused_vars(SE, S, EE)}; +expand_try(Meta, {'else', _} = Else, S, E) -> + Fun = expand_head('try', 'else'), + expand_clauses(Meta, 'try', Fun, Else, S, E); +expand_try(Meta, {'catch', _} = Catch, S, E) -> + expand_clauses_with_stacktrace(Meta, fun expand_catch/4, Catch, S, E); +expand_try(Meta, {'rescue', _} = Rescue, S, E) -> + expand_clauses_with_stacktrace(Meta, fun expand_rescue/4, Rescue, S, E); +expand_try(Meta, {Key, _}, _S, E) -> + file_error(Meta, E, ?MODULE, {unexpected_option, 'try', Key}). + +expand_clauses_with_stacktrace(Meta, Fun, Clauses, S, E) -> + OldStacktrace = S#elixir_ex.stacktrace, + SS = S#elixir_ex{stacktrace=true}, + {Ret, SE} = expand_clauses(Meta, 'try', Fun, Clauses, SS, E), + {Ret, SE#elixir_ex{stacktrace=OldStacktrace}}. + +expand_catch(Meta, [{'when', _, [_, _, _, _ | _]}], _, E) -> + Error = {wrong_number_of_args_for_clause, "one or two args", origin(Meta, 'try'), 'catch'}, + file_error(Meta, E, ?MODULE, Error); +expand_catch(Meta, [{'when', WhenMeta, [Arg1, Arg2, Guard]}], S, E) -> + guarded_head(Meta, WhenMeta, [Arg1, Arg2], Guard, S, E); +expand_catch(Meta, [{'when', WhenMeta, [Arg1, Guard]}], S, E) -> + guarded_head(Meta, WhenMeta, [throw, Arg1], Guard, S, E); +expand_catch(Meta, [Arg], S, E) -> + head(Meta, [throw, Arg], S, E); +expand_catch(Meta, [_, _] = Args, S, E) -> + head(Meta, Args, S, E); +expand_catch(Meta, _, _, E) -> + Error = {wrong_number_of_args_for_clause, "one or two args", origin(Meta, 'try'), 'catch'}, + file_error(Meta, E, ?MODULE, Error). + +expand_rescue(Meta, [Arg], S, E) -> + case expand_rescue(Arg, S, E) of + {EArg, SA, EA} -> + {[EArg], SA, EA}; + false -> + file_error(Meta, E, ?MODULE, {invalid_rescue_clause, Arg}) end; - -expand_clauses(_Line, [], [], _FinalVars, Acc, S) -> - {lists:reverse(Acc), S}. - -% Handle each key/value clause pair and translate them accordingly. - -each_clause(Export, {match, Meta, [Condition], Expr}, Return, S) -> - Fun = wrap_export_fun(Export, fun elixir_translator:translate_args/2), - {Arg, Guards} = extract_guards(Condition), - clause(?line(Meta), Fun, [Arg], Expr, Guards, Return, S); - -each_clause(Export, {expr, Meta, [Condition], Expr}, Return, S) -> - {TCondition, SC} = (wrap_export_fun(Export, fun elixir_translator:translate/2))(Condition, S), - {TExpr, SB} = elixir_translator:translate_block(Expr, Return, SC), - {{clause, ?line(Meta), [TCondition], [], unblock(TExpr)}, SB}. - -wrap_export_fun(Meta, Fun) -> - case lists:keyfind(export_head, 1, Meta) of - {export_head, true} -> - Fun; +expand_rescue(Meta, _, _, E) -> + Error = {wrong_number_of_args_for_clause, "one argument", origin(Meta, 'try'), 'rescue'}, + file_error(Meta, E, ?MODULE, Error). + +%% rescue var +expand_rescue({Name, Meta, Atom} = Var, S, E) when is_atom(Name), is_atom(Atom) -> + match(fun elixir_expand:expand/3, Meta, Var, S, S, E); + +%% rescue Alias => _ in [Alias] +expand_rescue({'__aliases__', _, [_ | _]} = Alias, S, E) -> + expand_rescue({in, [], [{'_', [], ?key(E, module)}, Alias]}, S, E); + +%% rescue var in _ +expand_rescue({in, _, [{Name, Meta, VarContext} = Var, {'_', _, UnderscoreContext}]}, S, E) + when is_atom(Name), is_atom(VarContext), is_atom(UnderscoreContext) -> + match(fun elixir_expand:expand/3, Meta, Var, S, S, E); + +%% rescue var in (list() or atom()) +expand_rescue({in, Meta, [Left, Right]}, S, E) -> + {ELeft, SL, EL} = match(fun elixir_expand:expand/3, Meta, Left, S, S, E), + {ERight, SR, ER} = elixir_expand:expand(Right, SL, EL), + + case ELeft of + {Name, _, Atom} when is_atom(Name), is_atom(Atom) -> + case normalize_rescue(ERight) of + false -> false; + Other -> {{in, Meta, [ELeft, Other]}, SR, ER} + end; _ -> - fun(Args, S) -> - {TArgs, TS} = Fun(Args, S), - {TArgs, TS#elixir_scope{export_vars = S#elixir_scope.export_vars}} - end - end. + false + end; -% Check if the given expression is a match tuple. -% This is a small optimization to allow us to change -% existing assignments instead of creating new ones every time. - -has_match_tuple({'receive', _, _, _, _}) -> - true; -has_match_tuple({'receive', _, _}) -> - true; -has_match_tuple({'case', _, _, _}) -> - true; -has_match_tuple({match, _, _, _}) -> - true; -has_match_tuple({'fun', _, {clauses, _}}) -> - false; -has_match_tuple(H) when is_tuple(H) -> - has_match_tuple(tuple_to_list(H)); -has_match_tuple(H) when is_list(H) -> - lists:any(fun has_match_tuple/1, H); -has_match_tuple(_) -> false. - -% Normalize the given var in between clauses -% by picking one value as reference and retriving -% its previous value. - -normalize_vars(Key, Value, #elixir_scope{vars=Vars,export_vars=ClauseVars} = S) -> - VS = S#elixir_scope{ - vars=orddict:store(Key, Value, Vars), - export_vars=orddict:store(Key, Value, ClauseVars) - }, +%% rescue expr() => rescue expanded_expr() +expand_rescue({_, Meta, _} = Arg, S, E) -> + case 'Elixir.Macro':expand_once(Arg, E#{line := ?line(Meta)}) of + Arg -> false; + NewArg -> expand_rescue(NewArg, S, E) + end; - Expr = case orddict:find(Key, Vars) of - {ok, {PreValue, _}} -> {var, 0, PreValue}; - error -> {atom, 0, nil} - end, +%% rescue list() or atom() => _ in (list() or atom()) +expand_rescue(Arg, S, E) -> + expand_rescue({in, [], [{'_', [], ?key(E, module)}, Arg]}, S, E). + +normalize_rescue(Atom) when is_atom(Atom) -> + [Atom]; +normalize_rescue(Other) -> + is_list(Other) andalso lists:all(fun is_atom/1, Other) andalso Other. + +%% Expansion helpers + +expand_head(Kind, Key) -> + fun + (Meta, [{'when', _, [_, _, _ | _]}], _, E) -> + file_error(Meta, E, ?MODULE, {wrong_number_of_args_for_clause, "one argument", Kind, Key}); + (Meta, [_] = Args, S, E) -> + head(Meta, Args, S, E); + (Meta, _, _, E) -> + file_error(Meta, E, ?MODULE, {wrong_number_of_args_for_clause, "one argument", Kind, Key}) + end. - {{Key, Value, Expr}, VS}. - -% Generate match vars by checking if they were updated -% or not and assigning the previous value. - -generate_match_vars([{Key, Value, Expr}|T], ClauseVars, Left, Right) -> - case orddict:find(Key, ClauseVars) of - {ok, Value} -> - generate_match_vars(T, ClauseVars, Left, Right); - {ok, Clause} -> - generate_match_vars(T, ClauseVars, - [{var, 0, element(1, Value)}|Left], - [{var, 0, element(1, Clause)}|Right]); - error -> - generate_match_vars(T, ClauseVars, - [{var, 0, element(1, Value)}|Left], [Expr|Right]) - end; +%% Returns a function that expands arguments +%% considering we have at maximum one entry. +expand_one(Meta, Kind, Key, Fun) -> + fun + ([_] = Args, S, E) -> + Fun(Args, S, E); + (_, _, E) -> + file_error(Meta, E, ?MODULE, {wrong_number_of_args_for_clause, "one argument", Kind, Key}) + end. -generate_match_vars([], _ClauseVars, Left, Right) -> - {Left, Right}. +%% Expands all -> pairs in a given key but do not keep the overall vars. +expand_clauses(Meta, Kind, Fun, Clauses, S, E) -> + NewKind = origin(Meta, Kind), + expand_clauses_origin(Meta, NewKind, Fun, Clauses, S, E). -generate_match(Line, [Left], [Right]) -> - {match, Line, Left, Right}; +expand_clauses_origin(Meta, Kind, Fun, {Key, [_ | _] = Clauses}, S, E) -> + Transformer = fun(Clause, SA) -> + {EClause, SAcc, EAcc} = + clause(Meta, {Kind, Key}, Fun, Clause, elixir_env:reset_unused_vars(SA), E), -generate_match(Line, LeftVars, RightVars) -> - {match, Line, {tuple, Line, LeftVars}, {tuple, Line, RightVars}}. + {EClause, elixir_env:merge_and_check_unused_vars(SAcc, SA, EAcc)} + end, + {Values, SE} = lists:mapfoldl(Transformer, S, Clauses), + {{Key, Values}, SE}; +expand_clauses_origin(Meta, Kind, _Fun, {Key, _}, _, E) -> + file_error(Meta, E, ?MODULE, {bad_or_missing_clauses, {Kind, Key}}). + +assert_at_most_once(_Kind, [], _Count, _Fun) -> ok; +assert_at_most_once(Kind, [{Kind, _} | _], 1, ErrorFun) -> + ErrorFun(Kind); +assert_at_most_once(Kind, [{Kind, _} | Rest], Count, Fun) -> + assert_at_most_once(Kind, Rest, Count + 1, Fun); +assert_at_most_once(Kind, [_ | Rest], Count, Fun) -> + assert_at_most_once(Kind, Rest, Count, Fun). + +warn_catch_before_rescue([], _, _, _) -> + ok; +warn_catch_before_rescue([{'rescue', _} | _], Meta, E, true) -> + file_warn(Meta, ?key(E, file), ?MODULE, {catch_before_rescue, origin(Meta, 'try')}); +warn_catch_before_rescue([{'catch', _} | Rest], Meta, E, _) -> + warn_catch_before_rescue(Rest, Meta, E, true); +warn_catch_before_rescue([_ | Rest], Meta, E, Found) -> + warn_catch_before_rescue(Rest, Meta, E, Found). + +origin(Meta, Default) -> + case lists:keyfind(origin, 1, Meta) of + {origin, Origin} -> Origin; + false -> Default + end. -unblock({'block', _, Exprs}) -> Exprs; -unblock(Exprs) -> [Exprs]. +format_error({duplicate_match, Expr}) -> + String = 'Elixir.Macro':to_string(Expr), + io_lib:format( + "this pattern is matched against itself inside a match: ~ts = ~ts", + [String, String] + ); + +format_error({recursive, Vars, TypeExpr}) -> + Code = + case TypeExpr of + {match, Expr} -> 'Elixir.Macro':to_string(Expr); + {{Name, _Arity}, Args} -> 'Elixir.Macro':to_string({Name, [], Args}) + end, + + Message = + case lists:map(fun({Name, Context}) -> elixir_utils:var_info(Name, Context) end, lists:sort(Vars)) of + [Var] -> + io_lib:format("the variable ~ts is defined in function of itself", [Var]); + [Var1, Var2] -> + io_lib:format("the variable ~ts is defined recursively in function of ~ts", [Var1, Var2]); + [Head | Tail] -> + List = lists:foldl(fun(X, Acc) -> [Acc, $,, $\s, X] end, Head, Tail), + io_lib:format("the following variables form a cycle: ~ts", [List]) + end, + + io_lib:format( + "recursive variable definition in patterns:~n~n~ts~n~n~ts", + [Code, Message] + ); + +format_error({bad_or_missing_clauses, {Kind, Key}}) -> + io_lib:format("invalid \"~ts\" block in \"~ts\", it expects \"pattern -> expr\" clauses", [Key, Kind]); + +format_error({duplicated_clauses, Kind, Key}) -> + io_lib:format("duplicate \"~ts\" clauses given for \"~ts\"", [Key, Kind]); + +format_error({unexpected_option, Kind, Option}) -> + io_lib:format("unexpected option ~ts in \"~ts\"", ['Elixir.Macro':to_string(Option), Kind]); + +format_error({wrong_number_of_args_for_clause, Expected, Kind, Key}) -> + io_lib:format("expected ~ts for \"~ts\" clauses (->) in \"~ts\"", [Expected, Key, Kind]); + +format_error(multiple_after_clauses_in_receive) -> + "expected a single -> clause for :after in \"receive\""; + +format_error({invalid_rescue_clause, Arg}) -> + io_lib:format( + "invalid \"rescue\" clause. The clause should match on an alias, a variable " + "or be in the \"var in [alias]\" format. Got: ~ts", + ['Elixir.Macro':to_string(Arg)] + ); + +format_error({catch_before_rescue, Origin}) -> + io_lib:format("\"catch\" should always come after \"rescue\" in ~ts", [Origin]); + +format_error({try_with_only_else_clause, Origin}) -> + io_lib:format("\"else\" shouldn't be used as the only clause in \"~ts\", use \"case\" instead", + [Origin]); + +format_error(unmatchable_else_in_with) -> + "\"else\" clauses will never match because all patterns in \"with\" will always match"; + +format_error({zero_list_length_in_guard, ListArg}) -> + Arg = 'Elixir.Macro':to_string(ListArg), + io_lib:format("do not use \"length(~ts) == 0\" to check if a list is empty since length " + "always traverses the whole list. Prefer to pattern match on an empty list or " + "use \"~ts == []\" as a guard", [Arg, Arg]); + +format_error({positive_list_length_in_guard, ListArg}) -> + Arg = 'Elixir.Macro':to_string(ListArg), + io_lib:format("do not use \"length(~ts) > 0\" to check if a list is not empty since length " + "always traverses the whole list. Prefer to pattern match on a non-empty list, " + "such as [_ | _], or use \"~ts != []\" as a guard", [Arg, Arg]). diff --git a/lib/elixir/src/elixir_code_server.erl b/lib/elixir/src/elixir_code_server.erl index 1a1f5e08bf5..4185cebd659 100644 --- a/lib/elixir/src/elixir_code_server.erl +++ b/lib/elixir/src/elixir_code_server.erl @@ -1,18 +1,18 @@ +%% SPDX-License-Identifier: Apache-2.0 +%% SPDX-FileCopyrightText: 2021 The Elixir Team +%% SPDX-FileCopyrightText: 2012 Plataformatec + -module(elixir_code_server). -export([call/1, cast/1]). -export([start_link/0, init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). -behaviour(gen_server). --define(timeout, 30000). +-define(timeout, infinity). -record(elixir_code_server, { - compilation_status=[], - argv=[], - loaded=[], - at_exit=[], - pool={[],0}, - compiler_options=[{docs,true},{debug_info,true},{warnings_as_errors,false}], - erl_compiler_options=nil + required=#{}, + mod_pool={[], [], 0}, + mod_ets=#{} }). call(Args) -> @@ -27,114 +27,112 @@ start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, ok, []). init(ok) -> - code:ensure_loaded('Elixir.Macro.Env'), - code:ensure_loaded('Elixir.Module.LocalsTracker'), - code:ensure_loaded('Elixir.Kernel.LexicalTracker'), + %% The table where we store module definitions + _ = ets:new(elixir_modules, [set, public, named_table, {read_concurrency, true}]), {ok, #elixir_code_server{}}. +handle_call({defmodule, Module, Pid, Tuple}, _From, Config) -> + case ets:lookup(elixir_modules, Module) of + [] -> + {Ref, NewConfig} = defmodule(Pid, Tuple, Config), + {reply, {ok, Ref}, NewConfig}; + [CurrentTuple] -> + {reply, {error, CurrentTuple}, Config} + end; + +handle_call({undefmodule, Ref}, _From, Config) -> + {reply, ok, undefmodule(Ref, Config)}; + handle_call({acquire, Path}, From, Config) -> - Current = Config#elixir_code_server.loaded, - case orddict:find(Path, Current) of + Current = Config#elixir_code_server.required, + case maps:find(Path, Current) of {ok, true} -> - {reply, loaded, Config}; - {ok, {Ref, List}} when is_list(List), is_reference(Ref) -> - Queued = orddict:store(Path, {Ref, [From|List]}, Current), - {reply, {queued, Ref}, Config#elixir_code_server{loaded=Queued}}; + {reply, required, Config}; + {ok, Queued} when is_list(Queued) -> + Required = maps:put(Path, [From | Queued], Current), + {noreply, Config#elixir_code_server{required=Required}}; error -> - Queued = orddict:store(Path, {make_ref(), []}, Current), - {reply, proceed, Config#elixir_code_server{loaded=Queued}} + Required = maps:put(Path, [], Current), + {reply, proceed, Config#elixir_code_server{required=Required}} end; -handle_call(loaded, _From, Config) -> - {reply, [F || {F, true} <- Config#elixir_code_server.loaded], Config}; - -handle_call(at_exit, _From, Config) -> - {reply, Config#elixir_code_server.at_exit, Config}; - -handle_call(flush_at_exit, _From, Config) -> - {reply, Config#elixir_code_server.at_exit, Config#elixir_code_server{at_exit=[]}}; +handle_call(required, _From, Config) -> + {reply, [F || {F, true} <- maps:to_list(Config#elixir_code_server.required)], Config}; -handle_call(argv, _From, Config) -> - {reply, Config#elixir_code_server.argv, Config}; - -handle_call(compiler_options, _From, Config) -> - {reply, Config#elixir_code_server.compiler_options, Config}; - -handle_call({compilation_status, CompilerPid}, _From, Config) -> - CompilationStatusList = Config#elixir_code_server.compilation_status, - CompilationStatusListNew = orddict:erase(CompilerPid, CompilationStatusList), - CompilationStatus = orddict:fetch(CompilerPid, CompilationStatusList), - {reply, CompilationStatus, Config#elixir_code_server{compilation_status=CompilationStatusListNew}}; - -handle_call(retrieve_module_name, _From, Config) -> - case Config#elixir_code_server.pool of - {[H|T], Counter} -> - {reply, module_tuple(H), Config#elixir_code_server{pool={T,Counter}}}; - {[], Counter} -> - {reply, module_tuple(Counter), Config#elixir_code_server{pool={[],Counter+1}}} +handle_call(retrieve_compiler_module, _From, Config) -> + case Config#elixir_code_server.mod_pool of + {Used, [Mod | Unused], Counter} -> + {reply, Mod, Config#elixir_code_server{mod_pool={Used, Unused, Counter}}}; + {Used, [], Counter} -> + {reply, compiler_module(Counter), Config#elixir_code_server{mod_pool={Used, [], Counter+1}}} end; -handle_call(erl_compiler_options, _From, Config) -> - case Config#elixir_code_server.erl_compiler_options of - nil -> - Opts = erl_compiler_options(), - {reply, Opts, Config#elixir_code_server{erl_compiler_options=Opts}}; - Opts -> - {reply, Opts, Config} - end; +handle_call(purge_compiler_modules, _From, Config) -> + {Used, Unused, Counter} = Config#elixir_code_server.mod_pool, + _ = [code:purge(Module) || Module <- Used], + ModPool = {[], Used ++ Unused, Counter}, + {reply, {ok, length(Used)}, Config#elixir_code_server{mod_pool=ModPool}}; handle_call(Request, _From, Config) -> {stop, {badcall, Request}, Config}. -handle_cast({at_exit, AtExit}, Config) -> - {noreply, Config#elixir_code_server{at_exit=[AtExit|Config#elixir_code_server.at_exit]}}; - -handle_cast({argv, Argv}, Config) -> - {noreply, Config#elixir_code_server{argv=Argv}}; - -handle_cast({compiler_options, Options}, Config) -> - Final = orddict:merge(fun(_,_,V) -> V end, Config#elixir_code_server.compiler_options, Options), - {noreply, Config#elixir_code_server{compiler_options=Final}}; - -handle_cast({register_warning, CompilerPid}, Config) -> - CompilationStatusCurrent = Config#elixir_code_server.compilation_status, - CompilationStatusNew = orddict:store(CompilerPid, error, CompilationStatusCurrent), - case orddict:find(warnings_as_errors, Config#elixir_code_server.compiler_options) of - {ok, true} -> {noreply, Config#elixir_code_server{compilation_status=CompilationStatusNew}}; - _ -> {noreply, Config} - end; - -handle_cast({reset_warnings, CompilerPid}, Config) -> - CompilationStatusCurrent = Config#elixir_code_server.compilation_status, - CompilationStatusNew = orddict:store(CompilerPid, ok, CompilationStatusCurrent), - {noreply, Config#elixir_code_server{compilation_status=CompilationStatusNew}}; - -handle_cast({loaded, Path}, Config) -> - Current = Config#elixir_code_server.loaded, - case orddict:find(Path, Current) of +handle_cast({required, Path}, Config) -> + Current = Config#elixir_code_server.required, + case maps:find(Path, Current) of {ok, true} -> {noreply, Config}; - {ok, {Ref, List}} when is_list(List), is_reference(Ref) -> - [Pid ! {elixir_code_server, Ref, loaded} || {Pid, _Tag} <- lists:reverse(List)], - Done = orddict:store(Path, true, Current), - {noreply, Config#elixir_code_server{loaded=Done}}; + {ok, Queued} -> + _ = [gen_server:reply(From, required) || From <- lists:reverse(Queued)], + Done = maps:put(Path, true, Current), + {noreply, Config#elixir_code_server{required=Done}}; error -> - Done = orddict:store(Path, true, Current), - {noreply, Config#elixir_code_server{loaded=Done}} + Done = maps:put(Path, true, Current), + {noreply, Config#elixir_code_server{required=Done}} end; -handle_cast({unload_files, Files}, Config) -> - Current = Config#elixir_code_server.loaded, - Unloaded = lists:foldl(fun(File, Acc) -> orddict:erase(File, Acc) end, Current, Files), - {noreply, Config#elixir_code_server{loaded=Unloaded}}; - -handle_cast({return_module_name, H}, #elixir_code_server{pool={T,Counter}} = Config) -> - {noreply, Config#elixir_code_server{pool={[H|T],Counter}}}; +handle_cast({unrequire_files, Files}, Config) -> + Current = Config#elixir_code_server.required, + Unrequired = maps:without(Files, Current), + {noreply, Config#elixir_code_server{required=Unrequired}}; + +handle_cast({return_compiler_module, Module}, Config) -> + {Used, Unused, Counter} = Config#elixir_code_server.mod_pool, + ModPool = {[Module | Used], Unused, Counter}, + {noreply, Config#elixir_code_server{mod_pool=ModPool}}; + +handle_cast(purge_compiler_modules, Config) -> + {Used, Unused, Counter} = Config#elixir_code_server.mod_pool, + + case Used of + [] -> ok; + _ -> + %% Purging modules became more expensive in Erlang/OTP 27+, + %% so we accumulate them all during compilation and then + %% purge them asynchronously, especially because they can + %% block the code server. Ideally we would purge them in + %% batches, but that's not supported at the moment. + Opts = [{monitor, [{tag, {purged, Used}}]}], + erlang:spawn_opt(fun() -> + [code:purge(Module) || Module <- Used], + ok + end, Opts) + end, + + ModPool = {[], Unused, Counter}, + {noreply, Config#elixir_code_server{mod_pool=ModPool}}; handle_cast(Request, Config) -> {stop, {badcast, Request}, Config}. -handle_info(_Request, Config) -> +handle_info({{purged, Purged}, _Ref, process, _Pid, _Reason}, Config) -> + {Used, Unused, Counter} = Config#elixir_code_server.mod_pool, + ModPool = {Used, Purged ++ Unused, Counter}, + {noreply, Config#elixir_code_server{mod_pool=ModPool}}; + +handle_info({'DOWN', Ref, process, _Pid, _Reason}, Config) -> + {noreply, undefmodule(Ref, Config)}; + +handle_info(_Msg, Config) -> {noreply, Config}. terminate(_Reason, _Config) -> @@ -143,25 +141,20 @@ terminate(_Reason, _Config) -> code_change(_Old, Config, _Extra) -> {ok, Config}. -module_tuple(I) -> - {list_to_atom("elixir_compiler_" ++ integer_to_list(I)), I}. - -erl_compiler_options() -> - Key = "ERL_COMPILER_OPTIONS", - case os:getenv(Key) of - false -> []; - Str when is_list(Str) -> - case erl_scan:string(Str) of - {ok,Tokens,_} -> - case erl_parse:parse_term(Tokens ++ [{dot, 1}]) of - {ok,List} when is_list(List) -> List; - {ok,Term} -> [Term]; - {error,_Reason} -> - io:format("Ignoring bad term in ~ts\n", [Key]), - [] - end; - {error, {_,_,_Reason}, _} -> - io:format("Ignoring bad term in ~ts\n", [Key]), - [] - end +compiler_module(I) -> + list_to_atom("elixir_compiler_" ++ integer_to_list(I)). + +defmodule(Pid, Tuple, #elixir_code_server{mod_ets=ModEts} = Config) -> + ets:insert(elixir_modules, Tuple), + Ref = erlang:monitor(process, Pid), + Mod = erlang:element(1, Tuple), + {Ref, Config#elixir_code_server{mod_ets=maps:put(Ref, Mod, ModEts)}}. + +undefmodule(Ref, #elixir_code_server{mod_ets=ModEts} = Config) -> + case maps:find(Ref, ModEts) of + {ok, Mod} -> + ets:delete(elixir_modules, Mod), + Config#elixir_code_server{mod_ets=maps:remove(Ref, ModEts)}; + error -> + Config end. diff --git a/lib/elixir/src/elixir_compiler.erl b/lib/elixir/src/elixir_compiler.erl index cd5f52700a2..f2b4add4801 100644 --- a/lib/elixir/src/elixir_compiler.erl +++ b/lib/elixir/src/elixir_compiler.erl @@ -1,224 +1,229 @@ +%% SPDX-License-Identifier: Apache-2.0 +%% SPDX-FileCopyrightText: 2021 The Elixir Team +%% SPDX-FileCopyrightText: 2012 Plataformatec + +%% Elixir compiler front-end to the Erlang backend. -module(elixir_compiler). --export([get_opt/1, string/2, quoted/2, file/1, file_to_path/2]). --export([core/0, module/4, eval_forms/3]). +-export([string/3, quoted/3, bootstrap/0, file/2, compile/4]). -include("elixir.hrl"). -%% Public API - -get_opt(Key) -> - Dict = elixir_code_server:call(compiler_options), - case lists:keyfind(Key, 1, Dict) of - false -> false; - {Key, Value} -> Value - end. +string(Contents, File, Callback) -> + Forms = elixir:'string_to_quoted!'(Contents, 1, 1, File, elixir_config:get(parser_options)), + quoted(Forms, File, Callback). -%% Compilation entry points. +quoted(Forms, File, Callback) -> + Previous = get(elixir_module_binaries), -string(Contents, File) when is_list(Contents), is_binary(File) -> - Forms = elixir:'string_to_quoted!'(Contents, 1, File, []), - quoted(Forms, File). + try + put(elixir_module_binaries, []), + Env = (elixir_env:new())#{line := 1, file := File, tracers := elixir_config:get(tracers)}, -quoted(Forms, File) when is_binary(File) -> - Previous = get(elixir_compiled), + elixir_lexical:run( + Env, + fun (LexicalEnv) -> maybe_fast_compile(Forms, LexicalEnv) end, + fun (#{lexical_tracker := Pid}) -> Callback(File, Pid) end + ), - try - put(elixir_compiled, []), - elixir_lexical:run(File, fun - (Pid) -> - Env = elixir:env_for_eval([{line,1},{file,File}]), - eval_forms(Forms, [], Env#{lexical_tracker := Pid}) - end), - lists:reverse(get(elixir_compiled)) + lists:reverse(get(elixir_module_binaries)) after - put(elixir_compiled, Previous) + put(elixir_module_binaries, Previous) end. -file(Relative) when is_binary(Relative) -> - File = filename:absname(Relative), +file(File, Callback) -> {ok, Bin} = file:read_file(File), - string(elixir_utils:characters_to_list(Bin), File). + string(elixir_utils:characters_to_list(Bin), File, Callback). + +%% Evaluates the given code through the Erlang compiler. +%% It may end-up evaluating the code if it is deemed a +%% more efficient strategy depending on the code snippet. +maybe_fast_compile(Forms, E) -> + case (?key(E, module) == nil) andalso allows_fast_compilation(Forms) andalso + (not elixir_config:is_bootstrap()) of + true -> fast_compile(Forms, E); + false -> compile(Forms, [], [], E) + end, + ok. -file_to_path(File, Path) when is_binary(File), is_binary(Path) -> - Lists = file(File), - [binary_to_path(X, Path) || X <- Lists], - Lists. +compile(Quoted, ArgsList, CompilerOpts, #{line := Line} = E) -> + Block = no_tail_optimize([{line, Line}], Quoted), + {Expanded, SE, EE} = elixir_expand:expand(Block, elixir_env:env_to_ex(E), E), + elixir_env:check_unused_vars(SE, EE), -%% Evaluation + {Module, Fun, LabelledLocals} = + elixir_erl_compiler:spawn(fun() -> spawned_compile(Expanded, CompilerOpts, E) end), -eval_forms(Forms, Vars, E) -> - case (?m(E, module) == nil) andalso allows_fast_compilation(Forms) of - true -> eval_compilation(Forms, Vars, E); - false -> code_loading_compilation(Forms, Vars, E) - end. + Args = list_to_tuple(ArgsList), + {dispatch(Module, Fun, Args, LabelledLocals), SE, EE}. + +spawned_compile(ExExprs, CompilerOpts, #{line := Line, file := File} = E) -> + {Vars, S} = elixir_erl_var:from_env(E), + {ErlExprs, _} = elixir_erl_pass:translate(ExExprs, erl_anno:new(Line), S), + + Module = retrieve_compiler_module(), + Fun = code_fun(?key(E, module)), + Forms = code_mod(Fun, ErlExprs, Line, File, Module, Vars), + + {Module, Binary} = elixir_erl_compiler:noenv_forms(Forms, File, [nowarn_nomatch | CompilerOpts]), + code:load_binary(Module, "", Binary), + {Module, Fun, is_purgeable(Binary)}. + +is_purgeable(<<"FOR1", _Size:32, "BEAM", Rest/binary>>) -> + do_is_purgeable(Rest). + +do_is_purgeable(<<>>) -> true; +do_is_purgeable(<<"LocT", 4:32, 0:32, _/binary>>) -> true; +do_is_purgeable(<<"LocT", _:32, _/binary>>) -> false; +do_is_purgeable(<<_:4/binary, Size:32, Beam/binary>>) -> + <<_:(4 * trunc((Size+3) / 4))/binary, Rest/binary>> = Beam, + do_is_purgeable(Rest). -eval_compilation(Forms, Vars, E) -> - Binding = [{Key, Value} || {_Name, _Kind, Key, Value} <- Vars], - {Result, _Binding, EE, _S} = elixir:eval_forms(Forms, Binding, E), - {Result, EE}. - -code_loading_compilation(Forms, Vars, #{line := Line} = E) -> - Dict = [{{Name, Kind}, {Value, 0}} || {Name, Kind, Value, _} <- Vars], - S = elixir_env:env_to_scope_with_vars(E, Dict), - {Expr, EE, _S} = elixir:quoted_to_erl(Forms, E, S), - - {Module, I} = retrieve_module_name(), - Fun = code_fun(?m(E, module)), - Form = code_mod(Fun, Expr, Line, ?m(E, file), Module, Vars), - Args = list_to_tuple([V || {_, _, _, V} <- Vars]), - - %% Pass {native, false} to speed up bootstrap - %% process when native is set to true - AllOpts = elixir_code_server:call(erl_compiler_options), - FinalOpts = AllOpts -- [native, warn_missing_spec], - module(Form, ?m(E, file), FinalOpts, true, fun(_, Binary) -> - %% If we have labeled locals, anonymous functions - %% were created and therefore we cannot ditch the - %% module - Purgeable = - case beam_lib:chunks(Binary, [labeled_locals]) of - {ok, {_, [{labeled_locals, []}]}} -> true; - _ -> false - end, - dispatch_loaded(Module, Fun, Args, Purgeable, I, EE) - end). - -dispatch_loaded(Module, Fun, Args, Purgeable, I, E) -> +dispatch(Module, Fun, Args, Purgeable) -> Res = Module:Fun(Args), code:delete(Module), - if Purgeable -> - code:purge(Module), - return_module_name(I); - true -> - ok - end, - {Res, E}. + Purgeable andalso return_compiler_module(Module), + Res. code_fun(nil) -> '__FILE__'; code_fun(_) -> '__MODULE__'. code_mod(Fun, Expr, Line, File, Module, Vars) when is_binary(File), is_integer(Line) -> - Tuple = {tuple, Line, [{var, Line, K} || {_, _, K, _} <- Vars]}, + Ann = erl_anno:new(Line), + Tuple = {tuple, Ann, [{var, Ann, Var} || {_, Var} <- Vars]}, Relative = elixir_utils:relative_to_cwd(File), - [ - {attribute, Line, file, {elixir_utils:characters_to_list(Relative), 1}}, - {attribute, Line, module, Module}, - {attribute, Line, export, [{Fun, 1}, {'__RELATIVE__', 0}]}, - {function, Line, Fun, 1, [ - {clause, Line, [Tuple], [], [Expr]} - ]}, - {function, Line, '__RELATIVE__', 0, [ - {clause, Line, [], [], [elixir_utils:elixir_to_erl(Relative)]} - ]} - ]. + [{attribute, Ann, file, {elixir_utils:characters_to_list(Relative), 1}}, + {attribute, Ann, module, Module}, + {attribute, Ann, compile, no_auto_import}, + {attribute, Ann, export, [{Fun, 1}, {'__RELATIVE__', 0}]}, + {function, Ann, Fun, 1, [ + {clause, Ann, [Tuple], [], [Expr]} + ]}, + {function, Ann, '__RELATIVE__', 0, [ + {clause, Ann, [], [], [elixir_erl:elixir_to_erl(Relative)]} + ]}]. -retrieve_module_name() -> - elixir_code_server:call(retrieve_module_name). +retrieve_compiler_module() -> + elixir_code_server:call(retrieve_compiler_module). -return_module_name(I) -> - elixir_code_server:cast({return_module_name, I}). +return_compiler_module(Module) -> + elixir_code_server:cast({return_compiler_module, Module}). allows_fast_compilation({'__block__', _, Exprs}) -> lists:all(fun allows_fast_compilation/1, Exprs); -allows_fast_compilation({defmodule,_,_}) -> true; -allows_fast_compilation(_) -> false. - -%% INTERNAL API - -%% Compile the module by forms based on the scope information -%% executes the callback in case of success. This automatically -%% handles errors and warnings. Used by this module and elixir_module. -module(Forms, File, Opts, Callback) -> - Final = - case (get_opt(debug_info) == true) orelse - lists:member(debug_info, Opts) of - true -> [debug_info] ++ elixir_code_server:call(erl_compiler_options); - false -> elixir_code_server:call(erl_compiler_options) - end, - module(Forms, File, Final, false, Callback). - -module(Forms, File, Options, Bootstrap, Callback) when - is_binary(File), is_list(Forms), is_list(Options), is_boolean(Bootstrap), is_function(Callback) -> - Listname = elixir_utils:characters_to_list(File), - - case compile:noenv_forms([no_auto_import()|Forms], [return,{source,Listname}|Options]) of - {ok, ModuleName, Binary, Warnings} -> - format_warnings(Bootstrap, Warnings), - code:load_binary(ModuleName, Listname, Binary), - Callback(ModuleName, Binary); - {error, Errors, Warnings} -> - format_warnings(Bootstrap, Warnings), - format_errors(Errors) - end. - -no_auto_import() -> - {attribute, 0, compile, no_auto_import}. - -%% CORE HANDLING - -core() -> - application:start(elixir), - elixir_code_server:cast({compiler_options, [{docs,false},{internal,true}]}), - [core_file(File) || File <- core_main()]. +allows_fast_compilation({defmodule, _, [_, [{do, _}]]}) -> + true; +allows_fast_compilation(_) -> + false. + +fast_compile({'__block__', _, Exprs}, E) -> + lists:foldl(fun(Expr, _) -> fast_compile(Expr, E) end, nil, Exprs); +fast_compile({defmodule, Meta, [Mod, [{do, Block}]]}, NoLineE) -> + E = NoLineE#{line := ?line(Meta)}, + + Expanded = case Mod of + {'__aliases__', AliasMeta, List} -> + case elixir_aliases:expand_or_concat(AliasMeta, List, E, true) of + Receiver when is_atom(Receiver) -> Receiver; + _ -> 'Elixir.Macro':expand(Mod, E) + end; + + _ -> + 'Elixir.Macro':expand(Mod, E) + end, -core_file(File) -> + ContextModules = [Expanded | ?key(E, context_modules)], + elixir_module:compile(Meta, Expanded, Block, [], false, E#{context_modules := ContextModules}). + +no_tail_optimize(Meta, Block) -> + {'__block__', Meta, [ + {'=', Meta, [{result, Meta, ?MODULE}, Block]}, + {{'.', Meta, [elixir_utils, noop]}, Meta, []}, + {result, Meta, ?MODULE} + ]}. + +%% Bootstrapper + +bootstrap() -> + {ok, _} = application:ensure_all_started(elixir), + elixir_config:static(#{bootstrap => true}), + elixir_config:put(docs, false), + elixir_config:put(ignore_module_conflict, true), + elixir_config:put(on_undefined_variable, raise), + elixir_config:put(parser_options, []), + elixir_config:put(relative_paths, false), + elixir_config:put(tracers, []), + elixir_config:put(infer_signatures, []), + {Init, Main} = bootstrap_files(), + {ok, Cwd} = file:get_cwd(), + Lib = filename:join(Cwd, "lib/elixir/lib"), + [bootstrap_file(Lib, File) || File <- [<<"kernel.ex">> | Init]], + [bootstrap_file(Lib, File) || File <- [<<"kernel.ex">> | Main]]. + +bootstrap_file(Lib, Suffix) -> try - Lists = file(File), - [binary_to_path(X, "lib/elixir/ebin") || X <- Lists], - io:format("Compiled ~ts~n", [File]) + File = filename:join(Lib, Suffix), + Mods = file(File, fun(_, _) -> ok end), + _ = [binary_to_path(X, "lib/elixir/ebin") || X <- Mods], + io:format("Compiled ~ts~n", [Suffix]) catch - Kind:Reason -> - io:format("~p: ~p~nstacktrace: ~p~n", [Kind, Reason, erlang:get_stacktrace()]), + Kind:Reason:Stacktrace -> + io:format("~p: ~p~nstacktrace: ~p~n", [Kind, Reason, Stacktrace]), erlang:halt(1) end. -core_main() -> - [<<"lib/elixir/lib/kernel.ex">>, - <<"lib/elixir/lib/macro/env.ex">>, - <<"lib/elixir/lib/keyword.ex">>, - <<"lib/elixir/lib/module.ex">>, - <<"lib/elixir/lib/list.ex">>, - <<"lib/elixir/lib/macro.ex">>, - <<"lib/elixir/lib/code.ex">>, - <<"lib/elixir/lib/module/locals_tracker.ex">>, - <<"lib/elixir/lib/kernel/typespec.ex">>, - <<"lib/elixir/lib/exception.ex">>, - <<"lib/elixir/lib/protocol.ex">>, - <<"lib/elixir/lib/stream/reducers.ex">>, - <<"lib/elixir/lib/enum.ex">>, - <<"lib/elixir/lib/inspect/algebra.ex">>, - <<"lib/elixir/lib/inspect.ex">>, - <<"lib/elixir/lib/range.ex">>, - <<"lib/elixir/lib/regex.ex">>, - <<"lib/elixir/lib/string.ex">>, - <<"lib/elixir/lib/string/chars.ex">>, - <<"lib/elixir/lib/io.ex">>, - <<"lib/elixir/lib/path.ex">>, - <<"lib/elixir/lib/file.ex">>, - <<"lib/elixir/lib/system.ex">>, - <<"lib/elixir/lib/kernel/cli.ex">>, - <<"lib/elixir/lib/kernel/error_handler.ex">>, - <<"lib/elixir/lib/kernel/parallel_compiler.ex">>, - <<"lib/elixir/lib/kernel/lexical_tracker.ex">>]. +bootstrap_files() -> + { + [ + <<"kernel/utils.ex">>, + <<"macro/env.ex">>, + <<"range.ex">>, + <<"keyword.ex">>, + <<"module.ex">>, + <<"list.ex">>, + <<"macro.ex">>, + <<"kernel/typespec.ex">>, + <<"code.ex">>, + <<"code/identifier.ex">>, + <<"protocol.ex">>, + <<"stream/reducers.ex">>, + <<"enum.ex">>, + <<"regex.ex">>, + <<"inspect/algebra.ex">>, + <<"inspect.ex">>, + <<"string.ex">>, + <<"string/chars.ex">> + ], + [ + <<"list/chars.ex">>, + <<"bitwise.ex">>, + <<"module/parallel_checker.ex">>, + <<"module/behaviour.ex">>, + <<"module/types/helpers.ex">>, + <<"module/types/descr.ex">>, + <<"module/types/of.ex">>, + <<"module/types/pattern.ex">>, + <<"module/types/apply.ex">>, + <<"module/types/expr.ex">>, + <<"module/types.ex">>, + <<"exception.ex">>, + <<"path.ex">>, + <<"file.ex">>, + <<"map.ex">>, + <<"access.ex">>, + <<"io.ex">>, + <<"system.ex">>, + <<"code/formatter.ex">>, + <<"code/normalizer.ex">>, + <<"kernel/cli.ex">>, + <<"kernel/error_handler.ex">>, + <<"kernel/parallel_compiler.ex">>, + <<"kernel/lexical_tracker.ex">> + ] + }. binary_to_path({ModuleName, Binary}, CompilePath) -> Path = filename:join(CompilePath, atom_to_list(ModuleName) ++ ".beam"), - ok = file:write_file(Path, Binary), - Path. - -%% ERROR HANDLING - -format_errors([]) -> - exit({nocompile, "compilation failed but no error was raised"}); - -format_errors(Errors) -> - lists:foreach(fun ({File, Each}) -> - BinFile = elixir_utils:characters_to_binary(File), - lists:foreach(fun (Error) -> elixir_errors:handle_file_error(BinFile, Error) end, Each) - end, Errors). - -format_warnings(Bootstrap, Warnings) -> - lists:foreach(fun ({File, Each}) -> - BinFile = elixir_utils:characters_to_binary(File), - lists:foreach(fun (Warning) -> elixir_errors:handle_file_warning(Bootstrap, BinFile, Warning) end, Each) - end, Warnings). + case file:write_file(Path, Binary) of + ok -> Path; + {error, Reason} -> error('Elixir.File.Error':exception([{action, "write to"}, {path, Path}, {reason, Reason}])) + end. diff --git a/lib/elixir/src/elixir_config.erl b/lib/elixir/src/elixir_config.erl new file mode 100644 index 00000000000..9b956365d83 --- /dev/null +++ b/lib/elixir/src/elixir_config.erl @@ -0,0 +1,92 @@ +%% SPDX-License-Identifier: Apache-2.0 +%% SPDX-FileCopyrightText: 2021 The Elixir Team +%% SPDX-FileCopyrightText: 2012 Plataformatec + +-module(elixir_config). +-compile({no_auto_import, [get/1]}). +-export([new/1, warn/2, serial/1]). +-export([static/1, is_bootstrap/0, identifier_tokenizer/0]). +-export([delete/1, put/2, get/1, get/2, update/2, get_and_put/2]). +-export([start_link/0, init/1, handle_call/3, handle_cast/2]). +-behaviour(gen_server). + +%% Persistent term + +static(Map) when is_map(Map) -> + persistent_term:put(?MODULE, maps:merge(persistent_term:get(?MODULE, #{}), Map)). +is_bootstrap() -> + maps:get(bootstrap, persistent_term:get(?MODULE, #{}), false). +identifier_tokenizer() -> + maps:get(identifier_tokenizer, persistent_term:get(?MODULE, #{}), 'Elixir.String.Tokenizer'). + +%% Key-value store (concurrent reads, serial writes) + +get(Key) -> + [{_, Value}] = ets:lookup(?MODULE, Key), + Value. + +get(Key, Default) -> + try ets:lookup(?MODULE, Key) of + [{_, Value}] -> Value; + [] -> Default + catch + _:_ -> Default + end. + +put(Key, Value) -> + gen_server:call(?MODULE, {put, Key, Value}, infinity). + +get_and_put(Key, Value) -> + gen_server:call(?MODULE, {get_and_put, Key, Value}, infinity). + +update(Key, Fun) -> + gen_server:call(?MODULE, {update, Key, Fun}, infinity). + +new(Opts) -> + Tab = ets:new(?MODULE, [named_table, public, {read_concurrency, true}]), + true = ets:insert_new(?MODULE, Opts), + Tab. + +delete(?MODULE) -> + ets:delete(?MODULE). + +%% MISC + +serial(Fun) -> + gen_server:call(?MODULE, {serial, Fun}, infinity). + +%% Used to guarantee warnings are emitted only once per caller. +warn(Key, [{Mod, Fun, ArgsOrArity, _} | _]) -> + EtsKey = {warn, Key, Mod, Fun, to_arity(ArgsOrArity)}, + ets:update_counter(?MODULE, EtsKey, {2, 1, 1, 1}, {EtsKey, -1}) =:= 0; + +warn(_, _) -> + true. + +to_arity(Args) when is_list(Args) -> length(Args); +to_arity(Arity) -> Arity. + +%% gen_server api + +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, ?MODULE, []). + +init(?MODULE) -> + {ok, []}. + +handle_call({serial, Fun}, _From, State) -> + {reply, Fun(), State}; + handle_call({put, Key, Value}, _From, State) -> + ets:insert(?MODULE, {Key, Value}), + {reply, ok, State}; +handle_call({update, Key, Fun}, _From, State) -> + Value = Fun(get(Key)), + ets:insert(?MODULE, {Key, Value}), + {reply, Value, State}; +handle_call({get_and_put, Key, Value}, _From, State) -> + OldValue = get(Key), + ets:insert(?MODULE, {Key, Value}), + {reply, OldValue, State}. + +handle_cast(Cast, Tab) -> + {stop, {bad_cast, Cast}, Tab}. diff --git a/lib/elixir/src/elixir_counter.erl b/lib/elixir/src/elixir_counter.erl deleted file mode 100644 index 35cfd160338..00000000000 --- a/lib/elixir/src/elixir_counter.erl +++ /dev/null @@ -1,38 +0,0 @@ --module(elixir_counter). --export([start_link/0, init/1, handle_call/3, handle_cast/2, - handle_info/2, terminate/2, code_change/3, next/0]). --behaviour(gen_server). - --define(timeout, 1000). %% 1 second --define(limit, 4294967296). %% 2^32 - -next() -> - gen_server:call(?MODULE, next, ?timeout). - -start_link() -> - gen_server:start_link({local, ?MODULE}, ?MODULE, 0, []). - -init(Counter) -> - {ok, Counter}. - -handle_call(next, _From, Counter) -> - {reply, Counter, bump(Counter)}; -handle_call(Request, _From, Counter) -> - {stop, {badcall, Request}, Counter}. - -handle_cast(Request, Counter) -> - {stop, {badcast, Request}, Counter}. - -handle_info(_Request, Counter) -> - {noreply, Counter}. - -terminate(_Reason, _Counter) -> - ok. - -code_change(_Old, Counter, _Extra) -> - {ok, Counter}. - -bump(Counter) when Counter < ?limit -> - Counter + 1; -bump(_Counter) -> - 0. diff --git a/lib/elixir/src/elixir_def.erl b/lib/elixir/src/elixir_def.erl index e7c134e851a..d4859dcb513 100644 --- a/lib/elixir/src/elixir_def.erl +++ b/lib/elixir/src/elixir_def.erl @@ -1,386 +1,505 @@ +%% SPDX-License-Identifier: Apache-2.0 +%% SPDX-FileCopyrightText: 2021 The Elixir Team +%% SPDX-FileCopyrightText: 2012 Plataformatec + % Holds the logic responsible for function definitions (def(p) and defmacro(p)). -module(elixir_def). --export([table/1, clauses_table/1, setup/1, - cleanup/1, reset_last/1, lookup_definition/2, - delete_definition/2, store_definition/6, unwrap_definitions/1, - store_each/8, format_error/1]). +-export([setup/1, reset_last/1, local_for/5, external_for/5, + take_definition/2, store_definition/3, store_definition/9, + fetch_definitions/2, format_error/1]). -include("elixir.hrl"). +-define(last_def, {elixir, last_def}). --define(attr, '__def_table'). --define(clauses_attr, '__clauses_table'). - -%% Table management functions. Called internally. +setup(DataTables) -> + reset_last(DataTables), + ok. -table(Module) -> - ets:lookup_element(Module, ?attr, 2). +reset_last({DataSet, _DataBag}) -> + ets:insert(DataSet, {?last_def, none}); -clauses_table(Module) -> - ets:lookup_element(Module, ?clauses_attr, 2). +reset_last(Module) when is_atom(Module) -> + reset_last(elixir_module:data_tables(Module)). -setup(Module) -> - ets:insert(Module, {?attr, ets:new(Module, [set, public])}), - ets:insert(Module, {?clauses_attr, ets:new(Module, [bag, public])}), - reset_last(Module), - ok. +%% Finds the local definition for the current module. +local_for(Meta, Name, Arity, Kinds, E) -> + External = fun({Mod, Fun}, Args) -> + invoke_external([{from_macro, true} | Meta], Mod, Fun, Args, E) + end, + fun_for(Meta, ?key(E, module), Name, Arity, Kinds, {value, External}). -cleanup(Module) -> - ets:delete(table(Module)), - ets:delete(clauses_table(Module)). +%% Finds the local definition for an external module. +external_for(Meta, Module, Name, Arity, Kinds) -> + fun_for(Meta, Module, Name, Arity, Kinds, none). -%% Reset the last item. Useful when evaling code. -reset_last(Module) -> - ets:insert(table(Module), {last, []}). +fun_for(Meta, Module, Name, Arity, Kinds, External) -> + Tuple = {Name, Arity}, -%% Looks up a definition from the database. -lookup_definition(Module, Tuple) -> - case ets:lookup(table(Module), Tuple) of - [Result] -> - CTable = clauses_table(Module), - {Result, [Clause || {_, Clause} <- ets:lookup(CTable, Tuple)]}; - _ -> + try + {Set, Bag} = elixir_module:data_tables(Module), + {ets:lookup(Set, {def, Tuple}), ets:lookup(Bag, {clauses, Tuple})} + of + {[{_, Kind, LocalMeta, _, _, _}], ClausesPairs} -> + case (Kinds == all) orelse (lists:member(Kind, Kinds)) of + true -> + (Kind == defmacrop) andalso track_defmacrop(Module, Tuple), + Local = {value, fun(Fun, Args) -> invoke_local(Meta, Module, Fun, Args, External) end}, + Clauses = [Clause || {_, Clause} <- ClausesPairs], + elixir_erl:definition_to_anonymous(Kind, LocalMeta, Clauses, Local, External); + false -> + false + end; + {[], _} -> false + catch + _:_ -> false + end. + +invoke_local(Meta, Module, ErlName, Args, External) -> + {Name, Arity} = elixir_utils:erl_fa_to_elixir_fa(ErlName, length(Args)), + + case fun_for(Meta, Module, Name, Arity, all, External) of + false -> + {current_stacktrace, [_ | T]} = erlang:process_info(self(), current_stacktrace), + erlang:raise(error, undef, [{Module, Name, Arity, []} | T]); + Fun -> + apply(Fun, Args) end. -delete_definition(Module, Tuple) -> - ets:delete(table(Module), Tuple), - ets:delete(clauses_table(Module), Tuple). +track_defmacrop(Module, FunArity) -> + {_, Bag} = elixir_module:data_tables(Module), + ets:insert(Bag, {used_private, FunArity}). -% Invoked by the wrap definition with the function abstract tree. -% Each function is then added to the function table. +invoke_external(Meta, Mod, Name, Args, E) -> + is_map(E) andalso elixir_env:trace({require, Meta, Mod, []}, E), + apply(Mod, Name, Args). -store_definition(Line, Kind, CheckClauses, Call, Body, Pos) -> - E = (elixir_locals:get_cached_env(Pos))#{line := Line}, - {NameAndArgs, Guards} = elixir_clauses:extract_guards(Call), +%% Take a definition out of the table - {Name, Args} = case NameAndArgs of - {N, _, A} when is_atom(N), is_atom(A) -> {N, []}; - {N, _, A} when is_atom(N), is_list(A) -> {N, A}; - _ -> elixir_errors:form_error(Line, ?m(E, file), ?MODULE, {invalid_def, Kind, NameAndArgs}) - end, +take_definition(Module, {Name, Arity} = Tuple) -> + {Set, Bag} = elixir_module:data_tables(Module), + case ets:take(Set, {def, Tuple}) of + [{{def, Tuple}, _, _, _, _, {Defaults, _, _}} = Result] -> + ets:delete_object(Bag, {defs, Tuple}), + ets:delete_object(Bag, {{default, Name}, Arity, Defaults}), + {Result, [Clause || {_, Clause} <- ets:take(Bag, {clauses, Tuple})]}; + [] -> + false + end. - %% Now that we have verified the call format, - %% extract meta information like file and context. - {_, Meta, _} = Call, - DoCheckClauses = (not lists:keymember(context, 1, Meta)) andalso (CheckClauses), +%% Fetch all available definitions - %% Check if there is a file information in the definition. - %% If so, we assume this come from another source and we need - %% to linify taking into account keep line numbers. - {File, Key} = case lists:keyfind(file, 1, Meta) of - {file, Bin} when is_binary(Bin) -> {Bin, keep}; - _ -> {nil, line} +fetch_definitions(Module, E) -> + {Set, Bag} = elixir_module:data_tables(Module), + + Entries = try + lists:sort(ets:lookup_element(Bag, defs, 2)) + catch + error:badarg -> [] end, - LinifyArgs = elixir_quote:linify(Line, Key, Args), - LinifyGuards = elixir_quote:linify(Line, Key, Guards), - LinifyBody = elixir_quote:linify(Line, Key, Body), + fetch_definition(Entries, E, Module, Set, Bag, [], []). + +fetch_definition([Tuple | T], E, Module, Set, Bag, All, Private) -> + [{_, Kind, Meta, _, Check, {MaxDefaults, _, Defaults}}] = ets:lookup(Set, {def, Tuple}), + + try ets:lookup_element(Bag, {clauses, Tuple}, 2) of + Clauses -> + NewAll = + [{Tuple, Kind, add_defaults_to_meta(MaxDefaults, Meta), Clauses} | All], + NewPrivate = + case (Kind == defp) orelse (Kind == defmacrop) of + true -> + Metas = head_and_definition_meta(Check, Meta, MaxDefaults - Defaults, All), + [{Tuple, Kind, Metas, MaxDefaults} | Private]; + false -> + Private + end, + fetch_definition(T, E, Module, Set, Bag, NewAll, NewPrivate) + catch + error:badarg -> + elixir_errors:module_error(Meta, E, ?MODULE, {function_head, Kind, Tuple}), + fetch_definition(T, E, Module, Set, Bag, All, Private) + end; - assert_no_aliases_name(Line, Name, Args, E), - store_definition(Line, Kind, DoCheckClauses, Name, - LinifyArgs, LinifyGuards, LinifyBody, File, E). +fetch_definition([], _E, _Module, _Set, _Bag, All, Private) -> + {All, Private}. -store_definition(Line, Kind, CheckClauses, Name, Args, Guards, Body, MetaFile, #{module := Module} = ER) -> - Arity = length(Args), - Tuple = {Name, Arity}, - E = ER#{function := Tuple}, - elixir_locals:record_definition(Tuple, Kind, Module), +add_defaults_to_meta(0, Meta) -> Meta; +add_defaults_to_meta(Defaults, Meta) -> [{defaults, Defaults} | Meta]. - Location = retrieve_location(Line, MetaFile, Module), - {Function, Defaults, Super} = translate_definition(Kind, Line, Module, Name, Args, Guards, Body, E), +head_and_definition_meta(none, _Meta, _HeadDefaults, _All) -> + false; +head_and_definition_meta(_, Meta, 0, _All) -> + Meta; +head_and_definition_meta(_, _Meta, _HeadDefaults, [{_, _, HeadMeta, _} | _]) -> + HeadMeta. + +%% Section for storing definitions + +store_definition(Kind, {Call, Body}, Pos) -> + E = elixir_module:get_cached_env(Pos), + store_definition(Kind, false, Call, Body, E); +store_definition(Kind, Key, Pos) -> + #{module := Module} = E = elixir_module:get_cached_env(Pos), + {Call, Body} = elixir_module:read_cache(Module, Key), + store_definition(Kind, true, Call, Body, E). + +store_definition(Kind, HasNoUnquote, Call, Body, #{line := Line} = E) -> + {NameAndArgs, Guards} = elixir_utils:extract_guards(Call), + + {Name, Meta, Args} = case NameAndArgs of + {N, M, A} when is_atom(N), is_atom(A) -> {N, M, []}; + {N, M, A} when is_atom(N), is_list(A) -> {N, M, A}; + _ -> elixir_errors:file_error([{line, Line}], E, ?MODULE, {invalid_def, Kind, NameAndArgs}) + end, - DefaultsLength = length(Defaults), - elixir_locals:record_defaults(Tuple, Kind, Module, DefaultsLength), + Context = case lists:keyfind(context, 1, Meta) of + {context, _} = ContextPair -> [ContextPair]; + _ -> [] + end, - File = ?m(E, file), - Table = table(Module), - CTable = clauses_table(Module), + Column = case lists:keyfind(column, 1, Meta) of + {column, _} = ColumnPair -> [ColumnPair | Context]; + _ -> Context + end, - compile_super(Module, Super, E), - check_previous_defaults(Table, Line, Name, Arity, Kind, DefaultsLength, E), + Generated = case lists:keyfind(generated, 1, Meta) of + {generated, true} = GeneratedPair -> [GeneratedPair | Column]; + _ -> Column + end, - store_each(CheckClauses, Kind, File, Location, - Table, CTable, DefaultsLength, Function), - [store_each(false, Kind, File, Location, Table, CTable, 0, - default_function_for(Kind, Name, Default)) || Default <- Defaults], + CheckClauses = if + Context /= [] -> none; + HasNoUnquote -> all; + true -> unused_only + end, - make_struct_available(Kind, Module, Name, Args), - {Name, Arity}. + %% Check if there is a file information in the definition. + %% If so, we assume this come from another source and + %% we need to linify taking into account keep line numbers. + %% + %% Line and File will always point to the caller. __ENV__.line + %% will always point to the quoted one and __ENV__.file will + %% always point to the one at @file or the quoted one. + {Location, LinifyArgs, LinifyGuards, LinifyBody} = + case elixir_utils:meta_keep(Meta) of + {_, _} = MetaFile -> + {MetaFile, + elixir_quote:linify(Line, keep, Args), + elixir_quote:linify(Line, keep, Guards), + elixir_quote:linify(Line, keep, Body)}; + nil -> + {nil, Args, Guards, Body} + end, -%% @on_definition + Arity = length(Args), -run_on_definition_callbacks(Kind, Line, Module, Name, Args, Guards, Expr, E) -> - case elixir_compiler:get_opt(internal) of - true -> - ok; - _ -> - Env = elixir_env:linify({Line, E}), - Callbacks = 'Elixir.Module':get_attribute(Module, on_definition), - [Mod:Fun(Env, Kind, Name, Args, Guards, Expr) || {Mod, Fun} <- Callbacks] - end. + {File, DefMeta} = + case retrieve_location(Location, ?key(E, module)) of + {AF, RF, L} -> + {AF, [{line, Line}, {file, {RF, L}} | Generated]}; + nil -> + {nil, [{line, Line} | Generated]} + end, -make_struct_available(def, Module, '__struct__', []) -> - case erlang:get(elixir_compiler_pid) of - undefined -> ok; - Pid -> Pid ! {struct_available, Module} - end; -make_struct_available(_, _, _, _) -> - ok. + run_with_location_change(File, E, fun(EL) -> + assert_no_aliases_name(DefMeta, Name, Args, EL), + assert_valid_name(DefMeta, Kind, Name, Args, EL), + store_definition(DefMeta, Kind, CheckClauses, Name, Arity, + LinifyArgs, LinifyGuards, LinifyBody, ?key(E, file), EL) + end). + +store_definition(Meta, Kind, CheckClauses, Name, Arity, DefaultsArgs, Guards, Body, File, ER) -> + Module = ?key(ER, module), + Tuple = {Name, Arity}, + {S, E} = env_for_expansion(Kind, Tuple, ER), -%% Retrieve location from meta file or @file, otherwise nil + {Args, Defaults} = unpack_defaults(Kind, Meta, Name, DefaultsArgs, S, E), + Clauses = [elixir_clauses:def(Clause, S, E) || + Clause <- def_to_clauses(Kind, Meta, Args, Guards, Body, E)], -retrieve_location(Line, File, Module) -> - case get_location_attribute(Module) of - nil when not is_binary(File) -> + DefaultsLength = length(Defaults), + check_previous_defaults(Meta, Module, Name, Arity, Kind, DefaultsLength, E), + + store_definition(CheckClauses, Kind, Meta, Name, Arity, File, + Module, DefaultsLength, Clauses), + [store_definition(none, Kind, Meta, Name, length(DefaultArgs), File, + Module, 0, [Default]) || {_, DefaultArgs, _, _} = Default <- Defaults], + + run_on_definition_callbacks(Meta, Kind, Module, Name, DefaultsArgs, Guards, Body, E), + Tuple. + +env_for_expansion(Kind, Tuple, E) when Kind =:= defmacro; Kind =:= defmacrop -> + S = elixir_env:env_to_ex(E), + {S#elixir_ex{caller=true}, E#{function := Tuple}}; +env_for_expansion(_Kind, Tuple, E) -> + {elixir_env:env_to_ex(E), E#{function := Tuple}}. + +retrieve_location(Location, Module) -> + {Set, _} = elixir_module:data_tables(Module), + case ets:take(Set, file) of + [] when is_tuple(Location) -> + {File, Line} = Location, + {filename:absname(File), elixir_utils:relative_to_cwd(File), Line}; + [] -> nil; - nil -> - {normalize_location(File), Line}; - X when is_binary(X) -> + [{file, File, _, _}] when is_binary(File) -> 'Elixir.Module':delete_attribute(Module, file), - {normalize_location(X), 0}; - {X, L} when is_binary(X) andalso is_integer(L) -> + {filename:absname(File), elixir_utils:relative_to_cwd(File), 0}; + [{file, {File, Line}, _, _}] when is_binary(File) andalso is_integer(Line) -> 'Elixir.Module':delete_attribute(Module, file), - {normalize_location(X), L} + {filename:absname(File), elixir_utils:relative_to_cwd(File), Line} end. -get_location_attribute(Module) -> - case elixir_compiler:get_opt(internal) of - true -> nil; - false -> 'Elixir.Module':get_attribute(Module, file) +run_with_location_change(nil, E, Callback) -> + Callback(E); +run_with_location_change(File, #{file := File} = E, Callback) -> + Callback(E); +run_with_location_change(File, E, Callback) -> + elixir_lexical:with_file(File, E, Callback). + +def_to_clauses(_Kind, Meta, Args, [], nil, E) -> + check_args_for_function_head(Meta, Args, E), + []; +def_to_clauses(_Kind, Meta, Args, Guards, [{do, Body}], _E) -> + [{Meta, Args, Guards, Body}]; +def_to_clauses(Kind, Meta, Args, Guards, Body, E) -> + case is_list(Body) andalso lists:keyfind(do, 1, Body) of + {do, _} -> + [{Meta, Args, Guards, {'try', [{origin, Kind} | Meta], [Body]}}]; + false -> + elixir_errors:file_error(Meta, E, elixir_expand, {missing_option, Kind, [do]}) end. -normalize_location(X) -> - elixir_utils:characters_to_list(elixir_utils:relative_to_cwd(X)). - -%% Compile super - -compile_super(Module, true, #{function := Function}) -> - elixir_def_overridable:store(Module, Function, true); -compile_super(_Module, _, _E) -> ok. - -%% Translate the given call and expression given -%% and then store it in memory. - -translate_definition(Kind, Line, Module, Name, Args, Guards, Body, E) when is_integer(Line) -> - Arity = length(Args), - - {EArgs, EGuards, EBody, _} = elixir_exp_clauses:def(fun elixir_def_defaults:expand/2, - Args, Guards, expr_from_body(Line, Body), E), - - Body == nil andalso check_args_for_bodyless_clause(Line, EArgs, E), - - S = elixir_env:env_to_scope(E), - {Unpacked, Defaults} = elixir_def_defaults:unpack(Kind, Name, EArgs, S), - {Clauses, Super} = translate_clause(Body, Line, Kind, Unpacked, EGuards, EBody, S), - - run_on_definition_callbacks(Kind, Line, Module, Name, EArgs, EGuards, EBody, E), - Function = {function, Line, Name, Arity, Clauses}, - {Function, Defaults, Super}. +run_on_definition_callbacks(Meta, Kind, Module, Name, Args, Guards, Body, E) -> + {_, Bag} = elixir_module:data_tables(Module), + Callbacks = ets:lookup_element(Bag, {accumulate, on_definition}, 2), + _ = [begin + elixir_env:trace({remote_function, Meta, Mod, Fun, 6}, E), + Mod:Fun(E, Kind, Name, Args, Guards, Body) + end || {Mod, Fun} <- lists:reverse(Callbacks)], + ok. -translate_clause(nil, _Line, _Kind, _Args, [], _Body, _S) -> - {[], false}; -translate_clause(nil, Line, Kind, _Args, _Guards, _Body, #elixir_scope{file=File}) -> - elixir_errors:form_error(Line, File, ?MODULE, {missing_do, Kind}); -translate_clause(_, Line, Kind, Args, Guards, Body, S) -> - {TClause, TS} = elixir_clauses:clause(Line, - fun elixir_translator:translate_args/2, Args, Body, Guards, true, S), +store_definition(CheckClauses, Kind, Meta, Name, Arity, File, Module, Defaults, Clauses) + when CheckClauses == all; CheckClauses == none; CheckClauses == unused_only -> + {Set, Bag} = elixir_module:data_tables(Module), + Tuple = {Name, Arity}, + HasBody = Clauses =/= [], + CheckAll = CheckClauses == all, - FClause = case is_macro(Kind) of + if + Defaults > 0 -> + ets:insert(Bag, {{default, Name}, Arity, Defaults}); true -> - FArgs = {var, Line, '_@CALLER'}, - MClause = setelement(3, TClause, [FArgs|element(3, TClause)]), - - case TS#elixir_scope.caller of - true -> - FBody = {'match', Line, - {'var', Line, '__CALLER__'}, - elixir_utils:erl_call(Line, elixir_env, linify, [{var, Line, '_@CALLER'}]) - }, - setelement(5, MClause, [FBody|element(5, TClause)]); - false -> - MClause - end; - false -> - TClause + ok end, - {[FClause], TS#elixir_scope.super}. - -expr_from_body(_Line, nil) -> nil; -expr_from_body(_Line, [{do, Expr}]) -> Expr; -expr_from_body(Line, Else) -> {'try', [{line,Line}], [Else]}. - -is_macro(defmacro) -> true; -is_macro(defmacrop) -> true; -is_macro(_) -> false. - -% Unwrap the functions stored in the functions table. -% It returns a list of all functions to be exported, plus the macros, -% and the body of all functions. -unwrap_definitions(Module) -> - Table = table(Module), - CTable = clauses_table(Module), - ets:delete(Table, last), - unwrap_definition(ets:tab2list(Table), CTable, [], [], [], [], [], [], []). - -unwrap_definition([Fun|T], CTable, All, Exports, Private, Def, Defmacro, Functions, Tail) -> - Tuple = element(1, Fun), - Clauses = [Clause || {_, Clause} <- ets:lookup(CTable, Tuple)], - - {NewFun, NewExports, NewPrivate, NewDef, NewDefmacro} = - case Clauses of - [] -> {false, Exports, Private, Def, Defmacro}; - _ -> unwrap_definition(element(2, Fun), Tuple, Fun, Exports, Private, Def, Defmacro) + {MaxDefaults, FirstMeta} = + case ets:lookup(Set, {def, Tuple}) of + [{_, StoredKind, StoredMeta, StoredFile, StoredCheck, {StoredDefaults, LastHasBody, LastDefaults}}] -> + check_valid_kind(Meta, File, Name, Arity, Kind, StoredKind, StoredFile, StoredMeta), + check_valid_defaults(Meta, File, Name, Arity, Kind, Defaults, StoredMeta, StoredDefaults, LastDefaults, HasBody, LastHasBody), + (CheckAll and (StoredCheck == all)) andalso + check_valid_clause(Meta, File, Name, Arity, Kind, Set, StoredMeta, StoredFile, Clauses), + + {max(Defaults, StoredDefaults), StoredMeta}; + [] -> + ets:insert(Bag, {defs, Tuple}), + {Defaults, Meta} end, - {NewFunctions, NewTail} = case NewFun of - false -> - NewAll = All, - {Functions, Tail}; - _ -> - NewAll = [Tuple|All], - function_for_stored_definition(NewFun, Clauses, Functions, Tail) - end, - - unwrap_definition(T, CTable, NewAll, NewExports, NewPrivate, - NewDef, NewDefmacro, NewFunctions, NewTail); -unwrap_definition([], _CTable, All, Exports, Private, Def, Defmacro, Functions, Tail) -> - {All, Exports, Private, ordsets:from_list(Def), - ordsets:from_list(Defmacro), lists:reverse(Tail ++ Functions)}. - -unwrap_definition(def, Tuple, Fun, Exports, Private, Def, Defmacro) -> - {Fun, [Tuple|Exports], Private, [Tuple|Def], Defmacro}; -unwrap_definition(defmacro, {Name, Arity} = Tuple, Fun, Exports, Private, Def, Defmacro) -> - Macro = {elixir_utils:macro_name(Name), Arity + 1}, - {setelement(1, Fun, Macro), [Macro|Exports], Private, Def, [Tuple|Defmacro]}; -unwrap_definition(defp, Tuple, Fun, Exports, Private, Def, Defmacro) -> - %% {Name, Arity}, Kind, Line, Check, Defaults - Info = {Tuple, defp, element(3, Fun), element(5, Fun), element(7, Fun)}, - {Fun, Exports, [Info|Private], Def, Defmacro}; -unwrap_definition(defmacrop, Tuple, Fun, Exports, Private, Def, Defmacro) -> - %% {Name, Arity}, Kind, Line, Check, Defaults - Info = {Tuple, defmacrop, element(3, Fun), element(5, Fun), element(7, Fun)}, - {false, Exports, [Info|Private], Def, Defmacro}. - -%% Helpers - -function_for_stored_definition({{Name,Arity}, _, Line, _, _, nil, _}, Clauses, Functions, Tail) -> - {[{function, Line, Name, Arity, Clauses}|Functions], Tail}; - -function_for_stored_definition({{Name,Arity}, _, Line, _, _, Location, _}, Clauses, Functions, Tail) -> - {Functions, [ - {function, Line, Name, Arity, Clauses}, - {attribute, Line, file, Location} | Tail - ]}. - -default_function_for(Kind, Name, {clause, Line, Args, _Guards, _Exprs} = Clause) - when Kind == defmacro; Kind == defmacrop -> - {function, Line, Name, length(Args) - 1, [Clause]}; - -default_function_for(_, Name, {clause, Line, Args, _Guards, _Exprs} = Clause) -> - {function, Line, Name, length(Args), [Clause]}. - -%% Store each definition in the table. -%% This function also checks and emit warnings in case -%% the kind, of the visibility of the function changes. - -store_each(Check, Kind, File, Location, Table, CTable, Defaults, {function, Line, Name, Arity, Clauses}) -> - Tuple = {Name, Arity}, - case ets:lookup(Table, Tuple) of - [{Tuple, StoredKind, StoredLine, StoredFile, StoredCheck, StoredLocation, StoredDefaults}] -> - FinalLine = StoredLine, - FinalLocation = StoredLocation, - FinalDefaults = max(Defaults, StoredDefaults), - check_valid_kind(Line, File, Name, Arity, Kind, StoredKind), - (Check and StoredCheck) andalso - check_valid_clause(Line, File, Name, Arity, Kind, Table, StoredLine, StoredFile), - check_valid_defaults(Line, File, Name, Arity, Kind, Defaults, StoredDefaults); - [] -> - FinalLine = Line, - FinalLocation = Location, - FinalDefaults = Defaults - end, - Check andalso ets:insert(Table, {last, {Name, Arity}}), - ets:insert(CTable, [{Tuple, Clause} || Clause <- Clauses ]), - ets:insert(Table, {Tuple, Kind, FinalLine, File, Check, FinalLocation, FinalDefaults}). + CheckAll andalso ets:insert(Set, {?last_def, Tuple}), + ets:insert(Bag, [{{clauses, Tuple}, Clause} || Clause <- Clauses]), + ets:insert(Set, {{def, Tuple}, Kind, FirstMeta, File, CheckClauses, {MaxDefaults, HasBody, Defaults}}). + +%% Handling of defaults + +unpack_defaults(Kind, Meta, Name, Args, S, E) -> + {Expanded, #elixir_ex{unused={_, VersionOffset}}} = expand_defaults(Args, S, E#{context := nil}, []), + unpack_expanded(Kind, Meta, Name, Expanded, VersionOffset, [], []). + +unpack_expanded(Kind, Meta, Name, [{'\\\\', DefaultMeta, [Expr, _]} | T] = List, VersionOffset, Acc, Clauses) -> + Base = match_defaults(Acc, length(Acc) + VersionOffset, []), + {Args, Invoke} = extract_defaults(List, length(Base) + VersionOffset, [], []), + Clause = {Meta, Base ++ Args, [], {super, [{super, {Kind, Name}}, {default, true} | DefaultMeta], Base ++ Invoke}}, + unpack_expanded(Kind, Meta, Name, T, VersionOffset, [Expr | Acc], [Clause | Clauses]); +unpack_expanded(Kind, Meta, Name, [H | T], VersionOffset, Acc, Clauses) -> + unpack_expanded(Kind, Meta, Name, T, VersionOffset, [H | Acc], Clauses); +unpack_expanded(_Kind, _Meta, _Name, [], _VersionOffset, Acc, Clauses) -> + {lists:reverse(Acc), lists:reverse(Clauses)}. + +expand_defaults([{'\\\\', Meta, [Expr, Default]} | Args], S, E, Acc) -> + {ExpandedDefault, SE, _} = elixir_expand:expand(Default, S, E), + expand_defaults(Args, SE, E, [{'\\\\', Meta, [Expr, ExpandedDefault]} | Acc]); +expand_defaults([Arg | Args], S, E, Acc) -> + expand_defaults(Args, S, E, [Arg | Acc]); +expand_defaults([], S, _E, Acc) -> + {lists:reverse(Acc), S}. + +extract_defaults([{'\\\\', _, [_Expr, Default]} | T], Counter, NewArgs, NewInvoke) -> + extract_defaults(T, Counter, NewArgs, [Default | NewInvoke]); +extract_defaults([_ | T], Counter, NewArgs, NewInvoke) -> + H = default_var(Counter), + extract_defaults(T, Counter + 1, [H | NewArgs], [H | NewInvoke]); +extract_defaults([], _Counter, NewArgs, NewInvoke) -> + {lists:reverse(NewArgs), lists:reverse(NewInvoke)}. + +match_defaults([], _Counter, Acc) -> + Acc; +match_defaults([_ | T], Counter, Acc) -> + NewCounter = Counter - 1, + match_defaults(T, NewCounter, [default_var(NewCounter) | Acc]). + +default_var(Counter) -> + {list_to_atom([$x | integer_to_list(Counter)]), [{generated, true}, {version, Counter}], ?var_context}. %% Validations -check_valid_kind(_Line, _File, _Name, _Arity, Kind, Kind) -> []; -check_valid_kind(Line, File, Name, Arity, Kind, StoredKind) -> - elixir_errors:form_error(Line, File, ?MODULE, - {changed_kind, {Name, Arity, StoredKind, Kind}}). +check_valid_kind(_Meta, _File, _Name, _Arity, Kind, Kind, _StoredFile, _StoredMeta) -> ok; +check_valid_kind(Meta, File, Name, Arity, Kind, StoredKind, StoredFile, StoredMeta) -> + elixir_errors:file_error(Meta, File, ?MODULE, + {changed_kind, {Name, Arity, StoredKind, Kind, StoredFile, ?line(StoredMeta)}}). -check_valid_clause(Line, File, Name, Arity, Kind, Table, StoredLine, StoredFile) -> - case ets:lookup_element(Table, last, 2) of - {Name,Arity} -> []; - [] -> []; +check_valid_clause(Meta, File, Name, Arity, Kind, Set, StoredMeta, StoredFile, Clauses) -> + case ets:lookup_element(Set, ?last_def, 2) of + none -> + ok; + {Name, Arity} when Clauses == [] -> + elixir_errors:file_warn(Meta, File, ?MODULE, + {late_function_head, {Kind, Name, Arity}}); + {Name, Arity} -> + ok; + {Name, _} -> + Relative = elixir_utils:relative_to_cwd(StoredFile), + elixir_errors:file_warn(Meta, File, ?MODULE, + {ungrouped_name, {Kind, Name, Arity, ?line(StoredMeta), Relative}}); _ -> Relative = elixir_utils:relative_to_cwd(StoredFile), - elixir_errors:handle_file_warning(File, {Line, ?MODULE, - {ungrouped_clause, {Kind, Name, Arity, StoredLine, Relative}}}) + elixir_errors:file_warn(Meta, File, ?MODULE, + {ungrouped_arity, {Kind, Name, Arity, ?line(StoredMeta), Relative}}) end. -check_valid_defaults(_Line, _File, _Name, _Arity, _Kind, 0, _) -> []; -check_valid_defaults(Line, File, Name, Arity, Kind, _, 0) -> - elixir_errors:handle_file_warning(File, {Line, ?MODULE, {out_of_order_defaults, {Kind, Name, Arity}}}); -check_valid_defaults(Line, File, Name, Arity, Kind, _, _) -> - elixir_errors:form_error(Line, File, ?MODULE, {clauses_with_defaults, {Kind, Name, Arity}}). +% Clause with defaults after clause with defaults +check_valid_defaults(Meta, File, Name, Arity, Kind, Defaults, StoredMeta, StoredDefaults, _, _, _) + when Defaults > 0, StoredDefaults > 0 -> + elixir_errors:file_error(Meta, File, ?MODULE, {duplicate_defaults, {Kind, Name, Arity, StoredMeta}}); +% Clause with defaults after clause without defaults +check_valid_defaults(Meta, File, Name, Arity, Kind, Defaults, StoredMeta, 0, _, _, _) when Defaults > 0 -> + elixir_errors:file_warn(Meta, File, ?MODULE, {mixed_defaults, {Kind, Name, Arity, StoredMeta}}); +% Clause without defaults directly after clause with defaults (bodiless does not count) +check_valid_defaults(Meta, File, Name, Arity, Kind, 0, StoredMeta, _, LastDefaults, true, true) when LastDefaults > 0 -> + elixir_errors:file_warn(Meta, File, ?MODULE, {mixed_defaults, {Kind, Name, Arity, StoredMeta}}); +% Clause without defaults +check_valid_defaults(_Meta, _File, _Name, _Arity, _Kind, 0, _StoredMeta, _StoredDefaults, _LastDefaults, _HasBody, _LastHasBody) -> + ok. + +check_args_for_function_head(Meta, Args, E) -> + [begin + elixir_errors:module_error(Meta, E, ?MODULE, invalid_args_for_function_head) + end || Arg <- Args, invalid_arg(Arg)]. + +invalid_arg({Name, _, Kind}) when is_atom(Name), is_atom(Kind) -> false; +invalid_arg(_) -> true. -check_previous_defaults(Table, Line, Name, Arity, Kind, Defaults, E) -> - Matches = ets:match(Table, {{Name, '$2'}, '$1', '_', '_', '_', '_', '$3'}), - [ begin - elixir_errors:form_error(Line, ?m(E, file), ?MODULE, - {defs_with_defaults, Name, {Kind, Arity}, {K, A}}) - end || [K, A, D] <- Matches, A /= Arity, D /= 0, defaults_conflict(A, D, Arity, Defaults)]. +check_previous_defaults(Meta, Module, Name, Arity, Kind, Defaults, E) -> + {_Set, Bag} = elixir_module:data_tables(Module), + Matches = ets:lookup(Bag, {default, Name}), + [begin + elixir_errors:file_error(Meta, E, ?MODULE, + {defs_with_defaults, Kind, Name, Arity, A}) + end || {_, A, D} <- Matches, A /= Arity, D /= 0, defaults_conflict(A, D, Arity, Defaults)]. defaults_conflict(A, D, Arity, Defaults) -> ((Arity >= (A - D)) andalso (Arity < A)) orelse ((A >= (Arity - Defaults)) andalso (A < Arity)). -check_args_for_bodyless_clause(Line, Args, E) -> - [ begin - elixir_errors:form_error(Line, ?m(E, file), ?MODULE, invalid_args_for_bodyless_clause) - end || Arg <- Args, invalid_arg(Arg) ]. - -invalid_arg({Name, _, Kind}) when is_atom(Name), is_atom(Kind) -> - false; -invalid_arg({'\\\\', _, [{Name, _, Kind}, _]}) when is_atom(Name), is_atom(Kind) -> - false; -invalid_arg(_) -> - true. - -assert_no_aliases_name(Line, '__aliases__', [Atom], #{file := File}) when is_atom(Atom) -> - elixir_errors:form_error(Line, File, ?MODULE, {no_alias, Atom}); - +assert_no_aliases_name(Meta, '__aliases__', [Atom], #{file := File}) when is_atom(Atom) -> + elixir_errors:file_error(Meta, File, ?MODULE, {no_alias, Atom}); assert_no_aliases_name(_Meta, _Aliases, _Args, _S) -> ok. -%% Format errors - -format_error({no_module,{Kind,Name,Arity}}) -> - io_lib:format("cannot define function outside module, invalid scope for ~ts ~ts/~B", [Kind, Name, Arity]); - -format_error({defs_with_defaults, Name, {Kind, Arity}, {K, A}}) when Arity > A -> - io_lib:format("~ts ~ts/~B defaults conflicts with ~ts ~ts/~B", - [Kind, Name, Arity, K, Name, A]); - -format_error({defs_with_defaults, Name, {Kind, Arity}, {K, A}}) when Arity < A -> - io_lib:format("~ts ~ts/~B conflicts with defaults from ~ts ~ts/~B", - [Kind, Name, Arity, K, Name, A]); +assert_valid_name(Meta, Kind, '__info__', [_], #{file := File}) -> + elixir_errors:file_error(Meta, File, ?MODULE, {'__info__', Kind}); +assert_valid_name(Meta, Kind, 'module_info', [], #{file := File}) -> + elixir_errors:file_error(Meta, File, ?MODULE, {module_info, Kind, 0}); +assert_valid_name(Meta, Kind, 'module_info', [_], #{file := File}) -> + elixir_errors:file_error(Meta, File, ?MODULE, {module_info, Kind, 1}); +assert_valid_name(Meta, Kind, is_record, [_, _], #{file := File}) when Kind == defp; Kind == def -> + elixir_errors:file_error(Meta, File, ?MODULE, {is_record, Kind}); +assert_valid_name(_Meta, _Kind, _Name, _Args, _S) -> + ok. -format_error({clauses_with_defaults,{Kind,Name,Arity}}) -> - io_lib:format("~ts ~ts/~B has default values and multiple clauses, " - "define a function head with the defaults", [Kind, Name, Arity]); +%% Format errors -format_error({out_of_order_defaults,{Kind,Name,Arity}}) -> - io_lib:format("clause with defaults should be the first clause in ~ts ~ts/~B", [Kind, Name, Arity]); +format_error({function_head, Kind, {Name, Arity}}) -> + io_lib:format("implementation not provided for predefined ~ts ~ts/~B", [Kind, Name, Arity]); -format_error({ungrouped_clause,{Kind,Name,Arity,OrigLine,OrigFile}}) -> - io_lib:format("clauses for the same ~ts should be grouped together, ~ts ~ts/~B was previously defined (~ts:~B)", - [Kind, Kind, Name, Arity, OrigFile, OrigLine]); +format_error({no_module, {Kind, Name, Arity}}) -> + io_lib:format("cannot define function outside module, invalid scope for ~ts ~ts/~B", [Kind, Name, Arity]); -format_error({changed_kind,{Name,Arity,Previous,Current}}) -> - io_lib:format("~ts ~ts/~B already defined as ~ts", [Current, Name, Arity, Previous]); +format_error({defs_with_defaults, Kind, Name, Arity, A}) when Arity > A -> + io_lib:format("~ts ~ts/~B defaults conflicts with ~ts/~B", + [Kind, Name, Arity, Name, A]); + +format_error({defs_with_defaults, Kind, Name, Arity, A}) when Arity < A -> + io_lib:format("~ts ~ts/~B conflicts with defaults from ~ts/~B", + [Kind, Name, Arity, Name, A]); + +format_error({duplicate_defaults, {Kind, Name, Arity, StoredMeta}}) -> + io_lib:format( + "~ts ~ts/~B defines defaults multiple times. " + "Elixir allows defaults to be declared once per definition. Instead of:\n" + "\n" + " def foo(:first_clause, b \\\\ :default) do ... end\n" + " def foo(:second_clause, b \\\\ :default) do ... end\n" + "\n" + "one should write:\n" + "\n" + " def foo(a, b \\\\ :default)\n" + " def foo(:first_clause, b) do ... end\n" + " def foo(:second_clause, b) do ... end\n" + "~ts", + [Kind, Name, Arity, maybe_stored_meta_line(StoredMeta)]); + +format_error({mixed_defaults, {Kind, Name, Arity, StoredMeta}}) -> + io_lib:format( + "~ts ~ts/~B has multiple clauses and also declares default values. " + "In such cases, the default values should be defined in a header. Instead of:\n" + "\n" + " def foo(:first_clause, b \\\\ :default) do ... end\n" + " def foo(:second_clause, b) do ... end\n" + "\n" + "one should write:\n" + "\n" + " def foo(a, b \\\\ :default)\n" + " def foo(:first_clause, b) do ... end\n" + " def foo(:second_clause, b) do ... end\n" + "~ts", + [Kind, Name, Arity, maybe_stored_meta_line(StoredMeta)]); + +format_error({ungrouped_name, {Kind, Name, Arity, OrigLine, OrigFile}}) -> + io_lib:format("clauses with the same name should be grouped together, \"~ts ~ts/~B\" was previously defined (~ts:~B)", + [Kind, Name, Arity, OrigFile, OrigLine]); + +format_error({ungrouped_arity, {Kind, Name, Arity, OrigLine, OrigFile}}) -> + io_lib:format("clauses with the same name and arity (number of arguments) should be grouped together, \"~ts ~ts/~B\" was previously defined (~ts:~B)", + [Kind, Name, Arity, OrigFile, OrigLine]); + +format_error({late_function_head, {Kind, Name, Arity}}) -> + io_lib:format("function head for ~ts ~ts/~B must come at the top of its direct implementation. Instead of:\n" + "\n" + " def add(a, b), do: a + b\n" + " def add(a, b)\n" + "\n" + "one should write:\n" + "\n" + " def add(a, b)\n" + " def add(a, b), do: a + b\n", + [Kind, Name, Arity]); + +format_error({changed_kind, {Name, Arity, Previous, Current, OrigFile, OrigLine}}) -> + OrigFileRelative = elixir_utils:relative_to_cwd(OrigFile), + io_lib:format("~ts ~ts/~B already defined as ~ts in ~ts:~B", [Current, Name, Arity, Previous, OrigFileRelative, OrigLine]); format_error({no_alias, Atom}) -> io_lib:format("function names should start with lowercase characters or underscore, invalid name ~ts", [Atom]); @@ -388,8 +507,30 @@ format_error({no_alias, Atom}) -> format_error({invalid_def, Kind, NameAndArgs}) -> io_lib:format("invalid syntax in ~ts ~ts", [Kind, 'Elixir.Macro':to_string(NameAndArgs)]); -format_error(invalid_args_for_bodyless_clause) -> - "can use only variables and \\\\ as arguments of bodyless clause"; - -format_error({missing_do, Kind}) -> - io_lib:format("missing do keyword in ~ts", [Kind]). +format_error(invalid_args_for_function_head) -> + "patterns are not allowed in function head, only variables and default arguments (using \\\\)\n" + "\n" + "If you did not intend to define a function head, make sure your function " + "definition has the proper syntax by wrapping the arguments in parentheses " + "and using the do instruction accordingly:\n\n" + " def add(a, b), do: a + b\n\n" + " def add(a, b) do\n" + " a + b\n" + " end\n"; + +format_error({'__info__', Kind}) -> + io_lib:format("cannot define ~ts __info__/1 as it is automatically defined by Elixir", [Kind]); + +format_error({module_info, Kind, Arity}) -> + io_lib:format("cannot define ~ts module_info/~B as it is automatically defined by Erlang", [Kind, Arity]); + +format_error({is_record, Kind}) -> + io_lib:format("cannot define ~ts is_record/2 due to compatibility " + "with the Erlang compiler (it is a known limitation)", [Kind]). + +maybe_stored_meta_line(StoredMeta) -> + case lists:keyfind(line, 1, StoredMeta) of + {line, Line} when Line > 0 -> + "\nthe previous clause is defined on line " ++ integer_to_list(Line) ++ "\n"; + _ -> "" + end. diff --git a/lib/elixir/src/elixir_def_defaults.erl b/lib/elixir/src/elixir_def_defaults.erl deleted file mode 100644 index 0b0a2683c55..00000000000 --- a/lib/elixir/src/elixir_def_defaults.erl +++ /dev/null @@ -1,73 +0,0 @@ -% Handle default clauses for function definitions. --module(elixir_def_defaults). --export([expand/2, unpack/4]). --include("elixir.hrl"). - -expand(Args, E) -> - lists:mapfoldl(fun - ({'\\\\', Meta, [Left, Right]}, Acc) -> - {ELeft, EL} = elixir_exp:expand(Left, Acc), - {ERight, _} = elixir_exp:expand(Right, Acc#{context := nil}), - {{'\\\\', Meta, [ELeft, ERight]}, EL}; - (Left, Acc) -> - elixir_exp:expand(Left, Acc) - end, E, Args). - -unpack(Kind, Name, Args, S) -> - unpack_each(Kind, Name, Args, [], [], S). - -%% Helpers - -%% Unpack default from given args. -%% Returns the given arguments without their default -%% clauses and a list of clauses for the default calls. -unpack_each(Kind, Name, [{'\\\\', DefMeta, [Expr, _]}|T] = List, Acc, Clauses, S) -> - Base = wrap_kind(Kind, build_match(Acc, [])), - {Args, Invoke} = extract_defaults(List, length(Base), [], []), - - {DefArgs, SA} = elixir_clauses:match(fun elixir_translator:translate_args/2, Base ++ Args, S), - {DefInvoke, _} = elixir_translator:translate_args(Base ++ Invoke, SA), - - Line = ?line(DefMeta), - - Call = {call, Line, - {atom, Line, name_for_kind(Kind, Name)}, - DefInvoke - }, - - Clause = {clause, Line, DefArgs, [], [Call]}, - unpack_each(Kind, Name, T, [Expr|Acc], [Clause|Clauses], S); - -unpack_each(Kind, Name, [H|T], Acc, Clauses, S) -> - unpack_each(Kind, Name, T, [H|Acc], Clauses, S); - -unpack_each(_Kind, _Name, [], Acc, Clauses, _S) -> - {lists:reverse(Acc), lists:reverse(Clauses)}. - -% Extract default values from args following the current default clause. - -extract_defaults([{'\\\\', _, [_Expr, Default]}|T], Counter, NewArgs, NewInvoke) -> - extract_defaults(T, Counter, NewArgs, [Default|NewInvoke]); - -extract_defaults([_|T], Counter, NewArgs, NewInvoke) -> - H = {elixir_utils:atom_concat(["x", Counter]), [], nil}, - extract_defaults(T, Counter + 1, [H|NewArgs], [H|NewInvoke]); - -extract_defaults([], _Counter, NewArgs, NewInvoke) -> - {lists:reverse(NewArgs), lists:reverse(NewInvoke)}. - -% Build matches for all the previous argument until the current default clause. - -build_match([], Acc) -> Acc; - -build_match([_|T], Acc) -> - Var = {elixir_utils:atom_concat(["x", length(T)]), [], nil}, - build_match(T, [Var|Acc]). - -% Given the invoked function name based on the kind - -wrap_kind(Kind, Args) when Kind == defmacro; Kind == defmacrop -> [{c, [], nil}|Args]; -wrap_kind(_Kind, Args) -> Args. - -name_for_kind(Kind, Name) when Kind == defmacro; Kind == defmacrop -> elixir_utils:macro_name(Name); -name_for_kind(_Kind, Name) -> Name. \ No newline at end of file diff --git a/lib/elixir/src/elixir_def_overridable.erl b/lib/elixir/src/elixir_def_overridable.erl deleted file mode 100644 index f0cf2b1a7a6..00000000000 --- a/lib/elixir/src/elixir_def_overridable.erl +++ /dev/null @@ -1,75 +0,0 @@ -% Holds the logic responsible for defining overridable functions and handling super. --module(elixir_def_overridable). --export([store_pending/1, ensure_defined/4, - name/2, store/3, format_error/1]). --include("elixir.hrl"). - -overridable(Module) -> - ets:lookup_element(elixir_module:data_table(Module), '__overridable', 2). - -overridable(Module, Value) -> - ets:insert(elixir_module:data_table(Module), {'__overridable', Value}). - -%% Check if an overridable function is defined. - -ensure_defined(Meta, Module, Tuple, S) -> - Overridable = overridable(Module), - case orddict:find(Tuple, Overridable) of - {ok, {_, _, _, _}} -> ok; - _ -> elixir_errors:form_error(Meta, S#elixir_scope.file, ?MODULE, {no_super, Module, Tuple}) - end. - -%% Gets the name based on the function and stored overridables - -name(Module, Function) -> - name(Module, Function, overridable(Module)). - -name(_Module, {Name, _} = Function, Overridable) -> - {Count, _, _, _} = orddict:fetch(Function, Overridable), - elixir_utils:atom_concat([Name, " (overridable ", Count, ")"]). - -%% Store - -store(Module, Function, GenerateName) -> - Overridable = overridable(Module), - case orddict:fetch(Function, Overridable) of - {_Count, _Clause, _Neighbours, true} -> ok; - {Count, Clause, Neighbours, false} -> - overridable(Module, orddict:store(Function, {Count, Clause, Neighbours, true}, Overridable)), - {{{Name, Arity}, Kind, Line, File, _Check, Location, Defaults}, Clauses} = Clause, - - {FinalKind, FinalName} = case GenerateName of - true -> {defp, name(Module, Function, Overridable)}; - false -> {Kind, Name} - end, - - case code:is_loaded('Elixir.Module.LocalsTracker') of - {_, _} -> - 'Elixir.Module.LocalsTracker':reattach(Module, Kind, {Name, Arity}, Neighbours); - _ -> - ok - end, - - Def = {function, Line, FinalName, Arity, Clauses}, - elixir_def:store_each(false, FinalKind, File, Location, - elixir_def:table(Module), elixir_def:clauses_table(Module), Defaults, Def) - end. - -%% Store pending declarations that were not manually made concrete. - -store_pending(Module) -> - [store(Module, X, false) || {X, {_, _, _, false}} <- overridable(Module), - not 'Elixir.Module':'defines?'(Module, X)]. - -%% Error handling - -format_error({no_super, Module, {Name, Arity}}) -> - Bins = [format_fa(X) || {X, {_, _, _, _}} <- overridable(Module)], - Joined = 'Elixir.Enum':join(Bins, <<", ">>), - io_lib:format("no super defined for ~ts/~B in module ~ts. Overridable functions available are: ~ts", - [Name, Arity, elixir_aliases:inspect(Module), Joined]). - -format_fa({Name, Arity}) -> - A = atom_to_binary(Name, utf8), - B = integer_to_binary(Arity), - << A/binary, $/, B/binary >>. \ No newline at end of file diff --git a/lib/elixir/src/elixir_dispatch.erl b/lib/elixir/src/elixir_dispatch.erl index e4cbac4610f..436396857b5 100644 --- a/lib/elixir/src/elixir_dispatch.erl +++ b/lib/elixir/src/elixir_dispatch.erl @@ -1,327 +1,406 @@ +%% SPDX-License-Identifier: Apache-2.0 +%% SPDX-FileCopyrightText: 2021 The Elixir Team +%% SPDX-FileCopyrightText: 2012 Plataformatec + %% Helpers related to dispatching to imports and references. %% This module access the information stored on the scope %% by elixir_import and therefore assumes it is normalized (ordsets) -module(elixir_dispatch). --export([dispatch_import/5, dispatch_require/6, +-export([dispatch_import/6, dispatch_require/7, require_function/5, import_function/4, - expand_import/5, expand_require/5, + expand_import/7, expand_require/6, check_deprecated/6, default_functions/0, default_macros/0, default_requires/0, - find_import/4, format_error/1]). + find_import/4, find_imports/3, format_error/1, stop_generated/1]). -include("elixir.hrl"). -import(ordsets, [is_element/2]). - --define(atom, 'Elixir.Atom'). --define(float, 'Elixir.Float'). --define(io, 'Elixir.IO'). --define(integer, 'Elixir.Integer'). -define(kernel, 'Elixir.Kernel'). --define(list, 'Elixir.List'). --define(map, 'Elixir.Map'). --define(node, 'Elixir.Node'). --define(process, 'Elixir.Process'). --define(string, 'Elixir.String'). --define(system, 'Elixir.System'). --define(tuple, 'Elixir.Tuple'). +-define(application, 'Elixir.Application'). default_functions() -> [{?kernel, elixir_imported_functions()}]. default_macros() -> [{?kernel, elixir_imported_macros()}]. default_requires() -> - ['Elixir.Kernel', 'Elixir.Kernel.Typespec']. + ['Elixir.Application', 'Elixir.Kernel']. +%% This is used by elixir_quote. Note we don't record the +%% import locally because at that point there is no +%% ambiguity. find_import(Meta, Name, Arity, E) -> Tuple = {Name, Arity}, - case find_dispatch(Meta, Tuple, [], E) of + case find_import_by_name_arity(Meta, Tuple, [], E) of {function, Receiver} -> - elixir_lexical:record_import(Receiver, ?m(E, lexical_tracker)), - elixir_locals:record_import(Tuple, Receiver, ?m(E, module), ?m(E, function)), + elixir_env:trace({imported_function, Meta, Receiver, Name, Arity}, E), Receiver; {macro, Receiver} -> - elixir_lexical:record_import(Receiver, ?m(E, lexical_tracker)), - elixir_locals:record_import(Tuple, Receiver, ?m(E, module), ?m(E, function)), + elixir_env:trace({imported_macro, Meta, Receiver, Name, Arity}, E), Receiver; + {ambiguous, _} = Ambiguous -> + elixir_errors:file_error(Meta, E, ?MODULE, {import, Ambiguous, Name, Arity}); _ -> false end. +find_imports(Meta, Name, E) -> + Funs = ?key(E, functions), + Macs = ?key(E, macros), + + Acc0 = #{}, + Acc1 = find_imports_by_name(Funs, Acc0, Name, Meta, E), + Acc2 = find_imports_by_name(Macs, Acc1, Name, Meta, E), + + lists:sort(maps:to_list(Acc2)). + %% Function retrieval import_function(Meta, Name, Arity, E) -> Tuple = {Name, Arity}, - case find_dispatch(Meta, Tuple, [], E) of + case find_import_by_name_arity(Meta, Tuple, [], E) of {function, Receiver} -> - elixir_lexical:record_import(Receiver, ?m(E, lexical_tracker)), - elixir_locals:record_import(Tuple, Receiver, ?m(E, module), ?m(E, function)), + elixir_env:trace({imported_function, Meta, Receiver, Name, Arity}, E), + elixir_import:record(Tuple, Receiver, ?key(E, module), ?key(E, function)), remote_function(Meta, Receiver, Name, Arity, E); {macro, _Receiver} -> false; {import, Receiver} -> require_function(Meta, Receiver, Name, Arity, E); + {ambiguous, _} = Ambiguous -> + elixir_errors:file_error(Meta, E, ?MODULE, {import, Ambiguous, Name, Arity}); false -> case elixir_import:special_form(Name, Arity) of - true -> false; + true -> + false; false -> - elixir_locals:record_local(Tuple, ?m(E, module), ?m(E, function)), - {local, Name, Arity} + Function = ?key(E, function), + + case (Function /= nil) andalso (Function /= Tuple) andalso + elixir_def:local_for(Meta, Name, Arity, [defmacro, defmacrop], E) of + false -> + elixir_env:trace({local_function, Meta, Name, Arity}, E), + {local, Name, Arity}; + _ -> + false + end end end. require_function(Meta, Receiver, Name, Arity, E) -> - case is_element({Name, Arity}, get_optional_macros(Receiver)) of + Required = is_element(Receiver, ?key(E, requires)), + + case is_macro(Name, Arity, Receiver, Required) of true -> false; - false -> remote_function(Meta, Receiver, Name, Arity, E) + false -> + elixir_env:trace({remote_function, Meta, Receiver, Name, Arity}, E), + remote_function(Meta, Receiver, Name, Arity, E) end. remote_function(Meta, Receiver, Name, Arity, E) -> - check_deprecation(Meta, Receiver, Name, Arity, E), + check_deprecated(function, Meta, Receiver, Name, Arity, E), - elixir_lexical:record_remote(Receiver, ?m(E, lexical_tracker)), - case inline(Receiver, Name, Arity) of + case elixir_rewrite:inline(Receiver, Name, Arity) of {AR, AN} -> {remote, AR, AN, Arity}; false -> {remote, Receiver, Name, Arity} end. %% Dispatches -dispatch_import(Meta, Name, Args, E, Callback) -> +dispatch_import(Meta, Name, Args, S, E, Callback) -> Arity = length(Args), - case expand_import(Meta, {Name, Arity}, Args, E, []) of - {ok, Receiver, Quoted} -> - expand_quoted(Meta, Receiver, Name, Arity, Quoted, E); - {ok, Receiver, NewName, NewArgs} -> - elixir_exp:expand({{'.', [], [Receiver, NewName]}, Meta, NewArgs}, E); - error -> - Callback() + + AllowLocals = + %% If we are inside a function, we support reading from locals. + case E of + #{function := {N, A}} when Name =/= N; Arity =/= A -> true; + _ -> false + end, + + case expand_import(Meta, Name, Arity, E, [], AllowLocals, true) of + {macro, Receiver, Expander} -> + check_deprecated(macro, Meta, Receiver, Name, Arity, E), + Caller = {?line(Meta), S, E}, + expand_quoted(Meta, Receiver, Name, Arity, Expander(stop_generated(Args), Caller), S, E); + {function, Receiver, NewName} -> + case elixir_rewrite:inline(Receiver, NewName, Arity) of + {AR, AN} -> + Callback({AR, AN}); + false -> + check_deprecated(function, Meta, Receiver, Name, Arity, E), + Callback({Receiver, NewName}) + end; + not_found -> + Callback(local); + Error -> + elixir_errors:file_error(Meta, E, ?MODULE, {import, Error, Name, Arity}) end. -dispatch_require(Meta, Receiver, Name, Args, E, Callback) when is_atom(Receiver) -> +stop_generated(Args) -> + lists:map(fun + ({Call, Meta, Ctx}) when is_list(Meta) -> {Call, [{stop_generated, true} | Meta], Ctx}; + (Other) -> Other + end, Args). + +dispatch_require(Meta, Receiver, Name, Args, S, E, Callback) when is_atom(Receiver) -> Arity = length(Args), - case rewrite(Receiver, Name, Args, Arity) of - {ok, AR, AN, AA} -> - Callback(AR, AN, AA); + case elixir_rewrite:inline(Receiver, Name, Arity) of + {AR, AN} -> + elixir_env:trace({remote_function, Meta, Receiver, Name, Arity}, E), + Callback(AR, AN); false -> - case expand_require(Meta, Receiver, {Name, Arity}, Args, E) of - {ok, Receiver, Quoted} -> expand_quoted(Meta, Receiver, Name, Arity, Quoted, E); - error -> Callback(Receiver, Name, Args) + case expand_require(Meta, Receiver, Name, Arity, E, true) of + {macro, Receiver, Expander} -> + check_deprecated(macro, Meta, Receiver, Name, Arity, E), + Caller = {?line(Meta), S, E}, + expand_quoted(Meta, Receiver, Name, Arity, Expander(Args, Caller), S, E); + error -> + check_deprecated(function, Meta, Receiver, Name, Arity, E), + elixir_env:trace({remote_function, Meta, Receiver, Name, Arity}, E), + Callback(Receiver, Name) end end; -dispatch_require(_Meta, Receiver, Name, Args, _E, Callback) -> - Callback(Receiver, Name, Args). +dispatch_require(_Meta, Receiver, Name, _Args, _S, _E, Callback) -> + Callback(Receiver, Name). %% Macros expansion -expand_import(Meta, {Name, Arity} = Tuple, Args, E, Extra) -> - Module = ?m(E, module), - Dispatch = find_dispatch(Meta, Tuple, Extra, E), - Function = ?m(E, function), - Local = (Function /= nil) andalso (Function /= Tuple) andalso - elixir_locals:macro_for(Module, Name, Arity), +expand_import(Meta, Name, Arity, E, Extra, AllowLocals, Trace) -> + Tuple = {Name, Arity}, + Module = ?key(E, module), + Dispatch = find_import_by_name_arity(Meta, Tuple, Extra, E), case Dispatch of - %% In case it is an import, we dispatch the import. - {import, _} -> - do_expand_import(Meta, Tuple, Args, Module, E, Dispatch); - - %% There is a local and an import. This is a conflict unless - %% the receiver is the same as module (happens on bootstrap). - {_, Receiver} when Local /= false, Receiver /= Module -> - Error = {macro_conflict, {Receiver, Name, Arity}}, - elixir_errors:form_error(Meta, ?m(E, file), ?MODULE, Error); + {ambiguous, Ambiguous} -> + {ambiguous, Ambiguous}; - %% There is no local. Dispatch the import. - _ when Local == false -> - do_expand_import(Meta, Tuple, Args, Module, E, Dispatch); + {import, _} -> + do_expand_import(Dispatch, Meta, Name, Arity, Module, E, Trace); - %% Dispatch to the local. _ -> - elixir_locals:record_local(Tuple, Module, Function), - {ok, Module, expand_macro_fun(Meta, Local(), Module, Name, Args, E)} + Local = case AllowLocals of + false -> false; + true -> elixir_def:local_for(Meta, Name, Arity, [defmacro, defmacrop], E); + Fun when is_function(Fun, 0) -> Fun() + end, + + case Dispatch of + %% There is a local and an import. This is a conflict unless + %% the receiver is the same as module (happens on bootstrap). + {_, Receiver} when Local /= false, Receiver /= Module -> + {conflict, Receiver}; + + %% There is no local. Dispatch the import. + _ when Local == false -> + do_expand_import(Dispatch, Meta, Name, Arity, Module, E, Trace); + + %% Dispatch to the local. + _ -> + Trace andalso elixir_env:trace({local_macro, Meta, Name, Arity}, E), + {macro, Module, expander_macro_fun(Meta, Local, Module, Name, E)} + end end. -do_expand_import(Meta, {Name, Arity} = Tuple, Args, Module, E, Result) -> +do_expand_import(Result, Meta, Name, Arity, Module, E, Trace) -> case Result of {function, Receiver} -> - elixir_lexical:record_import(Receiver, ?m(E, lexical_tracker)), - elixir_locals:record_import(Tuple, Receiver, Module, ?m(E, function)), - - case rewrite(Receiver, Name, Args, Arity) of - {ok, _, _, _} = Res -> Res; - false -> {ok, Receiver, Name, Args} - end; + Trace andalso begin + elixir_env:trace({imported_function, Meta, Receiver, Name, Arity}, E), + elixir_import:record({Name, Arity}, Receiver, Module, ?key(E, function)) + end, + {function, Receiver, Name}; {macro, Receiver} -> - check_deprecation(Meta, Receiver, Name, Arity, E), - elixir_lexical:record_import(Receiver, ?m(E, lexical_tracker)), - elixir_locals:record_import(Tuple, Receiver, Module, ?m(E, function)), - {ok, Receiver, expand_macro_named(Meta, Receiver, Name, Arity, Args, E)}; + Trace andalso begin + elixir_env:trace({imported_macro, Meta, Receiver, Name, Arity}, E), + elixir_import:record({Name, Arity}, Receiver, Module, ?key(E, function)) + end, + {macro, Receiver, expander_macro_named(Meta, Receiver, Name, Arity, E)}; {import, Receiver} -> - case expand_require([{require,false}|Meta], Receiver, Tuple, Args, E) of - {ok, _, _} = Response -> Response; - error -> {ok, Receiver, Name, Args} + case expand_require(true, Meta, Receiver, Name, Arity, E, Trace) of + {macro, _, _} = Response -> Response; + error -> + Trace andalso elixir_env:trace({remote_function, Meta, Receiver, Name, Arity}, E), + {function, Receiver, Name} end; false when Module == ?kernel -> - case rewrite(Module, Name, Args, Arity) of - {ok, _, _, _} = Res -> Res; - false -> error + case elixir_rewrite:inline(Module, Name, Arity) of + {AR, AN} -> {function, AR, AN}; + false -> not_found end; false -> - error + not_found end. -expand_require(Meta, Receiver, {Name, Arity} = Tuple, Args, E) -> - check_deprecation(Meta, Receiver, Name, Arity, E), - Module = ?m(E, module), +expand_require(Meta, Receiver, Name, Arity, E, Trace) -> + Required = (Receiver == ?key(E, module)) + orelse (lists:keyfind(required, 1, Meta) == {required, true}) + orelse is_element(Receiver, ?key(E, requires)), + + expand_require(Required, Meta, Receiver, Name, Arity, E, Trace). - case is_element(Tuple, get_optional_macros(Receiver)) of +expand_require(Required, Meta, Receiver, Name, Arity, E, Trace) -> + case is_macro(Name, Arity, Receiver, Required) of true -> - Requires = ?m(E, requires), - case (Receiver == Module) orelse is_element(Receiver, Requires) orelse skip_require(Meta) of - true -> - elixir_lexical:record_remote(Receiver, ?m(E, lexical_tracker)), - {ok, Receiver, expand_macro_named(Meta, Receiver, Name, Arity, Args, E)}; - false -> - Info = {unrequired_module, {Receiver, Name, length(Args), Requires}}, - elixir_errors:form_error(Meta, ?m(E, file), ?MODULE, Info) - end; + Trace andalso elixir_env:trace({remote_macro, Meta, Receiver, Name, Arity}, E), + {macro, Receiver, expander_macro_named(Meta, Receiver, Name, Arity, E)}; false -> error end. %% Expansion helpers -expand_macro_fun(Meta, Fun, Receiver, Name, Args, E) -> - Line = ?line(Meta), - EArg = {Line, E}, +expander_macro_fun(Meta, Fun, Receiver, Name, E) -> + fun(Args, Caller) -> expand_macro_fun(Meta, Fun, Receiver, Name, Args, Caller, E) end. +expander_macro_named(Meta, Receiver, Name, Arity, E) -> + ProperName = elixir_utils:macro_name(Name), + ProperArity = Arity + 1, + Fun = fun Receiver:ProperName/ProperArity, + fun(Args, Caller) -> expand_macro_fun(Meta, Fun, Receiver, Name, Args, Caller, E) end. + +expand_macro_fun(Meta, Fun, Receiver, Name, Args, Caller, E) -> try - apply(Fun, [EArg|Args]) + apply(Fun, [Caller | Args]) catch - Kind:Reason -> + Kind:Reason:Stacktrace -> Arity = length(Args), MFA = {Receiver, elixir_utils:macro_name(Name), Arity+1}, - Info = [{Receiver, Name, Arity, [{file, "expanding macro"}]}, caller(Line, E)], - erlang:raise(Kind, Reason, prune_stacktrace(erlang:get_stacktrace(), MFA, Info, EArg)) + Info = [{Receiver, Name, Arity, [{file, "expanding macro"}]}, caller(?line(Meta), E)], + erlang:raise(Kind, Reason, prune_stacktrace(Stacktrace, MFA, Info, {ok, Caller})) end. -expand_macro_named(Meta, Receiver, Name, Arity, Args, E) -> - ProperName = elixir_utils:macro_name(Name), - ProperArity = Arity + 1, - Fun = fun Receiver:ProperName/ProperArity, - expand_macro_fun(Meta, Fun, Receiver, Name, Args, E). - -expand_quoted(Meta, Receiver, Name, Arity, Quoted, E) -> - Line = ?line(Meta), - Next = elixir_counter:next(), +expand_quoted(Meta, Receiver, Name, Arity, Quoted, S, E) -> + Next = elixir_module:next_counter(?key(E, module)), try - elixir_exp:expand( - elixir_quote:linify_with_context_counter(Line, {Receiver, Next}, Quoted), - E) + ToExpand = elixir_quote:linify_with_context_counter(Meta, {Receiver, Next}, Quoted), + elixir_expand:expand(ToExpand, S, E) catch - Kind:Reason -> + Kind:Reason:Stacktrace -> MFA = {Receiver, elixir_utils:macro_name(Name), Arity+1}, - Info = [{Receiver, Name, Arity, [{file, "expanding macro"}]}, caller(Line, E)], - erlang:raise(Kind, Reason, prune_stacktrace(erlang:get_stacktrace(), MFA, Info, nil)) + Info = [{Receiver, Name, Arity, [{file, "expanding macro"}]}, caller(?line(Meta), E)], + erlang:raise(Kind, Reason, prune_stacktrace(Stacktrace, MFA, Info, error)) end. -caller(Line, #{module := nil} = E) -> - {elixir_compiler, '__FILE__', 2, location(Line, E)}; -caller(Line, #{module := Module, function := nil} = E) -> - {Module, '__MODULE__', 0, location(Line, E)}; -caller(Line, #{module := Module, function := {Name, Arity}} = E) -> - {Module, Name, Arity, location(Line, E)}. - -location(Line, E) -> - [{file, elixir_utils:characters_to_list(elixir_utils:relative_to_cwd(?m(E, file)))}, - {line, Line}]. +caller(Line, E) -> + elixir_utils:caller(Line, ?key(E, file), ?key(E, module), ?key(E, function)). %% Helpers -skip_require(Meta) -> - lists:keyfind(require, 1, Meta) == {require, false}. +find_imports_by_name([{Mod, Imports} | ModImports], Acc, Name, Meta, E) -> + NewAcc = find_imports_by_name(Name, Imports, Acc, Mod, Meta, E), + find_imports_by_name(ModImports, NewAcc, Name, Meta, E); +find_imports_by_name([], Acc, _Name, _Meta, _E) -> + Acc. + +find_imports_by_name(Name, [{Name, Arity} | Imports], Acc, Mod, Meta, E) -> + case Acc of + #{Arity := OtherMod} -> + Error = {import, {ambiguous, [Mod, OtherMod]}, Name, Arity}, + elixir_errors:file_error(Meta, E, ?MODULE, Error); -find_dispatch(Meta, Tuple, Extra, E) -> - case is_import(Meta) of + #{} -> + find_imports_by_name(Name, Imports, Acc#{Arity => Mod}, Mod, Meta, E) + end; +find_imports_by_name(Name, [{ImportName, _} | Imports], Acc, Mod, Meta, E) when Name > ImportName -> + find_imports_by_name(Name, Imports, Acc, Mod, Meta, E); +find_imports_by_name(_Name, _Imports, Acc, _Mod, _Meta, _E) -> + Acc. + +find_import_by_name_arity(Meta, {_Name, Arity} = Tuple, Extra, E) -> + case is_import(Meta, Arity) of {import, _} = Import -> Import; false -> - Funs = ?m(E, functions), - Macs = Extra ++ ?m(E, macros), - FunMatch = find_dispatch(Tuple, Funs), - MacMatch = find_dispatch(Tuple, Macs), + Funs = ?key(E, functions), + Macs = Extra ++ ?key(E, macros), + FunMatch = find_import_by_name_arity(Tuple, Funs), + MacMatch = find_import_by_name_arity(Tuple, Macs), case {FunMatch, MacMatch} of {[], [Receiver]} -> {macro, Receiver}; {[Receiver], []} -> {function, Receiver}; {[], []} -> false; - _ -> - {Name, Arity} = Tuple, - [First, Second|_] = FunMatch ++ MacMatch, - Error = {ambiguous_call, {First, Second, Name, Arity}}, - elixir_errors:form_error(Meta, ?m(E, file), ?MODULE, Error) + _ -> {ambiguous, FunMatch ++ MacMatch} end end. -find_dispatch(Tuple, List) -> +find_import_by_name_arity(Tuple, List) -> [Receiver || {Receiver, Set} <- List, is_element(Tuple, Set)]. -is_import(Meta) -> - case lists:keyfind(import, 1, Meta) of - {import, _} = Import -> +is_import(Meta, Arity) -> + case lists:keyfind(imports, 1, Meta) of + {imports, [_ | _] = Imports} -> case lists:keyfind(context, 1, Meta) of - {context, _} -> Import; - false -> - false + {context, _} -> + case lists:keyfind(Arity, 1, Imports) of + {Arity, Receiver} -> {import, Receiver}; + false -> false + end; + false -> false end; - false -> - false + _ -> false end. % %% We've reached the macro wrapper fun, skip it with the rest -prune_stacktrace([{_, _, [E|_], _}|_], _MFA, Info, E) -> +prune_stacktrace([{_, _, [Caller | _], _} | _], _MFA, Info, {ok, Caller}) -> Info; %% We've reached the invoked macro, skip it -prune_stacktrace([{M, F, A, _}|_], {M, F, A}, Info, _E) -> +prune_stacktrace([{M, F, A, _} | _], {M, F, A}, Info, _E) -> Info; %% We've reached the elixir_dispatch internals, skip it with the rest -prune_stacktrace([{Mod, _, _, _}|_], _MFA, Info, _E) when Mod == elixir_dispatch; Mod == elixir_exp -> +prune_stacktrace([{Mod, _, _, _} | _], _MFA, Info, _E) when Mod == elixir_dispatch; Mod == elixir_exp -> Info; -prune_stacktrace([H|T], MFA, Info, E) -> - [H|prune_stacktrace(T, MFA, Info, E)]; +prune_stacktrace([H | T], MFA, Info, E) -> + [H | prune_stacktrace(T, MFA, Info, E)]; prune_stacktrace([], _MFA, Info, _E) -> Info. %% ERROR HANDLING -format_error({unrequired_module, {Receiver, Name, Arity, _Required}}) -> - Module = elixir_aliases:inspect(Receiver), - io_lib:format("you must require ~ts before invoking the macro ~ts.~ts/~B", - [Module, Module, Name, Arity]); -format_error({macro_conflict, {Receiver, Name, Arity}}) -> +format_error({import, {conflict, Receiver}, Name, Arity}) -> io_lib:format("call to local macro ~ts/~B conflicts with imported ~ts.~ts/~B, " "please rename the local macro or remove the conflicting import", [Name, Arity, elixir_aliases:inspect(Receiver), Name, Arity]); -format_error({ambiguous_call, {Mod1, Mod2, Name, Arity}}) -> +format_error({import, {ambiguous, [Mod1, Mod2 | _]}, Name, Arity}) -> io_lib:format("function ~ts/~B imported from both ~ts and ~ts, call is ambiguous", - [Name, Arity, elixir_aliases:inspect(Mod1), elixir_aliases:inspect(Mod2)]). + [Name, Arity, elixir_aliases:inspect(Mod1), elixir_aliases:inspect(Mod2)]); +format_error({compile_env, Name, Arity}) -> + io_lib:format("Application.~s/~B is discouraged in the module body, use Application.compile_env/3 instead", [Name, Arity]); +format_error({deprecated, Mod, '__using__', 1, Message}) -> + io_lib:format("use ~s is deprecated. ~s", [elixir_aliases:inspect(Mod), Message]); +format_error({deprecated, Mod, Fun, Arity, Message}) -> + io_lib:format("~s.~s/~B is deprecated. ~s",[elixir_aliases:inspect(Mod), Fun, Arity, Message]). %% INTROSPECTION -%% Do not try to get macros from Erlang. Speeds up compilation a bit. -get_optional_macros(erlang) -> []; +is_macro(_Name, _Arity, _Module, false) -> + false; +is_macro(Name, Arity, Receiver, true) -> + try Receiver:'__info__'(macros) of + Macros -> is_element({Name, Arity}, Macros) + catch + error:_ -> false + end. -get_optional_macros(Receiver) -> +%% Deprecations checks only happen at the module body, +%% so in there we can try to at least load the module. +get_deprecations(Receiver) -> case code:ensure_loaded(Receiver) of - {module, Receiver} -> + {module, Receiver} -> get_info(Receiver, deprecated); + _ -> [] + end. + +get_info(Receiver, Key) -> + case erlang:function_exported(Receiver, '__info__', 1) of + true -> try - Receiver:'__info__'(macros) + Receiver:'__info__'(Key) catch - error:undef -> [] + error:_ -> [] end; - {error, _} -> [] + false -> + [] end. elixir_imported_functions() -> @@ -338,184 +417,31 @@ elixir_imported_macros() -> error:undef -> [] end. -rewrite(?atom, to_string, [Arg], _) -> - {ok, erlang, atom_to_binary, [Arg, utf8]}; -rewrite(?kernel, elem, [Tuple, Index], _) -> - {ok, erlang, element, [increment(Index), Tuple]}; -rewrite(?kernel, put_elem, [Tuple, Index, Value], _) -> - {ok, erlang, setelement, [increment(Index), Tuple, Value]}; -rewrite(?map, 'has_key?', [Map, Key], _) -> - {ok, maps, is_key, [Key, Map]}; -rewrite(?map, fetch, [Map, Key], _) -> - {ok, maps, find, [Key, Map]}; -rewrite(?map, put, [Map, Key, Value], _) -> - {ok, maps, put, [Key, Value, Map]}; -rewrite(?map, delete, [Map, Key], _) -> - {ok, maps, remove, [Key, Map]}; -rewrite(?process, monitor, [Arg], _) -> - {ok, erlang, monitor, [process, Arg]}; -rewrite(?string, to_atom, [Arg], _) -> - {ok, erlang, binary_to_atom, [Arg, utf8]}; -rewrite(?string, to_existing_atom, [Arg], _) -> - {ok, erlang, binary_to_existing_atom, [Arg, utf8]}; -rewrite(?tuple, insert_at, [Tuple, Index, Term], _) -> - {ok, erlang, insert_element, [increment(Index), Tuple, Term]}; -rewrite(?tuple, delete_at, [Tuple, Index], _) -> - {ok, erlang, delete_element, [increment(Index), Tuple]}; -rewrite(?tuple, duplicate, [Data, Size], _) -> - {ok, erlang, make_tuple, [Size, Data]}; - -rewrite(Receiver, Name, Args, Arity) -> - case inline(Receiver, Name, Arity) of - {AR, AN} -> {ok, AR, AN, Args}; - false -> false - end. - -increment(Number) when is_number(Number) -> - Number + 1; -increment(Other) -> - {{'.', [], [erlang, '+']}, [], [Other, 1]}. - -inline(?atom, to_char_list, 1) -> {erlang, atom_to_list}; -inline(?io, iodata_length, 1) -> {erlang, iolist_size}; -inline(?io, iodata_to_binary, 1) -> {erlang, iolist_to_binary}; -inline(?integer, to_string, 1) -> {erlang, integer_to_binary}; -inline(?integer, to_string, 2) -> {erlang, integer_to_binary}; -inline(?integer, to_char_list, 1) -> {erlang, integer_to_list}; -inline(?integer, to_char_list, 2) -> {erlang, integer_to_list}; -inline(?float, to_string, 1) -> {erlang, float_to_binary}; -inline(?float, to_char_list, 1) -> {erlang, float_to_list}; -inline(?list, to_atom, 1) -> {erlang, list_to_atom}; -inline(?list, to_existing_atom, 1) -> {erlang, list_to_existing_atom}; -inline(?list, to_float, 1) -> {erlang, list_to_float}; -inline(?list, to_integer, 1) -> {erlang, list_to_integer}; -inline(?list, to_integer, 2) -> {erlang, list_to_integer}; -inline(?list, to_tuple, 1) -> {erlang, list_to_tuple}; - -inline(?kernel, '+', 2) -> {erlang, '+'}; -inline(?kernel, '-', 2) -> {erlang, '-'}; -inline(?kernel, '+', 1) -> {erlang, '+'}; -inline(?kernel, '-', 1) -> {erlang, '-'}; -inline(?kernel, '*', 2) -> {erlang, '*'}; -inline(?kernel, '/', 2) -> {erlang, '/'}; -inline(?kernel, '++', 2) -> {erlang, '++'}; -inline(?kernel, '--', 2) -> {erlang, '--'}; -inline(?kernel, 'not', 1) -> {erlang, 'not'}; -inline(?kernel, '<', 2) -> {erlang, '<'}; -inline(?kernel, '>', 2) -> {erlang, '>'}; -inline(?kernel, '<=', 2) -> {erlang, '=<'}; -inline(?kernel, '>=', 2) -> {erlang, '>='}; -inline(?kernel, '==', 2) -> {erlang, '=='}; -inline(?kernel, '!=', 2) -> {erlang, '/='}; -inline(?kernel, '===', 2) -> {erlang, '=:='}; -inline(?kernel, '!==', 2) -> {erlang, '=/='}; -inline(?kernel, abs, 1) -> {erlang, abs}; -inline(?kernel, apply, 2) -> {erlang, apply}; -inline(?kernel, apply, 3) -> {erlang, apply}; -inline(?kernel, binary_part, 3) -> {erlang, binary_part}; -inline(?kernel, bit_size, 1) -> {erlang, bit_size}; -inline(?kernel, byte_size, 1) -> {erlang, byte_size}; -inline(?kernel, 'div', 2) -> {erlang, 'div'}; -inline(?kernel, exit, 1) -> {erlang, exit}; -inline(?kernel, hd, 1) -> {erlang, hd}; -inline(?kernel, is_atom, 1) -> {erlang, is_atom}; -inline(?kernel, is_binary, 1) -> {erlang, is_binary}; -inline(?kernel, is_bitstring, 1) -> {erlang, is_bitstring}; -inline(?kernel, is_boolean, 1) -> {erlang, is_boolean}; -inline(?kernel, is_float, 1) -> {erlang, is_float}; -inline(?kernel, is_function, 1) -> {erlang, is_function}; -inline(?kernel, is_function, 2) -> {erlang, is_function}; -inline(?kernel, is_integer, 1) -> {erlang, is_integer}; -inline(?kernel, is_list, 1) -> {erlang, is_list}; -inline(?kernel, is_map, 1) -> {erlang, is_map}; -inline(?kernel, is_number, 1) -> {erlang, is_number}; -inline(?kernel, is_pid, 1) -> {erlang, is_pid}; -inline(?kernel, is_port, 1) -> {erlang, is_port}; -inline(?kernel, is_reference, 1) -> {erlang, is_reference}; -inline(?kernel, is_tuple, 1) -> {erlang, is_tuple}; -inline(?kernel, length, 1) -> {erlang, length}; -inline(?kernel, make_ref, 0) -> {erlang, make_ref}; -inline(?kernel, map_size, 1) -> {erlang, map_size}; -inline(?kernel, max, 2) -> {erlang, max}; -inline(?kernel, min, 2) -> {erlang, min}; -inline(?kernel, node, 0) -> {erlang, node}; -inline(?kernel, node, 1) -> {erlang, node}; -inline(?kernel, 'rem', 2) -> {erlang, 'rem'}; -inline(?kernel, round, 1) -> {erlang, round}; -inline(?kernel, self, 0) -> {erlang, self}; -inline(?kernel, send, 2) -> {erlang, send}; -inline(?kernel, spawn, 1) -> {erlang, spawn}; -inline(?kernel, spawn, 3) -> {erlang, spawn}; -inline(?kernel, spawn_link, 1) -> {erlang, spawn_link}; -inline(?kernel, spawn_link, 3) -> {erlang, spawn_link}; -inline(?kernel, spawn_monitor, 1) -> {erlang, spawn_monitor}; -inline(?kernel, spawn_monitor, 3) -> {erlang, spawn_monitor}; -inline(?kernel, throw, 1) -> {erlang, throw}; -inline(?kernel, tl, 1) -> {erlang, tl}; -inline(?kernel, trunc, 1) -> {erlang, trunc}; -inline(?kernel, tuple_size, 1) -> {erlang, tuple_size}; - -inline(?map, keys, 1) -> {maps, keys}; -inline(?map, merge, 2) -> {maps, merge}; -inline(?map, size, 1) -> {maps, size}; -inline(?map, values, 1) -> {maps, values}; -inline(?map, to_list, 1) -> {maps, to_list}; - -inline(?node, spawn, 2) -> {erlang, spawn}; -inline(?node, spawn, 3) -> {erlang, spawn_opt}; -inline(?node, spawn, 4) -> {erlang, spawn}; -inline(?node, spawn, 5) -> {erlang, spawn_opt}; -inline(?node, spawn_link, 2) -> {erlang, spawn_link}; -inline(?node, spawn_link, 4) -> {erlang, spawn_link}; - -inline(?process, exit, 2) -> {erlang, exit}; -inline(?process, spawn, 2) -> {erlang, spawn_opt}; -inline(?process, spawn, 4) -> {erlang, spawn_opt}; -inline(?process, demonitor, 1) -> {erlang, demonitor}; -inline(?process, demonitor, 2) -> {erlang, demonitor}; -inline(?process, link, 1) -> {erlang, link}; -inline(?process, unlink, 1) -> {erlang, unlink}; - -inline(?string, to_float, 1) -> {erlang, binary_to_float}; -inline(?string, to_integer, 1) -> {erlang, binary_to_integer}; -inline(?string, to_integer, 2) -> {erlang, binary_to_integer}; -inline(?system, stacktrace, 0) -> {erlang, get_stacktrace}; -inline(?tuple, to_list, 1) -> {erlang, tuple_to_list}; - -inline(_, _, _) -> false. - -check_deprecation(Meta, Receiver, Name, Arity, #{file := File}) -> - case deprecation(Receiver, Name, Arity) of - false -> ok; - Message -> - Warning = deprecation_message(Receiver, Name, Arity, Message), - elixir_errors:warn(?line(Meta), File, Warning) - end. +check_deprecated(_, _, erlang, _, _, _) -> ok; +check_deprecated(_, _, elixir_def, _, _, _) -> ok; +check_deprecated(_, _, elixir_module, _, _, _) -> ok; +check_deprecated(_, _, ?kernel, _, _, _) -> ok; +check_deprecated(Kind, Meta, ?application, Name, Arity, E) -> + case E of + #{module := Module, function := nil} + when (Module /= nil) or (Kind == macro), (Name == get_env) orelse (Name == fetch_env) orelse (Name == 'fetch_env!') -> + elixir_errors:file_warn(Meta, E, ?MODULE, {compile_env, Name, Arity}); -deprecation_message(Receiver, '__using__', _Arity, Message) -> - Warning = io_lib:format("use ~s is deprecated", [elixir_aliases:inspect(Receiver)]), - deprecation_message(Warning, Message); + _ -> + ok + end; +check_deprecated(Kind, Meta, Receiver, Name, Arity, E) -> + %% Any compile time behavior cannot be verified by the runtime group pass. + case ((?key(E, function) == nil) or (Kind == macro)) andalso get_deprecations(Receiver) of + [_ | _] = Deprecations -> + case lists:keyfind({Name, Arity}, 1, Deprecations) of + {_, Message} -> + elixir_errors:file_warn(Meta, E, ?MODULE, {deprecated, Receiver, Name, Arity, Message}); -deprecation_message(Receiver, Name, Arity, Message) -> - Warning = io_lib:format("~s.~s/~B is deprecated", - [elixir_aliases:inspect(Receiver), Name, Arity]), - deprecation_message(Warning, Message). + false -> + false + end; -deprecation_message(Warning, Message) -> - case Message of - true -> Warning; - Message -> Warning ++ ", " ++ Message + _ -> + ok end. - -deprecation('Elixir.Mix.Generator', 'from_file', _) -> - "instead pass [from_file: file] to embed_text/2 and embed_template/2 macros. " - "Note that [from_file: file] expects paths relative to the current working " - "directory and not to the current file"; -deprecation('Elixir.EEx.TransformerEngine', '__using__', _) -> - "check EEx.SmartEngine for how to build custom engines"; -deprecation('Elixir.EEx.AssignsEngine', '__using__', _) -> - "check EEx.SmartEngine for how to build custom engines"; -deprecation('Elixir.Kernel', 'xor', _) -> - true; %% Remember to remove xor operator from tokenizer -deprecation(_, _, _) -> - false. diff --git a/lib/elixir/src/elixir_env.erl b/lib/elixir/src/elixir_env.erl index fc32e1b0599..ec6df8f6366 100644 --- a/lib/elixir/src/elixir_env.erl +++ b/lib/elixir/src/elixir_env.erl @@ -1,62 +1,164 @@ +%% SPDX-License-Identifier: Apache-2.0 +%% SPDX-FileCopyrightText: 2021 The Elixir Team +%% SPDX-FileCopyrightText: 2012 Plataformatec + -module(elixir_env). -include("elixir.hrl"). --export([new/0, linify/1, env_to_scope/1, env_to_scope_with_vars/2]). --export([mergea/2, mergev/2, merge_vars/2, merge_opt_vars/2]). +-export([ + new/0, to_caller/1, merge_vars/2, with_vars/2, reset_vars/1, env_to_ex/1, + reset_unused_vars/1, check_unused_vars/2, merge_and_check_unused_vars/3, calculate_span/2, + trace/2, format_error/1, + reset_read/2, prepare_write/1, prepare_write/2, close_write/2 +]). new() -> - #{'__struct__' => 'Elixir.Macro.Env', - module => nil, %% the current module - file => <<"nofile">>, %% the current filename - line => 1, %% the current line - function => nil, %% the current function - context => nil, %% can be match_vars, guards or nil - requires => [], %% a set with modules required - aliases => [], %% an orddict with aliases by new -> old names - functions => [], %% a list with functions imported from module - macros => [], %% a list with macros imported from module - macro_aliases => [], %% keep aliases defined inside a macro - context_modules => [], %% modules defined in the current context - vars => [], %% a set of defined variables - export_vars => nil, %% a set of variables to be exported in some constructs - lexical_tracker => nil, %% holds the lexical tracker pid - local => nil}. %% the module to delegate local functions to - -linify({Line, Env}) -> - Env#{line := Line}. - -env_to_scope(#{module := Module, file := File, function := Function, context := Context}) -> - #elixir_scope{module=Module, file=File, function=Function, context=Context}. - -env_to_scope_with_vars(Env, Vars) -> - (env_to_scope(Env))#elixir_scope{ - vars=orddict:from_list(Vars), - counter=[{'_',length(Vars)}] - }. - -%% SCOPE MERGING - -%% Receives two scopes and return a new scope based on the second -%% with their variables merged. -mergev(E1, E2) when is_list(E1) -> - E2#{ - vars := merge_vars(E1, ?m(E2, vars)), - export_vars := merge_opt_vars(E1, ?m(E2, export_vars)) - }; -mergev(E1, E2) -> - E2#{ - vars := merge_vars(?m(E1, vars), ?m(E2, vars)), - export_vars := merge_opt_vars(?m(E1, export_vars), ?m(E2, export_vars)) - }. - -%% Receives two scopes and return the later scope -%% keeping the variables from the first (imports -%% and everything else are passed forward). - -mergea(E1, E2) -> - E2#{vars := ?m(E1, vars)}. - -merge_vars(V1, V2) -> ordsets:union(V1, V2). - -merge_opt_vars(_V1, nil) -> nil; -merge_opt_vars(nil, _V2) -> nil; -merge_opt_vars(V1, V2) -> ordsets:union(V1, V2). + #{ + '__struct__' => 'Elixir.Macro.Env', + module => nil, %% the current module + file => <<"nofile">>, %% the current filename + line => 1, %% the current line + function => nil, %% the current function + context => nil, %% can be match, guard or nil + aliases => [], %% a list of aliases by new -> old names + requires => elixir_dispatch:default_requires(), %% a set with modules required + functions => elixir_dispatch:default_functions(), %% a list with functions imported from module + macros => elixir_dispatch:default_macros(), %% a list with macros imported from module + macro_aliases => [], %% keep aliases defined inside a macro + context_modules => [], %% modules defined in the current context + versioned_vars => #{}, %% a map of vars with their latest versions + lexical_tracker => nil, %% lexical tracker PID + tracers => [] %% available compilation tracers + }. + +trace(Event, #{tracers := Tracers} = E) -> + [ok = Tracer:trace(Event, E) || Tracer <- Tracers], + ok. + +to_caller({Line, #elixir_ex{vars={Read, _}}, Env}) -> + Env#{line := Line, versioned_vars := Read}; +to_caller(#{'__struct__' := 'Elixir.Macro.Env'} = Env) -> + Env. + +with_vars(Env, Vars) when is_list(Vars) -> + {ReversedVars, _} = + lists:foldl(fun(Var, {Acc, I}) -> {[{Var, I} | Acc], I + 1} end, {[], 0}, Vars), + Env#{versioned_vars := maps:from_list(ReversedVars)}; +with_vars(Env, #{} = Vars) -> + Env#{versioned_vars := Vars}. + +reset_vars(Env) -> + Env#{versioned_vars := #{}}. + +%% CONVERSIONS + +env_to_ex(#{context := match, versioned_vars := Vars}) -> + Counter = map_size(Vars), + #elixir_ex{ + prematch={Vars, {#{}, []}, Counter}, + vars={Vars, false}, + unused={#{}, Counter} + }; +env_to_ex(#{versioned_vars := Vars}) -> + #elixir_ex{ + vars={Vars, false}, + unused={#{}, map_size(Vars)} + }. + +%% VAR HANDLING + +reset_read(#elixir_ex{vars={_, Write}} = S, #elixir_ex{vars={Read, _}}) -> + S#elixir_ex{vars={Read, Write}}. + +prepare_write(S, #{context := nil}) -> + prepare_write(S); +prepare_write(S, _) -> + S. + +prepare_write(#elixir_ex{vars={Read, _}} = S) -> + S#elixir_ex{vars={Read, Read}}. + +close_write(#elixir_ex{vars={_Read, Write}} = S, #elixir_ex{vars={_, false}}) -> + S#elixir_ex{vars={Write, false}}; +close_write(#elixir_ex{vars={_Read, Write}} = S, #elixir_ex{vars={_, UpperWrite}}) -> + S#elixir_ex{vars={Write, merge_vars(UpperWrite, Write)}}. + +merge_vars(V, V) -> + V; +merge_vars(V1, V2) -> + maps:fold(fun(K, M2, Acc) -> + case Acc of + #{K := M1} when M1 >= M2 -> Acc; + _ -> Acc#{K => M2} + end + end, V1, V2). + +%% UNUSED VARS + +reset_unused_vars(#elixir_ex{unused={_Unused, Version}} = S) -> + S#elixir_ex{unused={#{}, Version}}. + +check_unused_vars(#elixir_ex{unused={Unused, _Version}}, E) -> + [elixir_errors:file_warn(calculate_span(Meta, Name), E, ?MODULE, {unused_var, Name, Overridden}) || + {{{Name, _Kind}, _Count}, {Meta, Overridden}} <- maps:to_list(Unused), is_unused_var(Name)], + E. + +calculate_span(Meta, Name) -> + case lists:keyfind(column, 1, Meta) of + {column, Column} -> + [{span, {?line(Meta), Column + string:length(atom_to_binary(Name))}} | Meta]; + + _ -> + Meta + end. + +merge_and_check_unused_vars(S, #elixir_ex{vars={Read, Write}, unused={Unused, _Version}}, E) -> + #elixir_ex{unused={ClauseUnused, Version}} = S, + NewUnused = merge_and_check_unused_vars(Read, Unused, ClauseUnused, E), + S#elixir_ex{unused={NewUnused, Version}, vars={Read, Write}}. + +merge_and_check_unused_vars(Current, Unused, ClauseUnused, E) -> + maps:fold(fun + ({Var, Count} = Key, false, Acc) -> + case Current of + #{Var := CurrentCount} when Count =< CurrentCount -> + %% The parent knows it, so we have to propagate it was used up. + Acc#{Key => false}; + + #{} -> + Acc + end; + + ({{Name, _Kind}, _Count}, {Meta, Overridden}, Acc) -> + case is_unused_var(Name) of + true -> + Warn = {unused_var, Name, Overridden}, + elixir_errors:file_warn(Meta, E, ?MODULE, Warn); + + false -> + ok + end, + + Acc + end, Unused, ClauseUnused). + +is_unused_var(Name) -> + case atom_to_list(Name) of + "_" ++ Rest -> is_compiler_var(Rest); + _ -> true + end. + +is_compiler_var([$_]) -> true; +is_compiler_var([Var | Rest]) when Var =:= $_; Var >= $A, Var =< $Z -> is_compiler_var(Rest); +is_compiler_var(_) -> false. + +format_error({unused_var, Name, Overridden}) -> + case atom_to_list(Name) of + "_" ++ _ -> + io_lib:format("unknown compiler variable \"~ts\" (expected one of __MODULE__, __ENV__, __DIR__, __CALLER__, __STACKTRACE__)", [Name]); + "&" ++ _ -> + io_lib:format("variable \"~ts\" is unused (this might happen when using a capture argument as a pattern)", [Name]); + _ when Overridden -> + io_lib:format("variable \"~ts\" is unused (there is a variable with the same name in the context, use the pin operator (^) to match on it or prefix this variable with underscore if it is not meant to be used)", [Name]); + _ -> + io_lib:format("variable \"~ts\" is unused (if the variable is not meant to be used, prefix it with an underscore)", [Name]) + end. diff --git a/lib/elixir/src/elixir_erl.erl b/lib/elixir/src/elixir_erl.erl new file mode 100644 index 00000000000..fc8da5e3bd6 --- /dev/null +++ b/lib/elixir/src/elixir_erl.erl @@ -0,0 +1,694 @@ +%% SPDX-License-Identifier: Apache-2.0 +%% SPDX-FileCopyrightText: 2021 The Elixir Team +%% SPDX-FileCopyrightText: 2012 Plataformatec + +%% Compiler backend to Erlang. + +-module(elixir_erl). +-export([elixir_to_erl/1, elixir_to_erl/2, definition_to_anonymous/5, compile/2, consolidate/4, + get_ann/1, debug_info/4, scope/2, checker_version/0, format_error/1]). +-include("elixir.hrl"). +-define(typespecs, 'Elixir.Kernel.Typespec'). + +checker_version() -> + elixir_checker_v4. + +%% debug_info callback + +debug_info(elixir_v1, _Module, none, _Opts) -> + {error, missing}; +debug_info(elixir_v1, _Module, {elixir_v1, Map, _Specs}, _Opts) -> + {ok, Map}; +debug_info(erlang_v1, _Module, {elixir_v1, Map, Specs}, _Opts) -> + {Prefix, Forms, _, _, _} = dynamic_form(Map, nil), + {ok, Prefix ++ Specs ++ Forms}; +debug_info(core_v1, _Module, {elixir_v1, Map, Specs}, Opts) -> + {Prefix, Forms, _, _, _} = dynamic_form(Map, nil), + #{compile_opts := CompileOpts} = Map, + AllOpts = CompileOpts ++ Opts, + + %% Do not rely on elixir_erl_compiler because we don't warn + %% warnings nor the other functionality provided there. + case elixir_erl_compiler:erl_to_core(Prefix ++ Specs ++ Forms, AllOpts) of + {ok, CoreForms, _} -> + try compile:noenv_forms(CoreForms, [no_spawn_compiler_process, from_core, to_core, return | AllOpts]) of + {ok, _, Core, _} -> {ok, Core}; + _What -> {error, failed_conversion} + catch + error:_ -> {error, failed_conversion} + end; + _ -> + {error, failed_conversion} + end; +debug_info(_, _, _, _) -> + {error, unknown_format}. + +%% Builds Erlang AST annotation. + +get_ann(Opts) when is_list(Opts) -> + get_ann(Opts, false, 0, undefined). + +get_ann([{generated, true} | T], _, Line, Column) -> get_ann(T, true, Line, Column); +get_ann([{line, Line} | T], Gen, _, Column) when is_integer(Line) -> get_ann(T, Gen, Line, Column); +get_ann([{column, Column} | T], Gen, Line, _) when is_integer(Column) -> get_ann(T, Gen, Line, Column); +get_ann([_ | T], Gen, Line, Column) -> get_ann(T, Gen, Line, Column); +get_ann([], Gen, Line, undefined) -> erl_anno:set_generated(Gen, erl_anno:new(Line)); +get_ann([], Gen, Line, Column) -> erl_anno:set_generated(Gen, erl_anno:new({Line, Column})). + +%% Converts an Elixir definition to an anonymous function. + +definition_to_anonymous(Kind, Meta, Clauses, LocalHandler, ExternalHandler) -> + ErlClauses = [translate_clause(Kind, 0, Clause, true) || Clause <- Clauses], + Fun = {'fun', ?ann(Meta), {clauses, ErlClauses}}, + {value, Result, _Binding} = erl_eval:expr(Fun, [], LocalHandler, ExternalHandler), + Result. + +%% Converts Elixir quoted literals to Erlang AST. +elixir_to_erl(Tree) -> + elixir_to_erl(Tree, erl_anno:new(0)). + +elixir_to_erl(Tree, Ann) when is_tuple(Tree) -> + {tuple, Ann, [elixir_to_erl(X, Ann) || X <- tuple_to_list(Tree)]}; +elixir_to_erl([], Ann) -> + {nil, Ann}; +elixir_to_erl(<<>>, Ann) -> + {bin, Ann, []}; +elixir_to_erl(#{} = Map, Ann) -> + Assocs = [{map_field_assoc, Ann, elixir_to_erl(K, Ann), elixir_to_erl(V, Ann)} + || {K, V} <- lists:sort(maps:to_list(Map))], + {map, Ann, Assocs}; +elixir_to_erl(Tree, Ann) when is_list(Tree) -> + elixir_to_erl_cons(Tree, Ann); +elixir_to_erl(Tree, Ann) when is_atom(Tree) -> + {atom, Ann, Tree}; +elixir_to_erl(Tree, Ann) when is_integer(Tree) -> + {integer, Ann, Tree}; +elixir_to_erl(Tree, Ann) when is_float(Tree), Tree == 0.0 -> + % 0.0 needs to be rewritten as the AST for +0.0 in matches + Op = + case <> of + <<1:1,_:63>> -> '-'; + _ -> '+' + end, + {op, Ann, Op, {float, Ann, 0.0}}; +elixir_to_erl(Tree, Ann) when is_float(Tree) -> + {float, Ann, Tree}; +elixir_to_erl(Tree, Ann) when is_binary(Tree) -> + %% Note that our binaries are UTF-8 encoded and we are converting + %% to a list using binary_to_list. The reason for this is that Erlang + %% considers a string in a binary to be encoded in latin1, so the bytes + %% are not changed in any fashion. + {bin, Ann, [{bin_element, Ann, {string, Ann, binary_to_list(Tree)}, default, default}]}; +elixir_to_erl(Tree, Ann) when is_bitstring(Tree) -> + Segments = [elixir_to_erl_bitstring_segment(X, Ann) || X <- bitstring_to_list(Tree)], + {bin, Ann, Segments}; +elixir_to_erl(Tree, Ann) when is_function(Tree) -> + case (erlang:fun_info(Tree, type) == {type, external}) andalso + (erlang:fun_info(Tree, env) == {env, []}) of + true -> + {module, Module} = erlang:fun_info(Tree, module), + {name, Name} = erlang:fun_info(Tree, name), + {arity, Arity} = erlang:fun_info(Tree, arity), + {'fun', Ann, {function, {atom, Ann, Module}, {atom, Ann, Name}, {integer, Ann, Arity}}}; + false -> + error(badarg, [Tree, Ann]) + end; +elixir_to_erl(Tree, Ann) when is_pid(Tree); is_port(Tree); is_reference(Tree) -> + ?remote(Ann, erlang, binary_to_term, [elixir_to_erl(term_to_binary(Tree), Ann)]); +elixir_to_erl(Tree, Ann) -> + error(badarg, [Tree, Ann]). + +elixir_to_erl_cons([H | T], Ann) -> {cons, Ann, elixir_to_erl(H, Ann), elixir_to_erl_cons(T, Ann)}; +elixir_to_erl_cons(T, Ann) -> elixir_to_erl(T, Ann). + +elixir_to_erl_bitstring_segment(Int, Ann) when is_integer(Int) -> + {bin_element, Ann, {integer, Ann, Int}, default, [integer]}; +elixir_to_erl_bitstring_segment(Rest, Ann) when is_bitstring(Rest) -> + Size = bit_size(Rest), + <> = Rest, + {bin_element, Ann, {integer, Ann, Int}, {integer, Ann, Size}, [integer]}. + +%% Returns a scope for translation. + +scope(_Meta, ExpandCaptures) -> + #elixir_erl{expand_captures=ExpandCaptures}. + +%% Static compilation hook, used in protocol consolidation + +consolidate(Map, Checker, TypeSpecs, DocsChunk) -> + {Prefix, Forms, _Def, _Defmacro, _Macros} = dynamic_form(Map, nil), + CheckerChunk = checker_chunk(Checker, chunk_opts(Map)), + load_form(Map, Prefix, Forms, TypeSpecs, DocsChunk ++ CheckerChunk). + +%% Dynamic compilation hook, used in regular compiler + +compile(#{module := Module, anno := Anno} = BaseMap, Signatures) -> + Map = + case elixir_erl_compiler:env_compiler_options() of + [] -> BaseMap; + EnvOptions -> BaseMap#{compile_opts := ?key(BaseMap, compile_opts) ++ EnvOptions} + end, + + {Set, Bag} = elixir_module:data_tables(Module), + + TranslatedTypespecs = + case elixir_config:is_bootstrap() andalso + (code:ensure_loaded(?typespecs) /= {module, ?typespecs}) of + true -> {[], [], [], [], []}; + false -> ?typespecs:translate_typespecs_for_module(Set, Bag) + end, + + MD5 = ets:lookup_element(Set, exports_md5, 2), + {Prefix, Forms, Def, Defmacro, Macros} = dynamic_form(Map, MD5), + {Types, Callbacks, TypeSpecs} = typespecs_form(Map, TranslatedTypespecs, Macros), + + ChunkOpts = chunk_opts(Map), + DocsChunk = docs_chunk(Map, Set, Module, Anno, Def, Defmacro, Types, Callbacks, ChunkOpts), + CheckerChunk = checker_chunk(Map, Def, Signatures, ChunkOpts), + load_form(Map, Prefix, Forms, TypeSpecs, DocsChunk ++ CheckerChunk). + +chunk_opts(Map) -> + case lists:member(deterministic, ?key(Map, compile_opts)) of + true -> [deterministic]; + false -> [] + end. + +dynamic_form(#{module := Module, relative_file := RelativeFile, + attributes := Attributes, definitions := Definitions, unreachable := Unreachable, + deprecated := Deprecated, compile_opts := Opts} = Map, MD5) -> + %% TODO: Match on anno directly in Elixir v1.22+ + Line = case Map of + #{anno := AnnoValue} -> erl_anno:line(AnnoValue); + #{line := LineValue} -> LineValue + end, + + {Def, Defmacro, Macros, Exports, Functions} = + split_definition(Definitions, Unreachable, Line, [], [], [], [], {[], []}), + + FilteredOpts = proplists:delete(debug_info, proplists:delete(no_warn_undefined, Opts)), + Location = {elixir_utils:characters_to_list(RelativeFile), Line}, + + Prefix = [{attribute, Line, file, Location}, + {attribute, Line, module, Module}, + {attribute, Line, compile, [no_auto_import | FilteredOpts]}], + + Struct = maps:get(struct, Map, nil), + Forms0 = functions_form(Line, Module, Def, Defmacro, Exports, Functions, Deprecated, Struct, MD5), + Forms1 = attributes_form(Line, Attributes, Forms0), + {Prefix, Forms1, Def, Defmacro, Macros}. + +% Definitions + +split_definition([{Tuple, Kind, Meta, Clauses} | T], Unreachable, Line, + Def, Defmacro, Macros, Exports, Functions) -> + case lists:member(Tuple, Unreachable) of + false -> + split_definition(Tuple, Kind, Meta, Clauses, T, Unreachable, Line, + Def, Defmacro, Macros, Exports, Functions); + true -> + split_definition(T, Unreachable, Line, Def, Defmacro, Macros, Exports, Functions) + end; + +split_definition([], _Unreachable, _Line, Def, Defmacro, Macros, Exports, {Head, Tail}) -> + {lists:sort(Def), lists:sort(Defmacro), Macros, Exports, Head ++ Tail}. + +split_definition(Tuple, def, Meta, Clauses, T, Unreachable, Line, + Def, Defmacro, Macros, Exports, Functions) -> + {_, _, N, A, _} = Entry = translate_definition(def, Line, Meta, Tuple, Clauses), + split_definition(T, Unreachable, Line, [{Tuple, Meta} | Def], Defmacro, Macros, [{N, A} | Exports], + add_definition(Meta, Entry, Functions)); + +split_definition(Tuple, defp, Meta, Clauses, T, Unreachable, Line, + Def, Defmacro, Macros, Exports, Functions) -> + Entry = translate_definition(defp, Line, Meta, Tuple, Clauses), + split_definition(T, Unreachable, Line, Def, Defmacro, Macros, Exports, + add_definition(Meta, Entry, Functions)); + +split_definition(Tuple, defmacro, Meta, Clauses, T, Unreachable, Line, + Def, Defmacro, Macros, Exports, Functions) -> + {_, _, N, A, _} = Entry = translate_definition(defmacro, Line, Meta, Tuple, Clauses), + split_definition(T, Unreachable, Line, Def, [{Tuple, Meta} | Defmacro], [Tuple | Macros], [{N, A} | Exports], + add_definition(Meta, Entry, Functions)); + +split_definition(Tuple, defmacrop, Meta, Clauses, T, Unreachable, Line, + Def, Defmacro, Macros, Exports, Functions) -> + Entry = translate_definition(defmacro, Line, Meta, Tuple, Clauses), + split_definition(T, Unreachable, Line, Def, Defmacro, [Tuple | Macros], Exports, + add_definition(Meta, Entry, Functions)). + +add_definition(Meta, Body, {Head, Tail}) -> + case lists:keyfind(file, 1, Meta) of + {file, {F, L}} -> + %% Erlang's epp attempts to perform offsetting when generated is set to true + %% and that causes cover to fail when processing modules. Therefore we never + %% pass the generated annotation forward for file attributes. The function + %% will still be marked as generated though if that's the case. + FileMeta = erl_anno:set_generated(false, ?ann(Meta)), + Attr = {attribute, FileMeta, file, {elixir_utils:characters_to_list(F), L}}, + {Head, [Attr, Body | Tail]}; + false -> + {[Body | Head], Tail} + end. + +translate_definition(Kind, Line, Meta, {Name, Arity}, Clauses) -> + ErlClauses = [translate_clause(Kind, Line, Clause, false) || Clause <- Clauses], + + case is_macro(Kind) of + true -> {function, ?ann(Meta), elixir_utils:macro_name(Name), Arity + 1, ErlClauses}; + false -> {function, ?ann(Meta), Name, Arity, ErlClauses} + end. + +translate_clause(Kind, Line, {Meta, Args, Guards, Body}, ExpandCaptures) -> + S = scope(Meta, ExpandCaptures), + + %% If the line matches the module line, then it is most likely an + %% auto-generated function and we don't want to track its contents. + Ann = + case ?line(Meta) of + Line -> erl_anno:set_generated(true, erl_anno:new(0)); + _ -> ?ann(Meta) + end, + + {TClause, TS} = + elixir_erl_clauses:clause(Ann, fun elixir_erl_pass:translate_args/3, Args, Body, Guards, S), + + case is_macro(Kind) of + true -> + FArgs = {var, Ann, '_@CALLER'}, + MClause = setelement(3, TClause, [FArgs | element(3, TClause)]), + + case TS#elixir_erl.caller of + true -> + FBody = {'match', Ann, + {'var', Ann, '__CALLER__'}, + ?remote(Ann, elixir_env, to_caller, [{var, Ann, '_@CALLER'}]) + }, + setelement(5, MClause, [FBody | element(5, TClause)]); + false -> + MClause + end; + false -> + TClause + end. + +is_macro(defmacro) -> true; +is_macro(defmacrop) -> true; +is_macro(_) -> false. + +% Functions + +functions_form(Line, Module, Def, Defmacro, Exports, Body, Deprecated, Struct, MD5) -> + {Spec, Info} = add_info_function(Line, Module, Def, Defmacro, Deprecated, Struct, MD5), + [{attribute, Line, export, lists:usort([{'__info__', 1} | Exports])}, Spec, Info | Body]. + +add_info_function(Line, Module, Def, Defmacro, Deprecated, Struct, MD5) -> + DefNA = [NA || {NA, _Meta} <- Def], + DefmacroNA = [NA || {NA, _Meta} <- Defmacro], + + AllowedAttrs = [attributes, compile, functions, macros, md5, exports_md5, module, deprecated, struct], + AllowedArgs = lists:map(fun(Atom) -> {atom, Line, Atom} end, AllowedAttrs), + + Spec = + {attribute, Line, spec, {{'__info__', 1}, + [{type, Line, 'fun', [ + {type, Line, product, [ + {type, Line, union, AllowedArgs} + ]}, + {type, Line, any, []} + ]}] + }}, + + Info = + {function, 0, '__info__', 1, [ + get_module_info(Module), + functions_info(DefNA), + macros_info(DefmacroNA), + struct_info(Struct), + exports_md5_info(MD5, DefNA, DefmacroNA, Struct), + get_module_info(Module, attributes), + get_module_info(Module, compile), + get_module_info(Module, md5), + deprecated_info(Deprecated) + ]}, + + {Spec, Info}. + +get_module_info(Module) -> + {clause, 0, [{atom, 0, module}], [], [{atom, 0, Module}]}. + +exports_md5_info(MD5Attr, Def, Defmacro, Struct) -> + MD5 = if + is_binary(MD5Attr) -> MD5Attr; + MD5Attr =:= nil -> elixir_module:exports_md5(Def, Defmacro, Struct) + end, + {clause, 0, [{atom, 0, exports_md5}], [], [elixir_to_erl(MD5)]}. + +functions_info(Def) -> + {clause, 0, [{atom, 0, functions}], [], [elixir_to_erl(Def)]}. + +macros_info(Defmacro) -> + {clause, 0, [{atom, 0, macros}], [], [elixir_to_erl(Defmacro)]}. + +struct_info(nil) -> + {clause, 0, [{atom, 0, struct}], [], [{atom, 0, nil}]}; +struct_info(Fields) -> + {clause, 0, [{atom, 0, struct}], [], [elixir_to_erl(Fields)]}. + +get_module_info(Module, Key) -> + Call = ?remote(0, erlang, get_module_info, [{atom, 0, Module}, {var, 0, 'Key'}]), + {clause, 0, [{match, 0, {var, 0, 'Key'}, {atom, 0, Key}}], [], [Call]}. + +deprecated_info(Deprecated) -> + {clause, 0, [{atom, 0, deprecated}], [], [elixir_to_erl(Deprecated)]}. + +% Typespecs + +typespecs_form(Map, TranslatedTypespecs, MacroNames) -> + {Types, Specs, Callbacks, MacroCallbacks, OptionalCallbacks} = TranslatedTypespecs, + + AllCallbacks = Callbacks ++ MacroCallbacks, + MacroCallbackNames = [NameArity || {_, NameArity, _, _} <- MacroCallbacks], + validate_behaviour_info_and_attributes(Map, AllCallbacks), + validate_optional_callbacks(Map, AllCallbacks, OptionalCallbacks), + + Forms0 = [], + Forms1 = types_form(Types, Forms0), + Forms2 = callspecs_form(spec, Specs, [], MacroNames, Forms1, Map), + Forms3 = callspecs_form(callback, AllCallbacks, OptionalCallbacks, MacroCallbackNames, Forms2, Map), + + AllCallbacksWithoutSpecs = usort_callbacks([ + {{Kind, Name, Arity}, Meta} || {Kind, {Name, Arity}, Meta, _Spec} <- AllCallbacks + ]), + + {Types, AllCallbacksWithoutSpecs, Forms3}. + +usort_callbacks(Callbacks) -> + % Sort and deduplicate callbacks. For duplicated callbacks we take + % the one with earliest line. + + LineComparator = fun + ({Callback1, Meta1}, {Callback1, Meta2}) -> ?line(Meta1) =< ?line(Meta2); + ({Callback1, _Meta1}, {Callback2, _Meta2}) -> Callback1 =< Callback2 + end, + + UniqFun = fun({Callback, _Meta}) -> Callback end, + + lists:uniq(UniqFun, lists:sort(LineComparator, Callbacks)). + +%% Types + +types_form(Types, Forms) -> + Fun = fun + ({Kind, NameArity, Meta, Expr, true}, Acc) -> + Line = ?line(Meta), + [{attribute, Line, export_type, [NameArity]}, {attribute, Line, Kind, Expr} | Acc]; + ({Kind, _NameArity, Meta, Expr, false}, Acc) -> + Line = ?line(Meta), + [{attribute, Line, Kind, Expr} | Acc] + end, + + lists:foldl(Fun, Forms, Types). + +%% Specs and callbacks + +validate_behaviour_info_and_attributes(#{definitions := Defs} = Map, AllCallbacks) -> + case {lists:keyfind({behaviour_info, 1}, 1, Defs), AllCallbacks} of + {false, _} -> + ok; + {_, [{Kind, {Name, Arity}, _, _} | _]} when Kind == callback; Kind == macrocallback -> + file_error(Map, {callbacks_but_also_behaviour_info, {Kind, Name, Arity}}); + {_, _} -> + ok + end. + +validate_optional_callbacks(Map, AllCallbacks, Optional) -> + lists:foldl(fun(Callback, Acc) -> + case Callback of + {Name, Arity} when is_atom(Name) and is_integer(Arity) -> ok; + _ -> file_error(Map, {ill_defined_optional_callback, Callback}) + end, + + case lists:keyfind(Callback, 2, AllCallbacks) of + false -> file_error(Map, {unknown_callback, Callback}); + _ -> ok + end, + + case Acc of + #{Callback := _} -> file_error(Map, {duplicate_optional_callback, Callback}); + _ -> ok + end, + + maps:put(Callback, true, Acc) + end, #{}, Optional). + +callspecs_form(_Kind, [], _Optional, _Macros, Forms, _ModuleMap) -> + Forms; +callspecs_form(Kind, Entries, Optional, Macros, Forms, ModuleMap) -> + #{unreachable := Unreachable} = ModuleMap, + + {SpecsMap, Signatures} = + lists:foldl(fun({_, NameArity, Meta, Spec}, {Acc, NA}) -> + Line = ?line(Meta), + + case Kind of + spec -> validate_spec_for_existing_function(ModuleMap, NameArity, Line); + _ -> ok + end, + + case lists:member(NameArity, Unreachable) of + false -> + case Acc of + #{NameArity := List} -> {Acc#{NameArity := [{Spec, Line} | List]}, NA}; + #{} -> {Acc#{NameArity => [{Spec, Line}]}, [NameArity | NA]} + end; + true -> + {Acc, NA} + end + end, {#{}, []}, Entries), + + lists:foldl(fun(NameArity, Acc) -> + #{NameArity := ExprsLines} = SpecsMap, + {Exprs, Lines} = lists:unzip(ExprsLines), + Line = lists:min(Lines), + + {Key, Value} = + case lists:member(NameArity, Macros) of + true -> + {Name, Arity} = NameArity, + {{elixir_utils:macro_name(Name), Arity + 1}, + lists:map(fun spec_for_macro/1, Exprs)}; + false -> + {NameArity, Exprs} + end, + + case lists:member(NameArity, Optional) of + true -> + [{attribute, Line, Kind, {Key, lists:reverse(Value)}}, + {attribute, Line, optional_callbacks, [Key]} | Acc]; + false -> + [{attribute, Line, Kind, {Key, lists:reverse(Value)}} | Acc] + end + end, Forms, lists:usort(Signatures)). + +spec_for_macro({type, Line, 'bounded_fun', [H | T]}) -> + {type, Line, 'bounded_fun', [spec_for_macro(H) | T]}; +spec_for_macro({type, Line, 'fun', [{type, _, product, Args} | T]}) -> + {type, Line, 'fun', [{type, Line, product, [{type, Line, term, []} | Args]} | T]}; +spec_for_macro(Else) -> + Else. + +validate_spec_for_existing_function(ModuleMap, NameAndArity, Line) -> + #{definitions := Defs, file := File} = ModuleMap, + + case lists:keymember(NameAndArity, 1, Defs) of + true -> ok; + false -> file_error(#{anno => erl_anno:new(Line), file => File}, {spec_for_undefined_function, NameAndArity}) + end. + +% Attributes + +attributes_form(Line, Attributes, Forms) -> + Fun = fun({Key, Value}, Acc) -> [{attribute, Line, Key, Value} | Acc] end, + lists:foldr(Fun, Forms, Attributes). + +% Loading forms + +load_form(#{file := File, compile_opts := Opts} = Map, Prefix, Forms, Specs, Chunks) -> + CompileOpts = extra_chunks_opts(Chunks, debug_opts(Map, Specs, Opts)), + {_, Binary} = elixir_erl_compiler:noenv_forms(Prefix ++ Specs ++ Forms, File, CompileOpts), + Binary. + +debug_opts(Map, Specs, Opts) -> + case take_debug_opts(Opts) of + {true, Rest} -> [{debug_info, {?MODULE, {elixir_v1, Map, Specs}}} | Rest]; + {false, Rest} -> [{debug_info, {?MODULE, none}} | Rest] + end. + +take_debug_opts(Opts) -> + case proplists:get_value(debug_info, Opts) of + true -> {true, proplists:delete(debug_info, Opts)}; + false -> {false, proplists:delete(debug_info, Opts)}; + undefined -> {elixir_config:get(debug_info), Opts} + end. + +extra_chunks_opts([], Opts) -> Opts; +extra_chunks_opts(Chunks, Opts) -> [{extra_chunks, Chunks} | Opts]. + +docs_chunk(Map, Set, Module, Anno, Def, Defmacro, Types, Callbacks, ChunkOpts) -> + #{file := File, attributes := Attributes} = Map, + + case elixir_config:get(docs) of + true -> + {ModuleDocLine, ModuleDoc} = get_moduledoc(erl_anno:line(Anno), Set), + ModuleDocMeta = get_moduledoc_meta(Set), + FunctionDocs = get_docs(Set, Module, Def, function), + MacroDocs = get_docs(Set, Module, Defmacro, macro), + CallbackDocs = get_callback_docs(Set, Callbacks), + TypeDocs = get_type_docs(Set, Types), + + ModuleMeta = ModuleDocMeta#{ + source_path => elixir_utils:characters_to_list(File), + source_annos => [Anno], + behaviours => [Mod || {behaviour, Mod} <- Attributes] + }, + + DocsChunkData = term_to_binary({docs_v1, + erl_anno:new(ModuleDocLine), + elixir, + <<"text/markdown">>, + ModuleDoc, + ModuleMeta, + FunctionDocs ++ MacroDocs ++ CallbackDocs ++ TypeDocs + }, [compressed | ChunkOpts]), + + [{<<"Docs">>, DocsChunkData}]; + + false -> + [] + end. + +doc_value(Doc, Name) -> + case Doc of + false -> + hidden; + nil -> + case erlang:atom_to_list(Name) of + [$_ | _] -> hidden; + _ -> none + end; + Doc -> + #{<<"en">> => Doc} + end. + +get_moduledoc(Line, Set) -> + case ets:lookup_element(Set, moduledoc, 2) of + nil -> {Line, none}; + {DocLine, false} -> {DocLine, hidden}; + {DocLine, nil} -> {DocLine, none}; + {DocLine, Doc} -> {DocLine, #{<<"en">> => Doc}} + end. + +get_moduledoc_meta(Set) -> + case ets:lookup(Set, {moduledoc, meta}) of + [] -> #{}; + [{{moduledoc, meta}, Map}] when is_map(Map) -> Map + end. + +get_docs(Set, Module, Definitions, Kind) -> + [{Key, + erl_anno:new(Line), + [signature_to_binary(Module, Name, Signature)], + doc_value(Doc, Name), + Meta#{source_annos => [?ann(DefinitionMeta)]} + } || {{Name, Arity}, DefinitionMeta} <- Definitions, + {Key, _Ctx, Line, Signature, Doc, Meta} <- ets:lookup(Set, {Kind, Name, Arity})]. + +get_callback_docs(Set, Callbacks) -> + [{Key, + erl_anno:new(Line), + [], + doc_value(Doc, Name), + Meta#{source_annos => [?ann(DefinitionMeta)]} + } || {{Kind, Name, Arity}, DefinitionMeta} <- Callbacks, {Key, Line, Doc, Meta} <- ets:lookup(Set, {Kind, Name, Arity})]. + +get_type_docs(Set, Types) -> + [{Key, + erl_anno:new(Line), + [], + doc_value(Doc, Name), + Meta#{source_annos => [?ann(DefinitionMeta)]} + } || {_Kind, {Name, Arity}, DefinitionMeta, _, true} <- Types, + {Key, Line, Doc, Meta} <- ets:lookup(Set, {type, Name, Arity})]. + +signature_to_binary(_Module, Name, _Signature) when Name == '__aliases__'; Name == '__block__' -> + <<(atom_to_binary(Name))/binary, "(args)">>; + +signature_to_binary(_Module, fn, _Signature) -> + <<"fn(clauses)">>; + +signature_to_binary(_Module, Name, _Signature) + when Name == '__CALLER__'; Name == '__DIR__'; Name == '__ENV__'; + Name == '__MODULE__'; Name == '__STACKTRACE__'; Name == '%{}' -> + atom_to_binary(Name); + +signature_to_binary(_Module, '%', _) -> + <<"%struct{}">>; + +signature_to_binary(Module, '__struct__', []) -> + <<"%", ('Elixir.Kernel':inspect(Module))/binary, "{}">>; + +signature_to_binary(_, Name, Signature) -> + Quoted = {Name, [{closing, []}], Signature}, + Doc = 'Elixir.Inspect.Algebra':format('Elixir.Code':quoted_to_algebra(Quoted), infinity), + 'Elixir.IO':iodata_to_binary(Doc). + +checker_chunk(nil, _ChunkOpts) -> + []; +checker_chunk(Contents, ChunkOpts) -> + [{<<"ExCk">>, term_to_binary({checker_version(), Contents}, ChunkOpts)}]. + +checker_chunk(Map, Def, Signatures, ChunkOpts) -> + #{deprecated := Deprecated, defines_behaviour := DefinesBehaviour, attributes := Attributes} = Map, + DeprecatedMap = maps:from_list(Deprecated), + + Exports = + [begin + Signature = maps:get(FA, Signatures, none), + Info = case DeprecatedMap of + #{FA := Reason} -> #{deprecated => Reason, sig => Signature}; + #{} -> #{sig => Signature} + end, + {FA, Info} + end || {FA, _Meta} <- prepend_behaviour_info(DefinesBehaviour, Def)], + + Contents = #{ + exports => Exports, + mode => case lists:keymember('__protocol__', 1, Attributes) of + true -> protocol; + false -> elixir + end + }, + + checker_chunk(Contents, ChunkOpts). + +prepend_behaviour_info(true, Def) -> [{{behaviour_info, 1}, []} | Def]; +prepend_behaviour_info(false, Def) -> Def. + +%% Errors + +file_error(#{anno := Anno, file := File}, Error) -> + Line = erl_anno:line(Anno), + elixir_errors:file_error([{line, Line}], File, ?MODULE, Error). + +format_error({ill_defined_optional_callback, Callback}) -> + io_lib:format("invalid optional callback ~ts. @optional_callbacks expects a " + "keyword list of callback names and arities", ['Elixir.Kernel':inspect(Callback)]); +format_error({unknown_callback, {Name, Arity}}) -> + io_lib:format("unknown callback ~ts/~B given as optional callback", [Name, Arity]); +format_error({duplicate_optional_callback, {Name, Arity}}) -> + io_lib:format("~ts/~B has been specified as optional callback more than once", [Name, Arity]); +format_error({callbacks_but_also_behaviour_info, {Type, Fun, Arity}}) -> + io_lib:format("cannot define @~ts attribute for ~ts/~B when behaviour_info/1 is defined", + [Type, Fun, Arity]); +format_error({spec_for_undefined_function, {Name, Arity}}) -> + io_lib:format("spec for undefined function ~ts/~B", [Name, Arity]). diff --git a/lib/elixir/src/elixir_erl_clauses.erl b/lib/elixir/src/elixir_erl_clauses.erl new file mode 100644 index 00000000000..a24e857e647 --- /dev/null +++ b/lib/elixir/src/elixir_erl_clauses.erl @@ -0,0 +1,69 @@ +%% SPDX-License-Identifier: Apache-2.0 +%% SPDX-FileCopyrightText: 2021 The Elixir Team +%% SPDX-FileCopyrightText: 2012 Plataformatec + +%% Handle code related to args, guard and -> matching for case, +%% fn, receive and friends. try is handled in elixir_erl_try. +-module(elixir_erl_clauses). +-export([match/4, clause/6, clauses/2, guards/4, get_clauses/3]). +-include("elixir.hrl"). + +%% Get clauses under the given key. + +get_clauses(Key, Keyword, As) -> + case lists:keyfind(Key, 1, Keyword) of + {Key, Clauses} when is_list(Clauses) -> + [{As, Meta, Left, Right} || {'->', Meta, [Left, Right]} <- Clauses]; + _ -> + [] + end. + +%% Translate matches + +match(Ann, Fun, Match, #elixir_erl{context=Context} = S) when Context =/= match -> + {Result, NewS} = Fun(Match, Ann, S#elixir_erl{context=match}), + {Result, NewS#elixir_erl{context=Context}}; +match(Ann, Fun, Match, S) -> + Fun(Match, Ann, S). + +%% Translate clauses with args, guards and expressions + +clause(Ann, Fun, Match, Expr, Guards, S) -> + {TMatch, SA} = match(Ann, Fun, Match, S), + SG = SA#elixir_erl{extra_guards=[]}, + TGuards = guards(Ann, Guards, SA#elixir_erl.extra_guards, SG), + {TExpr, SE} = elixir_erl_pass:translate(Expr, Ann, SG), + {{clause, Ann, TMatch, TGuards, unblock(TExpr)}, SE}. + +% Translate/Extract guards from the given expression. + +guards(Ann, Guards, Extra, S) -> + SG = S#elixir_erl{context=guard}, + case Guards of + [] -> case Extra of [] -> []; _ -> [Extra] end; + _ -> [translate_guard(Guard, Ann, SG, Extra) || Guard <- Guards] + end. + +translate_guard(Guard, Ann, S, Extra) -> + [element(1, elixir_erl_pass:translate(Guard, Ann, S)) | Extra]. + +% Function for translating macros with match style like case and receive. + +clauses([], S) -> + {[], S}; + +clauses(Clauses, S) -> + lists:mapfoldl(fun each_clause/2, S, Clauses). + +each_clause({match, Meta, [Condition], Expr}, S) -> + {Arg, Guards} = elixir_utils:extract_guards(Condition), + clause(?ann(Meta), fun elixir_erl_pass:translate_args/3, [Arg], Expr, Guards, S); + +each_clause({expr, Meta, [Condition], Expr}, S) -> + Ann = ?ann(Meta), + {TCondition, SC} = elixir_erl_pass:translate(Condition, Ann, S), + {TExpr, SB} = elixir_erl_pass:translate(Expr, Ann, SC), + {{clause, Ann, [TCondition], [], unblock(TExpr)}, SB}. + +unblock({'block', _, Exprs}) -> Exprs; +unblock(Exprs) -> [Exprs]. diff --git a/lib/elixir/src/elixir_erl_compiler.erl b/lib/elixir/src/elixir_erl_compiler.erl new file mode 100644 index 00000000000..3982b84ae35 --- /dev/null +++ b/lib/elixir/src/elixir_erl_compiler.erl @@ -0,0 +1,191 @@ +%% SPDX-License-Identifier: Apache-2.0 +%% SPDX-FileCopyrightText: 2021 The Elixir Team +%% SPDX-FileCopyrightText: 2012 Plataformatec + +-module(elixir_erl_compiler). +-export([spawn/1, noenv_forms/3, erl_to_core/2, env_compiler_options/0]). +-include("elixir.hrl"). + +spawn(Fun) -> + CompilerInfo = get(elixir_compiler_info), + {error_handler, ErrorHandler} = erlang:process_info(self(), error_handler), + + CodeDiagnostics = + case get(elixir_code_diagnostics) of + undefined -> undefined; + {_Tail, Log} -> {[], Log} + end, + + {_, Ref} = + spawn_monitor(fun() -> + erlang:process_flag(error_handler, ErrorHandler), + put(elixir_compiler_info, CompilerInfo), + put(elixir_code_diagnostics, CodeDiagnostics), + + try Fun() of + Result -> exit({ok, Result, get(elixir_code_diagnostics)}) + catch + Kind:Reason:Stack -> + exit({Kind, Reason, Stack, get(elixir_code_diagnostics)}) + end + end), + + receive + {'DOWN', Ref, process, _, {ok, Result, Diagnostics}} -> + copy_diagnostics(Diagnostics), + Result; + {'DOWN', Ref, process, _, {Kind, Reason, Stack, Diagnostics}} -> + copy_diagnostics(Diagnostics), + erlang:raise(Kind, Reason, Stack) + end. + +copy_diagnostics(undefined) -> + ok; +copy_diagnostics({Head, _}) -> + case get(elixir_code_diagnostics) of + undefined -> ok; + {Tail, Log} -> put(elixir_code_diagnostics, {Head ++ Tail, Log}) + end. + +env_compiler_options() -> + case persistent_term:get(?MODULE, undefined) of + undefined -> + Options = compile:env_compiler_options() -- [warnings_as_errors], + persistent_term:put(?MODULE, Options), + Options; + + Options -> + Options + end. + +erl_to_core(Forms, Opts) -> + %% TODO: Remove parse transform handling on Elixir v2.0 + case [M || {parse_transform, M} <- Opts] of + [] -> + v3_core:module(Forms, Opts); + _ -> + case compile:noenv_forms(Forms, [no_spawn_compiler_process, to_core0, return, no_auto_import | Opts]) of + {ok, _Module, Core, Warnings} -> {ok, Core, Warnings}; + {error, Errors, Warnings} -> {error, Errors, Warnings} + end + end. + +noenv_forms(Forms, File, Opts) when is_list(Forms), is_list(Opts), is_binary(File) -> + Source = elixir_utils:characters_to_list(File), + + case erl_to_core(Forms, Opts) of + {ok, CoreForms, CoreWarnings} -> + format_warnings(Opts, CoreWarnings), + CompileOpts = [no_spawn_compiler_process, from_core, no_core_prepare, + no_auto_import, return, {source, Source} | Opts], + + case compile:noenv_forms(CoreForms, CompileOpts) of + {ok, Module, Binary, Warnings} when is_binary(Binary) -> + format_warnings(Opts, Warnings), + {Module, Binary}; + + {ok, Module, _, _} -> + incompatible_options("could not compile module ~ts", [elixir_aliases:inspect(Module)], File); + + {ok, Module, _} -> + incompatible_options("could not compile module ~ts", [elixir_aliases:inspect(Module)], File); + + {error, Errors, Warnings} -> + format_warnings(Opts, Warnings), + format_errors(Errors); + + _ -> + incompatible_options("could not compile module", [], File) + end; + + {error, CoreErrors, CoreWarnings} -> + format_warnings(Opts, CoreWarnings), + format_errors(CoreErrors) + end. + +incompatible_options(Prefix, Args, File) -> + Message = io_lib:format( + Prefix ++ ". We expected the compiler to return a .beam binary but " + "got something else. This usually happens because ERL_COMPILER_OPTIONS or @compile " + "was set to change the compilation outcome in a way that is incompatible with Elixir", + Args + ), + + elixir_errors:compile_error([], File, Message). + +format_errors([]) -> + exit({nocompile, "compilation failed but no error was raised"}); +format_errors(Errors) -> + lists:foreach(fun + ({File, Each}) when is_list(File) -> + BinFile = elixir_utils:characters_to_binary(File), + lists:foreach(fun(Error) -> handle_file_error(BinFile, Error) end, Each); + ({Mod, Each}) when is_atom(Mod) -> + lists:foreach(fun(Error) -> handle_file_error(elixir_aliases:inspect(Mod), Error) end, Each) + end, Errors). + +format_warnings(Opts, Warnings) -> + NoWarnNoMatch = proplists:get_value(nowarn_nomatch, Opts, false), + lists:foreach(fun ({File, Each}) -> + BinFile = elixir_utils:characters_to_binary(File), + lists:foreach(fun(Warning) -> + handle_file_warning(NoWarnNoMatch, BinFile, Warning) + end, Each) + end, Warnings). + +%% Handle warnings from Erlang land + +%% Those we implement ourselves +handle_file_warning(_, _File, {_Line, v3_core, {map_key_repeated, _}}) -> ok; +handle_file_warning(_, _File, {_Line, sys_core_fold, {ignored, useless_building}}) -> ok; + +%% We skip all of no_match related to no_clause, clause_type, guard, shadow. +%% Those have too little information and they overlap with the type system. +%% We keep the remaining ones because the Erlang compiler performs analyses +%% on literals (including numbers), which the type system does not do. +handle_file_warning(_, _File, {_Line, sys_core_fold, {nomatch, Reason}}) when is_atom(Reason) -> ok; + +%% Ignore all linting errors (only come up on parse transforms) +handle_file_warning(_, _File, {_Line, erl_lint, _}) -> ok; + +handle_file_warning(_, File, {Line, Module, Desc}) -> + Message = custom_format(Module, Desc), + elixir_errors:erl_warn(Line, File, Message). + +%% Handle warnings + +handle_file_error(File, {beam_validator, Desc}) -> + elixir_errors:compile_error([{line, 0}], File, beam_validator:format_error(Desc)); +handle_file_error(File, {Line, Module, Desc}) -> + Message = custom_format(Module, Desc), + elixir_errors:compile_error([{line, Line}], File, Message). + +%% Mention the capture operator in make_fun +custom_format(sys_core_fold, {ignored, {no_effect, {erlang, make_fun, 3}}}) -> + "the result of the capture operator & (Function.capture/3) is never used"; + +%% Make no_effect clauses pretty +custom_format(sys_core_fold, {ignored, {no_effect, {erlang, F, A}}}) -> + {Fmt, Args} = case erl_internal:comp_op(F, A) of + true -> {"use of operator ~ts has no effect", [elixir_utils:erlang_comparison_op_to_elixir(F)]}; + false -> + case erl_internal:bif(F, A) of + false -> {"the call to :erlang.~ts/~B has no effect", [F, A]}; + true -> {"the call to ~ts/~B has no effect", [F, A]} + end + end, + io_lib:format(Fmt, Args); + +custom_format(sys_core_fold, {nomatch, {shadow, Line, {ErlName, ErlArity}}}) -> + {Name, Arity} = elixir_utils:erl_fa_to_elixir_fa(ErlName, ErlArity), + + io_lib:format( + "this clause for ~ts/~B cannot match because a previous clause at line ~B always matches", + [Name, Arity, Line] + ); + +custom_format([], Desc) -> + io_lib:format("~p", [Desc]); + +custom_format(Module, Desc) -> + Module:format_error(Desc). diff --git a/lib/elixir/src/elixir_erl_for.erl b/lib/elixir/src/elixir_erl_for.erl new file mode 100644 index 00000000000..61e558df549 --- /dev/null +++ b/lib/elixir/src/elixir_erl_for.erl @@ -0,0 +1,369 @@ +%% SPDX-License-Identifier: Apache-2.0 +%% SPDX-FileCopyrightText: 2021 The Elixir Team +%% SPDX-FileCopyrightText: 2012 Plataformatec + +-module(elixir_erl_for). +-export([translate/3]). +-include("elixir.hrl"). + +-define(empty_map_set_pattern, {map, _, [ + {map_field_assoc, _, {atom, _, '__struct__'}, {atom, _, 'Elixir.MapSet'}}, + {map_field_assoc, _, {atom, _, map}, {map, _, []}} + ]}). + +translate(Meta, Args, S) -> + {Cases, [{do, Expr} | Opts]} = elixir_utils:split_last(Args), + + case lists:keyfind(reduce, 1, Opts) of + {reduce, Reduce} -> translate_reduce(Meta, Cases, Expr, Reduce, S); + false -> translate_into(Meta, Cases, Expr, Opts, S) + end. + +translate_reduce(Meta, Cases, Expr, Reduce, S) -> + Ann = ?ann(Meta), + {TReduce, SR} = elixir_erl_pass:translate(Reduce, Ann, S), + {TCases, SC} = translate_gen(Meta, Cases, [], SR), + CaseExpr = {'case', Meta, [ok, [{do, Expr}]]}, + {TExpr, SE} = elixir_erl_pass:translate(CaseExpr, Ann, SC), + + InnerFun = fun + ({'case', CaseAnn, _, CaseBlock}, InnerAcc) -> {'case', CaseAnn, InnerAcc, CaseBlock} + end, + + build_reduce(Ann, TCases, InnerFun, TExpr, TReduce, false, SE). + +translate_into(Meta, Cases, Expr, Opts, S) -> + Ann = ?ann(Meta), + + {TInto, SI} = + case lists:keyfind(into, 1, Opts) of + {into, Into} -> elixir_erl_pass:translate(Into, Ann, S); + false -> {false, S} + end, + + TUniq = lists:keyfind(uniq, 1, Opts) == {uniq, true}, + + {TCases, SC} = translate_gen(Meta, Cases, [], SI), + {TExpr, SE} = elixir_erl_pass:translate(wrap_expr_if_unused(Expr, TInto), Ann, SC), + + case inline_or_into(TInto) of + inline -> build_inline(Ann, TCases, TExpr, TInto, TUniq, SE); + into -> build_into(Ann, TCases, TExpr, TInto, TUniq, SE) + end. + +%% In case we have no return, we wrap the expression +%% in a block that returns nil. +wrap_expr_if_unused(Expr, false) -> {'__block__', [], [Expr, nil]}; +wrap_expr_if_unused(Expr, _) -> Expr. + +translate_gen(ForMeta, [{'<-', Meta, [Left, Right]} | T], Acc, S) -> + {TLeft, TRight, TFilters, TT, TS} = translate_gen(Meta, Left, Right, T, S), + TAcc = [{enum, Meta, TLeft, TRight, TFilters} | Acc], + translate_gen(ForMeta, TT, TAcc, TS); +translate_gen(ForMeta, [{'<<>>', _, [{'<-', Meta, [Left, Right]}]} | T], Acc, S) -> + {TLeft, TRight, TFilters, TT, TS} = translate_gen(Meta, Left, Right, T, S), + TAcc = [{bin, Meta, TLeft, TRight, TFilters} | Acc], + translate_gen(ForMeta, TT, TAcc, TS); +translate_gen(_ForMeta, [], Acc, S) -> + {lists:reverse(Acc), S}. + +translate_gen(Meta, Left, Right, T, S) -> + Ann = ?ann(Meta), + {TRight, SR} = elixir_erl_pass:translate(Right, Ann, S), + {LeftArgs, LeftGuards} = elixir_utils:extract_guards(Left), + {TLeft, SL} = elixir_erl_clauses:match(Ann, fun elixir_erl_pass:translate/3, LeftArgs, + SR#elixir_erl{extra=pin_guard}), + + TLeftGuards = elixir_erl_clauses:guards(Ann, LeftGuards, [], SL), + ExtraGuards = [{nil, X} || X <- SL#elixir_erl.extra_guards], + {Filters, TT} = collect_filters(T, []), + + {TFilters, TS} = + lists:mapfoldr( + fun(F, SF) -> translate_filter(F, Ann, SF) end, + SL#elixir_erl{extra=S#elixir_erl.extra, extra_guards=[]}, + Filters + ), + + %% The list of guards is kept in reverse order + Guards = TFilters ++ translate_guards(TLeftGuards) ++ ExtraGuards, + {TLeft, TRight, Guards, TT, TS}. + +translate_guards([]) -> + []; +translate_guards([[Guards]]) -> + [{nil, Guards}]; +translate_guards([[Left], [Right] | Rest]) -> + translate_guards([[{op, element(2, Left), 'orelse', Left, Right}] | Rest]). + +translate_filter(Filter, Ann, S) -> + {TFilter, TS} = elixir_erl_pass:translate(Filter, Ann, S), + case elixir_utils:returns_boolean(Filter) of + true -> + {{nil, TFilter}, TS}; + false -> + {Name, VS} = elixir_erl_var:build('_', TS), + {{{var, 0, Name}, TFilter}, VS} + end. + +collect_filters([{'<-', _, [_, _]} | _] = T, Acc) -> + {Acc, T}; +collect_filters([{'<<>>', _, [{'<-', _, [_, _]}]} | _] = T, Acc) -> + {Acc, T}; +collect_filters([H | T], Acc) -> + collect_filters(T, [H | Acc]); +collect_filters([], Acc) -> + {Acc, []}. + +build_inline(Ann, Clauses, Expr, Into, Uniq, S) -> + case not Uniq and lists:all(fun(Clause) -> element(1, Clause) == bin end, Clauses) of + true -> {build_comprehension(Ann, Clauses, Expr, Into), S}; + false -> build_inline_each(Ann, Clauses, Expr, Into, Uniq, S) + end. + +build_inline_each(Ann, Clauses, Expr, false, Uniq, S) -> + InnerFun = fun(InnerExpr, _InnerAcc) -> InnerExpr end, + build_reduce(Ann, Clauses, InnerFun, Expr, {nil, Ann}, Uniq, S); +build_inline_each(Ann, [{enum, _, Left = {var, _, _}, Right, [] = _Filters}], Expr, {nil, _} = _Into, false, S) -> + Clauses = [{clause, Ann, [Left], [], [Expr]}], + Args = [Right, {'fun', Ann, {clauses, Clauses}}], + {?remote(Ann, 'Elixir.Enum', map, Args), S}; +build_inline_each(Ann, [{enum, _, Left = {var, _, _}, Right, [] = _Filters}], Expr, {map, _, []} = _Into, false, S) -> + Clauses = [{clause, Ann, [Left], [], [Expr]}], + Args = [Right, {'fun', Ann, {clauses, Clauses}}], + List = ?remote(Ann, 'Elixir.Enum', map, Args), + {?remote(Ann, maps, from_list, [List]), S}; +build_inline_each(Ann, Clauses, Expr, {nil, _} = Into, Uniq, S) -> + InnerFun = fun(InnerExpr, InnerAcc) -> {cons, Ann, InnerExpr, InnerAcc} end, + {ReduceExpr, SR} = build_reduce(Ann, Clauses, InnerFun, Expr, Into, Uniq, S), + {?remote(Ann, lists, reverse, [ReduceExpr]), SR}; +build_inline_each(Ann, Clauses, Expr, {bin, _, []}, Uniq, S) -> + {InnerValue, SV} = build_var(Ann, S), + Generated = erl_anno:set_generated(true, Ann), + + InnerFun = fun(InnerExpr, InnerAcc) -> + {'case', Ann, InnerExpr, [ + {clause, Generated, + [InnerValue], + [[?remote(Ann, erlang, is_bitstring, [InnerValue]), + ?remote(Ann, erlang, is_list, [InnerAcc])]], + [{cons, Generated, InnerAcc, InnerValue}]}, + {clause, Generated, + [InnerValue], + [], + [?remote(Ann, erlang, error, [{tuple, Ann, [{atom, Ann, badarg}, InnerValue]}])]} + ]} + end, + + {ReduceExpr, SR} = build_reduce(Ann, Clauses, InnerFun, Expr, {nil, Ann}, Uniq, SV), + {?remote(Ann, erlang, list_to_bitstring, [ReduceExpr]), SR}. + +build_into(Ann, Clauses, Expr, {map, _, []}, Uniq, S) -> + {ReduceExpr, SR} = build_inline_each(Ann, Clauses, Expr, {nil, Ann}, Uniq, S), + {?remote(Ann, maps, from_list, [ReduceExpr]), SR}; +build_into(Ann, Clauses, Expr, ?empty_map_set_pattern = _Into, Uniq, S) -> + InnerFun = fun(InnerExpr, InnerAcc) -> {cons, Ann, InnerExpr, InnerAcc} end, + {ReduceExpr, SR} = build_reduce(Ann, Clauses, InnerFun, Expr, {nil, Ann}, Uniq, S), + {?remote(Ann, 'Elixir.MapSet', new, [ReduceExpr]), SR}; +build_into(Ann, Clauses, Expr, Into, Uniq, S) -> + {Fun, SF} = build_var(Ann, S), + {Acc, SA} = build_var(Ann, SF), + {Kind, SK} = build_var(Ann, SA), + {Reason, SR} = build_var(Ann, SK), + {Stack, ST} = build_var(Ann, SR), + {Done, SD} = build_var(Ann, ST), + + InnerFun = fun(InnerExpr, InnerAcc) -> + {call, Ann, Fun, [InnerAcc, pair(Ann, cont, InnerExpr)]} + end, + + MatchExpr = {match, Ann, + {tuple, Ann, [Acc, Fun]}, + ?remote(Ann, 'Elixir.Collectable', into, [Into]) + }, + + {IntoReduceExpr, SN} = build_reduce(Ann, Clauses, InnerFun, Expr, Acc, Uniq, SD), + + TryExpr = + {'try', Ann, + [IntoReduceExpr], + [{clause, Ann, + [Done], + [], + [{call, Ann, Fun, [Done, {atom, Ann, done}]}]}], + [stacktrace_clause(Ann, Fun, Acc, Kind, Reason, Stack)], + []}, + + {{block, Ann, [MatchExpr, TryExpr]}, SN}. + +stacktrace_clause(Ann, Fun, Acc, Kind, Reason, Stack) -> + {clause, Ann, + [{tuple, Ann, [Kind, Reason, Stack]}], + [], + [{call, Ann, Fun, [Acc, {atom, Ann, halt}]}, + ?remote(Ann, erlang, raise, [Kind, Reason, Stack])]}. + +%% Helpers + +build_reduce(Ann, Clauses, InnerFun, Expr, Into, false, S) -> + {Acc, SA} = build_var(Ann, S), + {build_reduce_each(Clauses, InnerFun(Expr, Acc), Into, Acc, SA), SA}; +build_reduce(Ann, Clauses, InnerFun, Expr, Into, true, S) -> + %% Those variables are used only inside the anonymous function + %% so we don't need to worry about returning the scope. + {Acc, SA} = build_var(Ann, S), + {Value, SV} = build_var(Ann, SA), + {IntoAcc, SI} = build_var(Ann, SV), + {UniqAcc, SU} = build_var(Ann, SI), + + NewInto = {tuple, Ann, [Into, {map, Ann, []}]}, + AccTuple = {tuple, Ann, [IntoAcc, UniqAcc]}, + PutUniqExpr = {map, Ann, UniqAcc, [{map_field_assoc, Ann, Value, {atom, Ann, true}}]}, + + InnerExpr = {block, Ann, [ + {match, Ann, AccTuple, Acc}, + {match, Ann, Value, Expr}, + {'case', Ann, UniqAcc, [ + {clause, Ann, [{map, Ann, [{map_field_exact, Ann, Value, {atom, Ann, true}}]}], [], [AccTuple]}, + {clause, Ann, [{map, Ann, []}], [], [{tuple, Ann, [InnerFun(Value, IntoAcc), PutUniqExpr]}]} + ]} + ]}, + + EnumReduceCall = build_reduce_each(Clauses, InnerExpr, NewInto, Acc, SU), + {?remote(Ann, erlang, element, [{integer, Ann, 1}, EnumReduceCall]), SU}. + +build_reduce_each([{enum, Meta, Left, Right, Filters} | T], Expr, Arg, Acc, S) -> + Ann = ?ann(Meta), + True = build_reduce_each(T, Expr, Acc, Acc, S), + False = Acc, + Generated = erl_anno:set_generated(true, Ann), + + Clauses0 = + case is_var(Left) of + true -> []; + false -> + [{clause, Generated, + [{var, Ann, '_'}, Acc], [], + [False]}] + end, + + Clauses1 = + [{clause, Ann, + [Left, Acc], [], + [join_filters(Generated, Filters, True, False)]} | Clauses0], + + Args = [Right, Arg, {'fun', Ann, {clauses, Clauses1}}], + ?remote(Ann, 'Elixir.Enum', reduce, Args); + +build_reduce_each([{bin, Meta, Left, Right, Filters} | T], Expr, Arg, Acc, S) -> + Ann = ?ann(Meta), + Generated = erl_anno:set_generated(true, Ann), + {Tail, ST} = build_var(Ann, S), + {Fun, SF} = build_var(Ann, ST), + + True = build_reduce_each(T, Expr, Acc, Acc, SF), + False = Acc, + {bin, _, Elements} = Left, + TailElement = {bin_element, Ann, Tail, default, [bitstring]}, + + Clauses = + [{clause, Generated, + [{bin, Ann, [TailElement]}, Acc], [], + [Acc]}, + {clause, Generated, + [Tail, {var, Ann, '_'}], [], + [?remote(Ann, erlang, error, [pair(Ann, badarg, Tail)])]}], + + NoVarClauses = + case no_var(Generated, Elements) of + error -> + Clauses; + + NoVarElements -> + NoVarMatch = {bin, Ann, NoVarElements ++ [TailElement]}, + [{clause, Generated, [NoVarMatch, Acc], [], [{call, Ann, Fun, [Tail, False]}]} | Clauses] + end, + + BinMatch = {bin, Ann, Elements ++ [TailElement]}, + VarClauses = + [{clause, Ann, + [BinMatch, Acc], [], + [{call, Ann, Fun, [Tail, join_filters(Generated, Filters, True, False)]}]} | NoVarClauses], + + {call, Ann, + {named_fun, Ann, element(3, Fun), VarClauses}, + [Right, Arg]}; + +build_reduce_each([], Expr, _Arg, _Acc, _S) -> + Expr. + +is_var({var, _, _}) -> true; +is_var(_) -> false. + +pair(Ann, Atom, Arg) -> + {tuple, Ann, [{atom, Ann, Atom}, Arg]}. + +build_var(Ann, S) -> + {Name, ST} = elixir_erl_var:build('_', S), + {{var, Ann, Name}, ST}. + +no_var(ParentAnn, Elements) -> + try + [{bin_element, Ann, NoVarExpr, no_var_size(Size), Types} || + {bin_element, Ann, Expr, Size, Types} <- Elements, + NoVarExpr <- no_var_expr(ParentAnn, Expr)] + catch + unbound_size -> error + end. + +no_var_expr(Ann, {string, _, String}) -> [{var, Ann, '_'} || _ <- String]; +no_var_expr(Ann, _) -> [{var, Ann, '_'}]. +no_var_size({var, _, _}) -> throw(unbound_size); +no_var_size(Size) -> Size. + +build_comprehension(Ann, Clauses, Expr, Into) -> + {comprehension_kind(Into), Ann, Expr, comprehension_clause(Clauses)}. + +comprehension_clause([{bin, Meta, Left, Right, Filters} | T]) -> + Ann = ?ann(Meta), + [{b_generate, Ann, Left, Right}] ++ + comprehension_filter(Ann, Filters) ++ + comprehension_clause(T); +comprehension_clause([]) -> + []. + +comprehension_kind(false) -> lc; +comprehension_kind({nil, _}) -> lc; +comprehension_kind({bin, _, []}) -> bc. + +inline_or_into({bin, _, []}) -> inline; +inline_or_into({nil, _}) -> inline; +inline_or_into(false) -> inline; +inline_or_into(_) -> into. + +comprehension_filter(Ann, Filters) -> + [join_filter(Ann, Filter, {atom, Ann, true}, {atom, Ann, false}) || + Filter <- lists:reverse(Filters)]. + +join_filters(_Ann, [], True, _False) -> + True; +join_filters(Ann, [H | T], True, False) -> + lists:foldl(fun(Filter, Acc) -> + join_filter(Ann, Filter, Acc, False) + end, join_filter(Ann, H, True, False), T). + +join_filter(Ann, {nil, Filter}, True, False) -> + {'case', Ann, Filter, [ + {clause, Ann, [{atom, Ann, true}], [], [True]}, + {clause, Ann, [{atom, Ann, false}], [], [False]} + ]}; +join_filter(Ann, {Var, Filter}, True, False) -> + Guards = [[{op, Ann, 'orelse', + {op, Ann, '==', Var, {atom, Ann, false}}, + {op, Ann, '==', Var, {atom, Ann, nil}} + }]], + + {'case', Ann, Filter, [ + {clause, Ann, [Var], Guards, [False]}, + {clause, Ann, [{var, Ann, '_'}], [], [True]} + ]}. diff --git a/lib/elixir/src/elixir_erl_pass.erl b/lib/elixir/src/elixir_erl_pass.erl new file mode 100644 index 00000000000..a06c8c7ffeb --- /dev/null +++ b/lib/elixir/src/elixir_erl_pass.erl @@ -0,0 +1,736 @@ +%% SPDX-License-Identifier: Apache-2.0 +%% SPDX-FileCopyrightText: 2021 The Elixir Team +%% SPDX-FileCopyrightText: 2012 Plataformatec + +%% Translate Elixir quoted expressions to Erlang Abstract Format. +-module(elixir_erl_pass). +-export([translate/3, translate_args/3, no_parens_remote/2, parens_map_field/2]). +-include("elixir.hrl"). + +%% = + +translate({'=', Meta, [{'_', _, Atom}, Right]}, _Ann, S) when is_atom(Atom) -> + Ann = ?ann(Meta), + {TRight, SR} = translate(Right, Ann, S), + {{match, Ann, {var, Ann, '_'}, TRight}, SR}; + +translate({'=', Meta, [Left, Right]}, _Ann, S) -> + Ann = ?ann(Meta), + {TRight, SR} = translate(Right, Ann, S), + case elixir_erl_clauses:match(Ann, fun translate/3, Left, SR) of + {TLeft, #elixir_erl{extra_guards=ExtraGuards, context=Context} = SL0} + when ExtraGuards =/= [], Context =/= match -> + SL1 = SL0#elixir_erl{extra_guards=[]}, + {ResultVarName, SL2} = elixir_erl_var:build('_', SL1), + Match = {match, Ann, TLeft, TRight}, + Generated = erl_anno:set_generated(true, Ann), + ResultVar = {var, Generated, ResultVarName}, + ResultMatch = {match, Generated, ResultVar, Match}, + True = {atom, Generated, true}, + Reason = {tuple, Generated, [{atom, Generated, badmatch}, ResultVar]}, + RaiseExpr = ?remote(Generated, erlang, error, [Reason]), + GuardsExp = {'if', Generated, [ + {clause, Generated, [], [ExtraGuards], [ResultVar]}, + {clause, Generated, [], [[True]], [RaiseExpr]} + ]}, + {{block, Generated, [ResultMatch, GuardsExp]}, SL2}; + + {TLeft, SL} -> + {{match, Ann, TLeft, TRight}, SL} + end; + +%% Containers + +translate({'{}', Meta, Args}, _Ann, S) when is_list(Args) -> + Ann = ?ann(Meta), + {TArgs, SE} = translate_args(Args, Ann, S), + {{tuple, Ann, TArgs}, SE}; + +translate({'%{}', Meta, Args}, _Ann, S) when is_list(Args) -> + translate_map(?ann(Meta), Args, S); + +translate({'%', Meta, [{'^', _, [{Name, _, Context}]} = Left, Right]}, _Ann, S) when is_atom(Name), is_atom(Context) -> + translate_struct_var_name(?ann(Meta), Left, Right, S); + +translate({'%', Meta, [{Name, _, Context} = Left, Right]}, _Ann, S) when is_atom(Name), is_atom(Context) -> + translate_struct_var_name(?ann(Meta), Left, Right, S); + +translate({'%', Meta, [Left, Right]}, _Ann, S) -> + translate_struct(?ann(Meta), Left, Right, S); + +translate({'<<>>', Meta, Args}, _Ann, S) when is_list(Args) -> + translate_bitstring(Meta, Args, S); + +%% Blocks + +translate({'__block__', Meta, Args}, _Ann, S) when is_list(Args) -> + Ann = ?ann(Meta), + {TArgs, SA} = translate_block(Args, Ann, [], S), + {{block, Ann, lists:reverse(TArgs)}, SA}; + +%% Compilation environment macros + +translate({'__CALLER__', Meta, Atom}, _Ann, S) when is_atom(Atom) -> + {{var, ?ann(Meta), '__CALLER__'}, S#elixir_erl{caller=true}}; + +translate({'__STACKTRACE__', Meta, Atom}, _Ann, S = #elixir_erl{stacktrace=Var}) when is_atom(Atom) -> + {{var, ?ann(Meta), Var}, S}; + +translate({'super', Meta, Args}, _Ann, S) when is_list(Args) -> + Ann = ?ann(Meta), + + %% In the expanded AST, super is used to invoke a function + %% in the current module originated from a default clause + %% or a super call. + {TArgs, SA} = translate_args(Args, Ann, S), + {super, {Kind, Name}} = lists:keyfind(super, 1, Meta), + + if + Kind == defmacro; Kind == defmacrop -> + MacroName = elixir_utils:macro_name(Name), + {{call, Ann, {atom, Ann, MacroName}, [{var, Ann, '__CALLER__'} | TArgs]}, SA#elixir_erl{caller=true}}; + Kind == def; Kind == defp -> + {{call, Ann, {atom, Ann, Name}, TArgs}, SA} + end; + +%% Functions + +translate({'&', Meta, [{'/', _, [{{'.', _, [Remote, Fun]}, _, []}, Arity]}]}, _Ann, S) + when is_atom(Fun), is_integer(Arity) -> + Ann = ?ann(Meta), + {TRemote, SR} = translate(Remote, Ann, S), + TFun = {atom, Ann, Fun}, + TArity = {integer, Ann, Arity}, + {{'fun', Ann, {function, TRemote, TFun, TArity}}, SR}; +translate({'&', Meta, [{'/', _, [{Fun, _, Atom}, Arity]}]}, Ann, S) + when is_atom(Fun), is_atom(Atom), is_integer(Arity) -> + case S of + #elixir_erl{expand_captures=true} -> + {Vars, SV} = lists:mapfoldl(fun(_, Acc) -> + {Var, _, AccS} = elixir_erl_var:assign(Meta, Acc), + {Var, AccS} + end, S, tl(lists:seq(0, Arity))), + translate({'fn', Meta, [{'->', Meta, [Vars, {Fun, Meta, Vars}]}]}, Ann, SV); + + #elixir_erl{expand_captures=false} -> + {{'fun', ?ann(Meta), {function, Fun, Arity}}, S} + end; + +translate({fn, Meta, Clauses}, _Ann, S) -> + Transformer = fun({'->', CMeta, [ArgsWithGuards, Expr]}, Acc) -> + {Args, Guards} = elixir_utils:extract_splat_guards(ArgsWithGuards), + elixir_erl_clauses:clause(?ann(CMeta), fun translate_fn_match/3, Args, Expr, Guards, Acc) + end, + {TClauses, NS} = lists:mapfoldl(Transformer, S, Clauses), + {{'fun', ?ann(Meta), {clauses, TClauses}}, NS}; + +%% Cond + +translate({'cond', CondMeta, [[{do, Clauses}]]}, Ann, S) -> + [{'->', Meta, [[Condition], _]} = H | T] = lists:reverse(Clauses), + + FirstMeta = + if + is_atom(Condition), Condition /= false, Condition /= nil -> ?generated(Meta); + true -> Meta + end, + + Error = {{'.', Meta, [erlang, error]}, Meta, [cond_clause]}, + {Case, SC} = build_cond_clauses([H | T], Error, FirstMeta, S), + translate(replace_case_meta(CondMeta, Case), Ann, SC); + +%% Case + +translate({'case', Meta, [Expr, Opts]}, _Ann, S) -> + translate_case(Meta, Expr, Opts, S); + +%% Try + +translate({'try', Meta, [Opts]}, _Ann, S) -> + Ann = ?ann(Meta), + Do = proplists:get_value('do', Opts, nil), + {TDo, SB} = translate(Do, Ann, S), + + Catch = [Tuple || {X, _} = Tuple <- Opts, X == 'rescue' orelse X == 'catch'], + {TCatch, SC} = elixir_erl_try:clauses(Ann, Catch, SB), + + {TAfter, SA} = case lists:keyfind('after', 1, Opts) of + {'after', After} -> + {TBlock, SAExtracted} = translate(After, Ann, SC), + {unblock(TBlock), SAExtracted}; + false -> + {[], SC} + end, + + Else = elixir_erl_clauses:get_clauses('else', Opts, match), + {TElse, SE} = elixir_erl_clauses:clauses(Else, SA), + {{'try', ?ann(Meta), unblock(TDo), TElse, TCatch, TAfter}, SE}; + +%% Receive + +translate({'receive', Meta, [Opts]}, _Ann, S) -> + Do = elixir_erl_clauses:get_clauses(do, Opts, match), + + case lists:keyfind('after', 1, Opts) of + false -> + {TClauses, SC} = elixir_erl_clauses:clauses(Do, S), + {{'receive', ?ann(Meta), TClauses}, SC}; + _ -> + After = elixir_erl_clauses:get_clauses('after', Opts, expr), + {TClauses, SC} = elixir_erl_clauses:clauses(Do ++ After, S), + {FClauses, TAfter} = elixir_utils:split_last(TClauses), + {_, _, [FExpr], _, FAfter} = TAfter, + {{'receive', ?ann(Meta), FClauses, FExpr, FAfter}, SC} + end; + +%% Comprehensions and with + +translate({for, Meta, [_ | _] = Args}, _Ann, S) -> + elixir_erl_for:translate(Meta, Args, S); + +translate({with, Meta, [_ | _] = Args}, _Ann, S) -> + Ann = ?ann(Meta), + {Exprs, [{do, Do} | Opts]} = elixir_utils:split_last(Args), + {ElseClause, MaybeFun, SE} = translate_with_else(Meta, Opts, S), + {Case, SD} = translate_with_do(Exprs, Ann, Do, ElseClause, SE), + + case MaybeFun of + nil -> {Case, SD}; + FunAssign -> {{block, Ann, [FunAssign, Case]}, SD} + end; + +%% Variables + +translate({'^', _, [{Name, VarMeta, Kind}]}, _Ann, S) when is_atom(Name), is_atom(Kind) -> + {Var, VS} = elixir_erl_var:translate(VarMeta, Name, Kind, S), + + case S#elixir_erl.extra of + pin_guard -> + {PinVarName, PS} = elixir_erl_var:build('_', VS), + Ann = ?ann(?generated(VarMeta)), + PinVar = {var, Ann, PinVarName}, + Guard = {op, Ann, '=:=', Var, PinVar}, + {PinVar, PS#elixir_erl{extra_guards=[Guard | PS#elixir_erl.extra_guards]}}; + _ -> + {Var, VS} + end; + +translate({Name, Meta, Kind}, _Ann, S) when is_atom(Name), is_atom(Kind) -> + elixir_erl_var:translate(Meta, Name, Kind, S); + +%% Local calls + +translate({Name, Meta, Args}, _Ann, S) when is_atom(Name), is_list(Meta), is_list(Args) -> + Ann = ?ann(Meta), + {TArgs, NS} = translate_args(Args, Ann, S), + {{call, Ann, {atom, Ann, Name}, TArgs}, NS}; + +%% Remote calls + +translate({{'.', _, [Left, Right]}, Meta, []}, _Ann, #elixir_erl{context=guard} = S) + when is_tuple(Left), is_atom(Right), is_list(Meta) -> + Ann = ?ann(Meta), + {TLeft, SL} = translate(Left, Ann, S), + TRight = {atom, Ann, Right}, + {?remote(Ann, erlang, map_get, [TRight, TLeft]), SL}; + +translate({{'.', _, [Left, Right]}, Meta, []}, _Ann, S) + when is_tuple(Left) orelse Left =:= nil orelse is_boolean(Left), is_atom(Right), is_list(Meta) -> + Ann = ?ann(Meta), + {TLeft, SL} = translate(Left, Ann, S), + TRight = {atom, Ann, Right}, + + Generated = erl_anno:set_generated(true, Ann), + {InnerVar, SI} = elixir_erl_var:build('_', SL), + TInnerVar = {var, Generated, InnerVar}, + {Var, SV} = elixir_erl_var:build('_', SI), + TVar = {var, Generated, Var}, + + case proplists:get_value(no_parens, Meta, false) of + true -> + {{'case', Generated, TLeft, [ + {clause, Generated, + [{map, Ann, [{map_field_exact, Ann, TRight, TVar}]}], + [], + [TVar]}, + {clause, Generated, + [TVar], + [], + [{'case', Generated, ?remote(Generated, elixir_erl_pass, no_parens_remote, [TVar, TRight]), [ + {clause, Generated, + [{tuple, Generated, [{atom, Generated, ok}, TInnerVar]}], [], [TInnerVar]}, + {clause, Generated, + [{tuple, Generated, [{atom, Generated, error}, TInnerVar]}], [], [?remote(Ann, erlang, error, [TInnerVar])]} + ]}]} + ]}, SV}; + false -> + {{'case', Generated, TLeft, [ + {clause, Generated, + [{map, Ann, [{map_field_exact, Ann, TRight, TVar}]}], + [], + [?remote(Generated, elixir_erl_pass, parens_map_field, [TRight, TVar])]}, + {clause, Generated, + [TVar], + [], + [{call, Generated, {remote, Generated, TVar, TRight}, []}]} + ]}, SV} + end; + +translate({{'.', _, [Left, Right]}, Meta, Args}, _Ann, S) + when (is_tuple(Left) orelse is_atom(Left)), is_atom(Right), is_list(Meta), is_list(Args) -> + translate_remote(Left, Right, Meta, Args, S); + +%% Anonymous function calls + +translate({{'.', _, [Expr]}, Meta, Args}, _Ann, S) when is_list(Args) -> + Ann = ?ann(Meta), + {TExpr, SE} = translate(Expr, Ann, S), + {TArgs, SA} = translate_args(Args, Ann, SE), + {{call, Ann, TExpr, TArgs}, SA}; + +%% Literals + +translate({Left, Right}, Ann, S) -> + {TLeft, SL} = translate(Left, Ann, S), + {TRight, SR} = translate(Right, Ann, SL), + {{tuple, Ann, [TLeft, TRight]}, SR}; + +translate(List, Ann, S) when is_list(List) -> + translate_list(List, Ann, [], S); + +translate(Other, Ann, S) -> + {elixir_erl:elixir_to_erl(Other, Ann), S}. + +%% Helpers + +translate_case(Meta, Expr, Opts, S) -> + Ann = ?ann(Meta), + {TExpr, SE} = translate(Expr, Ann, S), + Clauses = elixir_erl_clauses:get_clauses(do, Opts, match), + RClauses = + %% For constructs that optimize booleans, we mark them as generated + %% to avoid reports from the Erlang compiler but specially Dialyzer. + case lists:member({optimize_boolean, true}, Meta) of + true -> [{N, ?generated(M), H, B} || {N, M, H, B} <- Clauses]; + false -> Clauses + end, + {TClauses, SC} = elixir_erl_clauses:clauses(RClauses, SE), + {{'case', Ann, TExpr, TClauses}, SC}. + +translate_list([{'|', _, [Left, Right]}], Ann, List, Acc) -> + {TLeft, LAcc} = translate(Left, Ann, Acc), + {TRight, TAcc} = translate(Right, Ann, LAcc), + {build_list([TLeft | List], TRight, Ann), TAcc}; +translate_list([H | T], Ann, List, Acc) -> + {TH, TAcc} = translate(H, Ann, Acc), + translate_list(T, Ann, [TH | List], TAcc); +translate_list([], Ann, List, Acc) -> + {build_list(List, {nil, Ann}, Ann), Acc}. + +build_list([H | T], Acc, Ann) -> + build_list(T, {cons, Ann, H, Acc}, Ann); +build_list([], Acc, _Ann) -> + Acc. + +%% Pack a list of expressions from a block. +unblock({'block', _, Exprs}) -> Exprs; +unblock(Expr) -> [Expr]. + +translate_fn_match(Arg, Ann, S) -> + {TArg, TS} = translate_args(Arg, Ann, S#elixir_erl{extra=pin_guard}), + {TArg, TS#elixir_erl{extra=S#elixir_erl.extra}}. + +%% Translate args + +translate_args(Args, Ann, S) -> + lists:mapfoldl(fun + (Arg, SA) when is_list(Arg) -> + translate_list(Arg, Ann, [], SA); + (Arg, SA) when is_tuple(Arg) -> + translate(Arg, Ann, SA); + (Arg, SA) -> + {elixir_erl:elixir_to_erl(Arg, Ann), SA} + end, S, Args). + +%% Translate blocks + +translate_block([], _Ann, Acc, S) -> + {Acc, S}; +translate_block([H], Ann, Acc, S) -> + {TH, TS} = translate(H, Ann, S), + translate_block([], Ann, [TH | Acc], TS); +translate_block([{'__block__', Meta, Args} | T], Ann, Acc, S) when is_list(Args) -> + {TAcc, SA} = translate_block(Args, ?ann(Meta), Acc, S), + translate_block(T, Ann, TAcc, SA); +translate_block([H | T], Ann, Acc, S) -> + {TH, TS} = translate(H, Ann, S), + translate_block(T, Ann, [TH | Acc], TS). + +%% Cond + +build_cond_clauses([{'->', NewMeta, [[Condition], Body]} | T], Acc, OldMeta, S) -> + {NewCondition, Truthy, Other, ST} = build_truthy_clause(NewMeta, Condition, Body, S), + Falsy = {'->', OldMeta, [[Other], Acc]}, + Case = {'case', NewMeta, [NewCondition, [{do, [Truthy, Falsy]}]]}, + build_cond_clauses(T, Case, NewMeta, ST); +build_cond_clauses([], Acc, _, S) -> + {Acc, S}. + +replace_case_meta(Meta, {'case', _, Args}) -> + {'case', Meta, Args}; +replace_case_meta(_Meta, Other) -> + Other. + +build_truthy_clause(Meta, Condition, Body, S) -> + case returns_boolean(Condition, Body) of + {NewCondition, NewBody} -> + {NewCondition, {'->', Meta, [[true], NewBody]}, false, S}; + false -> + {Var, _, SV} = elixir_erl_var:assign(Meta, S), + Head = {'when', [], [Var, + {{'.', [], [erlang, 'andalso']}, [], [ + {{'.', [], [erlang, '/=']}, [], [Var, nil]}, + {{'.', [], [erlang, '/=']}, [], [Var, false]} + ]} + ]}, + {Condition, {'->', Meta, [[Head], Body]}, {'_', [], nil}, SV} + end. + +%% In case a variable is defined to match in a condition +%% but a condition returns boolean, we can replace the +%% variable directly by the boolean result. +returns_boolean({'=', _, [{Var, _, Ctx}, Condition]}, {Var, _, Ctx}) when is_atom(Var), is_atom(Ctx) -> + case elixir_utils:returns_boolean(Condition) of + true -> {Condition, true}; + false -> false + end; + +%% For all other cases, we check the condition but +%% return both condition and body untouched. +returns_boolean(Condition, Body) -> + case elixir_utils:returns_boolean(Condition) of + true -> {Condition, Body}; + false -> false + end. + +%% with + +translate_with_else(Meta, [], S) -> + Ann = ?ann(Meta), + {VarName, SC} = elixir_erl_var:build('_', S), + Var = {var, Ann, VarName}, + Generated = erl_anno:set_generated(true, Ann), + {{clause, Generated, [Var], [], [Var]}, nil, SC}; +translate_with_else(Meta, [{'else', [{'->', _, [[{Var, VarMeta, Kind}], Clause]}]}], S) when is_atom(Var), is_atom(Kind) -> + Ann = ?ann(Meta), + {ElseVarErl, SV} = elixir_erl_var:translate(VarMeta, Var, Kind, S#elixir_erl{context=match}), + {TranslatedClause, SC} = translate(Clause, Ann, SV#elixir_erl{context=nil}), + Clauses = [{clause, Ann, [ElseVarErl], [], [TranslatedClause]}], + with_else_closure(Meta, Clauses, SC); +translate_with_else(Meta, [{'else', Else}], S) -> + Generated = ?generated(Meta), + {RaiseVar, _, SV} = elixir_erl_var:assign(Generated, S), + + RaiseExpr = {{'.', Generated, [erlang, error]}, Generated, [{else_clause, RaiseVar}]}, + RaiseClause = {'->', Generated, [[RaiseVar], RaiseExpr]}, + + Clauses = elixir_erl_clauses:get_clauses('else', [{'else', Else ++ [RaiseClause]}], match), + {TranslatedClauses, SC} = elixir_erl_clauses:clauses(Clauses, SV#elixir_erl{extra=pin_guard}), + with_else_closure(Generated, TranslatedClauses, SC#elixir_erl{extra=SV#elixir_erl.extra}). + +with_else_closure(Meta, TranslatedClauses, S) -> + Ann = ?ann(Meta), + {_, FunErlVar, SC} = elixir_erl_var:assign(Meta, S), + {_, ArgErlVar, SA} = elixir_erl_var:assign(Meta, SC), + Generated = erl_anno:set_generated(true, Ann), + FunAssign = {match, Ann, FunErlVar, {'fun', Generated, {clauses, TranslatedClauses}}}, + FunCall = {call, Ann, FunErlVar, [ArgErlVar]}, + {{clause, Generated, [ArgErlVar], [], [FunCall]}, FunAssign, SA}. + +translate_with_do([{'<-', Meta, [{Var, _, Ctx} = Left, Expr]} | Rest], Ann, Do, Else, S) when is_atom(Var), is_atom(Ctx) -> + translate_with_do([{'=', Meta, [Left, Expr]} | Rest], Ann, Do, Else, S); +translate_with_do([{'<-', Meta, [Left, Expr]} | Rest], _Ann, Do, Else, S) -> + Ann = ?ann(Meta), + {Args, Guards} = elixir_utils:extract_guards(Left), + {TExpr, SR} = translate(Expr, Ann, S), + {TArgs, SA} = elixir_erl_clauses:match(Ann, fun translate/3, Args, SR), + TGuards = elixir_erl_clauses:guards(Ann, Guards, SA#elixir_erl.extra_guards, SA), + {TBody, SB} = translate_with_do(Rest, Ann, Do, Else, SA#elixir_erl{extra_guards=[]}), + Clause = {clause, Ann, [TArgs], TGuards, unblock(TBody)}, + {{'case', Ann, TExpr, [Clause, Else]}, SB}; +translate_with_do([Expr | Rest], Ann, Do, Else, S) -> + {TExpr, TS} = translate(Expr, Ann, S), + {TRest, RS} = translate_with_do(Rest, Ann, Do, Else, TS), + {{block, Ann, [TExpr | unblock(TRest)]}, RS}; +translate_with_do([], Ann, Do, _Else, S) -> + translate(Do, Ann, S). + +%% Maps and structs + +translate_struct_var_name(Ann, Name, Args, S0) -> + {{map, MapAnn, TArgs0}, S1} = translate_struct(Ann, Name, Args, S0), + {TArgs1, S2} = generate_struct_name_guard(TArgs0, [], S1), + {{map, MapAnn, TArgs1}, S2}. + +translate_struct(Ann, _Name, {'%{}', _, [{'|', Meta, [Update, Assocs]}]}, S) -> + {TUpdate, SU} = translate(Update, Ann, S), + translate_map(?ann(Meta), Assocs, {ok, TUpdate}, SU); +translate_struct(Ann, Name, {'%{}', _, Assocs}, S) -> + translate_map(Ann, [{'__struct__', Name}] ++ Assocs, none, S). + +translate_map(Ann, [{'|', Meta, [Update, Assocs]}], S) -> + {TUpdate, SU} = translate(Update, Ann, S), + translate_map(?ann(Meta), Assocs, {ok, TUpdate}, SU); +translate_map(Ann, Assocs, S) -> + translate_map(Ann, Assocs, none, S). + +translate_map(Ann, Assocs, TUpdate, #elixir_erl{extra=Extra} = S) -> + Op = translate_key_val_op(TUpdate, S), + + {TArgs, SA} = lists:mapfoldl(fun({Key, Value}, Acc0) -> + {TKey, Acc1} = translate(Key, Ann, Acc0#elixir_erl{extra=map_key}), + {TValue, Acc2} = translate(Value, Ann, Acc1#elixir_erl{extra=Extra}), + {{Op, Ann, TKey, TValue}, Acc2} + end, S, Assocs), + + build_map(Ann, TUpdate, TArgs, SA). + +translate_key_val_op(_TUpdate, #elixir_erl{extra=map_key}) -> map_field_assoc; +translate_key_val_op(_TUpdate, #elixir_erl{context=match}) -> map_field_exact; +translate_key_val_op(none, _) -> map_field_assoc; +translate_key_val_op(_, _) -> map_field_exact. + +build_map(Ann, {ok, TUpdate}, TArgs, SA) -> {{map, Ann, TUpdate, TArgs}, SA}; +build_map(Ann, none, TArgs, SA) -> {{map, Ann, TArgs}, SA}. + +%% Translate bitstrings + +translate_bitstring(Meta, Args, S) -> + build_bitstr(Args, ?ann(Meta), S, []). + +build_bitstr([{'::', Meta, [H, V]} | T], Ann, S, Acc) -> + {Size, Types} = extract_bit_info(V, Meta, S#elixir_erl{context=nil, extra=nil}), + build_bitstr(T, Ann, S, Acc, H, Size, Types); +build_bitstr([], Ann, S, Acc) -> + {{bin, Ann, lists:reverse(Acc)}, S}. + +build_bitstr(T, Ann, S, Acc, H, default, Types) when is_binary(H) -> + Element = + case requires_utf_conversion(Types) of + false -> + %% See explanation in elixir_erl:elixir_to_erl/1 to + %% know why we can simply convert the binary to a list. + {bin_element, Ann, {string, 0, binary_to_list(H)}, default, default}; + true -> + %% UTF types require conversion. + {bin_element, Ann, {string, 0, elixir_utils:characters_to_list(H)}, default, Types} + end, + build_bitstr(T, Ann, S, [Element | Acc]); + +build_bitstr(T, Ann, S, Acc, H, Size, Types) -> + {Expr, NS} = translate(H, Ann, S), + build_bitstr(T, Ann, NS, [{bin_element, Ann, Expr, Size, Types} | Acc]). + +requires_utf_conversion([bitstring | _]) -> false; +requires_utf_conversion([binary | _]) -> false; +requires_utf_conversion(_) -> true. + +extract_bit_info({'-', _, [L, {size, _, [Size]}]}, Meta, S) -> + {extract_bit_size(Size, Meta, S), extract_bit_type(L, [])}; +extract_bit_info({size, _, [Size]}, Meta, S) -> + {extract_bit_size(Size, Meta, S), []}; +extract_bit_info(L, _Meta, _S) -> + {default, extract_bit_type(L, [])}. + +extract_bit_size(Size, Meta, S) -> + {TSize, _} = translate(Size, ?ann(Meta), S#elixir_erl{context=guard}), + TSize. + +extract_bit_type({'-', _, [L, R]}, Acc) -> + extract_bit_type(L, extract_bit_type(R, Acc)); +extract_bit_type({unit, _, [Arg]}, Acc) -> + [{unit, Arg} | Acc]; +extract_bit_type({Other, _, nil}, Acc) -> + [Other | Acc]; +%% TODO: Remove me on Elixir v2.0. +%% Elixir v1.14 and earlier emitted an empty list +%% and may still be processed via debug_info. +extract_bit_type({Other, _, []}, Acc) -> + [Other | Acc]. + +%% Optimizations that are specific to Erlang and change +%% the format of the AST. + +translate_remote('Elixir.String.Chars', to_string, Meta, [Arg], S) -> + Ann = ?ann(Meta), + {TArg, TS} = translate(Arg, Ann, S), + {VarName, VS} = elixir_erl_var:build('_', TS), + + Generated = erl_anno:set_generated(true, Ann), + Var = {var, Generated, VarName}, + Guard = ?remote(Generated, erlang, is_binary, [Var]), + Slow = ?remote(Generated, 'Elixir.String.Chars', to_string, [Var]), + Fast = Var, + + {{'case', Generated, TArg, [ + {clause, Generated, [Var], [[Guard]], [Fast]}, + {clause, Generated, [Var], [], [Slow]} + ]}, VS}; +translate_remote(maps, put, Meta, [Key, Value, Map], S) -> + Ann = ?ann(Meta), + + case translate_args([Key, Value, Map], Ann, S) of + {[TKey, TValue, {map, _, InnerMap, Pairs}], TS} -> + {{map, Ann, InnerMap, Pairs ++ [{map_field_assoc, Ann, TKey, TValue}]}, TS}; + + {[TKey, TValue, {map, _, Pairs}], TS} -> + {{map, Ann, Pairs ++ [{map_field_assoc, Ann, TKey, TValue}]}, TS}; + + {[TKey, TValue, TMap], TS} -> + {{map, Ann, TMap, [{map_field_assoc, Ann, TKey, TValue}]}, TS} + end; +translate_remote(maps, merge, Meta, [Map1, Map2], S) -> + Ann = ?ann(Meta), + + case translate_args([Map1, Map2], Ann, S) of + {[{map, _, Pairs1}, {map, _, Pairs2}], TS} -> + {{map, Ann, Pairs1 ++ Pairs2}, TS}; + + {[{map, _, InnerMap1, Pairs1}, {map, _, Pairs2}], TS} -> + {{map, Ann, InnerMap1, Pairs1 ++ Pairs2}, TS}; + + {[TMap1, {map, _, Pairs2}], TS} -> + {{map, Ann, TMap1, Pairs2}, TS}; + + {[TMap1, TMap2], TS} -> + {{call, Ann, {remote, Ann, {atom, Ann, maps}, {atom, Ann, merge}}, [TMap1, TMap2]}, TS} + end; +translate_remote(Left, Right, Meta, Args, S) -> + Ann = ?ann(Meta), + + case rewrite_strategy(Left, Right, Args) of + guard_op -> + {TArgs, SA} = translate_args(Args, Ann, S), + %% Rewrite Erlang function calls as operators so they + %% work in guards, matches and so on. + case TArgs of + [TOne] -> {{op, Ann, Right, TOne}, SA}; + [TOne, TTwo] -> {{op, Ann, Right, TOne, TTwo}, SA} + end; + {inline_pure, Result} -> + Generated = erl_anno:set_generated(true, Ann), + translate(Result, Generated, S); + {inline_args, NewArgs} -> + {TLeft, SL} = translate(Left, Ann, S), + {TArgs, SA} = translate_args(NewArgs, Ann, SL), + TRight = {atom, Ann, Right}, + {{call, Ann, {remote, Ann, TLeft, TRight}, TArgs}, SA}; + none -> + {TLeft, SL} = translate(Left, Ann, S), + {TArgs, SA} = translate_args(Args, Ann, SL), + TRight = {atom, Ann, Right}, + {{call, Ann, {remote, Ann, TLeft, TRight}, TArgs}, SA} + end. + +rewrite_strategy(erlang, Right, Args) -> + Arity = length(Args), + case elixir_utils:guard_op(Right, Arity) of + true -> guard_op; + false -> none + end; +rewrite_strategy(Left, shift, [Struct, Opts | RestArgs]) when + Left == 'Elixir.Date'; + Left == 'Elixir.DateTime'; + Left == 'Elixir.NaiveDateTime'; + Left == 'Elixir.Time' +-> + case basic_type_arg(Opts) of + true -> + try + {inline_args, [Struct, Left:'__duration__!'(Opts) | RestArgs]} + catch _:_ -> + % fail silently, will fail at runtime + none + end; + false -> + none + end; +rewrite_strategy(Left, Right, Args) -> + case inline_pure_function(Left, Right) andalso basic_type_arg(Args) of + true -> + try + {inline_pure, apply(Left, Right, Args)} + catch _:_ -> + % fail silently, will fail at runtime + none + end; + false -> + none + end. + +inline_pure_function('Elixir.Duration', 'new!') -> true; +inline_pure_function('Elixir.MapSet', new) -> true; +inline_pure_function('Elixir.String', length) -> true; +inline_pure_function('Elixir.String', graphemes) -> true; +inline_pure_function('Elixir.String', codepoints) -> true; +inline_pure_function('Elixir.String', split) -> true; +inline_pure_function('Elixir.Kernel', to_timeout) -> true; +inline_pure_function('Elixir.URI', new) -> true; +inline_pure_function('Elixir.URI', 'new!') -> true; +inline_pure_function('Elixir.URI', parse) -> true; +inline_pure_function('Elixir.URI', encode_query) -> true; +inline_pure_function('Elixir.URI', encode_www_form) -> true; +inline_pure_function('Elixir.URI', decode) -> true; +inline_pure_function('Elixir.URI', decode_www_form) -> true; +inline_pure_function('Elixir.Version', parse) -> true; +inline_pure_function('Elixir.Version', 'parse!') -> true; +inline_pure_function('Elixir.Version', parse_requirement) -> true; +inline_pure_function('Elixir.Version', 'parse_requirement!') -> true; +inline_pure_function(_Left, _Right) -> false. + +% we do not want to try and inline calls which might depend on protocols that might be overridden later +basic_type_arg(Term) when is_number(Term); is_atom(Term); is_binary(Term) -> true; +basic_type_arg(List) when is_list(List) -> lists:all(fun basic_type_arg/1, List); +basic_type_arg({Left, Right}) -> basic_type_arg(Left) and basic_type_arg(Right); +basic_type_arg(_) -> false. + +generate_struct_name_guard([{map_field_exact, Ann, {atom, _, '__struct__'} = Key, Var} | Rest], Acc, S0) -> + {ModuleVarName, S1} = elixir_erl_var:build('_', S0), + Generated = erl_anno:set_generated(true, Ann), + ModuleVar = {var, Generated, ModuleVarName}, + Match = {match, Generated, ModuleVar, Var}, + Guard = ?remote(Generated, erlang, is_atom, [ModuleVar]), + S2 = S1#elixir_erl{extra_guards=[Guard | S1#elixir_erl.extra_guards]}, + {lists:reverse(Acc, [{map_field_exact, Ann, Key, Match} | Rest]), S2}; +generate_struct_name_guard([Field | Rest], Acc, S) -> + generate_struct_name_guard(Rest, [Field | Acc], S). + +%% TODO: Make this a runtime error on Elixir v2.0 +no_parens_remote(nil, _Key) -> {error, {badmap, nil}}; +no_parens_remote(false, _Key) -> {error, {badmap, false}}; +no_parens_remote(true, _Key) -> {error, {badmap, true}}; +no_parens_remote(Atom, Fun) when is_atom(Atom) -> + Message = fun() -> + io_lib:format( + "using map.field notation (without parentheses) to invoke function ~ts.~ts() is deprecated, " + "you must add parentheses instead: remote.function()", + [elixir_aliases:inspect(Atom), Fun] + ) + end, + 'Elixir.IO':warn_once(?MODULE, Message, 3), + {ok, apply(Atom, Fun, [])}; +no_parens_remote(#{} = Map, Key) -> + {error, {badkey, Key, Map}}; +no_parens_remote(Other, _Key) -> + {error, {badmap, Other}}. + +parens_map_field(Key, Value) -> + Message = fun() -> + io_lib:format( + "using module.function() notation (with parentheses) to fetch map field ~ts is deprecated, " + "you must remove the parentheses: map.field", + [elixir_aliases:inspect(Key)] + ) + end, + 'Elixir.IO':warn_once(?MODULE, Message, 3), + Value. diff --git a/lib/elixir/src/elixir_erl_try.erl b/lib/elixir/src/elixir_erl_try.erl new file mode 100644 index 00000000000..36780a10409 --- /dev/null +++ b/lib/elixir/src/elixir_erl_try.erl @@ -0,0 +1,243 @@ +%% SPDX-License-Identifier: Apache-2.0 +%% SPDX-FileCopyrightText: 2021 The Elixir Team +%% SPDX-FileCopyrightText: 2012 Plataformatec + +-module(elixir_erl_try). +-export([clauses/3]). +-include("elixir.hrl"). +-define(REQUIRES_STACKTRACE, + ['Elixir.FunctionClauseError', 'Elixir.UndefinedFunctionError', + 'Elixir.KeyError', 'Elixir.ArgumentError', 'Elixir.SystemLimitError']). + +clauses(_Ann, Args, S) -> + Catch = elixir_erl_clauses:get_clauses('catch', Args, 'catch'), + Rescue = elixir_erl_clauses:get_clauses(rescue, Args, rescue), + {StackName, SV} = elixir_erl_var:build('__STACKTRACE__', S), + OldStack = SV#elixir_erl.stacktrace, + SS = SV#elixir_erl{stacktrace=StackName}, + reduce_clauses(Rescue ++ Catch, [], OldStack, SS, SS). + +reduce_clauses([H | T], Acc, OldStack, SAcc, S) -> + {TH, TS} = each_clause(H, SAcc), + reduce_clauses(T, [TH | Acc], OldStack, TS, S); +reduce_clauses([], Acc, OldStack, SAcc, _S) -> + {lists:reverse(Acc), SAcc#elixir_erl{stacktrace=OldStack}}. + +each_clause({'catch', Meta, Raw, Expr}, S) -> + {Args, Guards} = elixir_utils:extract_splat_guards(Raw), + + Match = + %% TODO: Remove me on Elixir v2.0. + %% Elixir v1.17 and earlier emitted single argument + %% and may still be processed via debug_info. + case Args of + [X] -> [throw, X]; + [X, Y] -> [X, Y] + end, + + {{clause, Line, [TKind, TMatches], TGuards, TBody}, TS} = + elixir_erl_clauses:clause(?ann(Meta), fun elixir_erl_pass:translate_args/3, Match, Expr, Guards, S), + + build_clause(Line, TKind, TMatches, TGuards, TBody, TS); + +each_clause({rescue, Meta, [{in, _, [Left, Right]}], Expr}, S) -> + {TempVar, _, CS} = elixir_erl_var:assign(Meta, S), + {Guards, ErlangAliases} = rescue_guards(Meta, TempVar, Right), + Body = normalize_rescue(Meta, TempVar, Left, Expr, ErlangAliases), + build_rescue(Meta, TempVar, Guards, Body, CS); + +each_clause({rescue, Meta, [{VarName, _, Context} = Left], Expr}, S) when is_atom(VarName), is_atom(Context) -> + {TempVar, _, CS} = elixir_erl_var:assign(Meta, S), + Body = normalize_rescue(Meta, TempVar, Left, Expr, ['Elixir.ErlangError']), + build_rescue(Meta, TempVar, [], Body, CS). + +normalize_rescue(_Meta, _Var, {'_', _, Atom}, Expr, _) when is_atom(Atom) -> + Expr; +normalize_rescue(Meta, Var, Pattern, Expr, []) -> + prepend_to_block(Meta, {'=', Meta, [Pattern, Var]}, Expr); +normalize_rescue(Meta, Var, Pattern, Expr, ErlangAliases) -> + Stacktrace = + case lists:member('Elixir.ErlangError', ErlangAliases) of + true -> + dynamic_normalize(Meta, Var, ?REQUIRES_STACKTRACE); + + false -> + case lists:partition(fun is_normalized_with_stacktrace/1, ErlangAliases) of + {[], _} -> []; + {_, []} -> {'__STACKTRACE__', Meta, nil}; + {Some, _} -> dynamic_normalize(Meta, Var, Some) + end + end, + + Normalized = {{'.', Meta, ['Elixir.Exception', normalize]}, Meta, [error, Var, Stacktrace]}, + prepend_to_block(Meta, {'=', Meta, [Pattern, Normalized]}, Expr). + +dynamic_normalize(Meta, Var, [H | T]) -> + Generated = ?generated(Meta), + + Guards = + lists:foldl(fun(Alias, Acc) -> + {'when', Generated, [erl_rescue_stacktrace_for(Generated, Var, Alias), Acc]} + end, erl_rescue_stacktrace_for(Generated, Var, H), T), + + {'case', Generated, [ + Var, + [{do, [ + {'->', Generated, [[{'when', Generated, [{'_', Generated, nil}, Guards]}], {'__STACKTRACE__', Generated, nil}]}, + {'->', Generated, [[{'_', Generated, nil}], []]} + ]}] + ]}. + +erl_rescue_stacktrace_for(_Meta, _Var, 'Elixir.ErlangError') -> + %% ErlangError is a "meta" exception, we should never expand it here. + error(badarg); +erl_rescue_stacktrace_for(Meta, Var, 'Elixir.KeyError') -> + %% Only the two-element tuple requires stacktrace. + erl_and(Meta, erl_tuple_size(Meta, Var, 2), erl_record_compare(Meta, Var, badkey)); +erl_rescue_stacktrace_for(Meta, Var, Module) -> + erl_rescue_guard_for(Meta, Var, Module). + +is_normalized_with_stacktrace(Module) -> + lists:member(Module, ?REQUIRES_STACKTRACE). + +%% Helpers + +build_rescue(Meta, Var, Guards, Body, S) -> + {{clause, Line, [TMatch], TGuards, TBody}, TS} = + elixir_erl_clauses:clause(?ann(Meta), fun elixir_erl_pass:translate_args/3, [Var], Body, Guards, S), + + build_clause(Line, {atom, Line, error}, TMatch, TGuards, TBody, TS). + +%% Convert rescue clauses ("var in [alias1, alias2]") into guards. +rescue_guards(_Meta, _Var, []) -> + {[], []}; +rescue_guards(Meta, Var, Aliases) -> + {ErlangGuards, ErlangAliases} = erl_rescue(Meta, Var, Aliases, [], []), + + ElixirGuards = + [erl_and(Meta, + {erl(Meta, '=='), Meta, [{erl(Meta, map_get), Meta, ['__struct__', Var]}, Alias]}, + {erl(Meta, map_get), Meta, ['__exception__', Var]} + ) || Alias <- Aliases], + + {ElixirGuards ++ ErlangGuards, ErlangAliases}. + +build_clause(Line, Kind, Expr, Guards, Body, #elixir_erl{stacktrace=Var} = TS) -> + Match = {tuple, Line, [Kind, Expr, {var, Line, Var}]}, + {{clause, Line, [Match], Guards, Body}, TS}. + +%% Rescue each atom name considering their Erlang or Elixir matches. +%% Matching of variables is done with Erlang exceptions is done in +%% function for optimization. + +erl_rescue(Meta, Var, [H | T], Guards, Aliases) when is_atom(H) -> + case erl_rescue_guard_for(Meta, Var, H) of + false -> erl_rescue(Meta, Var, T, Guards, Aliases); + Expr -> erl_rescue(Meta, Var, T, [Expr | Guards], [H | Aliases]) + end; +erl_rescue(_, _, [], Guards, Aliases) -> + {Guards, Aliases}. + +%% Handle Erlang rescue matches. + +erl_rescue_guard_for(Meta, Var, 'Elixir.UndefinedFunctionError') -> + {erl(Meta, '=='), Meta, [Var, undef]}; + +erl_rescue_guard_for(Meta, Var, 'Elixir.FunctionClauseError') -> + {erl(Meta, '=='), Meta, [Var, function_clause]}; + +erl_rescue_guard_for(Meta, Var, 'Elixir.SystemLimitError') -> + {erl(Meta, '=='), Meta, [Var, system_limit]}; + +erl_rescue_guard_for(Meta, Var, 'Elixir.ArithmeticError') -> + {erl(Meta, '=='), Meta, [Var, badarith]}; + +erl_rescue_guard_for(Meta, Var, 'Elixir.CondClauseError') -> + {erl(Meta, '=='), Meta, [Var, cond_clause]}; + +erl_rescue_guard_for(Meta, Var, 'Elixir.BadArityError') -> + erl_and(Meta, + erl_tuple_size(Meta, Var, 2), + erl_record_compare(Meta, Var, badarity)); + +erl_rescue_guard_for(Meta, Var, 'Elixir.BadFunctionError') -> + erl_and(Meta, + erl_tuple_size(Meta, Var, 2), + erl_record_compare(Meta, Var, badfun)); + +erl_rescue_guard_for(Meta, Var, 'Elixir.MatchError') -> + erl_and(Meta, + erl_tuple_size(Meta, Var, 2), + erl_record_compare(Meta, Var, badmatch)); + +erl_rescue_guard_for(Meta, Var, 'Elixir.CaseClauseError') -> + erl_and(Meta, + erl_tuple_size(Meta, Var, 2), + erl_record_compare(Meta, Var, case_clause)); + +erl_rescue_guard_for(Meta, Var, 'Elixir.WithClauseError') -> + erl_and(Meta, + erl_tuple_size(Meta, Var, 2), + erl_record_compare(Meta, Var, else_clause)); + +erl_rescue_guard_for(Meta, Var, 'Elixir.TryClauseError') -> + erl_and(Meta, + erl_tuple_size(Meta, Var, 2), + erl_record_compare(Meta, Var, try_clause)); + +erl_rescue_guard_for(Meta, Var, 'Elixir.BadMapError') -> + erl_and(Meta, + erl_tuple_size(Meta, Var, 2), + erl_record_compare(Meta, Var, badmap)); + +erl_rescue_guard_for(Meta, Var, 'Elixir.BadBooleanError') -> + erl_and(Meta, + erl_tuple_size(Meta, Var, 3), + erl_record_compare(Meta, Var, badbool)); + +erl_rescue_guard_for(Meta, Var, 'Elixir.KeyError') -> + erl_and(Meta, + erl_or(Meta, + erl_tuple_size(Meta, Var, 2), + erl_tuple_size(Meta, Var, 3)), + erl_record_compare(Meta, Var, badkey)); + +erl_rescue_guard_for(Meta, Var, 'Elixir.ArgumentError') -> + erl_or(Meta, + {erl(Meta, '=='), Meta, [Var, badarg]}, + erl_and(Meta, + erl_tuple_size(Meta, Var, 2), + erl_record_compare(Meta, Var, badarg))); + +erl_rescue_guard_for(Meta, Var, 'Elixir.ErlangError') -> + Condition = + erl_and( + Meta, + {erl(Meta, is_map), Meta, [Var]}, + {erl(Meta, is_map_key), Meta, ['__exception__', Var]} + ), + {erl(Meta, 'not'), Meta, [Condition]}; + +erl_rescue_guard_for(_, _, _) -> + false. + +%% Helpers + +erl_tuple_size(Meta, Var, Size) -> + {erl(Meta, '=='), Meta, [{erl(Meta, tuple_size), Meta, [Var]}, Size]}. + +erl_record_compare(Meta, Var, Expr) -> + {erl(Meta, '=='), Meta, [ + {erl(Meta, element), Meta, [1, Var]}, + Expr + ]}. + +prepend_to_block(_Meta, Expr, {'__block__', Meta, Args}) -> + {'__block__', Meta, [Expr | Args]}; + +prepend_to_block(Meta, Expr, Args) -> + {'__block__', Meta, [Expr, Args]}. + +erl(Meta, Op) -> {'.', Meta, [erlang, Op]}. +erl_or(Meta, Left, Right) -> {erl(Meta, 'orelse'), Meta, [Left, Right]}. +erl_and(Meta, Left, Right) -> {erl(Meta, 'andalso'), Meta, [Left, Right]}. diff --git a/lib/elixir/src/elixir_erl_var.erl b/lib/elixir/src/elixir_erl_var.erl new file mode 100644 index 00000000000..c3f731a0479 --- /dev/null +++ b/lib/elixir/src/elixir_erl_var.erl @@ -0,0 +1,127 @@ +%% SPDX-License-Identifier: Apache-2.0 +%% SPDX-FileCopyrightText: 2021 The Elixir Team +%% SPDX-FileCopyrightText: 2012 Plataformatec + +%% Convenience functions used to manipulate scope and its variables. +-module(elixir_erl_var). +-export([ + translate/4, assign/2, build/2, + load_binding/2, dump_binding/4, + from_env/1, from_env/2 +]). +-include("elixir.hrl"). + +%% VAR HANDLING + +translate(Meta, '_', _Kind, S) -> + {{var, ?ann(Meta), '_'}, S}; + +translate(Meta, Name, Kind, #elixir_erl{var_names=VarNames} = S) -> + {version, Version} = lists:keyfind(version, 1, Meta), + + case VarNames of + #{Version := ErlName} -> {{var, ?ann(Meta), ErlName}, S}; + #{} when Kind /= nil -> assign(Meta, '_', Version, S); + #{} -> assign(Meta, Name, Version, S) + end. + +assign(Meta, #elixir_erl{var_names=VarNames} = S) -> + Version = -(map_size(VarNames)+1), + ExVar = {var, [{version, Version} | Meta], ?var_context}, + {ErlVar, SV} = assign(Meta, '_', Version, S), + {ExVar, ErlVar, SV}. + +assign(Meta, Name, Version, #elixir_erl{var_names=VarNames} = S) -> + {NewVar, NS} = build(Name, S), + NewVarNames = VarNames#{Version => NewVar}, + {{var, ?ann(Meta), NewVar}, NS#elixir_erl{var_names=NewVarNames}}. + +build(Key, #elixir_erl{counter=Counter} = S) -> + Count = + case Counter of + #{Key := Val} -> Val + 1; + _ -> 1 + end, + {build_name(Key, Count), + S#elixir_erl{counter=Counter#{Key => Count}}}. + +build_name('_', Count) -> list_to_atom("_@" ++ integer_to_list(Count)); +build_name(Name, Count) -> list_to_atom("_" ++ atom_to_list(Name) ++ "@" ++ integer_to_list(Count)). + +%% BINDINGS + +from_env(#{versioned_vars := Read} = Env) -> + VarsList = to_erl_vars(maps:values(Read), 0), + {VarsList, from_env(Env, maps:from_list(VarsList))}. + +from_env(#{context := Context}, VarsMap) -> + #elixir_erl{ + context=Context, + var_names=VarsMap, + counter=#{'_' => map_size(VarsMap)} + }. + +to_erl_vars([Version | Versions], Counter) -> + [{Version, to_erl_var(Counter)} | to_erl_vars(Versions, Counter + 1)]; +to_erl_vars([], _Counter) -> + []. + +to_erl_var(Counter) -> + list_to_atom("_@" ++ integer_to_list(Counter)). + +load_binding(Binding, Prune) -> + load_binding(Binding, #{}, [], [], 0, Prune). + +load_binding([Binding | NextBindings], ExVars, ErlVars, Normalized, Counter, Prune) -> + {Pair, Value} = load_pair(Binding), + + case ExVars of + #{Pair := VarCounter} -> + ErlVar = to_erl_var(VarCounter), + load_binding(NextBindings, ExVars, ErlVars, [{ErlVar, Value} | Normalized], Counter, Prune); + + #{} -> + ErlVar = to_erl_var(Counter), + + load_binding( + NextBindings, + ExVars#{Pair => Counter}, + [{Counter, ErlVar} | ErlVars], + [{ErlVar, Value} | Normalized], + Counter + 1, + Prune + ) + end; +load_binding([], ExVars, ErlVars, Normalized, Counter, true) -> + load_binding([{{elixir, prune_binding}, true}], ExVars, ErlVars, Normalized, Counter, false); +load_binding([], ExVars, ErlVars, Normalized, _Counter, false) -> + {ExVars, maps:from_list(ErlVars), maps:from_list(lists:reverse(Normalized))}. + +load_pair({Key, Value}) when is_atom(Key) -> {{Key, nil}, Value}; +load_pair({Pair, Value}) -> {Pair, Value}. + +dump_binding(Binding, ErlS, ExS, PruneBefore) -> + #elixir_erl{var_names=ErlVars} = ErlS, + #elixir_ex{vars={ExVars, _}, unused={Unused, _}} = ExS, + + maps:fold(fun + ({Var, Kind} = Pair, Version, {B, V}) + when is_atom(Kind), + %% If the variable is part of the pruning (usually the input binding) + %% and is unused, we removed it from vars. + Version > PruneBefore orelse is_map_key({Pair, Version}, Unused) -> + Key = case Kind of + nil -> Var; + _ -> Pair + end, + + ErlName = maps:get(Version, ErlVars), + Value = maps:get(ErlName, Binding, nil), + {[{Key, Value} | B], V}; + + (Pair, _, {B, V}) when PruneBefore >= 0 -> + {B, maps:remove(Pair, V)}; + + (_, _, Acc) -> + Acc + end, {[], ExVars}, ExVars). diff --git a/lib/elixir/src/elixir_errors.erl b/lib/elixir/src/elixir_errors.erl index 84bbb6c7c3b..53b0c557c7e 100644 --- a/lib/elixir/src/elixir_errors.erl +++ b/lib/elixir/src/elixir_errors.erl @@ -1,217 +1,530 @@ -% A bunch of helpers to help to deal with errors in Elixir source code. -% This is not exposed in the Elixir language. +%% SPDX-License-Identifier: Apache-2.0 +%% SPDX-FileCopyrightText: 2021 The Elixir Team +%% SPDX-FileCopyrightText: 2012 Plataformatec + +%% A bunch of helpers to help to deal with errors in Elixir source code. +%% This is not exposed in the Elixir language. +%% +%% Note that this is also called by the Erlang backend, so we also support +%% the line number to be none (as it may happen in some erlang errors). -module(elixir_errors). --export([compile_error/3, compile_error/4, - form_error/4, parse_error/4, warn/2, warn/3, - handle_file_warning/2, handle_file_warning/3, handle_file_error/2]). +-export([compile_error/1, compile_error/3, parse_error/5]). +-export([function_error/4, module_error/4, file_error/4]). +-export([format_snippet/6]). +-export([erl_warn/3, file_warn/4]). +-export([prefix/1]). +-export([print_diagnostics/1, print_diagnostic/2, emit_diagnostic/6]). +-export([print_warning/3]). -include("elixir.hrl"). +-type location() :: non_neg_integer() | {non_neg_integer(), non_neg_integer()}. + +%% Diagnostic API + +%% TODO: Remove me on Elixir v2.0. +%% Called by deprecated Kernel.ParallelCompiler.print_warning. +print_warning(Position, File, Message) -> + Output = format_snippet(warning, Position, File, Message, nil, #{}), + io:put_chars(standard_error, [Output, $\n, $\n]). + +read_snippet(nil, _Position) -> + nil; +read_snippet(<<"nofile">>, _Position) -> + nil; +read_snippet(File, Position) -> + LineNumber = extract_line(Position), + get_file_line(File, LineNumber). + +get_file_line(File, LineNumber) when is_integer(LineNumber), LineNumber > 0 -> + case file:open(File, [read, binary]) of + {ok, IoDevice} -> + Line = traverse_file_line(IoDevice, LineNumber), + ok = file:close(IoDevice), + Line; + {error, _} -> + nil + end; +get_file_line(_, _) -> nil. --type line_or_meta() :: integer() | list(). - -warn(Warning) -> - CompilerPid = get(elixir_compiler_pid), - if - CompilerPid =/= undefined -> - elixir_code_server:cast({register_warning, CompilerPid}); - true -> false +traverse_file_line(IoDevice, 1) -> + case file:read_line(IoDevice) of + {ok, Line} -> binary:replace(Line, <<"\n">>, <<>>); + _ -> nil + end; +traverse_file_line(IoDevice, N) -> + file:read_line(IoDevice), + traverse_file_line(IoDevice, N - 1). + +%% Used by Module.ParallelChecker. +print_diagnostics([Diagnostic | Others]) -> + #{file := File, position := Position, message := Message} = Diagnostic, + Snippet = read_snippet(File, Position), + Formatted = format_snippet(warning, Position, File, Message, Snippet, Diagnostic), + LineNumber = extract_line(Position), + LineDigits = get_line_number_digits(LineNumber, 1), + Padding = case Snippet of + nil -> 0; + _ -> max(4, LineDigits + 2) end, - io:put_chars(standard_error, Warning). + Locations = [["\n", n_spaces(Padding), "└─ ", 'Elixir.Exception':format_stacktrace_entry(ES)] || #{stacktrace := [ES]} <- Others], + io:put_chars(standard_error, [Formatted, Locations, $\n, $\n]). + +print_diagnostic(#{severity := S, message := M, position := P, file := F} = Diagnostic, ReadSnippet) -> + Snippet = + case ReadSnippet of + true -> read_snippet(F, P); + false -> nil + end, -warn(Caller, Warning) -> - warn([Caller, "warning: ", Warning]). + Output = format_snippet(S, P, F, M, Snippet, Diagnostic), -warn(Line, File, Warning) when is_integer(Line) -> - warn(file_format(Line, File, "warning: " ++ Warning)). + MaybeStack = + case (F /= nil) orelse elixir_config:is_bootstrap() of + true -> []; + false -> [["\n ", 'Elixir.Exception':format_stacktrace_entry(E)] || E <- ?key(Diagnostic, stacktrace)] + end, -%% Raised during expansion/translation/compilation. + io:put_chars(standard_error, [Output, MaybeStack, $\n, $\n]), + Diagnostic. --spec form_error(line_or_meta(), binary(), module(), any()) -> no_return(). +emit_diagnostic(Severity, Position, File, Message, Stacktrace, Options) -> + ReadSnippet = proplists:get_value(read_snippet, Options, false), -form_error(Meta, File, Module, Desc) -> - compile_error(Meta, File, format_error(Module, Desc)). + Span = case lists:keyfind(span, 1, Options) of + {span, {EndLine, EndCol}} -> {EndLine, EndCol}; + _ -> nil + end, --spec compile_error(line_or_meta(), binary(), iolist()) -> no_return(). --spec compile_error(line_or_meta(), binary(), iolist(), list()) -> no_return(). + Diagnostic = #{ + severity => Severity, + source => case get(elixir_compiler_file) of + undefined -> File; + CompilerFile -> CompilerFile + end, + file => File, + position => Position, + message => unicode:characters_to_binary(Message), + stacktrace => Stacktrace, + span => Span + }, + + case get(elixir_code_diagnostics) of + undefined -> + case get(elixir_compiler_info) of + undefined -> print_diagnostic(Diagnostic, ReadSnippet); + {CompilerPid, _} -> CompilerPid ! {diagnostic, Diagnostic, ReadSnippet} + end; + + {Tail, true} -> + put(elixir_code_diagnostics, {[print_diagnostic(Diagnostic, ReadSnippet) | Tail], true}); + + {Tail, false} -> + put(elixir_code_diagnostics, {[Diagnostic | Tail], false}) + end, -compile_error(Meta, File, Message) when is_list(Message) -> - raise(Meta, File, 'Elixir.CompileError', elixir_utils:characters_to_binary(Message)). + ok. + +extract_line({L, _}) -> L; +extract_line(L) -> L. + +extract_column({_, C}) -> C; +extract_column(_) -> nil. + +%% Format snippets +%% "Snippet" here refers to the source code line where the diagnostic/error occurred + +format_snippet(Severity, _Position, nil, Message, nil, _Diagnostic) -> + Formatted = [prefix(Severity), " ", Message], + unicode:characters_to_binary(Formatted); + +format_snippet(Severity, Position, File, Message, nil, Diagnostic) -> + Location = location_format(Position, File, maps:get(stacktrace, Diagnostic, [])), + + Formatted = io_lib:format( + "~ts ~ts\n" + "└─ ~ts", + [prefix(Severity), Message, Location] + ), + + unicode:characters_to_binary(Formatted); + +format_snippet(Severity, Position, File, Message, Snippet, Diagnostic) -> + Column = extract_column(Position), + LineNumber = extract_line(Position), + LineDigits = get_line_number_digits(LineNumber, 1), + Spacing = n_spaces(max(2, LineDigits) + 1), + LineNumberSpacing = if LineDigits =:= 1 -> 1; true -> 0 end, + {FormattedLine, ColumnsTrimmed} = format_line(Snippet), + Location = location_format(Position, File, maps:get(stacktrace, Diagnostic, [])), + MessageDetail = format_detail(Diagnostic, Message), + + Highlight = + case Column of + nil -> + highlight_below_line(FormattedLine, Severity); + _ -> + Length = calculate_span_length({LineNumber, Column}, Diagnostic), + highlight_at_position(Column - ColumnsTrimmed, Severity, Length) + end, -compile_error(Meta, File, Format, Args) when is_list(Format) -> - compile_error(Meta, File, io_lib:format(Format, Args)). + Formatted = io_lib:format( + " ~ts~ts ~ts\n" + " ~ts│\n" + " ~ts~p │ ~ts\n" + " ~ts│ ~ts\n" + " ~ts│\n" + " ~ts└─ ~ts", + [ + Spacing, prefix(Severity), format_message(MessageDetail, LineDigits, 2 + LineNumberSpacing), + Spacing, + n_spaces(LineNumberSpacing), LineNumber, FormattedLine, + Spacing, Highlight, + Spacing, + Spacing, Location + ]), + + unicode:characters_to_binary(Formatted). + +format_detail(#{details := #{typing_traces := _}}, Message) -> [Message | "\ntyping violation found at:"]; +format_detail(_, Message) -> Message. + +calculate_span_length({StartLine, StartCol}, #{span := {StartLine, EndCol}}) -> EndCol - StartCol; +calculate_span_length({StartLine, _}, #{span := {EndLine, _}}) when EndLine > StartLine -> 1; +calculate_span_length({_, _}, #{}) -> 1. + +format_line(Line) -> + case trim_line(Line, 0) of + {Trimmed, SpacesMatched} when SpacesMatched >= 27 -> + ColumnsTrimmed = SpacesMatched - 22, + {["...", n_spaces(19), Trimmed], ColumnsTrimmed}; + + {_, _} -> + {Line, 0} + end. -%% Raised on tokenizing/parsing +trim_line(<<$\s, Rest/binary>>, Count) -> trim_line(Rest, Count + 1); +trim_line(<<$\t, Rest/binary>>, Count) -> trim_line(Rest, Count + 8); +trim_line(Rest, Count) -> {Rest, Count}. --spec parse_error(line_or_meta(), binary(), binary(), binary()) -> no_return(). +format_message(Message, NDigits, PaddingSize) -> + Padding = list_to_binary([$\n, n_spaces(NDigits + PaddingSize)]), + Bin = unicode:characters_to_binary(Message), + pad_line(binary:split(Bin, <<"\n">>, [global]), Padding). -parse_error(Meta, File, Error, <<>>) -> - Message = case Error of - <<"syntax error before: ">> -> <<"syntax error: expression is incomplete">>; - _ -> Error - end, - raise(Meta, File, 'Elixir.TokenMissingError', Message); - -%% Show a nicer message for missing end tokens -parse_error(Meta, File, <<"syntax error before: ">>, <<"'end'">>) -> - raise(Meta, File, 'Elixir.SyntaxError', <<"unexpected token: end">>); - -%% Binaries are wrapped in [<<...>>], so we need to unwrap them -parse_error(Meta, File, Error, <<"[", _/binary>> = Full) when is_binary(Error) -> - Rest = - case binary:split(Full, <<"<<">>) of - [Lead, Token] -> - case binary:split(Token, <<">>">>) of - [Part, _] when Lead == <<$[>> -> Part; - _ -> <<$">> - end; - [_] -> - <<$">> - end, - raise(Meta, File, 'Elixir.SyntaxError', <>); +pad_line([Last], _Padding) -> [Last]; +pad_line([First, <<"">> | Rest], Padding) -> [First, "\n" | pad_line([<<"">> | Rest], Padding)]; +pad_line([First | Rest], Padding) -> [First, Padding | pad_line(Rest, Padding)]. -%% Everything else is fine as is -parse_error(Meta, File, Error, Token) when is_binary(Error), is_binary(Token) -> - Message = <>, - raise(Meta, File, 'Elixir.SyntaxError', Message). +highlight_at_position(Column, Severity, Length) -> + Spacing = n_spaces(max(Column - 1, 0)), + case Severity of + warning -> highlight([Spacing, lists:duplicate(Length, $~)], warning); + error -> highlight([Spacing, lists:duplicate(Length, $^)], error) + end. -%% Handle warnings and errors (called during module compilation) +highlight_below_line(Line, Severity) -> + % Don't highlight leading whitespaces in line + {Rest, SpacesMatched} = trim_line(Line, 0), -%% Ignore on bootstrap -handle_file_warning(true, _File, {_Line, sys_core_fold, nomatch_guard}) -> []; -handle_file_warning(true, _File, {_Line, sys_core_fold, {nomatch_shadow, _}}) -> []; + Length = string:length(Rest), + Highlight = case Severity of + warning -> highlight(lists:duplicate(Length, $~), warning); + error -> highlight(lists:duplicate(Length, $^), error) + end, -%% Ignore always -handle_file_warning(_, _File, {_Line, sys_core_fold, useless_building}) -> []; + [n_spaces(SpacesMatched), Highlight]. -%% This is an Erlang bug, it considers {tuple, _}.call to always fail -handle_file_warning(_, _File, {_Line, v3_kernel, bad_call}) -> []; +get_line_number_digits(Number, Acc) when Number < 10 -> Acc; +get_line_number_digits(Number, Acc) -> + get_line_number_digits(Number div 10, Acc + 1). -%% We handle unused local warnings ourselves -handle_file_warning(_, _File, {_Line, erl_lint, {unused_function, _}}) -> []; +n_spaces(N) -> lists:duplicate(N, " "). -%% Make no_effect clauses pretty -handle_file_warning(_, File, {Line, sys_core_fold, {no_effect, {erlang, F, A}}}) -> - {Fmt, Args} = case erl_internal:comp_op(F, A) of - true -> {"use of operator ~ts has no effect", [translate_comp_op(F)]}; - false -> - case erl_internal:bif(F, A) of - false -> {"the call to :erlang.~ts/~B has no effect", [F,A]}; - true -> {"the call to ~ts/~B has no effect", [F,A]} - end - end, - Message = io_lib:format(Fmt, Args), - warn(Line, File, Message); - -%% Rewrite undefined behaviour to check for protocols -handle_file_warning(_, File, {Line,erl_lint,{undefined_behaviour_func,{Fun,Arity},Module}}) -> - {DefKind, Def, DefArity} = - case atom_to_list(Fun) of - "MACRO-" ++ Rest -> {macro, list_to_atom(Rest), Arity - 1}; - _ -> {function, Fun, Arity} - end, +%% Compilation error/warn handling. - Kind = protocol_or_behaviour(Module), - Raw = "undefined ~ts ~ts ~ts/~B (for ~ts ~ts)", - Message = io_lib:format(Raw, [Kind, DefKind, Def, DefArity, Kind, elixir_aliases:inspect(Module)]), - warn(Line, File, Message); +%% Low-level warning, should be used only from Erlang passes. +-spec erl_warn(location() | none, unicode:chardata(), unicode:chardata()) -> ok. +erl_warn(none, File, Warning) -> + erl_warn(0, File, Warning); +erl_warn(Location, File, Warning) when is_binary(File) -> + emit_diagnostic(warning, Location, File, Warning, [], [{read_snippet, true}]). -handle_file_warning(_, File, {Line,erl_lint,{undefined_behaviour,Module}}) -> - case elixir_compiler:get_opt(internal) of - true -> []; +-spec file_warn(list(), binary() | #{file := binary(), _ => _}, module(), any()) -> ok. +file_warn(Meta, File, Module, Desc) when is_list(Meta), is_binary(File) -> + file_warn(Meta, #{file => File}, Module, Desc); +file_warn(Meta, E, Module, Desc) when is_list(Meta) -> + % Skip warnings during bootstrap, they will be reported during recompilation + case elixir_config:is_bootstrap() of + true -> ok; false -> - Message = io_lib:format("behaviour ~ts undefined", [elixir_aliases:inspect(Module)]), - warn(Line, File, Message) - end; + {EnvPosition, EnvFile, EnvStacktrace} = env_format(Meta, E), + Message = Module:format_error(Desc), + emit_diagnostic(warning, EnvPosition, EnvFile, Message, EnvStacktrace, [{read_snippet, true} | Meta]) + end. -%% Ignore unused vars at "weird" lines (<= 0) -handle_file_warning(_, _File, {Line,erl_lint,{unused_var,_Var}}) when Line =< 0 -> - []; +-spec file_error(list(), binary() | #{file := binary(), _ => _}, module(), any()) -> no_return(). +file_error(Meta, File, Module, Desc) when is_list(Meta), is_binary(File) -> + file_error(Meta, #{file => File}, Module, Desc); +file_error(Meta, Env, Module, Desc) when is_list(Meta) -> + print_error(Meta, Env, Module, Desc), + compile_error(Env). + +%% A module error is one where it can continue if there is a module +%% being compiled. If there is no module, it is a regular file_error. +-spec module_error(list(), #{file := binary(), module => module() | nil, _ => _}, module(), any()) -> ok. +module_error(Meta, #{module := EnvModule} = Env, Module, Desc) when EnvModule /= nil -> + print_error(Meta, Env, Module, Desc), + case elixir_module:taint(EnvModule) of + true -> ok; + false -> compile_error(Env) + end; +module_error(Meta, Env, Module, Desc) -> + file_error(Meta, Env, Module, Desc). + +%% A function error is one where it can continue if there is a function +%% being compiled. If there is no function, it is falls back to file_error. +-spec function_error(list(), #{file := binary(), function => {term(), term()} | nil, _ => _}, module(), any()) -> ok. +function_error(Meta, #{function := {_, _}} = Env, Module, Desc) -> + module_error(Meta, Env, Module, Desc); +function_error(Meta, Env, Module, Desc) -> + file_error(Meta, Env, Module, Desc). + +print_error(Meta, Env, Module, Desc) -> + {EnvPosition, EnvFile, EnvStacktrace} = env_format(Meta, Env), + Message = Module:format_error(Desc), + emit_diagnostic(error, EnvPosition, EnvFile, Message, EnvStacktrace, [{read_snippet, true} | Meta]), + ok. + +%% Compilation error. + +-spec compile_error(#{file := binary(), _ => _}) -> no_return(). +%% We check for the lexical tracker because pry() inside a module +%% will have the environment but not a tracker. +compile_error(#{module := Module, file := File, lexical_tracker := LT}) when Module /= nil, LT /= nil -> + Inspected = elixir_aliases:inspect(Module), + Message = io_lib:format("cannot compile module ~ts (errors have been logged)", [Inspected]), + compile_error([], File, Message); +compile_error(#{file := File}) -> + compile_error([], File, "cannot compile file (errors have been logged)"). + +-spec compile_error(list(), binary(), binary() | unicode:charlist()) -> no_return(). +compile_error(Meta, File, Message) when is_binary(Message) -> + {File, Position} = meta_location(Meta, File), + raise('Elixir.CompileError', Message, [{file, File} | Position]); +compile_error(Meta, File, Message) when is_list(Message) -> + {File, Position} = meta_location(Meta, File), + raise('Elixir.CompileError', elixir_utils:characters_to_binary(Message), [{file, File} | Position]). -%% Ignore shadowed vars as we guarantee no conflicts ourselves -handle_file_warning(_, _File, {_Line,erl_lint,{shadowed_var,_Var,_Where}}) -> - []; +%% Tokenization parsing/errors. -%% Properly format other unused vars -handle_file_warning(_, File, {Line,erl_lint,{unused_var,Var}}) -> - Message = format_error(erl_lint, {unused_var, format_var(Var)}), - warn(Line, File, Message); +-spec parse_error(elixir:keyword(), binary() | {binary(), binary()}, + binary(), binary(), {unicode:charlist(), integer(), integer()}) -> no_return(). +parse_error(Location, File, Error, <<>>, Input) -> + Message = case Error of + <<"syntax error before: ">> -> <<"syntax error: expression is incomplete">>; + _ -> <> + end, -%% Default behaviour -handle_file_warning(_, File, {Line,Module,Desc}) -> - Message = format_error(Module, Desc), - warn(Line, File, Message). + raise_snippet(Location, File, Input, 'Elixir.TokenMissingError', Message); + +%% Show a nicer message for end of line +parse_error(Location, File, <<"syntax error before: ">>, <<"eol">>, Input) -> + raise_snippet(Location, File, Input, 'Elixir.SyntaxError', + <<"unexpectedly reached end of line. The current expression is invalid or incomplete">>); + +%% Show a nicer message for keywords pt1 (Erlang keywords show up wrapped in single quotes) +parse_error(Location, File, <<"syntax error before: ">>, Keyword, Input) + when Keyword == <<"'not'">>; + Keyword == <<"'and'">>; + Keyword == <<"'or'">>; + Keyword == <<"'when'">>; + Keyword == <<"'after'">>; + Keyword == <<"'catch'">>; + Keyword == <<"'end'">> -> + raise_reserved(Location, File, Input, binary_part(Keyword, 1, byte_size(Keyword) - 2)); + +%% Show a nicer message for keywords pt2 (Elixir keywords show up as is) +parse_error(Location, File, <<"syntax error before: ">>, Keyword, Input) + when Keyword == <<"fn">>; + Keyword == <<"else">>; + Keyword == <<"rescue">>; + Keyword == <<"true">>; + Keyword == <<"false">>; + Keyword == <<"nil">>; + Keyword == <<"in">> -> + raise_reserved(Location, File, Input, Keyword); + +%% Produce a human-readable message for errors before a sigil +parse_error(Location, File, <<"syntax error before: ">>, <<"{sigil,", _Rest/binary>> = Full, Input) -> + {ok, {sigil, _, Atom, [Content | _], _, _, _}} = parse_erl_term(Full), + Content2 = case is_binary(Content) of + true -> Content; + false -> <<>> + end, -handle_file_warning(File, Desc) -> - handle_file_warning(false, File, Desc). + % :static_atoms_encoder might encode :sigil_ atoms as arbitrary terms + MaybeSigil = case is_atom(Atom) of + true -> case atom_to_binary(Atom) of + <<"sigil_", Chars/binary>> -> <<"\~", Chars/binary, " ">>; + _ -> <<>> + end; + false -> <<>> + end, --spec handle_file_error(file:filename_all(), {non_neg_integer(), module(), any()}) -> no_return(). + Message = <<"syntax error before: sigil ", MaybeSigil/binary, "starting with content '", Content2/binary, "'">>, + raise_snippet(Location, File, Input, 'Elixir.SyntaxError', Message); -handle_file_error(File, {Line,erl_lint,{unsafe_var,Var,{In,_Where}}}) -> - Translated = case In of - 'orelse' -> 'or'; - 'andalso' -> 'and'; - _ -> In +%% Binaries (and interpolation) are wrapped in [<<...>>] +parse_error(Location, File, Error, <<"[", _/binary>> = Full, Input) when is_binary(Error) -> + Term = case parse_erl_term(Full) of + {ok, [H | _]} when is_binary(H) -> <<$", H/binary, $">>; + _ -> <<$">> end, - Message = io_lib:format("cannot define variable ~ts inside ~ts", [format_var(Var), Translated]), - raise(Line, File, 'Elixir.CompileError', iolist_to_binary(Message)); + raise_snippet(Location, File, Input, 'Elixir.SyntaxError', <>); + +%% Given a string prefix and suffix to insert the token inside the error message rather than append it +parse_error(Location, File, {ErrorPrefix, ErrorSuffix}, Token, Input) when is_binary(ErrorPrefix), is_binary(ErrorSuffix), is_binary(Token) -> + Message = <>, + raise_snippet(Location, File, Input, 'Elixir.SyntaxError', Message); + +%% Misplaced char tokens (for example, {char, _, 97}) are translated by Erlang into +%% the char literal (i.e., the token in the previous example becomes $a), +%% because {char, _, _} is a valid Erlang token for an Erlang char literal. We +%% want to represent that token as ?a in the error, according to the Elixir +%% syntax. +parse_error(Location, File, <<"syntax error before: ">>, <<$$, Char/binary>>, Input) -> + Message = <<"syntax error before: ?", Char/binary>>, + raise_snippet(Location, File, Input, 'Elixir.SyntaxError', Message); -handle_file_error(File, {Line,erl_lint,{spec_fun_undefined,{M,F,A}}}) -> - Message = io_lib:format("spec for undefined function ~ts.~ts/~B", [elixir_aliases:inspect(M), F, A]), - raise(Line, File, 'Elixir.CompileError', iolist_to_binary(Message)); +%% Everything else is fine as is +parse_error(Location, File, Error, Token, Input) when is_binary(Error), is_binary(Token) -> + Message = <>, + case lists:keytake(error_type, 1, Location) of + {value, {error_type, mismatched_delimiter}, Loc} -> + raise_snippet(Loc, File, Input, 'Elixir.MismatchedDelimiterError', Message); + _ -> + raise_snippet(Location, File, Input, 'Elixir.SyntaxError', Message) + end. -handle_file_error(File, {Line,Module,Desc}) -> - form_error(Line, File, Module, Desc). +parse_erl_term(Term) -> + case erl_scan:string(binary_to_list(Term)) of + {ok, Tokens, _} -> + case erl_parse:parse_term(Tokens ++ [{dot, 1}]) of + {ok, Parsed} -> {ok, Parsed}; + _ -> error + end; + _ -> error + end. -%% Helpers +raise_reserved(Location, File, Input, Keyword) -> + raise_snippet(Location, File, Input, 'Elixir.SyntaxError', + <<"syntax error before: ", Keyword/binary, ". \"", Keyword/binary, "\" is a " + "reserved word in Elixir and therefore its usage is limited. For instance, " + "it can't be used as a variable or be defined nor invoked as a regular function">>). + +raise_snippet(Location, File, Input, Kind, Message) when is_binary(File) -> + Snippet = cut_snippet(Location, Input), + raise(Kind, Message, [{file, File}, {snippet, Snippet} | Location]). + +cut_snippet(Location, Input) -> + case lists:keyfind(column, 1, Location) of + {column, _} -> + {line, Line} = lists:keyfind(line, 1, Location), + + case lists:keyfind(end_line, 1, Location) of + {end_line, EndLine} -> + cut_snippet(Input, Line, EndLine - Line + 1); + + false -> + Snippet = cut_snippet(Input, Line, 1), + case string:trim(Snippet, leading) of + <<>> -> nil; + _ -> Snippet + end + end; -raise(Meta, File, Kind, Message) when is_list(Meta) -> - raise(?line(Meta), File, Kind, Message); + false -> + nil + end. -raise(none, File, Kind, Message) -> - raise(0, File, Kind, Message); +cut_snippet({InputString, StartLine, StartColumn, Indentation}, Line, Span) -> + %% In case the code is indented, we need to add the indentation back + %% for the snippets to match the reported columns. + Prelude = lists:duplicate(max(StartColumn - Indentation - 1, 0), " "), + Lines = string:split(Prelude ++ InputString, "\n", all), + Indent = binary:copy(<<" ">>, Indentation), + [Head | Tail] = lists:nthtail(Line - StartLine, Lines), + IndentedTail = indent_n(Tail, Span - 1, <<"\n", Indent/binary>>), + elixir_utils:characters_to_binary([Indent, Head, IndentedTail]). -raise(Line, File, Kind, Message) when is_integer(Line), is_binary(File) -> - %% Populate the stacktrace so we can raise it - try - throw(ok) - catch - ok -> ok - end, - Stacktrace = erlang:get_stacktrace(), - Exception = Kind:exception([{description, Message}, {file, File}, {line, Line}]), - erlang:raise(error, Exception, tl(Stacktrace)). +indent_n([], _Count, _Indent) -> []; +indent_n(_Lines, 0, _Indent) -> []; +indent_n([H | T], Count, Indent) -> [Indent, H | indent_n(T, Count - 1, Indent)]. -file_format(0, File, Message) when is_binary(File) -> - io_lib:format("~ts: ~ts~n", [elixir_utils:relative_to_cwd(File), Message]); +%% Helpers -file_format(Line, File, Message) when is_binary(File) -> - io_lib:format("~ts:~w: ~ts~n", [elixir_utils:relative_to_cwd(File), Line, Message]). +prefix(warning) -> highlight(<<"warning:">>, warning); +prefix(error) -> highlight(<<"error:">>, error); +prefix(hint) -> <<"hint:">>. -format_var(Var) -> - list_to_atom(lists:takewhile(fun(X) -> X /= $@ end, atom_to_list(Var))). +highlight(Message, Severity) -> + case {Severity, application:get_env(elixir, ansi_enabled, false)} of + {warning, true} -> yellow(Message); + {error, true} -> red(Message); + _ -> Message + end. -format_error([], Desc) -> - io_lib:format("~p", [Desc]); +yellow(Msg) -> ["\e[33m", Msg, "\e[0m"]. +red(Msg) -> ["\e[31m", Msg, "\e[0m"]. + +env_format(Meta, #{file := EnvFile} = E) -> + {File, Position} = meta_location(Meta, EnvFile), + Line = ?line(Position), + + Stacktrace = + case E of + #{function := {Name, Arity}, module := Module} -> + [{Module, Name, Arity, [{file, elixir_utils:relative_to_cwd(File)} | Position ]}]; + #{module := Module} when Module /= nil -> + [{Module, '__MODULE__', 0, [{file, elixir_utils:relative_to_cwd(File)} | Position]}]; + #{} -> + [] + end, -format_error(Module, Desc) -> - Module:format_error(Desc). + case lists:keyfind(column, 1, Position) of + {column, Column} -> {{Line, Column}, File, Stacktrace}; + _ -> {Line, File, Stacktrace} + end. -protocol_or_behaviour(Module) -> - case is_protocol(Module) of - true -> protocol; - false -> behaviour +%% We prefer the stacktrace, if available, as it also contains module/function. +location_format(_Position, _File, [E | _]) -> + 'Elixir.Exception':format_stacktrace_entry(E); +location_format(Position, File, []) -> + file_format(Position, File). + +file_format({0, _Column}, File) -> + elixir_utils:relative_to_cwd(File); +file_format({Line, nil}, File) -> + file_format(Line, File); +file_format({Line, Column}, File) -> + io_lib:format("~ts:~w:~w", [elixir_utils:relative_to_cwd(File), Line, Column]); +file_format(0, File) -> + elixir_utils:relative_to_cwd(File); +file_format(Line, File) -> + io_lib:format("~ts:~w", [elixir_utils:relative_to_cwd(File), Line]). + +meta_location(Meta, File) -> + case elixir_utils:meta_keep(Meta) of + {F, L} -> {F, [{line, L}]}; + nil -> {File, maybe_add_col([{line, ?line(Meta)}], Meta)} end. -is_protocol(Module) -> - case code:ensure_loaded(Module) of - {module, _} -> - erlang:function_exported(Module, '__protocol__', 1) andalso - Module:'__protocol__'(name) == Module; - {error, _} -> - false +maybe_add_col(Position, Meta) -> + case lists:keyfind(column, 1, Meta) of + {column, Col} when is_integer(Col) -> [{column, Col} | Position]; + false -> Position end. -translate_comp_op('/=') -> '!='; -translate_comp_op('=<') -> '<='; -translate_comp_op('=:=') -> '==='; -translate_comp_op('=/=') -> '!=='; -translate_comp_op(Other) -> Other. +raise(Kind, Message, Opts) when is_binary(Message) -> + Stacktrace = try throw(ok) catch _:_:Stack -> Stack end, + Exception = Kind:exception([{description, Message} | Opts]), + erlang:raise(error, Exception, tl(Stacktrace)). diff --git a/lib/elixir/src/elixir_exp.erl b/lib/elixir/src/elixir_exp.erl deleted file mode 100644 index ba786b4b885..00000000000 --- a/lib/elixir/src/elixir_exp.erl +++ /dev/null @@ -1,571 +0,0 @@ --module(elixir_exp). --export([expand/2, expand_args/2, expand_arg/2]). --import(elixir_errors, [compile_error/3, compile_error/4]). --include("elixir.hrl"). - -%% = - -expand({'=', Meta, [Left, Right]}, E) -> - assert_no_guard_scope(Meta, '=', E), - {ERight, ER} = expand(Right, E), - {ELeft, EL} = elixir_exp_clauses:match(fun expand/2, Left, E), - {{'=', Meta, [ELeft, ERight]}, elixir_env:mergev(EL, ER)}; - -%% Literal operators - -expand({'{}', Meta, Args}, E) -> - {EArgs, EA} = expand_args(Args, E), - {{'{}', Meta, EArgs}, EA}; - -expand({'%{}', Meta, Args}, E) -> - elixir_map:expand_map(Meta, Args, E); - -expand({'%', Meta, [Left, Right]}, E) -> - elixir_map:expand_struct(Meta, Left, Right, E); - -expand({'<<>>', Meta, Args}, E) -> - elixir_bitstring:expand(Meta, Args, E); - -%% Other operators - -expand({'__op__', Meta, [_, _] = Args}, E) -> - {EArgs, EA} = expand_args(Args, E), - {{'__op__', Meta, EArgs}, EA}; - -expand({'__op__', Meta, [_, _, _] = Args}, E) -> - {EArgs, EA} = expand_args(Args, E), - {{'__op__', Meta, EArgs}, EA}; - -expand({'->', Meta, _Args}, E) -> - compile_error(Meta, ?m(E, file), "unhandled operator ->"); - -%% __block__ - -expand({'__block__', _Meta, []}, E) -> - {nil, E}; -expand({'__block__', _Meta, [Arg]}, E) -> - expand(Arg, E); -expand({'__block__', Meta, Args}, E) when is_list(Args) -> - {EArgs, EA} = expand_many(Args, E), - {{'__block__', Meta, EArgs}, EA}; - -%% __aliases__ - -expand({'__aliases__', _, _} = Alias, E) -> - case elixir_aliases:expand(Alias, ?m(E, aliases), - ?m(E, macro_aliases), ?m(E, lexical_tracker)) of - Receiver when is_atom(Receiver) -> - elixir_lexical:record_remote(Receiver, ?m(E, lexical_tracker)), - {Receiver, E}; - Aliases -> - {EAliases, EA} = expand_args(Aliases, E), - - case lists:all(fun is_atom/1, EAliases) of - true -> - Receiver = elixir_aliases:concat(EAliases), - elixir_lexical:record_remote(Receiver, ?m(E, lexical_tracker)), - {Receiver, EA}; - false -> - {{{'.', [], [elixir_aliases, concat]}, [], [EAliases]}, EA} - end - end; - -%% alias - -expand({alias, Meta, [Ref]}, E) -> - expand({alias, Meta, [Ref,[]]}, E); -expand({alias, Meta, [Ref, KV]}, E) -> - assert_no_match_or_guard_scope(Meta, alias, E), - {ERef, ER} = expand(Ref, E), - {EKV, ET} = expand_opts(Meta, alias, [as, warn], no_alias_opts(KV), ER), - - if - is_atom(ERef) -> - {{alias, Meta, [ERef, EKV]}, - expand_alias(Meta, true, ERef, EKV, ET)}; - true -> - compile_error(Meta, ?m(E, file), - "invalid argument for alias, expected a compile time atom or alias, got: ~ts", - ['Elixir.Kernel':inspect(ERef)]) - end; - -expand({require, Meta, [Ref]}, E) -> - expand({require, Meta, [Ref, []]}, E); -expand({require, Meta, [Ref, KV]}, E) -> - assert_no_match_or_guard_scope(Meta, require, E), - - {ERef, ER} = expand(Ref, E), - {EKV, ET} = expand_opts(Meta, require, [as, warn], no_alias_opts(KV), ER), - - if - is_atom(ERef) -> - elixir_aliases:ensure_loaded(Meta, ERef, ET), - {{require, Meta, [ERef, EKV]}, - expand_require(Meta, ERef, EKV, ET)}; - true -> - compile_error(Meta, ?m(E, file), - "invalid argument for require, expected a compile time atom or alias, got: ~ts", - ['Elixir.Kernel':inspect(ERef)]) - end; - -expand({import, Meta, [Left]}, E) -> - expand({import, Meta, [Left, []]}, E); - -expand({import, Meta, [Ref, KV]}, E) -> - assert_no_match_or_guard_scope(Meta, import, E), - {ERef, ER} = expand(Ref, E), - {EKV, ET} = expand_opts(Meta, import, [only, except, warn], KV, ER), - - if - is_atom(ERef) -> - elixir_aliases:ensure_loaded(Meta, ERef, ET), - {Functions, Macros} = elixir_import:import(Meta, ERef, EKV, ET), - {{import, Meta, [ERef, EKV]}, - expand_require(Meta, ERef, EKV, ET#{functions := Functions, macros := Macros})}; - true -> - compile_error(Meta, ?m(E, file), - "invalid argument for import, expected a compile time atom or alias, got: ~ts", - ['Elixir.Kernel':inspect(ERef)]) - end; - -%% Pseudo vars - -expand({'__MODULE__', _, Atom}, E) when is_atom(Atom) -> - {?m(E, module), E}; -expand({'__DIR__', _, Atom}, E) when is_atom(Atom) -> - {filename:dirname(?m(E, file)), E}; -expand({'__CALLER__', _, Atom} = Caller, E) when is_atom(Atom) -> - {Caller, E}; -expand({'__ENV__', Meta, Atom}, E) when is_atom(Atom) -> - Env = elixir_env:linify({?line(Meta), E}), - {{'%{}', [], maps:to_list(Env)}, E}; -expand({{'.', DotMeta, [{'__ENV__', Meta, Atom}, Field]}, CallMeta, []}, E) when is_atom(Atom), is_atom(Field) -> - Env = elixir_env:linify({?line(Meta), E}), - case maps:is_key(Field, Env) of - true -> {maps:get(Field, Env), E}; - false -> {{{'.', DotMeta, [{'%{}', [], maps:to_list(Env)}, Field]}, CallMeta, []}, E} - end; - -%% Quote - -expand({Unquote, Meta, [_]}, E) when Unquote == unquote; Unquote == unquote_splicing -> - compile_error(Meta, ?m(E, file), "~p called outside quote", [Unquote]); - -expand({quote, Meta, [Opts]}, E) when is_list(Opts) -> - case lists:keyfind(do, 1, Opts) of - {do, Do} -> - expand({quote, Meta, [lists:keydelete(do, 1, Opts), [{do,Do}]]}, E); - false -> - compile_error(Meta, ?m(E, file), "missing do keyword in quote") - end; - -expand({quote, Meta, [_]}, E) -> - compile_error(Meta, ?m(E, file), "invalid arguments for quote"); - -expand({quote, Meta, [KV, Do]}, E) when is_list(Do) -> - Exprs = - case lists:keyfind(do, 1, Do) of - {do, Expr} -> Expr; - false -> compile_error(Meta, E#elixir_scope.file, "missing do keyword in quote") - end, - - ValidOpts = [context, location, line, unquote, bind_quoted], - {EKV, ET} = expand_opts(Meta, quote, ValidOpts, KV, E), - - Context = case lists:keyfind(context, 1, EKV) of - {context, Ctx} when is_atom(Ctx) and (Ctx /= nil) -> - Ctx; - {context, Ctx} -> - compile_error(Meta, ?m(E, file), "invalid :context for quote, " - "expected non nil compile time atom or alias, got: ~ts", ['Elixir.Kernel':inspect(Ctx)]); - false -> - case ?m(E, module) of - nil -> 'Elixir'; - Mod -> Mod - end - end, - - Keep = lists:keyfind(location, 1, EKV) == {location, keep}, - Line = proplists:get_value(line, EKV, false), - - {Binding, DefaultUnquote} = case lists:keyfind(bind_quoted, 1, EKV) of - {bind_quoted, BQ} -> {BQ, false}; - false -> {nil, true} - end, - - Unquote = case lists:keyfind(unquote, 1, EKV) of - {unquote, Bool} when is_boolean(Bool) -> Bool; - false -> DefaultUnquote - end, - - Q = #elixir_quote{line=Line, keep=Keep, unquote=Unquote, context=Context}, - - {Quoted, _Q} = elixir_quote:quote(Exprs, Binding, Q, ET), - expand(Quoted, ET); - -expand({quote, Meta, [_, _]}, E) -> - compile_error(Meta, ?m(E, file), "invalid arguments for quote"); - -%% Functions - -expand({'&', _, [Arg]} = Original, E) when is_integer(Arg) -> - {Original, E}; -expand({'&', Meta, [Arg]}, E) -> - assert_no_match_or_guard_scope(Meta, '&', E), - case elixir_fn:capture(Meta, Arg, E) of - {local, Fun, Arity} -> - {{'&', Meta, [{'/', [], [{Fun, [], nil}, Arity]}]}, E}; - {expanded, Expr, EE} -> - expand(Expr, EE) - end; - -expand({fn, Meta, Pairs}, E) -> - assert_no_match_or_guard_scope(Meta, fn, E), - elixir_fn:expand(Meta, Pairs, E); - -%% Case/Receive/Try - -expand({'cond', Meta, [KV]}, E) -> - assert_no_match_or_guard_scope(Meta, 'cond', E), - {EClauses, EC} = elixir_exp_clauses:'cond'(Meta, KV, E), - {{'cond', Meta, [EClauses]}, EC}; - -expand({'case', Meta, [Expr, KV]}, E) -> - assert_no_match_or_guard_scope(Meta, 'case', E), - {EExpr, EE} = expand(Expr, E), - {EClauses, EC} = elixir_exp_clauses:'case'(Meta, KV, EE), - FClauses = - case (lists:keyfind(optimize_boolean, 1, Meta) == {optimize_boolean, true}) and - elixir_utils:returns_boolean(EExpr) of - true -> rewrite_case_clauses(EClauses); - false -> EClauses - end, - {{'case', Meta, [EExpr, FClauses]}, EC}; - -expand({'receive', Meta, [KV]}, E) -> - assert_no_match_or_guard_scope(Meta, 'receive', E), - {EClauses, EC} = elixir_exp_clauses:'receive'(Meta, KV, E), - {{'receive', Meta, [EClauses]}, EC}; - -expand({'try', Meta, [KV]}, E) -> - assert_no_match_or_guard_scope(Meta, 'try', E), - {EClauses, EC} = elixir_exp_clauses:'try'(Meta, KV, E), - {{'try', Meta, [EClauses]}, EC}; - -%% Comprehensions - -expand({for, Meta, [_|_] = Args}, E) -> - elixir_for:expand(Meta, Args, E); - -%% Super - -expand({super, Meta, Args}, E) when is_list(Args) -> - assert_no_match_or_guard_scope(Meta, super, E), - {EArgs, EA} = expand_args(Args, E), - {{super, Meta, EArgs}, EA}; - -%% Vars - -expand({'^', Meta, [Arg]}, #{context := match} = E) -> - case expand(Arg, E) of - {{Name, _, Kind} = EArg, EA} when is_atom(Name), is_atom(Kind) -> - {{'^', Meta, [EArg]}, EA}; - _ -> - Msg = "invalid argument for unary operator ^, expected an existing variable, got: ^~ts", - compile_error(Meta, ?m(E, file), Msg, ['Elixir.Macro':to_string(Arg)]) - end; -expand({'^', Meta, [Arg]}, E) -> - compile_error(Meta, ?m(E, file), - "cannot use ^~ts outside of match clauses", ['Elixir.Macro':to_string(Arg)]); - -expand({'_', _, Kind} = Var, E) when is_atom(Kind) -> - {Var, E}; -expand({Name, Meta, Kind} = Var, #{context := match, export_vars := Export} = E) when is_atom(Name), is_atom(Kind) -> - Pair = {Name, var_kind(Meta, Kind)}, - NewVars = ordsets:add_element(Pair, ?m(E, vars)), - NewExport = case (Export /= nil) of - true -> ordsets:add_element(Pair, Export); - false -> Export - end, - {Var, E#{vars := NewVars, export_vars := NewExport}}; -expand({Name, Meta, Kind} = Var, #{vars := Vars} = E) when is_atom(Name), is_atom(Kind) -> - case lists:member({Name, var_kind(Meta, Kind)}, Vars) of - true -> - {Var, E}; - false -> - VarMeta = lists:keyfind(var, 1, Meta), - if - VarMeta == {var, true} -> - Extra = case Kind of - nil -> ""; - _ -> io_lib:format(" (context ~ts)", [elixir_aliases:inspect(Kind)]) - end, - - compile_error(Meta, ?m(E, file), "expected var ~ts~ts to expand to an existing " - "variable or be a part of a match", [Name, Extra]); - true -> - expand({Name, Meta, []}, E) - end - end; - -%% Local calls - -expand({Atom, Meta, Args}, E) when is_atom(Atom), is_list(Meta), is_list(Args) -> - assert_no_ambiguous_op(Atom, Meta, Args, E), - - elixir_dispatch:dispatch_import(Meta, Atom, Args, E, fun() -> - expand_local(Meta, Atom, Args, E) - end); - -%% Remote calls - -expand({{'.', DotMeta, [Left, Right]}, Meta, Args}, E) - when (is_tuple(Left) orelse is_atom(Left)), is_atom(Right), is_list(Meta), is_list(Args) -> - {ELeft, EL} = expand(Left, E), - - elixir_dispatch:dispatch_require(Meta, ELeft, Right, Args, EL, fun(AR, AF, AA) -> - expand_remote(AR, DotMeta, AF, Meta, AA, E, EL) - end); - -%% Anonymous calls - -expand({{'.', DotMeta, [Expr]}, Meta, Args}, E) when is_list(Args) -> - {EExpr, EE} = expand(Expr, E), - if - is_atom(EExpr) -> - compile_error(Meta, ?m(E, file), "invalid function call :~ts.()", [EExpr]); - true -> - {EArgs, EA} = expand_args(Args, elixir_env:mergea(E, EE)), - {{{'.', DotMeta, [EExpr]}, Meta, EArgs}, elixir_env:mergev(EE, EA)} - end; - -%% Invalid calls - -expand({_, Meta, Args} = Invalid, E) when is_list(Meta) and is_list(Args) -> - compile_error(Meta, ?m(E, file), "invalid call ~ts", - ['Elixir.Macro':to_string(Invalid)]); - -expand({_, _, _} = Tuple, E) -> - compile_error([{line,0}], ?m(E, file), "invalid quoted expression: ~ts", - ['Elixir.Kernel':inspect(Tuple, [{records,false}])]); - -%% Literals - -expand({Left, Right}, E) -> - {[ELeft, ERight], EE} = expand_args([Left, Right], E), - {{ELeft, ERight}, EE}; - -expand(List, #{context := match} = E) when is_list(List) -> - expand_list(List, fun expand/2, E, []); - -expand(List, E) when is_list(List) -> - {EArgs, {EC, EV}} = expand_list(List, fun expand_arg/2, {E, E}, []), - {EArgs, elixir_env:mergea(EV, EC)}; - -expand(Function, E) when is_function(Function) -> - case (erlang:fun_info(Function, type) == {type, external}) andalso - (erlang:fun_info(Function, env) == {env, []}) of - true -> - {Function, E}; - false -> - compile_error([{line,0}], ?m(E, file), - "invalid quoted expression: ~ts", ['Elixir.Kernel':inspect(Function)]) - end; - -expand(Other, E) when is_number(Other); is_atom(Other); is_binary(Other); is_pid(Other) -> - {Other, E}; - -expand(Other, E) -> - compile_error([{line,0}], ?m(E, file), - "invalid quoted expression: ~ts", ['Elixir.Kernel':inspect(Other)]). - -%% Helpers - -expand_list([{'|', Meta, [_, _] = Args}], Fun, Acc, List) -> - {EArgs, EAcc} = lists:mapfoldl(Fun, Acc, Args), - expand_list([], Fun, EAcc, [{'|', Meta, EArgs}|List]); -expand_list([H|T], Fun, Acc, List) -> - {EArg, EAcc} = Fun(H, Acc), - expand_list(T, Fun, EAcc, [EArg|List]); -expand_list([], _Fun, Acc, List) -> - {lists:reverse(List), Acc}. - -expand_many(Args, E) -> - lists:mapfoldl(fun expand/2, E, Args). - -%% Variables in arguments are not propagated from one -%% argument to the other. For instance: -%% -%% x = 1 -%% foo(x = x + 2, x) -%% x -%% -%% Should be the same as: -%% -%% foo(3, 1) -%% 3 -%% -%% However, lexical information is. -expand_arg(Arg, Acc) when is_number(Arg); is_atom(Arg); is_binary(Arg); is_pid(Arg) -> - {Arg, Acc}; -expand_arg(Arg, {Acc1, Acc2}) -> - {EArg, EAcc} = expand(Arg, Acc1), - {EArg, {elixir_env:mergea(Acc1, EAcc), elixir_env:mergev(Acc2, EAcc)}}. - -expand_args([Arg], E) -> - {EArg, EE} = expand(Arg, E), - {[EArg], EE}; -expand_args(Args, #{context := match} = E) -> - expand_many(Args, E); -expand_args(Args, E) -> - {EArgs, {EC, EV}} = lists:mapfoldl(fun expand_arg/2, {E, E}, Args), - {EArgs, elixir_env:mergea(EV, EC)}. - -%% Match/var helpers - -var_kind(Meta, Kind) -> - case lists:keyfind(counter, 1, Meta) of - {counter, Counter} -> Counter; - false -> Kind - end. - -%% Locals - -assert_no_ambiguous_op(Name, Meta, [Arg], E) -> - case lists:keyfind(ambiguous_op, 1, Meta) of - {ambiguous_op, Kind} -> - case lists:member({Name, Kind}, ?m(E, vars)) of - true -> - compile_error(Meta, ?m(E, file), "\"~ts ~ts\" looks like a function call but " - "there is a variable named \"~ts\", please use explicit parenthesis or even spaces", - [Name, 'Elixir.Macro':to_string(Arg), Name]); - false -> - ok - end; - _ -> - ok - end; -assert_no_ambiguous_op(_Atom, _Meta, _Args, _E) -> - ok. - -expand_local(Meta, Name, Args, #{local := nil, function := nil} = E) -> - {EArgs, EA} = expand_args(Args, E), - {{Name, Meta, EArgs}, EA}; -expand_local(Meta, Name, Args, #{local := nil, module := Module, function := Function} = E) -> - elixir_locals:record_local({Name, length(Args)}, Module, Function), - {EArgs, EA} = expand_args(Args, E), - {{Name, Meta, EArgs}, EA}; -expand_local(Meta, Name, Args, E) -> - expand({{'.', Meta, [?m(E, local), Name]}, Meta, Args}, E). - -%% Remote - -expand_remote(Receiver, DotMeta, Right, Meta, Args, E, EL) -> - if - is_atom(Receiver) -> elixir_lexical:record_remote(Receiver, ?m(E, lexical_tracker)); - true -> ok - end, - {EArgs, EA} = expand_args(Args, E), - {{{'.', DotMeta, [Receiver, Right]}, Meta, EArgs}, elixir_env:mergev(EL, EA)}. - -%% Lexical helpers - -expand_opts(Meta, Kind, Allowed, Opts, E) -> - {EOpts, EE} = expand(Opts, E), - validate_opts(Meta, Kind, Allowed, EOpts, EE), - {EOpts, EE}. - -validate_opts(Meta, Kind, Allowed, Opts, E) when is_list(Opts) -> - [begin - compile_error(Meta, ?m(E, file), - "unsupported option ~ts given to ~s", ['Elixir.Kernel':inspect(Key), Kind]) - end || {Key, _} <- Opts, not lists:member(Key, Allowed)]; - -validate_opts(Meta, Kind, _Allowed, _Opts, S) -> - compile_error(Meta, S#elixir_scope.file, "invalid options for ~s, expected a keyword list", [Kind]). - -no_alias_opts(KV) when is_list(KV) -> - case lists:keyfind(as, 1, KV) of - {as, As} -> lists:keystore(as, 1, KV, {as, no_alias_expansion(As)}); - false -> KV - end; -no_alias_opts(KV) -> KV. - -no_alias_expansion({'__aliases__', Meta, [H|T]}) when (H /= 'Elixir') and is_atom(H) -> - {'__aliases__', Meta, ['Elixir',H|T]}; -no_alias_expansion(Other) -> - Other. - -expand_require(Meta, Ref, KV, E) -> - RE = E#{requires := ordsets:add_element(Ref, ?m(E, requires))}, - expand_alias(Meta, false, Ref, KV, RE). - -expand_alias(Meta, IncludeByDefault, Ref, KV, #{context_modules := Context} = E) -> - New = expand_as(lists:keyfind(as, 1, KV), Meta, IncludeByDefault, Ref, E), - - %% Add the alias to context_modules if defined is true. - %% This is used by defmodule in order to store the defined - %% module in context modules. - NewContext = - case lists:keyfind(defined, 1, Meta) of - {defined, Mod} when is_atom(Mod) -> [Mod|Context]; - false -> Context - end, - - {Aliases, MacroAliases} = elixir_aliases:store(Meta, New, Ref, KV, ?m(E, aliases), - ?m(E, macro_aliases), ?m(E, lexical_tracker)), - - E#{aliases := Aliases, macro_aliases := MacroAliases, context_modules := NewContext}. - -expand_as({as, true}, _Meta, _IncludeByDefault, Ref, _E) -> - elixir_aliases:last(Ref); -expand_as({as, false}, _Meta, _IncludeByDefault, Ref, _E) -> - Ref; -expand_as({as, Atom}, Meta, _IncludeByDefault, _Ref, E) when is_atom(Atom) -> - case length(string:tokens(atom_to_list(Atom), ".")) of - 1 -> compile_error(Meta, ?m(E, file), - "invalid value for keyword :as, expected an alias, got atom: ~ts", [elixir_aliases:inspect(Atom)]); - 2 -> Atom; - _ -> compile_error(Meta, ?m(E, file), - "invalid value for keyword :as, expected an alias, got nested alias: ~ts", [elixir_aliases:inspect(Atom)]) - end; -expand_as(false, _Meta, IncludeByDefault, Ref, _E) -> - if IncludeByDefault -> elixir_aliases:last(Ref); - true -> Ref - end; -expand_as({as, Other}, Meta, _IncludeByDefault, _Ref, E) -> - compile_error(Meta, ?m(E, file), - "invalid value for keyword :as, expected an alias, got: ~ts", ['Elixir.Macro':to_string(Other)]). - -%% Assertions - -rewrite_case_clauses([{do,[ - {'->', FalseMeta, [ - [{'when', _, [Var, {'__op__', _,[ - 'orelse', - {{'.', _, [erlang, '=:=']}, _, [Var, nil]}, - {{'.', _, [erlang, '=:=']}, _, [Var, false]} - ]}]}], - FalseExpr - ]}, - {'->', TrueMeta, [ - [{'_', _, _}], - TrueExpr - ]} -]}]) -> - [{do, [ - {'->', FalseMeta, [[false], FalseExpr]}, - {'->', TrueMeta, [[true], TrueExpr]} - ]}]; -rewrite_case_clauses(Clauses) -> - Clauses. - -assert_no_match_or_guard_scope(Meta, Kind, E) -> - assert_no_match_scope(Meta, Kind, E), - assert_no_guard_scope(Meta, Kind, E). -assert_no_match_scope(Meta, _Kind, #{context := match, file := File}) -> - compile_error(Meta, File, "invalid expression in match"); -assert_no_match_scope(_Meta, _Kind, _E) -> []. -assert_no_guard_scope(Meta, _Kind, #{context := guard, file := File}) -> - compile_error(Meta, File, "invalid expression in guard"); -assert_no_guard_scope(_Meta, _Kind, _E) -> []. diff --git a/lib/elixir/src/elixir_exp_clauses.erl b/lib/elixir/src/elixir_exp_clauses.erl deleted file mode 100644 index 577e7d6dc8a..00000000000 --- a/lib/elixir/src/elixir_exp_clauses.erl +++ /dev/null @@ -1,214 +0,0 @@ -%% Handle code related to args, guard and -> matching for case, -%% fn, receive and friends. try is handled in elixir_try. --module(elixir_exp_clauses). --export([match/3, clause/5, def/5, head/2, - 'case'/3, 'receive'/3, 'try'/3, 'cond'/3]). --import(elixir_errors, [compile_error/3, compile_error/4]). --include("elixir.hrl"). - -match(Fun, Expr, #{context := Context} = E) -> - {EExpr, EE} = Fun(Expr, E#{context := match}), - {EExpr, EE#{context := Context}}. - -def(Fun, Args, Guards, Body, E) -> - {EArgs, EA} = match(Fun, Args, E), - {EGuards, EG} = guard(Guards, EA#{context := guard}), - {EBody, EB} = elixir_exp:expand(Body, EG#{context := ?m(E, context)}), - {EArgs, EGuards, EBody, EB}. - -clause(Meta, Kind, Fun, {'->', ClauseMeta, [_, _]} = Clause, E) when is_function(Fun, 3) -> - clause(Meta, Kind, fun(X, Acc) -> Fun(ClauseMeta, X, Acc) end, Clause, E); -clause(_Meta, _Kind, Fun, {'->', Meta, [Left, Right]}, E) -> - {ELeft, EL} = Fun(Left, E), - {ERight, ER} = elixir_exp:expand(Right, EL), - {{'->', Meta, [ELeft, ERight]}, ER}; -clause(Meta, Kind, _Fun, _, E) -> - compile_error(Meta, ?m(E, file), "expected -> clauses in ~ts", [Kind]). - -head([{'when', Meta, [_,_|_] = All}], E) -> - {Args, Guard} = elixir_utils:split_last(All), - {EArgs, EA} = match(fun elixir_exp:expand_args/2, Args, E), - {EGuard, EG} = guard(Guard, EA#{context := guard}), - {[{'when', Meta, EArgs ++ [EGuard]}], EG#{context := ?m(E, context)}}; -head(Args, E) -> - match(fun elixir_exp:expand_args/2, Args, E). - -guard({'when', Meta, [Left, Right]}, E) -> - {ELeft, EL} = guard(Left, E), - {ERight, ER} = guard(Right, EL), - {{'when', Meta, [ELeft, ERight]}, ER}; -guard(Other, E) -> - elixir_exp:expand(Other, E). - -%% Case - -'case'(Meta, [], E) -> - compile_error(Meta, ?m(E, file), "missing do keyword in case"); -'case'(Meta, KV, E) when not is_list(KV) -> - compile_error(Meta, ?m(E, file), "invalid arguments for case"); -'case'(Meta, KV, E) -> - EE = E#{export_vars := []}, - {EClauses, EVars} = lists:mapfoldl(fun(X, Acc) -> do_case(Meta, X, Acc, EE) end, [], KV), - {EClauses, elixir_env:mergev(EVars, E)}. - -do_case(Meta, {'do', _} = Do, Acc, E) -> - Fun = expand_one(Meta, 'case', 'do', fun head/2), - expand_with_export(Meta, 'case', Fun, Do, Acc, E); -do_case(Meta, {Key, _}, _Acc, E) -> - compile_error(Meta, ?m(E, file), "unexpected keyword ~ts in case", [Key]). - -%% Cond - -'cond'(Meta, [], E) -> - compile_error(Meta, ?m(E, file), "missing do keyword in cond"); -'cond'(Meta, KV, E) when not is_list(KV) -> - compile_error(Meta, ?m(E, file), "invalid arguments for cond"); -'cond'(Meta, KV, E) -> - EE = E#{export_vars := []}, - {EClauses, EVars} = lists:mapfoldl(fun(X, Acc) -> do_cond(Meta, X, Acc, EE) end, [], KV), - {EClauses, elixir_env:mergev(EVars, E)}. - -do_cond(Meta, {'do', _} = Do, Acc, E) -> - Fun = expand_one(Meta, 'cond', 'do', fun elixir_exp:expand_args/2), - expand_with_export(Meta, 'cond', Fun, Do, Acc, E); -do_cond(Meta, {Key, _}, _Acc, E) -> - compile_error(Meta, ?m(E, file), "unexpected keyword ~ts in cond", [Key]). - -%% Receive - -'receive'(Meta, [], E) -> - compile_error(Meta, ?m(E, file), "missing do or after keyword in receive"); -'receive'(Meta, KV, E) when not is_list(KV) -> - compile_error(Meta, ?m(E, file), "invalid arguments for receive"); -'receive'(Meta, KV, E) -> - EE = E#{export_vars := []}, - {EClauses, EVars} = lists:mapfoldl(fun(X, Acc) -> do_receive(Meta, X, Acc, EE) end, [], KV), - {EClauses, elixir_env:mergev(EVars, E)}. - -do_receive(_Meta, {'do', nil} = Do, Acc, _E) -> - {Do, Acc}; -do_receive(Meta, {'do', _} = Do, Acc, E) -> - Fun = expand_one(Meta, 'receive', 'do', fun head/2), - expand_with_export(Meta, 'receive', Fun, Do, Acc, E); -do_receive(Meta, {'after', [_]} = After, Acc, E) -> - Fun = expand_one(Meta, 'receive', 'after', fun elixir_exp:expand_args/2), - expand_with_export(Meta, 'receive', Fun, After, Acc, E); -do_receive(Meta, {'after', _}, _Acc, E) -> - compile_error(Meta, ?m(E, file), "expected a single -> clause for after in receive"); -do_receive(Meta, {Key, _}, _Acc, E) -> - compile_error(Meta, ?m(E, file), "unexpected keyword ~ts in receive", [Key]). - -%% Try - -'try'(Meta, [], E) -> - compile_error(Meta, ?m(E, file), "missing do keywords in try"); -'try'(Meta, KV, E) when not is_list(KV) -> - elixir_errors:compile_error(Meta, ?m(E, file), "invalid arguments for try"); -'try'(Meta, KV, E) -> - {lists:map(fun(X) -> do_try(Meta, X, E) end, KV), E}. - -do_try(_Meta, {'do', Expr}, E) -> - {EExpr, _} = elixir_exp:expand(Expr, E), - {'do', EExpr}; -do_try(_Meta, {'after', Expr}, E) -> - {EExpr, _} = elixir_exp:expand(Expr, E), - {'after', EExpr}; -do_try(Meta, {'else', _} = Else, E) -> - Fun = expand_one(Meta, 'try', 'else', fun head/2), - expand_without_export(Meta, 'try', Fun, Else, E); -do_try(Meta, {'catch', _} = Catch, E) -> - expand_without_export(Meta, 'try', fun expand_catch/3, Catch, E); -do_try(Meta, {'rescue', _} = Rescue, E) -> - expand_without_export(Meta, 'try', fun expand_rescue/3, Rescue, E); -do_try(Meta, {Key, _}, E) -> - compile_error(Meta, ?m(E, file), "unexpected keyword ~ts in try", [Key]). - -expand_catch(_Meta, [_] = Args, E) -> - head(Args, E); -expand_catch(_Meta, [_, _] = Args, E) -> - head(Args, E); -expand_catch(Meta, _, E) -> - compile_error(Meta, ?m(E, file), "expected one or two args for catch clauses (->) in try"). - -expand_rescue(Meta, [Arg], E) -> - case expand_rescue(Arg, E) of - {EArg, EA} -> - {[EArg], EA}; - false -> - compile_error(Meta, ?m(E, file), "invalid rescue clause. The clause should " - "match on an alias, a variable or be in the `var in [alias]` format") - end; -expand_rescue(Meta, _, E) -> - compile_error(Meta, ?m(E, file), "expected one arg for rescue clauses (->) in try"). - -%% rescue var => var in _ -expand_rescue({Name, _, Atom} = Var, E) when is_atom(Name), is_atom(Atom) -> - expand_rescue({in, [], [Var, {'_', [], ?m(E, module)}]}, E); - -%% rescue var in [Exprs] -expand_rescue({in, Meta, [Left, Right]}, E) -> - {ELeft, EL} = match(fun elixir_exp:expand/2, Left, E), - {ERight, ER} = elixir_exp:expand(Right, EL), - - case ELeft of - {Name, _, Atom} when is_atom(Name), is_atom(Atom) -> - case normalize_rescue(ERight) of - false -> false; - Other -> {{in, Meta, [ELeft, Other]}, ER} - end; - _ -> - false - end; - -%% rescue Error => _ in [Error] -expand_rescue(Arg, E) -> - expand_rescue({in, [], [{'_', [], ?m(E, module)}, Arg]}, E). - -normalize_rescue({'_', _, Atom} = N) when is_atom(Atom) -> N; -normalize_rescue(Atom) when is_atom(Atom) -> [Atom]; -normalize_rescue(Other) -> - is_list(Other) andalso lists:all(fun is_atom/1, Other) andalso Other. - -%% Expansion helpers - -%% Returns a function that expands arguments -%% considering we have at maximum one entry. -expand_one(Meta, Kind, Key, Fun) -> - fun - ([_] = Args, E) -> - Fun(Args, E); - (_, E) -> - compile_error(Meta, ?m(E, file), - "expected one arg for ~ts clauses (->) in ~ts", [Key, Kind]) - end. - -%% Expands all -> pairs in a given key keeping the overall vars. -expand_with_export(Meta, Kind, Fun, {Key, Clauses}, Acc, E) when is_list(Clauses) -> - EFun = - case lists:keyfind(export_head, 1, Meta) of - {export_head, true} -> - Fun; - _ -> - fun(Args, #{export_vars := ExportVars} = EE) -> - {FArgs, FE} = Fun(Args, EE), - {FArgs, FE#{export_vars := ExportVars}} - end - end, - Transformer = fun(Clause, Vars) -> - {EClause, EC} = clause(Meta, Kind, EFun, Clause, E), - {EClause, elixir_env:merge_vars(Vars, ?m(EC, export_vars))} - end, - {EClauses, EVars} = lists:mapfoldl(Transformer, Acc, Clauses), - {{Key, EClauses}, EVars}; -expand_with_export(Meta, Kind, _Fun, {Key, _}, _Acc, E) -> - compile_error(Meta, ?m(E, file), "expected -> clauses for ~ts in ~ts", [Key, Kind]). - -%% Expands all -> pairs in a given key but do not keep the overall vars. -expand_without_export(Meta, Kind, Fun, {Key, Clauses}, E) when is_list(Clauses) -> - Transformer = fun(Clause) -> - {EClause, _} = clause(Meta, Kind, Fun, Clause, E), - EClause - end, - {Key, lists:map(Transformer, Clauses)}; -expand_without_export(Meta, Kind, _Fun, {Key, _}, E) -> - compile_error(Meta, ?m(E, file), "expected -> clauses for ~ts in ~ts", [Key, Kind]). diff --git a/lib/elixir/src/elixir_expand.erl b/lib/elixir/src/elixir_expand.erl new file mode 100644 index 00000000000..e1da7aee789 --- /dev/null +++ b/lib/elixir/src/elixir_expand.erl @@ -0,0 +1,1313 @@ +%% SPDX-License-Identifier: Apache-2.0 +%% SPDX-FileCopyrightText: 2021 The Elixir Team +%% SPDX-FileCopyrightText: 2012 Plataformatec + +-module(elixir_expand). +-export([expand/3, expand_args/3, expand_arg/3, format_error/1]). +-import(elixir_errors, [file_error/4, module_error/4, function_error/4]). +-include("elixir.hrl"). + +%% = + +expand({'=', Meta, [_, _]} = Expr, S, #{context := match} = E) -> + elixir_clauses:parallel_match(Meta, Expr, S, E); + +expand({'=', Meta, [Left, Right]}, S, E) -> + assert_no_guard_scope(Meta, "=", S, E), + {ERight, SR, ER} = expand(Right, S, E), + {ELeft, SL, EL} = elixir_clauses:match(fun expand/3, Meta, Left, SR, S, ER), + {{'=', Meta, [ELeft, ERight]}, SL, EL}; + +%% Literal operators + +expand({'{}', Meta, Args}, S, E) -> + {EArgs, SA, EA} = expand_args(Args, S, E), + {{'{}', Meta, EArgs}, SA, EA}; + +expand({'%{}', Meta, Args}, S, E) -> + elixir_map:expand_map(Meta, Args, S, E); + +expand({'%', Meta, [Left, Right]}, S, E) -> + elixir_map:expand_struct(Meta, Left, Right, S, E); + +expand({'<<>>', Meta, Args}, S, E) -> + elixir_bitstring:expand(Meta, Args, S, E, false); + +expand({'->', Meta, [_, _]}, _S, E) -> + file_error(Meta, E, ?MODULE, unhandled_arrow_op); + +expand({'::', Meta, [_, _]}, _S, E) -> + file_error(Meta, E, ?MODULE, unhandled_type_op); + +expand({'|', Meta, [_, _]}, _S, E) -> + file_error(Meta, E, ?MODULE, unhandled_cons_op); + +%% __block__ + +expand({'__block__', _Meta, []}, S, E) -> + {nil, S, E}; +expand({'__block__', _Meta, [Arg]}, S, E) -> + expand(Arg, S, E); +expand({'__block__', Meta, Args}, S, E) when is_list(Args) -> + {EArgs, SA, EA} = expand_block(Args, [], Meta, S, E), + {{'__block__', Meta, EArgs}, SA, EA}; + +%% __aliases__ + +expand({'__aliases__', _, _} = Alias, S, E) -> + expand_aliases(Alias, S, E, true); + +%% alias + +expand({Kind, Meta, [{{'.', _, [Base, '{}']}, _, Refs} | Rest]}, S, E) + when Kind == alias; Kind == require; Kind == import -> + case Rest of + [] -> + expand_multi_alias_call(Kind, Meta, Base, Refs, [], S, E); + [Opts] -> + lists:keymember(as, 1, Opts) andalso file_error(Meta, E, ?MODULE, as_in_multi_alias_call), + expand_multi_alias_call(Kind, Meta, Base, Refs, Opts, S, E) + end; +expand({alias, Meta, [Ref]}, S, E) -> + expand({alias, Meta, [Ref, []]}, S, E); +expand({alias, Meta, [Ref, Opts]}, S, E) -> + assert_no_match_or_guard_scope(Meta, "alias", S, E), + {ERef, SR, ER} = expand_without_aliases_report(Ref, S, E), + {EOpts, ST, ET} = expand_opts(Meta, alias, [as, warn], no_alias_opts(Opts), SR, ER), + + is_atom(ERef) orelse + file_error(Meta, E, ?MODULE, {expected_compile_time_module, alias, Ref}), + + {ok, New, EQ} = alias(Meta, ERef, true, EOpts, ET), + + Quoted = + case (New /= false) andalso should_warn(Meta, EOpts, EQ) of + false -> + ERef; + + Pid when ?key(EQ, function) /= nil -> + ?tracker:warn_alias(Pid, Meta, New, ERef); + + Pid -> + {{'.', Meta, [?tracker, warn_alias]}, Meta, [Pid, Meta, New, ERef]} + end, + + {Quoted, ST, EQ}; + +expand({require, Meta, [Ref]}, S, E) -> + expand({require, Meta, [Ref, []]}, S, E); +expand({require, Meta, [Ref, Opts]}, S, E) -> + assert_no_match_or_guard_scope(Meta, "require", S, E), + + {ERef, SR, ER} = expand_without_aliases_report(Ref, S, E), + {EOpts, ST, ET} = expand_opts(Meta, require, [as, warn], no_alias_opts(Opts), SR, ER), + + %% Add the alias to context_modules if defined is set. + %% This is used by defmodule in order to store the defined + %% module in context modules. + case lists:keyfind(defined, 1, Meta) of + {defined, Mod} when is_atom(Mod) -> + Mods = [Mod | ?key(ET, context_modules)], + {ok, _, EU} = alias(Meta, ERef, false, EOpts, ET#{context_modules := Mods}), + + SU = case E of + #{function := nil} -> ST; + _ -> ST#elixir_ex{runtime_modules=[Mod | ST#elixir_ex.runtime_modules]} + end, + + {ERef, SU, EU}; + + false when is_atom(ERef) -> + elixir_aliases:ensure_loaded(Meta, ERef, ET), + RE = elixir_aliases:require(Meta, ERef, EOpts, ET, true), + {ok, _, EU} = alias(Meta, ERef, false, EOpts, RE), + + Quoted = + case should_warn(Meta, EOpts, EU) of + false -> + ERef; + + Pid when ?key(EU, function) /= nil -> + ?tracker:warn_require(Pid, Meta, ERef); + + Pid -> + {{'.', Meta, [?tracker, warn_require]}, Meta, [Pid, Meta, ERef]} + end, + + {Quoted, ST, EU}; + + false -> + file_error(Meta, E, ?MODULE, {expected_compile_time_module, require, Ref}) + end; + +expand({import, Meta, [Left]}, S, E) -> + expand({import, Meta, [Left, []]}, S, E); + +expand({import, Meta, [Ref, Opts]}, S, E) -> + assert_no_match_or_guard_scope(Meta, "import", S, E), + {ERef, SR, ER} = expand_without_aliases_report(Ref, S, E), + {EOpts, ST, ET} = expand_opts(Meta, import, [only, except, warn], Opts, SR, ER), + + is_atom(ERef) orelse + file_error(Meta, E, ?MODULE, {expected_compile_time_module, import, Ref}), + + elixir_aliases:ensure_loaded(Meta, ERef, ET), + + case elixir_import:import(Meta, ERef, EOpts, ET, true, true) of + {ok, Imported, EI} -> + Quoted = + case Imported andalso should_warn(Meta, EOpts, ET) of + false -> + ERef; + + Pid -> + Only = + case lists:keyfind(only, 1, EOpts) of + {only, List} when is_list(List) -> List; + _ -> [] + end, + + %% If we are outside a function, we turn on the warnings at execution time. + case ET of + #{function := nil} -> + ?tracker:add_import(Pid, ERef, Only, Meta, false), + {{'.', Meta, [?tracker, warn_import]}, Meta, [Pid, ERef]}; + + #{} -> + ?tracker:add_import(Pid, ERef, Only, Meta, true), + ERef + end + end, + + {Quoted, ST, EI}; + + {error, Reason} -> + elixir_errors:file_error(Meta, E, elixir_import, Reason) + end; + +%% Compilation environment macros + +expand({'__MODULE__', _, Atom}, S, E) when is_atom(Atom) -> + {?key(E, module), S, E}; +expand({'__DIR__', _, Atom}, S, E) when is_atom(Atom) -> + {filename:dirname(?key(E, file)), S, E}; +expand({'__CALLER__', Meta, Atom} = Caller, S, E) when is_atom(Atom) -> + assert_no_match_scope(Meta, "__CALLER__", E), + (not S#elixir_ex.caller) andalso function_error(Meta, E, ?MODULE, caller_not_allowed), + {Caller, S, E}; +expand({'__STACKTRACE__', Meta, Atom} = Stacktrace, S, E) when is_atom(Atom) -> + assert_no_match_scope(Meta, "__STACKTRACE__", E), + (not S#elixir_ex.stacktrace) andalso function_error(Meta, E, ?MODULE, stacktrace_not_allowed), + {Stacktrace, S, E}; +expand({'__ENV__', Meta, Atom}, S, E) when is_atom(Atom) -> + assert_no_match_scope(Meta, "__ENV__", E), + {escape_map(escape_env_entries(Meta, S, E)), S, E}; +expand({{'.', DotMeta, [{'__ENV__', Meta, Atom}, Field]}, CallMeta, []}, S, E) + when is_atom(Atom), is_atom(Field) -> + assert_no_match_scope(Meta, "__ENV__", E), + Env = escape_env_entries(Meta, S, E), + case maps:is_key(Field, Env) of + true -> {maps:get(Field, Env), S, E}; + false -> {{{'.', DotMeta, [escape_map(Env), Field]}, CallMeta, []}, S, E} + end; +expand({'__cursor__', Meta, Args}, _S, E) when is_list(Args) -> + file_error(Meta, E, ?MODULE, '__cursor__'); + +%% Quote + +expand({Unquote, Meta, [_]}, _S, E) when Unquote == unquote; Unquote == unquote_splicing -> + file_error(Meta, E, ?MODULE, {unquote_outside_quote, Unquote}); + +expand({quote, Meta, [Opts]}, S, E) when is_list(Opts) -> + case lists:keytake(do, 1, Opts) of + {value, {do, Do}, DoOpts} -> + expand({quote, Meta, [DoOpts, [{do, Do}]]}, S, E); + false -> + file_error(Meta, E, ?MODULE, {missing_option, 'quote', [do]}) + end; + +expand({quote, Meta, [_]}, _S, E) -> + file_error(Meta, E, ?MODULE, {invalid_args, 'quote'}); + +expand({quote, Meta, [Opts, Do]}, S, E) when is_list(Do) -> + Exprs = + case lists:keyfind(do, 1, Do) of + {do, Expr} -> Expr; + false -> file_error(Meta, E, ?MODULE, {missing_option, 'quote', [do]}) + end, + + ValidOpts = [context, location, line, file, unquote, bind_quoted, generated], + {EOpts, ST, ET} = expand_opts(Meta, quote, ValidOpts, Opts, S, E), + + Context = proplists:get_value(context, EOpts, case ?key(E, module) of + nil -> 'Elixir'; + Mod -> Mod + end), + + {File, Line} = case lists:keyfind(location, 1, EOpts) of + {location, keep} -> + {?key(E, file), true}; + false -> + {proplists:get_value(file, EOpts, nil), proplists:get_value(line, EOpts, false)} + end, + + {Binding, DefaultUnquote} = case lists:keyfind(bind_quoted, 1, EOpts) of + {bind_quoted, BQ} -> + case is_list(BQ) andalso + lists:all(fun({Key, _}) when is_atom(Key) -> true; (_) -> false end, BQ) of + true -> {BQ, false}; + false -> file_error(Meta, E, ?MODULE, {invalid_bind_quoted_for_quote, BQ}) + end; + false -> + {[], true} + end, + + Unquote = proplists:get_value(unquote, EOpts, DefaultUnquote), + Generated = proplists:get_value(generated, EOpts, false), + + {Q, QContext, QPrelude} = elixir_quote:build(Meta, Line, File, Context, Unquote, Generated, ET), + {EPrelude, SP, EP} = expand(QPrelude, ST, ET), + {EContext, SC, EC} = expand(QContext, SP, EP), + Quoted = elixir_quote:quote(Exprs, Q), + {EQuoted, ES, EQ} = expand(Quoted, SC, EC), + BindingMeta = lists:keydelete(column, 1, Meta), + + EBinding = + [{'{}', [], + ['=', [], [ + {'{}', [], [K, BindingMeta, EContext]}, + V + ] + ]} || {K, V} <- Binding], + + EBindingQuoted = + case EBinding of + [] -> EQuoted; + _ -> {'{}', [], ['__block__', [], EBinding ++ [EQuoted]]} + end, + + case EPrelude of + [] -> {EBindingQuoted, ES, EQ}; + _ -> {{'__block__', [], EPrelude ++ [EBindingQuoted]}, ES, EQ} + end; + +expand({quote, Meta, [_, _]}, _S, E) -> + file_error(Meta, E, ?MODULE, {invalid_args, 'quote'}); + +%% Functions + +expand({'&', Meta, [{super, SuperMeta, Args} = Expr]}, S, E) when is_list(Args) -> + assert_no_match_or_guard_scope(Meta, "&", S, E), + + case resolve_super(Meta, length(Args), E) of + {Kind, Name, _} when Kind == def; Kind == defp -> + expand_fn_capture(Meta, {Name, SuperMeta, Args}, S, E); + _ -> + expand_fn_capture(Meta, Expr, S, E) + end; + +expand({'&', Meta, [{'/', ArityMeta, [{super, SuperMeta, Context}, Arity]} = Expr]}, S, E) when is_atom(Context), is_integer(Arity) -> + assert_no_match_or_guard_scope(Meta, "&", S, E), + + case resolve_super(Meta, Arity, E) of + {Kind, Name, _} when Kind == def; Kind == defp -> + expand({'&', Meta, [{'/', ArityMeta, [{Name, SuperMeta, Context}, Arity]}]}, S, E); + _ -> + expand_fn_capture(Meta, Expr, S, E) + end; + +expand({'&', Meta, [Arg]}, S, E) -> + assert_no_match_or_guard_scope(Meta, "&", S, E), + expand_fn_capture(Meta, Arg, S, E); + +expand({fn, Meta, Pairs}, S, E) -> + assert_no_match_or_guard_scope(Meta, "fn", S, E), + elixir_fn:expand(Meta, Pairs, S, E); + +%% Case/Receive/Try + +expand({'cond', Meta, [Opts]}, S, E) -> + assert_no_match_or_guard_scope(Meta, "cond", S, E), + assert_no_underscore_clause_in_cond(Opts, E), + {EClauses, SC, EC} = elixir_clauses:'cond'(Meta, Opts, S, E), + {{'cond', Meta, [EClauses]}, SC, EC}; + +expand({'case', Meta, [Expr, Options]}, S, E) -> + assert_no_match_or_guard_scope(Meta, "case", S, E), + expand_case(Meta, Expr, Options, S, E); + +expand({'receive', Meta, [Opts]}, S, E) -> + assert_no_match_or_guard_scope(Meta, "receive", S, E), + {EClauses, SC, EC} = elixir_clauses:'receive'(Meta, Opts, S, E), + {{'receive', Meta, [EClauses]}, SC, EC}; + +expand({'try', Meta, [Opts]}, S, E) -> + assert_no_match_or_guard_scope(Meta, "try", S, E), + {EClauses, SC, EC} = elixir_clauses:'try'(Meta, Opts, S, E), + {{'try', Meta, [EClauses]}, SC, EC}; + +%% Comprehensions + +expand({for, _, [_ | _] } = Expr, S, E) -> + expand_for(Expr, S, E, true); + +%% With + +expand({with, Meta, [_ | _] = Args}, S, E) -> + assert_no_match_or_guard_scope(Meta, "with", S, E), + elixir_clauses:with(Meta, Args, S, E); + +%% Super + +expand({super, Meta, Args}, S, E) when is_list(Args) -> + assert_no_match_or_guard_scope(Meta, "super", S, E), + Arity = length(Args), + {Kind, Name, _} = resolve_super(Meta, Arity, E), + elixir_env:trace({local_function, Meta, Name, Arity}, E), + {EArgs, SA, EA} = expand_args(Args, S, E), + {{super, [{super, {Kind, Name}} | Meta], EArgs}, SA, EA}; + +%% Vars + +expand({'^', Meta, [Arg]}, #elixir_ex{prematch={Prematch, _, _}, vars={_, Write}} = S, E) -> + NoMatchS = S#elixir_ex{prematch=pin, vars={Prematch, Write}}, + + case expand(Arg, NoMatchS, E#{context := nil}) of + {{Name, _, Kind} = Var, #elixir_ex{unused=Unused}, _} when is_atom(Name), is_atom(Kind) -> + {{'^', Meta, [Var]}, S#elixir_ex{unused=Unused}, E}; + + _ -> + function_error(Meta, E, ?MODULE, {invalid_arg_for_pin, Arg}), + {{'^', Meta, [Arg]}, S, E} + end; +expand({'^', Meta, [Arg]}, S, E) -> + function_error(Meta, E, ?MODULE, {pin_outside_of_match, Arg}), + {{'^', Meta, [Arg]}, S, E}; + +expand({'_', Meta, Kind} = Var, S, #{context := Context} = E) when is_atom(Kind) -> + (Context /= match) andalso function_error(Meta, E, ?MODULE, unbound_underscore), + {Var, S, E}; + +expand({Name, Meta, Kind}, S, #{context := match} = E) when is_atom(Name), is_atom(Kind) -> + #elixir_ex{ + prematch={_, _, PrematchVersion}, + unused={Unused, Version}, + vars={Read, Write} + } = S, + + Pair = {Name, elixir_utils:var_context(Meta, Kind)}, + + case Read of + %% Variable was already overridden + #{Pair := VarVersion} when VarVersion >= PrematchVersion -> + maybe_warn_underscored_var_repeat(Meta, Name, Kind, E), + NewUnused = var_used(Pair, Meta, VarVersion, Unused), + NewWrite = (Write /= false) andalso Write#{Pair => Version}, + Var = {Name, [{version, VarVersion} | Meta], Kind}, + {Var, S#elixir_ex{vars={Read, NewWrite}, unused={NewUnused, Version}}, E}; + + %% Variable is being overridden now + #{Pair := _} -> + NewUnused = var_unused(Pair, Meta, Version, Unused, true), + NewRead = Read#{Pair => Version}, + NewWrite = (Write /= false) andalso Write#{Pair => Version}, + Var = {Name, [{version, Version} | Meta], Kind}, + {Var, S#elixir_ex{vars={NewRead, NewWrite}, unused={NewUnused, Version + 1}}, E}; + + %% Variable defined for the first time + _ -> + NewUnused = var_unused(Pair, Meta, Version, Unused, false), + NewRead = Read#{Pair => Version}, + NewWrite = (Write /= false) andalso Write#{Pair => Version}, + Var = {Name, [{version, Version} | Meta], Kind}, + {Var, S#elixir_ex{vars={NewRead, NewWrite}, unused={NewUnused, Version + 1}}, E} + end; + +expand({Name, Meta, Kind}, S, E) when is_atom(Name), is_atom(Kind) -> + #elixir_ex{vars={Read, _Write}, unused={Unused, Version}, prematch=Prematch} = S, + Pair = {Name, elixir_utils:var_context(Meta, Kind)}, + + Result = + case Read of + #{Pair := CurrentVersion} -> + case Prematch of + {Pre, _Cycle, {bitsize, Original}} -> + if + map_get(Pair, Pre) /= CurrentVersion -> + {ok, CurrentVersion}; + + is_map_key(Pair, Pre) -> + %% TODO: Remove me on Elixir 2.0 + elixir_errors:file_warn(Meta, E, ?MODULE, {unpinned_bitsize_var, Name, Kind}), + {ok, CurrentVersion}; + + not is_map_key(Pair, Original) -> + {ok, CurrentVersion}; + + true -> + raise + end; + + _ -> + {ok, CurrentVersion} + end; + + _ -> + case E of + #{context := guard} -> raise; + #{} when S#elixir_ex.prematch =:= pin -> pin; + %% TODO: Remove fallback on on_undefined_variable + _ -> elixir_config:get(on_undefined_variable) + end + end, + + case Result of + {ok, PairVersion} -> + maybe_warn_underscored_var_access(Meta, Name, Kind, E), + Var = {Name, [{version, PairVersion} | Meta], Kind}, + {Var, S#elixir_ex{unused={var_used(Pair, Meta, PairVersion, Unused), Version}}, E}; + + Error -> + case lists:keyfind(if_undefined, 1, Meta) of + {if_undefined, apply} -> + expand({Name, Meta, []}, S, E); + + %% TODO: Remove this clause on v2.0 as we will raise by default + {if_undefined, raise} -> + function_error(Meta, E, ?MODULE, {undefined_var, Name, Kind}), + {{Name, Meta, Kind}, S, E}; + + %% TODO: Remove this clause on v2.0 as we will no longer support warn + _ when Error == warn -> + elixir_errors:file_warn(Meta, E, ?MODULE, {undefined_var_to_call, Name}), + expand({Name, [{if_undefined, warn} | Meta], []}, S, E); + + _ when Error == pin -> + function_error(Meta, E, ?MODULE, {undefined_var_pin, Name, Kind}), + {{Name, Meta, Kind}, S, E}; + + _ when Error == raise -> + SpanMeta = elixir_env:calculate_span(Meta, Name), + function_error(SpanMeta, E, ?MODULE, {undefined_var, Name, Kind}), + {{Name, SpanMeta, Kind}, S, E} + end + end; + +%% Local calls + +expand({Atom, Meta, Args}, S, E) when is_atom(Atom), is_list(Meta), is_list(Args) -> + assert_no_ambiguous_op(Atom, Meta, Args, S, E), + + elixir_dispatch:dispatch_import(Meta, Atom, Args, S, E, fun + ({AR, AF}) -> + expand_remote(AR, Meta, AF, Meta, Args, S, elixir_env:prepare_write(S, E), E); + + (local) -> + expand_local(Meta, Atom, Args, S, E) + end); + +%% Remote calls + +expand({{'.', DotMeta, [Left, Right]}, Meta, Args}, S, E) + when (is_tuple(Left) orelse is_atom(Left)), is_atom(Right), is_list(Meta), is_list(Args) -> + {ELeft, SL, EL} = expand(Left, elixir_env:prepare_write(S, E), E), + + elixir_dispatch:dispatch_require(Meta, ELeft, Right, Args, S, EL, fun(AR, AF) -> + expand_remote(AR, DotMeta, AF, Meta, Args, S, SL, EL) + end); + +%% Anonymous calls + +expand({{'.', DotMeta, [Expr]}, Meta, Args}, S, E) when is_list(Args) -> + assert_no_match_or_guard_scope(Meta, "anonymous call", S, E), + {[EExpr | EArgs], SA, EA} = expand_args([Expr | Args], S, E), + {{{'.', DotMeta, [EExpr]}, Meta, EArgs}, SA, EA}; + +%% Invalid calls + +expand({_, Meta, Args} = Invalid, _S, E) when is_list(Meta) and is_list(Args) -> + file_error(Meta, E, ?MODULE, {invalid_call, Invalid}); + +%% Literals + +expand({Left, Right}, S, E) -> + {[ELeft, ERight], SE, EE} = expand_args([Left, Right], S, E), + {{ELeft, ERight}, SE, EE}; + +expand(List, S, #{context := match} = E) when is_list(List) -> + expand_list(List, fun expand/3, S, E, []); + +expand(List, S, E) when is_list(List) -> + {EArgs, {SE, _}, EE} = + expand_list(List, fun expand_arg/3, {elixir_env:prepare_write(S), S}, E, []), + + {EArgs, elixir_env:close_write(SE, S), EE}; + +expand(Zero, S, #{context := match} = E) when is_float(Zero), Zero == 0.0 -> + elixir_errors:file_warn([], E, ?MODULE, invalid_match_on_zero_float), + {Zero, S, E}; + +expand(Other, S, E) when is_number(Other); is_atom(Other); is_binary(Other) -> + {Other, S, E}; + +expand(Function, S, E) when is_function(Function) -> + case (erlang:fun_info(Function, type) == {type, external}) andalso + (erlang:fun_info(Function, env) == {env, []}) of + true -> + {elixir_quote:fun_to_quoted(Function), S, E}; + false -> + file_error([{line, 0}], ?key(E, file), ?MODULE, {invalid_quoted_expr, Function}) + end; + +expand(Pid, S, E) when is_pid(Pid) -> + case ?key(E, function) of + nil -> + {Pid, S, E}; + Function -> + %% TODO: Make me an error on v2.0 + elixir_errors:file_warn([], E, ?MODULE, {invalid_pid_in_function, Pid, Function}), + {Pid, S, E} + end; + +expand(Other, _S, E) -> + file_error([{line, 0}], ?key(E, file), ?MODULE, {invalid_quoted_expr, Other}). + +%% Helpers + +escape_env_entries(Meta, #elixir_ex{vars={Read, _}}, Env0) -> + Env1 = case Env0 of + #{function := nil} -> Env0; + _ -> Env0#{lexical_tracker := nil, tracers := []} + end, + + Env1#{versioned_vars := escape_map(Read), line := ?line(Meta)}. + +escape_map(Map) -> {'%{}', [], lists:sort(maps:to_list(Map))}. + +expand_multi_alias_call(Kind, Meta, Base, Refs, Opts, S, E) -> + {BaseRef, SB, EB} = expand_without_aliases_report(Base, S, E), + + case is_atom(BaseRef) of + true -> + Fun = fun + ({'__aliases__', _, Ref}, SR, ER) -> + expand({Kind, Meta, [elixir_aliases:concat([BaseRef | Ref]), Opts]}, SR, ER); + + (Ref, SR, ER) when is_atom(Ref) -> + expand({Kind, Meta, [elixir_aliases:concat([BaseRef, Ref]), Opts]}, SR, ER); + + (Other, _SR, _ER) -> + file_error(Meta, E, ?MODULE, {expected_compile_time_module, Kind, Other}) + end, + + mapfold(Fun, SB, EB, Refs); + + false -> + file_error(Meta, E, ?MODULE, {invalid_alias, Base}) + end. + +resolve_super(Meta, Arity, E) -> + Module = assert_module_scope(Meta, super, E), + Function = assert_function_scope(Meta, super, E), + + case Function of + {_, Arity} -> + {Kind, Name, SuperMeta} = elixir_overridable:super(Meta, Module, Function, E), + maybe_warn_deprecated_super_in_gen_server_callback(Meta, Function, SuperMeta, E), + {Kind, Name, SuperMeta}; + + _ -> + file_error(Meta, E, ?MODULE, wrong_number_of_args_for_super) + end. + +expand_fn_capture(Meta, Arg, S, E) -> + case elixir_fn:capture(Meta, Arg, S, E) of + {{remote, Remote, Fun, Arity}, RequireMeta, DotMeta, SE, EE} -> + AttachedMeta = attach_runtime_module(Remote, RequireMeta, S, E), + {{'&', Meta, [{'/', [], [{{'.', DotMeta, [Remote, Fun]}, AttachedMeta, []}, Arity]}]}, SE, EE}; + {{local, Fun, Arity}, _, _, _SE, #{function := nil}} -> + file_error(Meta, E, ?MODULE, {undefined_local_capture, Fun, Arity}); + {{local, Fun, Arity}, LocalMeta, _, SE, EE} -> + {{'&', Meta, [{'/', [], [{Fun, LocalMeta, nil}, Arity]}]}, SE, EE}; + {expand, Expr, SE, EE} -> + expand(Expr, SE, EE) + end. + +expand_list([{'|', Meta, [_, _] = Args}], Fun, S, E, List) -> + {EArgs, SAcc, EAcc} = mapfold(Fun, S, E, Args), + expand_list([], Fun, SAcc, EAcc, [{'|', Meta, EArgs} | List]); +expand_list([H | T], Fun, S, E, List) -> + {EArg, SAcc, EAcc} = Fun(H, S, E), + expand_list(T, Fun, SAcc, EAcc, [EArg | List]); +expand_list([], _Fun, S, E, List) -> + {lists:reverse(List), S, E}. + +expand_block([], Acc, _Meta, S, E) -> + {lists:reverse(Acc), S, E}; +expand_block([H], Acc, Meta, S, E) -> + {EH, SE, EE} = expand(H, S, E), + expand_block([], [EH | Acc], Meta, SE, EE); +expand_block([{for, _, [_ | _]} = H | T], Acc, Meta, S, E) -> + {EH, SE, EE} = expand_for(H, S, E, false), + expand_block(T, [EH | Acc], Meta, SE, EE); +expand_block([{'=', _, [{'_', _, Ctx}, {for, _, [_ | _]} = H]} | T], Acc, Meta, S, E) when is_atom(Ctx) -> + {EH, SE, EE} = expand_for(H, S, E, false), + expand_block(T, [EH | Acc], Meta, SE, EE); +expand_block([H | T], Acc, Meta, S, E) -> + {EH, SE, EE} = expand(H, S, E), + + %% Note that checks rely on the code BEFORE expansion + %% instead of relying on Erlang checks. + %% + %% That's because expansion may generate useless + %% terms on their own (think compile time removed + %% logger calls) and we don't want to catch those. + %% + %% Or, similarly, the work is all in the expansion + %% (for example, to register something) and it is + %% simply returning something as replacement. + case is_useless_building(H, EH, Meta) of + {UselessMeta, UselessTerm} -> + elixir_errors:file_warn(UselessMeta, E, ?MODULE, UselessTerm); + + false -> + ok + end, + + expand_block(T, [EH | Acc], Meta, SE, EE). + +%% Note that we don't handle atoms on purpose. They are common +%% when unquoting AST and it is unlikely that we would catch +%% bugs as we don't do binary operations on them like in +%% strings or numbers. +is_useless_building(H, _, Meta) when is_binary(H); is_number(H) -> + {Meta, {useless_literal, H}}; +is_useless_building({'@', Meta, [{Var, _, Ctx}]}, _, _) when is_atom(Ctx); Ctx == [] -> + {Meta, {useless_attr, Var}}; +is_useless_building({Var, Meta, Ctx}, {Var, _, Ctx}, _) when is_atom(Ctx) -> + {Meta, {useless_var, Var}}; +is_useless_building(_, _, _) -> + false. + +%% Variables in arguments are not propagated from one +%% argument to the other. For instance: +%% +%% x = 1 +%% foo(x = x + 2, x) +%% x +%% +%% Should be the same as: +%% +%% foo(3, 1) +%% 3 +%% +%% However, lexical information is. +expand_arg(Arg, Acc, E) when is_number(Arg); is_atom(Arg); is_binary(Arg); is_pid(Arg) -> + {Arg, Acc, E}; +expand_arg(Arg, {Acc, S}, E) -> + {EArg, SAcc, EAcc} = expand(Arg, elixir_env:reset_read(Acc, S), E), + {EArg, {SAcc, S}, EAcc}. + +expand_args([Arg], S, E) -> + {EArg, SE, EE} = expand(Arg, S, E), + {[EArg], SE, EE}; +expand_args(Args, S, #{context := match} = E) -> + mapfold(fun expand/3, S, E, Args); +expand_args(Args, S, E) -> + {EArgs, {SA, _}, EA} = mapfold(fun expand_arg/3, {elixir_env:prepare_write(S), S}, E, Args), + {EArgs, elixir_env:close_write(SA, S), EA}. + +mapfold(Fun, S, E, List) -> + mapfold(Fun, S, E, List, []). + +mapfold(Fun, S, E, [H | T], Acc) -> + {RH, RS, RE} = Fun(H, S, E), + mapfold(Fun, RS, RE, T, [RH | Acc]); +mapfold(_Fun, S, E, [], Acc) -> + {lists:reverse(Acc), S, E}. + +%% Match/var helpers + +var_unused({_, Kind} = Pair, Meta, Version, Unused, Override) -> + case (Kind == nil) andalso should_warn(Meta) of + true -> Unused#{{Pair, Version} => {Meta, Override}}; + false -> Unused + end. + +var_used({_, Kind} = Pair, Meta, Version, Unused) -> + KeepUnused = lists:keymember(keep_unused, 1, Meta), + + if + KeepUnused -> Unused; + is_atom(Kind) -> Unused#{{Pair, Version} => false}; + true -> Unused + end. + +maybe_warn_underscored_var_repeat(Meta, Name, Kind, E) -> + case should_warn(Meta) andalso atom_to_list(Name) of + "_" ++ _ -> + elixir_errors:file_warn(Meta, E, ?MODULE, {underscored_var_repeat, Name, Kind}); + _ -> + ok + end. + +maybe_warn_underscored_var_access(Meta, Name, Kind, E) -> + case (Kind == nil) andalso should_warn(Meta) andalso atom_to_list(Name) of + "_" ++ _ -> + elixir_errors:file_warn(Meta, E, ?MODULE, {underscored_var_access, Name}); + _ -> + ok + end. + +%% TODO: Remove this on Elixir v2.0 and make all GenServer callbacks optional +maybe_warn_deprecated_super_in_gen_server_callback(Meta, Function, SuperMeta, E) -> + case lists:keyfind(context, 1, SuperMeta) of + {context, 'Elixir.GenServer'} -> + case Function of + {child_spec, 1} -> + ok; + + _ -> + elixir_errors:file_warn(Meta, E, ?MODULE, {super_in_genserver, Function}) + end; + + _ -> + ok + end. + +should_warn(Meta) -> + lists:keyfind(generated, 1, Meta) /= {generated, true}. + +%% Case + +expand_case(Meta, Expr, Opts, S, E) -> + {EExpr, SE, EE} = expand(Expr, S, E), + + ROpts = + case lists:member({optimize_boolean, true}, Meta) andalso elixir_utils:returns_boolean(EExpr) of + true -> rewrite_case_clauses(Opts); + false -> Opts + end, + + {EOpts, SO, EO} = elixir_clauses:'case'(Meta, ROpts, SE, EE), + {{'case', Meta, [EExpr, EOpts]}, SO, EO}. + +rewrite_case_clauses([{do, [ + {'->', FalseMeta, [ + [{'when', _, [Var, {{'.', _, ['Elixir.Kernel', 'in']}, _, [Var, [false, nil]]}]}], + FalseExpr + ]}, + {'->', TrueMeta, [ + [{'_', _, _}], + TrueExpr + ]} +]}]) -> + rewrite_case_clauses(FalseMeta, FalseExpr, TrueMeta, TrueExpr); + +rewrite_case_clauses([{do, [ + {'->', FalseMeta, [[false], FalseExpr]}, + {'->', TrueMeta, [[true], TrueExpr]} | _ +]}]) -> + rewrite_case_clauses(FalseMeta, FalseExpr, TrueMeta, TrueExpr); + +rewrite_case_clauses(Opts) -> + Opts. + +rewrite_case_clauses(FalseMeta, FalseExpr, TrueMeta, TrueExpr) -> + [{do, [ + {'->', FalseMeta, [[false], FalseExpr]}, + {'->', TrueMeta, [[true], TrueExpr]} + ]}]. + +%% Comprehensions + +expand_for({for, Meta, [_ | _] = Args}, S, E, Return) -> + assert_no_match_or_guard_scope(Meta, "for", S, E), + {Cases, Block} = elixir_utils:split_opts(Args), + validate_opts(Meta, for, [do, into, uniq, reduce], Block, E), + + {Expr, Opts} = + case lists:keytake(do, 1, Block) of + {value, {do, Do}, DoOpts} -> + {Do, DoOpts}; + false -> + file_error(Meta, E, ?MODULE, {missing_option, for, [do]}) + end, + + {EOpts, SO, EO} = expand(Opts, elixir_env:reset_unused_vars(S), E), + {ECases, SC, EC} = mapfold(fun expand_for_generator/3, SO, EO, Cases), + assert_generator_start(Meta, ECases, E), + + {{EExpr, SE, EE}, NormalizedOpts} = + case validate_for_options(EOpts, false, false, false, Return, Meta, E, []) of + {ok, MaybeReduce, NOpts} -> {expand_for_do_block(Meta, Expr, SC, EC, MaybeReduce), NOpts}; + {error, Error} -> {file_error(Meta, E, ?MODULE, Error), EOpts} + end, + + {{for, Meta, ECases ++ [[{do, EExpr} | NormalizedOpts]]}, + elixir_env:merge_and_check_unused_vars(SE, S, EE), + E}. + +validate_for_options([{into, _} = Pair | Opts], _Into, Uniq, Reduce, Return, Meta, E, Acc) -> + validate_for_options(Opts, Pair, Uniq, Reduce, Return, Meta, E, [Pair | Acc]); +validate_for_options([{uniq, Boolean} = Pair | Opts], Into, _Uniq, Reduce, Return, Meta, E, Acc) when is_boolean(Boolean) -> + validate_for_options(Opts, Into, Pair, Reduce, Return, Meta, E, [Pair | Acc]); +validate_for_options([{uniq, Value} | _], _, _, _, _, _, _, _) -> + {error, {for_invalid_uniq, Value}}; +validate_for_options([{reduce, _} = Pair | Opts], Into, Uniq, _Reduce, Return, Meta, E, Acc) -> + validate_for_options(Opts, Into, Uniq, Pair, Return, Meta, E, [Pair | Acc]); +validate_for_options([], Into, Uniq, {reduce, _}, _Return, _Meta, _E, _Acc) when Into /= false; Uniq /= false -> + {error, for_conflicting_reduce_into_uniq}; +validate_for_options([], _Into = false, Uniq, Reduce = false, Return = true, Meta, E, Acc) -> + Pair = {into, []}, + validate_for_options([Pair], Pair, Uniq, Reduce, Return, Meta, E, Acc); +validate_for_options([], Into = false, {uniq, true}, Reduce = false, Return = false, Meta, E, Acc) -> + elixir_errors:file_warn(Meta, E, ?MODULE, for_with_unused_uniq), + AccWithoutUniq = lists:keydelete(uniq, 1, Acc), + validate_for_options([], Into, false, Reduce, Return, Meta, E, AccWithoutUniq); +validate_for_options([], _Into, _Uniq, Reduce, _Return, _Meta, _E, Acc) -> + {ok, Reduce, lists:reverse(Acc)}. + +expand_for_do_block(Meta, [{'->', _, _} | _], _S, E, false) -> + file_error(Meta, E, ?MODULE, for_without_reduce_bad_block); +expand_for_do_block(_Meta, Expr, S, E, false) -> + expand(Expr, S, E); +expand_for_do_block(Meta, [{'->', _, _} | _] = Clauses, S, E, {reduce, _}) -> + Transformer = fun + ({_, _, [[{'when', _, [_, _, _ | _]}], _]}, _) -> + file_error(Meta, E, ?MODULE, for_with_reduce_bad_block); + + ({_, _, [[_], _]} = Clause, SA) -> + SReset = elixir_env:reset_unused_vars(SA), + + {EClause, SAcc, EAcc} = + elixir_clauses:clause(Meta, fn, fun elixir_clauses:head/4, Clause, SReset, E), + + {EClause, elixir_env:merge_and_check_unused_vars(SAcc, SA, EAcc)}; + + (_, _) -> + file_error(Meta, E, ?MODULE, for_with_reduce_bad_block) + end, + + {Do, SA} = lists:mapfoldl(Transformer, S, Clauses), + {Do, SA, E}; +expand_for_do_block(Meta, _Expr, _S, E, {reduce, _}) -> + file_error(Meta, E, ?MODULE, for_with_reduce_bad_block). + +%% Locals + +assert_no_ambiguous_op(Name, Meta, [Arg], S, E) -> + case lists:keyfind(ambiguous_op, 1, Meta) of + {ambiguous_op, Kind} -> + Pair = {Name, Kind}, + case S#elixir_ex.vars of + {#{Pair := _}, _} -> + file_error(Meta, E, ?MODULE, {op_ambiguity, Name, Arg}); + _ -> + ok + end; + _ -> + ok + end; +assert_no_ambiguous_op(_Atom, _Meta, _Args, _S, _E) -> + ok. + +expand_local(Meta, Name, Args, S, #{function := Function, context := Context} = E) + when Function /= nil -> + %% In case we have the wrong context, we log a module error + %% so we can print multiple entries at the same time. + case Context of + match -> + module_error(Meta, E, ?MODULE, {invalid_local_invocation, "match", {Name, Meta, Args}}); + + guard -> + module_error(Meta, E, ?MODULE, {invalid_local_invocation, elixir_utils:guard_info(S), {Name, Meta, Args}}); + + nil -> + elixir_env:trace({local_function, Meta, Name, length(Args)}, E) + end, + + {EArgs, SA, EA} = expand_args(Args, S, E), + {{Name, Meta, EArgs}, SA, EA}; +expand_local(Meta, Name, Args, _S, #{function := nil} = E) -> + file_error(Meta, E, ?MODULE, {undefined_function, Name, Args}). + +%% Remote + +expand_remote(Receiver, DotMeta, Right, Meta, Args, S, SL, #{context := Context} = E) + when is_atom(Receiver) or is_tuple(Receiver) -> + if + Context =:= guard, is_tuple(Receiver) -> + (lists:keyfind(no_parens, 1, Meta) /= {no_parens, true}) andalso + function_error(Meta, E, ?MODULE, {parens_map_lookup, Receiver, Right, elixir_utils:guard_info(S)}), + + {{{'.', DotMeta, [Receiver, Right]}, Meta, []}, SL, E}; + + Context =:= nil -> + AttachedMeta = attach_runtime_module(Receiver, Meta, S, E), + {EArgs, {SA, _}, EA} = mapfold(fun expand_arg/3, {SL, S}, E, Args), + Rewritten = elixir_rewrite:rewrite(Receiver, DotMeta, Right, AttachedMeta, EArgs), + {Rewritten, elixir_env:close_write(SA, S), EA}; + + true -> + case {Receiver, Right, Args} of + {erlang, '+', [Arg]} when is_number(Arg) -> {+Arg, SL, E}; + {erlang, '-', [Arg]} when is_number(Arg) -> {-Arg, SL, E}; + _ -> + {EArgs, SA, EA} = mapfold(fun expand/3, SL, E, Args), + + case elixir_rewrite:Context(Receiver, DotMeta, Right, Meta, EArgs, S) of + {ok, Rewritten} -> {Rewritten, SA, EA}; + {error, Error} -> file_error(Meta, E, elixir_rewrite, Error) + end + end + end; +expand_remote(Receiver, DotMeta, Right, Meta, Args, _, _, E) -> + Call = {{'.', DotMeta, [Receiver, Right]}, Meta, Args}, + file_error(Meta, E, ?MODULE, {invalid_call, Call}). + +attach_runtime_module(Receiver, Meta, S, _E) -> + case lists:member(Receiver, S#elixir_ex.runtime_modules) of + true -> [{runtime_module, true} | Meta]; + false -> Meta + end. + +%% Lexical helpers + +expand_opts(Meta, Kind, Allowed, Opts, S, E) -> + {EOpts, SE, EE} = expand(Opts, S, E), + validate_opts(Meta, Kind, Allowed, EOpts, EE), + {EOpts, SE, EE}. + +validate_opts(Meta, Kind, Allowed, Opts, E) when is_list(Opts) -> + [begin + file_error(Meta, E, ?MODULE, {unsupported_option, Kind, Key}) + end || {Key, _} <- Opts, not lists:member(Key, Allowed)]; + +validate_opts(Meta, Kind, _Allowed, Opts, E) -> + file_error(Meta, E, ?MODULE, {options_are_not_keyword, Kind, Opts}). + +no_alias_opts(Opts) when is_list(Opts) -> + case lists:keyfind(as, 1, Opts) of + {as, As} -> lists:keystore(as, 1, Opts, {as, no_alias_expansion(As)}); + false -> Opts + end; +no_alias_opts(Opts) -> Opts. + +no_alias_expansion({'__aliases__', _, [H | T]}) when is_atom(H) -> + elixir_aliases:concat([H | T]); +no_alias_expansion(Other) -> + Other. + +should_warn(_Meta, _Opts, #{lexical_tracker := nil}) -> + false; +should_warn(Meta, Opts, #{lexical_tracker := Pid}) -> + case lists:keyfind(warn, 1, Opts) of + {warn, false} -> false; + {warn, true} -> Pid; + false -> + case lists:keymember(context, 1, Meta) of + true -> false; + false -> Pid + end + end. + +%% Aliases + +alias(Meta, Ref, IncludeByDefault, Opts, E) -> + case elixir_aliases:alias(Meta, Ref, IncludeByDefault, Opts, E, true) of + {ok, New, EA} -> {ok, New, EA}; + {error, Reason} -> elixir_errors:file_error(Meta, E, elixir_aliases, Reason) + end. + +expand_without_aliases_report({'__aliases__', _, _} = Alias, S, E) -> + expand_aliases(Alias, S, E, false); +expand_without_aliases_report(Other, S, E) -> + expand(Other, S, E). + +expand_aliases({'__aliases__', Meta, List} = Alias, S, E, Report) -> + case elixir_aliases:expand_or_concat(Meta, List, E, true) of + Receiver when is_atom(Receiver) -> + if + Receiver =:= 'Elixir.True'; Receiver =:= 'Elixir.False'; Receiver =:= 'Elixir.Nil' -> + elixir_errors:file_warn(Meta, E, ?MODULE, {commonly_mistaken_alias, Receiver}); + true -> + ok + end, + Report andalso elixir_env:trace({alias_reference, Meta, Receiver}, E), + {Receiver, S, E}; + + [Head | Tail] -> + {EHead, SA, EA} = expand(Head, S, E), + + case is_atom(EHead) of + true -> + Receiver = elixir_aliases:concat([EHead | Tail]), + Report andalso elixir_env:trace({alias_reference, Meta, Receiver}, E), + {Receiver, SA, EA}; + + false -> + file_error(Meta, E, ?MODULE, {invalid_alias, Alias}) + end + end. + +%% Comprehensions + +expand_for_generator({'<-', Meta, [Left, Right]}, S, E) -> + {ERight, SR, ER} = expand(Right, S, E), + SM = elixir_env:reset_read(SR, S), + {[ELeft], SL, EL} = elixir_clauses:head(Meta, [Left], SM, ER), + {{'<-', Meta, [ELeft, ERight]}, SL, EL}; +expand_for_generator({'<<>>', Meta, Args} = X, S, E) when is_list(Args) -> + case elixir_utils:split_last(Args) of + {LeftStart, {'<-', OpMeta, [LeftEnd, Right]}} -> + {ERight, SR, ER} = expand(Right, S, E), + SM = elixir_env:reset_read(SR, S), + {ELeft, SL, EL} = elixir_clauses:match(fun(BArg, BS, BE) -> + elixir_bitstring:expand(Meta, BArg, BS, BE, true) + end, Meta, LeftStart ++ [LeftEnd], SM, SM, ER), + {{'<<>>', Meta, [{'<-', OpMeta, [ELeft, ERight]}]}, SL, EL}; + _ -> + expand(X, S, E) + end; +expand_for_generator(X, S, E) -> + expand(X, S, E). + +assert_generator_start(_, [{'<-', _, [_, _]} | _], _) -> + ok; +assert_generator_start(_, [{'<<>>', _, [{'<-', _, [_, _]}]} | _], _) -> + ok; +assert_generator_start(Meta, _, E) -> + elixir_errors:file_error(Meta, E, ?MODULE, for_generator_start). + +%% Assertions + +assert_module_scope(Meta, Kind, #{module := nil, file := File}) -> + file_error(Meta, File, ?MODULE, {invalid_expr_in_scope, "module", Kind}); +assert_module_scope(_Meta, _Kind, #{module:=Module}) -> Module. + +assert_function_scope(Meta, Kind, #{function := nil, file := File}) -> + file_error(Meta, File, ?MODULE, {invalid_expr_in_scope, "function", Kind}); +assert_function_scope(_Meta, _Kind, #{function := Function}) -> Function. + +assert_no_match_or_guard_scope(Meta, Kind, S, E) -> + assert_no_match_scope(Meta, Kind, E), + assert_no_guard_scope(Meta, Kind, S, E). +assert_no_match_scope(Meta, Kind, #{context := match, file := File}) -> + file_error(Meta, File, ?MODULE, {invalid_pattern_in_match, Kind}); +assert_no_match_scope(_Meta, _Kind, _E) -> ok. +assert_no_guard_scope(Meta, Kind, S, #{context := guard, file := File}) -> + Key = + case S of + #elixir_ex{prematch={_, _, {bitsize, _}}} -> invalid_expr_in_bitsize; + _ -> invalid_expr_in_guard + end, + file_error(Meta, File, ?MODULE, {Key, Kind}); +assert_no_guard_scope(_Meta, _Kind, _S, _E) -> ok. + +%% Here we look into the Clauses "optimistically", that is, we don't check for +%% multiple "do"s and similar stuff. After all, the error we're gonna give here +%% is just a friendlier version of the "undefined variable _" error that we +%% would raise if we found a "_ -> ..." clause in a "cond". For this reason, if +%% Clauses has a bad shape, we just do nothing and let future functions catch +%% this. +assert_no_underscore_clause_in_cond([{do, Clauses}], E) when is_list(Clauses) -> + case lists:last(Clauses) of + {'->', Meta, [[{'_', _, Atom}], _]} when is_atom(Atom) -> + file_error(Meta, E, ?MODULE, underscore_in_cond); + _Other -> + ok + end; +assert_no_underscore_clause_in_cond(_Other, _E) -> + ok. + +%% Errors + +format_error(invalid_match_on_zero_float) -> + "pattern matching on 0.0 is equivalent to matching only on +0.0 from Erlang/OTP 27+. Instead you must match on +0.0 or -0.0"; +format_error({useless_literal, Term}) -> + io_lib:format("code block contains unused literal ~ts " + "(remove the literal or assign it to _ to avoid warnings)", + ['Elixir.Macro':to_string(Term)]); +format_error({useless_var, Var}) -> + io_lib:format("variable ~ts in code block has no effect as it is never returned " + "(remove the variable or assign it to _ to avoid warnings)", + [Var]); +format_error({useless_attr, Attr}) -> + io_lib:format("module attribute @~ts in code block has no effect as it is never returned " + "(remove the attribute or assign it to _ to avoid warnings)", + [Attr]); +format_error({missing_option, Construct, Opts}) when is_list(Opts) -> + StringOpts = lists:map(fun(Opt) -> [$: | atom_to_list(Opt)] end, Opts), + io_lib:format("missing ~ts option in \"~ts\"", [string:join(StringOpts, "/"), Construct]); +format_error({invalid_args, Construct}) -> + io_lib:format("invalid arguments for \"~ts\"", [Construct]); +format_error({for_invalid_uniq, Value}) -> + io_lib:format(":uniq option for comprehensions only accepts a boolean, got: ~ts", ['Elixir.Macro':to_string(Value)]); +format_error(for_conflicting_reduce_into_uniq) -> + "cannot use :reduce alongside :into/:uniq in comprehension"; +format_error(for_with_reduce_bad_block) -> + "when using :reduce with comprehensions, the do block must be written using acc -> expr clauses, where each clause expects the accumulator as a single argument"; +format_error(for_without_reduce_bad_block) -> + "the do block was written using acc -> expr clauses but the :reduce option was not given"; +format_error(for_generator_start) -> + "for comprehensions must start with a generator"; +format_error(for_with_unused_uniq) -> + "the :uniq option has no effect since the result of the for comprehension is not used"; +format_error(unhandled_arrow_op) -> + "misplaced operator ->\n\n" + "This typically means invalid syntax or a macro is not available in scope"; +format_error(unhandled_cons_op) -> + "misplaced operator |/2\n\n" + "The | operator is typically used between brackets to mark the tail of a list:\n\n" + " [head | tail]\n" + " [head, middle, ... | tail]\n\n" + "It is also used to update maps and structs, via the %{map | key: value} notation, " + "and in typespecs, such as @type and @spec, to express the union of two types"; +format_error(unhandled_type_op) -> + "misplaced operator ::/2\n\n" + "The :: operator is typically used in bitstrings to specify types and sizes of segments:\n\n" + " <>\n\n" + "It is also used in typespecs, such as @type and @spec, to describe inputs and outputs"; +format_error(as_in_multi_alias_call) -> + ":as option is not supported by multi-alias call"; +format_error({commonly_mistaken_alias, Ref}) -> + Module = 'Elixir.Macro':to_string(Ref), + io_lib:format("reserved alias \"~ts\" expands to the atom :\"Elixir.~ts\". Perhaps you meant to write \"~ts\" instead?", [Module, Module, string:casefold(Module)]); +format_error({expected_compile_time_module, Kind, GivenTerm}) -> + io_lib:format("invalid argument for ~ts, expected a compile time atom or alias, got: ~ts", + [Kind, 'Elixir.Macro':to_string(GivenTerm)]); +format_error({unquote_outside_quote, Unquote}) -> + %% Unquote can be "unquote" or "unquote_splicing". + io_lib:format("~p called outside quote", [Unquote]); +format_error({invalid_bind_quoted_for_quote, BQ}) -> + io_lib:format("invalid :bind_quoted for quote, expected a keyword list of variable names, got: ~ts", + ['Elixir.Macro':to_string(BQ)]); +format_error(wrong_number_of_args_for_super) -> + "super must be called with the same number of arguments as the current definition"; +format_error({invalid_arg_for_pin, Arg}) -> + io_lib:format("invalid argument for unary operator ^, expected an existing variable, got: ^~ts", + ['Elixir.Macro':to_string(Arg)]); +format_error({pin_outside_of_match, Arg}) -> + io_lib:format( + "misplaced operator ^~ts\n\n" + "The pin operator ^ is supported only inside matches or inside custom macros. " + "Make sure you are inside a match or all necessary macros have been required", + ['Elixir.Macro':to_string(Arg)] + ); +format_error(unbound_underscore) -> + "invalid use of _. _ can only be used inside patterns to ignore values and cannot be used in expressions. Make sure you are inside a pattern or change it accordingly"; +format_error({undefined_var, Name, Kind}) -> + io_lib:format("undefined variable ~ts", [elixir_utils:var_info(Name, Kind)]); +format_error({undefined_var_pin, Name, Kind}) -> + Message = "undefined variable ^~ts. No variable ~ts has been defined before the current pattern", + io_lib:format(Message, [Name, elixir_utils:var_info(Name, Kind)]); +format_error(underscore_in_cond) -> + "invalid use of _ inside \"cond\". If you want the last clause to always match, " + "you probably meant to use: true ->"; +format_error({invalid_pattern_in_match, Kind}) -> + io_lib:format("invalid pattern in match, ~ts is not allowed in matches", [Kind]); +format_error({invalid_expr_in_scope, Scope, Kind}) -> + io_lib:format("cannot invoke ~ts outside ~ts", [Kind, Scope]); +format_error({invalid_expr_in_guard, Kind}) -> + Message = + "invalid expression in guards, ~ts is not allowed in guards. To learn more about " + "guards, visit: https://hexdocs.pm/elixir/patterns-and-guards.html#guards", + io_lib:format(Message, [Kind]); +format_error({invalid_expr_in_bitsize, Kind}) -> + Message = + "~ts is not allowed inside a bitstring size specifier. The size specifier in matches works like guards. " + "To learn more about guards, visit: https://hexdocs.pm/elixir/patterns-and-guards.html#guards", + io_lib:format(Message, [Kind]); +format_error({invalid_alias, Expr}) -> + Message = + "invalid alias: \"~ts\". If you wanted to define an alias, an alias must expand " + "to an atom at compile time but it did not, you may use Module.concat/2 to build " + "it at runtime. If instead you wanted to invoke a function or access a field, " + "wrap the function or field name in double quotes", + io_lib:format(Message, ['Elixir.Macro':to_string(Expr)]); +format_error({op_ambiguity, Name, Arg}) -> + NameString = atom_to_binary(Name), + ArgString = 'Elixir.Macro':to_string(Arg), + + Message = + "\"~ts ~ts\" looks like a function call but there is a variable named \"~ts\". " + "If you want to perform a function call, use parentheses:\n" + "\n" + " ~ts(~ts)\n" + "\n" + "If you want to perform an operation on the variable ~ts, use spaces " + "around the unary operator", + io_lib:format(Message, [NameString, ArgString, NameString, NameString, ArgString, NameString]); +format_error({invalid_clauses, Name}) -> + Message = + "the function \"~ts\" cannot handle clauses with the -> operator because it is not a macro. " + "Please make sure you are invoking the proper name and that it is a macro", + io_lib:format(Message, [Name]); +format_error({invalid_call, Call}) -> + io_lib:format("invalid call ~ts", ['Elixir.Macro':to_string(Call)]); +format_error({invalid_quoted_expr, Expr}) -> + Message = + "invalid quoted expression: ~ts\n\n" + "Please make sure your quoted expressions are made of valid AST nodes. " + "If you would like to introduce a value into the AST, such as a four-element " + "tuple or a map, make sure to call Macro.escape/1 before", + io_lib:format(Message, ['Elixir.Kernel':inspect(Expr, [])]); +format_error({invalid_local_invocation, Context, {Name, _, Args} = Call}) -> + Message = + "cannot find or invoke local ~ts/~B inside a ~ts. " + "Only macros can be invoked inside a ~ts and they must be defined before their invocation. Called as: ~ts", + io_lib:format(Message, [Name, length(Args), Context, Context, 'Elixir.Macro':to_string(Call)]); +format_error({invalid_pid_in_function, Pid, {Name, Arity}}) -> + io_lib:format("cannot compile PID ~ts inside quoted expression for function ~ts/~B", + ['Elixir.Kernel':inspect(Pid, []), Name, Arity]); +format_error({unsupported_option, Kind, Key}) -> + io_lib:format("unsupported option ~ts given to ~s", + ['Elixir.Macro':to_string(Key), Kind]); +format_error({options_are_not_keyword, Kind, Opts}) -> + io_lib:format("invalid options for ~s, expected a keyword list, got: ~ts", + [Kind, 'Elixir.Macro':to_string(Opts)]); +format_error({undefined_function, Name, Args}) -> + io_lib:format("undefined function ~ts/~B (there is no such import)", [Name, length(Args)]); +format_error({unpinned_bitsize_var, Name, Kind}) -> + io_lib:format("the variable ~ts is accessed inside size(...) of a bitstring " + "but it was defined outside of the match. You must precede it with the " + "pin operator", [elixir_utils:var_info(Name, Kind)]); +format_error({underscored_var_repeat, Name, Kind}) -> + io_lib:format("the underscored variable ~ts appears more than once in a " + "match. This means the pattern will only match if all \"~ts\" bind " + "to the same value. If this is the intended behaviour, please " + "remove the leading underscore from the variable name, otherwise " + "give the variables different names", [elixir_utils:var_info(Name, Kind), Name]); +format_error({underscored_var_access, Name}) -> + io_lib:format("the underscored variable \"~ts\" is used after being set. " + "A leading underscore indicates that the value of the variable " + "should be ignored. If this is intended please rename the " + "variable to remove the underscore", [Name]); +format_error({nested_comparison, CompExpr}) -> + String = 'Elixir.Macro':to_string(CompExpr), + io_lib:format("Elixir does not support nested comparisons. Something like\n\n" + " x < y < z\n\n" + "is equivalent to\n\n" + " (x < y) < z\n\n" + "which ultimately compares z with the boolean result of (x < y). " + "Instead, consider joining together each comparison segment with an \"and\", for example,\n\n" + " x < y and y < z\n\n" + "You wrote: ~ts", [String]); +format_error({undefined_local_capture, Fun, Arity}) -> + io_lib:format("undefined function ~ts/~B (there is no such import)", [Fun, Arity]); +format_error(caller_not_allowed) -> + "__CALLER__ is available only inside defmacro and defmacrop"; +format_error(stacktrace_not_allowed) -> + "__STACKTRACE__ is available only inside catch and rescue clauses of try expressions"; +format_error({undefined_var_to_call, Name}) -> + io_lib:format("variable \"~ts\" does not exist and is being expanded to \"~ts()\"," + " please use parentheses to remove the ambiguity or change the variable name", [Name, Name]); +format_error({parens_map_lookup, Map, Field, Context}) -> + io_lib:format("cannot invoke remote function inside a ~ts. " + "If you want to do a map lookup instead, please remove parens from ~ts.~ts()", + [Context, 'Elixir.Macro':to_string(Map), Field]); +format_error({super_in_genserver, {Name, Arity}}) -> + io_lib:format("calling super for GenServer callback ~ts/~B is deprecated", [Name, Arity]); +format_error('__cursor__') -> + "reserved special form __cursor__ cannot be expanded, it is used exclusively to annotate ASTs". \ No newline at end of file diff --git a/lib/elixir/src/elixir_fn.erl b/lib/elixir/src/elixir_fn.erl index d51d5702818..9a2a436d67f 100644 --- a/lib/elixir/src/elixir_fn.erl +++ b/lib/elixir/src/elixir_fn.erl @@ -1,164 +1,231 @@ +%% SPDX-License-Identifier: Apache-2.0 +%% SPDX-FileCopyrightText: 2021 The Elixir Team +%% SPDX-FileCopyrightText: 2012 Plataformatec + -module(elixir_fn). --export([translate/3, capture/3, expand/3]). --import(elixir_errors, [compile_error/3, compile_error/4]). +-export([capture/4, expand/4, format_error/1]). +-import(elixir_errors, [file_error/4]). -include("elixir.hrl"). -translate(Meta, Clauses, S) -> - Transformer = fun({'->', CMeta, [ArgsWithGuards, Expr]}, Acc) -> - {Args, Guards} = elixir_clauses:extract_splat_guards(ArgsWithGuards), - {TClause, TS } = elixir_clauses:clause(?line(CMeta), fun translate_fn_match/2, - Args, Expr, Guards, true, Acc), - {TClause, elixir_scope:mergef(S, TS)} +%% Anonymous functions + +expand(Meta, Clauses, S, E) when is_list(Clauses) -> + Transformer = fun({_, _, [Left, _Right]} = Clause, SA) -> + case lists:any(fun is_invalid_arg/1, Left) of + true -> + file_error(Meta, E, ?MODULE, defaults_in_args); + false -> + SReset = elixir_env:reset_unused_vars(SA), + + {EClause, SAcc, EAcc} = + elixir_clauses:clause(Meta, fn, fun elixir_clauses:head/4, Clause, SReset, E), + + {EClause, elixir_env:merge_and_check_unused_vars(SAcc, SA, EAcc)} + end end, - {TClauses, NS} = lists:mapfoldl(Transformer, S, Clauses), - Arities = [length(Args) || {clause, _Line, Args, _Guards, _Exprs} <- TClauses], + {EClauses, SE} = lists:mapfoldl(Transformer, S, Clauses), + EArities = [fn_arity(Args) || {'->', _, [Args, _]} <- EClauses], - case lists:usort(Arities) of + case lists:usort(EArities) of [_] -> - {{'fun', ?line(Meta), {clauses, TClauses}}, NS}; + {{fn, Meta, EClauses}, SE, E}; _ -> - compile_error(Meta, S#elixir_scope.file, - "cannot mix clauses with different arities in function definition") + file_error(Meta, E, ?MODULE, clauses_with_different_arities) end. -translate_fn_match(Arg, S) -> - {TArg, TS} = elixir_translator:translate_args(Arg, S#elixir_scope{backup_vars=orddict:new()}), - {TArg, TS#elixir_scope{backup_vars=S#elixir_scope.backup_vars}}. - -%% Expansion +is_invalid_arg({'\\\\', _, _}) -> true; +is_invalid_arg(_) -> false. -expand(Meta, Clauses, E) when is_list(Clauses) -> - Transformer = fun(Clause) -> - {EClause, _} = elixir_exp_clauses:clause(Meta, fn, fun elixir_exp_clauses:head/2, Clause, E), - EClause - end, - {{fn, Meta, lists:map(Transformer, Clauses)}, E}. +fn_arity([{'when', _, Args}]) -> length(Args) - 1; +fn_arity(Args) -> length(Args). %% Capture -capture(Meta, {'/', _, [{{'.', _, [_, F]} = Dot, RequireMeta , []}, A]}, E) when is_atom(F), is_integer(A) -> - Args = [{'&', [], [X]} || X <- lists:seq(1, A)], - capture_require(Meta, {Dot, RequireMeta, Args}, E, true); +capture(Meta, {'/', _, [{{'.', _, [M, F]} = Dot, RequireMeta, []}, A]}, S, E) when is_atom(F), is_integer(A) -> + Args = args_from_arity(Meta, A, E), + handle_capture_possible_warning(Meta, RequireMeta, M, F, A, E), + capture_require({Dot, RequireMeta, Args}, S, E, arity); -capture(Meta, {'/', _, [{F, _, C}, A]}, E) when is_atom(F), is_integer(A), is_atom(C) -> - ImportMeta = - case lists:keyfind(import_fa, 1, Meta) of - {import_fa, {Receiver, Context}} -> - lists:keystore(context, 1, - lists:keystore(import, 1, Meta, {import, Receiver}), - {context, Context} - ); - false -> Meta - end, - Args = [{'&', [], [X]} || X <- lists:seq(1, A)], - capture_import(Meta, {F, ImportMeta, Args}, E, true); +capture(Meta, {'/', _, [{F, ImportMeta, C}, A]}, S, E) when is_atom(F), is_integer(A), is_atom(C) -> + Args = args_from_arity(Meta, A, E), + capture_import({F, ImportMeta, Args}, S, E, arity); -capture(Meta, {{'.', _, [_, Fun]}, _, Args} = Expr, E) when is_atom(Fun), is_list(Args) -> - capture_require(Meta, Expr, E, is_sequential_and_not_empty(Args)); +capture(_Meta, {{'.', _, [_, Fun]}, _, Args} = Expr, S, E) when is_atom(Fun), is_list(Args) -> + capture_require(Expr, S, E, check_sequential_and_not_empty(Args)); -capture(Meta, {{'.', _, [_]}, _, Args} = Expr, E) when is_list(Args) -> - do_capture(Meta, Expr, E, false); +capture(Meta, {{'.', _, [_]}, _, Args} = Expr, S, E) when is_list(Args) -> + capture_expr(Meta, Expr, S, E, non_sequential); -capture(Meta, {'__block__', _, [Expr]}, E) -> - capture(Meta, Expr, E); +capture(Meta, {'__block__', _, [Expr]}, S, E) -> + capture(Meta, Expr, S, E); -capture(Meta, {'__block__', _, _} = Expr, E) -> - Message = "invalid args for &, block expressions are not allowed, got: ~ts", - compile_error(Meta, ?m(E, file), Message, ['Elixir.Macro':to_string(Expr)]); +capture(Meta, {'__block__', _, _} = Expr, _S, E) -> + file_error(Meta, E, ?MODULE, {block_expr_in_capture, Expr}); -capture(Meta, {Atom, _, Args} = Expr, E) when is_atom(Atom), is_list(Args) -> - capture_import(Meta, Expr, E, is_sequential_and_not_empty(Args)); +capture(_Meta, {Atom, _, Args} = Expr, S, E) when is_atom(Atom), is_list(Args) -> + capture_import(Expr, S, E, check_sequential_and_not_empty(Args)); -capture(Meta, {Left, Right}, E) -> - capture(Meta, {'{}', Meta, [Left, Right]}, E); +capture(Meta, {Left, Right}, S, E) -> + capture(Meta, {'{}', Meta, [Left, Right]}, S, E); -capture(Meta, List, E) when is_list(List) -> - do_capture(Meta, List, E, is_sequential_and_not_empty(List)); +capture(Meta, List, S, E) when is_list(List) -> + capture_expr(Meta, List, S, E, check_sequential_and_not_empty(List)); -capture(Meta, Arg, E) -> +capture(Meta, Integer, _S, E) when is_integer(Integer) -> + file_error(Meta, E, ?MODULE, {capture_arg_outside_of_capture, Integer}); + +capture(Meta, Arg, _S, E) -> invalid_capture(Meta, Arg, E). -capture_import(Meta, {Atom, ImportMeta, Args} = Expr, E, Sequential) -> - Res = Sequential andalso +capture_import({Atom, ImportMeta, Args} = Expr, S, E, ArgsType) -> + Res = ArgsType /= non_sequential andalso elixir_dispatch:import_function(ImportMeta, Atom, length(Args), E), - handle_capture(Res, Meta, Expr, E, Sequential). - -capture_require(Meta, {{'.', _, [Left, Right]}, RequireMeta, Args} = Expr, E, Sequential) -> - {Mod, EE} = elixir_exp:expand(Left, E), - Res = Sequential andalso case Mod of - {Name, _, Context} when is_atom(Name), is_atom(Context) -> - {remote, Mod, Right, length(Args)}; - _ when is_atom(Mod) -> - elixir_dispatch:require_function(RequireMeta, Mod, Right, length(Args), EE); - _ -> - false - end, - handle_capture(Res, Meta, Expr, EE, Sequential). - -handle_capture({local, Fun, Arity}, _Meta, _Expr, _E, _Sequential) -> - {local, Fun, Arity}; -handle_capture({remote, Receiver, Fun, Arity}, Meta, _Expr, E, _Sequential) -> - Tree = {{'.', [], [erlang, make_fun]}, Meta, [Receiver, Fun, Arity]}, - {expanded, Tree, E}; -handle_capture(false, Meta, Expr, E, Sequential) -> - do_capture(Meta, Expr, E, Sequential). - -do_capture(Meta, Expr, E, Sequential) -> - case do_escape(Expr, elixir_counter:next(), E, []) of - {_, []} when not Sequential -> + handle_capture(Res, ImportMeta, ImportMeta, Expr, S, E, ArgsType). + +capture_require({{'.', DotMeta, [Left, Right]}, RequireMeta, Args}, S, E, ArgsType) -> + case escape(Left, E, []) of + {EscLeft, []} -> + {ELeft, SE, EE} = elixir_expand:expand(EscLeft, S, E), + + case ELeft of + _ when ArgsType /= arity -> ok; + Atom when is_atom(Atom) -> ok; + {Var, _, Ctx} when is_atom(Var), is_atom(Ctx) -> ok; + %% TODO: Raise on Elixir v2.0 + _ -> elixir_errors:file_warn(RequireMeta, E, ?MODULE, {complex_module_capture, Left}) + end, + + Res = ArgsType /= non_sequential andalso case ELeft of + {Name, _, Context} when is_atom(Name), is_atom(Context) -> + {remote, ELeft, Right, length(Args)}; + _ when is_atom(ELeft) -> + elixir_dispatch:require_function(RequireMeta, ELeft, Right, length(Args), EE); + _ -> + false + end, + + Dot = {{'.', DotMeta, [ELeft, Right]}, RequireMeta, Args}, + handle_capture(Res, RequireMeta, DotMeta, Dot, SE, EE, ArgsType); + + {EscLeft, Escaped} -> + Dot = {{'.', DotMeta, [EscLeft, Right]}, RequireMeta, Args}, + capture_expr(RequireMeta, Dot, S, E, Escaped, ArgsType) + end. + +handle_capture(false, Meta, _DotMeta, Expr, S, E, ArgsType) -> + capture_expr(Meta, Expr, S, E, ArgsType); +handle_capture(LocalOrRemote, Meta, DotMeta, _Expr, S, E, _ArgsType) -> + {LocalOrRemote, Meta, DotMeta, S, E}. + +capture_expr(Meta, Expr, S, E, ArgsType) -> + capture_expr(Meta, Expr, S, E, [], ArgsType). +capture_expr(Meta, Expr, S, E, Escaped, ArgsType) -> + case escape(Expr, E, Escaped) of + {_, []} when ArgsType == non_sequential -> invalid_capture(Meta, Expr, E); + %% TODO: Remove this clause once we raise on complex module captures like &get_mod().fun/0 + {{{'.', _, [_, _]} = Dot, _, Args}, []} -> + Meta2 = lists:keydelete(no_parens, 1, Meta), + Fn = {fn, Meta2, [{'->', Meta2, [[], {Dot, Meta2, Args}]}]}, + {expand, Fn, S, E}; {EExpr, EDict} -> EVars = validate(Meta, EDict, 1, E), - Fn = {fn, Meta, [{'->', Meta, [EVars, EExpr]}]}, - {expanded, Fn, E} + Fn = {fn, [{capture, true} | Meta], [{'->', Meta, [EVars, EExpr]}]}, + {expand, Fn, S, E} end. invalid_capture(Meta, Arg, E) -> - Message = "invalid args for &, expected an expression in the format of &Mod.fun/arity, " - "&local/arity or a capture containing at least one argument as &1, got: ~ts", - compile_error(Meta, ?m(E, file), Message, ['Elixir.Macro':to_string(Arg)]). - -validate(Meta, [{Pos, Var}|T], Pos, E) -> - [Var|validate(Meta, T, Pos + 1, E)]; - -validate(Meta, [{Pos, _}|_], Expected, E) -> - compile_error(Meta, ?m(E, file), "capture &~B cannot be defined without &~B", [Pos, Expected]); + file_error(Meta, E, ?MODULE, {invalid_args_for_capture, Arg}). +validate(Meta, [{Pos, Var} | T], Pos, E) -> + [Var | validate(Meta, T, Pos + 1, E)]; +validate(Meta, [{Pos, _} | _], Expected, E) -> + file_error(Meta, E, ?MODULE, {capture_arg_without_predecessor, Pos, Expected}); validate(_Meta, [], _Pos, _E) -> []. -do_escape({'&', _, [Pos]}, Counter, _E, Dict) when is_integer(Pos), Pos > 0 -> - Var = {list_to_atom([$x, $@+Pos]), [{counter, Counter}], elixir_fn}, - {Var, orddict:store(Pos, Var, Dict)}; - -do_escape({'&', Meta, [Pos]}, _Counter, E, _Dict) when is_integer(Pos) -> - compile_error(Meta, ?m(E, file), "capture &~B is not allowed", [Pos]); - -do_escape({'&', Meta, _} = Arg, _Counter, E, _Dict) -> - Message = "nested captures via & are not allowed: ~ts", - compile_error(Meta, ?m(E, file), Message, ['Elixir.Macro':to_string(Arg)]); - -do_escape({Left, Meta, Right}, Counter, E, Dict0) -> - {TLeft, Dict1} = do_escape(Left, Counter, E, Dict0), - {TRight, Dict2} = do_escape(Right, Counter, E, Dict1), +escape({'&', Meta, [Pos]}, E, Dict) when is_integer(Pos), Pos > 0 -> + % Using a nil context here to emit warnings when variable is unused. + % This might pollute user space but is unlikely because variables + % named :"&1" are not valid syntax. + case orddict:find(Pos, Dict) of + {ok, Var} -> + {Var, Dict}; + error -> + Next = elixir_module:next_counter(?key(E, module)), + Var = {capture, [{counter, Next}, {capture, Pos} | Meta], nil}, + {Var, orddict:store(Pos, Var, Dict)} + end; +escape({'&', Meta, [Pos]}, E, _Dict) when is_integer(Pos) -> + file_error(Meta, E, ?MODULE, {invalid_arity_for_capture, Pos}); +escape({'&', Meta, _} = Arg, E, _Dict) -> + file_error(Meta, E, ?MODULE, {nested_capture, Arg}); +escape({Left, Meta, Right}, E, Dict0) -> + {TLeft, Dict1} = escape(Left, E, Dict0), + {TRight, Dict2} = escape(Right, E, Dict1), {{TLeft, Meta, TRight}, Dict2}; - -do_escape({Left, Right}, Counter, E, Dict0) -> - {TLeft, Dict1} = do_escape(Left, Counter, E, Dict0), - {TRight, Dict2} = do_escape(Right, Counter, E, Dict1), +escape({Left, Right}, E, Dict0) -> + {TLeft, Dict1} = escape(Left, E, Dict0), + {TRight, Dict2} = escape(Right, E, Dict1), {{TLeft, TRight}, Dict2}; +escape(List, E, Dict) when is_list(List) -> + lists:mapfoldl(fun(X, Acc) -> escape(X, E, Acc) end, Dict, List); +escape(Other, _E, Dict) -> + {Other, Dict}. -do_escape(List, Counter, E, Dict) when is_list(List) -> - lists:mapfoldl(fun(X, Acc) -> do_escape(X, Counter, E, Acc) end, Dict, List); +args_from_arity(_Meta, A, _E) when is_integer(A), A >= 0, A =< 255 -> + [{'&', [], [X]} || X <- lists:seq(1, A)]; +args_from_arity(Meta, A, E) -> + file_error(Meta, E, ?MODULE, {invalid_arity_for_capture, A}). -do_escape(Other, _Counter, _E, Dict) -> - {Other, Dict}. +check_sequential_and_not_empty([]) -> non_sequential; +check_sequential_and_not_empty(List) -> check_sequential(List, 1). + +check_sequential([{'&', _, [Int]} | T], Int) -> check_sequential(T, Int + 1); +check_sequential([], _Int) -> sequential; +check_sequential(_, _Int) -> non_sequential. -is_sequential_and_not_empty([]) -> false; -is_sequential_and_not_empty(List) -> is_sequential(List, 1). +handle_capture_possible_warning(Meta, DotMeta, Mod, Fun, Arity, E) -> + case (Arity =:= 0) andalso (lists:keyfind(no_parens, 1, DotMeta) /= {no_parens, true}) of + true -> + elixir_errors:file_warn(Meta, E, ?MODULE, {parens_remote_capture, Mod, Fun}); + + false -> ok + end. -is_sequential([{'&', _, [Int]}|T], Int) -> - is_sequential(T, Int + 1); -is_sequential([], _Int) -> true; -is_sequential(_, _Int) -> false. +%% TODO: Raise on Elixir v2.0 +format_error({parens_remote_capture, Mod, Fun}) -> + io_lib:format("extra parentheses on a remote function capture &~ts.~ts()/0 have been " + "deprecated. Please remove the parentheses: &~ts.~ts/0", + ['Elixir.Macro':to_string(Mod), Fun, 'Elixir.Macro':to_string(Mod), Fun]); +format_error({complex_module_capture, Mod}) -> + io_lib:format("expected the module in &module.fun/arity to expand to a variable or an atom, got: ~ts\n" + "You can either compute the module name outside of & or convert it to a regular anonymous function.", + ['Elixir.Macro':to_string(Mod)]); +format_error(clauses_with_different_arities) -> + "cannot mix clauses with different arities in anonymous functions"; +format_error(defaults_in_args) -> + "anonymous functions cannot have optional arguments"; +format_error({block_expr_in_capture, Expr}) -> + io_lib:format("block expressions are not allowed inside the capture operator &, got: ~ts", + ['Elixir.Macro':to_string(Expr)]); +format_error({nested_capture, Arg}) -> + io_lib:format("nested captures are not allowed. You cannot define a function using " + "the capture operator & inside another function defined via &. Got invalid nested " + "capture: ~ts", ['Elixir.Macro':to_string(Arg)]); +format_error({invalid_arity_for_capture, Arity}) -> + io_lib:format("capture argument &~B must be numbered between 1 and 255", [Arity]); +format_error({capture_arg_outside_of_capture, Integer}) -> + io_lib:format("capture argument &~B must be used within the capture operator &", [Integer]); +format_error({capture_arg_without_predecessor, Pos, Expected}) -> + io_lib:format("capture argument &~B cannot be defined without &~B " + "(you cannot skip arguments, all arguments must be numbered)", [Pos, Expected]); +format_error({invalid_args_for_capture, Arg}) -> + Message = + "invalid args for &, expected one of:\n\n" + " * &Mod.fun/arity to capture a remote function, such as &Enum.map/2\n" + " * &fun/arity to capture a local or imported function, such as &is_atom/1\n" + " * &some_code(&1, ...) containing at least one argument as &1, such as &List.flatten(&1)\n\n" + "Got: ~ts", + io_lib:format(Message, ['Elixir.Macro':to_string(Arg)]). diff --git a/lib/elixir/src/elixir_for.erl b/lib/elixir/src/elixir_for.erl deleted file mode 100644 index d017f6db4d0..00000000000 --- a/lib/elixir/src/elixir_for.erl +++ /dev/null @@ -1,338 +0,0 @@ --module(elixir_for). --export([expand/3, translate/3]). --include("elixir.hrl"). - -%% Expansion - -expand(Meta, Args, E) -> - {Cases, Block} = - case elixir_utils:split_last(Args) of - {OuterCases, OuterOpts} when is_list(OuterOpts) -> - case elixir_utils:split_last(OuterCases) of - {InnerCases, InnerOpts} when is_list(InnerOpts) -> - {InnerCases, InnerOpts ++ OuterOpts}; - _ -> - {OuterCases, OuterOpts} - end; - _ -> - {Args, []} - end, - - {Expr, Opts} = - case lists:keyfind(do, 1, Block) of - {do, Do} -> {Do, lists:keydelete(do, 1, Block)}; - _ -> elixir_errors:compile_error(Meta, ?m(E, file), - "missing do keyword in for comprehension") - end, - - {EOpts, EO} = elixir_exp:expand(Opts, E), - {ECases, EC} = lists:mapfoldl(fun expand/2, EO, Cases), - {EExpr, _} = elixir_exp:expand(Expr, EC), - {{for, Meta, ECases ++ [[{do,EExpr}|EOpts]]}, E}. - -expand({'<-', Meta, [Left, Right]}, E) -> - {ERight, ER} = elixir_exp:expand(Right, E), - {ELeft, EL} = elixir_exp_clauses:match(fun elixir_exp:expand/2, Left, E), - {{'<-', Meta, [ELeft, ERight]}, elixir_env:mergev(EL, ER)}; -expand({'<<>>', Meta, Args} = X, E) when is_list(Args) -> - case elixir_utils:split_last(Args) of - {LeftStart, {'<-', OpMeta, [LeftEnd, Right]}} -> - {ERight, ER} = elixir_exp:expand(Right, E), - Left = {'<<>>', Meta, LeftStart ++ [LeftEnd]}, - {ELeft, EL} = elixir_exp_clauses:match(fun elixir_exp:expand/2, Left, E), - {{'<<>>', [], [ {'<-', OpMeta, [ELeft, ERight]}]}, elixir_env:mergev(EL, ER)}; - _ -> - elixir_exp:expand(X, E) - end; -expand(X, E) -> - elixir_exp:expand(X, E). - -%% Translation - -translate(Meta, Args, #elixir_scope{return=Return} = RS) -> - S = RS#elixir_scope{return=true}, - {AccName, _, SA} = elixir_scope:build_var('_', S), - {VarName, _, SV} = elixir_scope:build_var('_', SA), - - Line = ?line(Meta), - Acc = {var, Line, AccName}, - Var = {var, Line, VarName}, - - {Cases, [{do,Expr}|Opts]} = elixir_utils:split_last(Args), - - {TInto, SI} = - case lists:keyfind(into, 1, Opts) of - {into, Into} -> elixir_translator:translate(Into, SV); - false when Return -> {{nil, Line}, SV}; - false -> {false, SV} - end, - - {TCases, SC} = translate_gen(Meta, Cases, [], SI), - {TExpr, SE} = elixir_translator:translate_block(Expr, Return, SC), - SF = elixir_scope:mergec(SI, SE), - - case comprehension_expr(TInto, TExpr) of - {inline, TIntoExpr} -> - {build_inline(Line, TCases, TIntoExpr, TInto, Var, Acc, SE), SF}; - {into, TIntoExpr} -> - build_into(Line, TCases, TIntoExpr, TInto, Var, Acc, SF) - end. - -translate_gen(ForMeta, [{'<-', Meta, [Left, Right]}|T], Acc, S) -> - {TLeft, TRight, TFilters, TT, TS} = translate_gen(Meta, Left, Right, T, S), - TAcc = [{enum, Meta, TLeft, TRight, TFilters}|Acc], - translate_gen(ForMeta, TT, TAcc, TS); -translate_gen(ForMeta, [{'<<>>', _, [ {'<-', Meta, [Left, Right]} ]}|T], Acc, S) -> - {TLeft, TRight, TFilters, TT, TS} = translate_gen(Meta, Left, Right, T, S), - TAcc = [{bin, Meta, TLeft, TRight, TFilters}|Acc], - case elixir_bitstring:has_size(TLeft) of - true -> translate_gen(ForMeta, TT, TAcc, TS); - false -> - elixir_errors:compile_error(Meta, S#elixir_scope.file, - "bitstring fields without size are not allowed in bitstring generators") - end; -translate_gen(_ForMeta, [], Acc, S) -> - {lists:reverse(Acc), S}; -translate_gen(ForMeta, _, _, S) -> - elixir_errors:compile_error(ForMeta, S#elixir_scope.file, - "for comprehensions must start with a generator"). - -translate_gen(_Meta, Left, Right, T, S) -> - {TRight, SR} = elixir_translator:translate(Right, S), - {TLeft, SL} = elixir_clauses:match(fun elixir_translator:translate/2, Left, SR), - {TT, {TFilters, TS}} = translate_filters(T, SL), - {TLeft, TRight, TFilters, TT, TS}. - -translate_filters(T, S) -> - {Filters, Rest} = collect_filters(T, []), - {Rest, lists:mapfoldr(fun translate_filter/2, S, Filters)}. - -translate_filter(Filter, S) -> - {TFilter, TS} = elixir_translator:translate(Filter, S), - case elixir_utils:returns_boolean(Filter) of - true -> - {{nil, TFilter}, TS}; - false -> - {Name, _, VS} = elixir_scope:build_var('_', TS), - {{{var, 0, Name}, TFilter}, VS} - end. - -collect_filters([{'<-', _, [_, _]}|_] = T, Acc) -> - {Acc, T}; -collect_filters([{'<<>>', _, [{'<-', _, [_, _]}]}|_] = T, Acc) -> - {Acc, T}; -collect_filters([H|T], Acc) -> - collect_filters(T, [H|Acc]); -collect_filters([], Acc) -> - {Acc, []}. - -%% If all we have is one enum generator, we check if it is a list -%% for optimization otherwise fallback to the reduce generator. -build_inline(Line, [{enum, Meta, Left, Right, Filters}] = Orig, Expr, Into, Var, Acc, S) -> - case Right of - {cons, _, _, _} -> - build_comprehension(Line, Orig, Expr, Into); - {Other, _, _} when Other == tuple; Other == map -> - build_reduce(Orig, Expr, Into, Acc, S); - _ -> - Clauses = [{enum, Meta, Left, Var, Filters}], - - {'case', -1, Right, [ - {clause, -1, - [Var], - [[elixir_utils:erl_call(Line, erlang, is_list, [Var])]], - [build_comprehension(Line, Clauses, Expr, Into)]}, - {clause, -1, - [Var], - [], - [build_reduce(Clauses, Expr, Into, Acc, S)]} - ]} - end; - -build_inline(Line, Clauses, Expr, Into, _Var, Acc, S) -> - case lists:all(fun(Clause) -> element(1, Clause) == bin end, Clauses) of - true -> build_comprehension(Line, Clauses, Expr, Into); - false -> build_reduce(Clauses, Expr, Into, Acc, S) - end. - -build_into(Line, Clauses, Expr, Into, Fun, Acc, S) -> - {Kind, SK} = build_var(Line, S), - {Reason, SR} = build_var(Line, SK), - {Stack, ST} = build_var(Line, SR), - {Done, SD} = build_var(Line, ST), - - IntoExpr = {call, Line, Fun, [Acc, pair(Line, cont, Expr)]}, - MatchExpr = {match, Line, - {tuple, Line, [Acc, Fun]}, - elixir_utils:erl_call(Line, 'Elixir.Collectable', into, [Into]) - }, - - TryExpr = - {'try', Line, - [build_reduce_clause(Clauses, IntoExpr, Acc, Acc, SD)], - [{clause, Line, - [Done], - [], - [{call, Line, Fun, [Done, {atom, Line, done}]}]}], - [{clause, Line, - [{tuple, Line, [Kind, Reason, {var, Line, '_'}]}], - [], - [{match, Line, Stack, elixir_utils:erl_call(Line, erlang, get_stacktrace, [])}, - {call, Line, Fun, [Acc, {atom, Line, halt}]}, - elixir_utils:erl_call(Line, erlang, raise, [Kind, Reason, Stack])]}], - []}, - - {{block, Line, [MatchExpr, TryExpr]}, SD}. - -%% Helpers - -build_reduce(Clauses, Expr, false, Acc, S) -> - build_reduce_clause(Clauses, Expr, {nil, 0}, Acc, S); -build_reduce(Clauses, Expr, {nil, Line} = Into, Acc, S) -> - ListExpr = {cons, Line, Expr, Acc}, - elixir_utils:erl_call(Line, lists, reverse, - [build_reduce_clause(Clauses, ListExpr, Into, Acc, S)]); -build_reduce(Clauses, Expr, {bin, _, _} = Into, Acc, S) -> - {bin, Line, Elements} = Expr, - BinExpr = {bin, Line, [{bin_element, Line, Acc, default, [bitstring]}|Elements]}, - build_reduce_clause(Clauses, BinExpr, Into, Acc, S). - -build_reduce_clause([{enum, Meta, Left, Right, Filters}|T], Expr, Arg, Acc, S) -> - Line = ?line(Meta), - Inner = build_reduce_clause(T, Expr, Acc, Acc, S), - - True = pair(Line, cont, Inner), - False = pair(Line, cont, Acc), - - Clauses0 = - case is_var(Left) of - true -> []; - false -> - [{clause, -1, - [{var, Line, '_'}, Acc], [], - [False]}] - end, - - Clauses1 = - [{clause, Line, - [Left, Acc], [], - [join_filters(Line, Filters, True, False)]}|Clauses0], - - Args = [Right, pair(Line, cont, Arg), {'fun', Line, {clauses, Clauses1}}], - Tuple = elixir_utils:erl_call(Line, 'Elixir.Enumerable', reduce, Args), - - %% Use -1 because in case of no returns we don't care about the result - elixir_utils:erl_call(-1, erlang, element, [{integer, Line, 2}, Tuple]); - -build_reduce_clause([{bin, Meta, Left, Right, Filters}|T], Expr, Arg, Acc, S) -> - Line = ?line(Meta), - {Tail, ST} = build_var(Line, S), - {Fun, SF} = build_var(Line, ST), - - True = build_reduce_clause(T, Expr, Acc, Acc, SF), - False = Acc, - - {bin, _, Elements} = Left, - - BinMatch = - {bin, Line, Elements ++ [{bin_element, Line, Tail, default, [bitstring]}]}, - NoVarMatch = - {bin, Line, no_var(Elements) ++ [{bin_element, Line, Tail, default, [bitstring]}]}, - - Clauses = - [{clause, Line, - [BinMatch, Acc], [], - [{call, Line, Fun, [Tail, join_filters(Line, Filters, True, False)]}]}, - {clause, -1, - [NoVarMatch, Acc], [], - [{call, Line, Fun, [Tail, False]}]}, - {clause, -1, - [{bin, Line, []}, Acc], [], - [Acc]}, - {clause, -1, - [Tail, {var, Line, '_'}], [], - [elixir_utils:erl_call(Line, erlang, error, [pair(Line, badarg, Tail)])]}], - - {call, Line, - {named_fun, Line, element(3, Fun), Clauses}, - [Right, Arg]}; - -build_reduce_clause([], Expr, _Arg, _Acc, _S) -> - Expr. - -is_var({var, _, _}) -> true; -is_var(_) -> false. - -pair(Line, Atom, Arg) -> - {tuple, Line, [{atom, Line, Atom}, Arg]}. - -build_var(Line, S) -> - {Name, _, ST} = elixir_scope:build_var('_', S), - {{var, Line, Name}, ST}. - -no_var(Elements) -> - [{bin_element, Line, no_var_expr(Expr), Size, Types} || - {bin_element, Line, Expr, Size, Types} <- Elements]. -no_var_expr({var, Line, _}) -> - {var, Line, '_'}. - -build_comprehension(Line, Clauses, Expr, false) -> - {block, Line, [ - build_comprehension(Line, Clauses, Expr, {nil, Line}), - {nil, Line} - ]}; -build_comprehension(Line, Clauses, Expr, Into) -> - {comprehension_kind(Into), Line, Expr, comprehension_clause(Clauses)}. - -comprehension_clause([{Kind, Meta, Left, Right, Filters}|T]) -> - Line = ?line(Meta), - [{comprehension_generator(Kind), Line, Left, Right}] ++ - comprehension_filter(Line, Filters) ++ - comprehension_clause(T); -comprehension_clause([]) -> - []. - -comprehension_kind({nil, _}) -> lc; -comprehension_kind({bin, _, []}) -> bc. - -comprehension_generator(enum) -> generate; -comprehension_generator(bin) -> b_generate. - -comprehension_expr({bin, _, []}, {bin, _, _} = Expr) -> - {inline, Expr}; -comprehension_expr({bin, Line, []}, Expr) -> - BinExpr = {bin, Line, [{bin_element, Line, Expr, default, [bitstring]}]}, - {inline, BinExpr}; -comprehension_expr({nil, _}, Expr) -> - {inline, Expr}; -comprehension_expr(false, Expr) -> - {inline, Expr}; -comprehension_expr(_, Expr) -> - {into, Expr}. - -comprehension_filter(Line, Filters) -> - [join_filter(Line, Filter, {atom, Line, true}, {atom, Line, false}) || - Filter <- lists:reverse(Filters)]. - -join_filters(_Line, [], True, _False) -> - True; -join_filters(Line, [H|T], True, False) -> - lists:foldl(fun(Filter, Acc) -> - join_filter(Line, Filter, Acc, False) - end, join_filter(Line, H, True, False), T). - -join_filter(Line, {nil, Filter}, True, False) -> - {'case', Line, Filter, [ - {clause, Line, [{atom, Line, true}], [], [True]}, - {clause, Line, [{atom, Line, false}], [], [False]} - ]}; -join_filter(Line, {Var, Filter}, True, False) -> - Guard = - {op, Line, 'orelse', - {op, Line, '==', Var, {atom, Line, false}}, - {op, Line, '==', Var, {atom, Line, nil}}}, - - {'case', Line, Filter, [ - {clause, Line, [Var], [[Guard]], [False]}, - {clause, Line, [{var, Line, '_'}], [], [True]} - ]}. diff --git a/lib/elixir/src/elixir_import.erl b/lib/elixir/src/elixir_import.erl index ece17e12828..ea7e07833cc 100644 --- a/lib/elixir/src/elixir_import.erl +++ b/lib/elixir/src/elixir_import.erl @@ -1,166 +1,285 @@ +%% SPDX-License-Identifier: Apache-2.0 +%% SPDX-FileCopyrightText: 2021 The Elixir Team +%% SPDX-FileCopyrightText: 2012 Plataformatec + %% Module responsible for handling imports and conflicts -%% in between local functions and imports. +%% between local functions and imports. %% For imports dispatch, please check elixir_dispatch. -module(elixir_import). --export([import/4, special_form/2, format_error/1]). +-export([import/6, import/7, special_form/2, + record/4, ensure_no_local_conflict/3, + format_error/1]). +-compile(inline_list_funcs). -include("elixir.hrl"). -%% IMPORT - -import(Meta, Ref, Opts, E) -> - Res = - case keyfind(only, Opts) of - {only, functions} -> - {import_functions(Meta, Ref, Opts, E), - ?m(E, macros)}; - {only, macros} -> - {?m(E, functions), - import_macros(true, Meta, Ref, Opts, E)}; - {only, List} when is_list(List) -> - {import_functions(Meta, Ref, Opts, E), - import_macros(false, Meta, Ref, Opts, E)}; - false -> - {import_functions(Meta, Ref, Opts, E), - import_macros(false, Meta, Ref, Opts, E)} - end, +import(Meta, Ref, Opts, E, Warn, Trace) -> + import(Meta, Ref, Opts, E, Warn, Trace, fun Ref:'__info__'/1). - record_warn(Meta, Ref, Opts, E), - Res. +import(Meta, Ref, Opts, E, Warn, Trace, InfoCallback) -> + case import_only_except(Meta, Ref, Opts, E, Warn, InfoCallback) of + {Functions, Macros, Added} -> + Trace andalso elixir_env:trace({import, Meta, Ref, Opts}, E), + EI = E#{functions := Functions, macros := Macros}, + {ok, Added, elixir_aliases:require(Meta, Ref, [{warn, false} | Opts], EI, Trace)}; -import_functions(Meta, Ref, Opts, E) -> - calculate(Meta, Ref, Opts, ?m(E, functions), E, fun() -> get_functions(Ref) end). + {error, Reason} -> + {error, Reason} + end. -import_macros(Force, Meta, Ref, Opts, E) -> - calculate(Meta, Ref, Opts, ?m(E, macros), E, fun() -> - case Force of - true -> get_macros(Meta, Ref, E); - false -> get_optional_macros(Ref) - end +import_only_except(Meta, Ref, Opts, E, Warn, InfoCallback) -> + MaybeOnly = lists:keyfind(only, 1, Opts), + + case lists:keyfind(except, 1, Opts) of + false -> + import_only_except(Meta, Ref, MaybeOnly, false, E, Warn, InfoCallback); + + {except, DupExcept} when is_list(DupExcept) -> + case ensure_keyword_list(DupExcept) of + ok -> + Except = ensure_no_duplicates(DupExcept, except, Meta, E, Warn), + import_only_except(Meta, Ref, MaybeOnly, Except, E, Warn, InfoCallback); + + error -> + {error, {invalid_option, except, DupExcept}} + end; + + {except, Other} -> + {error, {invalid_option, except, Other}} + end. + +import_only_except(Meta, Ref, MaybeOnly, Except, E, Warn, InfoCallback) -> + case MaybeOnly of + {only, functions} -> + {Added1, _Used1, Funs} = import_functions(Meta, Ref, Except, E, Warn, InfoCallback), + {Funs, keydelete(Ref, ?key(E, macros)), Added1}; + + {only, macros} -> + {Added2, _Used2, Macs} = import_macros(Meta, Ref, Except, E, Warn, InfoCallback), + {keydelete(Ref, ?key(E, functions)), Macs, Added2}; + + {only, sigils} -> + {Added1, _Used1, Funs} = import_sigil_functions(Meta, Ref, Except, E, Warn, InfoCallback), + {Added2, _Used2, Macs} = import_sigil_macros(Meta, Ref, Except, E, Warn, InfoCallback), + {Funs, Macs, Added1 or Added2}; + + {only, DupOnly} when is_list(DupOnly) -> + case ensure_keyword_list(DupOnly) of + ok when Except =:= false -> + Only = ensure_no_duplicates(DupOnly, only, Meta, E, Warn), + {Added1, Used1, Funs} = import_listed_functions(Meta, Ref, Only, E, Warn, InfoCallback), + {Added2, Used2, Macs} = import_listed_macros(Meta, Ref, Only, E, Warn, InfoCallback), + [Warn andalso elixir_errors:file_warn(Meta, E, ?MODULE, {invalid_import, {Ref, Name, Arity}}) || + {Name, Arity} <- (Only -- Used1) -- Used2], + {Funs, Macs, Added1 or Added2}; + + ok -> + {error, only_and_except_given}; + + error -> + {error, {invalid_option, only, DupOnly}} + end; + + {only, Other} -> + {error, {invalid_option, only, Other}}; + + false -> + {Added1, _Used1, Funs} = import_functions(Meta, Ref, Except, E, Warn, InfoCallback), + {Added2, _Used2, Macs} = import_macros(Meta, Ref, Except, E, Warn, InfoCallback), + {Funs, Macs, Added1 or Added2} + end. + +import_listed_functions(Meta, Ref, Only, E, Warn, InfoCallback) -> + New = intersection(Only, get_functions(Ref, InfoCallback)), + calculate_key(Meta, Ref, ?key(E, functions), New, E, Warn). + +import_listed_macros(Meta, Ref, Only, E, Warn, InfoCallback) -> + New = intersection(Only, get_macros(InfoCallback)), + calculate_key(Meta, Ref, ?key(E, macros), New, E, Warn). + +import_functions(Meta, Ref, Except, E, Warn, InfoCallback) -> + calculate_except(Meta, Ref, Except, ?key(E, functions), E, Warn, fun() -> + get_functions(Ref, InfoCallback) + end). + +import_macros(Meta, Ref, Except, E, Warn, InfoCallback) -> + calculate_except(Meta, Ref, Except, ?key(E, macros), E, Warn, fun() -> + get_macros(InfoCallback) + end). + +import_sigil_functions(Meta, Ref, Except, E, Warn, InfoCallback) -> + calculate_except(Meta, Ref, Except, ?key(E, functions), E, Warn, fun() -> + filter_sigils(InfoCallback(functions)) + end). + +import_sigil_macros(Meta, Ref, Except, E, Warn, InfoCallback) -> + calculate_except(Meta, Ref, Except, ?key(E, macros), E, Warn, fun() -> + filter_sigils(InfoCallback(macros)) end). -record_warn(Meta, Ref, Opts, E) -> - Warn = - case keyfind(warn, Opts) of - {warn, false} -> false; - {warn, true} -> true; - false -> not lists:keymember(context, 1, Meta) +calculate_except(Meta, Key, false, Old, E, Warn, Existing) -> + New = remove_underscored(Existing()), + calculate_key(Meta, Key, Old, New, E, Warn); + +calculate_except(Meta, Key, Except, Old, E, Warn, Existing) -> + %% We are not checking existence of exports listed in :except + %% option on purpose: to support backwards compatible code. + %% For example, "import String, except: [trim: 1]" + %% should work across all Elixir versions. + New = + case lists:keyfind(Key, 1, Old) of + false -> remove_underscored(Existing()) -- Except; + {Key, OldImports} -> OldImports -- Except end, - elixir_lexical:record_import(Ref, ?line(Meta), Warn, ?m(E, lexical_tracker)). - -%% Calculates the imports based on only and except - -calculate(Meta, Key, Opts, Old, E, Existing) -> - New = case keyfind(only, Opts) of - {only, Only} when is_list(Only) -> - case Only -- get_exports(Key) of - [{Name,Arity}|_] -> - Tuple = {invalid_import, {Key, Name, Arity}}, - elixir_errors:form_error(Meta, ?m(E, file), ?MODULE, Tuple); - _ -> - intersection(Only, Existing()) - end; - _ -> - case keyfind(except, Opts) of - false -> remove_underscored(Existing()); - {except, []} -> remove_underscored(Existing()); - {except, Except} when is_list(Except) -> - case keyfind(Key, Old) of - false -> remove_underscored(Existing()) -- Except; - {Key,OldImports} -> OldImports -- Except - end - end - end, - - %% Normalize the data before storing it - Set = ordsets:from_list(New), - Final = remove_internals(Set), - - case Final of - [] -> keydelete(Key, Old); - _ -> - ensure_no_special_form_conflict(Meta, ?m(E, file), Key, Final), - [{Key, Final}|keydelete(Key, Old)] + calculate_key(Meta, Key, Old, New, E, Warn). + +calculate_key(Meta, Key, Old, New, E, Warn) -> + case ordsets:from_list(New) of + [] -> + {false, [], keydelete(Key, Old)}; + Set -> + FinalSet = ensure_no_special_form_conflict(Set, Key, Meta, E, Warn), + {true, FinalSet, [{Key, FinalSet} | keydelete(Key, Old)]} end. -%% Retrieve functions and macros from modules +%% Record function calls for local conflicts -get_exports(Module) -> +record(_Tuple, Receiver, Module, Function) + when Function == nil; Module == Receiver -> false; +record(Tuple, Receiver, Module, _Function) -> try - Module:'__info__'(functions) ++ Module:'__info__'(macros) + {Set, _Bag} = elixir_module:data_tables(Module), + ets:insert(Set, {{import, Tuple}, Receiver}), + true catch - error:undef -> Module:module_info(exports) + error:badarg -> false end. -get_functions(Module) -> +ensure_no_local_conflict('Elixir.Kernel', _All, _E) -> + ok; +ensure_no_local_conflict(Module, AllDefinitions, E) -> + {Set, _} = elixir_module:data_tables(Module), + + [try + Receiver = ets:lookup_element(Set, {import, Pair}, 2), + elixir_errors:module_error(Meta, E, ?MODULE, {import_conflict, Receiver, Pair}) + catch + error:badarg -> false + end || {Pair, _, Meta, _} <- AllDefinitions]. + +%% Retrieve functions and macros from modules + +get_functions(Module, InfoCallback) -> try - Module:'__info__'(functions) + InfoCallback(functions) catch - error:undef -> Module:module_info(exports) + error:undef -> remove_internals(Module:module_info(exports)) end. -get_macros(Meta, Module, E) -> +get_macros(InfoCallback) -> try - Module:'__info__'(macros) + InfoCallback(macros) catch - error:undef -> - Tuple = {no_macros, Module}, - elixir_errors:form_error(Meta, ?m(E, file), ?MODULE, Tuple) + error:undef -> [] end. -get_optional_macros(Module) -> - case code:ensure_loaded(Module) of - {module, Module} -> - try - Module:'__info__'(macros) - catch - error:undef -> [] +filter_sigils(Funs) -> + lists:filter(fun is_sigil/1, Funs). + +is_sigil({Name, 2}) -> + case atom_to_list(Name) of + "sigil_" ++ Letters -> + case Letters of + [L] when L >= $a, L =< $z -> true; + [] -> false; + [H|T] when H >= $A, H =< $Z -> + lists:all(fun(L) -> (L >= $0 andalso L =< $9) + orelse (L>= $A andalso L =< $Z) + end, T) end; - {error, _} -> [] - end. - -%% VALIDATION HELPERS - -ensure_no_special_form_conflict(Meta, File, Key, [{Name,Arity}|T]) -> - case special_form(Name, Arity) of - true -> - Tuple = {special_form_conflict, {Key, Name, Arity}}, - elixir_errors:form_error(Meta, File, ?MODULE, Tuple); - false -> - ensure_no_special_form_conflict(Meta, File, Key, T) + _ -> + false end; - -ensure_no_special_form_conflict(_Meta, _File, _Key, []) -> ok. +is_sigil(_) -> + false. + +%% VALIDATION HELPERS\ + +ensure_keyword_list([]) -> + ok; +ensure_keyword_list([{Key, Value} | Rest]) when is_atom(Key), is_integer(Value) -> + ensure_keyword_list(Rest); +ensure_keyword_list(_Other) -> + error. + +ensure_no_special_form_conflict(Set, Key, Meta, E, Warn) -> + lists:filter(fun({Name, Arity}) -> + case special_form(Name, Arity) of + true -> + Warn andalso elixir_errors:file_warn(Meta, E, ?MODULE, {special_form_conflict, {Key, Name, Arity}}), + false; + false -> + true + end + end, Set). + +ensure_no_duplicates(Option, Kind, Meta, E, Warn) -> + lists:foldl(fun({Name, Arity}, Acc) -> + case lists:member({Name, Arity}, Acc) of + true -> + Warn andalso elixir_errors:file_warn(Meta, E, ?MODULE, {duplicated_import, {Kind, Name, Arity}}), + Acc; + false -> + [{Name, Arity} | Acc] + end + end, [], Option). %% ERROR HANDLING -format_error({invalid_import,{Receiver, Name, Arity}}) -> - io_lib:format("cannot import ~ts.~ts/~B because it doesn't exist", +format_error(only_and_except_given) -> + ":only and :except can only be given together to import " + "when :only is :functions, :macros, or :sigils"; + +format_error({duplicated_import, {Option, Name, Arity}}) -> + io_lib:format("invalid :~s option for import, ~ts/~B is duplicated", [Option, Name, Arity]); + +format_error({invalid_import, {Receiver, Name, Arity}}) -> + io_lib:format("cannot import ~ts.~ts/~B because it is undefined or private", [elixir_aliases:inspect(Receiver), Name, Arity]); -format_error({special_form_conflict,{Receiver, Name, Arity}}) -> - io_lib:format("cannot import ~ts.~ts/~B because it conflicts with Elixir special forms", +format_error({invalid_option, only, Value}) -> + Message = "invalid :only option for import, expected value to be an atom :functions, :macros" + ", or a literal keyword list of function names with arity as values, got: ~s", + io_lib:format(Message, ['Elixir.Macro':to_string(Value)]); + +format_error({invalid_option, except, Value}) -> + Message = "invalid :except option for import, expected value to be a literal keyword list of function names with arity as values, got: ~s", + io_lib:format(Message, ['Elixir.Macro':to_string(Value)]); + +format_error({special_form_conflict, {Receiver, Name, Arity}}) -> + io_lib:format("cannot import ~ts.~ts/~B because it conflicts with Elixir special forms, the import has been discarded", [elixir_aliases:inspect(Receiver), Name, Arity]); format_error({no_macros, Module}) -> - io_lib:format("could not load macros from module ~ts", [elixir_aliases:inspect(Module)]). + io_lib:format("could not load macros from module ~ts", [elixir_aliases:inspect(Module)]); -%% LIST HELPERS +format_error({import_conflict, Receiver, {Name, Arity}}) -> + io_lib:format("imported ~ts.~ts/~B conflicts with local function", + [elixir_aliases:inspect(Receiver), Name, Arity]). -keyfind(Key, List) -> - lists:keyfind(Key, 1, List). +%% LIST HELPERS keydelete(Key, List) -> lists:keydelete(Key, 1, List). -intersection([H|T], All) -> +intersection([H | T], All) -> case lists:member(H, All) of - true -> [H|intersection(T, All)]; + true -> [H | intersection(T, All)]; false -> intersection(T, All) end; intersection([], _All) -> []. -%% Internal funs that are never imported etc. +%% Internal funs that are never imported, and the like remove_underscored(List) -> lists:filter(fun({Name, _}) -> @@ -171,8 +290,7 @@ remove_underscored(List) -> end, List). remove_internals(Set) -> - ordsets:del_element({module_info, 1}, - ordsets:del_element({module_info, 0}, Set)). + Set -- [{behaviour_info, 1}, {module_info, 1}, {module_info, 0}]. %% Special forms @@ -180,9 +298,12 @@ special_form('&', 1) -> true; special_form('^', 1) -> true; special_form('=', 2) -> true; special_form('%', 2) -> true; -special_form('__op__', 2) -> true; -special_form('__op__', 3) -> true; +special_form('|', 2) -> true; +special_form('.', 2) -> true; +special_form('::', 2) -> true; +special_form('__aliases__', _) -> true; special_form('__block__', _) -> true; +special_form('__cursor__', _) -> true; special_form('->', _) -> true; special_form('<<>>', _) -> true; special_form('{}', _) -> true; @@ -195,9 +316,9 @@ special_form('import', 1) -> true; special_form('import', 2) -> true; special_form('__ENV__', 0) -> true; special_form('__CALLER__', 0) -> true; +special_form('__STACKTRACE__', 0) -> true; special_form('__MODULE__', 0) -> true; special_form('__DIR__', 0) -> true; -special_form('__aliases__', _) -> true; special_form('quote', 1) -> true; special_form('quote', 2) -> true; special_form('unquote', 1) -> true; @@ -205,8 +326,9 @@ special_form('unquote_splicing', 1) -> true; special_form('fn', _) -> true; special_form('super', _) -> true; special_form('for', _) -> true; +special_form('with', _) -> true; special_form('cond', 1) -> true; special_form('case', 2) -> true; -special_form('try', 2) -> true; +special_form('try', 1) -> true; special_form('receive', 1) -> true; special_form(_, _) -> false. diff --git a/lib/elixir/src/elixir_interpolation.erl b/lib/elixir/src/elixir_interpolation.erl index 1221811b7a0..90321f4fe76 100644 --- a/lib/elixir/src/elixir_interpolation.erl +++ b/lib/elixir/src/elixir_interpolation.erl @@ -1,139 +1,258 @@ +%% SPDX-License-Identifier: Apache-2.0 +%% SPDX-FileCopyrightText: 2021 The Elixir Team +%% SPDX-FileCopyrightText: 2012 Plataformatec + % Handle string and string-like interpolations. -module(elixir_interpolation). --export([extract/5, unescape_chars/1, unescape_chars/2, -unescape_tokens/1, unescape_tokens/2, unescape_map/1]). +-export([extract/6, unescape_string/1, unescape_string/2, +unescape_tokens/1, unescape_map/1]). -include("elixir.hrl"). +-include("elixir_tokenizer.hrl"). %% Extract string interpolations -extract(Line, Raw, Interpol, String, Last) -> - %% Ignore whatever is in the scope and enable terminator checking. - Scope = Raw#elixir_tokenizer{terminators=[], check_terminators=true}, - extract(Line, Scope, Interpol, String, [], [], Last). +extract(Line, Column, Scope, Interpol, String, Last) -> + extract(String, [], [], Line, Column, Scope, Interpol, Last). %% Terminators -extract(Line, _Scope, _Interpol, [], Buffer, Output, []) -> - finish_extraction(Line, Buffer, Output, []); +extract([], _Buffer, _Output, Line, Column, #elixir_tokenizer{cursor_completion=false}, _Interpol, Last) -> + {error, {string, Line, Column, io_lib:format("missing terminator: ~ts", [[Last]]), []}}; -extract(Line, _Scope, _Interpol, [], _Buffer, _Output, Last) -> - {error, {string, Line, io_lib:format("missing terminator: ~ts", [[Last]]), []}}; +extract([], Buffer, Output, Line, Column, Scope, _Interpol, _Last) -> + finish_extraction([], Buffer, Output, Line, Column, Scope); -extract(Line, _Scope, _Interpol, [Last|Remaining], Buffer, Output, Last) -> - finish_extraction(Line, Buffer, Output, Remaining); +extract([Last | Rest], Buffer, Output, Line, Column, Scope, _Interpol, Last) -> + finish_extraction(Rest, Buffer, Output, Line, Column + 1, Scope); %% Going through the string -extract(Line, Scope, Interpol, [$\\, $\n|Rest], Buffer, Output, Last) -> - extract(Line+1, Scope, Interpol, Rest, Buffer, Output, Last); +extract([$\\, $\r, $\n | Rest], Buffer, Output, Line, _Column, Scope, Interpol, Last) -> + extract_nl(Rest, [$\n, $\r, $\\ | Buffer], Output, Line, Scope, Interpol, Last); + +extract([$\\, $\n | Rest], Buffer, Output, Line, _Column, Scope, Interpol, Last) -> + extract_nl(Rest, [$\n, $\\ | Buffer], Output, Line, Scope, Interpol, Last); + +extract([$\n | Rest], Buffer, Output, Line, _Column, Scope, Interpol, Last) -> + extract_nl(Rest, [$\n | Buffer], Output, Line, Scope, Interpol, Last); + +extract([$\\, Last | Rest], Buffer, Output, Line, Column, Scope, Interpol, Last) -> + NewScope = + %% TODO: Remove this on Elixir v2.0 + case Interpol of + true -> + Scope; + false -> + Msg = "using \\~ts to escape the closing of an uppercase sigil is deprecated, please use another delimiter or a lowercase sigil instead", + prepend_warning(Line, Column, io_lib:format(Msg, [[Last]]), Scope) + end, + + extract(Rest, [Last | Buffer], Output, Line, Column+2, NewScope, Interpol, Last); + +extract([$\\, Last, Last, Last | Rest], Buffer, Output, Line, Column, Scope, Interpol, [Last, Last, Last] = All) -> + extract(Rest, [Last, Last, Last | Buffer], Output, Line, Column+4, Scope, Interpol, All); + +extract([$\\, $#, ${ | Rest], Buffer, Output, Line, Column, Scope, true, Last) -> + extract(Rest, [${, $#, $\\ | Buffer], Output, Line, Column+3, Scope, true, Last); + +extract([$#, ${ | Rest], Buffer, Output, Line, Column, Scope, true, Last) -> + Output1 = build_string(Buffer, Output), + case elixir_tokenizer:tokenize(Rest, Line, Column + 2, Scope#elixir_tokenizer{terminators=[]}) of + {error, {Location, _, "}"}, [$} | NewRest], Warnings, Tokens} -> + NewScope = Scope#elixir_tokenizer{warnings=Warnings}, + {line, EndLine} = lists:keyfind(line, 1, Location), + {column, EndColumn} = lists:keyfind(column, 1, Location), + Output2 = build_interpol(Line, Column, EndLine, EndColumn, lists:reverse(Tokens), Output1), + extract(NewRest, [], Output2, EndLine, EndColumn + 1, NewScope, true, Last); + {error, Reason, _, _, _} -> + {error, Reason}; + {ok, EndLine, EndColumn, Warnings, Tokens, Terminators} when Scope#elixir_tokenizer.cursor_completion /= false -> + NewScope = Scope#elixir_tokenizer{warnings=Warnings, cursor_completion=noprune}, + {CursorTerminators, _} = cursor_complete(EndLine, EndColumn, Terminators), + Output2 = build_interpol(Line, Column, EndLine, EndColumn, lists:reverse(Tokens, CursorTerminators), Output1), + extract([], [], Output2, EndLine, EndColumn, NewScope, true, Last); + {ok, _, _, _, _, _} -> + {error, {string, Line, Column, "missing interpolation terminator: \"}\"", []}} + end; -extract(Line, Scope, Interpol, [$\\, $\r, $\n|Rest], Buffer, Output, Last) -> - extract(Line+1, Scope, Interpol, Rest, Buffer, Output, Last); +extract([$\\ | Rest], Buffer, Output, Line, Column, Scope, Interpol, Last) -> + extract_char(Rest, [$\\ | Buffer], Output, Line, Column + 1, Scope, Interpol, Last); -extract(Line, Scope, Interpol, [$\n|Rest], Buffer, Output, Last) -> - extract(Line+1, Scope, Interpol, Rest, [$\n|Buffer], Output, Last); +%% Catch all clause -extract(Line, Scope, Interpol, [$\\, $#, ${|Rest], Buffer, Output, Last) -> - extract(Line, Scope, Interpol, Rest, [${,$#|Buffer], Output, Last); +extract([Char1, Char2 | Rest], Buffer, Output, Line, Column, Scope, Interpol, Last) + when Char1 =< 255, Char2 =< 255 -> + extract([Char2 | Rest], [Char1 | Buffer], Output, Line, Column + 1, Scope, Interpol, Last); -extract(Line, Scope, Interpol, [$\\,Char|Rest], Buffer, Output, Last) -> - extract(Line, Scope, Interpol, Rest, [Char,$\\|Buffer], Output, Last); +extract(Rest, Buffer, Output, Line, Column, Scope, Interpol, Last) -> + extract_char(Rest, Buffer, Output, Line, Column, Scope, Interpol, Last). -extract(Line, Scope, true, [$#, ${|Rest], Buffer, Output, Last) -> - Output1 = build_string(Line, Buffer, Output), +extract_char(Rest, Buffer, Output, Line, Column, Scope, Interpol, Last) -> + case unicode_util:gc(Rest) of + [Char | _] when ?bidi(Char); ?break(Char) -> + Token = io_lib:format("\\u~4.16.0B", [Char]), + Pre = if + ?bidi(Char) -> "invalid bidirectional formatting character in string: "; + true -> "invalid line break character in string: " + end, + Pos = io_lib:format(". If you want to use such character, use it in its escaped ~ts form instead", [Token]), + {error, {?LOC(Line, Column), {Pre, Pos}, Token}}; - case elixir_tokenizer:tokenize(Rest, Line, Scope) of - {error, {EndLine, _, "}"}, [$}|NewRest], Tokens} -> - Output2 = build_interpol(Line, Tokens, Output1), - extract(EndLine, Scope, true, NewRest, [], Output2, Last); - {error, Reason, _, _} -> - {error, Reason}; - {ok, _EndLine, _} -> - {error, {string, Line, "missing interpolation terminator:}", []}} - end; + [Char | NewRest] when is_list(Char) -> + extract(NewRest, lists:reverse(Char, Buffer), Output, Line, Column + 1, Scope, Interpol, Last); -%% Catch all clause + [Char | NewRest] when is_integer(Char) -> + extract(NewRest, [Char | Buffer], Output, Line, Column + 1, Scope, Interpol, Last); + + [] -> + extract([], Buffer, Output, Line, Column, Scope, Interpol, Last) + end. -extract(Line, Scope, Interpol, [Char|Rest], Buffer, Output, Last) -> - extract(Line, Scope, Interpol, Rest, [Char|Buffer], Output, Last). +%% Handle newlines. Heredocs require special attention + +extract_nl(Rest, Buffer, Output, Line, Scope, Interpol, [H,H,H] = Last) -> + case strip_horizontal_space(Rest, Buffer, 1) of + {[H,H,H|NewRest], _NewBuffer, Column} -> + finish_extraction(NewRest, Buffer, Output, Line + 1, Column + 3, Scope); + {NewRest, NewBuffer, Column} -> + extract(NewRest, NewBuffer, Output, Line + 1, Column, Scope, Interpol, Last) + end; +extract_nl(Rest, Buffer, Output, Line, Scope, Interpol, Last) -> + extract(Rest, Buffer, Output, Line + 1, Scope#elixir_tokenizer.column, Scope, Interpol, Last). + +strip_horizontal_space([H | T], Buffer, Counter) when H =:= $\s; H =:= $\t -> + strip_horizontal_space(T, [H | Buffer], Counter + 1); +strip_horizontal_space(T, Buffer, Counter) -> + {T, Buffer, Counter}. + +cursor_complete(Line, Column, Terminators) -> + lists:mapfoldl( + fun({Start, _, _}, AccColumn) -> + End = elixir_tokenizer:terminator(Start), + {{End, {Line, AccColumn, nil}}, AccColumn + length(erlang:atom_to_list(End))} + end, + Column, + Terminators + ). %% Unescape a series of tokens as returned by extract. unescape_tokens(Tokens) -> - unescape_tokens(Tokens, fun unescape_map/1). + try [unescape_token(Token, fun unescape_map/1) || Token <- Tokens] of + Unescaped -> {ok, Unescaped} + catch + {error, _Reason, _Token} = Error -> Error + end. -unescape_tokens(Tokens, Map) -> - [unescape_token(Token, Map) || Token <- Tokens]. +unescape_token(Token, Map) when is_list(Token) -> + unescape_chars(elixir_utils:characters_to_binary(Token), Map); +unescape_token(Token, Map) when is_binary(Token) -> + unescape_chars(Token, Map); +unescape_token(Other, _Map) -> + Other. -unescape_token(Token, Map) when is_binary(Token) -> unescape_chars(Token, Map); -unescape_token(Other, _Map) -> Other. +% Unescape string. This is called by Elixir. Wrapped by convenience. -% Unescape chars. For instance, "\" "n" (two chars) needs to be converted to "\n" (one char). +unescape_string(String) -> + unescape_string(String, fun unescape_map/1). -unescape_chars(String) -> - unescape_chars(String, fun unescape_map/1). +unescape_string(String, Map) -> + try + unescape_chars(String, Map) + catch + {error, Reason, _} -> + Message = elixir_utils:characters_to_binary(Reason), + error('Elixir.ArgumentError':exception([{message, Message}])) + end. + +% Unescape chars. For instance, "\" "n" (two chars) needs to be converted to "\n" (one char). unescape_chars(String, Map) -> - Octals = Map($0) /= false, - Hex = Map($x) /= false, - unescape_chars(String, Map, Octals, Hex, <<>>). + unescape_chars(String, Map, <<>>). -unescape_chars(<<$\\,A,B,C,Rest/binary>>, Map, true, Hex, Acc) when ?is_octal(A), A =< $3, ?is_octal(B), ?is_octal(C) -> - append_escaped(Rest, Map, [A,B,C], true, Hex, Acc, 8); +unescape_chars(<<$\\, $x, Rest/binary>>, Map, Acc) -> + case Map(hex) of + true -> unescape_hex(Rest, Map, Acc); + false -> unescape_chars(Rest, Map, <>) + end; -unescape_chars(<<$\\,A,B,Rest/binary>>, Map, true, Hex, Acc) when ?is_octal(A), ?is_octal(B) -> - append_escaped(Rest, Map, [A,B], true, Hex, Acc, 8); +unescape_chars(<<$\\, $u, Rest/binary>>, Map, Acc) -> + case Map(unicode) of + true -> unescape_unicode(Rest, Map, Acc); + false -> unescape_chars(Rest, Map, <>) + end; -unescape_chars(<<$\\,A,Rest/binary>>, Map, true, Hex, Acc) when ?is_octal(A) -> - append_escaped(Rest, Map, [A], true, Hex, Acc, 8); +unescape_chars(<<$\\, $\n, Rest/binary>>, Map, Acc) -> + case Map(newline) of + true -> unescape_chars(Rest, Map, Acc); + false -> unescape_chars(Rest, Map, <>) + end; -unescape_chars(<<$\\,P,A,B,Rest/binary>>, Map, Octal, true, Acc) when (P == $x orelse P == $X), ?is_hex(A), ?is_hex(B) -> - append_escaped(Rest, Map, [A,B], Octal, true, Acc, 16); +unescape_chars(<<$\\, $\r, $\n, Rest/binary>>, Map, Acc) -> + case Map(newline) of + true -> unescape_chars(Rest, Map, Acc); + false -> unescape_chars(Rest, Map, <>) + end; -unescape_chars(<<$\\,P,A,Rest/binary>>, Map, Octal, true, Acc) when (P == $x orelse P == $X), ?is_hex(A) -> - append_escaped(Rest, Map, [A], Octal, true, Acc, 16); +unescape_chars(<<$\\, Escaped, Rest/binary>>, Map, Acc) -> + case Map(Escaped) of + false -> unescape_chars(Rest, Map, <>); + Other -> unescape_chars(Rest, Map, <>) + end; -unescape_chars(<<$\\,P,${,A,$},Rest/binary>>, Map, Octal, true, Acc) when (P == $x orelse P == $X), ?is_hex(A) -> - append_escaped(Rest, Map, [A], Octal, true, Acc, 16); +unescape_chars(<>, Map, Acc) -> + unescape_chars(Rest, Map, <>); -unescape_chars(<<$\\,P,${,A,B,$},Rest/binary>>, Map, Octal, true, Acc) when (P == $x orelse P == $X), ?is_hex(A), ?is_hex(B) -> - append_escaped(Rest, Map, [A,B], Octal, true, Acc, 16); +unescape_chars(<<>>, _Map, Acc) -> Acc. -unescape_chars(<<$\\,P,${,A,B,C,$},Rest/binary>>, Map, Octal, true, Acc) when (P == $x orelse P == $X), ?is_hex(A), ?is_hex(B), ?is_hex(C) -> - append_escaped(Rest, Map, [A,B,C], Octal, true, Acc, 16); +% Unescape Helpers -unescape_chars(<<$\\,P,${,A,B,C,D,$},Rest/binary>>, Map, Octal, true, Acc) when (P == $x orelse P == $X), ?is_hex(A), ?is_hex(B), ?is_hex(C), ?is_hex(D) -> - append_escaped(Rest, Map, [A,B,C,D], Octal, true, Acc, 16); +unescape_hex(<>, Map, Acc) when ?is_hex(A), ?is_hex(B) -> + Bytes = list_to_integer([A, B], 16), + unescape_chars(Rest, Map, <>); -unescape_chars(<<$\\,P,${,A,B,C,D,E,$},Rest/binary>>, Map, Octal, true, Acc) when (P == $x orelse P == $X), ?is_hex(A), ?is_hex(B), ?is_hex(C), ?is_hex(D), ?is_hex(E) -> - append_escaped(Rest, Map, [A,B,C,D,E], Octal, true, Acc, 16); +unescape_hex(<<_/binary>>, _Map, _Acc) -> + throw({error, "invalid hex escape character, expected \\xHH where H is a hexadecimal digit", "\\x"}). -unescape_chars(<<$\\,P,${,A,B,C,D,E,F,$},Rest/binary>>, Map, Octal, true, Acc) when (P == $x orelse P == $X), ?is_hex(A), ?is_hex(B), ?is_hex(C), ?is_hex(D), ?is_hex(E), ?is_hex(F) -> - append_escaped(Rest, Map, [A,B,C,D,E,F], Octal, true, Acc, 16); +%% Finish deprecated sequences -unescape_chars(<<$\\,Escaped,Rest/binary>>, Map, Octals, Hex, Acc) -> - case Map(Escaped) of - false -> unescape_chars(Rest, Map, Octals, Hex, <>); - Other -> unescape_chars(Rest, Map, Octals, Hex, <>) - end; +unescape_unicode(<>, Map, Acc) when ?is_hex(A), ?is_hex(B), ?is_hex(C), ?is_hex(D) -> + append_codepoint(Rest, Map, [A, B, C, D], Acc, 16); -unescape_chars(<>, Map, Octals, Hex, Acc) -> - unescape_chars(Rest, Map, Octals, Hex, <>); +unescape_unicode(<<${, A, $}, Rest/binary>>, Map, Acc) when ?is_hex(A) -> + append_codepoint(Rest, Map, [A], Acc, 16); -unescape_chars(<<>>, _Map, _Octals, _Hex, Acc) -> Acc. +unescape_unicode(<<${, A, B, $}, Rest/binary>>, Map, Acc) when ?is_hex(A), ?is_hex(B) -> + append_codepoint(Rest, Map, [A, B], Acc, 16); -append_escaped(Rest, Map, List, Octal, Hex, Acc, Base) -> +unescape_unicode(<<${, A, B, C, $}, Rest/binary>>, Map, Acc) when ?is_hex(A), ?is_hex(B), ?is_hex(C) -> + append_codepoint(Rest, Map, [A, B, C], Acc, 16); + +unescape_unicode(<<${, A, B, C, D, $}, Rest/binary>>, Map, Acc) when ?is_hex(A), ?is_hex(B), ?is_hex(C), ?is_hex(D) -> + append_codepoint(Rest, Map, [A, B, C, D], Acc, 16); + +unescape_unicode(<<${, A, B, C, D, E, $}, Rest/binary>>, Map, Acc) when ?is_hex(A), ?is_hex(B), ?is_hex(C), ?is_hex(D), ?is_hex(E) -> + append_codepoint(Rest, Map, [A, B, C, D, E], Acc, 16); + +unescape_unicode(<<${, A, B, C, D, E, F, $}, Rest/binary>>, Map, Acc) when ?is_hex(A), ?is_hex(B), ?is_hex(C), ?is_hex(D), ?is_hex(E), ?is_hex(F) -> + append_codepoint(Rest, Map, [A, B, C, D, E, F], Acc, 16); + +unescape_unicode(<<_/binary>>, _Map, _Acc) -> + throw({error, "invalid Unicode escape character, expected \\uHHHH or \\u{H*} where H is a hexadecimal digit", "\\u"}). + +append_codepoint(Rest, Map, List, Acc, Base) -> Codepoint = list_to_integer(List, Base), try <> of - Binary -> unescape_chars(Rest, Map, Octal, Hex, Binary) + Binary -> unescape_chars(Rest, Map, Binary) catch error:badarg -> - Msg = <<"invalid or reserved unicode codepoint ", (integer_to_binary(Codepoint))/binary>>, - error('Elixir.ArgumentError':exception([{message,Msg}])) + throw({error, "invalid or reserved Unicode code point \\u{" ++ List ++ "}", "\\u"}) end. -% Unescape Helpers - +unescape_map(newline) -> true; +unescape_map(unicode) -> true; +unescape_map(hex) -> true; +unescape_map($0) -> 0; unescape_map($a) -> 7; unescape_map($b) -> $\b; unescape_map($d) -> $\d; @@ -148,16 +267,19 @@ unescape_map(E) -> E. % Extract Helpers -finish_extraction(Line, Buffer, Output, Remaining) -> - case build_string(Line, Buffer, Output) of - [] -> Final = [<<>>]; - Final -> [] +finish_extraction(Remaining, Buffer, Output, Line, Column, Scope) -> + Final = case build_string(Buffer, Output) of + [] -> [[]]; + F -> F end, - {Line, lists:reverse(Final), Remaining}. -build_string(_Line, [], Output) -> Output; -build_string(_Line, Buffer, Output) -> - [elixir_utils:characters_to_binary(lists:reverse(Buffer))|Output]. + {Line, Column, lists:reverse(Final), Remaining, Scope}. + +build_string([], Output) -> Output; +build_string(Buffer, Output) -> [lists:reverse(Buffer) | Output]. + +build_interpol(Line, Column, EndLine, EndColumn, Buffer, Output) -> + [{{Line, Column, nil}, {EndLine, EndColumn, nil}, Buffer} | Output]. -build_interpol(Line, Buffer, Output) -> - [{Line, lists:reverse(Buffer)}|Output]. +prepend_warning(Line, Column, Msg, #elixir_tokenizer{warnings=Warnings} = Scope) -> + Scope#elixir_tokenizer{warnings = [{{Line, Column}, Msg} | Warnings]}. diff --git a/lib/elixir/src/elixir_json.erl b/lib/elixir/src/elixir_json.erl new file mode 100644 index 00000000000..6413fb1e8d9 --- /dev/null +++ b/lib/elixir/src/elixir_json.erl @@ -0,0 +1,1140 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2024-2024. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(elixir_json). + +-dialyzer(no_improper_lists). + +-export([ + encode/1, encode/2, + encode_value/2, + encode_atom/2, + encode_integer/1, + encode_float/1, + encode_list/2, + encode_map/2, + encode_map_checked/2, + encode_key_value_list/2, + encode_key_value_list_checked/2, + encode_binary/1, + encode_binary_escape_all/1 +]). +-export_type([encoder/0, encode_value/0]). + +-export([ + decode/1, decode/3, decode_start/3, decode_continue/2 +]). +-export_type([ + from_binary_fun/0, + array_start_fun/0, + array_push_fun/0, + array_finish_fun/0, + object_start_fun/0, + object_push_fun/0, + object_finish_fun/0, + decoders/0, + decode_value/0, + continuation_state/0 +]). + +-compile({inline, [ + encode_atom/2, + encode_integer/1, + encode_float/1, + encode_object/1, + escape/1, + escape_binary/1, + escape_all/1, + utf8t/0, + utf8s/0, + utf8s0/0, + hex_to_int/4, + string/6 +]}). + +%% A lot of the macros below use multi-value comparisons where +%% range checks would have worked just fine. This is because +%% the compiler & JIT can emit better code in some cases when +%% multiple clauses are to be dispatched based on such sets +%% of values. They'll generate an efficient "jump table", +%% which gets to the correct clause in one go, rather +%% than going through a set of comparisons. +%% However, this might not always be the best way (see is_0_to_9), +%% so as always with any performance work - measure, don't guess! + +-define(is_1_to_9(X), + X =:= $1 orelse + X =:= $2 orelse + X =:= $3 orelse + X =:= $4 orelse + X =:= $5 orelse + X =:= $6 orelse + X =:= $7 orelse + X =:= $8 orelse + X =:= $9 +). + +-define(is_0_to_9(X), X >= $0 andalso X =< $9). + +-define(is_ws(X), X =:= $\s; X =:= $\t; X =:= $\r; X =:= $\n). + +-define(is_ascii_escape(Byte), + Byte =:= 0 orelse + Byte =:= 1 orelse + Byte =:= 2 orelse + Byte =:= 3 orelse + Byte =:= 4 orelse + Byte =:= 5 orelse + Byte =:= 6 orelse + Byte =:= 7 orelse + Byte =:= 8 orelse + Byte =:= 9 orelse + Byte =:= 10 orelse + Byte =:= 11 orelse + Byte =:= 12 orelse + Byte =:= 13 orelse + Byte =:= 14 orelse + Byte =:= 15 orelse + Byte =:= 16 orelse + Byte =:= 17 orelse + Byte =:= 18 orelse + Byte =:= 19 orelse + Byte =:= 20 orelse + Byte =:= 21 orelse + Byte =:= 22 orelse + Byte =:= 23 orelse + Byte =:= 24 orelse + Byte =:= 25 orelse + Byte =:= 26 orelse + Byte =:= 27 orelse + Byte =:= 28 orelse + Byte =:= 29 orelse + Byte =:= 30 orelse + Byte =:= 31 orelse + Byte =:= 34 orelse + Byte =:= 92 +). +-define(is_ascii_plain(Byte), + Byte =:= 32 orelse + Byte =:= 33 orelse + Byte =:= 35 orelse + Byte =:= 36 orelse + Byte =:= 37 orelse + Byte =:= 38 orelse + Byte =:= 39 orelse + Byte =:= 40 orelse + Byte =:= 41 orelse + Byte =:= 42 orelse + Byte =:= 43 orelse + Byte =:= 44 orelse + Byte =:= 45 orelse + Byte =:= 46 orelse + Byte =:= 47 orelse + Byte =:= 48 orelse + Byte =:= 49 orelse + Byte =:= 50 orelse + Byte =:= 51 orelse + Byte =:= 52 orelse + Byte =:= 53 orelse + Byte =:= 54 orelse + Byte =:= 55 orelse + Byte =:= 56 orelse + Byte =:= 57 orelse + Byte =:= 58 orelse + Byte =:= 59 orelse + Byte =:= 60 orelse + Byte =:= 61 orelse + Byte =:= 62 orelse + Byte =:= 63 orelse + Byte =:= 64 orelse + Byte =:= 65 orelse + Byte =:= 66 orelse + Byte =:= 67 orelse + Byte =:= 68 orelse + Byte =:= 69 orelse + Byte =:= 70 orelse + Byte =:= 71 orelse + Byte =:= 72 orelse + Byte =:= 73 orelse + Byte =:= 74 orelse + Byte =:= 75 orelse + Byte =:= 76 orelse + Byte =:= 77 orelse + Byte =:= 78 orelse + Byte =:= 79 orelse + Byte =:= 80 orelse + Byte =:= 81 orelse + Byte =:= 82 orelse + Byte =:= 83 orelse + Byte =:= 84 orelse + Byte =:= 85 orelse + Byte =:= 86 orelse + Byte =:= 87 orelse + Byte =:= 88 orelse + Byte =:= 89 orelse + Byte =:= 90 orelse + Byte =:= 91 orelse + Byte =:= 93 orelse + Byte =:= 94 orelse + Byte =:= 95 orelse + Byte =:= 96 orelse + Byte =:= 97 orelse + Byte =:= 98 orelse + Byte =:= 99 orelse + Byte =:= 100 orelse + Byte =:= 101 orelse + Byte =:= 102 orelse + Byte =:= 103 orelse + Byte =:= 104 orelse + Byte =:= 105 orelse + Byte =:= 106 orelse + Byte =:= 107 orelse + Byte =:= 108 orelse + Byte =:= 109 orelse + Byte =:= 110 orelse + Byte =:= 111 orelse + Byte =:= 112 orelse + Byte =:= 113 orelse + Byte =:= 114 orelse + Byte =:= 115 orelse + Byte =:= 116 orelse + Byte =:= 117 orelse + Byte =:= 118 orelse + Byte =:= 119 orelse + Byte =:= 120 orelse + Byte =:= 121 orelse + Byte =:= 122 orelse + Byte =:= 123 orelse + Byte =:= 124 orelse + Byte =:= 125 orelse + Byte =:= 126 orelse + Byte =:= 127 +). + +-define(are_all_ascii_plain(B1, B2, B3, B4, B5, B6, B7, B8), + (?is_ascii_plain(B1)) andalso + (?is_ascii_plain(B2)) andalso + (?is_ascii_plain(B3)) andalso + (?is_ascii_plain(B4)) andalso + (?is_ascii_plain(B5)) andalso + (?is_ascii_plain(B6)) andalso + (?is_ascii_plain(B7)) andalso + (?is_ascii_plain(B8)) +). + +-define(UTF8_ACCEPT, 0). +-define(UTF8_REJECT, 12). + +%% +%% Encoding implementation +%% + +-type encoder() :: fun((any(), encoder()) -> iodata()). + +-type encode_value() :: + integer() + | float() + | boolean() + | null + | binary() + | atom() + | list(encode_value()) + | encode_map(encode_value()). + +-type encode_map(Value) :: #{binary() | atom() | integer() => Value}. + +-spec encode(encode_value()) -> iodata(). +encode(Term) -> encode(Term, fun do_encode/2). + +-spec encode(any(), encoder()) -> iodata(). +encode(Term, Encoder) when is_function(Encoder, 2) -> + Encoder(Term, Encoder). + +-spec encode_value(any(), encoder()) -> iodata(). +encode_value(Value, Encode) -> + do_encode(Value, Encode). + +-spec do_encode(any(), encoder()) -> iodata(). +do_encode(Value, Encode) when is_atom(Value) -> + encode_atom(Value, Encode); +do_encode(Value, _Encode) when is_binary(Value) -> + escape_binary(Value); +do_encode(Value, _Encode) when is_integer(Value) -> + encode_integer(Value); +do_encode(Value, _Encode) when is_float(Value) -> + encode_float(Value); +do_encode(Value, Encode) when is_list(Value) -> + do_encode_list(Value, Encode); +do_encode(Value, Encode) when is_map(Value) -> + do_encode_map(Value, Encode); +do_encode(Other, _Encode) -> + error({unsupported_type, Other}). + +-spec encode_atom(atom(), encoder()) -> iodata(). +encode_atom(null, _Encode) -> <<"null">>; +encode_atom(true, _Encode) -> <<"true">>; +encode_atom(false, _Encode) -> <<"false">>; +encode_atom(Other, Encode) -> Encode(atom_to_binary(Other, utf8), Encode). + +-spec encode_integer(integer()) -> iodata(). +encode_integer(Integer) -> integer_to_binary(Integer). + +-spec encode_float(float()) -> iodata(). +encode_float(Float) -> float_to_binary(Float, [short]). + +-spec encode_list(list(), encoder()) -> iodata(). +encode_list(List, Encode) when is_list(List) -> + do_encode_list(List, Encode). + +do_encode_list([], _Encode) -> + <<"[]">>; +do_encode_list([First | Rest], Encode) when is_function(Encode, 2) -> + [$[, Encode(First, Encode) | list_loop(Rest, Encode)]. + +list_loop([], _Encode) -> "]"; +list_loop([Elem | Rest], Encode) -> [$,, Encode(Elem, Encode) | list_loop(Rest, Encode)]. + +-spec encode_map(encode_map(any()), encoder()) -> iodata(). +encode_map(Map, Encode) when is_map(Map) -> + do_encode_map(Map, Encode). + +do_encode_map(Map, Encode) when is_function(Encode, 2) -> + encode_object([[$,, key(Key, Encode), $: | Encode(Value, Encode)] || Key := Value <- Map]). + +-spec encode_map_checked(map(), encoder()) -> iodata(). +encode_map_checked(Map, Encode) -> + do_encode_checked(maps:to_list(Map), Encode). + +-spec encode_key_value_list([{term(), term()}], encoder()) -> iodata(). +encode_key_value_list(List, Encode) when is_function(Encode, 2) -> + encode_object([[$,, key(Key, Encode), $: | Encode(Value, Encode)] || {Key, Value} <- List]). + +-spec encode_key_value_list_checked([{term(), term()}], encoder()) -> iodata(). +encode_key_value_list_checked(List, Encode) -> + do_encode_checked(List, Encode). + +do_encode_checked(List, Encode) when is_function(Encode, 2) -> + encode_object(do_encode_checked(List, Encode, #{})). + +do_encode_checked([{Key, Value} | Rest], Encode, Visited0) -> + EncodedKey = iolist_to_binary(key(Key, Encode)), + case is_map_key(EncodedKey, Visited0) of + true -> + error({duplicate_key, Key}); + _ -> + Visited = Visited0#{EncodedKey => true}, + [[$,, EncodedKey, $: | Encode(Value, Encode)] | do_encode_checked(Rest, Encode, Visited)] + end; +do_encode_checked([], _, _) -> + []. + +%% Dispatching any value through `Encode` could allow incorrect +%% JSON to be emitted (with keys not being strings). To avoid this, +%% the default encoder only supports binaries, atoms, and numbers. +%% Customisation is possible by overriding how maps are encoded in general. +key(Key, Encode) when is_binary(Key) -> Encode(Key, Encode); +key(Key, Encode) when is_atom(Key) -> Encode(atom_to_binary(Key, utf8), Encode); +key(Key, _Encode) when is_integer(Key) -> [$", encode_integer(Key), $"]; +key(Key, _Encode) when is_float(Key) -> [$", encode_float(Key), $"]. + +encode_object([]) -> <<"{}">>; +encode_object([[_Comma | Entry] | Rest]) -> ["{", Entry, Rest, "}"]. + +-spec encode_binary(binary()) -> iodata(). +encode_binary(Bin) when is_binary(Bin) -> + escape_binary(Bin). + +-spec encode_binary_escape_all(binary()) -> iodata(). +encode_binary_escape_all(Bin) when is_binary(Bin) -> + escape_all(Bin). + +escape_binary(Bin) -> escape_binary_ascii(Bin, [$"], Bin, 0, 0). + +escape_binary_ascii(Binary, Acc, Orig, Skip, Len) -> + case Binary of + <> when ?are_all_ascii_plain(B1, B2, B3, B4, B5, B6, B7, B8) -> + escape_binary_ascii(Rest, Acc, Orig, Skip, Len + 8); + Other -> + escape_binary(Other, Acc, Orig, Skip, Len) + end. + +escape_binary(<>, Acc, Orig, Skip, Len) when ?is_ascii_plain(Byte) -> + %% we got here because there were either less than 8 bytes left + %% or we have an escape in the next 8 bytes, + %% escape_binary_ascii would fail and dispatch here anyway + escape_binary(Rest, Acc, Orig, Skip, Len + 1); +escape_binary(<>, Acc, Orig, Skip0, Len) when ?is_ascii_escape(Byte) -> + Escape = escape(Byte), + Skip = Skip0 + Len + 1, + case Len of + 0 -> + escape_binary_ascii(Rest, [Acc | Escape], Orig, Skip, 0); + _ -> + Part = binary_part(Orig, Skip0, Len), + escape_binary_ascii(Rest, [Acc, Part | Escape], Orig, Skip, 0) + end; +escape_binary(<>, Acc, Orig, Skip, Len) -> + case element(Byte - 127, utf8s0()) of + ?UTF8_REJECT -> invalid_byte(Orig, Skip + Len); + %% all accept cases are ASCII, already covered above + State -> escape_binary_utf8(Rest, Acc, Orig, Skip, Len, State) + end; +escape_binary(_, _Acc, Orig, 0, _Len) -> + [$", Orig, $"]; +escape_binary(_, Acc, _Orig, _Skip, 0) -> + [Acc, $"]; +escape_binary(_, Acc, Orig, Skip, Len) -> + Part = binary_part(Orig, Skip, Len), + [Acc, Part, $"]. + +escape_binary_utf8(<>, Acc, Orig, Skip, Len, State0) -> + Type = element(Byte + 1, utf8t()), + case element(State0 + Type, utf8s()) of + ?UTF8_ACCEPT -> escape_binary_ascii(Rest, Acc, Orig, Skip, Len + 2); + ?UTF8_REJECT -> invalid_byte(Orig, Skip + Len + 1); + State -> escape_binary_utf8(Rest, Acc, Orig, Skip, Len + 1, State) + end; +escape_binary_utf8(_, _Acc, Orig, Skip, Len, _State) -> + unexpected_utf8(Orig, Skip + Len + 1). + +escape_all(Bin) -> escape_all_ascii(Bin, [$"], Bin, 0, 0). + +escape_all_ascii(Binary, Acc, Orig, Skip, Len) -> + case Binary of + <> when ?are_all_ascii_plain(B1, B2, B3, B4, B5, B6, B7, B8) -> + escape_all_ascii(Rest, Acc, Orig, Skip, Len + 8); + Other -> + escape_all(Other, Acc, Orig, Skip, Len) + end. + +escape_all(<>, Acc, Orig, Skip, Len) when ?is_ascii_plain(Byte) -> + escape_all(Rest, Acc, Orig, Skip, Len + 1); +escape_all(<>, Acc, Orig, Skip, Len) when ?is_ascii_escape(Byte) -> + Escape = escape(Byte), + case Len of + 0 -> + escape_all(Rest, [Acc | Escape], Orig, Skip + 1, 0); + _ -> + Part = binary_part(Orig, Skip, Len), + escape_all(Rest, [Acc, Part | Escape], Orig, Skip + Len + 1, 0) + end; +escape_all(<>, Acc, Orig, Skip, 0) -> + escape_char(Rest, Acc, Orig, Skip, Char); +escape_all(<>, Acc, Orig, Skip, Len) -> + Part = binary_part(Orig, Skip, Len), + escape_char(Rest, [Acc | Part], Orig, Skip + Len, Char); +escape_all(<<>>, _Acc, Orig, 0, _Len) -> + [$", Orig, $"]; +escape_all(<<>>, Acc, _Orig, _Skip, 0) -> + [Acc, $"]; +escape_all(<<>>, Acc, Orig, Skip, Len) -> + Part = binary_part(Orig, Skip, Len), + [Acc, Part, $"]; +escape_all(_Other, _Acc, Orig, Skip, Len) -> + invalid_byte(Orig, Skip + Len). + +escape_char(<>, Acc, Orig, Skip, Char) when Char =< 16#FF -> + Acc1 = [Acc, "\\u00" | integer_to_binary(Char, 16)], + escape_all(Rest, Acc1, Orig, Skip + 2, 0); +escape_char(<>, Acc, Orig, Skip, Char) when Char =< 16#7FF -> + Acc1 = [Acc, "\\u0" | integer_to_binary(Char, 16)], + escape_all(Rest, Acc1, Orig, Skip + 2, 0); +escape_char(<>, Acc, Orig, Skip, Char) when Char =< 16#FFF -> + Acc1 = [Acc, "\\u0" | integer_to_binary(Char, 16)], + escape_all(Rest, Acc1, Orig, Skip + 3, 0); +escape_char(<>, Acc, Orig, Skip, Char) when Char =< 16#FFFF -> + Acc1 = [Acc, "\\u" | integer_to_binary(Char, 16)], + escape_all(Rest, Acc1, Orig, Skip + 3, 0); +escape_char(<>, Acc, Orig, Skip, Char0) -> + Char = Char0 - 16#10000, + First = integer_to_binary(16#800 bor (Char bsr 10), 16), + Second = integer_to_binary(16#C00 bor (Char band 16#3FF), 16), + Acc1 = [Acc, "\\uD", First, "\\uD" | Second], + escape_all(Rest, Acc1, Orig, Skip + 4, 0). + +-spec escape(byte()) -> binary() | no. +escape($\x00) -> <<"\\u0000">>; +escape($\x01) -> <<"\\u0001">>; +escape($\x02) -> <<"\\u0002">>; +escape($\x03) -> <<"\\u0003">>; +escape($\x04) -> <<"\\u0004">>; +escape($\x05) -> <<"\\u0005">>; +escape($\x06) -> <<"\\u0006">>; +escape($\x07) -> <<"\\u0007">>; +escape($\b) -> <<"\\b">>; +escape($\t) -> <<"\\t">>; +escape($\n) -> <<"\\n">>; +escape($\x0b) -> <<"\\u000B">>; +escape($\f) -> <<"\\f">>; +escape($\r) -> <<"\\r">>; +escape($\x0e) -> <<"\\u000E">>; +escape($\x0f) -> <<"\\u000F">>; +escape($\x10) -> <<"\\u0010">>; +escape($\x11) -> <<"\\u0011">>; +escape($\x12) -> <<"\\u0012">>; +escape($\x13) -> <<"\\u0013">>; +escape($\x14) -> <<"\\u0014">>; +escape($\x15) -> <<"\\u0015">>; +escape($\x16) -> <<"\\u0016">>; +escape($\x17) -> <<"\\u0017">>; +escape($\x18) -> <<"\\u0018">>; +escape($\x19) -> <<"\\u0019">>; +escape($\x1A) -> <<"\\u001A">>; +escape($\x1B) -> <<"\\u001B">>; +escape($\x1C) -> <<"\\u001C">>; +escape($\x1D) -> <<"\\u001D">>; +escape($\x1E) -> <<"\\u001E">>; +escape($\x1F) -> <<"\\u001F">>; +escape($") -> <<"\\\"">>; +escape($\\) -> <<"\\\\">>; +escape(_) -> no. + +%% This is an adapted table from "Flexible and Economical UTF-8 Decoding" by Bjoern Hoehrmann. +%% http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ + +%% Map character to character class +utf8t() -> + { + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, + 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, + 8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, + 10,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 11,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8 + }. + +%% Transition table mapping combination of state & class to next state +utf8s() -> + { + 12,24,36,60,96,84,12,12,12,48,72, 12,12,12,12,12,12,12,12,12,12,12,12, + 12, 0,12,12,12,12,12, 0,12, 0,12,12, 12,24,12,12,12,12,12,24,12,24,12,12, + 12,12,12,12,12,12,12,24,12,12,12,12, 12,24,12,12,12,12,12,12,12,24,12,12, + 12,12,12,12,12,12,12,36,12,36,12,12, 12,36,12,12,12,12,12,36,12,36,12,12, + 12,36,12,12,12,12,12,12,12,12,12,12 + }. + +%% Optimisation for 1st byte direct state lookup, +%% we know starting state is 0 and ASCII bytes were already handled +utf8s0() -> + { + 12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12, + 12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12, + 12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12, + 12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12, + 12,12,24,24,24,24,24,24,24,24,24,24,24,24,24,24, + 24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24, + 48,36,36,36,36,36,36,36,36,36,36,36,36,60,36,36, + 72,84,84,84,96,12,12,12,12,12,12,12,12,12,12,12 + }. + +invalid_byte(Bin, Skip) -> + Byte = binary:at(Bin, Skip), + error({invalid_byte, Byte}, none, error_info(Skip)). + +error_info(Skip) -> + [{error_info, #{cause => #{position => Skip}}}]. + +%% +%% Decoding implementation +%% + +-define(ARRAY, array). +-define(OBJECT, object). + +-type from_binary_fun() :: fun((binary()) -> any()). +-type array_start_fun() :: fun((Acc :: any()) -> ArrayAcc :: any()). +-type array_push_fun() :: fun((Value :: any(), Acc :: any()) -> NewAcc :: any()). +-type array_finish_fun() :: fun((ArrayAcc :: any(), OldAcc :: any()) -> {any(), any()}). +-type object_start_fun() :: fun((Acc :: any()) -> ObjectAcc :: any()). +-type object_push_fun() :: fun((Key :: any(), Value :: any(), Acc :: any()) -> NewAcc :: any()). +-type object_finish_fun() :: fun((ObjectAcc :: any(), OldAcc :: any()) -> {any(), any()}). + +-type decoders() :: #{ + array_start => array_start_fun(), + array_push => array_push_fun(), + array_finish => array_finish_fun(), + object_start => object_start_fun(), + object_push => object_push_fun(), + object_finish => object_finish_fun(), + float => from_binary_fun(), + integer => from_binary_fun(), + string => from_binary_fun(), + null => term() +}. + +-record(decode, { + array_start :: array_start_fun() | undefined, + array_push :: array_push_fun() | undefined, + array_finish :: array_finish_fun() | undefined, + object_start :: object_start_fun() | undefined, + object_push :: object_push_fun() | undefined, + object_finish :: object_finish_fun() | undefined, + float = fun erlang:binary_to_float/1 :: from_binary_fun(), + integer = fun erlang:binary_to_integer/1 :: from_binary_fun(), + string :: from_binary_fun() | undefined, + null = null :: term() +}). + +-type acc() :: any(). +-type stack() :: [?ARRAY | ?OBJECT | binary() | acc()]. +-type decode() :: #decode{}. + +-opaque continuation_state() :: tuple(). + +-type decode_value() :: + integer() + | float() + | boolean() + | null + | binary() + | list(decode_value()) + | #{binary() => decode_value()}. + +-spec decode(binary()) -> decode_value(). +decode(Binary) when is_binary(Binary) -> + case value(Binary, Binary, 0, ok, [], #decode{}) of + {Result, _Acc, <<>>} -> + Result; + {_, _, Rest} -> + invalid_byte(Rest, 0); + {continue, {_Bin, _Acc, [], _Decode, {number, Number}}} -> + Number; + {continue, {_, _, _, _, {float_error, Token, Skip}}} -> + unexpected_sequence(Token, Skip); + {continue, _} -> + error(unexpected_end) + end. + +-spec decode(binary(), any(), decoders()) -> + {Result :: any(), Acc :: any(), binary()}. +decode(Binary, Acc0, Decoders) when is_binary(Binary) -> + Decode = maps:fold(fun parse_decoder/3, #decode{}, Decoders), + case value(Binary, Binary, 0, Acc0, [], Decode) of + {continue, {_Bin, Acc, [], _Decode, {number, Val}}} -> + {Val, Acc, <<>>}; + {continue, {_, _, _, _, {float_error, Token, Skip}}} -> + unexpected_sequence(Token, Skip); + {continue, _} -> + error(unexpected_end); + Result -> + Result + end. + +-spec decode_start(binary(), any(), decoders()) -> + {Result :: any(), Acc :: any(), binary()} | {continue, continuation_state()}. +decode_start(Binary, Acc, Decoders) when is_binary(Binary) -> + Decode = maps:fold(fun parse_decoder/3, #decode{}, Decoders), + value(Binary, Binary, 0, Acc, [], Decode). + +-spec decode_continue(binary() | end_of_input, Opaque::term()) -> + {Result :: any(), Acc :: any(), binary()} | {continue, continuation_state()}. +decode_continue(end_of_input, State) -> + case State of + {_, Acc, [], _Decode, {number, Val}} -> + {Val, Acc, <<>>}; + {_, _, _, _, {float_error, Token, Skip}} -> + unexpected_sequence(Token, Skip); + _ -> + error(unexpected_end) + end; +decode_continue(Cont, {Rest, Acc, Stack, #decode{} = Decode, FuncData}) when is_binary(Cont) -> + Binary = <>, + case FuncData of + value -> + value(Binary, Binary, 0, Acc, Stack, Decode); + {number, _} -> + value(Binary, Binary, 0, Acc, Stack, Decode); + {float_error, _Token, _Skip} -> + value(Binary, Binary, 0, Acc, Stack, Decode); + {array_push, Val} -> + array_push(Binary, Binary, 0, Acc, Stack, Decode, Val); + {object_value, Key} -> + object_value(Binary, Binary, 0, Acc, Stack, Decode, Key); + {object_push, Value, Key} -> + object_push(Binary, Binary, 0, Acc, Stack, Decode, Value, Key); + object_key -> + object_key(Binary, Binary, 0, Acc, Stack, Decode) + end. + +parse_decoder(array_start, Fun, Decode) when is_function(Fun, 1) -> + Decode#decode{array_start = Fun}; +parse_decoder(array_push, Fun, Decode) when is_function(Fun, 2) -> + Decode#decode{array_push = Fun}; +parse_decoder(array_finish, Fun, Decode) when is_function(Fun, 2) -> + Decode#decode{array_finish = Fun}; +parse_decoder(object_start, Fun, Decode) when is_function(Fun, 1) -> + Decode#decode{object_start = Fun}; +parse_decoder(object_push, Fun, Decode) when is_function(Fun, 3) -> + Decode#decode{object_push = Fun}; +parse_decoder(object_finish, Fun, Decode) when is_function(Fun, 2) -> + Decode#decode{object_finish = Fun}; +parse_decoder(float, Fun, Decode) when is_function(Fun, 1) -> + Decode#decode{float = Fun}; +parse_decoder(integer, Fun, Decode) when is_function(Fun, 1) -> + Decode#decode{integer = Fun}; +parse_decoder(string, Fun, Decode) when is_function(Fun, 1) -> + Decode#decode{string = Fun}; +parse_decoder(null, Null, Decode) -> + Decode#decode{null = Null}. + +value(<>, Original, Skip, Acc, Stack, Decode) when ?is_ws(Byte) -> + value(Rest, Original, Skip + 1, Acc, Stack, Decode); +value(<<$0, Rest/bits>>, Original, Skip, Acc, Stack, Decode) -> + number_zero(Rest, Original, Skip, Acc, Stack, Decode, 1); +value(<>, Original, Skip, Acc, Stack, Decode) when ?is_1_to_9(Byte) -> + number(Rest, Original, Skip, Acc, Stack, Decode, 1); +value(<<$-, Rest/bits>>, Original, Skip, Acc, Stack, Decode) -> + number_minus(Rest, Original, Skip, Acc, Stack, Decode); +value(<<$t, Rest/bits>>, Original, Skip, Acc, Stack, Decode) -> + true(Rest, Original, Skip, Acc, Stack, Decode); +value(<<$f, Rest/bits>>, Original, Skip, Acc, Stack, Decode) -> + false(Rest, Original, Skip, Acc, Stack, Decode); +value(<<$n, Rest/bits>>, Original, Skip, Acc, Stack, Decode) -> + null(Rest, Original, Skip, Acc, Stack, Decode); +value(<<$", Rest/bits>>, Original, Skip, Acc, Stack, Decode) -> + string(Rest, Original, Skip + 1, Acc, Stack, Decode); +value(<<$[, Rest/bits>>, Original, Skip, Acc, Stack, Decode) -> + array_start(Rest, Original, Skip, Acc, Stack, Decode, 1); +value(<<${, Rest/bits>>, Original, Skip, Acc, Stack, Decode) -> + object_start(Rest, Original, Skip, Acc, Stack, Decode, 1); +value(<>, Original, Skip, _Acc, _Stack, _Decode) when ?is_ascii_plain(Byte) -> + %% this clause is effectively the same as the last one, but necessary to + %% force compiler to emit a jump table dispatch, rather than binary search + invalid_byte(Original, Skip); +value(_, Original, Skip, Acc, Stack, Decode) -> + unexpected(Original, Skip, Acc, Stack, Decode, 0, 0, value). + +true(<<"rue", Rest/bits>>, Original, Skip, Acc, Stack, Decode) -> + continue(Rest, Original, Skip+4, Acc, Stack, Decode, true); +true(_Rest, Original, Skip, Acc, Stack, Decode) -> + unexpected(Original, Skip, Acc, Stack, Decode, 1, 3, value). + +false(<<"alse", Rest/bits>>, Original, Skip, Acc, Stack, Decode) -> + continue(Rest, Original, Skip+5, Acc, Stack, Decode, false); +false(_Rest, Original, Skip, Acc, Stack, Decode) -> + unexpected(Original, Skip, Acc, Stack, Decode, 1, 4, value). + +null(<<"ull", Rest/bits>>, Original, Skip, Acc, Stack, Decode) -> + continue(Rest, Original, Skip+4, Acc, Stack, Decode, Decode#decode.null); +null(_Rest, Original, Skip, Acc, Stack, Decode) -> + unexpected(Original, Skip, Acc, Stack, Decode, 1, 3, value). + +number_minus(<<$0, Rest/bits>>, Original, Skip, Acc, Stack, Decode) -> + number_zero(Rest, Original, Skip, Acc, Stack, Decode, 2); +number_minus(<>, Original, Skip, Acc, Stack, Decode) when ?is_1_to_9(Num) -> + number(Rest, Original, Skip, Acc, Stack, Decode, 2); +number_minus(_Rest, Original, Skip, Acc, Stack, Decode) -> + unexpected(Original, Skip, Acc, Stack, Decode, 1, 0, value). + +number_zero(<<$., Rest/bits>>, Original, Skip, Acc, Stack, Decode, Len) -> + number_frac(Rest, Original, Skip, Acc, Stack, Decode, Len + 1); +number_zero(<>, Original, Skip, Acc, Stack, Decode, Len) when E =:= $E; E =:= $e -> + number_exp_copy(Rest, Original, Skip, Acc, Stack, Decode, Len + 1, <<"0">>); +number_zero(<<>>, Original, Skip, Acc, Stack, Decode, Len) -> + Value = (Decode#decode.integer)(<<"0">>), + unexpected(Original, Skip, Acc, Stack, Decode, Len, 0, {number, Value}); +number_zero(Rest, Original, Skip, Acc, Stack, Decode, Len) -> + Value = (Decode#decode.integer)(<<"0">>), + continue(Rest, Original, Skip+Len, Acc, Stack, Decode, Value). + +number(<>, Original, Skip, Acc, Stack, Decode, Len) when ?is_0_to_9(Num) -> + number(Rest, Original, Skip, Acc, Stack, Decode, Len + 1); +number(<<$., Rest/bits>>, Original, Skip, Acc, Stack, Decode, Len) -> + number_frac(Rest, Original, Skip, Acc, Stack, Decode, Len + 1); +number(<>, Original, Skip, Acc, Stack, Decode, Len) when E =:= $E; E =:= $e -> + Prefix = binary_part(Original, Skip, Len), + number_exp_copy(Rest, Original, Skip, Acc, Stack, Decode, Len + 1, Prefix); +number(<<>>, Original, Skip, Acc, Stack, Decode, Len) -> + Int = (Decode#decode.integer)(binary_part(Original, Skip, Len)), + unexpected(Original, Skip, Acc, Stack, Decode, Len, 0, {number, Int}); +number(Rest, Original, Skip, Acc, Stack, Decode, Len) -> + Int = (Decode#decode.integer)(binary_part(Original, Skip, Len)), + continue(Rest, Original, Skip+Len, Acc, Stack, Decode, Int). + +number_frac(<>, Original, Skip, Acc, Stack, Decode, Len) when ?is_0_to_9(Byte) -> + number_frac_cont(Rest, Original, Skip, Acc, Stack, Decode, Len + 1); +number_frac(_, Original, Skip, Acc, Stack, Decode, Len) -> + unexpected(Original, Skip, Acc, Stack, Decode, Len, 0, value). + +number_frac_cont(<>, Original, Skip, Acc, Stack, Decode, Len) when ?is_0_to_9(Byte) -> + number_frac_cont(Rest, Original, Skip, Acc, Stack, Decode, Len + 1); +number_frac_cont(<>, Original, Skip, Acc, Stack, Decode, Len) when E =:= $e; E =:= $E -> + number_exp(Rest, Original, Skip, Acc, Stack, Decode, Len + 1); +number_frac_cont(Rest, Original, Skip, Acc, Stack, Decode, Len) -> + Token = binary_part(Original, Skip, Len), + float_decode(Rest, Original, Skip, Acc, Stack, Decode, Len, Token). + +float_decode(<<>>, Original, Skip, Acc, Stack, Decode, Len, Token) -> + try (Decode#decode.float)(Token) of + Float -> unexpected(Original, Skip, Acc, Stack, Decode, Len, 0, {number, Float}) + catch + _:_ -> unexpected(Original, Skip, Acc, Stack, Decode, Len, 0, {float_error, Token, Skip}) + end; +float_decode(<>, Original, Skip, Acc, Stack, Decode, Len, Token) -> + try (Decode#decode.float)(Token) of + Float -> + continue(Rest, Original, Skip+Len, Acc, Stack, Decode, Float) + catch + _:_ -> unexpected_sequence(Token, Skip) + end. + +number_exp(<>, Original, Skip, Acc, Stack, Decode, Len) when ?is_0_to_9(Byte) -> + number_exp_cont(Rest, Original, Skip, Acc, Stack, Decode, Len + 1); +number_exp(<>, Original, Skip, Acc, Stack, Decode, Len) when Sign =:= $+; Sign =:= $- -> + number_exp_sign(Rest, Original, Skip, Acc, Stack, Decode, Len + 1); +number_exp(_, Original, Skip, Acc, Stack, Decode, Len) -> + unexpected(Original, Skip, Acc, Stack, Decode, Len, 0, value). + +number_exp_sign(<>, Original, Skip, Acc, Stack, Decode, Len) when ?is_0_to_9(Byte) -> + number_exp_cont(Rest, Original, Skip, Acc, Stack, Decode, Len + 1); +number_exp_sign(_, Original, Skip, Acc, Stack, Decode, Len) -> + unexpected(Original, Skip, Acc, Stack, Decode, Len, 0, value). + +number_exp_cont(<>, Original, Skip, Acc, Stack, Decode, Len) when ?is_0_to_9(Byte) -> + number_exp_cont(Rest, Original, Skip, Acc, Stack, Decode, Len + 1); +number_exp_cont(Rest, Original, Skip, Acc, Stack, Decode, Len) -> + Token = binary_part(Original, Skip, Len), + float_decode(Rest, Original, Skip, Acc, Stack, Decode, Len, Token). + +number_exp_copy(<>, Original, Skip, Acc, Stack, Decode, Len, Prefix) when ?is_0_to_9(Byte) -> + number_exp_cont(Rest, Original, Skip, Acc, Stack, Decode, Len, Prefix, 1); +number_exp_copy(<>, Original, Skip, Acc, Stack, Decode, Len, Prefix) when Sign =:= $+; Sign =:= $- -> + number_exp_sign(Rest, Original, Skip, Acc, Stack, Decode, Len, Prefix, 1); +number_exp_copy(_, Original, Skip, Acc, Stack, Decode, Len, _Prefix) -> + unexpected(Original, Skip, Acc, Stack, Decode, Len, 0, value). + +number_exp_sign(<>, Original, Skip, Acc, Stack, Decode, Len, Prefix, ExpLen) when ?is_0_to_9(Byte) -> + number_exp_cont(Rest, Original, Skip, Acc, Stack, Decode, Len, Prefix, ExpLen + 1); +number_exp_sign(_, Original, Skip, Acc, Stack, Decode, Len, _Prefix, ExpLen) -> + unexpected(Original, Skip, Acc, Stack, Decode, Len + ExpLen, 0, value). + +number_exp_cont(<>, Original, Skip, Acc, Stack, Decode, Len, Prefix, ExpLen) when ?is_0_to_9(Byte) -> + number_exp_cont(Rest, Original, Skip, Acc, Stack, Decode, Len, Prefix, ExpLen + 1); +number_exp_cont(Rest, Original, Skip, Acc, Stack, Decode, Len, Prefix, ExpLen) -> + Suffix = binary_part(Original, Skip + Len, ExpLen), + Token = <>, + float_decode(Rest, Original, Skip, Acc, Stack, Decode, Len + ExpLen, Token). + +string(Binary, Original, Skip, Acc, Stack, Decode) -> + string_ascii(Binary, Original, Skip, Acc, Stack, Decode, 0). + +string_ascii(Binary, Original, Skip, Acc, Stack, Decode, Len) -> + case Binary of + <> when ?are_all_ascii_plain(B1, B2, B3, B4, B5, B6, B7, B8) -> + string_ascii(Rest, Original, Skip, Acc, Stack, Decode, Len + 8); + Other -> + string(Other, Original, Skip, Acc, Stack, Decode, Len) + end. + +-spec string(binary(), binary(), integer(), acc(), stack(), decode(), integer()) -> any(). +string(<>, Orig, Skip, Acc, Stack, Decode, Len) when ?is_ascii_plain(Byte) -> + string(Rest, Orig, Skip, Acc, Stack, Decode, Len + 1); +string(<<$\\, Rest/bits>>, Orig, Skip, Acc, Stack, Decode, Len) -> + Part = binary_part(Orig, Skip, Len), + SAcc = <<>>, + unescape(Rest, Orig, Skip, Acc, Stack, Decode, Skip-1, Len, <>); +string(<<$", Rest/bits>>, Orig, Skip0, Acc, Stack, Decode, Len) -> + Value = binary_part(Orig, Skip0, Len), + Skip = Skip0 + Len + 1, + case Decode#decode.string of + undefined -> continue(Rest, Orig, Skip, Acc, Stack, Decode, Value); + Fun -> continue(Rest, Orig, Skip, Acc, Stack, Decode, Fun(Value)) + end; +string(<>, Orig, Skip, _Acc, _Stack, _Decode, Len) when ?is_ascii_escape(Byte) -> + invalid_byte(Orig, Skip + Len); +string(<>, Orig, Skip, Acc, Stack, Decode, Len) -> + case element(Byte - 127, utf8s0()) of + ?UTF8_REJECT -> invalid_byte(Orig, Skip + Len); + %% all accept cases are ASCII, already covered above + State -> string_utf8(Rest, Orig, Skip, Acc, Stack, Decode, Len, State) + end; +string(_, Orig, Skip, Acc, Stack, Decode, Len) -> + unexpected(Orig, Skip-1, Acc, Stack, Decode, Len + 1, 0, value). + +string_utf8(<>, Orig, Skip, Acc, Stack, Decode, Len, State0) -> + Type = element(Byte + 1, utf8t()), + case element(State0 + Type, utf8s()) of + ?UTF8_ACCEPT -> string_ascii(Rest, Orig, Skip, Acc, Stack, Decode, Len + 2); + ?UTF8_REJECT -> invalid_byte(Orig, Skip + Len + 1); + State -> string_utf8(Rest, Orig, Skip, Acc, Stack, Decode, Len + 1, State) + end; +string_utf8(_, Orig, Skip, Acc, Stack, Decode, Len, _State0) -> + unexpected(Orig, Skip-1, Acc, Stack, Decode, Len + 2, 0, value). + +string_ascii(Binary, Original, Skip, Acc, Stack, Decode, Start, Len, SAcc) -> + case Binary of + <> when ?are_all_ascii_plain(B1, B2, B3, B4, B5, B6, B7, B8) -> + string_ascii(Rest, Original, Skip, Acc, Stack, Decode, Start, Len + 8, SAcc); + Other -> + string(Other, Original, Skip, Acc, Stack, Decode, Start, Len, SAcc) + end. + +-spec string(binary(), binary(), integer(), acc(), stack(), decode(), integer(), integer(), binary()) -> any(). +string(<>, Orig, Skip, Acc, Stack, Decode, Start, Len, SAcc) when ?is_ascii_plain(Byte) -> + string(Rest, Orig, Skip, Acc, Stack, Decode, Start, Len + 1, SAcc); +string(<<$\\, Rest/bits>>, Orig, Skip, Acc, Stack, Decode, Start, Len, SAcc) -> + Part = binary_part(Orig, Skip, Len), + unescape(Rest, Orig, Skip, Acc, Stack, Decode, Start, Len, <>); +string(<<$", Rest/bits>>, Orig, Skip0, Acc, Stack, Decode, _Start, Len, SAcc) -> + Part = binary_part(Orig, Skip0, Len), + Value = <>, + Skip = Skip0 + Len + 1, + case Decode#decode.string of + undefined -> continue(Rest, Orig, Skip, Acc, Stack, Decode, Value); + Fun -> continue(Rest, Orig, Skip, Acc, Stack, Decode, Fun(Value)) + end; +string(<>, Orig, Skip, _Acc, _Stack, _Decode, _Start, Len, _SAcc) when ?is_ascii_escape(Byte) -> + invalid_byte(Orig, Skip + Len); +string(<>, Orig, Skip, Acc, Stack, Decode, Start, Len, SAcc) -> + case element(Byte - 127, utf8s0()) of + ?UTF8_REJECT -> invalid_byte(Orig, Skip + Len); + %% all accept cases are ASCII, already covered above + State -> string_utf8(Rest, Orig, Skip, Acc, Stack, Decode, Start, Len, SAcc, State) + end; +string(_, Orig, Skip, Acc, Stack, Decode, Start, Len, _SAcc) -> + Extra = Skip - Start, + unexpected(Orig, Start, Acc, Stack, Decode, Len+Extra, 0, value). + +string_utf8(<>, Orig, Skip, Acc, Stack, Decode, Start, Len, SAcc, State0) -> + Type = element(Byte + 1, utf8t()), + case element(State0 + Type, utf8s()) of + ?UTF8_ACCEPT -> string_ascii(Rest, Orig, Skip, Acc, Stack, Decode, Start, Len + 2, SAcc); + ?UTF8_REJECT -> invalid_byte(Orig, Skip + Len + 1); + State -> string_utf8(Rest, Orig, Skip, Acc, Stack, Decode, Start, Len + 1, SAcc, State) + end; +string_utf8(_, Orig, Skip, Acc, Stack, Decode, Start, Len, _SAcc, _State0) -> + Extra = Skip - Start, + unexpected(Orig, Start, Acc, Stack, Decode, Len + 1 + Extra, 0, value). + +unescape(<>, Original, Skip, Acc, Stack, Decode, Start, Len, SAcc) -> + Val = + case Byte of + $b -> $\b; + $f -> $\f; + $n -> $\n; + $r -> $\r; + $t -> $\t; + $" -> $"; + $\\ -> $\\; + $/ -> $/; + $u -> unicode; + _ -> error + end, + case Val of + unicode -> unescapeu(Rest, Original, Skip, Acc, Stack, Decode, Start, Len, SAcc); + error -> invalid_byte(Original, Skip+Len+1); + Int -> string_ascii(Rest, Original, Skip + Len + 2, Acc, Stack, Decode, Start, 0, <>) + end; +unescape(_, Original, Skip, Acc, Stack, Decode, Start, Len, _SAcc) -> + Extra = Skip - Start, + unexpected(Original, Start, Acc, Stack, Decode, Len + 1 + Extra, 0, value). + +unescapeu(<>, Original, Skip, Acc, Stack, Decode, Start, Len, SAcc) -> + try hex_to_int(E1, E2, E3, E4) of + CP when CP >= 16#D800, CP =< 16#DBFF -> + unescape_surrogate(Rest, Original, Skip, Acc, Stack, Decode, Start, Len, SAcc, CP); + CP -> + try <> of + SAcc1 -> string_ascii(Rest, Original, Skip + Len + 6, Acc, Stack, Decode, Start, 0, SAcc1) + catch + _:_ -> unexpected_sequence(binary_part(Original, Skip + Len, 6), Skip + Len) + end + catch + _:_ -> + unexpected_sequence(binary_part(Original, Skip + Len, 6), Skip + Len) + end; +unescapeu(_Rest, Original, Skip, Acc, Stack, Decode, Start, Len, _SAcc) -> + Extra = Skip - Start, + unexpected(Original, Start, Acc, Stack, Decode, Len + 2 + Extra, 4, value). + +unescape_surrogate(<<"\\u", E1, E2, E3, E4, Rest/bits>>, Original, Skip, Acc, Stack, Decode, Start, Len, SAcc, Hi) -> + try hex_to_int(E1, E2, E3, E4) of + Lo when Lo >= 16#DC00, Lo =< 16#DFFF -> + CP = 16#10000 + ((Hi band 16#3FF) bsl 10) + (Lo band 16#3FF), + try <> of + SAcc1 -> string_ascii(Rest, Original, Skip + Len + 12, Acc, Stack, Decode, Start, 0, SAcc1) + catch + _:_ -> unexpected_sequence(binary_part(Original, Skip + Len, 12), Skip + Len) + end; + _ -> + unexpected_sequence(binary_part(Original, Skip + Len, 12), Skip + Len) + catch + _:_ -> unexpected_sequence(binary_part(Original, Skip + Len, 12), Skip + Len) + end; +unescape_surrogate(_Rest, Original, Skip, Acc, Stack, Decode, Start, Len, _SAcc, _Hi) -> + Extra = Skip - Start, + unexpected(Original, Start, Acc, Stack, Decode, Len + 6 + Extra, 5, value). + +%% erlfmt-ignore +%% this is a macro instead of an inlined function - compiler refused to inline +-define(hex_digit(C), element(C - $0 + 1, { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, n, n, n, n, n, %% 0x30 + n, n, 10,11,12,13,14,15,n, n, n, n, n, n, n, %% 0x40 + n, n, n, n, n, n, n, n, n, n, n, n, n, n, n, %% 0x50 + n, n, n, n, 10,11,12,13,14,15 %% 0x60 +})). + +-spec hex_to_int(byte(), byte(), byte(), byte()) -> integer(). +hex_to_int(H1, H2, H3, H4) -> + ?hex_digit(H4) + 16 * (?hex_digit(H3) + 16 * (?hex_digit(H2) + 16 * ?hex_digit(H1))). + +array_start(<>, Original, Skip, Acc, Stack, Decode, Len) when ?is_ws(Byte) -> + array_start(Rest, Original, Skip, Acc, Stack, Decode, Len+1); +array_start(<<"]", Rest/bits>>, Original, Skip, Acc, Stack, Decode, Len) -> + {Value, NewAcc} = + case {Decode#decode.array_start, Decode#decode.array_finish} of + {undefined, undefined} -> {[], Acc}; + {Start, undefined} -> {lists:reverse(Start(Acc)), Acc}; + {undefined, Finish} -> Finish([], Acc); + {Start, Finish} -> Finish(Start(Acc), Acc) + end, + continue(Rest, Original, Skip+Len+1, NewAcc, Stack, Decode, Value); +array_start(<<>>, Original, Skip, Acc, Stack, Decode, Len) -> + %% Handles empty array [] in continuation mode + unexpected(Original, Skip, Acc, Stack, Decode, Len, 0, value); +array_start(Rest, Original, Skip, OldAcc, Stack, Decode, Len) -> + case Decode#decode.array_start of + undefined -> value(Rest, Original, Skip+Len, [], [?ARRAY, OldAcc | Stack], Decode); + Fun -> value(Rest, Original, Skip+Len, Fun(OldAcc), [?ARRAY, OldAcc | Stack], Decode) + end. + +array_push(<>, Original, Skip, Acc, Stack, Decode, Value) when ?is_ws(Byte) -> + array_push(Rest, Original, Skip + 1, Acc, Stack, Decode, Value); +array_push(<<"]", Rest/bits>>, Original, Skip, Acc0, Stack0, Decode, Value) -> + Acc = + case Decode#decode.array_push of + undefined -> [Value | Acc0]; + Push -> Push(Value, Acc0) + end, + [_, OldAcc | Stack] = Stack0, + {ArrayValue, NewAcc} = + case Decode#decode.array_finish of + undefined -> {lists:reverse(Acc), OldAcc}; + Finish -> Finish(Acc, OldAcc) + end, + continue(Rest, Original, Skip + 1, NewAcc, Stack, Decode, ArrayValue); +array_push(<<$,, Rest/bits>>, Original, Skip0, Acc, Stack, Decode, Value) -> + Skip = Skip0 + 1, + case Decode#decode.array_push of + undefined -> value(Rest, Original, Skip, [Value | Acc], Stack, Decode); + Fun -> value(Rest, Original, Skip, Fun(Value, Acc), Stack, Decode) + end; +array_push(_, Original, Skip, Acc, Stack, Decode, Value) -> + unexpected(Original, Skip, Acc, Stack, Decode, 0, 0, {?FUNCTION_NAME, Value}). + + +object_start(<>, Original, Skip, Acc, Stack, Decode, Len) when ?is_ws(Byte) -> + object_start(Rest, Original, Skip, Acc, Stack, Decode, Len+1); +object_start(<<"}", Rest/bits>>, Original, Skip, Acc, Stack, Decode, Len) -> + {Value, NewAcc} = + case {Decode#decode.object_start, Decode#decode.object_finish} of + {undefined, undefined} -> {#{}, Acc}; + {Start, undefined} -> {maps:from_list(Start(Acc)), Acc}; + {undefined, Finish} -> Finish([], Acc); + {Start, Finish} -> Finish(Start(Acc), Acc) + end, + continue(Rest, Original, Skip+Len+1, NewAcc, Stack, Decode, Value); +object_start(<<$", Rest/bits>>, Original, Skip0, OldAcc, Stack0, Decode, Len) -> + Stack = [?OBJECT, OldAcc | Stack0], + Skip = Skip0 + Len + 1, + case Decode#decode.object_start of + undefined -> + string(Rest, Original, Skip, [], Stack, Decode); + Fun -> + Acc = Fun(OldAcc), + string(Rest, Original, Skip, Acc, Stack, Decode) + end; +object_start(_, Original, Skip, Acc, Stack, Decode, Len) -> + unexpected(Original, Skip, Acc, Stack, Decode, Len, 0, value). + +object_value(<>, Original, Skip, Acc, Stack, Decode, Key) when ?is_ws(Byte) -> + object_value(Rest, Original, Skip + 1, Acc, Stack, Decode, Key); +object_value(<<$:, Rest/bits>>, Original, Skip, Acc, Stack, Decode, Key) -> + value(Rest, Original, Skip + 1, Acc, [Key | Stack], Decode); +object_value(_, Original, Skip, Acc, Stack, Decode, Key) -> + unexpected(Original, Skip, Acc, Stack, Decode, 0, 0, {?FUNCTION_NAME, Key}). + +object_push(<>, Original, Skip, Acc, Stack, Decode, Value, Key) when ?is_ws(Byte) -> + object_push(Rest, Original, Skip + 1, Acc, Stack, Decode, Value, Key); +object_push(<<"}", Rest/bits>>, Original, Skip, Acc0, Stack0, Decode, Value, Key) -> + Acc = + case Decode#decode.object_push of + undefined -> [{Key, Value} | Acc0]; + Fun -> Fun(Key, Value, Acc0) + end, + [_, OldAcc | Stack] = Stack0, + {ObjectValue, NewAcc} = + case Decode#decode.object_finish of + undefined -> {maps:from_list(Acc), OldAcc}; + Finish -> Finish(Acc, OldAcc) + end, + continue(Rest, Original, Skip + 1, NewAcc, Stack, Decode, ObjectValue); +object_push(<<$,, Rest/bits>>, Original, Skip, Acc0, Stack, Decode, Value, Key) -> + case Decode#decode.object_push of + undefined -> object_key(Rest, Original, Skip + 1, [{Key, Value} | Acc0], Stack, Decode); + Fun -> object_key(Rest, Original, Skip + 1, Fun(Key, Value, Acc0), Stack, Decode) + end; +object_push(_, Original, Skip, Acc, Stack, Decode, Value, Key) -> + unexpected(Original, Skip, Acc, Stack, Decode, 0, 0, {?FUNCTION_NAME, Value, Key}). + +object_key(<>, Original, Skip, Acc, Stack, Decode) when ?is_ws(Byte) -> + object_key(Rest, Original, Skip + 1, Acc, Stack, Decode); +object_key(<<$", Rest/bits>>, Original, Skip, Acc, Stack, Decode) -> + string(Rest, Original, Skip + 1, Acc, Stack, Decode); +object_key(_, Original, Skip, Acc, Stack, Decode) -> + unexpected(Original, Skip, Acc, Stack, Decode, 0, 0, ?FUNCTION_NAME). + +continue(<>, Original, Skip, Acc, Stack0, Decode, Value) -> + case Stack0 of + [] -> terminate(Rest, Original, Skip, Acc, Value); + [?ARRAY | _] -> array_push(Rest, Original, Skip, Acc, Stack0, Decode, Value); + [?OBJECT | _] -> object_value(Rest, Original, Skip, Acc, Stack0, Decode, Value); + [Key | Stack] -> object_push(Rest, Original, Skip, Acc, Stack, Decode, Value, Key) + end. + +terminate(<>, Original, Skip, Acc, Value) when ?is_ws(Byte) -> + terminate(Rest, Original, Skip, Acc, Value); +terminate(<<>>, _, _Skip, Acc, Value) -> + {Value, Acc, <<>>}; +terminate(<<_/bits>>, Original, Skip, Acc, Value) -> + <<_:Skip/binary, Rest/binary>> = Original, + {Value, Acc, Rest}. + +-spec unexpected_utf8(binary(), non_neg_integer()) -> no_return(). +unexpected_utf8(Original, Skip) when byte_size(Original) =:= Skip -> + error(unexpected_end); +unexpected_utf8(Original, Skip) -> + invalid_byte(Original, Skip). + +unexpected(Original, Skip, Acc, Stack, Decode, Pos, Len, FuncData) -> + RequiredSize = Skip+Pos+Len, + OrigSize = byte_size(Original), + case OrigSize =< RequiredSize of + true -> + <<_:Skip/binary, Rest/binary>> = Original, + {continue, {Rest, Acc, Stack, Decode, FuncData}}; + false -> + invalid_byte(Original, Skip+Pos) + end. + +-spec unexpected_sequence(binary(), non_neg_integer()) -> no_return(). +unexpected_sequence(Value, Skip) -> + error({unexpected_sequence, Value}, none, error_info(Skip)). diff --git a/lib/elixir/src/elixir_lexical.erl b/lib/elixir/src/elixir_lexical.erl index 7f299442deb..b4c5d8d6f9f 100644 --- a/lib/elixir/src/elixir_lexical.erl +++ b/lib/elixir/src/elixir_lexical.erl @@ -1,79 +1,122 @@ +%% SPDX-License-Identifier: Apache-2.0 +%% SPDX-FileCopyrightText: 2021 The Elixir Team +%% SPDX-FileCopyrightText: 2012 Plataformatec + %% Module responsible for tracking lexical information. -module(elixir_lexical). --export([run/2, - record_alias/4, record_alias/2, - record_import/4, record_import/2, - record_remote/2, format_error/1 -]). +-export([run/3, with_file/3, trace/2, format_error/1]). -include("elixir.hrl"). --define(tracker, 'Elixir.Kernel.LexicalTracker'). +run(#{tracers := Tracers} = E, ExecutionCallback, AfterExecutionCallback) -> + case elixir_config:is_bootstrap() of + false -> + {ok, Pid} = ?tracker:start_link(), + LexEnv = E#{lexical_tracker := Pid, tracers := [?MODULE | Tracers]}, + elixir_env:trace(start, LexEnv), -run(File, Callback) -> - case code:is_loaded(?tracker) of - {file, _} -> - Pid = ?tracker:start_link(), - try - Callback(Pid) + try ExecutionCallback(LexEnv) of + Res -> + warn_unused_aliases(Pid, LexEnv), + warn_unused_imports(Pid, LexEnv), + warn_unused_requires(Pid, LexEnv), + AfterExecutionCallback(LexEnv), + Res after - warn_unused_aliases(File, Pid), - warn_unused_imports(File, Pid), - unlink(Pid), ?tracker:stop(Pid) + elixir_env:trace(stop, LexEnv), + unlink(Pid), + ?tracker:stop(Pid) end; - false -> - Callback(nil) - end. - -%% RECORD - -record_alias(Module, Line, Warn, Ref) -> - if_tracker(Ref, fun(Pid) -> - ?tracker:add_alias(Pid, Module, Line, Warn), - true - end). -record_import(Module, Line, Warn, Ref) -> - if_tracker(Ref, fun(Pid) -> - ?tracker:add_import(Pid, Module, Line, Warn), - true - end). + true -> + ExecutionCallback(E), + AfterExecutionCallback(E) + end. +trace({alias_expansion, _Meta, Lookup, _Result}, #{lexical_tracker := Pid}) -> + ?tracker:alias_dispatch(Pid, Lookup), + ok; +trace({require, Meta, Module, _Opts}, #{lexical_tracker := Pid}) -> + case lists:keyfind(from_macro, 1, Meta) of + {from_macro, true} -> ?tracker:remote_dispatch(Pid, Module, compile); + _ -> ?tracker:add_export(Pid, Module) + end, + ok; +trace({struct_expansion, _Meta, Module, _Keys}, #{lexical_tracker := Pid}) -> + ?tracker:add_export(Pid, Module), + ok; +trace({alias_reference, _Meta, Module}, #{lexical_tracker := Pid} = E) -> + case E of + %% Alias references inside patterns and guards in functions are not + %% compile time dependencies. + #{function := nil} -> ?tracker:remote_dispatch(Pid, Module, compile); + #{context := nil} -> ?tracker:remote_dispatch(Pid, Module, runtime); + #{} -> ok + end, + ok; +trace({remote_function, _Meta, Module, _Function, _Arity}, #{lexical_tracker := Pid} = E) -> + ?tracker:remote_dispatch(Pid, Module, mode(E)), + ok; +trace({remote_macro, _Meta, Module, _Function, _Arity}, #{lexical_tracker := Pid}) -> + ?tracker:remote_dispatch(Pid, Module, compile), + ok; +trace({imported_function, _Meta, Module, Function, Arity}, #{lexical_tracker := Pid} = E) -> + ?tracker:import_dispatch(Pid, Module, {Function, Arity}, mode(E)), + ok; +trace({imported_macro, _Meta, Module, Function, Arity}, #{lexical_tracker := Pid}) -> + ?tracker:import_dispatch(Pid, Module, {Function, Arity}, compile), + ok; +trace({imported_quoted, _Meta, Module, Function, Arities}, #{lexical_tracker := Pid}) -> + ?tracker:import_quoted(Pid, Module, Function, Arities), + ok; +trace({compile_env, App, Path, Return}, #{lexical_tracker := Pid}) -> + ?tracker:add_compile_env(Pid, App, Path, Return), + ok; +trace(_, _) -> + ok. -record_alias(Module, Ref) -> - if_tracker(Ref, fun(Pid) -> - ?tracker:alias_dispatch(Pid, Module), - true - end). +mode(#{function := nil}) -> compile; +mode(#{}) -> runtime. -record_import(Module, Ref) -> - if_tracker(Ref, fun(Pid) -> - ?tracker:import_dispatch(Pid, Module), - true - end). +%% EXTERNAL SOURCES -record_remote(Module, Ref) -> - if_tracker(Ref, fun(Pid) -> - ?tracker:remote_dispatch(Pid, Module), - true - end). +with_file(File, #{lexical_tracker := nil} = E, Callback) -> + Callback(E#{file := File}); +with_file(File, #{lexical_tracker := Pid} = E, Callback) -> + try + ?tracker:set_file(Pid, File), + Callback(E#{file := File}) + after + ?tracker:reset_file(Pid) + end. -%% HELPERS +%% ERROR HANDLING -if_tracker(nil, _Callback) -> false; -if_tracker(Pid, Callback) when is_pid(Pid) -> Callback(Pid). +warn_unused_imports(Pid, E) -> + [elixir_errors:file_warn(Meta, ?key(E, file), ?MODULE, {unused_import, ModOrMFA}) + || {Module, Imports} <- ?tracker:collect_unused_imports(Pid), + {ModOrMFA, Meta} <- unused_imports_for_module(Module, Imports)], + ok. -%% ERROR HANDLING +warn_unused_requires(Pid, E) -> + [elixir_errors:file_warn(Meta, ?key(E, file), ?MODULE, {unused_require, Module}) + || {Module, Meta} <- ?tracker:collect_unused_requires(Pid)], + ok. -warn_unused_imports(File, Pid) -> - [ begin - elixir_errors:handle_file_warning(File, {L, ?MODULE, {unused_import, M}}) - end || {M, L} <- ?tracker:collect_unused_imports(Pid)]. +unused_imports_for_module(Module, Imports) -> + case Imports of + #{Module := Meta} -> [{Module, Meta}]; + #{} -> [{{Module, Fun, Arity}, Meta} || {{Fun, Arity}, Meta} <- maps:to_list(Imports)] + end. -warn_unused_aliases(File, Pid) -> - [ begin - elixir_errors:handle_file_warning(File, {L, ?MODULE, {unused_alias, M}}) - end || {M, L} <- ?tracker:collect_unused_aliases(Pid)]. +warn_unused_aliases(Pid, E) -> + [elixir_errors:file_warn(Meta, ?key(E, file), ?MODULE, {unused_alias, Module}) + || {Module, Meta} <- ?tracker:collect_unused_aliases(Pid)], + ok. format_error({unused_alias, Module}) -> io_lib:format("unused alias ~ts", [elixir_aliases:inspect(Module)]); +format_error({unused_import, {Module, Function, Arity}}) -> + io_lib:format("unused import ~ts.~ts/~w", [elixir_aliases:inspect(Module), Function, Arity]); format_error({unused_import, Module}) -> - io_lib:format("unused import ~ts", [elixir_aliases:inspect(Module)]). + io_lib:format("unused import ~ts", [elixir_aliases:inspect(Module)]); +format_error({unused_require, Module}) -> + io_lib:format("unused require ~ts", [elixir_aliases:inspect(Module)]). diff --git a/lib/elixir/src/elixir_locals.erl b/lib/elixir/src/elixir_locals.erl deleted file mode 100644 index 92991d772a0..00000000000 --- a/lib/elixir/src/elixir_locals.erl +++ /dev/null @@ -1,181 +0,0 @@ -%% Module responsible for tracking invocations of module calls. --module(elixir_locals). --export([ - setup/1, cleanup/1, cache_env/1, get_cached_env/1, - record_local/2, record_local/3, record_import/4, - record_definition/3, record_defaults/4, - ensure_no_function_conflict/4, warn_unused_local/3, format_error/1 -]). --export([macro_for/3, local_for/3, local_for/4]). - --include("elixir.hrl"). --define(attr, '__locals_tracker'). --define(tracker, 'Elixir.Module.LocalsTracker'). - -macro_for(Module, Name, Arity) -> - Tuple = {Name, Arity}, - try elixir_def:lookup_definition(Module, Tuple) of - {{Tuple, Kind, Line, _, _, _, _}, [_|_] = Clauses} - when Kind == defmacro; Kind == defmacrop -> - fun() -> get_function(Line, Module, Clauses) end; - _ -> - false - catch - error:badarg -> false - end. - -local_for(Module, Name, Arity) -> - local_for(Module, Name, Arity, nil). -local_for(Module, Name, Arity, Given) -> - Tuple = {Name, Arity}, - case elixir_def:lookup_definition(Module, Tuple) of - {{Tuple, Kind, Line, _, _, _, _}, [_|_] = Clauses} - when Given == nil; Kind == Given -> - get_function(Line, Module, Clauses); - _ -> - [_|T] = erlang:get_stacktrace(), - erlang:raise(error, undef, [{Module,Name,Arity,[]}|T]) - end. - -get_function(Line, Module, Clauses) -> - RewrittenClauses = [rewrite_clause(Clause, Module) || Clause <- Clauses], - Fun = {'fun', Line, {clauses, RewrittenClauses}}, - {value, Result, _Binding} = erl_eval:exprs([Fun], []), - Result. - -rewrite_clause({call, Line, {atom, Line, RawName}, Args}, Module) -> - Remote = {remote, Line, - {atom, Line, ?MODULE}, - {atom, Line, local_for} - }, - - %% If we have a macro, its arity in the table is - %% actually one less than in the function call - {Name, Arity} = case atom_to_list(RawName) of - "MACRO-" ++ Rest -> {list_to_atom(Rest), length(Args) - 1}; - _ -> {RawName, length(Args)} - end, - - FunCall = {call, Line, Remote, [ - {atom, Line, Module}, {atom, Line, Name}, {integer, Line, Arity} - ]}, - {call, Line, FunCall, Args}; - -rewrite_clause(Tuple, Module) when is_tuple(Tuple) -> - list_to_tuple(rewrite_clause(tuple_to_list(Tuple), Module)); - -rewrite_clause(List, Module) when is_list(List) -> - [rewrite_clause(Item, Module) || Item <- List]; - -rewrite_clause(Else, _) -> Else. - -%% TRACKING - -setup(Module) -> - case code:is_loaded(?tracker) of - {file, _} -> ets:insert(Module, {?attr, ?tracker:start_link()}); - false -> ok - end. - -cleanup(Module) -> - if_tracker(Module, fun(Pid) -> unlink(Pid), ?tracker:stop(Pid) end). - -record_local(Tuple, Module) when is_atom(Module) -> - if_tracker(Module, fun(Pid) -> - ?tracker:add_local(Pid, Tuple), - true - end). -record_local(Tuple, _Module, Function) - when Function == nil; Function == Tuple -> false; -record_local(Tuple, Module, Function) -> - if_tracker(Module, fun(Pid) -> - ?tracker:add_local(Pid, Function, Tuple), - true - end). - -record_import(_Tuple, Receiver, Module, _Function) - when Module == nil; Module == Receiver -> false; -record_import(Tuple, Receiver, Module, Function) -> - if_tracker(Module, fun(Pid) -> - ?tracker:add_import(Pid, Function, Receiver, Tuple), - true - end). - -record_definition(Tuple, Kind, Module) -> - if_tracker(Module, fun(Pid) -> - ?tracker:add_definition(Pid, Kind, Tuple), - true - end). - -record_defaults(_Tuple, _Kind, _Module, 0) -> - true; -record_defaults(Tuple, Kind, Module, Defaults) -> - if_tracker(Module, fun(Pid) -> - ?tracker:add_defaults(Pid, Kind, Tuple, Defaults), - true - end). - -if_tracker(Module, Callback) -> - try ets:lookup_element(Module, ?attr, 2) of - Pid -> Callback(Pid) - catch - error:badarg -> false - end. - -%% CACHING - -cache_env(#{module := Module} = RE) -> - E = RE#{line := nil,vars := []}, - try ets:lookup_element(Module, ?attr, 2) of - Pid -> - {Pid, ?tracker:cache_env(Pid, E)} - catch - error:badarg -> - {Escaped, _} = elixir_quote:escape(E, false), - Escaped - end. - -get_cached_env({Pid,Ref}) -> ?tracker:get_cached_env(Pid, Ref); -get_cached_env(Env) -> Env. - -%% ERROR HANDLING - -ensure_no_function_conflict(Meta, File, Module, AllDefined) -> - if_tracker(Module, fun(Pid) -> - [ begin - elixir_errors:form_error(Meta, File, ?MODULE, {function_conflict, Error}) - end || Error <- ?tracker:collect_imports_conflicts(Pid, AllDefined) ] - end), - ok. - -warn_unused_local(File, Module, Private) -> - if_tracker(Module, fun(Pid) -> - Args = [ {Fun, Kind, Defaults} || - {Fun, Kind, _Line, true, Defaults} <- Private], - - Unused = ?tracker:collect_unused_locals(Pid, Args), - - [ begin - {_, _, Line, _, _} = lists:keyfind(element(2, Error), 1, Private), - elixir_errors:handle_file_warning(File, {Line, ?MODULE, Error}) - end || Error <- Unused ] - end). - -format_error({function_conflict,{Receivers, Name, Arity}}) -> - io_lib:format("imported ~ts.~ts/~B conflicts with local function", - [elixir_aliases:inspect(hd(Receivers)), Name, Arity]); - -format_error({unused_args,{Name, Arity}}) -> - io_lib:format("default arguments in ~ts/~B are never used", [Name, Arity]); - -format_error({unused_args,{Name, Arity},1}) -> - io_lib:format("the first default argument in ~ts/~B is never used", [Name, Arity]); - -format_error({unused_args,{Name, Arity},Count}) -> - io_lib:format("the first ~B default arguments in ~ts/~B are never used", [Count, Name, Arity]); - -format_error({unused_def,{Name, Arity},defp}) -> - io_lib:format("function ~ts/~B is unused", [Name, Arity]); - -format_error({unused_def,{Name, Arity},defmacrop}) -> - io_lib:format("macro ~ts/~B is unused", [Name, Arity]). diff --git a/lib/elixir/src/elixir_map.erl b/lib/elixir/src/elixir_map.erl index 957530e8761..d38597e5864 100644 --- a/lib/elixir/src/elixir_map.erl +++ b/lib/elixir/src/elixir_map.erl @@ -1,175 +1,302 @@ +%% SPDX-License-Identifier: Apache-2.0 +%% SPDX-FileCopyrightText: 2021 The Elixir Team +%% SPDX-FileCopyrightText: 2012 Plataformatec + -module(elixir_map). --export([expand_map/3, translate_map/3, expand_struct/4, translate_struct/4]). --import(elixir_errors, [compile_error/4]). +-export([expand_map/4, expand_struct/5, format_error/1, maybe_load_struct_info/5]). +-import(elixir_errors, [function_error/4, file_error/4, file_warn/4]). -include("elixir.hrl"). -expand_map(Meta, [{'|', UpdateMeta, [Left, Right]}], E) -> - {[ELeft|ERight], EA} = elixir_exp:expand_args([Left|Right], E), - {{'%{}', Meta, [{'|', UpdateMeta, [ELeft, ERight]}]}, EA}; -expand_map(Meta, Args, E) -> - {EArgs, EA} = elixir_exp:expand_args(Args, E), - {{'%{}', Meta, EArgs}, EA}. +expand_map(Meta, [{'|', UpdateMeta, [Left, Right]}], S, #{context := nil} = E) -> + {[ELeft | ERight], SE, EE} = elixir_expand:expand_args([Left | Right], S, E), + validate_kv(Meta, ERight, Right, E), + {{'%{}', Meta, [{'|', UpdateMeta, [ELeft, ERight]}]}, SE, EE}; +expand_map(Meta, [{'|', _, [_, _]}] = Args, _S, #{context := Context, file := File}) -> + file_error(Meta, File, ?MODULE, {update_syntax_in_wrong_context, Context, {'%{}', Meta, Args}}); +expand_map(Meta, Args, S, E) -> + {EArgs, SE, EE} = elixir_expand:expand_args(Args, S, E), + validate_kv(Meta, EArgs, Args, E), + {{'%{}', Meta, EArgs}, SE, EE}. + +expand_struct(Meta, Left, {'%{}', MapMeta, MapArgs}, S, #{context := Context} = E) -> + CleanMapArgs = delete_struct_key(Meta, MapArgs, E), + {[ELeft, ERight], SE, EE} = elixir_expand:expand_args([Left, {'%{}', MapMeta, CleanMapArgs}], S, E), + + case validate_struct(ELeft, Context) of + true when is_atom(ELeft) -> + case ERight of + {'%{}', MapMeta, [{'|', _, [_, Assocs]}]} -> + _ = load_struct_info(Meta, ELeft, Assocs, EE), + {{'%', Meta, [ELeft, ERight]}, SE, EE}; -expand_struct(Meta, Left, Right, E) -> - {[ELeft, ERight], EE} = elixir_exp:expand_args([Left, Right], E), + {'%{}', MapMeta, Assocs} when Context /= match -> + AssocKeys = [K || {K, _} <- Assocs], + Struct = load_struct(Meta, ELeft, Assocs, EE), + Keys = ['__struct__'] ++ AssocKeys, + WithoutKeys = lists:sort(maps:to_list(maps:without(Keys, Struct))), + StructAssocs = elixir_quote:escape(WithoutKeys, escape, false), + {{'%', Meta, [ELeft, {'%{}', MapMeta, StructAssocs ++ Assocs}]}, SE, EE}; + + {'%{}', MapMeta, Assocs} -> + _ = load_struct_info(Meta, ELeft, Assocs, EE), + {{'%', Meta, [ELeft, ERight]}, SE, EE} + end; + + true -> + {{'%', Meta, [ELeft, ERight]}, SE, EE}; + + false when Context == match -> + file_error(Meta, E, ?MODULE, {invalid_struct_name_in_match, ELeft}); - case is_atom(ELeft) of - true -> ok; false -> - compile_error(Meta, ?m(E, file), "expected struct name to be a compile " - "time atom or alias, got: ~ts", ['Elixir.Macro':to_string(ELeft)]) - end, - - EMeta = - case lists:member(ELeft, ?m(E, context_modules)) of - true -> - case (ELeft == ?m(E, module)) and - (?m(E, function) == nil) of - true -> - compile_error(Meta, ?m(E, file), - "cannot access struct ~ts in body of the module that defines it as " - "the struct fields are not yet accessible", - [elixir_aliases:inspect(ELeft)]); - false -> - [{struct, context}|Meta] - end; - false -> - Meta - end, + file_error(Meta, E, ?MODULE, {invalid_struct_name, ELeft}) + end; +expand_struct(Meta, _Left, Right, _S, E) -> + file_error(Meta, E, ?MODULE, {non_map_after_struct, Right}). + +delete_struct_key(Meta, [{'|', PipeMeta, [Left, MapAssocs]}], E) -> + [{'|', PipeMeta, [Left, delete_struct_key_assoc(Meta, MapAssocs, E)]}]; +delete_struct_key(Meta, MapAssocs, E) -> + delete_struct_key_assoc(Meta, MapAssocs, E). - case ERight of - {'%{}', _, _} -> ok; - _ -> compile_error(Meta, ?m(E, file), - "expected struct to be followed by a map, got: ~ts", - ['Elixir.Macro':to_string(ERight)]) - end, +delete_struct_key_assoc(Meta, Assocs, E) -> + case lists:keytake('__struct__', 1, Assocs) of + {value, _, CleanAssocs} -> + file_warn(Meta, ?key(E, file), ?MODULE, ignored_struct_key_in_struct), + CleanAssocs; + false -> + Assocs + end. - {{'%', EMeta, [ELeft, ERight]}, EE}. +validate_match_key(Meta, {Name, _, Context}, E) when is_atom(Name), is_atom(Context) -> + file_error(Meta, E, ?MODULE, {invalid_variable_in_map_key_match, Name}); +validate_match_key(Meta, {'::', _, [Left, _]}, E) -> + validate_match_key(Meta, Left, E); +validate_match_key(_, {'^', _, [{Name, _, Context}]}, _) when is_atom(Name), is_atom(Context) -> + ok; +validate_match_key(_, {'%{}', _, [_ | _]}, _) -> + ok; +validate_match_key(Meta, {Left, _, Right}, E) -> + validate_match_key(Meta, Left, E), + validate_match_key(Meta, Right, E); +validate_match_key(Meta, {Left, Right}, E) -> + validate_match_key(Meta, Left, E), + validate_match_key(Meta, Right, E); +validate_match_key(Meta, List, E) when is_list(List) -> + [validate_match_key(Meta, Each, E) || Each <- List]; +validate_match_key(_, _, _) -> + ok. -translate_map(Meta, Args, S) -> - {Assocs, TUpdate, US} = extract_assoc_update(Args, S), - translate_map(Meta, Assocs, TUpdate, US). +validate_not_repeated(Meta, Key, Used, E) -> + case is_literal(Key) andalso Used of + #{Key := true} -> + case E of + #{context := match} -> function_error(Meta, ?key(E, file), ?MODULE, {repeated_key, Key}); + _ -> file_warn(Meta, ?key(E, file), ?MODULE, {repeated_key, Key}) + end, + Used; -translate_struct(Meta, Name, {'%{}', MapMeta, Args}, S) -> - {Assocs, TUpdate, US} = extract_assoc_update(Args, S), - Struct = load_struct(Meta, Name, S), + #{} -> + Used#{Key => true}; - case is_map(Struct) of - true -> - assert_struct_keys(Meta, Name, Struct, Assocs, S); false -> - compile_error(Meta, S#elixir_scope.file, "expected ~ts.__struct__/0 to " - "return a map, got: ~ts", [elixir_aliases:inspect(Name), 'Elixir.Kernel':inspect(Struct)]) - end, + Used + end. - if - TUpdate /= nil -> - Line = ?line(Meta), - {VarName, _, VS} = elixir_scope:build_var('_', US), +is_literal({_, _, _}) -> false; +is_literal({Left, Right}) -> is_literal(Left) andalso is_literal(Right); +is_literal([_ | _] = List) -> lists:all(fun is_literal/1, List); +is_literal(_) -> true. - Var = {var, Line, VarName}, - Map = {map, Line, [{map_field_exact, Line, {atom, Line, '__struct__'}, {atom, Line, Name}}]}, +validate_kv(Meta, KV, Original, #{context := Context} = E) -> + lists:foldl(fun + ({K, _V}, {Index, Used}) -> + (Context == match) andalso validate_match_key(Meta, K, E), + NewUsed = validate_not_repeated(Meta, K, Used, E), + {Index + 1, NewUsed}; + (_, {Index, _Used}) -> + file_error(Meta, E, ?MODULE, {not_kv_pair, lists:nth(Index, Original)}) + end, {1, #{}}, KV). - Match = {match, Line, Var, Map}, - Error = {tuple, Line, [{atom, Line, badstruct}, {atom, Line, Name}, Var]}, +validate_struct({'^', _, [{Var, _, Ctx}]}, match) when is_atom(Var), is_atom(Ctx) -> true; +validate_struct({Var, _Meta, Ctx}, match) when is_atom(Var), is_atom(Ctx) -> true; +validate_struct(Atom, _) when is_atom(Atom) -> true; +validate_struct(_, _) -> false. - {TMap, TS} = translate_map(MapMeta, Assocs, Var, VS), +load_struct_info(Meta, Name, Assocs, E) -> + assert_struct_assocs(Meta, Assocs, E), - {{'case', Line, TUpdate, [ - {clause, Line, [Match], [], [TMap]}, - {clause, Line, [Var], [], [elixir_utils:erl_call(Line, erlang, error, [Error])]} - ]}, TS}; - S#elixir_scope.context == match -> - translate_map(MapMeta, Assocs ++ [{'__struct__', Name}], nil, US); - true -> - Keys = [K || {K,_} <- Assocs], - {StructAssocs, _} = elixir_quote:escape(maps:to_list(maps:without(Keys, Struct)), false), - translate_map(MapMeta, StructAssocs ++ Assocs ++ [{'__struct__', Name}], nil, US) + case maybe_load_struct_info(Meta, Name, Assocs, true, E) of + {ok, Info} -> Info; + {error, Desc} -> file_error(Meta, E, ?MODULE, Desc) + end. + +maybe_load_struct_info(Meta, Name, Assocs, Trace, E) -> + try + case is_open(Name, Meta, E) andalso lookup_struct_info_from_data_tables(Name) of + %% If I am accessing myself and there is no attribute, + %% don't invoke the fallback to avoid calling loaded code. + false when ?key(E, module) =:= Name -> nil; + false -> Name:'__info__'(struct); + InfoList -> InfoList + end + of + nil -> + {error, detail_undef(Name, E)}; + + Info -> + Keys = [begin + lists:any(fun(Field) -> ?key(Field, field) =:= Key end, Info) orelse + function_error(Meta, E, ?MODULE, {unknown_key_for_struct, Name, Key}), + Key + end || {Key, _} <- Assocs], + Trace andalso elixir_env:trace({struct_expansion, Meta, Name, Keys}, E), + {ok, Info} + catch + error:undef -> {error, detail_undef(Name, E)} end. -%% Helpers +lookup_struct_info_from_data_tables(Module) -> + try + {Set, _} = elixir_module:data_tables(Module), + ets:lookup_element(Set, {elixir, struct}, 2) + catch + _:_ -> false + end. -load_struct(Meta, Name, S) -> - Local = - elixir_module:is_open(Name) andalso - (case lists:keyfind(struct, 1, Meta) of - {struct, context} -> true; - _ -> wait_for_struct(Name) - end), +load_struct(Meta, Name, Assocs, E) -> + assert_struct_assocs(Meta, Assocs, E), try - case Local of - true -> + maybe_load_struct(Meta, Name, Assocs, E) + of + {ok, Struct} -> Struct; + {error, Desc} -> file_error(Meta, E, ?MODULE, Desc) + catch + Kind:Reason -> + Info = [{Name, '__struct__', 1, [{file, "expanding struct"}]}, + elixir_utils:caller(?line(Meta), ?key(E, file), ?key(E, module), ?key(E, function))], + erlang:raise(Kind, Reason, Info) + end. + +maybe_load_struct(Meta, Name, Assocs, E) -> + try + case is_open(Name, Meta, E) andalso elixir_def:external_for(Meta, Name, '__struct__', 1, [def]) of + %% If I am accessing myself and there is no __struct__ function, + %% don't invoke the fallback to avoid calling loaded code. + false when ?key(E, module) =:= Name -> + error(undef); + + false -> + Name:'__struct__'(Assocs); + + ExternalFun -> + %% There is an inherent race condition when using external_for. + %% By the time we got to execute the function, the ETS table + %% with temporary definitions for the given module may no longer + %% be available, so any function invocation happening inside the + %% local function will fail. In this case, we need to fall back to + %% the regular dispatching since the module will be available if + %% the table has not been deleted (unless compilation of that + %% module failed which should then cause this call to fail too). try - (elixir_locals:local_for(Name, '__struct__', 0, def))() + ExternalFun(Assocs) catch - error:undef -> Name:'__struct__'(); - error:badarg -> Name:'__struct__'() - end; - false -> - Name:'__struct__'() + error:undef -> Name:'__struct__'(Assocs) + end end + of + #{'__struct__' := Name} = Struct -> + Keys = [begin + maps:is_key(Key, Struct) orelse + function_error(Meta, E, ?MODULE, {unknown_key_for_struct, Name, Key}), + Key + end || {Key, _} <- Assocs], + elixir_env:trace({struct_expansion, Meta, Name, Keys}, E), + {ok, Struct}; + + #{'__struct__' := StructName} when is_atom(StructName) -> + {error, {struct_name_mismatch, Name, StructName}}; + + Other -> + {error, {invalid_struct_return_value, Name, Other}} catch - error:undef -> - Inspected = elixir_aliases:inspect(Name), - compile_error(Meta, S#elixir_scope.file, "~ts.__struct__/0 is undefined, " - "cannot expand struct ~ts", [Inspected, Inspected]) + error:undef -> {error, detail_undef(Name, E)} end. -wait_for_struct(Module) -> - case erlang:get(elixir_compiler_pid) of - undefined -> - false; - Pid -> - Ref = erlang:make_ref(), - Pid ! {waiting, struct, self(), Ref, Module}, - receive - {Ref, ready} -> - true; - {Ref, release} -> - 'Elixir.Kernel.ErrorHandler':release(), - false - end +assert_struct_assocs(Meta, Assocs, E) -> + [function_error(Meta, E, ?MODULE, {invalid_key_for_struct, K}) + || {K, _} <- Assocs, not is_atom(K)]. + +is_open(Name, Meta, E) -> + in_context(Name, E) orelse ((code:ensure_loaded(Name) /= {module, Name}) andalso wait_for_struct(Name, Meta, E)). + +in_context(Name, E) -> + %% We also include the current module because it won't be present + %% in context module in case the module name is defined dynamically. + lists:member(Name, [?key(E, module) | ?key(E, context_modules)]). + +wait_for_struct(Module, Meta, E) -> + (erlang:get(elixir_compiler_info) /= undefined) andalso + ('Elixir.Kernel.ErrorHandler':ensure_compiled(Module, struct, hard, elixir_utils:get_line(Meta, E)) =:= found). + +detail_undef(Name, E) -> + case in_context(Name, E) andalso (?key(E, function) == nil) of + true -> {inaccessible_struct, Name}; + false -> {undefined_struct, Name} end. -translate_map(Meta, Assocs, TUpdate, #elixir_scope{extra=Extra} = S) -> - {Op, KeyFun, ValFun} = extract_key_val_op(TUpdate, S), - - Line = ?line(Meta), - - {TArgs, SA} = lists:mapfoldl(fun - ({Key, Value}, Acc) -> - {TKey, Acc1} = KeyFun(Key, Acc), - {TValue, Acc2} = ValFun(Value, Acc1#elixir_scope{extra=Extra}), - {{Op, ?line(Meta), TKey, TValue}, Acc2}; - (Other, _Acc) -> - compile_error(Meta, S#elixir_scope.file, "expected key-value pairs in map, got: ~ts", - ['Elixir.Macro':to_string(Other)]) - end, S, Assocs), - - build_map(Line, TUpdate, TArgs, SA). - -extract_assoc_update([{'|', _Meta, [Update, Args]}], S) -> - {TArg, SA} = elixir_translator:translate_arg(Update, S, S), - {Args, TArg, SA}; -extract_assoc_update(Args, SA) -> {Args, nil, SA}. - -extract_key_val_op(_TUpdate, #elixir_scope{context=match}) -> - {map_field_exact, - fun(X, Acc) -> elixir_translator:translate(X, Acc#elixir_scope{extra=map_key}) end, - fun elixir_translator:translate/2}; -extract_key_val_op(TUpdate, S) -> - KS = S#elixir_scope{extra=map_key}, - Op = if TUpdate == nil -> map_field_assoc; true -> map_field_exact end, - {Op, - fun(X, Acc) -> elixir_translator:translate_arg(X, Acc, KS) end, - fun(X, Acc) -> elixir_translator:translate_arg(X, Acc, S) end}. - -build_map(Line, nil, TArgs, SA) -> {{map, Line, TArgs}, SA}; -build_map(Line, TUpdate, TArgs, SA) -> {{map, Line, TUpdate, TArgs}, SA}. - -assert_struct_keys(Meta, Name, Struct, Assocs, S) -> - [begin - compile_error(Meta, S#elixir_scope.file, "unknown key ~ts for struct ~ts", - ['Elixir.Kernel':inspect(Key), elixir_aliases:inspect(Name)]) - end || {Key, _} <- Assocs, not maps:is_key(Key, Struct)]. +format_error({update_syntax_in_wrong_context, Context, Expr}) -> + io_lib:format("cannot use map/struct update syntax in ~ts, got: ~ts", + [Context, 'Elixir.Macro':to_string(Expr)]); +format_error({invalid_struct_name_in_match, Expr}) -> + Message = + "expected struct name in a match to be a compile time atom, alias or a " + "variable, got: ~ts", + io_lib:format(Message, ['Elixir.Macro':to_string(Expr)]); +format_error({invalid_struct_name, Expr}) -> + Message = "expected struct name to be a compile time atom or alias, got: ~ts", + io_lib:format(Message, ['Elixir.Macro':to_string(Expr)]); +format_error({invalid_variable_in_map_key_match, Name}) -> + Message = + "cannot use variable ~ts as map key inside a pattern. Map keys in patterns can only be literals " + "(such as atoms, strings, tuples, and the like) or an existing variable matched with the pin operator " + "(such as ^some_var)", + io_lib:format(Message, [Name]); +format_error({repeated_key, Key}) -> + io_lib:format("key ~ts will be overridden in map", ['Elixir.Macro':to_string(Key)]); +format_error({not_kv_pair, Expr}) -> + io_lib:format("expected key-value pairs in a map, got: ~ts", + ['Elixir.Macro':to_string(Expr)]); +format_error({non_map_after_struct, Expr}) -> + io_lib:format("expected struct to be followed by a map, got: ~ts", + ['Elixir.Macro':to_string(Expr)]); +format_error({struct_name_mismatch, Module, StructName}) -> + Name = elixir_aliases:inspect(Module), + Message = "expected struct name returned by ~ts.__struct__/1 to be ~ts, got: ~ts", + io_lib:format(Message, [Name, Name, elixir_aliases:inspect(StructName)]); +format_error({invalid_struct_return_value, Module, Value}) -> + Message = + "expected ~ts.__struct__/1 to return a map with a :__struct__ key that holds the " + "name of the struct (atom), got: ~ts", + io_lib:format(Message, [elixir_aliases:inspect(Module), 'Elixir.Kernel':inspect(Value)]); +format_error({inaccessible_struct, Module}) -> + Message = + "cannot access struct ~ts, the struct was not yet defined or the struct is " + "being accessed in the same context that defines it", + io_lib:format(Message, [elixir_aliases:inspect(Module)]); +format_error({undefined_struct, Module}) -> + Name = elixir_aliases:inspect(Module), + io_lib:format( + "~ts.__struct__/1 is undefined, cannot expand struct ~ts. " + "Make sure the struct name is correct. If the struct name exists and is correct " + "but it still cannot be found, you likely have cyclic module usage in your code", + [Name, Name]); +format_error({unknown_key_for_struct, Module, Key}) -> + io_lib:format("unknown key ~ts for struct ~ts", + ['Elixir.Macro':to_string(Key), elixir_aliases:inspect(Module)]); +format_error({invalid_key_for_struct, Key}) -> + io_lib:format("invalid key ~ts for struct, struct keys must be atoms, got: ", + ['Elixir.Macro':to_string(Key)]); +format_error(ignored_struct_key_in_struct) -> + "key :__struct__ is ignored when using structs". \ No newline at end of file diff --git a/lib/elixir/src/elixir_module.erl b/lib/elixir/src/elixir_module.erl index 971b50eb472..643291c9a83 100644 --- a/lib/elixir/src/elixir_module.erl +++ b/lib/elixir/src/elixir_module.erl @@ -1,365 +1,380 @@ +%% SPDX-License-Identifier: Apache-2.0 +%% SPDX-FileCopyrightText: 2021 The Elixir Team +%% SPDX-FileCopyrightText: 2012 Plataformatec + -module(elixir_module). --export([compile/4, data_table/1, docs_table/1, is_open/1, - expand_callback/6, add_beam_chunk/3, format_error/1]). +-export([file/1, data_tables/1, is_open/1, mode/1, delete_definition_attributes/6, + compile/6, expand_callback/6, format_error/1, compiler_modules/0, exports_md5/3, + write_cache/3, read_cache/2, next_counter/1, taint/1, cache_env/1, get_cached_env/1]). -include("elixir.hrl"). +-define(counter_attr, {elixir, counter}). +-define(cache_key, {elixir, cache_env}). --define(acc_attr, '__acc_attributes'). --define(docs_attr, '__docs_table'). --define(lexical_attr, '__lexical_tracker'). --define(persisted_attr, '__persisted_attributes'). --define(overridable_attr, '__overridable'). --define(location_attr, '__location'). - -%% TABLE METHODS - -data_table(Module) -> - Module. - -docs_table(Module) -> - ets:lookup_element(Module, ?docs_attr, 2). - -is_open(Module) -> - Module == ets:info(Module, name). - -%% Compilation hook - -compile(Module, Block, Vars, #{line := Line} = Env) when is_atom(Module) -> - %% In case we are generating a module from inside a function, - %% we get rid of the lexical tracker information as, at this - %% point, the lexical tracker process is long gone. - LexEnv = case ?m(Env, function) of - nil -> Env#{module := Module, local := nil}; - _ -> Env#{lexical_tracker := nil, function := nil, module := Module, local := nil} - end, - - case ?m(LexEnv, lexical_tracker) of - nil -> - elixir_lexical:run(?m(LexEnv, file), fun(Pid) -> - do_compile(Line, Module, Block, Vars, LexEnv#{lexical_tracker := Pid}) - end); - _ -> - do_compile(Line, Module, Block, Vars, LexEnv) - end; - -compile(Module, _Block, _Vars, #{line := Line, file := File}) -> - elixir_errors:form_error(Line, File, ?MODULE, {invalid_module, Module}). +%% Stores modules currently being defined by the compiler -do_compile(Line, Module, Block, Vars, E) -> - File = ?m(E, file), - check_module_availability(Line, File, Module), - build(Line, File, Module, ?m(E, lexical_tracker)), - - try - {Result, NE} = eval_form(Line, Module, Block, Vars, E), - {Base, Export, Private, Def, Defmacro, Functions} = elixir_def:unwrap_definitions(Module), - - {All, Forms0} = functions_form(Line, File, Module, Base, Export, Def, Defmacro, Functions), - Forms1 = specs_form(Module, Private, Defmacro, Forms0), - Forms2 = types_form(Module, Forms1), - Forms3 = attributes_form(Line, File, Module, Forms2), - - case ets:lookup(data_table(Module), 'on_load') of - [] -> ok; - [{on_load,OnLoad}] -> - [elixir_locals:record_local(Tuple, Module) || Tuple <- OnLoad] - end, - - AllFunctions = Def ++ [T || {T, defp, _, _, _} <- Private], - elixir_locals:ensure_no_function_conflict(Line, File, Module, AllFunctions), - elixir_locals:warn_unused_local(File, Module, Private), - warn_invalid_clauses(Line, File, Module, All), - warn_unused_docs(Line, File, Module), - - Location = {elixir_utils:relative_to_cwd(elixir_utils:characters_to_list(File)), Line}, - - Final = [ - {attribute, Line, file, Location}, - {attribute, Line, module, Module} | Forms3 - ], - - Binary = load_form(Line, Final, compile_opts(Module), NE), - {module, Module, Binary, Result} - after - elixir_locals:cleanup(Module), - elixir_def:cleanup(Module), - ets:delete(docs_table(Module)), - ets:delete(data_table(Module)) +compiler_modules() -> + case erlang:get(elixir_compiler_modules) of + undefined -> []; + M when is_list(M) -> M end. -%% Hook that builds both attribute and functions and set up common hooks. +put_compiler_modules([]) -> + erlang:erase(elixir_compiler_modules); +put_compiler_modules(M) when is_list(M) -> + erlang:put(elixir_compiler_modules, M). -build(Line, File, Module, Lexical) -> - %% Table with meta information about the module. - DataTable = data_table(Module), +exports_md5(Def, Defmacro, Struct) -> + erlang:md5(term_to_binary({lists:sort(Def), lists:sort(Defmacro), Struct}, [deterministic])). - OldTable = ets:info(DataTable, name), - case OldTable == DataTable of - true -> - [{OldFile, OldLine}] = ets:lookup_element(OldTable, ?location_attr, 2), - Error = {module_in_definition, Module, OldFile, OldLine}, - elixir_errors:form_error(Line, File, ?MODULE, Error); - false -> - [] - end, +%% Table functions - ets:new(DataTable, [set, named_table, public]), - ets:insert(DataTable, {before_compile, []}), - ets:insert(DataTable, {after_compile, []}), +file(Module) -> + ets:lookup_element(elixir_modules, Module, 4). - case elixir_compiler:get_opt(docs) of - true -> ets:insert(DataTable, {on_definition, [{'Elixir.Module', compile_doc}]}); - _ -> ets:insert(DataTable, {on_definition, []}) - end, +data_tables(Module) -> + ets:lookup_element(elixir_modules, Module, 2). - Attributes = [behaviour, on_load, spec, type, typep, opaque, callback, compile, external_resource], - ets:insert(DataTable, {?acc_attr, [before_compile, after_compile, on_definition, derive|Attributes]}), - ets:insert(DataTable, {?persisted_attr, [vsn|Attributes]}), - ets:insert(DataTable, {?docs_attr, ets:new(DataTable, [ordered_set, public])}), - ets:insert(DataTable, {?lexical_attr, Lexical}), - ets:insert(DataTable, {?overridable_attr, []}), - ets:insert(DataTable, {?location_attr, [{File, Line}]}), - - %% Setup other modules - elixir_def:setup(Module), - elixir_locals:setup(Module). - -%% Receives the module representation and evaluates it. - -eval_form(Line, Module, Block, Vars, E) -> - {Value, EE} = elixir_compiler:eval_forms(Block, Vars, E), - elixir_def_overridable:store_pending(Module), - EV = elixir_env:linify({Line, EE#{vars := [], export_vars := nil}}), - EC = eval_callbacks(Line, Module, before_compile, [EV], EV), - elixir_def_overridable:store_pending(Module), - {Value, EC}. - -eval_callbacks(Line, Module, Name, Args, E) -> - Callbacks = lists:reverse(ets:lookup_element(data_table(Module), Name, 2)), - - lists:foldl(fun({M,F}, Acc) -> - expand_callback(Line, M, F, Args, Acc#{vars := [], export_vars := nil}, - fun(AM, AF, AA) -> apply(AM, AF, AA) end) - end, E, Callbacks). +is_open(Module) -> + ets:member(elixir_modules, Module). -%% Return the form with exports and function declarations. +mode(Module) -> + try ets:lookup_element(elixir_modules, Module, 5) of + Mode -> Mode + catch + _:badarg -> closed + end. -functions_form(Line, File, Module, BaseAll, BaseExport, Def, Defmacro, BaseFunctions) -> - {InfoSpec, Info} = add_info_function(Line, File, Module, BaseExport, Def, Defmacro), +make_readonly(Module) -> + ets:update_element(elixir_modules, Module, {5, readonly}). - All = [{'__info__', 1}|BaseAll], - Export = [{'__info__', 1}|BaseExport], - Functions = [InfoSpec,Info|BaseFunctions], +delete_definition_attributes(#{module := Module}, _, _, _, _, _) -> + {DataSet, _} = data_tables(Module), + ets:delete(DataSet, doc), + ets:delete(DataSet, deprecated), + ets:delete(DataSet, impl). - {All, [ - {attribute, Line, export, lists:sort(Export)} | Functions - ]}. +write_cache(Module, Key, Value) -> + {DataSet, _} = data_tables(Module), + ets:insert(DataSet, {{cache, Key}, Value}). -%% Add attributes handling to the form +read_cache(Module, Key) -> + {DataSet, _} = data_tables(Module), + ets:lookup_element(DataSet, {cache, Key}, 2). -attributes_form(Line, File, Module, Current) -> - Table = data_table(Module), +next_counter(nil) -> erlang:unique_integer(); +next_counter(Module) -> + try + {DataSet, _} = data_tables(Module), + {Module, ets:update_counter(DataSet, ?counter_attr, 1)} + catch + _:_ -> erlang:unique_integer() + end. - AccAttrs = ets:lookup_element(Table, '__acc_attributes', 2), - PersistedAttrs = ets:lookup_element(Table, '__persisted_attributes', 2), +taint(Module) -> + try + {DataSet, _} = data_tables(Module), + ets:insert(DataSet, [{{elixir, taint}}]), + true + catch + _:_ -> false + end. - Transform = fun({Key, Value}, Acc) -> - case lists:member(Key, PersistedAttrs) of - false -> Acc; - true -> - Values = - case lists:member(Key, AccAttrs) of - true -> Value; - false -> [Value] - end, +cache_env(#{line := Line, module := Module} = E) -> + {Set, _} = data_tables(Module), + Cache = elixir_env:reset_vars(E#{line := nil}), + PrevKey = ets:lookup_element(Set, ?cache_key, 2), - lists:foldl(fun(X, Final) -> - [{attribute, Line, Key, X}|Final] - end, Acc, process_attribute(Line, File, Key, Values)) - end - end, + Pos = + case ets:lookup(Set, {cache_env, PrevKey}) of + [{_, Cache}] -> + PrevKey; + _ -> + NewKey = PrevKey + 1, + ets:insert(Set, [{{cache_env, NewKey}, Cache}, {?cache_key, NewKey}]), + NewKey + end, - ets:foldl(Transform, Current, Table). + {Module, {Line, Pos}}. -process_attribute(Line, File, external_resource, Values) -> - lists:usort([process_external_resource(Line, File, Value) || Value <- Values]); -process_attribute(_Line, _File, _Key, Values) -> - Values. +get_cached_env({Module, {Line, Pos}}) -> + {Set, _} = data_tables(Module), + (ets:lookup_element(Set, {cache_env, Pos}, 2))#{line := Line}; +get_cached_env(Env) -> + Env. -process_external_resource(_Line, _File, Value) when is_binary(Value) -> - Value; -process_external_resource(Line, File, Value) -> - elixir_errors:handle_file_error(File, - {Line, ?MODULE, {invalid_external_resource, Value}}). +%% Compilation hook -%% Types +compile(Meta, Module, Block, Vars, Prune, Env) -> + ModuleAsCharlist = validate_module_name(Module), + #{function := Function, versioned_vars := OldVerVars} = Env, -types_form(Module, Forms0) -> - case code:ensure_loaded('Elixir.Kernel.Typespec') of - {module, 'Elixir.Kernel.Typespec'} -> - Types0 = 'Elixir.Module':get_attribute(Module, type) ++ - 'Elixir.Module':get_attribute(Module, typep) ++ - 'Elixir.Module':get_attribute(Module, opaque), + {VerVars, _} = + lists:mapfoldl(fun({Var, _}, I) -> {{Var, I}, I + 1} end, 0, maps:to_list(OldVerVars)), - Types1 = ['Elixir.Kernel.Typespec':translate_type(Kind, Expr, Doc, Caller) || - {Kind, Expr, Doc, Caller} <- Types0], + BaseEnv = Env#{module := Module, versioned_vars := maps:from_list(VerVars)}, - 'Elixir.Module':delete_attribute(Module, type), - 'Elixir.Module':delete_attribute(Module, typep), - 'Elixir.Module':delete_attribute(Module, opaque), + MaybeLexEnv = + case Function of + nil -> BaseEnv; + _ -> BaseEnv#{lexical_tracker := nil, tracers := [], function := nil} + end, - Forms1 = types_attributes(Types1, Forms0), - Forms2 = export_types_attributes(Types1, Forms1), - typedocs_attributes(Types1, Forms2); + case MaybeLexEnv of + #{lexical_tracker := nil} -> + elixir_lexical:run( + MaybeLexEnv, + fun(LexEnv) -> compile(Meta, Module, ModuleAsCharlist, Block, Vars, Prune, LexEnv) end, + fun(_LexEnv) -> ok end + ); + _ -> + compile(Meta, Module, ModuleAsCharlist, Block, Vars, Prune, MaybeLexEnv) + end. - {error, _} -> - Forms0 +validate_module_name(Module) when Module == nil; is_boolean(Module); not is_atom(Module) -> + invalid_module_name(Module); +validate_module_name(Module) -> + Charlist = atom_to_list(Module), + case lists:any(fun(Char) -> (Char =:= $/) or (Char =:= $\\) end, Charlist) of + true -> invalid_module_name(Module); + false -> Charlist end. -types_attributes(Types, Forms) -> - Fun = fun({{Kind, _NameArity, Expr}, Line, _Export, _Doc}, Acc) -> - [{attribute, Line, Kind, Expr}|Acc] - end, - lists:foldl(Fun, Forms, Types). - -export_types_attributes(Types, Forms) -> - Fun = fun - ({{_Kind, NameArity, _Expr}, Line, true, _Doc}, Acc) -> - [{attribute, Line, export_type, [NameArity]}|Acc]; - ({_Type, _Line, false, _Doc}, Acc) -> - Acc - end, - lists:foldl(Fun, Forms, Types). - -typedocs_attributes(Types, Forms) -> - Fun = fun - ({{_Kind, NameArity, _Expr}, Line, true, Doc}, Acc) when Doc =/= nil -> - [{attribute, Line, typedoc, {NameArity, Doc}}|Acc]; - ({_Type, _Line, _Export, _Doc}, Acc) -> - Acc - end, - lists:foldl(Fun, Forms, Types). +invalid_module_name(Module) -> + %% We raise an argument error to keep it close to Elixir errors before it starts. + erlang:error('Elixir.ArgumentError':exception( + <<"invalid module name: ", + ('Elixir.Kernel':inspect(Module))/binary>> + )). -%% Specs +compile(Meta, Module, ModuleAsCharlist, Block, Vars, Prune, E) -> + Anno = ?ann(Meta), + Line = erl_anno:line(Anno), -specs_form(Module, Private, Defmacro, Forms) -> - case code:ensure_loaded('Elixir.Kernel.Typespec') of - {module, 'Elixir.Kernel.Typespec'} -> - Defmacrop = [Tuple || {Tuple, defmacrop, _, _, _} <- Private], + File = ?key(E, file), + check_module_availability(Module, Line, E), + elixir_env:trace(defmodule, E), - Specs0 = 'Elixir.Module':get_attribute(Module, spec) ++ - 'Elixir.Module':get_attribute(Module, callback), + CompilerModules = compiler_modules(), + {Tables, Ref} = build(Module, Line, File, E), + {DataSet, DataBag} = Tables, - Specs1 = ['Elixir.Kernel.Typespec':translate_spec(Kind, Expr, Caller) || - {Kind, Expr, Caller} <- Specs0], - Specs2 = lists:flatmap(fun(Spec) -> - translate_macro_spec(Spec, Defmacro, Defmacrop) - end, Specs1), + try + put_compiler_modules([Module | CompilerModules]), + {Result, ModuleE, CallbackE} = eval_form(Line, Module, DataBag, Block, Vars, Prune, E), + CheckerInfo = checker_info(), + {BeamLocation, Forceload} = beam_location(ModuleAsCharlist), + + {Binary, PersistedAttributes, Autoload} = + elixir_erl_compiler:spawn(fun() -> + PersistedAttributes = ets:lookup_element(DataBag, persisted_attributes, 2), + Attributes = attributes(DataSet, DataBag, PersistedAttributes), + {AllDefinitions, Private} = elixir_def:fetch_definitions(Module, E), + + OnLoadAttribute = lists:keyfind(on_load, 1, Attributes), + validate_on_load_attribute(OnLoadAttribute, AllDefinitions, DataBag, Line, E), + + DialyzerAttribute = lists:keyfind(dialyzer, 1, Attributes), + validate_dialyzer_attribute(DialyzerAttribute, AllDefinitions, Line, E), + + NifsAttribute = lists:keyfind(nifs, 1, Attributes), + validate_nifs_attribute(NifsAttribute, AllDefinitions, Line, E), + elixir_import:ensure_no_local_conflict(Module, AllDefinitions, E), + make_readonly(Module), + + (not elixir_config:is_bootstrap()) andalso + 'Elixir.Module':'__check_attributes__'(E, DataSet, DataBag), + + AfterVerify = bag_lookup_element(DataBag, {accumulate, after_verify}, 2), + [elixir_env:trace({remote_function, [{line, Line}], VerifyMod, VerifyFun, 1}, CallbackE) || + {VerifyMod, VerifyFun} <- AfterVerify], + + %% Ensure there are no errors before we infer types + compile_error_if_tainted(DataSet, E), + + {Signatures, Unreachable} = + case elixir_config:is_bootstrap() of + true -> {#{}, []}; + false -> + UsedPrivate = bag_lookup_element(DataBag, used_private, 2), + 'Elixir.Module.Types':infer(Module, File, Attributes, AllDefinitions, Private, UsedPrivate, E, CheckerInfo) + end, - 'Elixir.Module':delete_attribute(Module, spec), - 'Elixir.Module':delete_attribute(Module, callback), - specs_attributes(Forms, Specs2); + RawCompileOpts = bag_lookup_element(DataBag, {accumulate, compile}, 2), + CompileOpts = validate_compile_opts(RawCompileOpts, AllDefinitions, Unreachable, Line, E), + Impls = bag_lookup_element(DataBag, impls, 2), + + Struct = get_struct(DataSet), + set_exports_md5(DataSet, AllDefinitions, Struct), + + ModuleMap = #{ + struct => Struct, + module => Module, + anno => Anno, + file => File, + relative_file => elixir_utils:relative_to_cwd(File), + attributes => Attributes, + definitions => AllDefinitions, + after_verify => AfterVerify, + compile_opts => CompileOpts, + deprecated => get_deprecated(DataBag), + defines_behaviour => defines_behaviour(DataBag), + impls => Impls, + unreachable => Unreachable + }, + + compile_error_if_tainted(DataSet, E), + Binary = elixir_erl:compile(ModuleMap, Signatures), + Autoload = Forceload or proplists:get_value(autoload, CompileOpts, false), + spawn_parallel_checker(CheckerInfo, Module, ModuleMap, Signatures, BeamLocation), + {Binary, PersistedAttributes, Autoload} + end), + + Autoload andalso code:load_binary(Module, BeamLocation, Binary), + make_module_available(Module, Binary, Autoload), + put_compiler_modules(CompilerModules), + eval_callbacks(Line, DataBag, after_compile, [CallbackE, Binary], CallbackE), + elixir_env:trace({on_module, Binary, none}, ModuleE), + warn_unused_attributes(DataSet, DataBag, PersistedAttributes, E), + (element(2, CheckerInfo) == nil) andalso + [VerifyMod:VerifyFun(Module) || + {VerifyMod, VerifyFun} <- bag_lookup_element(DataBag, {accumulate, after_verify}, 2)], + {module, Module, Binary, Result} + catch + error:undef:Stacktrace -> + case Stacktrace of + [{Module, Fun, Args, _Info} | _] = Stack when is_list(Args) -> + compile_undef(Module, Fun, length(Args), Stack); + [{Module, Fun, Arity, _Info} | _] = Stack -> + compile_undef(Module, Fun, Arity, Stack); + Stack -> + erlang:raise(error, undef, Stack) + end + after + put_compiler_modules(CompilerModules), + ets:delete(DataSet), + ets:delete(DataBag), + elixir_code_server:call({undefmodule, Ref}) + end. - {error, _} -> - Forms +compile_error_if_tainted(DataSet, E) -> + case ets:member(DataSet, {elixir, taint}) of + true -> elixir_errors:compile_error(E); + false -> ok end. -specs_attributes(Forms, Specs) -> - Dict = lists:foldl(fun({{Kind, NameArity, Spec}, Line}, Acc) -> - dict:append({Kind, NameArity}, {Spec, Line}, Acc) - end, dict:new(), Specs), - dict:fold(fun({Kind, NameArity}, ExprsLines, Acc) -> - {Exprs, Lines} = lists:unzip(ExprsLines), - Line = lists:min(Lines), - [{attribute, Line, Kind, {NameArity, Exprs}}|Acc] - end, Forms, Dict). - -translate_macro_spec({{spec, NameArity, Spec}, Line}, Defmacro, Defmacrop) -> - case ordsets:is_element(NameArity, Defmacrop) of - true -> []; +set_exports_md5(DataSet, AllDefinitions, Struct) -> + {Funs, Macros} = + lists:foldl(fun + ({Tuple, def, _Meta, _Clauses}, {Funs, Macros}) -> {[Tuple | Funs], Macros}; + ({Tuple, defmacro, _Meta, _Clauses}, {Funs, Macros}) -> {Funs, [Tuple | Macros]}; + ({_Tuple, _Kind, _Meta, _Clauses}, {Funs, Macros}) -> {Funs, Macros} + end, {[], []}, AllDefinitions), + MD5 = exports_md5(Funs, Macros, Struct), + ets:insert(DataSet, {exports_md5, MD5, nil, []}). + +validate_compile_opts(Opts, Defs, Unreachable, Line, E) -> + lists:flatmap(fun (Opt) -> validate_compile_opt(Opt, Defs, Unreachable, Line, E) end, Opts). + +%% TODO: Make this an error on v2.0 +validate_compile_opt({parse_transform, Module} = Opt, _Defs, _Unreachable, Line, E) -> + elixir_errors:file_warn([{line, Line}], E, ?MODULE, {parse_transform, Module}), + [Opt]; +validate_compile_opt({inline, Inlines}, Defs, Unreachable, Line, E) -> + case validate_inlines(Inlines, Defs, Unreachable, []) of + {ok, []} -> + []; + {ok, FilteredInlines} -> + [{inline, FilteredInlines}]; + {error, Reason} -> + elixir_errors:module_error([{line, Line}], E, ?MODULE, Reason), + [] + end; +validate_compile_opt(Opt, Defs, Unreachable, Line, E) when is_list(Opt) -> + validate_compile_opts(Opt, Defs, Unreachable, Line, E); +validate_compile_opt(Opt, _Defs, _Unreachable, _Line, _E) -> + [Opt]. + +validate_inlines([Inline | Inlines], Defs, Unreachable, Acc) -> + case lists:keyfind(Inline, 1, Defs) of false -> - case ordsets:is_element(NameArity, Defmacro) of - true -> - {Name, Arity} = NameArity, - [{{spec, {elixir_utils:macro_name(Name), Arity + 1}, spec_for_macro(Spec)}, Line}]; - false -> - [{{spec, NameArity, Spec}, Line}] + {error, {undefined_function, {compile, inline}, Inline}}; + {_Def, Kind, _Meta, _Clauses} when Kind == defmacro; Kind == defmacrop -> + {error, {bad_macro, {compile, inline}, Inline}}; + _ -> + case lists:member(Inline, Unreachable) of + true -> validate_inlines(Inlines, Defs, Unreachable, Acc); + false -> validate_inlines(Inlines, Defs, Unreachable, [Inline | Acc]) end end; +validate_inlines([], _Defs, _Unreachable, Acc) -> {ok, Acc}. -translate_macro_spec({{callback, NameArity, Spec}, Line}, _Defmacro, _Defmacrop) -> - [{{callback, NameArity, Spec}, Line}]. - -spec_for_macro({type, Line, 'fun', [{type, _, product, Args}|T]}) -> - NewArgs = [{type, Line, term, []}|Args], - {type, Line, 'fun', [{type, Line, product, NewArgs}|T]}; - -spec_for_macro(Else) -> Else. - -%% Loads the form into the code server. - -compile_opts(Module) -> - case ets:lookup(data_table(Module), compile) of - [{compile,Opts}] when is_list(Opts) -> Opts; - [] -> [] - end. - -load_form(Line, Forms, Opts, #{file := File} = E) -> - elixir_compiler:module(Forms, File, Opts, fun(Module, Binary0) -> - Docs = elixir_compiler:get_opt(docs), - Binary = add_docs_chunk(Binary0, Module, Line, Docs), - eval_callbacks(Line, Module, after_compile, [E, Binary], E), - - case get(elixir_compiled) of - Current when is_list(Current) -> - put(elixir_compiled, [{Module,Binary}|Current]), - - case get(elixir_compiler_pid) of - undefined -> []; - PID -> - Ref = make_ref(), - PID ! {module_available, self(), Ref, File, Module, Binary}, - receive {Ref, ack} -> ok end - end; - _ -> - [] - end, +validate_on_load_attribute({on_load, Def}, Defs, Bag, Line, E) -> + case lists:keyfind(Def, 1, Defs) of + false -> + elixir_errors:module_error([{line, Line}], E, ?MODULE, {undefined_function, on_load, Def}); + {_Def, Kind, _Meta, _Clauses} when Kind == defmacro; Kind == defmacrop -> + elixir_errors:module_error([{line, Line}], E, ?MODULE, {bad_macro, on_load, Def}); + {{Name, Arity}, Kind, _Meta, _Clauses} -> + elixir_env:trace({local_function, [{line, Line}], Name, Arity}, E), + (Kind == defp) andalso ets:insert(Bag, {used_private, Def}) + end; +validate_on_load_attribute(false, _Defs, _Bag, _Line, _E) -> ok. - Binary - end). +validate_dialyzer_attribute({dialyzer, Dialyzer}, Defs, Line, E) -> + [validate_definition({dialyzer, Key}, Fun, Defs, Line, E) + || {Key, Funs} <- lists:flatten([Dialyzer]), Fun <- lists:flatten([Funs])]; +validate_dialyzer_attribute(false, _Defs, _Line, _E) -> + ok. -add_docs_chunk(Bin, Module, Line, true) -> - ChunkData = term_to_binary({elixir_docs_v1, [ - {docs, get_docs(Module)}, - {moduledoc, get_moduledoc(Line, Module)} - ]}), - add_beam_chunk(Bin, "ExDc", ChunkData); +validate_nifs_attribute({nifs, Funs}, Defs, Line, E) -> + [validate_definition(nifs, Fun, Defs, Line, E) || Fun <- lists:flatten([Funs])]; +validate_nifs_attribute(false, _Defs, _Line, _E) -> + ok. -add_docs_chunk(Bin, _, _, _) -> Bin. +validate_definition(Key, Fun, Defs, Line, E) -> + case lists:keyfind(Fun, 1, Defs) of + false -> + elixir_errors:module_error([{line, Line}], E, ?MODULE, {undefined_function, Key, Fun}); + {Fun, Type, _Meta, _Clauses} when Type == defmacro; Type == defmacrop -> + elixir_errors:module_error([{line, Line}], E, ?MODULE, {bad_macro, Key, Fun}); + _ -> + ok + end. -get_docs(Module) -> - ordsets:from_list( - [{Tuple, Line, Kind, Sig, Doc} || - {Tuple, Line, Kind, Sig, Doc} <- ets:tab2list(docs_table(Module)), - Kind =/= type, Kind =/= opaque]). +defines_behaviour(DataBag) -> + ets:member(DataBag, {accumulate, callback}) orelse ets:member(DataBag, {accumulate, macrocallback}). -get_moduledoc(Line, Module) -> - {Line, 'Elixir.Module':get_attribute(Module, moduledoc)}. +%% An undef error for a function in the module being compiled might result in an +%% exception message suggesting the current module is not loaded. This is +%% misleading so use a custom reason. +compile_undef(Module, Fun, Arity, Stack) -> + case elixir_config:is_bootstrap() of + false -> + Opts = [{module, Module}, {function, Fun}, {arity, Arity}, + {reason, 'function not available'}], + Exception = 'Elixir.UndefinedFunctionError':exception(Opts), + erlang:raise(error, Exception, Stack); + true -> + erlang:raise(error, undef, Stack) + end. +%% Handle reserved modules and duplicates. -check_module_availability(Line, File, Module) -> - Reserved = ['Elixir.Any', 'Elixir.BitString', 'Elixir.Function', 'Elixir.PID', +check_module_availability(Module, Line, E) -> + Reserved = ['Elixir.True', 'Elixir.False', 'Elixir.Nil', + 'Elixir.Any', 'Elixir.BitString', 'Elixir.PID', 'Elixir.Reference', 'Elixir.Elixir', 'Elixir'], case lists:member(Module, Reserved) of - true -> elixir_errors:handle_file_error(File, {Line, ?MODULE, {module_reserved, Module}}); + true -> elixir_errors:file_error([{line, Line}], E, ?MODULE, {module_reserved, Module}); false -> ok end, - case elixir_compiler:get_opt(ignore_module_conflict) of + case elixir_config:get(ignore_module_conflict) of false -> case code:ensure_loaded(Module) of {module, _} -> - elixir_errors:handle_file_warning(File, {Line, ?MODULE, {module_defined, Module}}); + elixir_errors:file_warn([{line, Line}], E, ?MODULE, {module_defined, Module}); {error, _} -> ok end; @@ -367,132 +382,280 @@ check_module_availability(Line, File, Module) -> ok end. -warn_invalid_clauses(_Line, _File, 'Elixir.Kernel.SpecialForms', _All) -> ok; -warn_invalid_clauses(_Line, File, Module, All) -> - ets:foldl(fun - ({_, _, Kind, _, _}, _) when Kind == type; Kind == opaque -> - ok; - ({Tuple, Line, _, _, _}, _) -> - case lists:member(Tuple, All) of - false -> - elixir_errors:handle_file_warning(File, {Line, ?MODULE, {invalid_clause, Tuple}}); - true -> - ok - end - end, ok, docs_table(Module)). - -warn_unused_docs(Line, File, Module) -> - lists:foreach(fun(Attribute) -> - case ets:member(data_table(Module), Attribute) of - true -> - elixir_errors:handle_file_warning(File, {Line, ?MODULE, {unused_doc, Attribute}}); - _ -> - ok - end - end, [typedoc]). - -% EXTRA FUNCTIONS - -add_info_function(Line, File, Module, Export, Def, Defmacro) -> - Pair = {'__info__', 1}, - case lists:member(Pair, Export) of - true -> - elixir_errors:form_error(Line, File, ?MODULE, {internal_function_overridden, Pair}); - false -> - { - {attribute, Line, spec, {{'__info__', 1}, - [{type, Line, 'fun', [{type, Line, product, [ {type, Line, atom, []}]}, {type, Line, term, []} ]}] - }}, - {function, 0, '__info__', 1, [ - functions_clause(Def), - macros_clause(Defmacro), - module_clause(Module), - else_clause() - ]} - } - end. - -functions_clause(Def) -> - {clause, 0, [{atom, 0, functions}], [], [elixir_utils:elixir_to_erl(Def)]}. - -macros_clause(Defmacro) -> - {clause, 0, [{atom, 0, macros}], [], [elixir_utils:elixir_to_erl(Defmacro)]}. - -module_clause(Module) -> - {clause, 0, [{atom, 0, module}], [], [{atom, 0, Module}]}. - -else_clause() -> - Info = {call, 0, {atom, 0, module_info}, [{var, 0, atom}]}, - {clause, 0, [{var, 0, atom}], [], [Info]}. +%% Hook that builds both attribute and functions and set up common hooks. -% HELPERS +build(Module, Line, File, E) -> + %% In the set table we store: + %% + %% * {Attribute, Value, AccumulateOrUnsetOrReadOrUnreadline, TraceLineOrNil} + %% * {{elixir, ...}, ...} + %% * {{cache, ...}, ...} + %% * {{function, Tuple}, ...}, {{macro, Tuple}, ...} + %% * {{type, Tuple}, ...}, {{opaque, Tuple}, ...} + %% * {{callback, Tuple}, ...}, {{macrocallback, Tuple}, ...} + %% * {{def, Tuple}, ...} (from elixir_def) + %% * {{overridable, Tuple}, ...} (from elixir_overridable) + %% + DataSet = ets:new(Module, [set, public]), + + %% In the bag table we store: + %% + %% * {{accumulate, Attribute}, ...} (includes typespecs) + %% * {warn_attributes, ...} + %% * {impls, ...} + %% * {deprecated, ...} + %% * {persisted_attributes, ...} + %% * {defs, ...} (from elixir_def) + %% * {overridables, ...} (from elixir_overridable) + %% * {{default, Name}, ...} (from elixir_def) + %% * {{clauses, Tuple}, ...} (from elixir_def) + %% + DataBag = ets:new(Module, [duplicate_bag, public]), + + ets:insert(DataSet, [ + % {Key, Value, ReadOrUnreadLine, TraceLine} + {moduledoc, nil, nil, []}, + + % {Key, Value, accumulate, TraceLine} + {after_compile, [], accumulate, []}, + {after_verify, [], accumulate, []}, + {before_compile, [], accumulate, []}, + {behaviour, [], accumulate, []}, + {compile, [], accumulate, []}, + {derive, [], accumulate, []}, + {dialyzer, [], accumulate, []}, + {external_resource, [], accumulate, []}, + {nifs, [], accumulate, []}, + {on_definition, [], accumulate, []}, + {opaque, [], accumulate, []}, + {type, [], accumulate, []}, + {typep, [], accumulate, []}, + {spec, [], accumulate, []}, + {callback, [], accumulate, []}, + {macrocallback, [], accumulate, []}, + {optional_callbacks, [], accumulate, []}, + + % Others + {?cache_key, 0}, + {?counter_attr, 0} + ]), + + Persisted = [behaviour, dialyzer, external_resource, nifs, on_load, vsn], + ets:insert(DataBag, [{persisted_attributes, Attr} || Attr <- Persisted]), + + OnDefinition = + case elixir_config:is_bootstrap() of + false -> {'Elixir.Module', compile_definition_attributes}; + _ -> {elixir_module, delete_definition_attributes} + end, + ets:insert(DataBag, {{accumulate, on_definition}, OnDefinition}), + + %% Setup definition related modules + Tables = {DataSet, DataBag}, + elixir_def:setup(Tables), + Tuple = {Module, Tables, Line, File, all}, + + Ref = + case elixir_code_server:call({defmodule, Module, self(), Tuple}) of + {ok, ModuleRef} -> + ModuleRef; + {error, {Module, _, OldLine, OldFile, _}} -> + ets:delete(DataSet), + ets:delete(DataBag), + Error = {module_in_definition, Module, OldFile, OldLine}, + elixir_errors:file_error([{line, Line}], E, ?MODULE, Error) + end, -%% Adds custom chunk to a .beam binary -add_beam_chunk(Bin, Id, ChunkData) - when is_binary(Bin), is_list(Id), is_binary(ChunkData) -> - {ok, _, Chunks0} = beam_lib:all_chunks(Bin), - NewChunk = {Id, ChunkData}, - Chunks = [NewChunk|Chunks0], - {ok, NewBin} = beam_lib:build_module(Chunks), - NewBin. + {Tables, Ref}. + +%% Handles module and callback evaluations. + +eval_form(Line, Module, DataBag, Block, Vars, Prune, E) -> + %% Given Elixir modules can get very long to compile due to metaprogramming, + %% we disable expansions that take linear time to code size. + {Value, ExS, EE} = elixir_compiler:compile(Block, Vars, [no_bool_opt, no_ssa_opt], E), + elixir_overridable:store_not_overridden(Module), + EV = (elixir_env:reset_vars(EE))#{line := Line}, + EC = eval_callbacks(Line, DataBag, before_compile, [EV], EV), + elixir_overridable:store_not_overridden(Module), + {Value, maybe_prune_versioned_vars(Prune, Vars, ExS, E), EC}. + +maybe_prune_versioned_vars(false, _Vars, _Exs, E) -> + E; +maybe_prune_versioned_vars(true, Vars, ExS, E) -> + PruneBefore = length(Vars), + #elixir_ex{vars={ExVars, _}, unused={Unused, _}} = ExS, + + VersionedVars = + maps:filter(fun + (Pair, Version) when Version < PruneBefore, not is_map_key({Pair, Version}, Unused) -> false; + (_, _) -> true + end, ExVars), + + E#{versioned_vars := VersionedVars}. + +eval_callbacks(Line, DataBag, Name, Args, E) -> + Callbacks = bag_lookup_element(DataBag, {accumulate, Name}, 2), + lists:foldl(fun({M, F}, Acc) -> + expand_callback(Line, M, F, Args, Acc, fun(AM, AF, AA) -> apply(AM, AF, AA) end) + end, E, Callbacks). -%% Expands a callback given by M:F(Args). In case -%% the callback can't be expanded, invokes the given -%% fun passing a possibly expanded AM:AF(Args). -expand_callback(Line, M, F, Args, E, Fun) -> - Meta = [{line,Line},{require,false}], +expand_callback(Line, M, F, Args, Acc, Fun) -> + E = elixir_env:reset_vars(Acc), + S = elixir_env:env_to_ex(E), + Meta = [{line, Line}, {required, true}], - {EE, ET} = elixir_dispatch:dispatch_require(Meta, M, F, Args, E, fun(AM, AF, AA) -> - Fun(AM, AF, AA), - {ok, E} - end), + {EE, _S, ET} = + elixir_dispatch:dispatch_require(Meta, M, F, Args, S, E, fun(AM, AF) -> + Fun(AM, AF, Args), + {ok, S, E} + end), if is_atom(EE) -> ET; true -> try - {_Value, _Binding, EF, _S} = elixir:eval_forms(EE, [], ET), + {_Value, _Binding, EF} = elixir:eval_forms(EE, [], ET), EF catch - Kind:Reason -> + Kind:Reason:Stacktrace -> Info = {M, F, length(Args), location(Line, E)}, - erlang:raise(Kind, Reason, prune_stacktrace(Info, erlang:get_stacktrace())) + erlang:raise(Kind, Reason, prune_stacktrace(Info, Stacktrace)) end end. -location(Line, E) -> - [{file, elixir_utils:characters_to_list(?m(E, file))}, {line, Line}]. +%% Add attributes handling to the form + +attributes(DataSet, DataBag, PersistedAttributes) -> + [{Key, Value} || Key <- PersistedAttributes, Value <- lookup_attribute(DataSet, DataBag, Key)]. + +lookup_attribute(DataSet, DataBag, Key) when is_atom(Key) -> + case ets:lookup(DataSet, Key) of + [{_, _, accumulate, _}] -> bag_lookup_element(DataBag, {accumulate, Key}, 2); + [{_, _, unset, _}] -> []; + [{_, Value, _, _}] -> [Value]; + [] -> [] + end. + +warn_unused_attributes(DataSet, DataBag, PersistedAttrs, E) -> + StoredAttrs = bag_lookup_element(DataBag, warn_attributes, 2), + %% This is the same list as in Module.put_attribute + %% without moduledoc which are never warned on. + Attrs = [doc, typedoc, impl, deprecated | StoredAttrs -- PersistedAttrs], + Query = [{{Attr, '_', '$1', '_'}, [{is_integer, '$1'}], [[Attr, '$1']]} || Attr <- Attrs], + [elixir_errors:file_warn([{line, Line}], E, ?MODULE, {unused_attribute, Key}) + || [Key, Line] <- ets:select(DataSet, Query)]. + +get_struct(Set) -> + case ets:lookup(Set, {elixir, struct}) of + [] -> nil; + [{_, Fields}] -> Fields + end. + +get_deprecated(Bag) -> + lists:usort(bag_lookup_element(Bag, deprecated, 2)). + +bag_lookup_element(Table, Name, Pos) -> + try + ets:lookup_element(Table, Name, Pos) + catch + error:badarg -> [] + end. + +beam_location(ModuleAsCharlist) -> + case get(elixir_compiler_dest) of + {Dest, Forceload} when is_binary(Dest) -> + {filename:join(elixir_utils:characters_to_list(Dest), ModuleAsCharlist ++ ".beam"), + Forceload}; + _ -> + {"", true} + end. + +%% Integration with elixir_compiler that makes the module available + +checker_info() -> + case get(elixir_checker_info) of + undefined -> {self(), nil}; + _ -> 'Elixir.Module.ParallelChecker':get() + end. + +spawn_parallel_checker({_, nil}, _Module, _ModuleMap, _Signatures, _BeamLocation) -> + ok; +spawn_parallel_checker(CheckerInfo, Module, ModuleMap, Signatures, BeamLocation) -> + Log = + case erlang:get(elixir_code_diagnostics) of + {_, false} -> false; + _ -> true + end, + 'Elixir.Module.ParallelChecker':spawn(CheckerInfo, Module, ModuleMap, Signatures, BeamLocation, Log). + +make_module_available(Module, Binary, Loaded) -> + case get(elixir_module_binaries) of + Current when is_list(Current) -> + put(elixir_module_binaries, [{Module, Binary} | Current]); + _ -> + ok + end, + + case get(elixir_compiler_info) of + undefined -> + ok; + {PID, _} -> + Ref = make_ref(), + PID ! {module_available, self(), Ref, get(elixir_compiler_file), Module, Binary, Loaded}, + receive {Ref, ack} -> ok end + end. + +%% Error handling and helpers. %% We've reached the elixir_module or eval internals, skip it with the rest -prune_stacktrace(Info, [{elixir, eval_forms, _, _}|_]) -> +prune_stacktrace(Info, [{elixir, eval_forms, _, _} | _]) -> [Info]; -prune_stacktrace(Info, [{elixir_module, _, _, _}|_]) -> +prune_stacktrace(Info, [{elixir_module, _, _, _} | _]) -> [Info]; -prune_stacktrace(Info, [H|T]) -> - [H|prune_stacktrace(Info, T)]; +prune_stacktrace(Info, [H | T]) -> + [H | prune_stacktrace(Info, T)]; prune_stacktrace(Info, []) -> [Info]. -% ERROR HANDLING - -format_error({invalid_clause, {Name, Arity}}) -> - io_lib:format("empty clause provided for nonexistent function or macro ~ts/~B", [Name, Arity]); -format_error({invalid_external_resource, Value}) -> - io_lib:format("expected a string value for @external_resource, got: ~p", - ['Elixir.Kernel':inspect(Value)]); -format_error({unused_doc, typedoc}) -> - "@typedoc provided but no type follows it"; -format_error({unused_doc, doc}) -> - "@doc provided but no definition follows it"; -format_error({internal_function_overridden, {Name, Arity}}) -> - io_lib:format("function ~ts/~B is internal and should not be overridden", [Name, Arity]); -format_error({invalid_module, Module}) -> - io_lib:format("invalid module name: ~ts", ['Elixir.Kernel':inspect(Module)]); +location(Line, E) -> + [{file, elixir_utils:characters_to_list(?key(E, file))}, {line, Line}]. + +format_error({unused_attribute, typedoc}) -> + "module attribute @typedoc was set but no type follows it"; +format_error({unused_attribute, doc}) -> + "module attribute @doc was set but no definition follows it"; +format_error({unused_attribute, impl}) -> + "module attribute @impl was set but no definition follows it"; +format_error({unused_attribute, deprecated}) -> + "module attribute @deprecated was set but no definition follows it"; +format_error({unused_attribute, Attr}) -> + io_lib:format("module attribute @~ts was set but never used", [Attr]); format_error({module_defined, Module}) -> - io_lib:format("redefining module ~ts", [elixir_aliases:inspect(Module)]); + Extra = + case code:which(Module) of + "" -> + " (current version defined in memory)"; + Path when is_list(Path) -> + io_lib:format(" (current version loaded from ~ts)", [elixir_utils:relative_to_cwd(Path)]); + _ -> + "" + end, + io_lib:format("redefining module ~ts~ts", [elixir_aliases:inspect(Module), Extra]); format_error({module_reserved, Module}) -> io_lib:format("module ~ts is reserved and cannot be defined", [elixir_aliases:inspect(Module)]); format_error({module_in_definition, Module, File, Line}) -> io_lib:format("cannot define module ~ts because it is currently being defined in ~ts:~B", - [elixir_aliases:inspect(Module), 'Elixir.Path':relative_to_cwd(File), Line]). + [elixir_aliases:inspect(Module), elixir_utils:relative_to_cwd(File), Line]); +format_error({undefined_function, {Attr, Key}, {Name, Arity}}) -> + io_lib:format("undefined function ~ts/~B given to @~ts :~ts", [Name, Arity, Attr, Key]); +format_error({undefined_function, Attr, {Name, Arity}}) -> + io_lib:format("undefined function ~ts/~B given to @~ts", [Name, Arity, Attr]); +format_error({bad_macro, {Attr, Key}, {Name, Arity}}) -> + io_lib:format("macro ~ts/~B given to @~ts :~ts (only functions are supported)", [Name, Arity, Attr, Key]); +format_error({bad_macro, Attr, {Name, Arity}}) -> + io_lib:format("macro ~ts/~B given to @~ts (only functions are supported)", [Name, Arity, Attr]); +format_error({parse_transform, Module}) -> + io_lib:format("@compile {:parse_transform, ~ts} is deprecated. Elixir will no longer support " + "Erlang-based transforms in future versions", [elixir_aliases:inspect(Module)]). diff --git a/lib/elixir/src/elixir_overridable.erl b/lib/elixir/src/elixir_overridable.erl new file mode 100644 index 00000000000..bac527a5bf3 --- /dev/null +++ b/lib/elixir/src/elixir_overridable.erl @@ -0,0 +1,137 @@ +%% SPDX-License-Identifier: Apache-2.0 +%% SPDX-FileCopyrightText: 2021 The Elixir Team +%% SPDX-FileCopyrightText: 2012 Plataformatec + +% Holds the logic responsible for defining overridable functions and handling super. +-module(elixir_overridable). +-export([overridables_for/1, overridable_for/2, + record_overridable/3, super/4, + store_not_overridden/1, format_error/1]). +-include("elixir.hrl"). +-define(overridden_pos, 4). + +overridables_for(Module) -> + {_, Bag} = elixir_module:data_tables(Module), + try + ets:lookup_element(Bag, overridables, 2) + catch + _:_ -> [] + end. + +overridable_for(Module, Tuple) -> + {Set, _} = elixir_module:data_tables(Module), + + case ets:lookup(Set, {overridable, Tuple}) of + [Overridable] -> Overridable; + [] -> not_overridable + end. + +record_overridable(Module, Tuple, Def) -> + {Set, Bag} = elixir_module:data_tables(Module), + + case ets:insert_new(Set, {{overridable, Tuple}, 1, Def, false}) of + true -> + ets:insert(Bag, {overridables, Tuple}); + false -> + [{_, Count, PreviousDef, _}] = ets:lookup(Set, {overridable, Tuple}), + {{_, Kind, Meta, File, _, _}, _} = Def, + {{_, PreviousKind, _, _, _, _}, _} = PreviousDef, + + case is_valid_kind(Kind, PreviousKind) of + true -> + ets:insert(Set, {{overridable, Tuple}, Count + 1, Def, false}); + false -> + elixir_errors:file_error(Meta, File, ?MODULE, {bad_kind, Module, Tuple, Kind}) + end + end, + + ok. + +super(Meta, Module, Tuple, E) -> + {Set, _} = elixir_module:data_tables(Module), + + case ets:lookup(Set, {overridable, Tuple}) of + [Overridable] -> + store(Set, Module, Tuple, Overridable, true); + [] -> + elixir_errors:file_error(Meta, E, ?MODULE, {no_super, Module, Tuple}) + end. + +store_not_overridden(Module) -> + {Set, Bag} = elixir_module:data_tables(Module), + + lists:foreach(fun({_, Tuple}) -> + [Overridable] = ets:lookup(Set, {overridable, Tuple}), + + case ets:lookup(Set, {def, Tuple}) of + [] -> + store(Set, Module, Tuple, Overridable, false); + [{_, Kind, Meta, File, _, _}] -> + {{_, OverridableKind, _, _, _, _}, _} = element(3, Overridable), + + case is_valid_kind(Kind, OverridableKind) of + true -> ok; + false -> elixir_errors:file_error(Meta, File, ?MODULE, {bad_kind, Module, Tuple, Kind}) + end + end + end, ets:lookup(Bag, overridables)). + +%% Private + +store(Set, Module, Tuple, {_, Count, Def, Overridden}, Hidden) -> + {{{def, {Name, Arity}}, Kind, BaseMeta, File, _Check, + {Defaults, _HasBody, _LastDefaults}}, Clauses} = Def, + Meta = [{from_super, Hidden} | BaseMeta], + + {FinalKind, FinalName, FinalArity, FinalClauses} = + case Hidden of + false -> + {Kind, Name, Arity, Clauses}; + true when Kind == defmacro; Kind == defmacrop -> + {defmacrop, name(Name, Count), Arity, Clauses}; + true -> + {defp, name(Name, Count), Arity, Clauses} + end, + + case Overridden of + false -> + ets:update_element(Set, {overridable, Tuple}, {?overridden_pos, true}), + elixir_def:store_definition(none, FinalKind, Meta, FinalName, FinalArity, + File, Module, Defaults, FinalClauses); + true -> + ok + end, + + {FinalKind, FinalName, Meta}. + +name(Name, Count) when is_integer(Count) -> + list_to_atom(atom_to_list(Name) ++ " (overridable " ++ integer_to_list(Count) ++ ")"). + +is_valid_kind(NewKind, PreviousKind) -> + is_macro(NewKind) =:= is_macro(PreviousKind). + +is_macro(defmacro) -> true; +is_macro(defmacrop) -> true; +is_macro(_) -> false. + +%% Error handling +format_error({bad_kind, Module, {Name, Arity}, Kind}) -> + case is_macro(Kind) of + true -> + io_lib:format("cannot override function (def, defp) ~ts/~B in module ~ts as a macro (defmacro, defmacrop)", + [Name, Arity, elixir_aliases:inspect(Module)]); + false -> + io_lib:format("cannot override macro (defmacro, defmacrop) ~ts/~B in module ~ts as a function (def, defp)", + [Name, Arity, elixir_aliases:inspect(Module)]) + end; + +format_error({no_super, Module, {Name, Arity}}) -> + Bins = [format_fa(Tuple) || Tuple <- overridables_for(Module)], + Joined = 'Elixir.Enum':join(Bins, <<", ">>), + io_lib:format("no super defined for ~ts/~B in module ~ts. Overridable functions available are: ~ts", + [Name, Arity, elixir_aliases:inspect(Module), Joined]). + +format_fa({Name, Arity}) -> + A = 'Elixir.Macro':inspect_atom(remote_call, Name), + B = integer_to_binary(Arity), + <>. diff --git a/lib/elixir/src/elixir_parser.yrl b/lib/elixir/src/elixir_parser.yrl index e8d4c9f45aa..adfedc5907d 100644 --- a/lib/elixir/src/elixir_parser.yrl +++ b/lib/elixir/src/elixir_parser.yrl @@ -1,69 +1,95 @@ +%% SPDX-License-Identifier: Apache-2.0 +%% SPDX-FileCopyrightText: 2021 The Elixir Team +%% SPDX-FileCopyrightText: 2012 Plataformatec + +%% REUSE-IgnoreStart +Header "%% SPDX-License-Identifier: Apache-2.0" +"%% SPDX-FileCopyrightText: 2021 The Elixir Team" +"%% SPDX-FileCopyrightText: 2012 Plataformatec". +%% REUSE-IgnoreEnd + Nonterminals grammar expr_list - expr container_expr block_expr no_parens_expr no_parens_one_expr access_expr - bracket_expr bracket_at_expr bracket_arg matched_expr unmatched_expr max_expr - op_expr matched_op_expr no_parens_op_expr + expr container_expr block_expr access_expr + no_parens_expr no_parens_zero_expr no_parens_one_expr no_parens_one_ambig_expr + bracket_expr bracket_at_expr bracket_arg matched_expr unmatched_expr sub_matched_expr + unmatched_op_expr matched_op_expr no_parens_op_expr no_parens_many_expr comp_op_eol at_op_eol unary_op_eol and_op_eol or_op_eol capture_op_eol - add_op_eol mult_op_eol hat_op_eol two_op_eol pipe_op_eol stab_op_eol - arrow_op_eol match_op_eol when_op_eol in_op_eol in_match_op_eol - type_op_eol rel_op_eol - open_paren close_paren empty_paren + dual_op_eol mult_op_eol power_op_eol concat_op_eol xor_op_eol pipe_op_eol + stab_op_eol arrow_op_eol match_op_eol when_op_eol in_op_eol in_match_op_eol + type_op_eol rel_op_eol range_op_eol ternary_op_eol + open_paren close_paren empty_paren eoe list list_args open_bracket close_bracket tuple open_curly close_curly - bit_string open_bit close_bit - map map_op map_close map_args map_expr struct_op - assoc_op_eol assoc_expr assoc_base assoc_update assoc_update_kw assoc - container_args_base container_args call_args_parens_base call_args_parens parens_call - call_args_no_parens_one call_args_no_parens_expr call_args_no_parens_comma_expr - call_args_no_parens_all call_args_no_parens_many call_args_no_parens_many_strict - stab stab_eol stab_expr stab_maybe_expr stab_parens_many - kw_eol kw_base kw call_args_no_parens_kw_expr call_args_no_parens_kw - dot_op dot_alias dot_identifier dot_op_identifier dot_do_identifier - dot_paren_identifier dot_bracket_identifier - do_block fn_eol do_eol end_eol block_eol block_item block_list + bitstring open_bit close_bit + map map_op map_base_expr map_close map_args + assoc_op_eol assoc_expr assoc_base assoc assoc_update assoc_update_kw + container_args_base container_args + call_args_parens_expr call_args_parens_base call_args_parens parens_call + call_args_no_parens_one call_args_no_parens_ambig call_args_no_parens_expr + call_args_no_parens_comma_expr call_args_no_parens_all call_args_no_parens_many + call_args_no_parens_many_strict + stab stab_eoe stab_expr stab_op_eol_and_expr stab_parens_many + kw_eol kw_base kw_data kw_call call_args_no_parens_kw_expr call_args_no_parens_kw + dot_op dot_alias dot_bracket_identifier dot_call_identifier + dot_identifier dot_op_identifier dot_do_identifier dot_paren_identifier + do_block fn_eoe do_eoe block_eoe block_item block_list . Terminals identifier kw_identifier kw_identifier_safe kw_identifier_unsafe bracket_identifier - paren_identifier do_identifier block_identifier - fn 'end' aliases - number signed_number atom atom_safe atom_unsafe bin_string list_string sigil - dot_call_op op_identifier - comp_op at_op unary_op and_op or_op arrow_op match_op in_op in_match_op - type_op dual_op add_op mult_op hat_op two_op pipe_op stab_op when_op assoc_op - capture_op rel_op - 'true' 'false' 'nil' 'do' eol ',' '.' + paren_identifier do_identifier block_identifier op_identifier + fn 'end' alias + atom atom_quoted atom_safe atom_unsafe bin_string list_string sigil + bin_heredoc list_heredoc + comp_op at_op unary_op and_op or_op arrow_op match_op in_op in_match_op ellipsis_op + type_op dual_op mult_op power_op concat_op range_op xor_op pipe_op stab_op when_op + capture_int capture_op assoc_op rel_op ternary_op dot_call_op + 'true' 'false' 'nil' 'do' eol ';' ',' '.' '(' ')' '[' ']' '{' '}' '<<' '>>' '%{}' '%' + int flt char . Rootsymbol grammar. -%% There are two shift/reduce conflicts coming from call_args_parens. -Expect 2. +%% Two shift/reduce conflicts coming from call_args_parens and +%% one coming from empty_paren on stab. +Expect 3. + +%% Changes in ops and precedence should be reflected on: +%% +%% 1. lib/elixir/lib/code/identifier.ex +%% 2. lib/elixir/pages/operators.md +%% 3. lib/iex/lib/iex/evaluator.ex +%% +%% Note though the operator => in practice has lower precedence +%% than all others, its entry in the table is only to support the +%% %{user | foo => bar} syntax. -%% Changes in ops and precedence should be reflected on lib/elixir/lib/macro.ex -%% Note though the operator => in practice has lower precedence than all others, -%% its entry in the table is only to support the %{user | foo => bar} syntax. Left 5 do. Right 10 stab_op_eol. %% -> Left 20 ','. -Nonassoc 30 capture_op_eol. %% & Left 40 in_match_op_eol. %% <-, \\ (allowed in matches along =) Right 50 when_op_eol. %% when Right 60 type_op_eol. %% :: Right 70 pipe_op_eol. %% | Right 80 assoc_op_eol. %% => -Right 90 match_op_eol. %% = -Left 130 or_op_eol. %% ||, |||, or, xor -Left 140 and_op_eol. %% &&, &&&, and -Left 150 comp_op_eol. %% ==, !=, =~, ===, !== -Left 160 rel_op_eol. %% <, >, <=, >= -Left 170 arrow_op_eol. %% < (op), (op) > (e.g |>, <<<, >>>) -Left 180 in_op_eol. %% in -Right 200 two_op_eol. %% ++, --, .., <> -Left 210 add_op_eol. %% + (op), - (op) -Left 220 mult_op_eol. %% * (op), / (op) -Left 250 hat_op_eol. %% ^ (op) (e.g ^^^) +Nonassoc 90 capture_op_eol. %% & +Nonassoc 90 ellipsis_op. %% ... +Right 100 match_op_eol. %% = +Left 120 or_op_eol. %% ||, |||, or +Left 130 and_op_eol. %% &&, &&&, and +Left 140 comp_op_eol. %% ==, !=, =~, ===, !== +Left 150 rel_op_eol. %% <, >, <=, >= +Left 160 arrow_op_eol. %% <<<, >>>, |>, <<~, ~>>, <~, ~>, <~>, <|> +Left 170 in_op_eol. %% in, not in +Left 180 xor_op_eol. %% ^^^ +Right 190 ternary_op_eol. %% // +Right 200 concat_op_eol. %% ++, --, +++, ---, <> +Right 200 range_op_eol. %% .. +Left 210 dual_op_eol. %% +, - +Left 220 mult_op_eol. %% *, / +Left 230 power_op_eol. %% ** Nonassoc 300 unary_op_eol. %% +, -, !, ^, not, ~~~ Left 310 dot_call_op. Left 310 dot_op. %% . @@ -72,18 +98,17 @@ Nonassoc 330 dot_identifier. %%% MAIN FLOW OF EXPRESSIONS -grammar -> eol : nil. -grammar -> expr_list : to_block('$1'). -grammar -> eol expr_list : to_block('$2'). -grammar -> expr_list eol : to_block('$1'). -grammar -> eol expr_list eol : to_block('$2'). -grammar -> '$empty' : nil. +grammar -> eoe : {'__block__', meta_from_location({1, 1, nil}), []}. +grammar -> expr_list : build_block(reverse('$1')). +grammar -> eoe expr_list : build_block(reverse('$2')). +grammar -> expr_list eoe : build_block(reverse(annotate_eoe('$2', '$1'))). +grammar -> eoe expr_list eoe : build_block(reverse(annotate_eoe('$3', '$2'))). +grammar -> '$empty' : {'__block__', meta_from_location({1, 1, nil}), []}. % Note expressions are on reverse order expr_list -> expr : ['$1']. -expr_list -> expr_list eol expr : ['$3'|'$1']. +expr_list -> expr_list eoe expr : ['$3' | annotate_eoe('$2', '$1')]. -expr -> empty_paren : nil. expr -> matched_expr : '$1'. expr -> no_parens_expr : '$1'. expr -> unmatched_expr : '$1'. @@ -92,6 +117,23 @@ expr -> unmatched_expr : '$1'. %% without parentheses and with do blocks. They are represented %% in the AST as matched, no_parens and unmatched. %% +%% Calls without parentheses are further divided according to how +%% problematic they are: +%% +%% (a) no_parens_one: a call with one unproblematic argument +%% (for example, `f a` or `f g a` and similar) (includes unary operators) +%% +%% (b) no_parens_many: a call with several arguments (for example, `f a, b`) +%% +%% (c) no_parens_one_ambig: a call with one argument which is +%% itself a no_parens_many or no_parens_one_ambig (for example, `f g a, b`, +%% `f g h a, b` and similar) +%% +%% Note, in particular, that no_parens_one_ambig expressions are +%% ambiguous and are interpreted such that the outer function has +%% arity 1. For instance, `f g a, b` is interpreted as `f(g(a, b))` rather +%% than `f(g(a), b)`. Hence the name, no_parens_one_ambig. +%% %% The distinction is required because we can't, for example, have %% a function call with a do block as argument inside another do %% block call, unless there are parentheses: @@ -104,270 +146,328 @@ expr -> unmatched_expr : '$1'. %% %% foo a, bar b, c #=> invalid %% foo(a, bar b, c) #=> invalid -%% foo a, bar b #=> valid +%% foo bar a, b #=> valid %% foo a, bar(b, c) #=> valid %% %% So the different grammar rules need to take into account %% if calls without parentheses are do blocks in particular %% segments and act accordingly. -matched_expr -> matched_expr matched_op_expr : build_op(element(1, '$2'), '$1', element(2, '$2')). -matched_expr -> matched_expr no_parens_op_expr : build_op(element(1, '$2'), '$1', element(2, '$2')). +matched_expr -> matched_expr matched_op_expr : build_op('$1', '$2'). matched_expr -> unary_op_eol matched_expr : build_unary_op('$1', '$2'). -matched_expr -> unary_op_eol no_parens_expr : build_unary_op('$1', '$2'). matched_expr -> at_op_eol matched_expr : build_unary_op('$1', '$2'). -matched_expr -> at_op_eol no_parens_expr : build_unary_op('$1', '$2'). matched_expr -> capture_op_eol matched_expr : build_unary_op('$1', '$2'). -matched_expr -> capture_op_eol no_parens_expr : build_unary_op('$1', '$2'). +matched_expr -> ellipsis_op matched_expr : build_unary_op('$1', '$2'). matched_expr -> no_parens_one_expr : '$1'. -matched_expr -> access_expr : '$1'. - -no_parens_expr -> dot_op_identifier call_args_no_parens_many_strict : build_identifier('$1', '$2'). -no_parens_expr -> dot_identifier call_args_no_parens_many_strict : build_identifier('$1', '$2'). +matched_expr -> sub_matched_expr : '$1'. -unmatched_expr -> empty_paren op_expr : build_op(element(1, '$2'), nil, element(2, '$2')). -unmatched_expr -> matched_expr op_expr : build_op(element(1, '$2'), '$1', element(2, '$2')). -unmatched_expr -> unmatched_expr op_expr : build_op(element(1, '$2'), '$1', element(2, '$2')). +unmatched_expr -> matched_expr unmatched_op_expr : build_op('$1', '$2'). +unmatched_expr -> unmatched_expr matched_op_expr : build_op('$1', '$2'). +unmatched_expr -> unmatched_expr unmatched_op_expr : build_op('$1', '$2'). +unmatched_expr -> unmatched_expr no_parens_op_expr : warn_no_parens_after_do_op('$2'), build_op('$1', '$2'). unmatched_expr -> unary_op_eol expr : build_unary_op('$1', '$2'). unmatched_expr -> at_op_eol expr : build_unary_op('$1', '$2'). unmatched_expr -> capture_op_eol expr : build_unary_op('$1', '$2'). +unmatched_expr -> ellipsis_op expr : build_unary_op('$1', '$2'). unmatched_expr -> block_expr : '$1'. -block_expr -> parens_call call_args_parens do_block : build_identifier('$1', '$2' ++ '$3'). -block_expr -> parens_call call_args_parens call_args_parens do_block : build_nested_parens('$1', '$2', '$3' ++ '$4'). -block_expr -> dot_do_identifier do_block : build_identifier('$1', '$2'). -block_expr -> dot_identifier call_args_no_parens_all do_block : build_identifier('$1', '$2' ++ '$3'). - -op_expr -> match_op_eol expr : {'$1', '$2'}. -op_expr -> add_op_eol expr : {'$1', '$2'}. -op_expr -> mult_op_eol expr : {'$1', '$2'}. -op_expr -> hat_op_eol expr : {'$1', '$2'}. -op_expr -> two_op_eol expr : {'$1', '$2'}. -op_expr -> and_op_eol expr : {'$1', '$2'}. -op_expr -> or_op_eol expr : {'$1', '$2'}. -op_expr -> in_op_eol expr : {'$1', '$2'}. -op_expr -> in_match_op_eol expr : {'$1', '$2'}. -op_expr -> type_op_eol expr : {'$1', '$2'}. -op_expr -> when_op_eol expr : {'$1', '$2'}. -op_expr -> pipe_op_eol expr : {'$1', '$2'}. -op_expr -> comp_op_eol expr : {'$1', '$2'}. -op_expr -> rel_op_eol expr : {'$1', '$2'}. -op_expr -> arrow_op_eol expr : {'$1', '$2'}. +no_parens_expr -> matched_expr no_parens_op_expr : build_op('$1', '$2'). +no_parens_expr -> unary_op_eol no_parens_expr : build_unary_op('$1', '$2'). +no_parens_expr -> at_op_eol no_parens_expr : build_unary_op('$1', '$2'). +no_parens_expr -> capture_op_eol no_parens_expr : build_unary_op('$1', '$2'). +no_parens_expr -> ellipsis_op no_parens_expr : build_unary_op('$1', '$2'). +no_parens_expr -> no_parens_one_ambig_expr : '$1'. +no_parens_expr -> no_parens_many_expr : '$1'. + +block_expr -> dot_call_identifier call_args_parens do_block : build_parens('$1', '$2', '$3'). +block_expr -> dot_call_identifier call_args_parens call_args_parens do_block : build_nested_parens('$1', '$2', '$3', '$4'). +block_expr -> dot_do_identifier do_block : build_no_parens_do_block('$1', [], '$2'). +block_expr -> dot_op_identifier call_args_no_parens_all do_block : build_no_parens_do_block('$1', '$2', '$3'). +block_expr -> dot_identifier call_args_no_parens_all do_block : build_no_parens_do_block('$1', '$2', '$3'). + +matched_op_expr -> match_op_eol matched_expr : {'$1', '$2'}. +matched_op_expr -> dual_op_eol matched_expr : {'$1', '$2'}. +matched_op_expr -> mult_op_eol matched_expr : {'$1', '$2'}. +matched_op_expr -> power_op_eol matched_expr : {'$1', '$2'}. +matched_op_expr -> concat_op_eol matched_expr : {'$1', '$2'}. +matched_op_expr -> range_op_eol matched_expr : {'$1', '$2'}. +matched_op_expr -> ternary_op_eol matched_expr : {'$1', '$2'}. +matched_op_expr -> xor_op_eol matched_expr : {'$1', '$2'}. +matched_op_expr -> and_op_eol matched_expr : {'$1', '$2'}. +matched_op_expr -> or_op_eol matched_expr : {'$1', '$2'}. +matched_op_expr -> in_op_eol matched_expr : {'$1', '$2'}. +matched_op_expr -> in_match_op_eol matched_expr : {'$1', '$2'}. +matched_op_expr -> type_op_eol matched_expr : {'$1', '$2'}. +matched_op_expr -> when_op_eol matched_expr : {'$1', '$2'}. +matched_op_expr -> pipe_op_eol matched_expr : {'$1', '$2'}. +matched_op_expr -> comp_op_eol matched_expr : {'$1', '$2'}. +matched_op_expr -> rel_op_eol matched_expr : {'$1', '$2'}. +matched_op_expr -> arrow_op_eol matched_expr : {'$1', '$2'}. + +%% We warn exclusively of |> and friends because they are used +%% in other languages with lower precedence than function application, +%% which can be the source of confusion. +matched_op_expr -> arrow_op_eol no_parens_one_expr : warn_pipe('$1', '$2'), {'$1', '$2'}. + +unmatched_op_expr -> match_op_eol unmatched_expr : {'$1', '$2'}. +unmatched_op_expr -> dual_op_eol unmatched_expr : {'$1', '$2'}. +unmatched_op_expr -> mult_op_eol unmatched_expr : {'$1', '$2'}. +unmatched_op_expr -> power_op_eol unmatched_expr : {'$1', '$2'}. +unmatched_op_expr -> concat_op_eol unmatched_expr : {'$1', '$2'}. +unmatched_op_expr -> range_op_eol unmatched_expr : {'$1', '$2'}. +unmatched_op_expr -> ternary_op_eol unmatched_expr : {'$1', '$2'}. +unmatched_op_expr -> xor_op_eol unmatched_expr : {'$1', '$2'}. +unmatched_op_expr -> and_op_eol unmatched_expr : {'$1', '$2'}. +unmatched_op_expr -> or_op_eol unmatched_expr : {'$1', '$2'}. +unmatched_op_expr -> in_op_eol unmatched_expr : {'$1', '$2'}. +unmatched_op_expr -> in_match_op_eol unmatched_expr : {'$1', '$2'}. +unmatched_op_expr -> type_op_eol unmatched_expr : {'$1', '$2'}. +unmatched_op_expr -> when_op_eol unmatched_expr : {'$1', '$2'}. +unmatched_op_expr -> pipe_op_eol unmatched_expr : {'$1', '$2'}. +unmatched_op_expr -> comp_op_eol unmatched_expr : {'$1', '$2'}. +unmatched_op_expr -> rel_op_eol unmatched_expr : {'$1', '$2'}. +unmatched_op_expr -> arrow_op_eol unmatched_expr : {'$1', '$2'}. no_parens_op_expr -> match_op_eol no_parens_expr : {'$1', '$2'}. -no_parens_op_expr -> add_op_eol no_parens_expr : {'$1', '$2'}. +no_parens_op_expr -> dual_op_eol no_parens_expr : {'$1', '$2'}. no_parens_op_expr -> mult_op_eol no_parens_expr : {'$1', '$2'}. -no_parens_op_expr -> hat_op_eol no_parens_expr : {'$1', '$2'}. -no_parens_op_expr -> two_op_eol no_parens_expr : {'$1', '$2'}. +no_parens_op_expr -> power_op_eol no_parens_expr : {'$1', '$2'}. +no_parens_op_expr -> concat_op_eol no_parens_expr : {'$1', '$2'}. +no_parens_op_expr -> range_op_eol no_parens_expr : {'$1', '$2'}. +no_parens_op_expr -> ternary_op_eol no_parens_expr : {'$1', '$2'}. +no_parens_op_expr -> xor_op_eol no_parens_expr : {'$1', '$2'}. no_parens_op_expr -> and_op_eol no_parens_expr : {'$1', '$2'}. no_parens_op_expr -> or_op_eol no_parens_expr : {'$1', '$2'}. no_parens_op_expr -> in_op_eol no_parens_expr : {'$1', '$2'}. no_parens_op_expr -> in_match_op_eol no_parens_expr : {'$1', '$2'}. no_parens_op_expr -> type_op_eol no_parens_expr : {'$1', '$2'}. +no_parens_op_expr -> when_op_eol no_parens_expr : {'$1', '$2'}. no_parens_op_expr -> pipe_op_eol no_parens_expr : {'$1', '$2'}. no_parens_op_expr -> comp_op_eol no_parens_expr : {'$1', '$2'}. no_parens_op_expr -> rel_op_eol no_parens_expr : {'$1', '$2'}. -no_parens_op_expr -> arrow_op_eol no_parens_expr : {'$1', '$2'}. +no_parens_op_expr -> arrow_op_eol no_parens_expr : warn_pipe('$1', '$2'), {'$1', '$2'}. %% Allow when (and only when) with keywords -no_parens_op_expr -> when_op_eol no_parens_expr : {'$1', '$2'}. no_parens_op_expr -> when_op_eol call_args_no_parens_kw : {'$1', '$2'}. -matched_op_expr -> match_op_eol matched_expr : {'$1', '$2'}. -matched_op_expr -> add_op_eol matched_expr : {'$1', '$2'}. -matched_op_expr -> mult_op_eol matched_expr : {'$1', '$2'}. -matched_op_expr -> hat_op_eol matched_expr : {'$1', '$2'}. -matched_op_expr -> two_op_eol matched_expr : {'$1', '$2'}. -matched_op_expr -> and_op_eol matched_expr : {'$1', '$2'}. -matched_op_expr -> or_op_eol matched_expr : {'$1', '$2'}. -matched_op_expr -> in_op_eol matched_expr : {'$1', '$2'}. -matched_op_expr -> in_match_op_eol matched_expr : {'$1', '$2'}. -matched_op_expr -> type_op_eol matched_expr : {'$1', '$2'}. -matched_op_expr -> when_op_eol matched_expr : {'$1', '$2'}. -matched_op_expr -> pipe_op_eol matched_expr : {'$1', '$2'}. -matched_op_expr -> comp_op_eol matched_expr : {'$1', '$2'}. -matched_op_expr -> rel_op_eol matched_expr : {'$1', '$2'}. -matched_op_expr -> arrow_op_eol matched_expr : {'$1', '$2'}. +no_parens_one_ambig_expr -> dot_op_identifier call_args_no_parens_ambig : build_no_parens('$1', '$2'). +no_parens_one_ambig_expr -> dot_identifier call_args_no_parens_ambig : build_no_parens('$1', '$2'). + +no_parens_many_expr -> dot_op_identifier call_args_no_parens_many_strict : build_no_parens('$1', '$2'). +no_parens_many_expr -> dot_identifier call_args_no_parens_many_strict : build_no_parens('$1', '$2'). -no_parens_one_expr -> dot_op_identifier call_args_no_parens_one : build_identifier('$1', '$2'). -no_parens_one_expr -> dot_identifier call_args_no_parens_one : build_identifier('$1', '$2'). -no_parens_one_expr -> dot_do_identifier : build_identifier('$1', nil). -no_parens_one_expr -> dot_identifier : build_identifier('$1', nil). +no_parens_one_expr -> dot_op_identifier call_args_no_parens_one : build_no_parens('$1', '$2'). +no_parens_one_expr -> dot_identifier call_args_no_parens_one : build_no_parens('$1', '$2'). +no_parens_zero_expr -> dot_do_identifier : build_identifier('$1'). +no_parens_zero_expr -> dot_identifier : build_identifier('$1'). + +sub_matched_expr -> no_parens_zero_expr : '$1'. +sub_matched_expr -> range_op : build_nullary_op('$1'). +sub_matched_expr -> ellipsis_op : build_nullary_op('$1'). +sub_matched_expr -> access_expr : '$1'. +sub_matched_expr -> access_expr kw_identifier : error_invalid_kw_identifier('$2'). %% From this point on, we just have constructs that can be -%% used with the access syntax. Notice that (dot_)identifier +%% used with the access syntax. Note that (dot_)identifier %% is not included in this list simply because the tokenizer %% marks identifiers followed by brackets as bracket_identifier. access_expr -> bracket_at_expr : '$1'. access_expr -> bracket_expr : '$1'. -access_expr -> at_op_eol number : build_unary_op('$1', ?exprs('$2')). -access_expr -> unary_op_eol number : build_unary_op('$1', ?exprs('$2')). -access_expr -> capture_op_eol number : build_unary_op('$1', ?exprs('$2')). -access_expr -> fn_eol stab end_eol : build_fn('$1', build_stab(reverse('$2'))). -access_expr -> open_paren stab close_paren : build_stab(reverse('$2')). -access_expr -> number : ?exprs('$1'). -access_expr -> signed_number : {element(4, '$1'), meta('$1'), ?exprs('$1')}. +access_expr -> capture_int int : build_unary_op('$1', number_value('$2')). +access_expr -> fn_eoe stab_eoe 'end' : build_fn('$1', '$2', '$3'). +access_expr -> open_paren stab_eoe ')' : build_paren_stab('$1', '$2', '$3'). +access_expr -> open_paren ';' stab_eoe ')' : build_paren_stab('$1', '$3', '$4'). +access_expr -> open_paren ';' close_paren : build_paren_stab('$1', [], '$3'). +access_expr -> empty_paren : warn_empty_paren('$1'), {'__block__', parens_meta('$1'), []}. +access_expr -> int : handle_number(number_value('$1'), '$1', ?exprs('$1')). +access_expr -> flt : handle_number(number_value('$1'), '$1', ?exprs('$1')). +access_expr -> char : handle_number(?exprs('$1'), '$1', number_value('$1')). access_expr -> list : element(1, '$1'). access_expr -> map : '$1'. access_expr -> tuple : '$1'. -access_expr -> 'true' : ?id('$1'). -access_expr -> 'false' : ?id('$1'). -access_expr -> 'nil' : ?id('$1'). -access_expr -> bin_string : build_bin_string('$1'). -access_expr -> list_string : build_list_string('$1'). -access_expr -> bit_string : '$1'. +access_expr -> 'true' : handle_literal(?id('$1'), '$1'). +access_expr -> 'false' : handle_literal(?id('$1'), '$1'). +access_expr -> 'nil' : handle_literal(?id('$1'), '$1'). +access_expr -> bin_string : build_bin_string('$1', delimiter(<<$">>)). +access_expr -> list_string : build_list_string('$1', delimiter(<<$'>>)). +access_expr -> bin_heredoc : build_bin_heredoc('$1'). +access_expr -> list_heredoc : build_list_heredoc('$1'). +access_expr -> bitstring : '$1'. access_expr -> sigil : build_sigil('$1'). -access_expr -> max_expr : '$1'. +access_expr -> atom : handle_literal(?exprs('$1'), '$1', atom_colon_meta('$1')). +access_expr -> atom_quoted : handle_literal(?exprs('$1'), '$1', atom_delimiter_meta('$1')). +access_expr -> atom_safe : build_quoted_atom('$1', true, atom_delimiter_meta('$1')). +access_expr -> atom_unsafe : build_quoted_atom('$1', false, atom_delimiter_meta('$1')). +access_expr -> dot_alias : '$1'. +access_expr -> parens_call : '$1'. -%% Aliases and properly formed calls. Used by map_expr. -max_expr -> atom : ?exprs('$1'). -max_expr -> atom_safe : build_quoted_atom('$1', true). -max_expr -> atom_unsafe : build_quoted_atom('$1', false). -max_expr -> parens_call call_args_parens : build_identifier('$1', '$2'). -max_expr -> parens_call call_args_parens call_args_parens : build_nested_parens('$1', '$2', '$3'). -max_expr -> dot_alias : '$1'. +%% Also used by maps and structs +parens_call -> dot_call_identifier call_args_parens : build_parens('$1', '$2', {[], []}). +parens_call -> dot_call_identifier call_args_parens call_args_parens : build_nested_parens('$1', '$2', '$3', {[], []}). -bracket_arg -> open_bracket ']' : build_list('$1', []). -bracket_arg -> open_bracket kw close_bracket : build_list('$1', '$2'). -bracket_arg -> open_bracket container_expr close_bracket : build_list('$1', '$2'). -bracket_arg -> open_bracket container_expr ',' close_bracket : build_list('$1', '$2'). +bracket_arg -> open_bracket kw_data close_bracket : build_access_arg('$1', '$2', '$3'). +bracket_arg -> open_bracket container_expr close_bracket : build_access_arg('$1', '$2', '$3'). +bracket_arg -> open_bracket container_expr ',' close_bracket : build_access_arg('$1', '$2', '$4'). +bracket_arg -> open_bracket container_expr ',' container_args close_bracket : error_too_many_access_syntax('$3'). -bracket_expr -> dot_bracket_identifier bracket_arg : build_access(build_identifier('$1', nil), '$2'). -bracket_expr -> access_expr bracket_arg : build_access('$1', '$2'). +bracket_expr -> dot_bracket_identifier bracket_arg : build_access(build_identifier('$1'), meta_with_from_brackets('$2')). +bracket_expr -> access_expr bracket_arg : build_access('$1', meta_with_from_brackets('$2')). bracket_at_expr -> at_op_eol dot_bracket_identifier bracket_arg : - build_access(build_unary_op('$1', build_identifier('$2', nil)), '$3'). + build_access(build_unary_op('$1', build_identifier('$2')), meta_with_from_brackets('$3')). bracket_at_expr -> at_op_eol access_expr bracket_arg : - build_access(build_unary_op('$1', '$2'), '$3'). + build_access(build_unary_op('$1', '$2'), meta_with_from_brackets('$3')). %% Blocks -do_block -> do_eol 'end' : [[{do,nil}]]. -do_block -> do_eol stab end_eol : [[{do, build_stab(reverse('$2'))}]]. -do_block -> do_eol block_list 'end' : [[{do, nil}|'$2']]. -do_block -> do_eol stab_eol block_list 'end' : [[{do, build_stab(reverse('$2'))}|'$3']]. +do_block -> do_eoe 'end' : + {do_end_meta('$1', '$2'), [[{handle_literal(do, '$1'), {'__block__', meta_from_token('$1'), []}}]]}. +do_block -> do_eoe stab_eoe 'end' : + {do_end_meta('$1', '$3'), [[{handle_literal(do, '$1'), build_stab('$2', meta_from_token('$1'))}]]}. +do_block -> do_eoe block_list 'end' : + {do_end_meta('$1', '$3'), [[{handle_literal(do, '$1'), {'__block__', meta_from_token('$1'), []}} | '$2']]}. +do_block -> do_eoe stab_eoe block_list 'end' : + {do_end_meta('$1', '$4'), [[{handle_literal(do, '$1'), build_stab('$2', meta_from_token('$1'))} | '$3']]}. -fn_eol -> 'fn' : '$1'. -fn_eol -> 'fn' eol : '$1'. +eoe -> eol : '$1'. +eoe -> ';' : '$1'. +eoe -> eol ';' : '$1'. -do_eol -> 'do' : '$1'. -do_eol -> 'do' eol : '$1'. +fn_eoe -> 'fn' : '$1'. +fn_eoe -> 'fn' eoe : next_is_eol('$1', '$2'). -end_eol -> 'end' : '$1'. -end_eol -> eol 'end' : '$2'. +do_eoe -> 'do' : '$1'. +do_eoe -> 'do' eoe : '$1'. -block_eol -> block_identifier : '$1'. -block_eol -> block_identifier eol : '$1'. +block_eoe -> block_identifier : '$1'. +block_eoe -> block_identifier eoe : '$1'. stab -> stab_expr : ['$1']. -stab -> stab eol stab_expr : ['$3'|'$1']. - -stab_eol -> stab : '$1'. -stab_eol -> stab eol : '$1'. - -stab_expr -> expr : '$1'. -stab_expr -> stab_op_eol stab_maybe_expr : build_op('$1', [], '$2'). -stab_expr -> call_args_no_parens_all stab_op_eol stab_maybe_expr : - build_op('$2', unwrap_when(unwrap_splice('$1')), '$3'). -stab_expr -> stab_parens_many stab_op_eol stab_maybe_expr : - build_op('$2', unwrap_splice('$1'), '$3'). -stab_expr -> stab_parens_many when_op expr stab_op_eol stab_maybe_expr : - build_op('$4', [{'when', meta('$2'), unwrap_splice('$1') ++ ['$3']}], '$5'). - -stab_maybe_expr -> 'expr' : '$1'. -stab_maybe_expr -> '$empty' : nil. - -block_item -> block_eol stab_eol : {?exprs('$1'), build_stab(reverse('$2'))}. -block_item -> block_eol : {?exprs('$1'), nil}. +stab -> stab eoe stab_expr : ['$3' | annotate_eoe('$2', '$1')]. + +stab_eoe -> stab : '$1'. +stab_eoe -> stab eoe : annotate_eoe('$2', '$1'). + +stab_expr -> expr : + '$1'. +stab_expr -> stab_op_eol_and_expr : + build_op([], '$1'). +stab_expr -> empty_paren stab_op_eol_and_expr : + build_op_with_meta([], '$2', parens_meta('$1')). +stab_expr -> empty_paren when_op expr stab_op_eol_and_expr : + build_op_with_meta([{'when', meta_from_token('$2'), ['$3']}], '$4', parens_meta('$1')). +stab_expr -> call_args_no_parens_all stab_op_eol_and_expr : + build_op(unwrap_when(unwrap_splice('$1')), '$2'). +stab_expr -> stab_parens_many stab_op_eol_and_expr : + build_op_with_meta(unwrap_splice(element(2, '$1')), '$2', parens_meta('$1')). +stab_expr -> stab_parens_many when_op expr stab_op_eol_and_expr : + build_op_with_meta([{'when', meta_from_token('$2'), unwrap_splice(element(2, '$1')) ++ ['$3']}], '$4', parens_meta('$1')). + +stab_op_eol_and_expr -> stab_op_eol expr : {'$1', '$2'}. +stab_op_eol_and_expr -> stab_op_eol : warn_empty_stab_clause('$1'), {'$1', handle_literal(nil, '$1')}. + +block_item -> block_eoe stab_eoe : + {handle_literal(?exprs('$1'), '$1'), build_stab('$2', [])}. +block_item -> block_eoe : + {handle_literal(?exprs('$1'), '$1'), {'__block__', [], []}}. block_list -> block_item : ['$1']. -block_list -> block_item block_list : ['$1'|'$2']. +block_list -> block_item block_list : ['$1' | '$2']. %% Helpers open_paren -> '(' : '$1'. -open_paren -> '(' eol : '$1'. +open_paren -> '(' eol : next_is_eol('$1', '$2'). close_paren -> ')' : '$1'. close_paren -> eol ')' : '$2'. -empty_paren -> open_paren ')' : '$1'. +empty_paren -> open_paren ')' : {'$1', '$2'}. -open_bracket -> '[' : '$1'. -open_bracket -> '[' eol : '$1'. -close_bracket -> ']' : '$1'. -close_bracket -> eol ']' : '$2'. +open_bracket -> '[' : '$1'. +open_bracket -> '[' eol : next_is_eol('$1', '$2'). +close_bracket -> ']' : '$1'. +close_bracket -> eol ']' : '$2'. -open_bit -> '<<' : '$1'. -open_bit -> '<<' eol : '$1'. -close_bit -> '>>' : '$1'. -close_bit -> eol '>>' : '$2'. +open_bit -> '<<' : '$1'. +open_bit -> '<<' eol : next_is_eol('$1', '$2'). +close_bit -> '>>' : '$1'. +close_bit -> eol '>>' : '$2'. open_curly -> '{' : '$1'. -open_curly -> '{' eol : '$1'. -close_curly -> '}' : '$1'. -close_curly -> eol '}' : '$2'. +open_curly -> '{' eol : next_is_eol('$1', '$2'). +close_curly -> '}' : '$1'. +close_curly -> eol '}' : '$2'. % Operators -add_op_eol -> add_op : '$1'. -add_op_eol -> add_op eol : '$1'. -add_op_eol -> dual_op : '$1'. -add_op_eol -> dual_op eol : '$1'. +unary_op_eol -> unary_op : '$1'. +unary_op_eol -> unary_op eol : '$1'. +unary_op_eol -> dual_op : '$1'. +unary_op_eol -> dual_op eol : '$1'. +unary_op_eol -> ternary_op : '$1'. +unary_op_eol -> ternary_op eol : '$1'. + +capture_op_eol -> capture_op : '$1'. +capture_op_eol -> capture_op eol : '$1'. + +at_op_eol -> at_op : '$1'. +at_op_eol -> at_op eol : '$1'. + +match_op_eol -> match_op : '$1'. +match_op_eol -> match_op eol : next_is_eol('$1', '$2'). + +dual_op_eol -> dual_op : '$1'. +dual_op_eol -> dual_op eol : next_is_eol('$1', '$2'). mult_op_eol -> mult_op : '$1'. -mult_op_eol -> mult_op eol : '$1'. +mult_op_eol -> mult_op eol : next_is_eol('$1', '$2'). -hat_op_eol -> hat_op : '$1'. -hat_op_eol -> hat_op eol : '$1'. +power_op_eol -> power_op : '$1'. +power_op_eol -> power_op eol : next_is_eol('$1', '$2'). -two_op_eol -> two_op : '$1'. -two_op_eol -> two_op eol : '$1'. +concat_op_eol -> concat_op : '$1'. +concat_op_eol -> concat_op eol : next_is_eol('$1', '$2'). -pipe_op_eol -> pipe_op : '$1'. -pipe_op_eol -> pipe_op eol : '$1'. +range_op_eol -> range_op : '$1'. +range_op_eol -> range_op eol : next_is_eol('$1', '$2'). -capture_op_eol -> capture_op : '$1'. -capture_op_eol -> capture_op eol : '$1'. +ternary_op_eol -> ternary_op : '$1'. +ternary_op_eol -> ternary_op eol : next_is_eol('$1', '$2'). -unary_op_eol -> unary_op : '$1'. -unary_op_eol -> unary_op eol : '$1'. -unary_op_eol -> dual_op : '$1'. -unary_op_eol -> dual_op eol : '$1'. +xor_op_eol -> xor_op : '$1'. +xor_op_eol -> xor_op eol : next_is_eol('$1', '$2'). -match_op_eol -> match_op : '$1'. -match_op_eol -> match_op eol : '$1'. +pipe_op_eol -> pipe_op : '$1'. +pipe_op_eol -> pipe_op eol : next_is_eol('$1', '$2'). and_op_eol -> and_op : '$1'. -and_op_eol -> and_op eol : '$1'. +and_op_eol -> and_op eol : next_is_eol('$1', '$2'). or_op_eol -> or_op : '$1'. -or_op_eol -> or_op eol : '$1'. +or_op_eol -> or_op eol : next_is_eol('$1', '$2'). in_op_eol -> in_op : '$1'. -in_op_eol -> in_op eol : '$1'. +in_op_eol -> in_op eol : next_is_eol('$1', '$2'). in_match_op_eol -> in_match_op : '$1'. -in_match_op_eol -> in_match_op eol : '$1'. +in_match_op_eol -> in_match_op eol : next_is_eol('$1', '$2'). type_op_eol -> type_op : '$1'. -type_op_eol -> type_op eol : '$1'. +type_op_eol -> type_op eol : next_is_eol('$1', '$2'). when_op_eol -> when_op : '$1'. -when_op_eol -> when_op eol : '$1'. +when_op_eol -> when_op eol : next_is_eol('$1', '$2'). stab_op_eol -> stab_op : '$1'. -stab_op_eol -> stab_op eol : '$1'. - -at_op_eol -> at_op : '$1'. -at_op_eol -> at_op eol : '$1'. +stab_op_eol -> stab_op eol : next_is_eol('$1', '$2'). comp_op_eol -> comp_op : '$1'. -comp_op_eol -> comp_op eol : '$1'. +comp_op_eol -> comp_op eol : next_is_eol('$1', '$2'). rel_op_eol -> rel_op : '$1'. -rel_op_eol -> rel_op eol : '$1'. +rel_op_eol -> rel_op eol : next_is_eol('$1', '$2'). arrow_op_eol -> arrow_op : '$1'. -arrow_op_eol -> arrow_op eol : '$1'. +arrow_op_eol -> arrow_op eol : next_is_eol('$1', '$2'). % Dot operator @@ -377,8 +477,10 @@ dot_op -> '.' eol : '$1'. dot_identifier -> identifier : '$1'. dot_identifier -> matched_expr dot_op identifier : build_dot('$2', '$1', '$3'). -dot_alias -> aliases : {'__aliases__', meta('$1', 0), ?exprs('$1')}. -dot_alias -> matched_expr dot_op aliases : build_dot_alias('$2', '$1', '$3'). +dot_alias -> alias : build_alias('$1'). +dot_alias -> matched_expr dot_op alias : build_dot_alias('$2', '$1', '$3'). +dot_alias -> matched_expr dot_op open_curly '}' : build_dot_container('$2', '$1', [], newlines_pair('$3', '$4')). +dot_alias -> matched_expr dot_op open_curly container_args close_curly : build_dot_container('$2', '$1', '$4', newlines_pair('$3', '$5')). dot_op_identifier -> op_identifier : '$1'. dot_op_identifier -> matched_expr dot_op op_identifier : build_dot('$2', '$1', '$3'). @@ -392,121 +494,146 @@ dot_bracket_identifier -> matched_expr dot_op bracket_identifier : build_dot('$2 dot_paren_identifier -> paren_identifier : '$1'. dot_paren_identifier -> matched_expr dot_op paren_identifier : build_dot('$2', '$1', '$3'). -parens_call -> dot_paren_identifier : '$1'. -parens_call -> matched_expr dot_call_op : {'.', meta('$2'), ['$1']}. % Fun/local calls +dot_call_identifier -> dot_paren_identifier : '$1'. +dot_call_identifier -> matched_expr dot_call_op : {'.', meta_from_token('$2'), ['$1']}. % Fun/local calls % Function calls with no parentheses call_args_no_parens_expr -> matched_expr : '$1'. -call_args_no_parens_expr -> empty_paren : nil. -call_args_no_parens_expr -> no_parens_expr : throw_no_parens_many_strict('$1'). +call_args_no_parens_expr -> no_parens_expr : error_no_parens_many_strict('$1'). call_args_no_parens_comma_expr -> matched_expr ',' call_args_no_parens_expr : ['$3', '$1']. -call_args_no_parens_comma_expr -> call_args_no_parens_comma_expr ',' call_args_no_parens_expr : ['$3'|'$1']. +call_args_no_parens_comma_expr -> call_args_no_parens_comma_expr ',' call_args_no_parens_expr : ['$3' | '$1']. call_args_no_parens_all -> call_args_no_parens_one : '$1'. +call_args_no_parens_all -> call_args_no_parens_ambig : '$1'. call_args_no_parens_all -> call_args_no_parens_many : '$1'. call_args_no_parens_one -> call_args_no_parens_kw : ['$1']. call_args_no_parens_one -> matched_expr : ['$1']. -call_args_no_parens_one -> no_parens_expr : ['$1']. + +%% This is the only no parens ambiguity where we don't +%% raise nor warn: "parent_call nested_call 1, 2, 3" +%% always assumes that all arguments are nested. +call_args_no_parens_ambig -> no_parens_expr : ['$1']. call_args_no_parens_many -> matched_expr ',' call_args_no_parens_kw : ['$1', '$3']. call_args_no_parens_many -> call_args_no_parens_comma_expr : reverse('$1'). -call_args_no_parens_many -> call_args_no_parens_comma_expr ',' call_args_no_parens_kw : reverse(['$3'|'$1']). +call_args_no_parens_many -> call_args_no_parens_comma_expr ',' call_args_no_parens_kw : reverse(['$3' | '$1']). call_args_no_parens_many_strict -> call_args_no_parens_many : '$1'. -call_args_no_parens_many_strict -> empty_paren : throw_no_parens_strict('$1'). -call_args_no_parens_many_strict -> open_paren call_args_no_parens_kw close_paren : throw_no_parens_strict('$1'). -call_args_no_parens_many_strict -> open_paren call_args_no_parens_many close_paren : throw_no_parens_strict('$1'). +call_args_no_parens_many_strict -> open_paren call_args_no_parens_kw close_paren : error_no_parens_strict('$1'). +call_args_no_parens_many_strict -> open_paren call_args_no_parens_many close_paren : error_no_parens_strict('$1'). -stab_parens_many -> empty_paren : []. -stab_parens_many -> open_paren call_args_no_parens_kw close_paren : ['$2']. -stab_parens_many -> open_paren call_args_no_parens_many close_paren : '$2'. +stab_parens_many -> open_paren call_args_no_parens_kw close_paren : {'$1', ['$2'], '$3'}. +stab_parens_many -> open_paren call_args_no_parens_many close_paren : {'$1', '$2', '$3'}. -% Containers and function calls with parentheses +% Containers -container_expr -> empty_paren : nil. container_expr -> matched_expr : '$1'. container_expr -> unmatched_expr : '$1'. -container_expr -> no_parens_expr : throw_no_parens_many_strict('$1'). +container_expr -> no_parens_expr : error_no_parens_container_strict('$1'). container_args_base -> container_expr : ['$1']. -container_args_base -> container_args_base ',' container_expr : ['$3'|'$1']. +container_args_base -> container_args_base ',' container_expr : ['$3' | '$1']. -container_args -> container_args_base : lists:reverse('$1'). -container_args -> container_args_base ',' : lists:reverse('$1'). -container_args -> container_args_base ',' kw : lists:reverse(['$3'|'$1']). +container_args -> container_args_base : reverse('$1'). +container_args -> container_args_base ',' : reverse('$1'). +container_args -> container_args_base ',' kw_data : reverse(['$3' | '$1']). -call_args_parens_base -> container_expr : ['$1']. -call_args_parens_base -> call_args_parens_base ',' container_expr : ['$3'|'$1']. +% Function calls with parentheses -call_args_parens -> empty_paren : []. -call_args_parens -> open_paren no_parens_expr close_paren : ['$2']. -call_args_parens -> open_paren kw close_paren : ['$2']. -call_args_parens -> open_paren call_args_parens_base close_paren : reverse('$2'). -call_args_parens -> open_paren call_args_parens_base ',' kw close_paren : reverse(['$4'|'$2']). +call_args_parens_expr -> matched_expr : '$1'. +call_args_parens_expr -> unmatched_expr : '$1'. +call_args_parens_expr -> no_parens_expr : error_no_parens_many_strict('$1'). + +call_args_parens_base -> call_args_parens_expr : ['$1']. +call_args_parens_base -> call_args_parens_base ',' call_args_parens_expr : ['$3' | '$1']. + +call_args_parens -> open_paren ')' : + {newlines_pair('$1', '$2'), []}. +call_args_parens -> open_paren no_parens_expr close_paren : + {newlines_pair('$1', '$3'), ['$2']}. +call_args_parens -> open_paren kw_call close_paren : + {newlines_pair('$1', '$3'), ['$2']}. +call_args_parens -> open_paren call_args_parens_base close_paren : + {newlines_pair('$1', '$3'), reverse('$2')}. +call_args_parens -> open_paren call_args_parens_base ',' kw_call close_paren : + {newlines_pair('$1', '$5'), reverse(['$4' | '$2'])}. % KV -kw_eol -> kw_identifier : ?exprs('$1'). -kw_eol -> kw_identifier eol : ?exprs('$1'). -kw_eol -> kw_identifier_safe : build_quoted_atom('$1', true). -kw_eol -> kw_identifier_safe eol : build_quoted_atom('$1', true). -kw_eol -> kw_identifier_unsafe : build_quoted_atom('$1', false). -kw_eol -> kw_identifier_unsafe eol : build_quoted_atom('$1', false). +kw_eol -> kw_identifier : handle_literal(?exprs('$1'), '$1', kw_identifier_meta('$1')). +kw_eol -> kw_identifier eol : handle_literal(?exprs('$1'), '$1', kw_identifier_meta('$1')). +kw_eol -> kw_identifier_safe : build_quoted_atom('$1', true, kw_identifier_meta('$1')). +kw_eol -> kw_identifier_safe eol : build_quoted_atom('$1', true, kw_identifier_meta('$1')). +kw_eol -> kw_identifier_unsafe : build_quoted_atom('$1', false, kw_identifier_meta('$1')). +kw_eol -> kw_identifier_unsafe eol : build_quoted_atom('$1', false, kw_identifier_meta('$1')). kw_base -> kw_eol container_expr : [{'$1', '$2'}]. -kw_base -> kw_base ',' kw_eol container_expr : [{'$3', '$4'}|'$1']. +kw_base -> kw_base ',' kw_eol container_expr : [{'$3', '$4'} | '$1']. + +kw_call -> kw_base : reverse('$1'). +kw_call -> kw_base ',' : warn_trailing_comma('$2'), reverse('$1'). +kw_call -> kw_base ',' matched_expr : maybe_bad_keyword_call_follow_up('$2', '$1', '$3'). -kw -> kw_base : reverse('$1'). -kw -> kw_base ',' : reverse('$1'). +kw_data -> kw_base : reverse('$1'). +kw_data -> kw_base ',' : reverse('$1'). +kw_data -> kw_base ',' matched_expr : maybe_bad_keyword_data_follow_up('$2', '$1', '$3'). + +call_args_no_parens_kw_expr -> kw_eol matched_expr : {'$1', '$2'}. +call_args_no_parens_kw_expr -> kw_eol no_parens_expr : warn_nested_no_parens_keyword('$1', '$2'), {'$1', '$2'}. -call_args_no_parens_kw_expr -> kw_eol call_args_no_parens_expr : {'$1','$2'}. call_args_no_parens_kw -> call_args_no_parens_kw_expr : ['$1']. -call_args_no_parens_kw -> call_args_no_parens_kw_expr ',' call_args_no_parens_kw : ['$1'|'$3']. +call_args_no_parens_kw -> call_args_no_parens_kw_expr ',' call_args_no_parens_kw : ['$1' | '$3']. +call_args_no_parens_kw -> call_args_no_parens_kw_expr ',' matched_expr : maybe_bad_keyword_call_follow_up('$2', ['$1'], '$3'). % Lists -list_args -> kw : '$1'. +list_args -> kw_data : '$1'. list_args -> container_args_base : reverse('$1'). list_args -> container_args_base ',' : reverse('$1'). -list_args -> container_args_base ',' kw : reverse('$1', '$3'). +list_args -> container_args_base ',' kw_data : reverse('$1', '$3'). -list -> open_bracket ']' : build_list('$1', []). -list -> open_bracket list_args close_bracket : build_list('$1', '$2'). +list -> open_bracket ']' : build_list('$1', [], '$2'). +list -> open_bracket list_args close_bracket : build_list('$1', '$2', '$3'). % Tuple -tuple -> open_curly '}' : build_tuple('$1', []). -tuple -> open_curly container_args close_curly : build_tuple('$1', '$2'). +tuple -> open_curly '}' : build_tuple('$1', [], '$2'). +tuple -> open_curly kw_data '}' : bad_keyword('$1', tuple, "'{'"). +tuple -> open_curly container_args close_curly : build_tuple('$1', '$2', '$3'). % Bitstrings -bit_string -> open_bit '>>' : build_bit('$1', []). -bit_string -> open_bit container_args close_bit : build_bit('$1', '$2'). +bitstring -> open_bit '>>' : build_bit('$1', [], '$2'). +bitstring -> open_bit kw_data '>>' : bad_keyword('$1', bitstring, "'<<'"). +bitstring -> open_bit container_args close_bit : build_bit('$1', '$2', '$3'). % Map and structs -map_expr -> max_expr : '$1'. -map_expr -> dot_identifier : build_identifier('$1', nil). -map_expr -> at_op_eol map_expr : build_unary_op('$1', '$2'). +map_base_expr -> sub_matched_expr : '$1'. +map_base_expr -> at_op_eol map_base_expr : build_unary_op('$1', '$2'). +map_base_expr -> unary_op_eol map_base_expr : build_unary_op('$1', '$2'). +map_base_expr -> ellipsis_op map_base_expr : build_unary_op('$1', '$2'). assoc_op_eol -> assoc_op : '$1'. assoc_op_eol -> assoc_op eol : '$1'. -assoc_expr -> container_expr assoc_op_eol container_expr : {'$1', '$3'}. -assoc_expr -> map_expr : '$1'. +assoc_expr -> matched_expr assoc_op_eol matched_expr : {with_assoc_meta('$1', '$2'), '$3'}. +assoc_expr -> unmatched_expr assoc_op_eol unmatched_expr : {with_assoc_meta('$1', '$2'), '$3'}. +assoc_expr -> matched_expr assoc_op_eol unmatched_expr : {with_assoc_meta('$1', '$2'), '$3'}. +assoc_expr -> unmatched_expr assoc_op_eol matched_expr : {with_assoc_meta('$1', '$2'), '$3'}. +assoc_expr -> map_base_expr : '$1'. -assoc_update -> matched_expr pipe_op_eol matched_expr assoc_op_eol matched_expr : {'$2', '$1', [{'$3', '$5'}]}. -assoc_update -> unmatched_expr pipe_op_eol expr assoc_op_eol expr : {'$2', '$1', [{'$3', '$5'}]}. -assoc_update -> matched_expr pipe_op_eol map_expr : {'$2', '$1', ['$3']}. +assoc_update -> matched_expr pipe_op_eol assoc_expr : {'$2', '$1', ['$3']}. +assoc_update -> unmatched_expr pipe_op_eol assoc_expr : {'$2', '$1', ['$3']}. -assoc_update_kw -> matched_expr pipe_op_eol kw : {'$2', '$1', '$3'}. -assoc_update_kw -> unmatched_expr pipe_op_eol kw : {'$2', '$1', '$3'}. +assoc_update_kw -> matched_expr pipe_op_eol kw_data : {'$2', '$1', '$3'}. +assoc_update_kw -> unmatched_expr pipe_op_eol kw_data : {'$2', '$1', '$3'}. assoc_base -> assoc_expr : ['$1']. -assoc_base -> assoc_base ',' assoc_expr : ['$3'|'$1']. +assoc_base -> assoc_base ',' assoc_expr : ['$3' | '$1']. assoc -> assoc_base : reverse('$1'). assoc -> assoc_base ',' : reverse('$1'). @@ -514,86 +641,268 @@ assoc -> assoc_base ',' : reverse('$1'). map_op -> '%{}' : '$1'. map_op -> '%{}' eol : '$1'. -map_close -> kw close_curly : '$1'. -map_close -> assoc close_curly : '$1'. -map_close -> assoc_base ',' kw close_curly : reverse('$1', '$3'). - -map_args -> open_curly '}' : build_map('$1', []). -map_args -> open_curly map_close : build_map('$1', '$2'). -map_args -> open_curly assoc_update close_curly : build_map_update('$1', '$2', []). -map_args -> open_curly assoc_update ',' close_curly : build_map_update('$1', '$2', []). -map_args -> open_curly assoc_update ',' map_close : build_map_update('$1', '$2', '$4'). -map_args -> open_curly assoc_update_kw close_curly : build_map_update('$1', '$2', []). +map_close -> kw_data close_curly : {'$1', '$2'}. +map_close -> assoc close_curly : {'$1', '$2'}. +map_close -> assoc_base ',' kw_data close_curly : {reverse('$1', '$3'), '$4'}. -struct_op -> '%' : '$1'. -struct_op -> '%' eol : '$1'. +map_args -> open_curly '}' : build_map('$1', [], '$2'). +map_args -> open_curly map_close : build_map('$1', element(1, '$2'), element(2, '$2')). +map_args -> open_curly assoc_update close_curly : build_map_update('$1', '$2', '$3', []). +map_args -> open_curly assoc_update ',' close_curly : build_map_update('$1', '$2', '$4', []). +map_args -> open_curly assoc_update ',' map_close : build_map_update('$1', '$2', element(2, '$4'), element(1, '$4')). +map_args -> open_curly assoc_update_kw close_curly : build_map_update('$1', '$2', '$3', []). map -> map_op map_args : '$2'. -map -> struct_op map_expr map_args : {'%', meta('$1'), ['$2', '$3']}. -map -> struct_op map_expr eol map_args : {'%', meta('$1'), ['$2', '$4']}. +map -> '%' map_base_expr map_args : {'%', meta_from_token('$1'), ['$2', '$3']}. +map -> '%' map_base_expr eol map_args : {'%', meta_from_token('$1'), ['$2', '$4']}. Erlang code. --define(id(Node), element(1, Node)). --define(line(Node), element(2, Node)). --define(exprs(Node), element(3, Node)). +-define(columns(), get(elixir_parser_columns)). +-define(token_metadata(), get(elixir_token_metadata)). + +-define(id(Token), element(1, Token)). +-define(location(Token), element(2, Token)). +-define(exprs(Token), element(3, Token)). +-define(meta(Node), element(2, Node)). -define(rearrange_uop(Op), (Op == 'not' orelse Op == '!')). -%% The following directive is needed for (significantly) faster -%% compilation of the generated .erl file by the HiPE compiler --compile([{hipe,[{regalloc,linear_scan}]}]). +-compile({inline, meta_from_token/1, meta_from_location/1, is_eol/1}). -import(lists, [reverse/1, reverse/2]). -meta(Line, Counter) -> [{counter,Counter}|meta(Line)]. -meta(Line) when is_integer(Line) -> [{line,Line}]; -meta(Node) -> meta(?line(Node)). +meta_from_token(Token) -> + meta_from_location(?location(Token)). + +meta_from_location({Line, Column, _}) -> + case ?columns() of + true -> [{line, Line}, {column, Column}]; + false -> [{line, Line}] + end. + +do_end_meta(Do, End) -> + case ?token_metadata() of + true -> + [{do, meta_from_token(Do)}, {'end', meta_from_token(End)}]; + false -> + [] + end. + +meta_from_token_with_closing(Begin, End) -> + case ?token_metadata() of + true -> + [{closing, meta_from_token(End)} | meta_from_token(Begin)]; + false -> + meta_from_token(Begin) + end. + +append_non_empty(Left, []) -> Left; +append_non_empty(Left, Right) -> Left ++ Right. + +%% Handle metadata in literals + +handle_literal(Literal, Token) -> + handle_literal(Literal, Token, []). + +handle_literal(Literal, Token, ExtraMeta) -> + case get(elixir_literal_encoder) of + false -> + Literal; + + Fun -> + Meta = ExtraMeta ++ meta_from_token(Token), + case Fun(Literal, Meta) of + {ok, EncodedLiteral} -> + EncodedLiteral; + {error, Reason} -> + return_error(?location(Token), elixir_utils:characters_to_list(Reason) ++ [": "], "literal") + end + end. + +handle_number(Number, Token, Original) -> + case ?token_metadata() of + true -> handle_literal(Number, Token, [{token, elixir_utils:characters_to_binary(Original)}]); + false -> handle_literal(Number, Token, []) + end. + +number_value({_, {_, _, Value}, _}) -> + Value. %% Operators -build_op({_Kind, Line, 'in'}, {UOp, _, [Left]}, Right) when ?rearrange_uop(UOp) -> - {UOp, meta(Line), [{'in', meta(Line), [Left, Right]}]}; +build_op_with_meta(Left, {Op, Right}, Meta) -> + {Op1, OpMeta, Args} = build_op(Left, Op, Right), + {Op1, Meta ++ OpMeta, Args}. + +build_op(Left, {Op, Right}) -> + build_op(Left, Op, Right). + +build_op(AST, {_Kind, Location, '//'}, Right) -> + case AST of + {'..', Meta, [Left, Middle]} -> + {'..//', Meta, [Left, Middle, Right]}; + + _ -> + return_error(Location, "the range step operator (//) must immediately follow the range definition operator (..), for example: 1..9//2. If you wanted to define a default argument, use (\\\\) instead. Syntax error before: ", "'//'") + end; + +build_op({UOp, UMeta, [Left]}, {_Kind, {Line, Column, _} = Location, 'in'}, Right) when ?rearrange_uop(UOp) -> + %% TODO: Remove "not left in right" rearrangement on v2.0 + warn({Line, Column}, case UOp of + '!' -> "\"!expr1 in expr2\" is deprecated, use \"expr1 not in expr2\" instead"; + 'not' -> "\"not expr1 in expr2\" is deprecated, use \"expr1 not in expr2\" instead" + end), + Meta = meta_from_location(Location), + {UOp, UMeta, [{'in', Meta, [Left, Right]}]}; + +build_op(Left, {in_op, NotLocation, 'not in', InLocation}, Right) -> + NotMeta = newlines_op(NotLocation) ++ meta_from_location(NotLocation), + InMeta = meta_from_location(InLocation), + {'not', NotMeta, [{'in', InMeta, [Left, Right]}]}; + +build_op(Left, {_Kind, Location, Op}, Right) -> + {Op, newlines_op(Location) ++ meta_from_location(Location), [Left, Right]}. + +build_unary_op({_Kind, {Line, Column, _}, '//'}, Expr) -> + {Outer, Inner} = + case ?columns() of + true -> {[{column, Column+1}], [{column, Column}]}; + false -> {[], []} + end, + {'/', [{line, Line} | Outer], [{'/', [{line, Line} | Inner], nil}, Expr]}; -build_op({_Kind, Line, Op}, Left, Right) -> - {Op, meta(Line), [Left, Right]}. +build_unary_op({_Kind, Location, Op}, Expr) -> + {Op, meta_from_location(Location), [Expr]}. -build_unary_op({_Kind, Line, Op}, Expr) -> - {Op, meta(Line), [Expr]}. +build_nullary_op({_Kind, Location, Op}) -> + {Op, meta_from_location(Location), []}. -build_list(Marker, Args) -> - {Args, ?line(Marker)}. +build_list(Left, Args, Right) -> + {handle_literal(Args, Left, newlines_pair(Left, Right)), ?location(Left)}. -build_tuple(_Marker, [Left, Right]) -> - {Left, Right}; -build_tuple(Marker, Args) -> - {'{}', meta(Marker), Args}. +build_tuple(Left, [Arg1, Arg2], Right) -> + handle_literal({Arg1, Arg2}, Left, newlines_pair(Left, Right)); +build_tuple(Left, Args, Right) -> + {'{}', newlines_pair(Left, Right) ++ meta_from_token(Left), Args}. -build_bit(Marker, Args) -> - {'<<>>', meta(Marker), Args}. +build_bit(Left, Args, Right) -> + {'<<>>', newlines_pair(Left, Right) ++ meta_from_token(Left), Args}. -build_map(Marker, Args) -> - {'%{}', meta(Marker), Args}. +build_map(Left, Args, Right) -> + {'%{}', newlines_pair(Left, Right) ++ meta_from_token(Left), Args}. -build_map_update(Marker, {Pipe, Left, Right}, Extra) -> - {'%{}', meta(Marker), [build_op(Pipe, Left, Right ++ Extra)]}. +build_map_update(Left, {Pipe, Struct, Map}, Right, Extra) -> + Op = build_op(Struct, Pipe, append_non_empty(Map, Extra)), + {'%{}', newlines_pair(Left, Right) ++ meta_from_token(Left), [Op]}. %% Blocks -build_block([{Op,_,[_]}]=Exprs) when ?rearrange_uop(Op) -> {'__block__', [], Exprs}; -build_block([{unquote_splicing,_,Args}]=Exprs) when - length(Args) =< 2 -> {'__block__', [], Exprs}; -build_block([Expr]) -> Expr; -build_block(Exprs) -> {'__block__', [], Exprs}. +build_block(Exprs) -> build_block(Exprs, []). -%% Dots +build_block([{unquote_splicing, _, [_]}]=Exprs, Meta) -> + {'__block__', Meta, Exprs}; +build_block([{Op, ExprMeta, Args}], Meta) -> + ExprMetaWithExtra = + case ?token_metadata() of + true when Meta /= [] -> [{parens, Meta} | ExprMeta]; + _ -> ExprMeta + end, + {Op, ExprMetaWithExtra, Args}; +build_block([Expr], _Meta) -> + Expr; +build_block(Exprs, Meta) -> + {'__block__', Meta, Exprs}. + +%% Newlines + +newlines_pair(Left, Right) -> + case ?token_metadata() of + true -> + newlines(?location(Left), [{closing, meta_from_token(Right)}]); + false -> + [] + end. + +newlines_op(Location) -> + case ?token_metadata() of + true -> newlines(Location, []); + false -> [] + end. -build_dot_alias(Dot, {'__aliases__', _, Left}, {'aliases', _, Right}) -> - {'__aliases__', meta(Dot), Left ++ Right}; +next_is_eol(Token, {_, {_, _, Count}}) -> + {Line, Column, _} = ?location(Token), + setelement(2, Token, {Line, Column, Count}). + +newlines({_, _, Count}, Meta) when is_integer(Count) and (Count > 0) -> + [{newlines, Count} | Meta]; +newlines(_, Meta) -> + Meta. + +annotate_eoe(Token, Stack) -> + case ?token_metadata() of + true -> + case {Token, Stack} of + {{_, Location}, [{'->', StabMeta, [StabArgs, {Left, Meta, Right}]} | Rest]} when is_list(Meta) -> + [{'->', StabMeta, [StabArgs, {Left, [{end_of_expression, end_of_expression(Location)} | Meta], Right}]} | Rest]; + + {{_, Location}, [{Left, Meta, Right} | Rest]} when is_list(Meta), Left =/= '->' -> + [{Left, [{end_of_expression, end_of_expression(Location)} | Meta], Right} | Rest]; + + _ -> + Stack + end; + false -> + Stack + end. -build_dot_alias(Dot, Other, {'aliases', _, Right}) -> - {'__aliases__', meta(Dot), [Other|Right]}. +end_of_expression({_, _, Count} = Location) when is_integer(Count) -> + [{newlines, Count} | meta_from_location(Location)]; +end_of_expression(Location) -> + meta_from_location(Location). -build_dot(Dot, Left, Right) -> - {'.', meta(Dot), [Left, extract_identifier(Right)]}. +%% Dots + +build_alias({'alias', Location, Alias}) -> + Meta = meta_from_location(Location), + MetaWithExtra = + case ?token_metadata() of + true -> [{last, meta_from_location(Location)} | Meta]; + false -> Meta + end, + {'__aliases__', MetaWithExtra, [Alias]}. + +build_dot_alias(_Dot, {'__aliases__', Meta, Left}, {'alias', SegmentLocation, Right}) -> + MetaWithExtra = + case ?token_metadata() of + true -> lists:keystore(last, 1, Meta, {last, meta_from_location(SegmentLocation)}); + false -> Meta + end, + {'__aliases__', MetaWithExtra, Left ++ [Right]}; +build_dot_alias(_Dot, Atom, Right) when is_atom(Atom) -> + error_bad_atom(Right); +build_dot_alias(Dot, Expr, {'alias', SegmentLocation, Right}) -> + Meta = meta_from_token(Dot), + MetaWithExtra = + case ?token_metadata() of + true -> [{last, meta_from_location(SegmentLocation)} | Meta]; + false -> Meta + end, + {'__aliases__', MetaWithExtra, [Expr, Right]}. + +build_dot_container(Dot, Left, Right, Extra) -> + Meta = meta_from_token(Dot), + {{'.', Meta, [Left, '{}']}, Extra ++ Meta, Right}. + +build_dot(Dot, Left, {_, Location, _} = Right) -> + Meta = meta_from_token(Dot), + IdentifierMeta0 = meta_from_location(Location), + IdentifierMeta1 = + case Location of + {_Line, _Column, Delimiter} when is_integer(Delimiter) -> + delimiter(<>) ++ IdentifierMeta0; + _ -> + IdentifierMeta0 + end, + {'.', Meta, IdentifierMeta1, [Left, extract_identifier(Right)]}. extract_identifier({Kind, _, Identifier}) when Kind == identifier; Kind == bracket_identifier; Kind == paren_identifier; @@ -602,72 +911,185 @@ extract_identifier({Kind, _, Identifier}) when %% Identifiers -build_nested_parens(Dot, Args1, Args2) -> - Identifier = build_identifier(Dot, Args1), - Meta = element(2, Identifier), - {Identifier, Meta, Args2}. +build_nested_parens(Dot, Args1, {Args2Meta, Args2}, {BlockMeta, Block}) -> + Identifier = build_parens(Dot, Args1, {[], []}), + %% Take line and column meta from the call target node + LocationMeta = lists:filter(fun({Key, _}) -> Key == line orelse Key == column end, ?meta(Identifier)), + Meta = BlockMeta ++ Args2Meta ++ LocationMeta, + {Identifier, Meta, append_non_empty(Args2, Block)}. + +build_parens(Expr, {ArgsMeta, Args}, {BlockMeta, Block}) -> + {BuiltExpr, BuiltMeta, BuiltArgs} = build_call(Expr, append_non_empty(Args, Block)), + {BuiltExpr, BlockMeta ++ ArgsMeta ++ BuiltMeta, BuiltArgs}. -build_identifier({'.', Meta, _} = Dot, Args) -> - FArgs = case Args of - nil -> []; - _ -> Args - end, - {Dot, Meta, FArgs}; +build_no_parens_do_block(Expr, Args, {BlockMeta, Block}) -> + {BuiltExpr, BuiltMeta, BuiltArgs} = build_call(Expr, Args ++ Block), + {BuiltExpr, BlockMeta ++ BuiltMeta, BuiltArgs}. -build_identifier({Keyword, Line}, Args) when Keyword == fn -> - {fn, meta(Line), Args}; +build_no_parens(Expr, Args) -> + build_call(Expr, Args). -build_identifier({op_identifier, Line, Identifier}, [Arg]) -> - {Identifier, [{ambiguous_op,nil}|meta(Line)], [Arg]}; +build_identifier({'.', Meta, IdentifierMeta, DotArgs}) -> + {{'.', Meta, DotArgs}, [{no_parens, true} | IdentifierMeta], []}; -build_identifier({_, Line, Identifier}, Args) -> - {Identifier, meta(Line), Args}. +build_identifier({'.', Meta, _} = Dot) -> + {Dot, [{no_parens, true} | Meta], []}; + +build_identifier({_, Location, Identifier}) -> + {Identifier, meta_from_location(Location), nil}. + +build_call({'.', Meta, IdentifierMeta, DotArgs}, Args) -> + {{'.', Meta, DotArgs}, IdentifierMeta, Args}; + +build_call({'.', Meta, _} = Dot, Args) -> + {Dot, Meta, Args}; + +build_call({op_identifier, Location, Identifier}, [Arg]) -> + {Identifier, [{ambiguous_op, nil} | meta_from_location(Location)], [Arg]}; + +build_call({_, Location, Identifier}, Args) -> + {Identifier, meta_from_location(Location), Args}. %% Fn -build_fn(Op, Stab) -> - {fn, meta(Op), Stab}. +build_fn(Fn, Stab, End) -> + case check_stab(Stab, none) of + stab -> + Meta = newlines_op(?location(Fn)) ++ meta_from_token_with_closing(Fn, End), + {fn, Meta, collect_stab(Stab, [], [])}; + block -> + return_error(?location(Fn), "expected anonymous functions to be defined with -> inside: ", "'fn'") + end. %% Access -build_access(Expr, {List, Line}) -> - Meta = meta(Line), +build_access_arg(Left, Args, Right) -> + {Args, newlines_pair(Left, Right) ++ meta_from_token(Left)}. + +build_access(Expr, {List, Meta}) -> {{'.', Meta, ['Elixir.Access', get]}, Meta, [Expr, List]}. %% Interpolation aware -build_sigil({sigil, Line, Sigil, Parts, Modifiers}) -> - Meta = meta(Line), - {list_to_atom("sigil_" ++ [Sigil]), Meta, [ {'<<>>', Meta, string_parts(Parts)}, Modifiers ]}. - -build_bin_string({bin_string, _Line, [H]}) when is_binary(H) -> - H; -build_bin_string({bin_string, Line, Args}) -> - {'<<>>', meta(Line), string_parts(Args)}. - -build_list_string({list_string, _Line, [H]}) when is_binary(H) -> - elixir_utils:characters_to_list(H); -build_list_string({list_string, Line, Args}) -> - Meta = meta(Line), - {{'.', Meta, ['Elixir.String', to_char_list]}, Meta, [{'<<>>', Meta, string_parts(Args)}]}. +build_sigil({sigil, Location, Atom, Parts, Modifiers, Indentation, Delimiter}) -> + Meta = meta_from_location(Location), + MetaWithDelimiter = [{delimiter, Delimiter} | Meta], + MetaWithIndentation = meta_with_indentation(Meta, Indentation), + {Atom, + MetaWithDelimiter, + [{'<<>>', MetaWithIndentation, string_parts(Parts)}, Modifiers]}. + +meta_with_indentation(Meta, nil) -> + Meta; +meta_with_indentation(Meta, Indentation) -> + [{indentation, Indentation} | Meta]. + +meta_with_from_brackets({List, Meta}) -> + {List, [{from_brackets, true} | Meta]}. + +build_bin_heredoc({bin_heredoc, Location, Indentation, Args}) -> + ExtraMeta = + case ?token_metadata() of + true -> [{delimiter, <<$", $", $">>}, {indentation, Indentation}]; + false -> [] + end, + build_bin_string({bin_string, Location, Args}, ExtraMeta). -build_quoted_atom({_, _Line, [H]}, Safe) when is_binary(H) -> - Op = binary_to_atom_op(Safe), erlang:Op(H, utf8); -build_quoted_atom({_, Line, Args}, Safe) -> - Meta = meta(Line), - {{'.', Meta, [erlang, binary_to_atom_op(Safe)]}, Meta, [{'<<>>', Meta, string_parts(Args)}, utf8]}. +build_list_heredoc({list_heredoc, Location, Indentation, Args}) -> + ExtraMeta = + case ?token_metadata() of + true -> [{delimiter, <<$', $', $'>>}, {indentation, Indentation}]; + false -> [] + end, + build_list_string({list_string, Location, Args}, ExtraMeta). + +build_bin_string({bin_string, _Location, [H]} = Token, ExtraMeta) when is_binary(H) -> + handle_literal(H, Token, ExtraMeta); +build_bin_string({bin_string, Location, Args}, ExtraMeta) -> + Meta = + case ?token_metadata() of + true -> ExtraMeta ++ meta_from_location(Location); + false -> meta_from_location(Location) + end, + {'<<>>', Meta, string_parts(Args)}. + +build_list_string({list_string, _Location, [H]} = Token, ExtraMeta) when is_binary(H) -> + try + List = elixir_utils:characters_to_list(H), + handle_literal(List, Token, ExtraMeta) + catch + error:#{'__struct__' := 'Elixir.UnicodeConversionError', message := Message} -> + return_error(?location(Token), elixir_utils:characters_to_list(Message), "'") + end; +build_list_string({list_string, Location, Args}, ExtraMeta) -> + Meta = meta_from_location(Location), + MetaWithExtra = + case ?token_metadata() of + true -> ExtraMeta ++ Meta; + false -> Meta + end, + {{'.', Meta, ['Elixir.List', to_charlist]}, MetaWithExtra, [charlist_parts(Args)]}. + +build_quoted_atom({_, _Location, [H]} = Token, Safe, ExtraMeta) when is_binary(H) -> + Op = binary_to_atom_op(Safe), + handle_literal(erlang:Op(H, utf8), Token, ExtraMeta); +build_quoted_atom({_, Location, Args}, Safe, ExtraMeta) -> + Meta = meta_from_location(Location), + MetaWithExtra = + case ?token_metadata() of + true -> ExtraMeta ++ Meta; + false -> Meta + end, + {{'.', Meta, [erlang, binary_to_atom_op(Safe)]}, MetaWithExtra, [{'<<>>', Meta, string_parts(Args)}, utf8]}. binary_to_atom_op(true) -> binary_to_existing_atom; binary_to_atom_op(false) -> binary_to_atom. +atom_colon_meta({atom, _Location, Atom}) when Atom =:= true orelse Atom =:= false orelse Atom =:= nil -> + [{format, atom}]; +atom_colon_meta(_) -> + []. + +atom_delimiter_meta({_Kind, {_Line, _Column, Delimiter}, _Args}) -> + case ?token_metadata() of + true -> [{delimiter, <>}]; + false -> [] + end. + +kw_identifier_meta({_Kind, {_Line, _Column, Delimiter}, _Args}) -> + Meta = [{format, keyword}], + case ?token_metadata() of + true when is_integer(Delimiter) -> [{delimiter, <>} | Meta]; + _ -> Meta + end. + +charlist_parts(Parts) -> + [charlist_part(Part) || Part <- Parts]. +charlist_part(Binary) when is_binary(Binary) -> + Binary; +charlist_part({Begin, End, Tokens}) -> + Form = string_tokens_parse(Tokens), + Meta = meta_from_location(Begin), + MetaWithExtra = + case ?token_metadata() of + true -> [{closing, meta_from_location(End)} | Meta]; + false -> Meta + end, + {{'.', Meta, ['Elixir.Kernel', to_string]}, [{from_interpolation, true} | MetaWithExtra], [Form]}. + string_parts(Parts) -> [string_part(Part) || Part <- Parts]. string_part(Binary) when is_binary(Binary) -> Binary; -string_part({Line, Tokens}) -> +string_part({Begin, End, Tokens}) -> Form = string_tokens_parse(Tokens), - Meta = meta(Line), - {'::', Meta, [{{'.', Meta, ['Elixir.Kernel', to_string]}, Meta, [Form]}, {binary, Meta, nil}]}. + Meta = meta_from_location(Begin), + MetaWithExtra = + case ?token_metadata() of + true -> [{closing, meta_from_location(End)} | Meta]; + false -> Meta + end, + {'::', Meta, [{{'.', Meta, ['Elixir.Kernel', to_string]}, [{from_interpolation, true} | MetaWithExtra], [Form]}, {binary, Meta, nil}]}. string_tokens_parse(Tokens) -> case parse(Tokens) of @@ -675,36 +1097,56 @@ string_tokens_parse(Tokens) -> {error, _} = Error -> throw(Error) end. +delimiter(Delimiter) -> + case ?token_metadata() of + true -> [{delimiter, Delimiter}]; + false -> [] + end. + %% Keywords -build_stab([{'->', Meta, [Left, Right]}|T]) -> - build_stab(Meta, T, Left, [Right], []); +check_stab([{'->', _, [_, _]}], _) -> stab; +check_stab([], none) -> block; +check_stab([_], none) -> block; +check_stab([_], Meta) -> error_invalid_stab(Meta); +check_stab([{'->', Meta, [_, _]} | T], _) -> check_stab(T, Meta); +check_stab([_ | T], MaybeMeta) -> check_stab(T, MaybeMeta). + +build_stab(Stab, BlockMeta) -> + case check_stab(Stab, none) of + block -> build_block(reverse(Stab), BlockMeta); + stab -> collect_stab(Stab, [], []) + end. -build_stab(Else) -> - build_block(Else). +build_paren_stab(_Before, [{Op, _, [_]}]=Exprs, _After) when ?rearrange_uop(Op) -> + {'__block__', [], Exprs}; +build_paren_stab(Before, Stab, After) -> + case check_stab(Stab, none) of + block -> build_block(reverse(Stab), meta_from_token_with_closing(Before, After)); + stab -> handle_literal(collect_stab(Stab, [], []), Before, newlines_pair(Before, After)) + end. -build_stab(Old, [{'->', New, [Left, Right]}|T], Marker, Temp, Acc) -> - H = {'->', Old, [Marker, build_block(reverse(Temp))]}, - build_stab(New, T, Left, [Right], [H|Acc]); +collect_stab([{'->', Meta, [Left, Right]} | T], Exprs, Stabs) -> + Stab = {'->', Meta, [Left, build_block([Right | Exprs])]}, + collect_stab(T, [], [Stab | Stabs]); -build_stab(Meta, [H|T], Marker, Temp, Acc) -> - build_stab(Meta, T, Marker, [H|Temp], Acc); +collect_stab([H | T], Exprs, Stabs) -> + collect_stab(T, [H | Exprs], Stabs); -build_stab(Meta, [], Marker, Temp, Acc) -> - H = {'->', Meta, [Marker, build_block(reverse(Temp))]}, - reverse([H|Acc]). +collect_stab([], [], Stabs) -> + Stabs. %% Every time the parser sees a (unquote_splicing()) %% it assumes that a block is being spliced, wrapping %% the splicing in a __block__. But in the stab clause, -%% we can have (unquote_splicing(1,2,3)) -> :ok, in such +%% we can have (unquote_splicing(1, 2, 3)) -> :ok, in such %% case, we don't actually want the block, since it is %% an arg style call. unwrap_splice unwraps the splice %% from such blocks. -unwrap_splice([{'__block__', [], [{unquote_splicing, _, _}] = Splice}]) -> +unwrap_splice([{'__block__', _, [{unquote_splicing, _, _}] = Splice}]) -> Splice; - -unwrap_splice(Other) -> Other. +unwrap_splice(Other) -> + Other. unwrap_when(Args) -> case elixir_utils:split_last(Args) of @@ -714,25 +1156,196 @@ unwrap_when(Args) -> Args end. -to_block([One]) -> One; -to_block(Other) -> {'__block__', [], reverse(Other)}. - -%% Errors - -throw(Line, Error, Token) -> - throw({error, {Line, ?MODULE, [Error, Token]}}). - -throw_no_parens_strict(Token) -> - throw(?line(Token), "unexpected parenthesis. If you are making a " - "function call, do not insert spaces in between the function name and the " +parens_meta({Open, Close}) -> + case ?token_metadata() of + true -> + ParensEntry = [{closing, meta_from_token(Close)} | meta_from_token(Open)], + [{parens, ParensEntry}]; + false -> + [] + end; +parens_meta({Open, _Args, Close}) -> + parens_meta({Open, Close}). + +with_assoc_meta({Target, Meta, Args}, AssocToken) -> + case ?token_metadata() of + true -> + {Target, [{assoc, meta_from_token(AssocToken)} | Meta], Args}; + false -> + {Target, Meta, Args} + end; + +with_assoc_meta(Left, _AssocToken) -> Left. + +%% Warnings and errors + +return_error({Line, Column, _}, ErrorMessage, ErrorToken) -> + return_error([{line, Line}, {column, Column}], [ErrorMessage, ErrorToken]). + +%% We should prefer to use return_error as it includes +%% Line and Column but that's not always possible. +return_error_with_meta(Meta, ErrorMessage, ErrorToken) -> + return_error(Meta, [ErrorMessage, ErrorToken]). + +error_invalid_stab(MetaStab) -> + return_error_with_meta(MetaStab, + "unexpected operator ->. If you want to define multiple clauses, the first expression must use ->. " + "Syntax error before: ", "'->'"). + +error_bad_atom(Token) -> + return_error(?location(Token), "atom cannot be followed by an alias. " + "If the '.' was meant to be part of the atom's name, " + "the atom name must be quoted. Syntax error before: ", "'.'"). + +bad_keyword(Token, Context, StartString) -> + return_error(?location(Token), + "unexpected keyword list inside " ++ atom_to_list(Context) ++ ". " + "Did you mean to write a map (using %{...}) or a list (using [...]) instead? " + "Syntax error after: ", StartString). + +maybe_bad_keyword_call_follow_up(_Token, KW, {'__cursor__', _, []} = Expr) -> + reverse([Expr | KW]); +maybe_bad_keyword_call_follow_up(Token, _KW, _Expr) -> + return_error(?location(Token), + "unexpected expression after keyword list. Keyword lists must always come as the last argument. Therefore, this is not allowed:\n\n" + " function_call(1, some: :option, 2)\n\n" + "Instead, wrap the keyword in brackets:\n\n" + " function_call(1, [some: :option], 2)\n\n" + "Syntax error after: ", "','"). + +maybe_bad_keyword_data_follow_up(_Token, KW, {'__cursor__', _, []} = Expr) -> + reverse([Expr | KW]); +maybe_bad_keyword_data_follow_up(Token, _KW, _Expr) -> + return_error(?location(Token), + "unexpected expression after keyword list. Keyword lists must always come last in lists and maps. Therefore, this is not allowed:\n\n" + " [some: :value, :another]\n" + " %{some: :value, another => value}\n\n" + "Instead, reorder it to be the last entry:\n\n" + " [:another, some: :value]\n" + " %{another => value, some: :value}\n\n" + "Syntax error after: ", "','"). + +error_no_parens_strict(Token) -> + return_error(?location(Token), "unexpected parentheses. If you are making a " + "function call, do not insert spaces between the function name and the " "opening parentheses. Syntax error before: ", "'('"). -throw_no_parens_many_strict(Token) -> - Line = - case lists:keyfind(line, 1, element(2, Token)) of - {line, L} -> L; - false -> 0 - end, - - throw(Line, "unexpected comma. Parentheses are required to solve ambiguity " - "in nested calls. Syntax error before: ", "','"). +error_no_parens_many_strict(Node) -> + return_error_with_meta(?meta(Node), + "unexpected comma. Parentheses are required to solve ambiguity in nested calls.\n\n" + "This error happens when you have nested function calls without parentheses. " + "For example:\n\n" + " parent_call a, nested_call b, c, d\n\n" + "In the example above, we don't know if the parameters \"c\" and \"d\" apply " + "to the function \"parent_call\" or \"nested_call\". You can solve this by " + "explicitly adding parentheses:\n\n" + " parent_call a, nested_call(b, c, d)\n\n" + "Or by adding commas (in case a nested call is not intended):\n\n" + " parent_call a, nested_call, b, c, d\n\n" + "Elixir cannot compile otherwise. Syntax error before: ", "','"). + +error_no_parens_container_strict(Node) -> + return_error_with_meta(?meta(Node), + "unexpected comma. Parentheses are required to solve ambiguity inside containers.\n\n" + "This error may happen when you forget a comma in a list or other container:\n\n" + " [a, b c, d]\n\n" + "Or when you have ambiguous calls:\n\n" + " [function a, b, c]\n\n" + "In the example above, we don't know if the values \"b\" and \"c\" " + "belongs to the list or the function \"function\". You can solve this by explicitly " + "adding parentheses:\n\n" + " [one, function(a, b, c)]\n\n" + "Elixir cannot compile otherwise. Syntax error before: ", "','"). + +error_too_many_access_syntax(Comma) -> + return_error(?location(Comma), "too many arguments when accessing a value. " + "The value[key] notation in Elixir expects either a single argument or a keyword list. " + "The following examples are allowed:\n\n" + " value[one]\n" + " value[one: 1, two: 2]\n" + " value[[one, two, three]]\n\n" + "These are invalid:\n\n" + " value[1, 2, 3]\n" + " value[one, two, three]\n\n" + "Syntax error after: ", "','"). + +error_invalid_kw_identifier({_, Location, do}) -> + return_error(Location, elixir_tokenizer:invalid_do_error("unexpected keyword: "), "do:"); +error_invalid_kw_identifier({_, Location, KW}) -> + return_error(Location, "syntax error before: ", "'" ++ atom_to_list(KW) ++ ":'"). + +%% TODO: Make this an error on v2.0 +warn_trailing_comma({',', {Line, Column, _}}) -> + warn({Line, Column}, "trailing commas are not allowed inside function/macro call arguments"). + +%% TODO: Make this an error on v2.0 +warn_pipe({arrow_op, {Line, Column, _}, Op}, {_, [_ | _], [_ | _]}) -> + warn( + {Line, Column}, + io_lib:format( + "parentheses are required when piping into a function call. For example:\n\n" + " foo 1 ~ts bar 2 ~ts baz 3\n\n" + "is ambiguous and should be written as\n\n" + " foo(1) ~ts bar(2) ~ts baz(3)\n\n" + "Ambiguous pipe found at:", + [Op, Op, Op, Op] + ) + ); +warn_pipe(_Token, _) -> + ok. + +%% TODO: Make this an error on v2.0 +warn_no_parens_after_do_op({{_Type, Location, Op}, _}) -> + {Line, _, _} = Location, + + warn( + Line, + "missing parentheses on expression following operator \"" ++ atom_to_list(Op) ++ "\", " + "you must add parentheses to avoid ambiguities" + ). + +%% TODO: Make this an error on v2.0 +warn_nested_no_parens_keyword(Key, Value) when is_atom(Key) -> + {line, Line} = lists:keyfind(line, 1, ?meta(Value)), + warn( + Line, + "missing parentheses for expression following \"" ++ atom_to_list(Key) ++ ":\" keyword. " + "Parentheses are required to solve ambiguity inside keywords.\n\n" + "This error happens when you have function calls without parentheses inside keywords. " + "For example:\n\n" + " function(arg, one: nested_call a, b, c)\n" + " function(arg, one: if expr, do: :this, else: :that)\n\n" + "In the examples above, we don't know if the arguments \"b\" and \"c\" apply " + "to the function \"function\" or \"nested_call\". Or if the keywords \"do\" and " + "\"else\" apply to the function \"function\" or \"if\". You can solve this by " + "explicitly adding parentheses:\n\n" + " function(arg, one: if(expr, do: :this, else: :that))\n" + " function(arg, one: nested_call(a, b, c))\n\n" + "Ambiguity found at:" + ); + +% Key might not be an atom when using literal_encoder, we just skip the warning +warn_nested_no_parens_keyword(_Key, _Value) -> + ok. + +warn_empty_paren({{_, {Line, Column, _}}, _}) -> + warn( + {Line, Column}, + "invalid expression (). " + "If you want to invoke or define a function, make sure there are " + "no spaces between the function name and its arguments. If you wanted " + "to pass an empty block or code, pass a value instead, such as a nil or an atom" + ). + +warn_empty_stab_clause({stab_op, {Line, Column, _}, '->'}) -> + warn( + {Line, Column}, + "an expression is always required on the right side of ->. " + "Please provide a value after ->" + ). + +warn(LineColumn, Message) -> + case get(elixir_parser_warning_file) of + nil -> ok; + File -> elixir_errors:erl_warn(LineColumn, File, Message) + end. diff --git a/lib/elixir/src/elixir_quote.erl b/lib/elixir/src/elixir_quote.erl index 9e8cbc7bd9f..b05eeab7a45 100644 --- a/lib/elixir/src/elixir_quote.erl +++ b/lib/elixir/src/elixir_quote.erl @@ -1,310 +1,639 @@ +%% SPDX-License-Identifier: Apache-2.0 +%% SPDX-FileCopyrightText: 2021 The Elixir Team +%% SPDX-FileCopyrightText: 2012 Plataformatec + -module(elixir_quote). --export([escape/2, linify/2, linify/3, linify_with_context_counter/3, quote/4]). --export([dot/6, tail_list/3, list/2]). %% Quote callbacks + +-feature(maybe_expr, enable). + +-export([escape/3, linify/3, linify_with_context_counter/3, build/7, quote/2, has_unquotes/1, fun_to_quoted/1]). +-export([dot/5, tail_list/3, list/2, validate_runtime/2, shallow_validate_ast/1]). %% Quote callbacks -include("elixir.hrl"). --define(defs(Kind), Kind == def; Kind == defp; Kind == defmacro; Kind == defmacrop). --define(lexical(Kind), Kind == import; Kind == alias; Kind == '__aliases__'). --compile({inline, [keyfind/2, keystore/3, keydelete/2, keyreplace/3, keynew/3]}). +-define(defs(Kind), Kind == def; Kind == defp; Kind == defmacro; Kind == defmacrop; Kind == '@'). +-define(lexical(Kind), Kind == import; Kind == alias; Kind == require). +-compile({inline, [keyfind/2, keystore/3, keydelete/2, keynew/3, do_tuple_linify/6]}). + +-record(elixir_quote, { + line=false, + file=nil, + context=nil, + op=escape, % escape | escape_and_prune | {struct, Module} | quote + aliases_hygiene=nil, + imports_hygiene=nil, + unquote=true, + generated=false, + shallow_validate=false +}). + +%% fun_to_quoted + +fun_to_quoted(Function) -> + Meta = [], + {module, Module} = erlang:fun_info(Function, module), + {name, Name} = erlang:fun_info(Function, name), + {arity, Arity} = erlang:fun_info(Function, arity), + {'&', Meta, [{'/', Meta, [{{'.', Meta, [Module, Name]}, [{no_parens, true} | Meta], []}, Arity]}]}. + +%% has_unquotes + +has_unquotes(Ast) -> has_unquotes(Ast, 0). + +has_unquotes({quote, _, [Child]}, QuoteLevel) -> + has_unquotes(Child, QuoteLevel + 1); +has_unquotes({quote, _, [QuoteOpts, Child]}, QuoteLevel) -> + case disables_unquote(QuoteOpts) of + true -> false; + _ -> has_unquotes(Child, QuoteLevel + 1) + end; +has_unquotes({Unquote, _, [Child]}, QuoteLevel) + when Unquote == unquote; Unquote == unquote_splicing -> + case QuoteLevel of + 0 -> true; + _ -> has_unquotes(Child, QuoteLevel - 1) +end; +has_unquotes({{'.', _, [_, unquote]}, _, [_]}, _) -> true; +has_unquotes({Var, _, Ctx}, _) when is_atom(Var), is_atom(Ctx) -> false; +has_unquotes({Name, _, Args}, QuoteLevel) when is_list(Args) -> + has_unquotes(Name) orelse lists:any(fun(Child) -> has_unquotes(Child, QuoteLevel) end, Args); +has_unquotes({Left, Right}, QuoteLevel) -> + has_unquotes(Left, QuoteLevel) orelse has_unquotes(Right, QuoteLevel); +has_unquotes(List, QuoteLevel) when is_list(List) -> + lists:any(fun(Child) -> has_unquotes(Child, QuoteLevel) end, List); +has_unquotes(_Other, _) -> false. + +disables_unquote([{unquote, false} | _]) -> true; +disables_unquote([{bind_quoted, _} | _]) -> true; +disables_unquote([_H | T]) -> disables_unquote(T); +disables_unquote(_) -> false. %% Apply the line from site call on quoted contents. %% Receives a Key to look for the default line as argument. -linify(Line, Exprs) when is_integer(Line) -> - do_linify(Line, line, nil, Exprs). - +linify(0, _Key, Exprs) -> + Exprs; linify(Line, Key, Exprs) when is_integer(Line) -> - do_linify(Line, Key, nil, Exprs). + Fun = + case Key of + line -> + fun(Meta) -> keynew(line, Meta, Line) end; + keep -> + fun(Meta) -> + case lists:keytake(keep, 1, Meta) of + {value, {keep, {_, Int}}, MetaNoFile} -> + [{line, Int} | keydelete(line, MetaNoFile)]; + _ -> + keynew(line, Meta, Line) + end + end + end, -%% Same as linify but also considers the context counter. -linify_with_context_counter(Line, Var, Exprs) when is_integer(Line) -> - do_linify(Line, line, Var, Exprs). + do_linify(Fun, Exprs, nil, false). -do_linify(Line, Key, {Receiver, Counter} = Var, {Left, Meta, Receiver}) - when is_atom(Left), is_list(Meta), Left /= '_' -> - do_tuple_linify(Line, Key, Var, keynew(counter, Meta, Counter), Left, Receiver); +%% Same as linify but also considers the context counter and generated. +linify_with_context_counter(ContextMeta, Var, Exprs) when is_list(ContextMeta) -> + Line = ?line(ContextMeta), -do_linify(Line, Key, {_, Counter} = Var, {Lexical, [_|_] = Meta, [_|_] = Args}) when ?lexical(Lexical) -> - do_tuple_linify(Line, Key, Var, keynew(counter, Meta, Counter), Lexical, Args); + Generated = keyfind(generated, ContextMeta) == {generated, true}, -do_linify(Line, Key, Var, {Left, Meta, Right}) when is_list(Meta) -> - do_tuple_linify(Line, Key, Var, Meta, Left, Right); + Fun = if + Line =:= 0 -> fun(Meta) -> Meta end; + true -> fun(Meta) -> keynew(line, Meta, Line) end + end, -do_linify(Line, Key, Var, {Left, Right}) -> - {do_linify(Line, Key, Var, Left), do_linify(Line, Key, Var, Right)}; + do_linify(Fun, Exprs, Var, Generated). -do_linify(Line, Key, Var, List) when is_list(List) -> - [do_linify(Line, Key, Var, X) || X <- List]; +do_linify(Fun, {quote, Meta, [_ | _] = Args}, {Receiver, Counter} = Var, Gen) + when is_list(Meta) -> + NewMeta = + case keyfind(context, Meta) == {context, Receiver} of + true -> keynew(counter, Meta, Counter); + false -> Meta + end, + do_tuple_linify(Fun, NewMeta, quote, Args, Var, Gen); -do_linify(_, _, _, Else) -> Else. +do_linify(Fun, {Left, Meta, Receiver}, {Receiver, Counter} = Var, Gen) + when is_atom(Left), is_list(Meta), Left /= '_' -> + do_tuple_linify(Fun, keynew(counter, Meta, Counter), Left, Receiver, Var, Gen); -do_tuple_linify(Line, Key, Var, Meta, Left, Right) -> - {do_linify(Line, Key, Var, Left), - do_linify_meta(Line, Key, Meta), - do_linify(Line, Key, Var, Right)}. +do_linify(Fun, {Lexical, Meta, [_ | _] = Args}, {_, Counter} = Var, Gen) + when ?lexical(Lexical); Lexical == '__aliases__' -> + do_tuple_linify(Fun, keynew(counter, Meta, Counter), Lexical, Args, Var, Gen); -do_linify_meta(0, line, Meta) -> - Meta; -do_linify_meta(Line, line, Meta) -> - case keyfind(line, Meta) of - {line, Int} when is_integer(Int), Int /= 0 -> - Meta; - _ -> - keystore(line, Meta, Line) - end; -do_linify_meta(Line, Key, Meta) -> - case keyfind(Key, Meta) of - {Key, Int} when is_integer(Int), Int /= 0 -> - keyreplace(Key, Meta, {line, Int}); - _ -> - do_linify_meta(Line, line, Meta) - end. +do_linify(Fun, {Left, Meta, Right}, Var, Gen) when is_list(Meta) -> + do_tuple_linify(Fun, Meta, Left, Right, Var, Gen); -%% Some expressions cannot be unquoted at compilation time. -%% This function is responsible for doing runtime unquoting. -dot(Meta, Left, Right, Args, Context, File) -> - annotate(dot(Meta, Left, Right, Args), Context, File). +do_linify(Fun, {Left, Right}, Var, Gen) -> + {do_linify(Fun, Left, Var, Gen), do_linify(Fun, Right, Var, Gen)}; -dot(Meta, Left, {'__aliases__', _, Args}, nil) -> - {'__aliases__', Meta, [Left|Args]}; +do_linify(Fun, List, Var, Gen) when is_list(List) -> + [do_linify(Fun, X, Var, Gen) || X <- List]; -dot(Meta, Left, Right, nil) when is_atom(Right) -> - case atom_to_list(Right) of - "Elixir." ++ _ -> - {'__aliases__', Meta, [Left, Right]}; - _ -> - {{'.', Meta, [Left, Right]}, Meta, []} - end; +do_linify(_, Else, _, _Gen) -> Else. -dot(Meta, Left, {Right, _, Context}, nil) when is_atom(Right), is_atom(Context) -> - {{'.', Meta, [Left, Right]}, Meta, []}; +do_tuple_linify(Fun, Meta, Left, Right, Var, Gen) -> + {NewMeta, NewGen} = + case keyfind(stop_generated, Meta) of + {stop_generated, true} -> {keydelete(stop_generated, Meta), false}; + _ when Gen -> {elixir_utils:generated(Meta), Gen}; + _ -> {Meta, Gen} + end, -dot(Meta, Left, {Right, _, Args}, nil) when is_atom(Right) -> - {{'.', Meta, [Left, Right]}, Meta, Args}; + {do_linify(Fun, Left, Var, NewGen), Fun(NewMeta), do_linify(Fun, Right, Var, NewGen)}. -dot(_Meta, _Left, Right, nil) -> - argument_error(<<"expected unquote after dot to return an atom, an alias or a quoted call, got: ", - ('Elixir.Macro':to_string(Right))/binary>>); +%% Escaping -dot(Meta, Left, Right, Args) when is_atom(Right) -> - {{'.', Meta, [Left, Right]}, Meta, Args}; +%% Escapes the given expression. It is similar to quote, but +%% lines are kept and hygiene mechanisms are disabled. +escape(Expr, Op, Unquote) -> + try + do_quote(Expr, #elixir_quote{ + line=true, + file=nil, + op=Op, + unquote=Unquote + }) + catch + Kind:Reason:Stacktrace -> + Pruned = lists:dropwhile(fun + ({?MODULE, _, _, _}) -> true; + (_) -> false + end, Stacktrace), + erlang:raise(Kind, Reason, Pruned) + end. -dot(Meta, Left, {Right, _, Context}, Args) when is_atom(Right), is_atom(Context) -> - {{'.', Meta, [Left, Right]}, Meta, Args}; +do_escape({Left, Meta, Right}, #elixir_quote{op=escape_and_prune} = Q) when is_list(Meta) -> + TM = [{K, V} || {K, V} <- Meta, (K == no_parens) orelse (K == line) orelse (K == delimiter)], + TL = do_quote(Left, Q), + TR = do_quote(Right, Q), + {'{}', [], [TL, TM, TR]}; + +do_escape({Left, Right}, Q) -> + {do_quote(Left, Q), do_quote(Right, Q)}; + +do_escape(Tuple, Q) when is_tuple(Tuple) -> + TT = do_quote(tuple_to_list(Tuple), Q), + {'{}', [], TT}; + +do_escape(BitString, _) when is_bitstring(BitString) -> + case bit_size(BitString) rem 8 of + 0 -> + BitString; + Size -> + <> = BitString, + {'<<>>', [], [{'::', [], [Bits, {size, [], [Size]}]}, {'::', [], [Bytes, {binary, [], nil}]}]} + end; -dot(_Meta, _Left, Right, _Args) -> - argument_error(<<"expected unquote after dot with args to return an atom or a quoted call, got: ", - ('Elixir.Macro':to_string(Right))/binary>>). +do_escape(Map, Q) when is_map(Map) -> + maybe + #{'__struct__' := Module} ?= Map, + true ?= is_atom(Module), + % We never escape ourselves (it can only happen during Elixir bootstrapping) + true ?= (Q#elixir_quote.op /= {struct, Module}), + {module, Module} ?= code:ensure_loaded(Module), + true ?= erlang:function_exported(Module, '__escape__', 1), + case Q#elixir_quote.op of + {struct, _Module} -> + argument_error(<<('Elixir.Kernel':inspect(Module))/binary, + " defines custom escaping rules which are not supported in struct defaults", + (bad_escape_hint())/binary>>); + + _ -> + Expr = Module:'__escape__'(Map), + case shallow_valid_ast(Expr) of + true -> Expr; + false -> argument_error( + <<('Elixir.Kernel':inspect(Module))/binary, ".__escape__/1 returned invalid AST: ", ('Elixir.Kernel':inspect(Expr))/binary>> + ) + end + end + else + _ -> + TT = [ + {do_quote(K, Q), do_quote(V, Q)} + || {K, V} <- lists:sort(maps:to_list(Map)) + ], + {'%{}', [], TT} + end; -list(Left, Right) when is_list(Right) -> - validate_list(Left), - Left ++ Right. +do_escape([], _) -> + []; + +do_escape([H | T], #elixir_quote{unquote=false} = Q) -> + do_quote_simple_list(T, do_escape(H, Q), Q); + +do_escape([H | T], Q) -> + %% The improper case is inefficient, but improper lists are rare. + try lists:reverse(T, [H]) of + L -> do_quote_tail(L, Q) + catch + _:_ -> + {L, R} = reverse_improper(T, [H]), + TL = do_quote_splice(L, Q, [], []), + TR = do_quote(R, Q), + update_last(TL, fun(X) -> {'|', [], [X, TR]} end) + end; -tail_list(Left, Right, Tail) when is_list(Right), is_list(Tail) -> - validate_list(Left), - Tail ++ Left ++ Right; +do_escape(Other, _) when is_number(Other); is_atom(Other); is_pid(Other) -> + Other; -tail_list(Left, Right, Tail) when is_list(Left) -> - validate_list(Left), - [H|T] = lists:reverse(Tail ++ Left), - lists:reverse([{'|', [], [H, Right]}|T]). +do_escape(Fun, _) when is_function(Fun) -> + case (erlang:fun_info(Fun, env) == {env, []}) andalso + (erlang:fun_info(Fun, type) == {type, external}) of + true -> fun_to_quoted(Fun); + false -> bad_escape(Fun) + end; -validate_list(List) when is_list(List) -> - ok; -validate_list(List) when not is_list(List) -> - argument_error(<<"expected a list with quoted expressions in unquote_splicing/1, got: ", - ('Elixir.Kernel':inspect(List))/binary>>). +do_escape(Other, _) -> + bad_escape(Other). + +bad_escape(Arg) -> + argument_error(<<"cannot escape ", ('Elixir.Kernel':inspect(Arg, []))/binary, + (bad_escape_hint())/binary>>). + +bad_escape_hint() -> + <<". The supported values are: lists, tuples, maps, atoms, numbers, bitstrings, ", + "PIDs and remote functions in the format &Mod.fun/arity">>. + +%% Quote entry points + +build(Meta, Line, File, Context, Unquote, Generated, E) -> + Acc0 = [], + {VLine, Acc1} = validate_compile(Meta, line, Line, Acc0), + {VFile, Acc2} = validate_compile(Meta, file, File, Acc1), + {VContext, Acc3} = validate_compile(Meta, context, Context, Acc2), + validate_runtime(unquote, Unquote), + validate_runtime(generated, Generated), + + Q = #elixir_quote{ + op=quote, + aliases_hygiene=E, + imports_hygiene=E, + line=VLine, + file=VFile, + unquote=Unquote, + context=VContext, + generated=Generated, + shallow_validate=true + }, + + {Q, VContext, Acc3}. + +validate_compile(_Meta, line, Value, Acc) when is_boolean(Value) -> + {Value, Acc}; +validate_compile(_Meta, file, nil, Acc) -> + {nil, Acc}; +validate_compile(Meta, Key, Value, Acc) -> + case is_valid(Key, Value) of + true -> + {Value, Acc}; + false -> + Var = {Key, Meta, ?MODULE}, + Call = {{'.', Meta, [?MODULE, validate_runtime]}, Meta, [Key, Value]}, + {Var, [{'=', Meta, [Var, Call]} | Acc]} + end. -argument_error(Message) -> - error('Elixir.ArgumentError':exception([{message,Message}])). +validate_runtime(Key, Value) -> + case is_valid(Key, Value) of + true -> + Value; -%% Annotates the AST with context and other info + false -> + erlang:error( + 'Elixir.ArgumentError':exception( + <<"invalid runtime value for option :", (erlang:atom_to_binary(Key))/binary, + " in quote, got: ", ('Elixir.Kernel':inspect(Value))/binary>> + ) + ) + end. -annotate({Def, Meta, [{H, M, A}|T]}, Context, File) when ?defs(Def) -> - %% Store the context information in the first element of the - %% definition tuple so we can access it later on. - MM = keystore(context, keystore(file, M, File), Context), - {Def, Meta, [{H, MM, A}|T]}; -annotate({{'.', _, [_, Def]} = Target, Meta, [{H, M, A}|T]}, Context, File) when ?defs(Def) -> - MM = keystore(context, keystore(file, M, File), Context), - {Target, Meta, [{H, MM, A}|T]}; +is_valid(line, Line) -> is_integer(Line); +is_valid(file, File) -> is_binary(File); +is_valid(context, Context) -> is_atom(Context) andalso (Context /= nil); +is_valid(generated, Generated) -> is_boolean(Generated); +is_valid(unquote, Unquote) -> is_boolean(Unquote). + +shallow_validate_ast(Expr) -> + case shallow_valid_ast(Expr) of + true -> Expr; + false -> argument_error( + <<"tried to unquote invalid AST: ", ('Elixir.Kernel':inspect(Expr))/binary, + "\nDid you forget to escape term using Macro.escape/1?">>) + end. -annotate({Lexical, Meta, [_|_] = Args}, Context, _File) when Lexical == import; Lexical == alias -> - NewMeta = keystore(context, keydelete(counter, Meta), Context), - {Lexical, NewMeta, Args}; -annotate(Tree, _Context, _File) -> Tree. +shallow_valid_ast(Expr) when is_list(Expr) -> valid_ast_list(Expr); +shallow_valid_ast(Expr) -> valid_ast_elem(Expr). -%% Escapes the given expression. It is similar to quote, but -%% lines are kept and hygiene mechanisms are disabled. -escape(Expr, Unquote) -> - {Res, Q} = quote(Expr, nil, #elixir_quote{ - line=true, - keep=false, - vars_hygiene=false, - aliases_hygiene=false, - imports_hygiene=false, - unquote=Unquote, - escape=true - }, nil), - {Res, Q#elixir_quote.unquoted}. +valid_ast_list([]) -> true; +valid_ast_list([Head | Tail]) -> valid_ast_elem(Head) andalso valid_ast_list(Tail); +valid_ast_list(_Improper) -> false. -%% Quotes an expression and return its quoted Elixir AST. +valid_ast_elem(Expr) when is_list(Expr); is_atom(Expr); is_binary(Expr); is_number(Expr); is_pid(Expr); is_function(Expr) -> true; +valid_ast_elem({Left, Right}) -> valid_ast_elem(Left) andalso valid_ast_elem(Right); +valid_ast_elem({Atom, Meta, Args}) when is_atom(Atom), is_list(Meta), is_atom(Args) orelse is_list(Args) -> true; +valid_ast_elem({Call, Meta, Args}) when is_list(Meta), is_list(Args) -> shallow_valid_ast(Call); +valid_ast_elem(_Term) -> false. -quote(Expr, nil, Q, E) -> - do_quote(Expr, Q, E); +quote({unquote_splicing, _, [_]}, #elixir_quote{unquote=true}) -> + argument_error(<<"unquote_splicing only works inside arguments and block contexts, " + "wrap it in parens if you want it to work with one-liners">>); +quote(Expr, Q) -> + do_quote(Expr, Q). -quote(Expr, Binding, Q, E) -> - Context = Q#elixir_quote.context, +%% quote/unquote - Vars = [ {'{}', [], - [ '=', [], [ - {'{}', [], [K, [], Context]}, - V - ] ] - } || {K, V} <- Binding], +do_quote({quote, Meta, [Arg]}, Q) when is_list(Meta) -> + TArg = do_quote(Arg, Q#elixir_quote{unquote=false}), - {TExprs, TQ} = do_quote(Expr, Q, E), - {{'{}',[], ['__block__',[], Vars ++ [TExprs] ]}, TQ}. + NewMeta = case Q of + #elixir_quote{op=quote, context=Context} -> keystore(context, Meta, Context); + _ -> Meta + end, -%% Actual quoting and helpers + {'{}', [], [quote, meta(NewMeta, Q), [TArg]]}; -do_quote({quote, _, Args} = Tuple, #elixir_quote{unquote=true} = Q, E) when length(Args) == 1; length(Args) == 2 -> - {TTuple, TQ} = do_quote_tuple(Tuple, Q#elixir_quote{unquote=false}, E), - {TTuple, TQ#elixir_quote{unquote=true}}; +do_quote({quote, Meta, [Opts, Arg]}, Q) when is_list(Meta) -> + TOpts = do_quote(Opts, Q), + TArg = do_quote(Arg, Q#elixir_quote{unquote=false}), -do_quote({unquote, _Meta, [Expr]}, #elixir_quote{unquote=true} = Q, _) -> - {Expr, Q#elixir_quote{unquoted=true}}; + NewMeta = case Q of + #elixir_quote{op=quote, context=Context} -> keystore(context, Meta, Context); + _ -> Meta + end, + + {'{}', [], [quote, meta(NewMeta, Q), [TOpts, TArg]]}; + +do_quote({unquote, Meta, [Expr]}, #elixir_quote{unquote=true, shallow_validate=Validate}) when is_list(Meta) -> + case Validate of + true -> {{'.', Meta, [?MODULE, shallow_validate_ast]}, Meta, [Expr]}; + false -> Expr + end; %% Aliases -do_quote({'__aliases__', Meta, [H|T]} = Alias, #elixir_quote{aliases_hygiene=true} = Q, E) when is_atom(H) and (H /= 'Elixir') -> - Annotation = case elixir_aliases:expand(Alias, ?m(E, aliases), - ?m(E, macro_aliases), ?m(E, lexical_tracker)) of - Atom when is_atom(Atom) -> Atom; - Aliases when is_list(Aliases) -> false - end, +do_quote({'__aliases__', Meta, [H | T]}, #elixir_quote{aliases_hygiene=(#{}=E)} = Q) + when is_list(Meta), is_atom(H), H /= 'Elixir' -> + Annotation = + case elixir_aliases:expand(Meta, [H | T], E, true) of + Atom when is_atom(Atom) -> Atom; + Aliases when is_list(Aliases) -> false + end, AliasMeta = keystore(alias, keydelete(counter, Meta), Annotation), - do_quote_tuple({'__aliases__', AliasMeta, [H|T]}, Q, E); + do_quote_tuple('__aliases__', AliasMeta, [H | T], Q); %% Vars -do_quote({Left, Meta, nil}, #elixir_quote{vars_hygiene=true} = Q, E) when is_atom(Left) -> - do_quote_tuple({Left, Meta, Q#elixir_quote.context}, Q, E); +do_quote({Name, Meta, nil}, #elixir_quote{op=quote} = Q) + when is_atom(Name), is_list(Meta) -> + ImportMeta = case Q#elixir_quote.imports_hygiene of + nil -> Meta; + E -> import_meta(Meta, Name, 0, Q, E) + end, + + {'{}', [], [Name, meta(ImportMeta, Q), Q#elixir_quote.context]}; %% Unquote -do_quote({{{'.', Meta, [Left, unquote]}, _, [Expr]}, _, Args}, #elixir_quote{unquote=true} = Q, E) -> - do_quote_call(Left, Meta, Expr, Args, Q, E); +do_quote({{{'.', Meta, [Left, unquote]}, _, [Expr]}, _, Args}, #elixir_quote{unquote=true} = Q) when is_list(Meta) -> + do_quote_call(Left, Meta, Expr, Args, Q); -do_quote({{'.', Meta, [Left, unquote]}, _, [Expr]}, #elixir_quote{unquote=true} = Q, E) -> - do_quote_call(Left, Meta, Expr, nil, Q, E); +do_quote({{'.', Meta, [Left, unquote]}, _, [Expr]}, #elixir_quote{unquote=true} = Q) when is_list(Meta) -> + do_quote_call(Left, Meta, Expr, nil, Q); %% Imports do_quote({'&', Meta, [{'/', _, [{F, _, C}, A]}] = Args}, - #elixir_quote{imports_hygiene=true} = Q, E) when is_atom(F), is_integer(A), is_atom(C) -> - do_quote_fa('&', Meta, Args, F, A, Q, E); + #elixir_quote{imports_hygiene=(#{}=E)} = Q) when is_atom(F), is_integer(A), is_atom(C), is_list(Meta) -> + NewMeta = + case elixir_dispatch:find_import(Meta, F, A, E) of + false -> + Meta; + + Receiver -> + keystore(context, keystore(imports, Meta, [{A, Receiver}]), Q#elixir_quote.context) + end, + do_quote_tuple('&', NewMeta, Args, Q); -do_quote({Name, Meta, ArgsOrAtom}, #elixir_quote{imports_hygiene=true} = Q, E) when is_atom(Name) -> - Arity = case is_atom(ArgsOrAtom) of - true -> 0; - false -> length(ArgsOrAtom) +do_quote({Name, Meta, ArgsOrContext}, #elixir_quote{imports_hygiene=(#{}=E)} = Q) + when is_atom(Name), is_list(Meta), is_list(ArgsOrContext) or is_atom(ArgsOrContext) -> + Arity = if + is_atom(ArgsOrContext) -> 0; + true -> length(ArgsOrContext) end, - NewMeta = case (keyfind(import, Meta) == false) andalso - elixir_dispatch:find_import(Meta, Name, Arity, E) of - false -> + ImportMeta = import_meta(Meta, Name, Arity, Q, E), + Annotated = annotate({Name, ImportMeta, ArgsOrContext}, Q#elixir_quote.context), + do_quote_tuple(Annotated, Q); + +%% Two-element tuples + +do_quote({Left, Right}, #elixir_quote{unquote=true} = Q) when + is_tuple(Left) andalso (element(1, Left) == unquote_splicing); + is_tuple(Right) andalso (element(1, Right) == unquote_splicing) -> + do_quote({'{}', [], [Left, Right]}, Q); + +do_quote({Left, Right}, Q) -> + TLeft = do_quote(Left, Q), + TRight = do_quote(Right, Q), + {TLeft, TRight}; + +%% Everything else + +do_quote(Other, #elixir_quote{op=Op} = Q) when Op =/= quote -> + do_escape(Other, Q); + +do_quote({_, _, _} = Tuple, Q) -> + Annotated = annotate(Tuple, Q#elixir_quote.context), + do_quote_tuple(Annotated, Q); + +do_quote([], _) -> + []; + +do_quote([H | T], #elixir_quote{unquote=false} = Q) -> + do_quote_simple_list(T, do_quote(H, Q), Q); + +do_quote([H | T], Q) -> + do_quote_tail(lists:reverse(T, [H]), Q); + +do_quote(Other, _) -> + Other. + +import_meta(Meta, Name, Arity, Q, E) -> + case (keyfind(imports, Meta) == false) andalso + elixir_dispatch:find_imports(Meta, Name, E) of + [_ | _] = Imports -> + trace_import_quoted(Imports, Meta, Name, E), + keystore(imports, keystore(context, Meta, Q#elixir_quote.context), Imports); + + _ -> case (Arity == 1) andalso keyfind(ambiguous_op, Meta) of {ambiguous_op, nil} -> keystore(ambiguous_op, Meta, Q#elixir_quote.context); _ -> Meta - end; - Receiver -> - keystore(import, keystore(context, Meta, Q#elixir_quote.context), Receiver) - end, + end + end. - Annotated = annotate({Name, NewMeta, ArgsOrAtom}, Q#elixir_quote.context, file(E, Q)), - do_quote_tuple(Annotated, Q, E); +trace_import_quoted([{Arity, Mod} | Imports], Meta, Name, E) -> + {Rest, Arities} = collect_trace_import_quoted(Imports, Mod, [], [Arity]), + elixir_env:trace({imported_quoted, Meta, Mod, Name, Arities}, E), + trace_import_quoted(Rest, Meta, Name, E); +trace_import_quoted([], _Meta, _Name, _E) -> + ok. + +collect_trace_import_quoted([{Arity, Mod} | Imports], Mod, Acc, Arities) -> + collect_trace_import_quoted(Imports, Mod, Acc, [Arity | Arities]); +collect_trace_import_quoted([Import | Imports], Mod, Acc, Arities) -> + collect_trace_import_quoted(Imports, Mod, [Import | Acc], Arities); +collect_trace_import_quoted([], _Mod, Acc, Arities) -> + {lists:reverse(Acc), lists:reverse(Arities)}. + +%% do_quote_* + +do_quote_call(Left, Meta, Expr, Args, Q) -> + All = [Left, {unquote, Meta, [Expr]}, Args, Q#elixir_quote.context], + TAll = [do_quote(X, Q) || X <- All], + {{'.', Meta, [elixir_quote, dot]}, Meta, [meta(Meta, Q) | TAll]}. + +do_quote_tuple({Left, Meta, Right}, Q) -> + do_quote_tuple(Left, Meta, Right, Q). + +do_quote_tuple(Left, Meta, Right, Q) -> + TLeft = do_quote(Left, Q), + TRight = do_quote(Right, Q), + {'{}', [], [TLeft, meta(Meta, Q), TRight]}. + +do_quote_simple_list([], Prev, _) -> [Prev]; +do_quote_simple_list([H | T], Prev, Q) -> + [Prev | do_quote_simple_list(T, do_quote(H, Q), Q)]; +do_quote_simple_list(Other, Prev, Q) -> + [{'|', [], [Prev, do_quote(Other, Q)]}]. + +do_quote_tail([{'|', Meta, [{unquote_splicing, _, [Left]}, Right]} | T], #elixir_quote{unquote=true} = Q) -> + %% Process the remaining entries on the list. + %% For [1, 2, 3, unquote_splicing(arg) | tail], this will quote + %% 1, 2 and 3, which could even be unquotes. + TT = do_quote_splice(T, Q, [], []), + TR = do_quote(Right, Q), + do_runtime_list(Meta, tail_list, [Left, TR, TT]); -do_quote({_, _, _} = Tuple, #elixir_quote{escape=false} = Q, E) -> - Annotated = annotate(Tuple, Q#elixir_quote.context, file(E, Q)), - do_quote_tuple(Annotated, Q, E); +do_quote_tail(List, Q) -> + do_quote_splice(List, Q, [], []). -%% Literals +do_quote_splice([{unquote_splicing, Meta, [Expr]} | T], #elixir_quote{unquote=true} = Q, Buffer, Acc) -> + Runtime = do_runtime_list(Meta, list, [Expr, do_list_concat(Buffer, Acc)]), + do_quote_splice(T, Q, [], Runtime); -do_quote({Left, Right}, #elixir_quote{unquote=true} = Q, E) when - is_tuple(Left) andalso (element(1, Left) == unquote_splicing); - is_tuple(Right) andalso (element(1, Right) == unquote_splicing) -> - do_quote({'{}', [], [Left, Right]}, Q, E); - -do_quote({Left, Right}, Q, E) -> - {TLeft, LQ} = do_quote(Left, Q, E), - {TRight, RQ} = do_quote(Right, LQ, E), - {{TLeft, TRight}, RQ}; - -do_quote(Map, #elixir_quote{escape=true} = Q, E) when is_map(Map) -> - {TT, TQ} = do_quote(maps:to_list(Map), Q, E), - {{'%{}', [], TT}, TQ}; - -do_quote(Tuple, #elixir_quote{escape=true} = Q, E) when is_tuple(Tuple) -> - {TT, TQ} = do_quote(tuple_to_list(Tuple), Q, E), - {{'{}', [], TT}, TQ}; - -do_quote(List, #elixir_quote{escape=true} = Q, E) when is_list(List) -> - % The improper case is pretty inefficient, but improper lists are are. - case reverse_improper(List) of - {L} -> do_splice(L, Q, E); - {L, R} -> - {TL, QL} = do_splice(L, Q, E, [], []), - {TR, QR} = do_quote(R, QL, E), - {update_last(TL, fun(X) -> {'|', [], [X, TR]} end), QR} +do_quote_splice([H | T], Q, Buffer, Acc) -> + TH = do_quote(H, Q), + do_quote_splice(T, Q, [TH | Buffer], Acc); + +do_quote_splice([], _Q, Buffer, Acc) -> + do_list_concat(Buffer, Acc). + +do_list_concat(Left, []) -> Left; +do_list_concat([], Right) -> Right; +do_list_concat(Left, Right) -> {{'.', [], [erlang, '++']}, [], [Left, Right]}. + +do_runtime_list(Meta, Fun, Args) -> + {{'.', Meta, [elixir_quote, Fun]}, Meta, Args}. + +%% Callbacks + +%% Some expressions cannot be unquoted at compilation time. +%% This function is responsible for doing runtime unquoting. +dot(Meta, Left, Right, Args, Context) -> + annotate(dot(Meta, Left, Right, Args), Context). + +dot(Meta, Left, {'__aliases__', _, Args}, nil) -> + {'__aliases__', Meta, [Left | Args]}; + +dot(Meta, Left, Right, nil) when is_atom(Right) -> + case atom_to_list(Right) of + "Elixir." ++ _ -> + {'__aliases__', Meta, [Left, Right]}; + _ -> + {{'.', Meta, [Left, Right]}, [{no_parens, true} | Meta], []} end; -do_quote(List, Q, E) when is_list(List) -> - do_splice(lists:reverse(List), Q, E); -do_quote(Other, Q, _) -> - {Other, Q}. +dot(Meta, Left, {Right, _, Context}, nil) when is_atom(Right), is_atom(Context) -> + {{'.', Meta, [Left, Right]}, [{no_parens, true} | Meta], []}; -%% Quote helpers +dot(Meta, Left, {Right, _, Args}, nil) when is_atom(Right) -> + {{'.', Meta, [Left, Right]}, Meta, Args}; -do_quote_call(Left, Meta, Expr, Args, Q, E) -> - All = [meta(Meta, Q), Left, {unquote, Meta, [Expr]}, Args, - Q#elixir_quote.context, file(E, Q)], - {TAll, TQ} = lists:mapfoldl(fun(X, Acc) -> do_quote(X, Acc, E) end, Q, All), - {{{'.', Meta, [elixir_quote, dot]}, Meta, TAll}, TQ}. +dot(_Meta, _Left, Right, nil) -> + argument_error(<<"expected unquote after dot to return an atom, an alias or a quoted call, got: ", + ('Elixir.Macro':to_string(Right))/binary>>); -do_quote_fa(Target, Meta, Args, F, A, Q, E) -> - NewMeta = - case (keyfind(import_fa, Meta) == false) andalso - elixir_dispatch:find_import(Meta, F, A, E) of - false -> Meta; - Receiver -> keystore(import_fa, Meta, {Receiver, Q#elixir_quote.context}) - end, - do_quote_tuple({Target, NewMeta, Args}, Q, E). +dot(Meta, Left, Right, Args) when is_atom(Right) -> + {{'.', Meta, [Left, Right]}, Meta, Args}; -do_quote_tuple({Left, Meta, Right}, Q, E) -> - {TLeft, LQ} = do_quote(Left, Q, E), - {TRight, RQ} = do_quote(Right, LQ, E), - {{'{}', [], [TLeft, meta(Meta, Q), TRight]}, RQ}. +dot(Meta, Left, {Right, _, Context}, Args) when is_atom(Right), is_atom(Context) -> + {{'.', Meta, [Left, Right]}, Meta, Args}; -file(#{file := File}, #elixir_quote{keep=true}) -> File; -file(_, _) -> nil. +dot(_Meta, _Left, Right, _Args) -> + argument_error(<<"expected unquote after dot with args to return an atom or a quoted call, got: ", + ('Elixir.Macro':to_string(Right))/binary>>). + +list(Left, Right) when is_list(Right) -> + validate_list(Left), + Left ++ Right. + +tail_list(Left, Right, Tail) when is_list(Right), is_list(Tail) -> + validate_list(Left), + Tail ++ Left ++ Right; + +tail_list(Left, Right, Tail) when is_list(Left) -> + validate_list(Left), + [H | T] = lists:reverse(Tail ++ Left), + lists:reverse([{'|', [], [H, Right]} | T]). + +validate_list(List) -> + case valid_ast_list(List) of + true -> ok; + false -> argument_error(<<"expected a list with quoted expressions in unquote_splicing/1, got: ", + ('Elixir.Kernel':inspect(List))/binary>>) + end. + +argument_error(Message) -> + error('Elixir.ArgumentError':exception([{message, Message}])). -meta(Meta, #elixir_quote{keep=true}) -> - [case KV of {line, V} -> {keep, V}; _ -> KV end || KV <- Meta]; -meta(Meta, #elixir_quote{line=true}) -> +%% Helpers + +meta(Meta, #elixir_quote{op=quote} = Q) -> + generated(keep(keydelete(column, Meta), Q), Q); +meta(Meta, Q) -> + do_quote(Meta, Q). + +generated(Meta, #elixir_quote{generated=true}) -> [{generated, true} | Meta]; +generated(Meta, #elixir_quote{generated=false}) -> Meta. + +keep(Meta, #elixir_quote{file=nil, line=Line}) -> + line(Meta, Line); +keep(Meta, #elixir_quote{file=File, line=true}) -> + case lists:keytake(line, 1, Meta) of + {value, {line, Line}, MetaNoLine} -> + [{keep, {File, Line}} | MetaNoLine]; + false -> + [{keep, {File, 0}} | Meta] + end; +keep(Meta, #elixir_quote{file=File, line=false}) -> + [{keep, {File, 0}} | keydelete(line, Meta)]; +keep(Meta, #elixir_quote{file=File, line=Line}) -> + [{keep, {File, Line}} | keydelete(line, Meta)]. + +line(Meta, true) -> Meta; -meta(Meta, #elixir_quote{line=false}) -> +line(Meta, false) -> keydelete(line, Meta); -meta(Meta, #elixir_quote{line=Line}) -> +line(Meta, Line) -> keystore(line, Meta, Line). -reverse_improper(L) -> reverse_improper(L, []). -reverse_improper([], Acc) -> {Acc}; -reverse_improper([H|T], Acc) when is_list(T) -> reverse_improper(T, [H|Acc]); -reverse_improper([H|T], Acc) -> {[H|Acc], T}. +reverse_improper([H | T], Acc) -> reverse_improper(T, [H | Acc]); +reverse_improper([], Acc) -> Acc; +reverse_improper(T, Acc) -> {Acc, T}. update_last([], _) -> []; update_last([H], F) -> [F(H)]; -update_last([H|T], F) -> [H|update_last(T,F)]. +update_last([H | T], F) -> [H | update_last(T, F)]. keyfind(Key, Meta) -> lists:keyfind(Key, 1, Meta). @@ -314,42 +643,31 @@ keystore(_Key, Meta, nil) -> Meta; keystore(Key, Meta, Value) -> lists:keystore(Key, 1, Meta, {Key, Value}). -keyreplace(Key, Meta, {Key, _V}) -> - Meta; -keyreplace(Key, Meta, Tuple) -> - lists:keyreplace(Key, 1, Meta, Tuple). keynew(Key, Meta, Value) -> - case keyfind(Key, Meta) of - {Key, _} -> Meta; - _ -> keystore(Key, Meta, Value) + case lists:keymember(Key, 1, Meta) of + true -> Meta; + false -> [{Key, Value} | Meta] end. -%% Quote splicing - -do_splice([{'|', Meta, [{unquote_splicing, _, [Left]}, Right]}|T], #elixir_quote{unquote=true} = Q, E) -> - %% Process the remaining entries on the list. - %% For [1, 2, 3, unquote_splicing(arg)|tail], this will quote - %% 1, 2 and 3, which could even be unquotes. - {TT, QT} = do_splice(T, Q, E, [], []), - {TR, QR} = do_quote(Right, QT, E), - {do_runtime_list(Meta, tail_list, [Left, TR, TT]), QR#elixir_quote{unquoted=true}}; - -do_splice(List, Q, E) -> - do_splice(List, Q, E, [], []). - -do_splice([{unquote_splicing, Meta, [Expr]}|T], #elixir_quote{unquote=true} = Q, E, Buffer, Acc) -> - do_splice(T, Q#elixir_quote{unquoted=true}, E, [], do_runtime_list(Meta, list, [Expr, do_join(Buffer, Acc)])); - -do_splice([H|T], Q, E, Buffer, Acc) -> - {TH, TQ} = do_quote(H, Q, E), - do_splice(T, TQ, E, [TH|Buffer], Acc); - -do_splice([], Q, _E, Buffer, Acc) -> - {do_join(Buffer, Acc), Q}. - -do_join(Left, []) -> Left; -do_join([], Right) -> Right; -do_join(Left, Right) -> {{'.', [], [erlang, '++']}, [], [Left, Right]}. - -do_runtime_list(Meta, Fun, Args) -> - {{'.', Meta, [elixir_quote, Fun]}, Meta, Args}. +%% Annotates the AST with context and other info. +%% +%% Note we need to delete the counter because linify +%% adds the counter recursively, even inside quoted +%% expressions, so we need to clean up the forms to +%% allow them to get a new counter on the next expansion. + +annotate({Def, Meta, [H | T]}, Context) when ?defs(Def) -> + {Def, Meta, [annotate_def(H, Context) | T]}; +annotate({{'.', _, [_, Def]} = Target, Meta, [H | T]}, Context) when ?defs(Def) -> + {Target, Meta, [annotate_def(H, Context) | T]}; +annotate({Lexical, Meta, [_ | _] = Args}, Context) when ?lexical(Lexical) -> + NewMeta = keystore(context, keydelete(counter, Meta), Context), + {Lexical, NewMeta, Args}; +annotate(Tree, _Context) -> Tree. + +annotate_def({'when', Meta, [Left, Right]}, Context) -> + {'when', Meta, [annotate_def(Left, Context), Right]}; +annotate_def({Fun, Meta, Args}, Context) -> + {Fun, keystore(context, Meta, Context), Args}; +annotate_def(Other, _Context) -> + Other. diff --git a/lib/elixir/src/elixir_rewrite.erl b/lib/elixir/src/elixir_rewrite.erl new file mode 100644 index 00000000000..697a1b9170c --- /dev/null +++ b/lib/elixir/src/elixir_rewrite.erl @@ -0,0 +1,386 @@ +%% SPDX-License-Identifier: Apache-2.0 +%% SPDX-FileCopyrightText: 2021 The Elixir Team +%% SPDX-FileCopyrightText: 2012 Plataformatec + +-module(elixir_rewrite). +-compile({inline, [inner_inline/4, inner_rewrite/5]}). +-compile(nowarn_shadow_vars). +-export([erl_to_ex/3, inline/3, rewrite/5, match/6, guard/6, format_error/1]). +-include("elixir.hrl"). + +%% Convenience variables + +-define(atom, 'Elixir.Atom'). +-define(bitwise, 'Elixir.Bitwise'). +-define(enum, 'Elixir.Enum'). +-define(float, 'Elixir.Float'). +-define(function, 'Elixir.Function'). +-define(integer, 'Elixir.Integer'). +-define(io, 'Elixir.IO'). +-define(kernel, 'Elixir.Kernel'). +-define(list, 'Elixir.List'). +-define(map, 'Elixir.Map'). +-define(node, 'Elixir.Node'). +-define(port, 'Elixir.Port'). +-define(process, 'Elixir.Process'). +-define(string, 'Elixir.String'). +-define(string_chars, 'Elixir.String.Chars'). +-define(system, 'Elixir.System'). +-define(tuple, 'Elixir.Tuple'). + +% Macros used to define inline and rewrite rules. +% Defines the rules from Elixir function to Erlang function +% and the reverse, rewrites that are not reversible or have +% complex rules are defined without the macros. +-define( + inline(ExMod, ExFun, Arity, ErlMod, ErlFun), + inner_inline(ex_to_erl, ExMod, ExFun, Arity) -> {ErlMod, ErlFun}; + inner_inline(erl_to_ex, ErlMod, ErlFun, Arity) -> {ExMod, ExFun} +). + +-define( + rewrite(ExMod, ExFun, ExArgs, ErlMod, ErlFun, ErlArgs), + inner_rewrite(ex_to_erl, _Meta, ExMod, ExFun, ExArgs) -> {ErlMod, ErlFun, ErlArgs}; + inner_rewrite(erl_to_ex, _Meta, ErlMod, ErlFun, ErlArgs) -> {ExMod, ExFun, ExArgs, fun(ErlArgs) -> ExArgs end} +). + +erl_to_ex(Mod, Fun, Args) when is_list(Args) -> + case inner_inline(erl_to_ex, Mod, Fun, length(Args)) of + false -> inner_rewrite(erl_to_ex, [], Mod, Fun, Args); + {ExMod, ExFun} -> {ExMod, ExFun, Args, fun identity/1} + end; + +erl_to_ex(Mod, Fun, Arity) when is_integer(Arity) -> + inner_inline(erl_to_ex, Mod, Fun, Arity). + +%% Inline rules +%% +%% Inline rules are straightforward, they keep the same +%% number and order of arguments and show up on captures. +inline(Mod, Fun, Arity) -> inner_inline(ex_to_erl, Mod, Fun, Arity). + +?inline(?atom, to_charlist, 1, erlang, atom_to_list); +?inline(?atom, to_string, 1, erlang, atom_to_binary); + +?inline(?bitwise, 'bnot', 1, erlang, 'bnot'); +?inline(?bitwise, 'band', 2, erlang, 'band'); +?inline(?bitwise, 'bor', 2, erlang, 'bor'); +?inline(?bitwise, 'bxor', 2, erlang, 'bxor'); +?inline(?bitwise, 'bsl', 2, erlang, 'bsl'); +?inline(?bitwise, 'bsr', 2, erlang, 'bsr'); + +?inline(?function, capture, 3, erlang, make_fun); +?inline(?function, info, 1, erlang, fun_info); +?inline(?function, info, 2, erlang, fun_info); + +?inline(?integer, to_charlist, 1, erlang, integer_to_list); +?inline(?integer, to_charlist, 2, erlang, integer_to_list); +?inline(?integer, to_string, 1, erlang, integer_to_binary); +?inline(?integer, to_string, 2, erlang, integer_to_binary); + +?inline(?io, iodata_length, 1, erlang, iolist_size); +?inline(?io, iodata_to_binary, 1, erlang, iolist_to_binary); + +?inline(?kernel, '!=', 2, erlang, '/='); +?inline(?kernel, '!==', 2, erlang, '=/='); +?inline(?kernel, '*', 2, erlang, '*'); +?inline(?kernel, '+', 1, erlang, '+'); +?inline(?kernel, '+', 2, erlang, '+'); +?inline(?kernel, '++', 2, erlang, '++'); +?inline(?kernel, '-', 1, erlang, '-'); +?inline(?kernel, '-', 2, erlang, '-'); +?inline(?kernel, '--', 2, erlang, '--'); +?inline(?kernel, '/', 2, erlang, '/'); +?inline(?kernel, '<', 2, erlang, '<'); +?inline(?kernel, '<=', 2, erlang, '=<'); +?inline(?kernel, '==', 2, erlang, '=='); +?inline(?kernel, '===', 2, erlang, '=:='); +?inline(?kernel, '>', 2, erlang, '>'); +?inline(?kernel, '>=', 2, erlang, '>='); +?inline(?kernel, abs, 1, erlang, abs); +?inline(?kernel, apply, 2, erlang, apply); +?inline(?kernel, apply, 3, erlang, apply); +?inline(?kernel, binary_part, 3, erlang, binary_part); +?inline(?kernel, bit_size, 1, erlang, bit_size); +?inline(?kernel, byte_size, 1, erlang, byte_size); +?inline(?kernel, ceil, 1, erlang, ceil); +?inline(?kernel, 'div', 2, erlang, 'div'); +?inline(?kernel, exit, 1, erlang, exit); +?inline(?kernel, floor, 1, erlang, floor); +?inline(?kernel, 'function_exported?', 3, erlang, function_exported); +?inline(?kernel, hd, 1, erlang, hd); +?inline(?kernel, is_atom, 1, erlang, is_atom); +?inline(?kernel, is_binary, 1, erlang, is_binary); +?inline(?kernel, is_bitstring, 1, erlang, is_bitstring); +?inline(?kernel, is_boolean, 1, erlang, is_boolean); +?inline(?kernel, is_float, 1, erlang, is_float); +?inline(?kernel, is_function, 1, erlang, is_function); +?inline(?kernel, is_function, 2, erlang, is_function); +?inline(?kernel, is_integer, 1, erlang, is_integer); +?inline(?kernel, is_list, 1, erlang, is_list); +?inline(?kernel, is_map, 1, erlang, is_map); +?inline(?kernel, is_number, 1, erlang, is_number); +?inline(?kernel, is_pid, 1, erlang, is_pid); +?inline(?kernel, is_port, 1, erlang, is_port); +?inline(?kernel, is_reference, 1, erlang, is_reference); +?inline(?kernel, is_tuple, 1, erlang, is_tuple); +?inline(?kernel, length, 1, erlang, length); +?inline(?kernel, make_ref, 0, erlang, make_ref); +?inline(?kernel, map_size, 1, erlang, map_size); +?inline(?kernel, max, 2, erlang, max); +?inline(?kernel, min, 2, erlang, min); +?inline(?kernel, node, 0, erlang, node); +?inline(?kernel, node, 1, erlang, node); +?inline(?kernel, 'not', 1, erlang, 'not'); +?inline(?kernel, 'rem', 2, erlang, 'rem'); +?inline(?kernel, round, 1, erlang, round); +?inline(?kernel, self, 0, erlang, self); +?inline(?kernel, send, 2, erlang, send); +?inline(?kernel, spawn, 1, erlang, spawn); +?inline(?kernel, spawn, 3, erlang, spawn); +?inline(?kernel, spawn_link, 1, erlang, spawn_link); +?inline(?kernel, spawn_link, 3, erlang, spawn_link); +?inline(?kernel, spawn_monitor, 1, erlang, spawn_monitor); +?inline(?kernel, spawn_monitor, 3, erlang, spawn_monitor); +?inline(?kernel, throw, 1, erlang, throw); +?inline(?kernel, tl, 1, erlang, tl); +?inline(?kernel, trunc, 1, erlang, trunc); +?inline(?kernel, tuple_size, 1, erlang, tuple_size); + +?inline(?list, to_atom, 1, erlang, list_to_atom); +?inline(?list, to_existing_atom, 1, erlang, list_to_existing_atom); +?inline(?list, to_float, 1, erlang, list_to_float); +?inline(?list, to_integer, 1, erlang, list_to_integer); +?inline(?list, to_integer, 2, erlang, list_to_integer); +?inline(?list, to_tuple, 1, erlang, list_to_tuple); + +?inline(?map, from_keys, 2, maps, from_keys); +?inline(?map, intersect, 2, maps, intersect); +?inline(?map, keys, 1, maps, keys); +?inline(?map, merge, 2, maps, merge); +?inline(?map, to_list, 1, maps, to_list); +?inline(?map, values, 1, maps, values); + +?inline(?node, list, 0, erlang, nodes); +?inline(?node, list, 1, erlang, nodes); +?inline(?node, spawn, 2, erlang, spawn); +?inline(?node, spawn, 3, erlang, spawn_opt); +?inline(?node, spawn, 4, erlang, spawn); +?inline(?node, spawn, 5, erlang, spawn_opt); +?inline(?node, spawn_link, 2, erlang, spawn_link); +?inline(?node, spawn_link, 4, erlang, spawn_link); +?inline(?node, spawn_monitor, 2, erlang, spawn_monitor); +?inline(?node, spawn_monitor, 4, erlang, spawn_monitor); + +?inline(?port, close, 1, erlang, port_close); +?inline(?port, command, 2, erlang, port_command); +?inline(?port, command, 3, erlang, port_command); +?inline(?port, connect, 2, erlang, port_connect); +?inline(?port, list, 0, erlang, ports); +?inline(?port, open, 2, erlang, open_port); + +?inline(?process, alias, 0, erlang, alias); +?inline(?process, alias, 1, erlang, alias); +?inline(?process, 'alive?', 1, erlang, is_process_alive); +?inline(?process, cancel_timer, 1, erlang, cancel_timer); +?inline(?process, cancel_timer, 2, erlang, cancel_timer); +?inline(?process, demonitor, 1, erlang, demonitor); +?inline(?process, demonitor, 2, erlang, demonitor); +?inline(?process, exit, 2, erlang, exit); +?inline(?process, flag, 2, erlang, process_flag); +?inline(?process, flag, 3, erlang, process_flag); +?inline(?process, get, 0, erlang, get); +?inline(?process, get_keys, 0, erlang, get_keys); +?inline(?process, get_keys, 1, erlang, get_keys); +?inline(?process, group_leader, 0, erlang, group_leader); +?inline(?process, hibernate, 3, erlang, hibernate); +?inline(?process, link, 1, erlang, link); +?inline(?process, list, 0, erlang, processes); +?inline(?process, read_timer, 1, erlang, read_timer); +?inline(?process, registered, 0, erlang, registered); +?inline(?process, send, 3, erlang, send); +?inline(?process, spawn, 2, erlang, spawn_opt); +?inline(?process, spawn, 4, erlang, spawn_opt); +?inline(?process, unalias, 1, erlang, unalias); +?inline(?process, unlink, 1, erlang, unlink); +?inline(?process, unregister, 1, erlang, unregister); + +?inline(?string, duplicate, 2, binary, copy); +?inline(?string, to_atom, 1, erlang, binary_to_atom); +?inline(?string, to_existing_atom, 1, erlang, binary_to_existing_atom); +?inline(?string, to_float, 1, erlang, binary_to_float); +?inline(?string, to_integer, 1, erlang, binary_to_integer); +?inline(?string, to_integer, 2, erlang, binary_to_integer); + +?inline(?system, monotonic_time, 0, erlang, monotonic_time); +?inline(?system, os_time, 0, os, system_time); +?inline(?system, system_time, 0, erlang, system_time); +?inline(?system, time_offset, 0, erlang, time_offset); +?inline(?system, unique_integer, 0, erlang, unique_integer); +?inline(?system, unique_integer, 1, erlang, unique_integer); +?inline(?tuple, to_list, 1, erlang, tuple_to_list); + +% Defined without macro to avoid conflict with Bitwise named operators +inner_inline(ex_to_erl, ?bitwise, '~~~', 1) -> {erlang, 'bnot'}; +inner_inline(ex_to_erl, ?bitwise, '&&&', 2) -> {erlang, 'band'}; +inner_inline(ex_to_erl, ?bitwise, '|||', 2) -> {erlang, 'bor'}; +inner_inline(ex_to_erl, ?bitwise, '^^^', 2) -> {erlang, 'bxor'}; +inner_inline(ex_to_erl, ?bitwise, '<<<', 2) -> {erlang, 'bsl'}; +inner_inline(ex_to_erl, ?bitwise, '>>>', 2) -> {erlang, 'bsr'}; + +% Defined without macro to avoid conflict with Process.demonitor +inner_inline(ex_to_erl, ?port, demonitor, 1) -> {erlang, demonitor}; +inner_inline(ex_to_erl, ?port, demonitor, 2) -> {erlang, demonitor}; + +inner_inline(_, _, _, _) -> false. + +%% Rewrite rules +%% +%% Rewrite rules are more complex than regular inlining code +%% as they may change the number of arguments. However, they +%% don't add new code (such as case expressions), at best they +%% perform dead code removal. +rewrite(?string_chars, DotMeta, to_string, Meta, [Arg]) -> + case is_always_string(Arg) of + true -> Arg; + false -> {{'.', DotMeta, [?string_chars, to_string]}, Meta, [Arg]} + end; +rewrite(erlang, _, '+', _, [Arg]) when is_number(Arg) -> +Arg; +rewrite(erlang, _, '-', _, [Arg]) when is_number(Arg) -> -Arg; +rewrite(Receiver, DotMeta, Right, Meta, Args) -> + {EReceiver, ERight, EArgs} = inner_rewrite(ex_to_erl, DotMeta, Receiver, Right, Args), + {{'.', DotMeta, [EReceiver, ERight]}, Meta, EArgs}. + +?rewrite(?float, to_charlist, [Arg], erlang, float_to_list, [Arg, [short]]); +?rewrite(?float, to_string, [Arg], erlang, float_to_binary, [Arg, [short]]); +?rewrite(?kernel, is_map_key, [Map, Key], erlang, is_map_key, [Key, Map]); +?rewrite(?map, delete, [Map, Key], maps, remove, [Key, Map]); +?rewrite(?map, fetch, [Map, Key], maps, find, [Key, Map]); +?rewrite(?map, 'fetch!', [Map, Key], maps, get, [Key, Map]); +?rewrite(?map, 'has_key?', [Map, Key], maps, is_key, [Key, Map]); +?rewrite(?map, put, [Map, Key, Value], maps, put, [Key, Value, Map]); +?rewrite(?map, 'replace!', [Map, Key, Value], maps, update, [Key, Value, Map]); +?rewrite(?port, monitor, [Arg], erlang, monitor, [port, Arg]); +?rewrite(?process, group_leader, [Pid, Leader], erlang, group_leader, [Leader, Pid]); +?rewrite(?process, monitor, [Arg], erlang, monitor, [process, Arg]); +?rewrite(?process, monitor, [Arg, Opts], erlang, monitor, [process, Arg, Opts]); +?rewrite(?process, send_after, [Dest, Msg, Time], erlang, send_after, [Time, Dest, Msg]); +?rewrite(?process, send_after, [Dest, Msg, Time, Opts], erlang, send_after, [Time, Dest, Msg, Opts]); +?rewrite(?tuple, duplicate, [Data, Size], erlang, make_tuple, [Size, Data]); + +inner_rewrite(ex_to_erl, Meta, ?tuple, delete_at, [Tuple, Index]) -> + {erlang, delete_element, [increment(Meta, Index), Tuple]}; +inner_rewrite(ex_to_erl, Meta, ?tuple, insert_at, [Tuple, Index, Term]) -> + {erlang, insert_element, [increment(Meta, Index), Tuple, Term]}; +inner_rewrite(ex_to_erl, Meta, ?kernel, elem, [Tuple, Index]) -> + {erlang, element, [increment(Meta, Index), Tuple]}; +inner_rewrite(ex_to_erl, Meta, ?kernel, put_elem, [Tuple, Index, Value]) -> + {erlang, setelement, [increment(Meta, Index), Tuple, Value]}; + +inner_rewrite(erl_to_ex, _Meta, erlang, delete_element, [Index, Tuple]) when is_number(Index) -> + {?tuple, delete_at, [Tuple, Index - 1], fun([Index, Tuple]) -> [Tuple, Index] end}; +inner_rewrite(erl_to_ex, _Meta, erlang, insert_element, [Index, Tuple, Term]) when is_number(Index) -> + {?tuple, insert_at, [Tuple, Index - 1, Term], fun([Index, Tuple, Term]) -> [Tuple, Index, Term] end}; +inner_rewrite(erl_to_ex, _Meta, erlang, element, [Index, Tuple]) when is_number(Index) -> + {?kernel, elem, [Tuple, Index - 1], fun([Index, Tuple]) -> [Tuple, Index] end}; +inner_rewrite(erl_to_ex, _Meta, erlang, setelement, [Index, Tuple, Term]) when is_number(Index) -> + {?kernel, put_elem, [Tuple, Index - 1, Term], fun([Index, Tuple, Term]) -> [Tuple, Index, Term] end}; + +inner_rewrite(erl_to_ex, _Meta, erlang, delete_element, [{{'.', _, [erlang, '+']}, _, [Index, 1]}, Tuple]) -> + {?tuple, delete_at, [Tuple, Index], fun([Index, Tuple]) -> [Tuple, Index] end}; +inner_rewrite(erl_to_ex, _Meta, erlang, insert_element, [{{'.', _, [erlang, '+']}, _, [Index, 1]}, Tuple, Term]) -> + {?tuple, insert_at, [Tuple, Index, Term], fun([Index, Tuple, Term]) -> [Tuple, Index, Term] end}; +inner_rewrite(erl_to_ex, _Meta, erlang, element, [{{'.', _, [erlang, '+']}, _, [Index, 1]}, Tuple]) -> + {?kernel, elem, [Tuple, Index], fun([Index, Tuple]) -> [Tuple, Index] end}; +inner_rewrite(erl_to_ex, _Meta, erlang, setelement, [{{'.', _, [erlang, '+']}, _, [Index, 1]}, Tuple, Term]) -> + {?kernel, put_elem, [Tuple, Index, Term], fun([Index, Tuple, Term]) -> [Tuple, Index, Term] end}; + +inner_rewrite(erl_to_ex, _Meta, erlang, 'orelse', [_, _] = Args) -> + {?kernel, 'or', Args, fun identity/1}; +inner_rewrite(erl_to_ex, _Meta, erlang, 'andalso', [_, _] = Args) -> + {?kernel, 'and', Args, fun identity/1}; + +inner_rewrite(ex_to_erl, _Meta, Mod, Fun, Args) -> {Mod, Fun, Args}; +inner_rewrite(erl_to_ex, _Meta, Mod, Fun, Args) -> {Mod, Fun, Args, fun identity/1}. + +identity(Arg) -> Arg. + +increment(_Meta, Number) when is_number(Number) -> + Number + 1; +increment(Meta, Other) -> + {{'.', Meta, [erlang, '+']}, Meta, [Other, 1]}. + +%% Match rewrite +%% +%% Match rewrite is similar to regular rewrite, except +%% it also verifies the rewrite rule applies in a match context. +%% The allowed operations are very limited. +%% The Kernel operators are already inlined by now, we only need to +%% care about Erlang ones. +match(erlang, _, '++', Meta, [Left, Right], _S) -> + try {ok, static_append(Left, Right, Meta)} + catch impossible -> {error, {invalid_match_append, Left}} + end; +match(Receiver, _, Right, _, Args, _S) -> + {error, {invalid_match, Receiver, Right, length(Args)}}. + +static_append([], Right, _Meta) -> Right; +static_append([{'|', InnerMeta, [Head, Tail]}], Right, Meta) when is_list(Tail) -> + [{'|', InnerMeta, [Head, static_append(Tail, Right, Meta)]}]; +static_append([{'|', _, [_, _]}], _, _) -> throw(impossible); +static_append([Last], Right, Meta) -> [{'|', Meta, [Last, Right]}]; +static_append([Head | Tail], Right, Meta) -> [Head | static_append(Tail, Right, Meta)]; +static_append(_, _, _) -> throw(impossible). + +%% Guard rewrite +%% +%% Guard rewrite is similar to regular rewrite, except +%% it also verifies the resulting function is supported in +%% guard context - only certain BIFs and operators are. +guard(Receiver, DotMeta, Right, Meta, Args, S) -> + case inner_rewrite(ex_to_erl, DotMeta, Receiver, Right, Args) of + {erlang, RRight, RArgs} -> + case allowed_guard(RRight, length(RArgs)) of + true -> {ok, {{'.', DotMeta, [erlang, RRight]}, Meta, RArgs}}; + false -> {error, {invalid_guard, Receiver, Right, length(Args), elixir_utils:guard_info(S)}} + end; + _ -> {error, {invalid_guard, Receiver, Right, length(Args), elixir_utils:guard_info(S)}} + end. + +%% erlang:is_record/2-3 are compiler guards in Erlang which we +%% need to explicitly forbid as they are allowed in erl_internal. +allowed_guard(is_record, 2) -> false; +allowed_guard(is_record, 3) -> false; +allowed_guard(Right, Arity) -> + erl_internal:guard_bif(Right, Arity) orelse elixir_utils:guard_op(Right, Arity). + +format_error({invalid_guard, Receiver, Right, Arity, Context}) -> + io_lib:format(cannot_invoke_or_maybe_require(Receiver, Right, Arity) ++ " ~ts.~ts/~B inside a ~ts", + ['Elixir.Macro':to_string(Receiver), Right, Arity, Context]); +format_error({invalid_match, Receiver, Right, Arity}) -> + io_lib:format(cannot_invoke_or_maybe_require(Receiver, Right, Arity) ++ " ~ts.~ts/~B inside a match", + ['Elixir.Macro':to_string(Receiver), Right, Arity]); +format_error({invalid_match_append, Arg}) -> + io_lib:format("invalid argument for ++ operator inside a match, expected a literal proper list, got: ~ts", + ['Elixir.Macro':to_string(Arg)]). + +cannot_invoke_or_maybe_require(Receiver, Fun, Arity) -> + try + true = lists:member({Fun, Arity}, Receiver:'__info__'(macros)), + ["you must require the module", 'Elixir.Macro':to_string(Receiver), " before invoking macro"] + catch + _:_ -> "cannot invoke remote function" + end. + +is_always_string({{'.', _, [Module, Function]}, _, Args}) -> + is_always_string(Module, Function, length(Args)); +is_always_string(Ast) -> + is_binary(Ast). + +is_always_string('Elixir.Enum', join, _) -> true; +is_always_string('Elixir.Enum', map_join, _) -> true; +is_always_string('Elixir.Kernel', inspect, _) -> true; +is_always_string('Elixir.Macro', to_string, _) -> true; +is_always_string('Elixir.String.Chars', to_string, _) -> true; +is_always_string('Elixir.Path', join, _) -> true; +is_always_string(_Module, _Function, _Args) -> false. diff --git a/lib/elixir/src/elixir_scope.erl b/lib/elixir/src/elixir_scope.erl deleted file mode 100644 index c9f02e3dd0f..00000000000 --- a/lib/elixir/src/elixir_scope.erl +++ /dev/null @@ -1,140 +0,0 @@ -%% Convenience functions used to manipulate scope and its variables. --module(elixir_scope). --export([translate_var/4, build_var/2, - load_binding/2, dump_binding/2, - mergev/2, mergec/2, mergef/2, - merge_vars/2, merge_opt_vars/2 -]). --include("elixir.hrl"). - -%% VAR HANDLING - -translate_var(Meta, Name, Kind, S) when is_atom(Kind); is_integer(Kind) -> - Line = ?line(Meta), - Tuple = {Name, Kind}, - Vars = S#elixir_scope.vars, - - case orddict:find({Name, Kind}, Vars) of - {ok, {Current, _}} -> Exists = true; - error -> Current = nil, Exists = false - end, - - case S#elixir_scope.context of - match -> - MatchVars = S#elixir_scope.match_vars, - - case Exists andalso ordsets:is_element(Tuple, MatchVars) of - true -> - {{var, Line, Current}, S}; - false -> - %% We attempt to give vars a nice name because we - %% still use the unused vars warnings from erl_lint. - %% - %% Once we move the warning to Elixir compiler, we - %% can name vars as _@COUNTER. - {NewVar, Counter, NS} = - if - Kind /= nil -> - build_var('_', S); - true -> - build_var(Name, S) - end, - - FS = NS#elixir_scope{ - vars=orddict:store(Tuple, {NewVar, Counter}, Vars), - match_vars=ordsets:add_element(Tuple, MatchVars), - export_vars=case S#elixir_scope.export_vars of - nil -> nil; - EV -> orddict:store(Tuple, {NewVar, Counter}, EV) - end - }, - - {{var, Line, NewVar}, FS} - end; - _ when Exists -> - {{var, Line, Current}, S} - end. - -build_var(Key, S) -> - New = orddict:update_counter(Key, 1, S#elixir_scope.counter), - Cnt = orddict:fetch(Key, New), - {elixir_utils:atom_concat([Key, "@", Cnt]), Cnt, S#elixir_scope{counter=New}}. - -%% SCOPE MERGING - -%% Receives two scopes and return a new scope based on -%% the second with their variables merged. - -mergev(S1, S2) -> - S2#elixir_scope{ - vars=merge_vars(S1#elixir_scope.vars, S2#elixir_scope.vars), - export_vars=merge_opt_vars(S1#elixir_scope.export_vars, S2#elixir_scope.export_vars) - }. - -%% Receives two scopes and return the first scope with -%% counters and flags from the later. - -mergec(S1, S2) -> - S1#elixir_scope{ - counter=S2#elixir_scope.counter, - super=S2#elixir_scope.super, - caller=S2#elixir_scope.caller - }. - -%% Similar to mergec but does not merge the user vars counter. - -mergef(S1, S2) -> - S1#elixir_scope{ - super=S2#elixir_scope.super, - caller=S2#elixir_scope.caller - }. - -%% Mergers. - -merge_vars(V, V) -> V; -merge_vars(V1, V2) -> - orddict:merge(fun var_merger/3, V1, V2). - -merge_opt_vars(nil, _C2) -> nil; -merge_opt_vars(_C1, nil) -> nil; -merge_opt_vars(C, C) -> C; -merge_opt_vars(C1, C2) -> - orddict:merge(fun var_merger/3, C1, C2). - -var_merger(_Var, {_, V1} = K1, {_, V2}) when V1 > V2 -> K1; -var_merger(_Var, _K1, K2) -> K2. - -%% BINDINGS - -load_binding(Binding, Scope) -> - {NewBinding, NewVars, NewCounter} = load_binding(Binding, [], [], 0), - {NewBinding, Scope#elixir_scope{ - vars=NewVars, - counter=[{'_',NewCounter}] - }}. - -load_binding([{Key,Value}|T], Binding, Vars, Counter) -> - Actual = case Key of - {_Name, _Kind} -> Key; - Name when is_atom(Name) -> {Name, nil} - end, - InternalName = elixir_utils:atom_concat(["_@", Counter]), - load_binding(T, - orddict:store(InternalName, Value, Binding), - orddict:store(Actual, {InternalName, 0}, Vars), Counter + 1); -load_binding([], Binding, Vars, Counter) -> - {Binding, Vars, Counter}. - -dump_binding(Binding, #elixir_scope{vars=Vars}) -> - dump_binding(Vars, Binding, []). - -dump_binding([{{Var, Kind} = Key, {InternalName,_}}|T], Binding, Acc) when is_atom(Kind) -> - Actual = case Kind of - nil -> Var; - _ -> Key - end, - Value = proplists:get_value(InternalName, Binding, nil), - dump_binding(T, Binding, orddict:store(Actual, Value, Acc)); -dump_binding([_|T], Binding, Acc) -> - dump_binding(T, Binding, Acc); -dump_binding([], _Binding, Acc) -> Acc. diff --git a/lib/elixir/src/elixir_sup.erl b/lib/elixir/src/elixir_sup.erl index 80767df73c1..8071647f06c 100644 --- a/lib/elixir/src/elixir_sup.erl +++ b/lib/elixir/src/elixir_sup.erl @@ -1,3 +1,7 @@ +%% SPDX-License-Identifier: Apache-2.0 +%% SPDX-FileCopyrightText: 2021 The Elixir Team +%% SPDX-FileCopyrightText: 2012 Plataformatec + -module(elixir_sup). -behaviour(supervisor). -export([init/1, start_link/0]). @@ -8,24 +12,24 @@ start_link() -> init(ok) -> Workers = [ { - elixir_code_server, - {elixir_code_server, start_link, []}, + elixir_config, + {elixir_config, start_link, []}, permanent, % Restart = permanent | transient | temporary 2000, % Shutdown = brutal_kill | int() >= 0 | infinity worker, % Type = worker | supervisor - [elixir_code_server] % Modules = [Module] | dynamic - }, + [elixir_config] % Modules = [Module] | dynamic + }, { - elixir_counter, - {elixir_counter, start_link, []}, + elixir_code_server, + {elixir_code_server, start_link, []}, permanent, % Restart = permanent | transient | temporary 2000, % Shutdown = brutal_kill | int() >= 0 | infinity worker, % Type = worker | supervisor - [elixir_counter] % Modules = [Module] | dynamic - } + [elixir_code_server] % Modules = [Module] | dynamic + } ], {ok, {{one_for_one, 3, 10}, Workers}}. diff --git a/lib/elixir/src/elixir_tokenizer.erl b/lib/elixir/src/elixir_tokenizer.erl index 84310831eb6..14ea3ca7922 100644 --- a/lib/elixir/src/elixir_tokenizer.erl +++ b/lib/elixir/src/elixir_tokenizer.erl @@ -1,894 +1,1637 @@ +%% SPDX-License-Identifier: Apache-2.0 +%% SPDX-FileCopyrightText: 2021 The Elixir Team +%% SPDX-FileCopyrightText: 2012 Plataformatec + -module(elixir_tokenizer). -include("elixir.hrl"). --export([tokenize/3]). --import(elixir_interpolation, [unescape_tokens/1]). +-include("elixir_tokenizer.hrl"). +-export([tokenize/1, tokenize/3, tokenize/4, invalid_do_error/1, terminator/1]). -define(at_op(T), - T == $@). + T =:= $@). -define(capture_op(T), - T == $&). + T =:= $&). -define(unary_op(T), - T == $!; - T == $^). + T =:= $!; + T =:= $^). --define(unary_op3(T1, T2, T3), - T1 == $~, T2 == $~, T3 == $~). +-define(range_op(T1, T2), + T1 =:= $., T2 =:= $.). --define(hat_op3(T1, T2, T3), - T1 == $^, T2 == $^, T3 == $^). +-define(concat_op(T1, T2), + T1 =:= $+, T2 =:= $+; + T1 =:= $-, T2 =:= $-; + T1 =:= $<, T2 =:= $>). --define(two_op(T1, T2), - T1 == $+, T2 == $+; - T1 == $-, T2 == $-; - T1 == $<, T2 == $>; - T1 == $., T2 == $.). +-define(concat_op3(T1, T2, T3), + T1 =:= $+, T2 =:= $+, T3 =:= $+; + T1 =:= $-, T2 =:= $-, T3 =:= $-). + +-define(power_op(T1, T2), + T1 =:= $*, T2 =:= $*). -define(mult_op(T), - T == $* orelse T == $/). + T =:= $* orelse T =:= $/). -define(dual_op(T), - T == $+ orelse T == $-). + T =:= $+ orelse T =:= $-). -define(arrow_op3(T1, T2, T3), - T1 == $<, T2 == $<, T3 == $<; - T1 == $>, T2 == $>, T3 == $>). + T1 =:= $<, T2 =:= $<, T3 =:= $<; + T1 =:= $>, T2 =:= $>, T3 =:= $>; + T1 =:= $~, T2 =:= $>, T3 =:= $>; + T1 =:= $<, T2 =:= $<, T3 =:= $~; + T1 =:= $<, T2 =:= $~, T3 =:= $>; + T1 =:= $<, T2 =:= $|, T3 =:= $>). -define(arrow_op(T1, T2), - T1 == $|, T2 == $>). + T1 =:= $|, T2 =:= $>; + T1 =:= $~, T2 =:= $>; + T1 =:= $<, T2 =:= $~). -define(rel_op(T), - T == $<; - T == $>). + T =:= $<; + T =:= $>). -define(rel_op2(T1, T2), - T1 == $<, T2 == $=; - T1 == $>, T2 == $=). + T1 =:= $<, T2 =:= $=; + T1 =:= $>, T2 =:= $=). -define(comp_op2(T1, T2), - T1 == $=, T2 == $=; - T1 == $=, T2 == $~; - T1 == $!, T2 == $=). + T1 =:= $=, T2 =:= $=; + T1 =:= $=, T2 =:= $~; + T1 =:= $!, T2 =:= $=). -define(comp_op3(T1, T2, T3), - T1 == $=, T2 == $=, T3 == $=; - T1 == $!, T2 == $=, T3 == $=). + T1 =:= $=, T2 =:= $=, T3 =:= $=; + T1 =:= $!, T2 =:= $=, T3 =:= $=). + +-define(ternary_op(T1, T2), + T1 =:= $/, T2 =:= $/). -define(and_op(T1, T2), - T1 == $&, T2 == $&). + T1 =:= $&, T2 =:= $&). -define(or_op(T1, T2), - T1 == $|, T2 == $|). + T1 =:= $|, T2 =:= $|). -define(and_op3(T1, T2, T3), - T1 == $&, T2 == $&, T3 == $&). + T1 =:= $&, T2 =:= $&, T3 =:= $&). -define(or_op3(T1, T2, T3), - T1 == $|, T2 == $|, T3 == $|). + T1 =:= $|, T2 =:= $|, T3 =:= $|). -define(match_op(T), - T == $=). + T =:= $=). -define(in_match_op(T1, T2), - T1 == $<, T2 == $-; - T1 == $\\, T2 == $\\). + T1 =:= $<, T2 =:= $-; + T1 =:= $\\, T2 =:= $\\). -define(stab_op(T1, T2), - T1 == $-, T2 == $>). + T1 =:= $-, T2 =:= $>). -define(type_op(T1, T2), - T1 == $:, T2 == $:). + T1 =:= $:, T2 =:= $:). --define(pipe_op(T1), - T == $|). +-define(pipe_op(T), + T =:= $|). -tokenize(String, Line, #elixir_tokenizer{} = Scope) -> - tokenize(String, Line, Scope, []); +-define(ellipsis_op3(T1, T2, T3), + T1 =:= $., T2 =:= $., T3 =:= $.). -tokenize(String, Line, Opts) -> - File = case lists:keyfind(file, 1, Opts) of - {file, V1} -> V1; - false -> <<"nofile">> - end, +%% Deprecated operators - Existing = case lists:keyfind(existing_atoms_only, 1, Opts) of - {existing_atoms_only, true} -> true; - false -> false - end, +-define(unary_op3(T1, T2, T3), + T1 =:= $~, T2 =:= $~, T3 =:= $~). + +-define(xor_op3(T1, T2, T3), + T1 =:= $^, T2 =:= $^, T3 =:= $^). + +tokenize(String, Line, Column, #elixir_tokenizer{} = Scope) -> + tokenize(String, Line, Column, Scope, []); + +tokenize(String, Line, Column, Opts) -> + IdentifierTokenizer = elixir_config:identifier_tokenizer(), + + Scope = + lists:foldl(fun + ({check_terminators, false}, Acc) -> + Acc#elixir_tokenizer{cursor_completion=false, terminators=none}; + ({check_terminators, {cursor, Terminators}}, Acc) -> + Acc#elixir_tokenizer{cursor_completion=prune_and_cursor, terminators=Terminators}; + ({existing_atoms_only, ExistingAtomsOnly}, Acc) when is_boolean(ExistingAtomsOnly) -> + Acc#elixir_tokenizer{existing_atoms_only=ExistingAtomsOnly}; + ({static_atoms_encoder, StaticAtomsEncoder}, Acc) when is_function(StaticAtomsEncoder) -> + Acc#elixir_tokenizer{static_atoms_encoder=StaticAtomsEncoder}; + ({preserve_comments, PreserveComments}, Acc) when is_function(PreserveComments) -> + Acc#elixir_tokenizer{preserve_comments=PreserveComments}; + ({unescape, Unescape}, Acc) when is_boolean(Unescape) -> + Acc#elixir_tokenizer{unescape=Unescape}; + ({indentation, Indentation}, Acc) when Indentation >= 0 -> + Acc#elixir_tokenizer{column=Indentation+1}; + (_, Acc) -> + Acc + end, #elixir_tokenizer{identifier_tokenizer=IdentifierTokenizer}, Opts), + + tokenize(String, Line, Column, Scope, []). - Check = case lists:keyfind(check_terminators, 1, Opts) of - {check_terminators, false} -> false; - false -> true - end, +tokenize(String, Line, Opts) -> + tokenize(String, Line, 1, Opts). - tokenize(String, Line, #elixir_tokenizer{ - file=File, - existing_atoms_only=Existing, - check_terminators=Check - }). +tokenize([], Line, Column, #elixir_tokenizer{cursor_completion=Cursor} = Scope, Tokens) when Cursor /= false -> + #elixir_tokenizer{ascii_identifiers_only=Ascii, terminators=Terminators, warnings=Warnings} = Scope, -tokenize([], Line, #elixir_tokenizer{terminators=[]}, Tokens) -> - {ok, Line, lists:reverse(Tokens)}; + {CursorColumn, AccTerminators, AccTokens} = + add_cursor(Line, Column, Cursor, Terminators, Tokens), -tokenize([], EndLine, #elixir_tokenizer{terminators=[{Start, StartLine}|_]}, Tokens) -> - End = terminator(Start), - Message = io_lib:format("missing terminator: ~ts (for \"~ts\" starting at line ~B)", [End, Start, StartLine]), - {error, {EndLine, Message, []}, [], Tokens}; + AllWarnings = maybe_unicode_lint_warnings(Ascii, Tokens, Warnings), + {ok, Line, CursorColumn, AllWarnings, AccTokens, AccTerminators}; + +tokenize([], EndLine, EndColumn, #elixir_tokenizer{terminators=[{Start, {StartLine, StartColumn, _}, _} | _]} = Scope, Tokens) -> + End = terminator(Start), + Hint = missing_terminator_hint(Start, End, Scope), + Message = "missing terminator: ~ts", + Formatted = io_lib:format(Message, [End]), + Meta = [ + {opening_delimiter, Start}, + {expected_delimiter, End}, + {line, StartLine}, + {column, StartColumn}, + {end_line, EndLine}, + {end_column, EndColumn} + ], + error({Meta, [Formatted, Hint], []}, [], Scope, Tokens); + +tokenize([], Line, Column, #elixir_tokenizer{} = Scope, Tokens) -> + #elixir_tokenizer{ascii_identifiers_only=Ascii, warnings=Warnings} = Scope, + AllWarnings = maybe_unicode_lint_warnings(Ascii, Tokens, Warnings), + {ok, Line, Column, AllWarnings, Tokens, []}; + +% VC merge conflict + +tokenize(("<<<<<<<" ++ _) = Original, Line, 1, Scope, Tokens) -> + FirstLine = lists:takewhile(fun(C) -> C =/= $\n andalso C =/= $\r end, Original), + Reason = {?LOC(Line, 1), "found an unexpected version control marker, please resolve the conflicts: ", FirstLine}, + error(Reason, Original, Scope, Tokens); % Base integers -tokenize([$0,X,H|T], Line, Scope, Tokens) when (X == $x orelse X == $X), ?is_hex(H) -> - {Rest, Number} = tokenize_hex([H|T], []), - tokenize(Rest, Line, Scope, [{number, Line, Number}|Tokens]); +tokenize([$0, $x, H | T], Line, Column, Scope, Tokens) when ?is_hex(H) -> + {Rest, Number, OriginalRepresentation, Length} = tokenize_hex(T, [H], 1), + Token = {int, {Line, Column, Number}, OriginalRepresentation}, + tokenize(Rest, Line, Column + 2 + Length, Scope, [Token | Tokens]); -tokenize([$0,B,H|T], Line, Scope, Tokens) when (B == $b orelse B == $B), ?is_bin(H) -> - {Rest, Number} = tokenize_bin([H|T], []), - tokenize(Rest, Line, Scope, [{number, Line, Number}|Tokens]); +tokenize([$0, $b, H | T], Line, Column, Scope, Tokens) when ?is_bin(H) -> + {Rest, Number, OriginalRepresentation, Length} = tokenize_bin(T, [H], 1), + Token = {int, {Line, Column, Number}, OriginalRepresentation}, + tokenize(Rest, Line, Column + 2 + Length, Scope, [Token | Tokens]); -tokenize([$0,H|T], Line, Scope, Tokens) when ?is_octal(H) -> - {Rest, Number} = tokenize_octal([H|T], []), - tokenize(Rest, Line, Scope, [{number, Line, Number}|Tokens]); +tokenize([$0, $o, H | T], Line, Column, Scope, Tokens) when ?is_octal(H) -> + {Rest, Number, OriginalRepresentation, Length} = tokenize_octal(T, [H], 1), + Token = {int, {Line, Column, Number}, OriginalRepresentation}, + tokenize(Rest, Line, Column + 2 + Length, Scope, [Token | Tokens]); % Comments -tokenize([$#|String], Line, Scope, Tokens) -> - Rest = tokenize_comment(String), - tokenize(Rest, Line, Scope, Tokens); +tokenize([$# | String], Line, Column, Scope, Tokens) -> + case tokenize_comment(String, [$#]) of + {error, Char, Reason} -> + error_comment(Char, Reason, [$# | String], Line, Column, Scope, Tokens); + {Rest, Comment} -> + preserve_comments(Line, Column, Tokens, Comment, Rest, Scope), + tokenize(Rest, Line, Column, Scope, reset_eol(Tokens)) + end; % Sigils -tokenize([$~,S,H,H,H|T] = Original, Line, Scope, Tokens) when ?is_quote(H), ?is_upcase(S) orelse ?is_downcase(S) -> - case extract_heredoc_with_interpolation(Line, Scope, ?is_downcase(S), T, H) of - {ok, NewLine, Parts, Rest} -> - {Final, Modifiers} = collect_modifiers(Rest, []), - tokenize(Final, NewLine, Scope, [{sigil, Line, S, Parts, Modifiers}|Tokens]); - {error, Reason} -> - {error, Reason, Original, Tokens} - end; - -tokenize([$~,S,H|T] = Original, Line, Scope, Tokens) when ?is_sigil(H), ?is_upcase(S) orelse ?is_downcase(S) -> - case elixir_interpolation:extract(Line, Scope, ?is_downcase(S), T, sigil_terminator(H)) of - {NewLine, Parts, Rest} -> - {Final, Modifiers} = collect_modifiers(Rest, []), - tokenize(Final, NewLine, Scope, [{sigil, Line, S, Parts, Modifiers}|Tokens]); - {error, Reason} -> - Sigil = [$~,S,H], - interpolation_error(Reason, Original, Tokens, " (for sigil ~ts starting at line ~B)", [Sigil, Line]) - end; +tokenize([$~, H | _T] = Original, Line, Column, Scope, Tokens) when ?is_upcase(H) orelse ?is_downcase(H) -> + tokenize_sigil(Original, Line, Column, Scope, Tokens); % Char tokens -tokenize([$?,$\\,P,${,A,B,C,D,E,F,$}|T], Line, Scope, Tokens) - when (P == $x orelse P == $X), ?is_hex(A), ?is_hex(B), ?is_hex(C), ?is_hex(D), ?is_hex(E), ?is_hex(F) -> - Char = escape_char([$\\,P,${,A,B,C,D,E,F,$}]), - tokenize(T, Line, Scope, [{number, Line, Char}|Tokens]); - -tokenize([$?,$\\,P,${,A,B,C,D,E,$}|T], Line, Scope, Tokens) - when (P == $x orelse P == $X), ?is_hex(A), ?is_hex(B), ?is_hex(C), ?is_hex(D), ?is_hex(E) -> - Char = escape_char([$\\,P,${,A,B,C,D,E,$}]), - tokenize(T, Line, Scope, [{number, Line, Char}|Tokens]); - -tokenize([$?,$\\,P,${,A,B,C,D,$}|T], Line, Scope, Tokens) - when (P == $x orelse P == $X), ?is_hex(A), ?is_hex(B), ?is_hex(C), ?is_hex(D) -> - Char = escape_char([$\\,P,${,A,B,C,D,$}]), - tokenize(T, Line, Scope, [{number, Line, Char}|Tokens]); - -tokenize([$?,$\\,P,${,A,B,C,$}|T], Line, Scope, Tokens) - when (P == $x orelse P == $X), ?is_hex(A), ?is_hex(B), ?is_hex(C) -> - Char = escape_char([$\\,P,${,A,B,C,$}]), - tokenize(T, Line, Scope, [{number, Line, Char}|Tokens]); - -tokenize([$?,$\\,P,${,A,B,$}|T], Line, Scope, Tokens) - when (P == $x orelse P == $X), ?is_hex(A), ?is_hex(B) -> - Char = escape_char([$\\,P,${,A,B,$}]), - tokenize(T, Line, Scope, [{number, Line, Char}|Tokens]); - -tokenize([$?,$\\,P,${,A,$}|T], Line, Scope, Tokens) - when (P == $x orelse P == $X), ?is_hex(A) -> - Char = escape_char([$\\,P,${,A,$}]), - tokenize(T, Line, Scope, [{number, Line, Char}|Tokens]); - -tokenize([$?,$\\,P,A,B|T], Line, Scope, Tokens) - when (P == $x orelse P == $X), ?is_hex(A), ?is_hex(B) -> - Char = escape_char([$\\,P,A,B]), - tokenize(T, Line, Scope, [{number, Line, Char}|Tokens]); - -tokenize([$?,$\\,P,A|T], Line, Scope, Tokens) - when (P == $x orelse P == $X), ?is_hex(A) -> - Char = escape_char([$\\,P,A]), - tokenize(T, Line, Scope, [{number, Line, Char}|Tokens]); - -tokenize([$?,$\\,A,B,C|T], Line, Scope, Tokens) - when ?is_octal(A), A =< $3,?is_octal(B), ?is_octal(C) -> - Char = escape_char([$\\,A,B,C]), - tokenize(T, Line, Scope, [{number, Line, Char}|Tokens]); - -tokenize([$?,$\\,A,B|T], Line, Scope, Tokens) - when ?is_octal(A), ?is_octal(B) -> - Char = escape_char([$\\,A,B]), - tokenize(T, Line, Scope, [{number, Line, Char}|Tokens]); - -tokenize([$?,$\\,A|T], Line, Scope, Tokens) - when ?is_octal(A) -> - Char = escape_char([$\\,A]), - tokenize(T, Line, Scope, [{number, Line, Char}|Tokens]); - -tokenize([$?,$\\,H|T], Line, Scope, Tokens) -> +% We tokenize char literals (?a) as {char, _, CharInt} instead of {number, _, +% CharInt}. This is exactly what Erlang does with Erlang char literals +% ($a). This means we'll have to adjust the error message for char literals in +% elixir_errors.erl as by default {char, _, _} tokens are "hijacked" by Erlang +% and printed with Erlang syntax ($a) in the parser's error messages. + +tokenize([$?, $\\, H | T], Line, Column, Scope, Tokens) -> Char = elixir_interpolation:unescape_map(H), - tokenize(T, Line, Scope, [{number, Line, Char}|Tokens]); -tokenize([$?,Char|T], Line, Scope, Tokens) -> - tokenize(T, Line, Scope, [{number, Line, Char}|Tokens]); + NewScope = if + H =:= Char, H =/= $\\ -> + case handle_char(Char) of + {Escape, Name} -> + Msg = io_lib:format("found ?\\ followed by code point 0x~.16B (~ts), please use ?~ts instead", + [Char, Name, Escape]), + prepend_warning(Line, Column, Msg, Scope); -% Heredocs + false when ?is_downcase(H); ?is_upcase(H) -> + Msg = io_lib:format("unknown escape sequence ?\\~tc, use ?~tc instead", [H, H]), + prepend_warning(Line, Column, Msg, Scope); -tokenize("\"\"\"" ++ T, Line, Scope, Tokens) -> - handle_heredocs(T, Line, $", Scope, Tokens); + false -> + Scope + end; + true -> + Scope + end, -tokenize("'''" ++ T, Line, Scope, Tokens) -> - handle_heredocs(T, Line, $', Scope, Tokens); + Token = {char, {Line, Column, [$?, $\\, H]}, Char}, + case H of + $\n -> + %% If original char is a literal line feed, we already emit a warning, + %% but we need to bump the line without emitting an EOL token. + tokenize_eol(T, Line, NewScope, [Token | Tokens]); + _ -> + tokenize(T, Line, Column + 3, NewScope, [Token | Tokens]) + end; -% Strings +tokenize([$?, Char | T], Line, Column, Scope, Tokens) -> + NewScope = case handle_char(Char) of + {Escape, Name} -> + Msg = io_lib:format("found ? followed by code point 0x~.16B (~ts), please use ?~ts instead", + [Char, Name, Escape]), + prepend_warning(Line, Column, Msg, Scope); + false -> + Scope + end, + Token = {char, {Line, Column, [$?, Char]}, Char}, + case Char of + $\n -> + %% If original char is a literal line feed, we already emit a warning, + %% but we need to bump the line without emitting an EOL token. + tokenize_eol(T, Line, NewScope, [Token | Tokens]); + _ -> + tokenize(T, Line, Column + 2, NewScope, [Token | Tokens]) + end; -tokenize([$"|T], Line, Scope, Tokens) -> - handle_strings(T, Line, $", Scope, Tokens); -tokenize([$'|T], Line, Scope, Tokens) -> - handle_strings(T, Line, $', Scope, Tokens); +% Heredocs -% Atoms +tokenize("\"\"\"" ++ T, Line, Column, Scope, Tokens) -> + handle_heredocs(T, Line, Column, $", Scope, Tokens); -tokenize([$:,H|T] = Original, Line, Scope, Tokens) when ?is_quote(H) -> - case elixir_interpolation:extract(Line, Scope, true, T, H) of - {NewLine, Parts, Rest} -> - Unescaped = unescape_tokens(Parts), - Key = case Scope#elixir_tokenizer.existing_atoms_only of - true -> atom_safe; - false -> atom_unsafe - end, - tokenize(Rest, NewLine, Scope, [{Key, Line, Unescaped}|Tokens]); - {error, Reason} -> - interpolation_error(Reason, Original, Tokens, " (for atom starting at line ~B)", [Line]) - end; +%% TODO: Remove me in Elixir v2.0 +tokenize("'''" ++ T, Line, Column, Scope, Tokens) -> + NewScope = prepend_warning(Line, Column, "single-quoted string represent charlists. Use ~c''' if you indeed want a charlist or use \"\"\" instead", Scope), + handle_heredocs(T, Line, Column, $', NewScope, Tokens); -tokenize([$:,T|String] = Original, Line, Scope, Tokens) when ?is_atom_start(T) -> - {Rest, Part} = tokenize_atom([T|String], []), - case unsafe_to_atom(Part, Line, Scope) of - {ok, Atom} -> - tokenize(Rest, Line, Scope, [{atom, Line, Atom}|Tokens]); - {error, Reason} -> - {error, Reason, Original, Tokens} - end; +% Strings -% %% Special atom identifiers / operators - -tokenize(":..." ++ Rest, Line, Scope, Tokens) -> - tokenize(Rest, Line, Scope, [{atom, Line, '...'}|Tokens]); -tokenize(":<<>>" ++ Rest, Line, Scope, Tokens) -> - tokenize(Rest, Line, Scope, [{atom, Line, '<<>>'}|Tokens]); -tokenize(":%{}" ++ Rest, Line, Scope, Tokens) -> - tokenize(Rest, Line, Scope, [{atom, Line, '%{}'}|Tokens]); -tokenize(":%" ++ Rest, Line, Scope, Tokens) -> - tokenize(Rest, Line, Scope, [{atom, Line, '%'}|Tokens]); -tokenize(":{}" ++ Rest, Line, Scope, Tokens) -> - tokenize(Rest, Line, Scope, [{atom, Line, '{}'}|Tokens]); - -tokenize("...:" ++ Rest, Line, Scope, Tokens) when ?is_space(hd(Rest)) -> - tokenize(Rest, Line, Scope, [{kw_identifier, Line, '...'}|Tokens]); -tokenize("<<>>:" ++ Rest, Line, Scope, Tokens) when ?is_space(hd(Rest)) -> - tokenize(Rest, Line, Scope, [{kw_identifier, Line, '<<>>'}|Tokens]); -tokenize("%{}:" ++ Rest, Line, Scope, Tokens) when ?is_space(hd(Rest)) -> - tokenize(Rest, Line, Scope, [{kw_identifier, Line, '%{}'}|Tokens]); -tokenize("%:" ++ Rest, Line, Scope, Tokens) when ?is_space(hd(Rest)) -> - tokenize(Rest, Line, Scope, [{kw_identifier, Line, '%'}|Tokens]); -tokenize("{}:" ++ Rest, Line, Scope, Tokens) when ?is_space(hd(Rest)) -> - tokenize(Rest, Line, Scope, [{kw_identifier, Line, '{}'}|Tokens]); +tokenize([$" | T], Line, Column, Scope, Tokens) -> + handle_strings(T, Line, Column + 1, $", Scope, Tokens); + +%% TODO: Remove me in Elixir v2.0 +tokenize([$' | T], Line, Column, Scope, Tokens) -> + handle_strings(T, Line, Column + 1, $', Scope, Tokens); + +% Operator atoms + +tokenize(".:" ++ Rest, Line, Column, Scope, Tokens) when ?is_space(hd(Rest)) -> + tokenize(Rest, Line, Column + 2, Scope, [{kw_identifier, {Line, Column, nil}, '.'} | Tokens]); + +tokenize("<<>>:" ++ Rest, Line, Column, Scope, Tokens) when ?is_space(hd(Rest)) -> + tokenize(Rest, Line, Column + 5, Scope, [{kw_identifier, {Line, Column, nil}, '<<>>'} | Tokens]); +tokenize("%{}:" ++ Rest, Line, Column, Scope, Tokens) when ?is_space(hd(Rest)) -> + tokenize(Rest, Line, Column + 4, Scope, [{kw_identifier, {Line, Column, nil}, '%{}'} | Tokens]); +tokenize("%:" ++ Rest, Line, Column, Scope, Tokens) when ?is_space(hd(Rest)) -> + tokenize(Rest, Line, Column + 2, Scope, [{kw_identifier, {Line, Column, nil}, '%'} | Tokens]); +tokenize("&:" ++ Rest, Line, Column, Scope, Tokens) when ?is_space(hd(Rest)) -> + tokenize(Rest, Line, Column + 2, Scope, [{kw_identifier, {Line, Column, nil}, '&'} | Tokens]); +tokenize("{}:" ++ Rest, Line, Column, Scope, Tokens) when ?is_space(hd(Rest)) -> + tokenize(Rest, Line, Column + 3, Scope, [{kw_identifier, {Line, Column, nil}, '{}'} | Tokens]); +tokenize("..//:" ++ Rest, Line, Column, Scope, Tokens) when ?is_space(hd(Rest)) -> + tokenize(Rest, Line, Column + 5, Scope, [{kw_identifier, {Line, Column, nil}, '..//'} | Tokens]); + +tokenize(":<<>>" ++ Rest, Line, Column, Scope, Tokens) -> + tokenize(Rest, Line, Column + 5, Scope, [{atom, {Line, Column, nil}, '<<>>'} | Tokens]); +tokenize(":%{}" ++ Rest, Line, Column, Scope, Tokens) -> + tokenize(Rest, Line, Column + 4, Scope, [{atom, {Line, Column, nil}, '%{}'} | Tokens]); +tokenize(":%" ++ Rest, Line, Column, Scope, Tokens) -> + tokenize(Rest, Line, Column + 2, Scope, [{atom, {Line, Column, nil}, '%'} | Tokens]); +tokenize(":{}" ++ Rest, Line, Column, Scope, Tokens) -> + tokenize(Rest, Line, Column + 3, Scope, [{atom, {Line, Column, nil}, '{}'} | Tokens]); +tokenize(":..//" ++ Rest, Line, Column, Scope, Tokens) -> + tokenize(Rest, Line, Column + 5, Scope, [{atom, {Line, Column, nil}, '..//'} | Tokens]); % ## Three Token Operators -tokenize([$:,T1,T2,T3|Rest], Line, Scope, Tokens) when +tokenize([$:, T1, T2, T3 | Rest], Line, Column, Scope, Tokens) when ?unary_op3(T1, T2, T3); ?comp_op3(T1, T2, T3); ?and_op3(T1, T2, T3); ?or_op3(T1, T2, T3); - ?arrow_op3(T1, T2, T3); ?hat_op3(T1, T2, T3) -> - tokenize(Rest, Line, Scope, [{atom, Line, list_to_atom([T1,T2,T3])}|Tokens]); + ?arrow_op3(T1, T2, T3); ?xor_op3(T1, T2, T3); ?concat_op3(T1, T2, T3); ?ellipsis_op3(T1, T2, T3) -> + Token = {atom, {Line, Column, nil}, list_to_atom([T1, T2, T3])}, + tokenize(Rest, Line, Column + 4, Scope, [Token | Tokens]); % ## Two Token Operators -tokenize([$:,T1,T2|Rest], Line, Scope, Tokens) when + +tokenize([$:, $:, $: | Rest], Line, Column, Scope, Tokens) -> + Message = "atom ::: must be written between quotes, as in :\"::\", to avoid ambiguity", + NewScope = prepend_warning(Line, Column, Message, Scope), + Token = {atom, {Line, Column, nil}, '::'}, + tokenize(Rest, Line, Column + 3, NewScope, [Token | Tokens]); + +tokenize([$:, T1, T2 | Rest], Line, Column, Scope, Tokens) when ?comp_op2(T1, T2); ?rel_op2(T1, T2); ?and_op(T1, T2); ?or_op(T1, T2); - ?arrow_op(T1, T2); ?in_match_op(T1, T2); ?two_op(T1, T2); ?stab_op(T1, T2); - ?type_op(T1, T2) -> - tokenize(Rest, Line, Scope, [{atom, Line, list_to_atom([T1,T2])}|Tokens]); + ?arrow_op(T1, T2); ?in_match_op(T1, T2); ?concat_op(T1, T2); ?power_op(T1, T2); + ?stab_op(T1, T2); ?range_op(T1, T2) -> + Token = {atom, {Line, Column, nil}, list_to_atom([T1, T2])}, + tokenize(Rest, Line, Column + 3, Scope, [Token | Tokens]); % ## Single Token Operators -tokenize([$:,T|Rest], Line, Scope, Tokens) when +tokenize([$:, T | Rest], Line, Column, Scope, Tokens) when ?at_op(T); ?unary_op(T); ?capture_op(T); ?dual_op(T); ?mult_op(T); - ?rel_op(T); ?match_op(T); ?pipe_op(T); T == $. -> - tokenize(Rest, Line, Scope, [{atom, Line, list_to_atom([T])}|Tokens]); + ?rel_op(T); ?match_op(T); ?pipe_op(T); T =:= $. -> + Token = {atom, {Line, Column, nil}, list_to_atom([T])}, + tokenize(Rest, Line, Column + 2, Scope, [Token | Tokens]); + +% ## Stand-alone tokens + +tokenize("=>" ++ Rest, Line, Column, Scope, Tokens) -> + Token = {assoc_op, {Line, Column, previous_was_eol(Tokens)}, '=>'}, + tokenize(Rest, Line, Column + 2, Scope, add_token_with_eol(Token, Tokens)); + +tokenize("..//" ++ Rest = String, Line, Column, Scope, Tokens) -> + case strip_horizontal_space(Rest, Line, Column + 4, Scope) of + {[$/ | _] = Remaining, NewLine, NewColumn} -> + Token = {identifier, {Line, Column, nil}, '..//'}, + tokenize(Remaining, NewLine, NewColumn, Scope, [Token | Tokens]); + {_, _, _} -> + unexpected_token(String, Line, Column, Scope, Tokens) + end; -% End of line +% ## Ternary operator -tokenize(";" ++ Rest, Line, Scope, []) -> - tokenize(Rest, Line, Scope, eol(Line, ';', [])); +% ## Three token operators +tokenize([T1, T2, T3 | Rest], Line, Column, Scope, Tokens) when ?unary_op3(T1, T2, T3) -> + handle_unary_op(Rest, Line, Column, unary_op, 3, list_to_atom([T1, T2, T3]), Scope, Tokens); -tokenize(";" ++ Rest, Line, Scope, [Top|Tokens]) when element(1, Top) /= eol -> - tokenize(Rest, Line, Scope, eol(Line, ';', [Top|Tokens])); +tokenize([T1, T2, T3 | Rest], Line, Column, Scope, Tokens) when ?ellipsis_op3(T1, T2, T3) -> + handle_unary_op(Rest, Line, Column, ellipsis_op, 3, list_to_atom([T1, T2, T3]), Scope, Tokens); -tokenize("\\\n" ++ Rest, Line, Scope, Tokens) -> - tokenize(Rest, Line + 1, Scope, Tokens); +tokenize([T1, T2, T3 | Rest], Line, Column, Scope, Tokens) when ?comp_op3(T1, T2, T3) -> + handle_op(Rest, Line, Column, comp_op, 3, list_to_atom([T1, T2, T3]), Scope, Tokens); -tokenize("\\\r\n" ++ Rest, Line, Scope, Tokens) -> - tokenize(Rest, Line + 1, Scope, Tokens); +tokenize([T1, T2, T3 | Rest], Line, Column, Scope, Tokens) when ?and_op3(T1, T2, T3) -> + NewScope = maybe_warn_too_many_of_same_char([T1, T2, T3], Rest, Line, Column, Scope), + handle_op(Rest, Line, Column, and_op, 3, list_to_atom([T1, T2, T3]), NewScope, Tokens); -tokenize("\n" ++ Rest, Line, Scope, Tokens) -> - tokenize(Rest, Line + 1, Scope, eol(Line, newline, Tokens)); +tokenize([T1, T2, T3 | Rest], Line, Column, Scope, Tokens) when ?or_op3(T1, T2, T3) -> + NewScope = maybe_warn_too_many_of_same_char([T1, T2, T3], Rest, Line, Column, Scope), + handle_op(Rest, Line, Column, or_op, 3, list_to_atom([T1, T2, T3]), NewScope, Tokens); -tokenize("\r\n" ++ Rest, Line, Scope, Tokens) -> - tokenize(Rest, Line + 1, Scope, eol(Line, newline, Tokens)); +tokenize([T1, T2, T3 | Rest], Line, Column, Scope, Tokens) when ?xor_op3(T1, T2, T3) -> + NewScope = maybe_warn_too_many_of_same_char([T1, T2, T3], Rest, Line, Column, Scope), + handle_op(Rest, Line, Column, xor_op, 3, list_to_atom([T1, T2, T3]), NewScope, Tokens); -% Stand-alone tokens +tokenize([T1, T2, T3 | Rest], Line, Column, Scope, Tokens) when ?concat_op3(T1, T2, T3) -> + NewScope = maybe_warn_too_many_of_same_char([T1, T2, T3], Rest, Line, Column, Scope), + handle_op(Rest, Line, Column, concat_op, 3, list_to_atom([T1, T2, T3]), NewScope, Tokens); -tokenize("..." ++ Rest, Line, Scope, Tokens) -> - Token = check_call_identifier(identifier, Line, '...', Rest), - tokenize(Rest, Line, Scope, [Token|Tokens]); +tokenize([T1, T2, T3 | Rest], Line, Column, Scope, Tokens) when ?arrow_op3(T1, T2, T3) -> + handle_op(Rest, Line, Column, arrow_op, 3, list_to_atom([T1, T2, T3]), Scope, Tokens); -tokenize("=>" ++ Rest, Line, Scope, Tokens) -> - tokenize(Rest, Line, Scope, add_token_with_nl({assoc_op, Line, '=>'}, Tokens)); +% ## Containers + punctuation tokens +tokenize([$, | Rest], Line, Column, Scope, Tokens) -> + Token = {',', {Line, Column, 0}}, + tokenize(Rest, Line, Column + 1, Scope, [Token | Tokens]); + +tokenize([$<, $< | Rest], Line, Column, Scope, Tokens) -> + Token = {'<<', {Line, Column, nil}}, + handle_terminator(Rest, Line, Column + 2, Scope, Token, Tokens); + +tokenize([$>, $> | Rest], Line, Column, Scope, Tokens) -> + Token = {'>>', {Line, Column, previous_was_eol(Tokens)}}, + handle_terminator(Rest, Line, Column + 2, Scope, Token, Tokens); + +tokenize([${ | Rest], Line, Column, Scope, [{'%', _} | _] = Tokens) -> + Message = + "unexpected space between % and {\n\n" + "If you want to define a map, write %{...}, with no spaces.\n" + "If you want to define a struct, write %StructName{...}.\n\n" + "Syntax error before: ", + error({?LOC(Line, Column), Message, [${]}, Rest, Scope, Tokens); + +tokenize([T | Rest], Line, Column, Scope, Tokens) when T =:= $(; T =:= ${; T =:= $[ -> + Token = {list_to_atom([T]), {Line, Column, nil}}, + handle_terminator(Rest, Line, Column + 1, Scope, Token, Tokens); + +tokenize([T | Rest], Line, Column, Scope, Tokens) when T =:= $); T =:= $}; T =:= $] -> + Token = {list_to_atom([T]), {Line, Column, previous_was_eol(Tokens)}}, + handle_terminator(Rest, Line, Column + 1, Scope, Token, Tokens); -% ## Three token operators -tokenize([T1,T2,T3|Rest], Line, Scope, Tokens) when ?unary_op3(T1, T2, T3) -> - handle_unary_op(Rest, Line, unary_op, list_to_atom([T1,T2,T3]), Scope, Tokens); +% ## Two Token Operators +tokenize([T1, T2 | Rest], Line, Column, Scope, Tokens) when ?ternary_op(T1, T2) -> + Op = list_to_atom([T1, T2]), + Token = {ternary_op, {Line, Column, previous_was_eol(Tokens)}, Op}, + tokenize(Rest, Line, Column + 2, Scope, add_token_with_eol(Token, Tokens)); -tokenize([T1,T2,T3|Rest], Line, Scope, Tokens) when ?comp_op3(T1, T2, T3) -> - handle_op(Rest, Line, comp_op, list_to_atom([T1,T2,T3]), Scope, Tokens); +tokenize([T1, T2 | Rest], Line, Column, Scope, Tokens) when ?power_op(T1, T2) -> + handle_op(Rest, Line, Column, power_op, 2, list_to_atom([T1, T2]), Scope, Tokens); -tokenize([T1,T2,T3|Rest], Line, Scope, Tokens) when ?and_op3(T1, T2, T3) -> - handle_op(Rest, Line, and_op, list_to_atom([T1,T2,T3]), Scope, Tokens); +tokenize([T1, T2 | Rest], Line, Column, Scope, Tokens) when ?range_op(T1, T2) -> + handle_op(Rest, Line, Column, range_op, 2, list_to_atom([T1, T2]), Scope, Tokens); -tokenize([T1,T2,T3|Rest], Line, Scope, Tokens) when ?or_op3(T1, T2, T3) -> - handle_op(Rest, Line, or_op, list_to_atom([T1,T2,T3]), Scope, Tokens); +tokenize([T1, T2 | Rest], Line, Column, Scope, Tokens) when ?concat_op(T1, T2) -> + handle_op(Rest, Line, Column, concat_op, 2, list_to_atom([T1, T2]), Scope, Tokens); -tokenize([T1,T2,T3|Rest], Line, Scope, Tokens) when ?arrow_op3(T1, T2, T3) -> - handle_op(Rest, Line, arrow_op, list_to_atom([T1,T2,T3]), Scope, Tokens); +tokenize([T1, T2 | Rest], Line, Column, Scope, Tokens) when ?arrow_op(T1, T2) -> + handle_op(Rest, Line, Column, arrow_op, 2, list_to_atom([T1, T2]), Scope, Tokens); -tokenize([T1,T2,T3|Rest], Line, Scope, Tokens) when ?hat_op3(T1, T2, T3) -> - handle_op(Rest, Line, hat_op, list_to_atom([T1,T2,T3]), Scope, Tokens); +tokenize([T1, T2 | Rest], Line, Column, Scope, Tokens) when ?comp_op2(T1, T2) -> + handle_op(Rest, Line, Column, comp_op, 2, list_to_atom([T1, T2]), Scope, Tokens); -% ## Containers + punctuation tokens -tokenize([T,T|Rest], Line, Scope, Tokens) when T == $<; T == $> -> - Token = {list_to_atom([T,T]), Line}, - handle_terminator(Rest, Line, Scope, Token, Tokens); +tokenize([T1, T2 | Rest], Line, Column, Scope, Tokens) when ?rel_op2(T1, T2) -> + handle_op(Rest, Line, Column, rel_op, 2, list_to_atom([T1, T2]), Scope, Tokens); -tokenize([T|Rest], Line, Scope, Tokens) when T == $(; - T == ${; T == $}; T == $[; T == $]; T == $); T == $, -> - Token = {list_to_atom([T]), Line}, - handle_terminator(Rest, Line, Scope, Token, Tokens); +tokenize([T1, T2 | Rest], Line, Column, Scope, Tokens) when ?and_op(T1, T2) -> + handle_op(Rest, Line, Column, and_op, 2, list_to_atom([T1, T2]), Scope, Tokens); -% ## Two Token Operators -tokenize([T1,T2|Rest], Line, Scope, Tokens) when ?two_op(T1, T2) -> - handle_op(Rest, Line, two_op, list_to_atom([T1, T2]), Scope, Tokens); +tokenize([T1, T2 | Rest], Line, Column, Scope, Tokens) when ?or_op(T1, T2) -> + handle_op(Rest, Line, Column, or_op, 2, list_to_atom([T1, T2]), Scope, Tokens); -tokenize([T1,T2|Rest], Line, Scope, Tokens) when ?arrow_op(T1, T2) -> - handle_op(Rest, Line, arrow_op, list_to_atom([T1, T2]), Scope, Tokens); +tokenize([T1, T2 | Rest], Line, Column, Scope, Tokens) when ?in_match_op(T1, T2) -> + handle_op(Rest, Line, Column, in_match_op, 2, list_to_atom([T1, T2]), Scope, Tokens); -tokenize([T1,T2|Rest], Line, Scope, Tokens) when ?comp_op2(T1, T2) -> - handle_op(Rest, Line, comp_op, list_to_atom([T1, T2]), Scope, Tokens); +tokenize([T1, T2 | Rest], Line, Column, Scope, Tokens) when ?type_op(T1, T2) -> + handle_op(Rest, Line, Column, type_op, 2, list_to_atom([T1, T2]), Scope, Tokens); -tokenize([T1,T2|Rest], Line, Scope, Tokens) when ?rel_op2(T1, T2) -> - handle_op(Rest, Line, rel_op, list_to_atom([T1, T2]), Scope, Tokens); +tokenize([T1, T2 | Rest], Line, Column, Scope, Tokens) when ?stab_op(T1, T2) -> + handle_op(Rest, Line, Column, stab_op, 2, list_to_atom([T1, T2]), Scope, Tokens); -tokenize([T1,T2|Rest], Line, Scope, Tokens) when ?and_op(T1, T2) -> - handle_op(Rest, Line, and_op, list_to_atom([T1, T2]), Scope, Tokens); +% ## Single Token Operators -tokenize([T1,T2|Rest], Line, Scope, Tokens) when ?or_op(T1, T2) -> - handle_op(Rest, Line, or_op, list_to_atom([T1, T2]), Scope, Tokens); +tokenize([$& | Rest], Line, Column, Scope, Tokens) -> + Kind = + case strip_horizontal_space(Rest, Line, 0, Scope) of + {[Int | _], Line, 0} when ?is_digit(Int) -> + capture_int; -tokenize([T1,T2|Rest], Line, Scope, Tokens) when ?in_match_op(T1, T2) -> - handle_op(Rest, Line, in_match_op, list_to_atom([T1, T2]), Scope, Tokens); + {[$/ | NewRest], _, _} -> + case strip_horizontal_space(NewRest, Line, 0, Scope) of + {[$/ | _], _, _} -> capture_op; + {_, _, _} -> identifier + end; -tokenize([T1,T2|Rest], Line, Scope, Tokens) when ?type_op(T1, T2) -> - handle_op(Rest, Line, type_op, list_to_atom([T1, T2]), Scope, Tokens); + {_, _, _} -> + capture_op + end, -tokenize([T1,T2|Rest], Line, Scope, Tokens) when ?stab_op(T1, T2) -> - handle_op(Rest, Line, stab_op, list_to_atom([T1, T2]), Scope, Tokens); + Token = {Kind, {Line, Column, nil}, '&'}, + tokenize(Rest, Line, Column + 1, Scope, [Token | Tokens]); -% ## Single Token Operators +tokenize([T | Rest], Line, Column, Scope, Tokens) when ?at_op(T) -> + handle_unary_op(Rest, Line, Column, at_op, 1, list_to_atom([T]), Scope, Tokens); -tokenize([T|Rest], Line, Scope, Tokens) when ?at_op(T) -> - handle_unary_op(Rest, Line, at_op, list_to_atom([T]), Scope, Tokens); +tokenize([T | Rest], Line, Column, Scope, Tokens) when ?unary_op(T) -> + handle_unary_op(Rest, Line, Column, unary_op, 1, list_to_atom([T]), Scope, Tokens); -tokenize([T|Rest], Line, Scope, Tokens) when ?capture_op(T) -> - handle_unary_op(Rest, Line, capture_op, list_to_atom([T]), Scope, Tokens); +tokenize([T | Rest], Line, Column, Scope, Tokens) when ?rel_op(T) -> + handle_op(Rest, Line, Column, rel_op, 1, list_to_atom([T]), Scope, Tokens); -tokenize([T|Rest], Line, Scope, Tokens) when ?unary_op(T) -> - handle_unary_op(Rest, Line, unary_op, list_to_atom([T]), Scope, Tokens); +tokenize([T | Rest], Line, Column, Scope, Tokens) when ?dual_op(T) -> + handle_unary_op(Rest, Line, Column, dual_op, 1, list_to_atom([T]), Scope, Tokens); -tokenize([T|Rest], Line, Scope, Tokens) when ?rel_op(T) -> - handle_op(Rest, Line, rel_op, list_to_atom([T]), Scope, Tokens); +tokenize([T | Rest], Line, Column, Scope, Tokens) when ?mult_op(T) -> + handle_op(Rest, Line, Column, mult_op, 1, list_to_atom([T]), Scope, Tokens); -tokenize([T|Rest], Line, Scope, Tokens) when ?dual_op(T) -> - handle_unary_op(Rest, Line, dual_op, list_to_atom([T]), Scope, Tokens); +tokenize([T | Rest], Line, Column, Scope, Tokens) when ?match_op(T) -> + handle_op(Rest, Line, Column, match_op, 1, list_to_atom([T]), Scope, Tokens); -tokenize([T|Rest], Line, Scope, Tokens) when ?mult_op(T) -> - handle_op(Rest, Line, mult_op, list_to_atom([T]), Scope, Tokens); +tokenize([T | Rest], Line, Column, Scope, Tokens) when ?pipe_op(T) -> + handle_op(Rest, Line, Column, pipe_op, 1, list_to_atom([T]), Scope, Tokens); -tokenize([T|Rest], Line, Scope, Tokens) when ?match_op(T) -> - handle_op(Rest, Line, match_op, list_to_atom([T]), Scope, Tokens); +% Non-operator Atoms -tokenize([T|Rest], Line, Scope, Tokens) when ?pipe_op(T) -> - handle_op(Rest, Line, pipe_op, list_to_atom([T]), Scope, Tokens); +tokenize([$:, H | T] = Original, Line, Column, BaseScope, Tokens) when ?is_quote(H) -> + Scope = case H == $' of + true -> + prepend_warning(Line, Column, "single quotes around atoms are deprecated. Use double quotes instead", BaseScope); -% Others + false -> + BaseScope + end, + + case elixir_interpolation:extract(Line, Column + 2, Scope, true, T, H) of + {NewLine, NewColumn, Parts, Rest, InterScope} -> + NewScope = case is_unnecessary_quote(Parts, InterScope) of + true -> + WarnMsg = io_lib:format( + "found quoted atom \"~ts\" but the quotes are not required. " + "Atoms made exclusively of ASCII letters, numbers, underscores, " + "beginning with a letter or underscore, and optionally ending with ! or ? " + "do not require quotes", + [hd(Parts)] + ), + prepend_warning(Line, Column, WarnMsg, InterScope); + + false -> + InterScope + end, -tokenize([$%|T], Line, Scope, Tokens) -> - case strip_space(T, 0) of - {[${|_] = Rest, Counter} -> tokenize(Rest, Line + Counter, Scope, [{'%{}', Line}|Tokens]); - {Rest, Counter} -> tokenize(Rest, Line + Counter, Scope, [{'%', Line}|Tokens]) + case unescape_tokens(Parts, Line, Column, NewScope) of + {ok, [Part]} when is_binary(Part) -> + case unsafe_to_atom(Part, Line, Column, Scope) of + {ok, Atom} -> + Token = {atom_quoted, {Line, Column, H}, Atom}, + tokenize(Rest, NewLine, NewColumn, NewScope, [Token | Tokens]); + + {error, Reason} -> + error(Reason, Rest, NewScope, Tokens) + end; + + {ok, Unescaped} -> + Key = case Scope#elixir_tokenizer.existing_atoms_only of + true -> atom_safe; + false -> atom_unsafe + end, + Token = {Key, {Line, Column, H}, Unescaped}, + tokenize(Rest, NewLine, NewColumn, NewScope, [Token | Tokens]); + + {error, Reason} -> + error(Reason, Rest, NewScope, Tokens) + end; + + {error, Reason} -> + Message = " (for atom starting at line ~B)", + interpolation_error(Reason, Original, Scope, Tokens, Message, [Line], Line, Column + 1, [H], [H]) end; -tokenize([$.|T], Line, Scope, Tokens) -> - {Rest, Counter} = strip_space(T, 0), - handle_dot([$.|Rest], Line + Counter, Scope, Tokens); +tokenize([$: | String] = Original, Line, Column, Scope, Tokens) -> + case tokenize_identifier(String, Line, Column, Scope, false) of + {_Kind, Unencoded, Atom, Rest, Length, Ascii, _Special} -> + NewScope = maybe_warn_for_ambiguous_bang_before_equals(atom, Unencoded, Rest, Line, Column, Scope), + TrackedScope = track_ascii(Ascii, NewScope), + Token = {atom, {Line, Column, Unencoded}, Atom}, + tokenize(Rest, Line, Column + 1 + Length, TrackedScope, [Token | Tokens]); + empty when Scope#elixir_tokenizer.cursor_completion == false -> + unexpected_token(Original, Line, Column, Scope, Tokens); + empty -> + tokenize([], Line, Column, Scope, Tokens); + {unexpected_token, Length} -> + unexpected_token(lists:nthtail(Length - 1, String), Line, Column + Length - 1, Scope, Tokens); + {error, Reason} -> + error(Reason, Original, Scope, Tokens) + end; % Integers and floats +% We use int and flt otherwise elixir_parser won't format them +% properly in case of errors. + +tokenize([H | T], Line, Column, Scope, Tokens) when ?is_digit(H) -> + case tokenize_number(T, [H], 1, false) of + {error, Reason, Original} -> + error({?LOC(Line, Column), Reason, Original}, T, Scope, Tokens); + {[I | Rest], Number, Original, _Length} when ?is_upcase(I); ?is_downcase(I); I == $_ -> + if + Number == 0, (I =:= $x) orelse (I =:= $o) orelse (I =:= $b), Rest == [], + Scope#elixir_tokenizer.cursor_completion /= false -> + tokenize([], Line, Column, Scope, Tokens); + + true -> + Msg = + io_lib:format( + "invalid character \"~ts\" after number ~ts. If you intended to write a number, " + "make sure to separate the number from the character (using comma, space, etc). " + "If you meant to write a function name or a variable, note that identifiers in " + "Elixir cannot start with numbers. Unexpected token: ", + [[I], Original] + ), + + error({?LOC(Line, Column), Msg, [I]}, T, Scope, Tokens) + end; + {Rest, Number, Original, Length} when is_integer(Number) -> + Token = {int, {Line, Column, Number}, Original}, + tokenize(Rest, Line, Column + Length, Scope, [Token | Tokens]); + {Rest, Number, Original, Length} -> + Token = {flt, {Line, Column, Number}, Original}, + tokenize(Rest, Line, Column + Length, Scope, [Token | Tokens]) + end; -tokenize([H|_] = String, Line, Scope, Tokens) when ?is_digit(H) -> - {Rest, Number} = tokenize_number(String, [], false), - tokenize(Rest, Line, Scope, [{number, Line, Number}|Tokens]); +% Spaces -% Aliases +tokenize([T | Rest], Line, Column, Scope, Tokens) when ?is_horizontal_space(T) -> + {Remaining, NewLine, NewColumn} = strip_horizontal_space(Rest, Line, Column + 1, Scope), + handle_space_sensitive_tokens(Remaining, NewLine, NewColumn, Scope, Tokens); + +% End of line + +tokenize(";" ++ Rest, Line, Column, Scope, []) -> + tokenize(Rest, Line, Column + 1, Scope, [{';', {Line, Column, 0}}]); + +tokenize(";" ++ Rest, Line, Column, Scope, [Top | _] = Tokens) when element(1, Top) /= ';' -> + tokenize(Rest, Line, Column + 1, Scope, [{';', {Line, Column, 0}} | Tokens]); + +tokenize("\\" = Original, Line, Column, Scope, Tokens) -> + error({?LOC(Line, Column), "invalid escape \\ at end of file", []}, Original, Scope, Tokens); + +tokenize("\\\n" = Original, Line, Column, Scope, Tokens) -> + error({?LOC(Line, Column), "invalid escape \\ at end of file", []}, Original, Scope, Tokens); + +tokenize("\\\r\n" = Original, Line, Column, Scope, Tokens) -> + error({?LOC(Line, Column), "invalid escape \\ at end of file", []}, Original, Scope, Tokens); + +tokenize("\\\n" ++ Rest, Line, _Column, Scope, Tokens) -> + tokenize_eol(Rest, Line, Scope, Tokens); + +tokenize("\\\r\n" ++ Rest, Line, _Column, Scope, Tokens) -> + tokenize_eol(Rest, Line, Scope, Tokens); + +tokenize("\n" ++ Rest, Line, Column, Scope, Tokens) -> + tokenize_eol(Rest, Line, Scope, eol(Line, Column, Tokens)); + +tokenize("\r\n" ++ Rest, Line, Column, Scope, Tokens) -> + tokenize_eol(Rest, Line, Scope, eol(Line, Column, Tokens)); + +% Others + +tokenize([$%, $( | Rest], Line, Column, Scope, Tokens) -> + Reason = {?LOC(Line, Column), "expected %{ to define a map, got: ", [$%, $(]}, + error(Reason, Rest, Scope, Tokens); + +tokenize([$%, $[ | Rest], Line, Column, Scope, Tokens) -> + Reason = {?LOC(Line, Column), "expected %{ to define a map, got: ", [$%, $[]}, + error(Reason, Rest, Scope, Tokens); + +tokenize([$%, ${ | T], Line, Column, Scope, Tokens) -> + Token = {'{', {Line, Column, nil}}, + handle_terminator(T, Line, Column + 2, Scope, Token, [{'%{}', {Line, Column, nil}} | Tokens]); + +tokenize([$% | T], Line, Column, Scope, Tokens) -> + tokenize(T, Line, Column + 1, Scope, [{'%', {Line, Column, nil}} | Tokens]); + +tokenize([$. | T], Line, Column, Scope, Tokens) -> + tokenize_dot(T, Line, Column + 1, {Line, Column, nil}, Scope, Tokens); + +% Identifiers + +tokenize(String, Line, Column, OriginalScope, Tokens) -> + case tokenize_identifier(String, Line, Column, OriginalScope, not previous_was_dot(Tokens)) of + {Kind, Unencoded, Atom, Rest, Length, Ascii, Special} -> + HasAt = lists:member(at, Special), + Scope = track_ascii(Ascii, OriginalScope), -tokenize([H|_] = Original, Line, Scope, Tokens) when ?is_upcase(H) -> - {Rest, Alias} = tokenize_identifier(Original, []), - case unsafe_to_atom(Alias, Line, Scope) of - {ok, Atom} -> case Rest of - [$:|T] when ?is_space(hd(T)) -> - tokenize(T, Line, Scope, [{kw_identifier, Line, Atom}|Tokens]); + [$: | T] when ?is_space(hd(T)) -> + Token = {kw_identifier, {Line, Column, Unencoded}, Atom}, + tokenize(T, Line, Column + Length + 1, Scope, [Token | Tokens]); + + [$: | T] when hd(T) =/= $: -> + AtomName = atom_to_list(Atom) ++ [$:], + Reason = {?LOC(Line, Column), "keyword argument must be followed by space after: ", AtomName}, + error(Reason, String, Scope, Tokens); + + _ when HasAt -> + Reason = {?LOC(Line, Column), invalid_character_error(Kind, $@), atom_to_list(Atom)}, + error(Reason, String, Scope, Tokens); + + _ when Atom == '__aliases__'; Atom == '__block__' -> + error({?LOC(Line, Column), "reserved token: ", atom_to_list(Atom)}, Rest, Scope, Tokens); + + _ when Kind == alias -> + tokenize_alias(Rest, Line, Column, Unencoded, Atom, Length, Ascii, Special, Scope, Tokens); + + _ when Kind == identifier -> + NewScope = maybe_warn_for_ambiguous_bang_before_equals(identifier, Unencoded, Rest, Line, Column, Scope), + Token = check_call_identifier(Line, Column, Unencoded, Atom, Rest), + tokenize(Rest, Line, Column + Length, NewScope, [Token | Tokens]); + _ -> - tokenize(Rest, Line, Scope, [{aliases, Line, [Atom]}|Tokens]) + unexpected_token(String, Line, Column, Scope, Tokens) end; - {error, Reason} -> - {error, Reason, Original, Tokens} - end; - -% Identifier -tokenize([H|_] = String, Line, Scope, Tokens) when ?is_downcase(H); H == $_ -> - case tokenize_any_identifier(String, Line, Scope, Tokens) of - {keyword, Rest, Check, T} -> - handle_terminator(Rest, Line, Scope, Check, T); - {identifier, Rest, Token} -> - tokenize(Rest, Line, Scope, [Token|Tokens]); - {error, _, _, _} = Error -> - Error - end; + {keyword, Atom, Type, Rest, Length} -> + tokenize_keyword(Type, Rest, Line, Column, Atom, Length, OriginalScope, Tokens); -% Ambiguous unary/binary operators tokens + empty when OriginalScope#elixir_tokenizer.cursor_completion == false -> + unexpected_token(String, Line, Column, OriginalScope, Tokens); -tokenize([Space, Sign, NotMarker|T], Line, Scope, [{Identifier, _, _} = H|Tokens]) when - ?dual_op(Sign), - ?is_horizontal_space(Space), - not(?is_space(NotMarker)), - NotMarker /= $(, NotMarker /= $[, NotMarker /= $<, NotMarker /= ${, %% containers - NotMarker /= $%, NotMarker /= $+, NotMarker /= $-, NotMarker /= $/, NotMarker /= $>, %% operators - Identifier == identifier -> - Rest = [NotMarker|T], - tokenize(Rest, Line, Scope, [{dual_op, Line, list_to_atom([Sign])}, setelement(1, H, op_identifier)|Tokens]); + empty -> + case String of + [$~, L] when ?is_upcase(L); ?is_downcase(L) -> tokenize([], Line, Column, OriginalScope, Tokens); + [$~] -> tokenize([], Line, Column, OriginalScope, Tokens); + _ -> unexpected_token(String, Line, Column, OriginalScope, Tokens) + end; -% Spaces + {unexpected_token, Length} -> + unexpected_token(lists:nthtail(Length - 1, String), Line, Column + Length - 1, OriginalScope, Tokens); -tokenize([T|Rest], Line, Scope, Tokens) when ?is_horizontal_space(T) -> - tokenize(strip_horizontal_space(Rest), Line, Scope, Tokens); -tokenize(T, Line, _Scope, Tokens) -> - {error, {Line, "invalid token: ", until_eol(T)}, T, Tokens}. - -strip_horizontal_space([H|T]) when ?is_horizontal_space(H) -> - strip_horizontal_space(T); -strip_horizontal_space(T) -> - T. - -strip_space(T, Counter) -> - case strip_horizontal_space(T) of - "\r\n" ++ Rest -> strip_space(Rest, Counter + 1); - "\n" ++ Rest -> strip_space(Rest, Counter + 1); - Rest -> {Rest, Counter} + {error, Reason} -> + error(Reason, String, OriginalScope, Tokens) end. -until_eol("\r\n" ++ _) -> []; -until_eol("\n" ++ _) -> []; -until_eol([]) -> []; -until_eol([H|T]) -> [H|until_eol(T)]. +previous_was_dot([{'.', _} | _]) -> true; +previous_was_dot(_) -> false. + +unexpected_token([T | Rest], Line, Column, Scope, Tokens) -> + Message = + case handle_char(T) of + {_Escaped, Explanation} -> + io_lib:format("~ts (column ~p, code point U+~4.16.0B)", [Explanation, Column, T]); + false -> + io_lib:format("\"~ts\" (column ~p, code point U+~4.16.0B)", [[T], Column, T]) + end, + error({?LOC(Line, Column), "unexpected token: ", Message}, Rest, Scope, Tokens). + +tokenize_eol(Rest, Line, Scope, Tokens) -> + {StrippedRest, NewLine, NewColumn} = + strip_horizontal_space(Rest, Line + 1, Scope#elixir_tokenizer.column, Scope), + IndentedScope = Scope#elixir_tokenizer{indentation=NewColumn-1}, + tokenize(StrippedRest, NewLine, NewColumn, IndentedScope, Tokens). + +strip_horizontal_space([H | T], Line, Counter, Scope) when ?is_horizontal_space(H) -> + strip_horizontal_space(T, Line, Counter + 1, Scope); +%% \\ at the end of lines is treated as horizontal whitespace +%% except at the very end of the buffer, which we treat as incomplete +strip_horizontal_space("\\\n" ++ T, Line, _Counter, Scope) when T /= [] -> + strip_horizontal_space(T, Line+1, Scope#elixir_tokenizer.column, Scope); +strip_horizontal_space("\\\r\n" ++ T, Line, _Counter, Scope) when T /= [] -> + strip_horizontal_space(T, Line+1, Scope#elixir_tokenizer.column, Scope); +strip_horizontal_space(T, Line, Counter, _Scope) -> + {T, Line, Counter}. + +tokenize_dot(T, Line, Column, DotInfo, Scope, Tokens) -> + case strip_horizontal_space(T, Line, Column, Scope) of + {[$# | R], NewLine, NewColumn} -> + case tokenize_comment(R, [$#]) of + {error, Char, Reason} -> + error_comment(Char, Reason, [$# | R], NewLine, NewColumn, Scope, Tokens); + + {Rest, Comment} -> + preserve_comments(NewLine, NewColumn, Tokens, Comment, Rest, Scope), + tokenize_dot(Rest, NewLine, Scope#elixir_tokenizer.column, DotInfo, Scope, Tokens) + end; + {"\r\n" ++ Rest, NewLine, _NewColumn} -> + tokenize_dot(Rest, NewLine + 1, Scope#elixir_tokenizer.column, DotInfo, Scope, Tokens); + {"\n" ++ Rest, NewLine, _NewColumn} -> + tokenize_dot(Rest, NewLine + 1, Scope#elixir_tokenizer.column, DotInfo, Scope, Tokens); + {Rest, NewLine, NewColumn} -> + handle_dot([$. | Rest], NewLine, NewColumn, DotInfo, Scope, Tokens) + end. -escape_char(List) -> - << Char/utf8 >> = elixir_interpolation:unescape_chars(list_to_binary(List)), - Char. +handle_char(0) -> {"\\0", "null byte"}; +handle_char(7) -> {"\\a", "alert"}; +handle_char($\b) -> {"\\b", "backspace"}; +handle_char($\d) -> {"\\d", "delete"}; +handle_char($\e) -> {"\\e", "escape"}; +handle_char($\f) -> {"\\f", "form feed"}; +handle_char($\n) -> {"\\n", "newline"}; +handle_char($\r) -> {"\\r", "carriage return"}; +handle_char($\s) -> {"\\s", "space"}; +handle_char($\t) -> {"\\t", "tab"}; +handle_char($\v) -> {"\\v", "vertical tab"}; +handle_char(_) -> false. %% Handlers -handle_heredocs(T, Line, H, Scope, Tokens) -> - case extract_heredoc_with_interpolation(Line, Scope, true, T, H) of - {ok, NewLine, Parts, Rest} -> - Token = {string_type(H), Line, unescape_tokens(Parts)}, - tokenize(Rest, NewLine, Scope, [Token|Tokens]); +handle_heredocs(T, Line, Column, H, Scope, Tokens) -> + case extract_heredoc_with_interpolation(Line, Column, Scope, true, T, H) of + {ok, NewLine, NewColumn, Parts, Rest, NewScope} -> + case unescape_tokens(Parts, Line, Column, NewScope) of + {ok, Unescaped} -> + Token = {heredoc_type(H), {Line, Column, nil}, NewColumn - 4, Unescaped}, + tokenize(Rest, NewLine, NewColumn, NewScope, [Token | Tokens]); + + {error, Reason} -> + error(Reason, Rest, Scope, Tokens) + end; + {error, Reason} -> - {error, Reason, [H, H, H] ++ T, Tokens} + error(Reason, [H, H, H] ++ T, Scope, Tokens) end. -handle_strings(T, Line, H, Scope, Tokens) -> - case elixir_interpolation:extract(Line, Scope, true, T, H) of +handle_strings(T, Line, Column, H, Scope, Tokens) -> + case elixir_interpolation:extract(Line, Column, Scope, true, T, H) of {error, Reason} -> - interpolation_error(Reason, [H|T], Tokens, " (for string starting at line ~B)", [Line]); - {NewLine, Parts, [$:|Rest]} when ?is_space(hd(Rest)) -> - Unescaped = unescape_tokens(Parts), - Key = case Scope#elixir_tokenizer.existing_atoms_only of - true -> kw_identifier_safe; - false -> kw_identifier_unsafe + interpolation_error(Reason, [H | T], Scope, Tokens, " (for string starting at line ~B)", [Line], Line, Column-1, [H], [H]); + + {NewLine, NewColumn, Parts, [$: | Rest], InterScope} when ?is_space(hd(Rest)) -> + NewScope = case is_unnecessary_quote(Parts, InterScope) of + true -> + WarnMsg = io_lib:format( + "found quoted keyword \"~ts\" but the quotes are not required. " + "Note that keywords are always atoms, even when quoted. " + "Similar to atoms, keywords made exclusively of ASCII " + "letters, numbers, and underscores and not beginning with a " + "number do not require quotes", + [hd(Parts)] + ), + prepend_warning(Line, Column-1, WarnMsg, InterScope); + + false when H =:= $' -> + WarnMsg = "single quotes around keywords are deprecated. Use double quotes instead", + prepend_warning(Line, Column-1, WarnMsg, InterScope); + + false -> + InterScope end, - tokenize(Rest, NewLine, Scope, [{Key, Line, Unescaped}|Tokens]); - {NewLine, Parts, Rest} -> - Token = {string_type(H), Line, unescape_tokens(Parts)}, - tokenize(Rest, NewLine, Scope, [Token|Tokens]) - end. -handle_unary_op([$:|Rest], Line, _Kind, Op, Scope, Tokens) when ?is_space(hd(Rest)) -> - tokenize(Rest, Line, Scope, [{kw_identifier, Line, Op}|Tokens]); + case unescape_tokens(Parts, Line, Column, NewScope) of + {ok, [Part]} when is_binary(Part) -> + case unsafe_to_atom(Part, Line, Column - 1, Scope) of + {ok, Atom} -> + Token = {kw_identifier, {Line, Column - 1, H}, Atom}, + tokenize(Rest, NewLine, NewColumn + 1, NewScope, [Token | Tokens]); + {error, Reason} -> + error(Reason, Rest, NewScope, Tokens) + end; + + {ok, Unescaped} -> + Key = case Scope#elixir_tokenizer.existing_atoms_only of + true -> kw_identifier_safe; + false -> kw_identifier_unsafe + end, + Token = {Key, {Line, Column - 1, H}, Unescaped}, + tokenize(Rest, NewLine, NewColumn + 1, NewScope, [Token | Tokens]); -handle_unary_op(Rest, Line, Kind, Op, Scope, Tokens) -> - case strip_horizontal_space(Rest) of - [$/|_] -> tokenize(Rest, Line, Scope, [{identifier, Line, Op}|Tokens]); - _ -> tokenize(Rest, Line, Scope, [{Kind, Line, Op}|Tokens]) + {error, Reason} -> + error(Reason, Rest, NewScope, Tokens) + end; + + {NewLine, NewColumn, Parts, Rest, InterScope} -> + NewScope = + case H of + $' -> + Message = "using single-quoted strings to represent charlists is deprecated.\n" + "Use ~c\"\" if you indeed want a charlist or use \"\" instead.\n" + "You may run \"mix format --migrate\" to change all single-quoted\n" + "strings to use the ~c sigil and fix this warning.", + prepend_warning(Line, Column-1, Message, InterScope); + + _ -> + InterScope + end, + + case unescape_tokens(Parts, Line, Column, NewScope) of + {ok, Unescaped} -> + Token = {string_type(H), {Line, Column - 1, nil}, Unescaped}, + tokenize(Rest, NewLine, NewColumn, NewScope, [Token | Tokens]); + + {error, Reason} -> + error(Reason, Rest, NewScope, Tokens) + end end. -handle_op([$:|Rest], Line, _Kind, Op, Scope, Tokens) when ?is_space(hd(Rest)) -> - tokenize(Rest, Line, Scope, [{kw_identifier, Line, Op}|Tokens]); +handle_unary_op([$: | Rest], Line, Column, _Kind, Length, Op, Scope, Tokens) when ?is_space(hd(Rest)) -> + Token = {kw_identifier, {Line, Column, nil}, Op}, + tokenize(Rest, Line, Column + Length + 1, Scope, [Token | Tokens]); + +handle_unary_op(Rest, Line, Column, Kind, Length, Op, Scope, Tokens) -> + case strip_horizontal_space(Rest, Line, Column + Length, Scope) of + {[$/ | _] = Remaining, NewLine, NewColumn} -> + Token = {identifier, {Line, Column, nil}, Op}, + tokenize(Remaining, NewLine, NewColumn, Scope, [Token | Tokens]); + {Remaining, NewLine, NewColumn} -> + NewScope = + %% TODO: Remove these deprecations on Elixir v2.0 + case Op of + '~~~' -> + Msg = "~~~ is deprecated. Use Bitwise.bnot/1 instead for clarity", + prepend_warning(Line, Column, Msg, Scope); + _ -> + Scope + end, + + Token = {Kind, {Line, Column, nil}, Op}, + tokenize(Remaining, NewLine, NewColumn, NewScope, [Token | Tokens]) + end. -handle_op(Rest, Line, Kind, Op, Scope, Tokens) -> - case strip_horizontal_space(Rest) of - [$/|_] -> tokenize(Rest, Line, Scope, [{identifier, Line, Op}|Tokens]); - _ -> tokenize(Rest, Line, Scope, add_token_with_nl({Kind, Line, Op}, Tokens)) +handle_op([$: | Rest], Line, Column, _Kind, Length, Op, Scope, Tokens) when ?is_space(hd(Rest)) -> + Token = {kw_identifier, {Line, Column, nil}, Op}, + tokenize(Rest, Line, Column + Length + 1, Scope, [Token | Tokens]); + +handle_op(Rest, Line, Column, Kind, Length, Op, Scope, Tokens) -> + case strip_horizontal_space(Rest, Line, Column + Length, Scope) of + {[$/ | _] = Remaining, NewLine, NewColumn} -> + Token = {identifier, {Line, Column, nil}, Op}, + tokenize(Remaining, NewLine, NewColumn, Scope, [Token | Tokens]); + {Remaining, NewLine, NewColumn} -> + NewScope = + %% TODO: Remove these deprecations on Elixir v2.0 + case Op of + '^^^' -> + Msg = "^^^ is deprecated. It is typically used as xor but it has the wrong precedence, use Bitwise.bxor/2 instead", + prepend_warning(Line, Column, Msg, Scope); + + '<|>' -> + Msg = "<|> is deprecated. Use another pipe-like operator", + prepend_warning(Line, Column, Msg, Scope); + + _ -> + Scope + end, + + Token = {Kind, {Line, Column, previous_was_eol(Tokens)}, Op}, + tokenize(Remaining, NewLine, NewColumn, NewScope, add_token_with_eol(Token, Tokens)) end. % ## Three Token Operators -handle_dot([$.,T1,T2,T3|Rest], Line, Scope, Tokens) when +handle_dot([$., T1, T2, T3 | Rest], Line, Column, DotInfo, Scope, Tokens) when ?unary_op3(T1, T2, T3); ?comp_op3(T1, T2, T3); ?and_op3(T1, T2, T3); ?or_op3(T1, T2, T3); - ?arrow_op3(T1, T2, T3); ?hat_op3(T1, T2, T3) -> - handle_call_identifier(Rest, Line, list_to_atom([T1, T2, T3]), Scope, Tokens); + ?arrow_op3(T1, T2, T3); ?xor_op3(T1, T2, T3); ?concat_op3(T1, T2, T3) -> + handle_call_identifier(Rest, Line, Column, DotInfo, 3, [T1, T2, T3], Scope, Tokens); % ## Two Token Operators -handle_dot([$.,T1,T2|Rest], Line, Scope, Tokens) when +handle_dot([$., T1, T2 | Rest], Line, Column, DotInfo, Scope, Tokens) when ?comp_op2(T1, T2); ?rel_op2(T1, T2); ?and_op(T1, T2); ?or_op(T1, T2); - ?arrow_op(T1, T2); ?in_match_op(T1, T2); ?two_op(T1, T2); ?stab_op(T1, T2); + ?arrow_op(T1, T2); ?in_match_op(T1, T2); ?concat_op(T1, T2); ?power_op(T1, T2); ?type_op(T1, T2) -> - handle_call_identifier(Rest, Line, list_to_atom([T1, T2]), Scope, Tokens); + handle_call_identifier(Rest, Line, Column, DotInfo, 2, [T1, T2], Scope, Tokens); % ## Single Token Operators -handle_dot([$.,T|Rest], Line, Scope, Tokens) when +handle_dot([$., T | Rest], Line, Column, DotInfo, Scope, Tokens) when ?at_op(T); ?unary_op(T); ?capture_op(T); ?dual_op(T); ?mult_op(T); - ?rel_op(T); ?match_op(T); ?pipe_op(T); T == $% -> - handle_call_identifier(Rest, Line, list_to_atom([T]), Scope, Tokens); + ?rel_op(T); ?match_op(T); ?pipe_op(T) -> + handle_call_identifier(Rest, Line, Column, DotInfo, 1, [T], Scope, Tokens); % ## Exception for .( as it needs to be treated specially in the parser -handle_dot([$.,$(|Rest], Line, Scope, Tokens) -> - tokenize([$(|Rest], Line, Scope, add_token_with_nl({dot_call_op, Line, '.'}, Tokens)); +handle_dot([$., $( | Rest], Line, Column, DotInfo, Scope, Tokens) -> + TokensSoFar = add_token_with_eol({dot_call_op, DotInfo, '.'}, Tokens), + tokenize([$( | Rest], Line, Column, Scope, TokensSoFar); + +handle_dot([$., H | T] = Original, Line, Column, DotInfo, BaseScope, Tokens) when ?is_quote(H) -> + Scope = case H == $' of + true -> + prepend_warning(Line, Column, "single quotes around calls are deprecated. Use double quotes instead", BaseScope); + + false -> + BaseScope + end, + + case elixir_interpolation:extract(Line, Column + 1, Scope, true, T, H) of + {NewLine, NewColumn, [Part], Rest, InterScope} when is_list(Part) -> + NewScope = case is_unnecessary_quote([Part], InterScope) of + true -> + WarnMsg = io_lib:format( + "found quoted call \"~ts\" but the quotes are not required. " + "Calls made exclusively of Unicode letters, numbers, and underscores " + "and not beginning with a number do not require quotes", + [Part] + ), + prepend_warning(Line, Column, WarnMsg, InterScope); + + false -> + InterScope + end, + + case unescape_tokens([Part], Line, Column, NewScope) of + {ok, [UnescapedPart]} -> + case unsafe_to_atom(UnescapedPart, Line, Column, NewScope) of + {ok, Atom} -> + Token = check_call_identifier(Line, Column, H, Atom, Rest), + TokensSoFar = add_token_with_eol({'.', DotInfo}, Tokens), + tokenize(Rest, NewLine, NewColumn, NewScope, [Token | TokensSoFar]); + + {error, Reason} -> + error(Reason, Original, NewScope, Tokens) + end; -handle_dot([$.,H|T] = Original, Line, Scope, Tokens) when ?is_quote(H) -> - case elixir_interpolation:extract(Line, Scope, true, T, H) of - {NewLine, [Part], Rest} when is_binary(Part) -> - case unsafe_to_atom(Part, Line, Scope) of - {ok, Atom} -> - Token = check_call_identifier(identifier, Line, Atom, Rest), - tokenize(Rest, NewLine, Scope, [Token|add_token_with_nl({'.', Line}, Tokens)]); {error, Reason} -> - {error, Reason, Original, Tokens} + error(Reason, Original, NewScope, Tokens) end; + + {_NewLine, _NewColumn, _Parts, Rest, NewScope} -> + Message = "interpolation is not allowed when calling function/macro. Found interpolation in a call starting with: ", + error({?LOC(Line, Column), Message, [H]}, Rest, NewScope, Tokens); {error, Reason} -> - interpolation_error(Reason, Original, Tokens, " (for function name starting at line ~B)", [Line]) + interpolation_error(Reason, Original, Scope, Tokens, " (for function name starting at line ~B)", [Line], Line, Column, [H], [H]) end; -handle_dot([$.|Rest], Line, Scope, Tokens) -> - tokenize(Rest, Line, Scope, add_token_with_nl({'.', Line}, Tokens)). - -handle_call_identifier(Rest, Line, Op, Scope, Tokens) -> - Token = check_call_identifier(identifier, Line, Op, Rest), - tokenize(Rest, Line, Scope, [Token|add_token_with_nl({'.', Line}, Tokens)]). +handle_dot([$. | Rest], Line, Column, DotInfo, Scope, Tokens) -> + TokensSoFar = add_token_with_eol({'.', DotInfo}, Tokens), + tokenize(Rest, Line, Column, Scope, TokensSoFar). + +handle_call_identifier(Rest, Line, Column, DotInfo, Length, UnencodedOp, Scope, Tokens) -> + Token = check_call_identifier(Line, Column, UnencodedOp, list_to_atom(UnencodedOp), Rest), + TokensSoFar = add_token_with_eol({'.', DotInfo}, Tokens), + tokenize(Rest, Line, Column + Length, Scope, [Token | TokensSoFar]). + +% ## Ambiguous unary/binary operators tokens +% Keywords are not ambiguous operators +handle_space_sensitive_tokens([Sign, $:, Space | _] = String, Line, Column, Scope, Tokens) when + ?dual_op(Sign), ?is_space(Space) -> + tokenize(String, Line, Column, Scope, Tokens); + +% But everything else, except other operators, are +handle_space_sensitive_tokens([Sign, NotMarker | T], Line, Column, Scope, [{identifier, _, _} = H | Tokens]) when + ?dual_op(Sign), not(?is_space(NotMarker)), + %% Do not match ++ or -- + NotMarker =/= Sign, + %% Do not match +/2 or -/2 + NotMarker =/= $/, + %% Do not match -> + NotMarker =/= $>, + %% Do not match +\\n or -\\n (it should be treated as if a space is there) + NotMarker =/= $\\ -> + Rest = [NotMarker | T], + DualOpToken = {dual_op, {Line, Column, nil}, list_to_atom([Sign])}, + tokenize(Rest, Line, Column + 1, Scope, [DualOpToken, setelement(1, H, op_identifier) | Tokens]); + +% Handle cursor completion +handle_space_sensitive_tokens([], Line, Column, + #elixir_tokenizer{cursor_completion=Cursor} = Scope, + [{identifier, Info, Identifier} | Tokens]) when Cursor /= false -> + tokenize([$(], Line, Column+1, Scope, [{paren_identifier, Info, Identifier} | Tokens]); + +handle_space_sensitive_tokens(String, Line, Column, Scope, Tokens) -> + tokenize(String, Line, Column, Scope, Tokens). %% Helpers -eol(_Line, _Mod, [{',',_}|_] = Tokens) -> Tokens; -eol(_Line, _Mod, [{eol,_,_}|_] = Tokens) -> Tokens; -eol(Line, Mod, Tokens) -> [{eol,Line,Mod}|Tokens]. +eol(_Line, _Column, [{',', {Line, Column, Count}} | Tokens]) -> + [{',', {Line, Column, Count + 1}} | Tokens]; +eol(_Line, _Column, [{';', {Line, Column, Count}} | Tokens]) -> + [{';', {Line, Column, Count + 1}} | Tokens]; +eol(_Line, _Column, [{eol, {Line, Column, Count}} | Tokens]) -> + [{eol, {Line, Column, Count + 1}} | Tokens]; +eol(Line, Column, Tokens) -> + [{eol, {Line, Column, 1}} | Tokens]. + +is_unnecessary_quote([Part], Scope) when is_list(Part) -> + case (Scope#elixir_tokenizer.identifier_tokenizer):tokenize(Part) of + {identifier, _, [], _, true, Special} -> not lists:member(at, Special); + _ -> false + end; +is_unnecessary_quote(_Parts, _Scope) -> + false. -unsafe_to_atom(Part, Line, #elixir_tokenizer{}) when - is_binary(Part) andalso size(Part) > 255; +unsafe_to_atom(Part, Line, Column, #elixir_tokenizer{}) when + is_binary(Part) andalso byte_size(Part) > 255; is_list(Part) andalso length(Part) > 255 -> - {error, {Line, "atom length must be less than system limit", ":"}}; -unsafe_to_atom(Binary, _Line, #elixir_tokenizer{existing_atoms_only=true}) when is_binary(Binary) -> - {ok, binary_to_existing_atom(Binary, utf8)}; -unsafe_to_atom(Binary, _Line, #elixir_tokenizer{}) when is_binary(Binary) -> - {ok, binary_to_atom(Binary, utf8)}; -unsafe_to_atom(List, _Line, #elixir_tokenizer{existing_atoms_only=true}) when is_list(List) -> - {ok, list_to_existing_atom(List)}; -unsafe_to_atom(List, _Line, #elixir_tokenizer{}) when is_list(List) -> - {ok, list_to_atom(List)}. - -collect_modifiers([H|T], Buffer) when ?is_downcase(H) -> - collect_modifiers(T, [H|Buffer]); + try + PartList = elixir_utils:characters_to_list(Part), + {error, {?LOC(Line, Column), "atom length must be less than system limit: ", PartList}} + catch + error:#{'__struct__' := 'Elixir.UnicodeConversionError', message := Message} -> + {error, {?LOC(Line, Column), "invalid encoding in atom: ", elixir_utils:characters_to_list(Message)}} + end; +unsafe_to_atom(Part, Line, Column, #elixir_tokenizer{static_atoms_encoder=StaticAtomsEncoder}) when + is_function(StaticAtomsEncoder) -> + EncodeResult = try + ValueEncBin = elixir_utils:characters_to_binary(Part), + ValueEncList = elixir_utils:characters_to_list(Part), + {ok, ValueEncBin, ValueEncList} + catch + error:#{'__struct__' := 'Elixir.UnicodeConversionError', message := Message} -> + {error, {?LOC(Line, Column), "invalid encoding in atom: ", elixir_utils:characters_to_list(Message)}} + end, + + case EncodeResult of + {ok, Value, ValueList} -> + case StaticAtomsEncoder(Value, [{line, Line}, {column, Column}]) of + {ok, Term} -> + {ok, Term}; + {error, Reason} when is_binary(Reason) -> + {error, {?LOC(Line, Column), elixir_utils:characters_to_list(Reason) ++ ": ", ValueList}} + end; + EncError -> EncError + end; +unsafe_to_atom(Binary, Line, Column, #elixir_tokenizer{existing_atoms_only=true}) when is_binary(Binary) -> + try + {ok, binary_to_existing_atom(Binary, utf8)} + catch + error:badarg -> + % Check if it's a UTF-8 issue by trying to convert to list + try + List = elixir_utils:characters_to_list(Binary), + % If we get here, it's not a UTF-8 issue + {error, {?LOC(Line, Column), "unsafe atom does not exist: ", List}} + catch + error:#{'__struct__' := 'Elixir.UnicodeConversionError', message := Message} -> + {error, {?LOC(Line, Column), "invalid encoding in atom: ", elixir_utils:characters_to_list(Message)}} + end + end; +unsafe_to_atom(Binary, Line, Column, #elixir_tokenizer{}) when is_binary(Binary) -> + try + {ok, binary_to_atom(Binary, utf8)} + catch + error:badarg -> + % Try to convert using elixir_utils to get proper UnicodeConversionError + try + List = elixir_utils:characters_to_list(Binary), + % If we get here, it's not a UTF-8 issue, so it's some other badarg + {error, {?LOC(Line, Column), "invalid atom: ", List}} + catch + error:#{'__struct__' := 'Elixir.UnicodeConversionError', message := Message} -> + {error, {?LOC(Line, Column), "invalid encoding in atom: ", elixir_utils:characters_to_list(Message)}} + end + end; +unsafe_to_atom(List, Line, Column, #elixir_tokenizer{existing_atoms_only=true}) when is_list(List) -> + try + {ok, list_to_existing_atom(List)} + catch + error:badarg -> + % Try to convert using elixir_utils to get proper UnicodeConversionError + try + elixir_utils:characters_to_binary(List), + % If we get here, it's not a UTF-8 issue + {error, {?LOC(Line, Column), "unsafe atom does not exist: ", List}} + catch + error:#{'__struct__' := 'Elixir.UnicodeConversionError', message := Message} -> + {error, {?LOC(Line, Column), "invalid encoding in atom: ", elixir_utils:characters_to_list(Message)}} + end + end; +unsafe_to_atom(List, Line, Column, #elixir_tokenizer{}) when is_list(List) -> + try + {ok, list_to_atom(List)} + catch + error:badarg -> + % Try to convert using elixir_utils to get proper UnicodeConversionError + try + elixir_utils:characters_to_binary(List), + % If we get here, it's not a UTF-8 issue, so it's some other badarg + {error, {?LOC(Line, Column), "invalid atom: ", List}} + catch + error:#{'__struct__' := 'Elixir.UnicodeConversionError', message := Message} -> + {error, {?LOC(Line, Column), "invalid encoding in atom: ", elixir_utils:characters_to_list(Message)}} + end + end. + +collect_modifiers([H | T], Buffer) when ?is_downcase(H) or ?is_upcase(H) or ?is_digit(H) -> + collect_modifiers(T, [H | Buffer]); collect_modifiers(Rest, Buffer) -> {Rest, lists:reverse(Buffer)}. %% Heredocs -extract_heredoc_with_interpolation(Line, Scope, Interpol, T, H) -> - case extract_heredoc(Line, T, H) of - {ok, NewLine, Body, Rest} -> - case elixir_interpolation:extract(Line + 1, Scope, Interpol, Body, 0) of - {error, Reason} -> - {error, interpolation_format(Reason, " (for heredoc starting at line ~B)", [Line])}; - {_, Parts, []} -> - {ok, NewLine, Parts, Rest} - end; - {error, _} = Error -> - Error - end. - -extract_heredoc(Line0, Rest0, Marker) -> - case extract_heredoc_header(Rest0) of - {ok, Rest1} -> +extract_heredoc_with_interpolation(Line, Column, Scope, Interpol, T, H) -> + case extract_heredoc_header(T) of + {ok, Headerless} -> %% We prepend a new line so we can transparently remove - %% spaces later. This new line is removed by calling `tl` + %% spaces later. This new line is removed by calling "tl" %% in the final heredoc body three lines below. - case extract_heredoc_body(Line0, Marker, [$\n|Rest1], []) of - {ok, Line1, Body, Rest2, Spaces} -> - {ok, Line1, tl(remove_heredoc_spaces(Body, Spaces)), Rest2}; - {error, ErrorLine} -> - Terminator = [Marker, Marker, Marker], - Message = "missing terminator: ~ts (for heredoc starting at line ~B)", - {error, {ErrorLine, io_lib:format(Message, [Terminator, Line0]), []}} + case elixir_interpolation:extract(Line, Column, Scope, Interpol, [$\n|Headerless], [H,H,H]) of + {NewLine, NewColumn, Parts0, Rest, InterScope} -> + Indent = NewColumn - 4, + Fun = fun(Part, Acc) -> extract_heredoc_indent(Part, Acc, Indent) end, + {Parts1, {ShouldWarn, _}} = lists:mapfoldl(Fun, {false, Line}, Parts0), + Parts2 = extract_heredoc_head(Parts1), + NewScope = maybe_heredoc_warn(ShouldWarn, Column, InterScope, H), + try + {ok, NewLine, NewColumn, tokens_to_binary(Parts2), Rest, NewScope} + catch + error:#{'__struct__' := 'Elixir.UnicodeConversionError', message := Message} -> + {error, interpolation_format(Message, " (for heredoc starting at line ~B)", [Line], Line, Column, [H, H, H], [H, H, H])} + end; + + {error, Reason} -> + {error, interpolation_format(Reason, " (for heredoc starting at line ~B)", [Line], Line, Column, [H, H, H], [H, H, H])} end; + error -> - Message = "heredoc start must be followed by a new line after ", - {error, {Line0, io_lib:format(Message, []), [Marker, Marker, Marker]}} + Message = "heredoc allows only whitespace characters followed by a new line after opening ", + {error, {?LOC(Line, Column + 3), io_lib:format(Message, []), [H, H, H]}} end. -%% Remove spaces from heredoc based on the position of the final quotes. - -remove_heredoc_spaces(Body, 0) -> - lists:reverse([0|Body]); -remove_heredoc_spaces(Body, Spaces) -> - remove_heredoc_spaces([0|Body], [], Spaces, Spaces). -remove_heredoc_spaces([H,$\n|T], [Backtrack|Buffer], Spaces, Original) when Spaces > 0, ?is_horizontal_space(H) -> - remove_heredoc_spaces([Backtrack,$\n|T], Buffer, Spaces - 1, Original); -remove_heredoc_spaces([$\n=H|T], Buffer, _Spaces, Original) -> - remove_heredoc_spaces(T, [H|Buffer], Original, Original); -remove_heredoc_spaces([H|T], Buffer, Spaces, Original) -> - remove_heredoc_spaces(T, [H|Buffer], Spaces, Original); -remove_heredoc_spaces([], Buffer, _Spaces, _Original) -> - Buffer. - -%% Extract the heredoc header. - extract_heredoc_header("\r\n" ++ Rest) -> {ok, Rest}; extract_heredoc_header("\n" ++ Rest) -> {ok, Rest}; -extract_heredoc_header([H|T]) when ?is_horizontal_space(H) -> +extract_heredoc_header([H | T]) when ?is_horizontal_space(H) -> extract_heredoc_header(T); extract_heredoc_header(_) -> error. -%% Extract heredoc body. It returns the heredoc body (in reverse order), -%% the remaining of the document and the number of spaces the heredoc -%% is aligned. - -extract_heredoc_body(Line, Marker, Rest, Buffer) -> - case extract_heredoc_line(Marker, Rest, Buffer, 0) of - {ok, NewBuffer, NewRest} -> - extract_heredoc_body(Line + 1, Marker, NewRest, NewBuffer); - {ok, NewBuffer, NewRest, Spaces} -> - {ok, Line, NewBuffer, NewRest, Spaces}; - {error, eof} -> - {error, Line} +extract_heredoc_indent(Part, {Warned, Line}, Indent) when is_list(Part) -> + extract_heredoc_indent(Part, [], Warned, Line, Indent); +extract_heredoc_indent({_, {EndLine, _, _}, _} = Part, {Warned, _Line}, _Indent) -> + {Part, {Warned, EndLine}}. + +extract_heredoc_indent([$\n | Rest], Acc, Warned, Line, Indent) -> + {Trimmed, ShouldWarn} = trim_space(Rest, Indent), + Warn = if ShouldWarn, not Warned -> Line + 1; true -> Warned end, + extract_heredoc_indent(Trimmed, [$\n | Acc], Warn, Line + 1, Indent); +extract_heredoc_indent([Head | Rest], Acc, Warned, Line, Indent) -> + extract_heredoc_indent(Rest, [Head | Acc], Warned, Line, Indent); +extract_heredoc_indent([], Acc, Warned, Line, _Indent) -> + {lists:reverse(Acc), {Warned, Line}}. + +trim_space(Rest, 0) -> {Rest, false}; +trim_space([$\r, $\n | _] = Rest, _) -> {Rest, false}; +trim_space([$\n | _] = Rest, _) -> {Rest, false}; +trim_space([H | T], Spaces) when ?is_horizontal_space(H) -> trim_space(T, Spaces - 1); +trim_space([], _Spaces) -> {[], false}; +trim_space(Rest, _Spaces) -> {Rest, true}. + +maybe_heredoc_warn(false, _Column, Scope, _Marker) -> + Scope; +maybe_heredoc_warn(Line, Column, Scope, Marker) -> + Msg = io_lib:format("outdented heredoc line. The contents inside the heredoc should be indented " + "at the same level as the closing ~ts. The following is forbidden:~n~n" + " def text do~n" + " \"\"\"~n" + " contents~n" + " \"\"\"~n" + " end~n~n" + "Instead make sure the contents are indented as much as the heredoc closing:~n~n" + " def text do~n" + " \"\"\"~n" + " contents~n" + " \"\"\"~n" + " end~n~n" + "The current heredoc line is indented too little", [[Marker, Marker, Marker]]), + + prepend_warning(Line, Column, Msg, Scope). + +extract_heredoc_head([[$\n|H]|T]) -> [H|T]. + +unescape_tokens(Tokens, Line, Column, #elixir_tokenizer{unescape=true}) -> + case elixir_interpolation:unescape_tokens(Tokens) of + {ok, Result} -> + {ok, Result}; + + {error, Message, Token} -> + {error, {?LOC(Line, Column), Message ++ ". Syntax error after: ", Token}} + end; +unescape_tokens(Tokens, Line, Column, #elixir_tokenizer{unescape=false}) -> + try + {ok, tokens_to_binary(Tokens)} + catch + error:#{'__struct__' := 'Elixir.UnicodeConversionError', message := Message} -> + {error, {?LOC(Line, Column), "invalid encoding in tokens: ", elixir_utils:characters_to_list(Message)}} end. -%% Extract a line from the heredoc prepending its contents to a buffer. - -extract_heredoc_line("\r\n" ++ Rest, Buffer) -> - {ok, [$\n|Buffer], Rest}; -extract_heredoc_line("\n" ++ Rest, Buffer) -> - {ok, [$\n|Buffer], Rest}; -extract_heredoc_line([H|T], Buffer) -> - extract_heredoc_line(T, [H|Buffer]); -extract_heredoc_line(_, _) -> - {error, eof}. - -%% Extract each heredoc line trying to find a match according to the marker. - -extract_heredoc_line(Marker, [H|T], Buffer, Counter) when ?is_horizontal_space(H) -> - extract_heredoc_line(Marker, T, [H|Buffer], Counter + 1); -extract_heredoc_line(Marker, [Marker,Marker,Marker|T], Buffer, Counter) -> - {ok, Buffer, T, Counter}; -extract_heredoc_line(_Marker, Rest, Buffer, _Counter) -> - extract_heredoc_line(Rest, Buffer). +tokens_to_binary(Tokens) -> + [if is_list(Token) -> elixir_utils:characters_to_binary(Token); true -> Token end + || Token <- Tokens]. %% Integers and floats %% At this point, we are at least sure the first digit is a number. %% Check if we have a point followed by a number; -tokenize_number([$.,H|T], Acc, false) when ?is_digit(H) -> - tokenize_number(T, [H,$.|Acc], true); +tokenize_number([$., H | T], Acc, Length, false) when ?is_digit(H) -> + tokenize_number(T, [H, $. | Acc], Length + 2, true); %% Check if we have an underscore followed by a number; -tokenize_number([$_,H|T], Acc, Bool) when ?is_digit(H) -> - tokenize_number(T, [H|Acc], Bool); +tokenize_number([$_, H | T], Acc, Length, Bool) when ?is_digit(H) -> + tokenize_number(T, [H, $_ | Acc], Length + 2, Bool); %% Check if we have e- followed by numbers (valid only for floats); -tokenize_number([E,S,H|T], Acc, true) - when (E == $E) or (E == $e), ?is_digit(H), S == $+ orelse S == $- -> - tokenize_number(T, [H,S,$e|Acc], true); +tokenize_number([E, S, H | T], Acc, Length, true) + when (E =:= $E) or (E =:= $e), ?is_digit(H), S =:= $+ orelse S =:= $- -> + tokenize_number(T, [H, S, E | Acc], Length + 3, true); %% Check if we have e followed by numbers (valid only for floats); -tokenize_number([E,H|T], Acc, true) - when (E == $E) or (E == $e), ?is_digit(H) -> - tokenize_number(T, [H,$e|Acc], true); +tokenize_number([E, H | T], Acc, Length, true) + when (E =:= $E) or (E =:= $e), ?is_digit(H) -> + tokenize_number(T, [H, E | Acc], Length + 2, true); %% Finally just numbers. -tokenize_number([H|T], Acc, Bool) when ?is_digit(H) -> - tokenize_number(T, [H|Acc], Bool); +tokenize_number([H | T], Acc, Length, Bool) when ?is_digit(H) -> + tokenize_number(T, [H | Acc], Length + 1, Bool); %% Cast to float... -tokenize_number(Rest, Acc, true) -> - {Rest, list_to_float(lists:reverse(Acc))}; +tokenize_number(Rest, Acc, Length, true) -> + try + {Number, Original} = reverse_number(Acc, [], []), + {Rest, list_to_float(Number), Original, Length} + catch + error:badarg -> {error, "invalid float number ", lists:reverse(Acc)} + end; %% Or integer. -tokenize_number(Rest, Acc, false) -> - {Rest, list_to_integer(lists:reverse(Acc))}. +tokenize_number(Rest, Acc, Length, false) -> + {Number, Original} = reverse_number(Acc, [], []), + {Rest, list_to_integer(Number), Original, Length}. + +tokenize_hex([H | T], Acc, Length) when ?is_hex(H) -> + tokenize_hex(T, [H | Acc], Length + 1); +tokenize_hex([$_, H | T], Acc, Length) when ?is_hex(H) -> + tokenize_hex(T, [H, $_ | Acc], Length + 2); +tokenize_hex(Rest, Acc, Length) -> + {Number, Original} = reverse_number(Acc, [], []), + {Rest, list_to_integer(Number, 16), [$0, $x | Original], Length}. + +tokenize_octal([H | T], Acc, Length) when ?is_octal(H) -> + tokenize_octal(T, [H | Acc], Length + 1); +tokenize_octal([$_, H | T], Acc, Length) when ?is_octal(H) -> + tokenize_octal(T, [H, $_ | Acc], Length + 2); +tokenize_octal(Rest, Acc, Length) -> + {Number, Original} = reverse_number(Acc, [], []), + {Rest, list_to_integer(Number, 8), [$0, $o | Original], Length}. + +tokenize_bin([H | T], Acc, Length) when ?is_bin(H) -> + tokenize_bin(T, [H | Acc], Length + 1); +tokenize_bin([$_, H | T], Acc, Length) when ?is_bin(H) -> + tokenize_bin(T, [H, $_ | Acc], Length + 2); +tokenize_bin(Rest, Acc, Length) -> + {Number, Original} = reverse_number(Acc, [], []), + {Rest, list_to_integer(Number, 2), [$0, $b | Original], Length}. + +reverse_number([$_ | T], Number, Original) -> + reverse_number(T, Number, [$_ | Original]); +reverse_number([H | T], Number, Original) -> + reverse_number(T, [H | Number], [H | Original]); +reverse_number([], Number, Original) -> + {Number, Original}. -tokenize_hex([H|T], Acc) when ?is_hex(H) -> tokenize_hex(T, [H|Acc]); -tokenize_hex(Rest, Acc) -> {Rest, list_to_integer(lists:reverse(Acc), 16)}. +%% Comments -tokenize_octal([H|T], Acc) when ?is_octal(H) -> tokenize_octal(T, [H|Acc]); -tokenize_octal(Rest, Acc) -> {Rest, list_to_integer(lists:reverse(Acc), 8)}. +reset_eol([{eol, {Line, Column, _}} | Rest]) -> [{eol, {Line, Column, 0}} | Rest]; +reset_eol(Rest) -> Rest. + +tokenize_comment("\r\n" ++ _ = Rest, Acc) -> + {Rest, lists:reverse(Acc)}; +tokenize_comment("\n" ++ _ = Rest, Acc) -> + {Rest, lists:reverse(Acc)}; +tokenize_comment([H | _Rest], _) when ?bidi(H) -> + {error, H, "invalid bidirectional formatting character in comment: "}; +tokenize_comment([H | _Rest], _) when ?break(H) -> + {error, H, "invalid line break character in comment: "}; +tokenize_comment([H | Rest], Acc) -> + tokenize_comment(Rest, [H | Acc]); +tokenize_comment([], Acc) -> + {[], lists:reverse(Acc)}. + +error_comment(Char, Reason, Comment, Line, Column, Scope, Tokens) -> + Token = io_lib:format("\\u~4.16.0B", [Char]), + error({?LOC(Line, Column), Reason, Token}, Comment, Scope, Tokens). + +preserve_comments(Line, Column, Tokens, Comment, Rest, Scope) -> + case Scope#elixir_tokenizer.preserve_comments of + Fun when is_function(Fun) -> + Fun(Line, Column, Tokens, Comment, Rest); + nil -> + ok + end. -tokenize_bin([H|T], Acc) when ?is_bin(H) -> tokenize_bin(T, [H|Acc]); -tokenize_bin(Rest, Acc) -> {Rest, list_to_integer(lists:reverse(Acc), 2)}. +%% Identifiers -%% Comments +tokenize([H | T]) when ?is_upcase(H) -> + {Acc, Rest, Length, Special} = tokenize_continue(T, [H], 1, []), + {alias, lists:reverse(Acc), Rest, Length, true, Special}; +tokenize([H | T]) when ?is_downcase(H); H =:= $_ -> + {Acc, Rest, Length, Special} = tokenize_continue(T, [H], 1, []), + {identifier, lists:reverse(Acc), Rest, Length, true, Special}; +tokenize(_List) -> + {error, empty}. + +tokenize_continue([$@ | T], Acc, Length, Special) -> + tokenize_continue(T, [$@ | Acc], Length + 1, [at | lists:delete(at, Special)]); +tokenize_continue([$! | T], Acc, Length, Special) -> + {[$! | Acc], T, Length + 1, [punctuation | Special]}; +tokenize_continue([$? | T], Acc, Length, Special) -> + {[$? | Acc], T, Length + 1, [punctuation | Special]}; +tokenize_continue([H | T], Acc, Length, Special) when ?is_upcase(H); ?is_downcase(H); ?is_digit(H); H =:= $_ -> + tokenize_continue(T, [H | Acc], Length + 1, Special); +tokenize_continue(Rest, Acc, Length, Special) -> + {Acc, Rest, Length, Special}. + +tokenize_identifier(String, Line, Column, Scope, MaybeKeyword) -> + case (Scope#elixir_tokenizer.identifier_tokenizer):tokenize(String) of + {Kind, Acc, Rest, Length, Ascii, Special} -> + Keyword = MaybeKeyword andalso maybe_keyword(Rest), + + case keyword_or_unsafe_to_atom(Keyword, Acc, Line, Column, Scope) of + {keyword, Atom, Type} -> + {keyword, Atom, Type, Rest, Length}; + {ok, Atom} -> + {Kind, Acc, Atom, Rest, Length, Ascii, Special}; + {error, _Reason} = Error -> + Error + end; -tokenize_comment("\r\n" ++ _ = Rest) -> Rest; -tokenize_comment("\n" ++ _ = Rest) -> Rest; -tokenize_comment([_|Rest]) -> tokenize_comment(Rest); -tokenize_comment([]) -> []. + {error, {mixed_script, Wrong, {Prefix, Suffix}}} -> + WrongColumn = Column + length(Wrong) - 1, + case suggest_simpler_unexpected_token_in_error(Wrong, Line, WrongColumn, Scope) of + no_suggestion -> + %% we append a pointer to more info if we aren't appending a suggestion + MoreInfo = "\nSee https://hexdocs.pm/elixir/unicode-syntax.html for more information.", + {error, {?LOC(Line, Column), {Prefix, Suffix ++ MoreInfo}, Wrong}}; -%% Atoms -%% Handle atoms specially since they support @ + {_, {Location, _, SuggestionMessage}} = _SuggestionError -> + {error, {Location, {Prefix, Suffix ++ SuggestionMessage}, Wrong}} + end; -tokenize_atom([H|T], Acc) when ?is_atom(H) -> - tokenize_atom(T, [H|Acc]); + {error, {unexpected_token, Wrong}} -> + WrongColumn = Column + length(Wrong) - 1, + case suggest_simpler_unexpected_token_in_error(Wrong, Line, WrongColumn, Scope) of + no_suggestion -> + [T | _] = lists:reverse(Wrong), + case suggest_simpler_unexpected_token_in_error([T], Line, WrongColumn, Scope) of + no_suggestion -> {unexpected_token, length(Wrong)}; + SuggestionError -> SuggestionError + end; + + SuggestionError -> + SuggestionError + end; -tokenize_atom([H|T], Acc) when H == $?; H == $! -> - {T, lists:reverse([H|Acc])}; + {error, empty} -> + empty + end. -tokenize_atom(Rest, Acc) -> - {Rest, lists:reverse(Acc)}. +%% heuristic: try nfkc; try confusability skeleton; try calling this again w/just failed codepoint +suggest_simpler_unexpected_token_in_error(Wrong, Line, WrongColumn, Scope) -> + NFKC = unicode:characters_to_nfkc_list(Wrong), + case (Scope#elixir_tokenizer.identifier_tokenizer):tokenize(NFKC) of + {error, _Reason} -> + ConfusableSkeleton = 'Elixir.String.Tokenizer.Security':confusable_skeleton(Wrong), + case (Scope#elixir_tokenizer.identifier_tokenizer):tokenize(ConfusableSkeleton) of + {_, Simpler, _, _, _, _} -> + Message = suggest_change("Codepoint failed identifier tokenization, but a simpler form was found.", + Wrong, + "You could write the above in a similar way that is accepted by Elixir:", + Simpler, + "See https://hexdocs.pm/elixir/unicode-syntax.html for more information."), + {error, {?LOC(Line, WrongColumn), "unexpected token: ", Message}}; + _other -> + no_suggestion + end; + {_, _NFKC, _, _, _, _} -> + Message = suggest_change("Elixir expects unquoted Unicode atoms, variables, and calls to use allowed codepoints and to be in NFC form.", + Wrong, + "You could write the above in a compatible format that is accepted by Elixir:", + NFKC, + "See https://hexdocs.pm/elixir/unicode-syntax.html for more information."), + {error, {?LOC(Line, WrongColumn), "unexpected token: ", Message}} + end. + +suggest_change(Intro, WrongForm, Hint, HintedForm, Ending) -> + WrongCodepoints = list_to_codepoint_hex(WrongForm), + HintedCodepoints = list_to_codepoint_hex(HintedForm), + io_lib:format("~ts\n\nGot:\n\n \"~ts\" (code points~ts)\n\n" + "Hint: ~ts\n\n \"~ts\" (code points~ts)\n\n~ts", + [Intro, WrongForm, WrongCodepoints, Hint, HintedForm, HintedCodepoints, Ending]). + +maybe_keyword([]) -> true; +maybe_keyword([$:, $: | _]) -> true; +maybe_keyword([$: | _]) -> false; +maybe_keyword(_) -> true. + +list_to_codepoint_hex(List) -> + [io_lib:format(" 0x~5.16.0B", [Codepoint]) || Codepoint <- List]. + +tokenize_alias(Rest, Line, Column, Unencoded, Atom, Length, Ascii, Special, Scope, Tokens) -> + if + not Ascii or (Special /= []) -> + Invalid = hd([C || C <- Unencoded, (C < $A) or (C > 127)]), + Reason = {?LOC(Line, Column), invalid_character_error("alias (only ASCII characters, without punctuation, are allowed)", Invalid), Unencoded}, + error(Reason, Unencoded ++ Rest, Scope, Tokens); + + true -> + AliasesToken = {alias, {Line, Column, Unencoded}, Atom}, + tokenize(Rest, Line, Column + Length, Scope, [AliasesToken | Tokens]) + end. -%% Identifiers -%% At this point, the validity of the first character was already verified. +%% Check if it is a call identifier (paren | bracket | do) -tokenize_identifier([H|T], Acc) when ?is_identifier(H) -> - tokenize_identifier(T, [H|Acc]); +check_call_identifier(Line, Column, Info, Atom, [$( | _]) -> + {paren_identifier, {Line, Column, Info}, Atom}; +check_call_identifier(Line, Column, Info, Atom, [$[ | _]) -> + {bracket_identifier, {Line, Column, Info}, Atom}; +check_call_identifier(Line, Column, Info, Atom, _Rest) -> + {identifier, {Line, Column, Info}, Atom}. -tokenize_identifier(Rest, Acc) -> - {Rest, lists:reverse(Acc)}. +add_token_with_eol({unary_op, _, _} = Left, T) -> [Left | T]; +add_token_with_eol(Left, [{eol, _} | T]) -> [Left | T]; +add_token_with_eol(Left, T) -> [Left | T]. -%% Tokenize any identifier, handling kv, punctuated, paren, bracket and do identifiers. +previous_was_eol([{',', {_, _, Count}} | _]) when Count > 0 -> Count; +previous_was_eol([{';', {_, _, Count}} | _]) when Count > 0 -> Count; +previous_was_eol([{eol, {_, _, Count}} | _]) when Count > 0 -> Count; +previous_was_eol(_) -> nil. -tokenize_any_identifier(Original, Line, Scope, Tokens) -> - {Rest, Identifier} = tokenize_identifier(Original, []), +%% Error handling - {AllIdentifier, AllRest} = - case Rest of - [H|T] when H == $?; H == $! -> {Identifier ++ [H], T}; - _ -> {Identifier, Rest} - end, +interpolation_error(Reason, Rest, Scope, Tokens, Extension, Args, Line, Column, Opening, Closing) -> + error(interpolation_format(Reason, Extension, Args, Line, Column, Opening, Closing), Rest, Scope, Tokens). + +interpolation_format({string, EndLine, EndColumn, Message, Token}, Extension, Args, Line, Column, Opening, Closing) -> + Meta = [ + {opening_delimiter, list_to_atom(Opening)}, + {expected_delimiter, list_to_atom(Closing)}, + {line, Line}, + {column, Column}, + {end_line, EndLine}, + {end_column, EndColumn} + ], + {Meta, [Message, io_lib:format(Extension, Args)], Token}; +interpolation_format({_, _, _} = Reason, _Extension, _Args, _Line, _Column, _Opening, _Closing) -> + Reason. - case unsafe_to_atom(AllIdentifier, Line, Scope) of - {ok, Atom} -> - tokenize_kw_or_other(AllRest, identifier, Line, Atom, Tokens); +%% Terminators + +handle_terminator(Rest, _, _, Scope, {'(', {Line, Column, _}}, [{alias, _, Alias} | Tokens]) when is_atom(Alias) -> + Reason = + io_lib:format( + "unexpected ( after alias ~ts. Function names and identifiers in Elixir " + "start with lowercase characters or underscore. For example:\n\n" + " hello_world()\n" + " _starting_with_underscore()\n" + " numb3rs_are_allowed()\n" + " may_finish_with_question_mark?()\n" + " may_finish_with_exclamation_mark!()\n\n" + "Unexpected token: ", + [Alias] + ), + + error({?LOC(Line, Column), Reason, ["("]}, atom_to_list(Alias) ++ [$( | Rest], Scope, Tokens); +handle_terminator(Rest, Line, Column, #elixir_tokenizer{terminators=none} = Scope, Token, Tokens) -> + tokenize(Rest, Line, Column, Scope, [Token | Tokens]); +handle_terminator(Rest, Line, Column, Scope, Token, Tokens) -> + #elixir_tokenizer{terminators=Terminators} = Scope, + + case check_terminator(Token, Terminators, Scope) of {error, Reason} -> - {error, Reason, Original, Tokens} + error(Reason, atom_to_list(element(1, Token)) ++ Rest, Scope, Tokens); + {ok, New} -> + tokenize(Rest, Line, Column, New, [Token | Tokens]) end. -tokenize_kw_or_other([$:,H|T], _Kind, Line, Atom, _Tokens) when ?is_space(H) -> - {identifier, [H|T], {kw_identifier, Line, Atom}}; - -tokenize_kw_or_other([$:,H|T], _Kind, Line, Atom, Tokens) when ?is_atom_start(H); ?is_digit(H) -> - Original = atom_to_list(Atom) ++ [$:], - Reason = {Line, "keyword argument must be followed by space after: ", Original}, - {error, Reason, Original ++ [H|T], Tokens}; - -tokenize_kw_or_other(Rest, Kind, Line, Atom, Tokens) -> - case check_keyword(Line, Atom, Tokens) of - nomatch -> - {identifier, Rest, check_call_identifier(Kind, Line, Atom, Rest)}; - {ok, [Check|T]} -> - {keyword, Rest, Check, T}; - {error, Token} -> - {error, {Line, "syntax error before: ", Token}, atom_to_list(Atom) ++ Rest, Tokens} - end. +check_terminator({Start, Meta}, Terminators, Scope) + when Start == '('; Start == '['; Start == '{'; Start == '<<' -> + Indentation = Scope#elixir_tokenizer.indentation, + {ok, Scope#elixir_tokenizer{terminators=[{Start, Meta, Indentation} | Terminators]}}; -%% Check if it is a call identifier (paren | bracket | do) +check_terminator({Start, Meta}, Terminators, Scope) when Start == 'fn'; Start == 'do' -> + Indentation = Scope#elixir_tokenizer.indentation, -check_call_identifier(_Kind, Line, Atom, [$(|_]) -> {paren_identifier, Line, Atom}; -check_call_identifier(_Kind, Line, Atom, [$[|_]) -> {bracket_identifier, Line, Atom}; -check_call_identifier(Kind, Line, Atom, _Rest) -> {Kind, Line, Atom}. + NewScope = + case Terminators of + %% If the do is indented equally or less than the previous do, it may be a missing end error! + [{Start, _, PreviousIndentation} = Previous | _] when Indentation =< PreviousIndentation -> + Scope#elixir_tokenizer{mismatch_hints=[Previous | Scope#elixir_tokenizer.mismatch_hints]}; -add_token_with_nl(Left, [{eol,_,newline}|T]) -> [Left|T]; -add_token_with_nl(Left, T) -> [Left|T]. + _ -> + Scope + end, -%% Error handling + {ok, NewScope#elixir_tokenizer{terminators=[{Start, Meta, Indentation} | Terminators]}}; -interpolation_error(Reason, Rest, Tokens, Extension, Args) -> - {error, interpolation_format(Reason, Extension, Args), Rest, Tokens}. +check_terminator({'end', {EndLine, _, _}}, [{'do', _, Indentation} | Terminators], Scope) -> + NewScope = + %% If the end is more indented than the do, it may be a missing do error! + case Scope#elixir_tokenizer.indentation > Indentation of + true -> + Hint = {'end', EndLine, Scope#elixir_tokenizer.indentation}, + Scope#elixir_tokenizer{mismatch_hints=[Hint | Scope#elixir_tokenizer.mismatch_hints]}; -interpolation_format({string, Line, Message, Token}, Extension, Args) -> - {Line, io_lib:format("~ts" ++ Extension, [Message|Args]), Token}; -interpolation_format({_, _, _} = Reason, _Extension, _Args) -> - Reason. + false -> + Scope + end, -%% Terminators + {ok, NewScope#elixir_tokenizer{terminators=Terminators}}; + +check_terminator({End, {EndLine, EndColumn, _}}, [{Start, {StartLine, StartColumn, _}, _} | Terminators], Scope) + when End == 'end'; End == ')'; End == ']'; End == '}'; End == '>>' -> + case terminator(Start) of + End -> + {ok, Scope#elixir_tokenizer{terminators=Terminators}}; + + ExpectedEnd -> + Meta = [ + {line, StartLine}, + {column, StartColumn}, + {end_line, EndLine}, + {end_column, EndColumn}, + {error_type, mismatched_delimiter}, + {opening_delimiter, Start}, + {closing_delimiter, End}, + {expected_delimiter, ExpectedEnd} + ], + {error, {Meta, unexpected_token_or_reserved(End), [atom_to_list(End)]}} + end; -handle_terminator(Rest, Line, Scope, Token, Tokens) -> - case handle_terminator(Token, Scope) of - {error, Reason} -> - {error, Reason, atom_to_list(element(1, Token)) ++ Rest, Tokens}; - New -> - tokenize(Rest, Line, New, [Token|Tokens]) - end. +check_terminator({'end', {Line, Column, _}}, [], #elixir_tokenizer{mismatch_hints=Hints}) -> + Suffix = + case lists:keyfind('end', 1, Hints) of + {'end', HintLine, _Indentation} -> + io_lib:format("\n~ts the \"end\" on line ~B may not have a matching \"do\" " + "defined before it (based on indentation)", [elixir_errors:prefix(hint), HintLine]); + false -> + "" + end, -handle_terminator(_, #elixir_tokenizer{check_terminators=false} = Scope) -> - Scope; -handle_terminator(Token, #elixir_tokenizer{terminators=Terminators} = Scope) -> - case check_terminator(Token, Terminators) of - {error, _} = Error -> Error; - New -> Scope#elixir_tokenizer{terminators=New} - end. + {error, {?LOC(Line, Column), {"unexpected reserved word: ", Suffix}, "end"}}; -check_terminator({S, Line}, Terminators) when S == 'fn' -> - [{fn, Line}|Terminators]; - -check_terminator({S, _} = New, Terminators) when - S == 'do'; - S == '('; - S == '['; - S == '{'; - S == '<<' -> - [New|Terminators]; - -check_terminator({E, _}, [{S, _}|Terminators]) when - S == 'do', E == 'end'; - S == 'fn', E == 'end'; - S == '(', E == ')'; - S == '[', E == ']'; - S == '{', E == '}'; - S == '<<', E == '>>' -> - Terminators; - -check_terminator({E, Line}, [{Start, StartLine}|_]) when - E == 'end'; E == ')'; E == ']'; E == '}'; E == '>>' -> - End = terminator(Start), - Message = io_lib:format("\"~ts\" starting at line ~B is missing terminator \"~ts\". " - "Unexpected token: ", [Start, StartLine, End]), - {error, {Line, Message, atom_to_list(E)}}; +check_terminator({End, {Line, Column, _}}, [], _Scope) + when End == ')'; End == ']'; End == '}'; End == '>>' -> + {error, {?LOC(Line, Column), "unexpected token: ", atom_to_list(End)}}; -check_terminator({E, Line}, []) when - E == 'end'; E == ')'; E == ']'; E == '}'; E == '>>' -> - {error, {Line, "unexpected token: ", atom_to_list(E)}}; +check_terminator(_, _, Scope) -> + {ok, Scope}. -check_terminator(_, Terminators) -> - Terminators. +unexpected_token_or_reserved('end') -> "unexpected reserved word: "; +unexpected_token_or_reserved(_) -> "unexpected token: ". + +missing_terminator_hint(Start, End, #elixir_tokenizer{mismatch_hints=Hints}) -> + case lists:keyfind(Start, 1, Hints) of + {Start, {HintLine, _, _}, _} -> + io_lib:format("\n~ts it looks like the \"~ts\" on line ~B does not have a matching \"~ts\"", + [elixir_errors:prefix(hint), Start, HintLine, End]); + false -> + "" + end. string_type($") -> bin_string; string_type($') -> list_string. +heredoc_type($") -> bin_heredoc; +heredoc_type($') -> list_heredoc. + sigil_terminator($() -> $); sigil_terminator($[) -> $]; sigil_terminator(${) -> $}; @@ -904,55 +1647,335 @@ terminator('<<') -> '>>'. %% Keywords checking -check_keyword(_Line, _Atom, [{'.', _}|_]) -> - nomatch; -check_keyword(DoLine, do, [{Identifier, Line, Atom}|T]) when Identifier == identifier -> - {ok, add_token_with_nl({do, DoLine}, [{do_identifier, Line, Atom}|T])}; -check_keyword(Line, do, Tokens) -> - case do_keyword_valid(Tokens) of - true -> {ok, add_token_with_nl({do, Line}, Tokens)}; - false -> {error, "do"} +keyword_or_unsafe_to_atom(true, "fn", _Line, _Column, _Scope) -> {keyword, 'fn', terminator}; +keyword_or_unsafe_to_atom(true, "do", _Line, _Column, _Scope) -> {keyword, 'do', terminator}; +keyword_or_unsafe_to_atom(true, "end", _Line, _Column, _Scope) -> {keyword, 'end', terminator}; +keyword_or_unsafe_to_atom(true, "true", _Line, _Column, _Scope) -> {keyword, 'true', token}; +keyword_or_unsafe_to_atom(true, "false", _Line, _Column, _Scope) -> {keyword, 'false', token}; +keyword_or_unsafe_to_atom(true, "nil", _Line, _Column, _Scope) -> {keyword, 'nil', token}; + +keyword_or_unsafe_to_atom(true, "not", _Line, _Column, _Scope) -> {keyword, 'not', unary_op}; +keyword_or_unsafe_to_atom(true, "and", _Line, _Column, _Scope) -> {keyword, 'and', and_op}; +keyword_or_unsafe_to_atom(true, "or", _Line, _Column, _Scope) -> {keyword, 'or', or_op}; +keyword_or_unsafe_to_atom(true, "when", _Line, _Column, _Scope) -> {keyword, 'when', when_op}; +keyword_or_unsafe_to_atom(true, "in", _Line, _Column, _Scope) -> {keyword, 'in', in_op}; + +keyword_or_unsafe_to_atom(true, "after", _Line, _Column, _Scope) -> {keyword, 'after', block}; +keyword_or_unsafe_to_atom(true, "else", _Line, _Column, _Scope) -> {keyword, 'else', block}; +keyword_or_unsafe_to_atom(true, "catch", _Line, _Column, _Scope) -> {keyword, 'catch', block}; +keyword_or_unsafe_to_atom(true, "rescue", _Line, _Column, _Scope) -> {keyword, 'rescue', block}; + +keyword_or_unsafe_to_atom(_, Part, Line, Column, Scope) -> + unsafe_to_atom(Part, Line, Column, Scope). + +tokenize_keyword(terminator, Rest, Line, Column, Atom, Length, Scope, Tokens) -> + case tokenize_keyword_terminator(Line, Column, Atom, Tokens) of + {ok, [Check | T]} -> + handle_terminator(Rest, Line, Column + Length, Scope, Check, T); + {error, Message, Token} -> + error({?LOC(Line, Column), Message, Token}, Token ++ Rest, Scope, Tokens) end; -check_keyword(Line, Atom, Tokens) -> - case keyword(Atom) of - false -> nomatch; - token -> {ok, [{Atom, Line}|Tokens]}; - block -> {ok, [{block_identifier, Line, Atom}|Tokens]}; - unary_op -> {ok, [{unary_op, Line, Atom}|Tokens]}; - Kind -> {ok, add_token_with_nl({Kind, Line, Atom}, Tokens)} + +tokenize_keyword(token, Rest, Line, Column, Atom, Length, Scope, Tokens) -> + Token = {Atom, {Line, Column, nil}}, + tokenize(Rest, Line, Column + Length, Scope, [Token | Tokens]); + +tokenize_keyword(block, Rest, Line, Column, Atom, Length, Scope, Tokens) -> + Token = {block_identifier, {Line, Column, nil}, Atom}, + tokenize(Rest, Line, Column + Length, Scope, [Token | Tokens]); + +tokenize_keyword(Kind, Rest, Line, Column, Atom, Length, Scope, Tokens) -> + NewTokens = + case strip_horizontal_space(Rest, Line, Column, Scope) of + {[$/ | _], _, _} -> + [{identifier, {Line, Column, nil}, Atom} | Tokens]; + + _ -> + Info = {Line, Column, previous_was_eol(Tokens)}, + + case {Kind, Tokens} of + {in_op, [{unary_op, NotInfo, 'not'} | T]} -> + add_token_with_eol({in_op, NotInfo, 'not in', Info}, T); + + {_, _} -> + add_token_with_eol({Kind, Info, Atom}, Tokens) + end + end, + + tokenize(Rest, Line, Column + Length, Scope, NewTokens). + +tokenize_sigil([$~ | T], Line, Column, Scope, Tokens) -> + case tokenize_sigil_name(T, [], Line, Column + 1, Scope, Tokens) of + {ok, Name, Rest, NewLine, NewColumn, NewScope, NewTokens} -> + tokenize_sigil_contents(Rest, Name, NewLine, NewColumn, NewScope, NewTokens); + + {error, Message, Token} -> + Reason = {?LOC(Line, Column), Message, Token}, + error(Reason, T, Scope, Tokens) + end. + +% A one-letter sigil is ok both as upcase as well as downcase. +tokenize_sigil_name([S | T], [], Line, Column, Scope, Tokens) when ?is_downcase(S) -> + tokenize_lower_sigil_name(T, [S], Line, Column + 1, Scope, Tokens); +tokenize_sigil_name([S | T], [], Line, Column, Scope, Tokens) when ?is_upcase(S) -> + tokenize_upper_sigil_name(T, [S], Line, Column + 1, Scope, Tokens). + +tokenize_lower_sigil_name([S | _T] = Original, [_ | _] = NameAcc, _Line, _Column, _Scope, _Tokens) when ?is_downcase(S) -> + SigilName = lists:reverse(NameAcc) ++ Original, + {error, sigil_name_error(), [$~] ++ SigilName}; +tokenize_lower_sigil_name(T, NameAcc, Line, Column, Scope, Tokens) -> + {ok, lists:reverse(NameAcc), T, Line, Column, Scope, Tokens}. + +% If we have an uppercase letter, we keep tokenizing the name. +% A digit is allowed but an uppercase letter or digit must proceed it. +tokenize_upper_sigil_name([S | T], NameAcc, Line, Column, Scope, Tokens) when ?is_upcase(S); ?is_digit(S) -> + tokenize_upper_sigil_name(T, [S | NameAcc], Line, Column + 1, Scope, Tokens); +% With a lowercase letter and a non-empty NameAcc we return an error. +tokenize_upper_sigil_name([S | _T] = Original, [_ | _] = NameAcc, _Line, _Column, _Scope, _Tokens) when ?is_downcase(S) -> + SigilName = lists:reverse(NameAcc) ++ Original, + {error, sigil_name_error(), [$~] ++ SigilName}; +% We finished the letters, so the name is over. +tokenize_upper_sigil_name(T, NameAcc, Line, Column, Scope, Tokens) -> + {ok, lists:reverse(NameAcc), T, Line, Column, Scope, Tokens}. + +sigil_name_error() -> + "invalid sigil name, it should be either a one-letter lowercase letter or an " ++ + "uppercase letter optionally followed by uppercase letters and digits, got: ". + +tokenize_sigil_contents([H, H, H | T] = Original, [S | _] = SigilName, Line, Column, Scope, Tokens) + when ?is_quote(H) -> + case extract_heredoc_with_interpolation(Line, Column, Scope, ?is_downcase(S), T, H) of + {ok, NewLine, NewColumn, Parts, Rest, NewScope} -> + Indentation = NewColumn - 4, + add_sigil_token(SigilName, Line, Column, NewLine, NewColumn, Parts, Rest, NewScope, Tokens, Indentation, <>); + + {error, Reason} -> + error(Reason, [$~] ++ SigilName ++ Original, Scope, Tokens) + end; + +tokenize_sigil_contents([H | T] = Original, [S | _] = SigilName, Line, Column, Scope, Tokens) + when ?is_sigil(H) -> + case elixir_interpolation:extract(Line, Column + 1, Scope, ?is_downcase(S), T, sigil_terminator(H)) of + {NewLine, NewColumn, Parts, Rest, NewScope} -> + Indentation = nil, + add_sigil_token(SigilName, Line, Column, NewLine, NewColumn, tokens_to_binary(Parts), Rest, NewScope, Tokens, Indentation, <>); + + {error, Reason} -> + Sigil = [$~, S, H], + Message = " (for sigil ~ts starting at line ~B)", + interpolation_error(Reason, [$~] ++ SigilName ++ Original, Scope, Tokens, Message, [Sigil, Line], Line, Column, [H], [sigil_terminator(H)]) + end; + +tokenize_sigil_contents([H | _] = Original, SigilName, Line, Column, Scope, Tokens) -> + MessageString = + "\"~ts\" (column ~p, code point U+~4.16.0B). The available delimiters are: " + "//, ||, \"\", '', (), [], {}, <>", + Message = io_lib:format(MessageString, [[H], Column, H]), + ErrorColumn = Column - 1 - length(SigilName), + error({?LOC(Line, ErrorColumn), "invalid sigil delimiter: ", Message}, [$~] ++ SigilName ++ Original, Scope, Tokens); + +% Incomplete sigil. +tokenize_sigil_contents([], _SigilName, Line, Column, Scope, Tokens) -> + tokenize([], Line, Column, Scope, Tokens). + +add_sigil_token(SigilName, Line, Column, NewLine, NewColumn, Parts, Rest, Scope, Tokens, Indentation, Delimiter) -> + TokenColumn = Column - 1 - length(SigilName), + MaybeEncoded = case SigilName of + % Single-letter sigils present no risk of atom exhaustion (limited possibilities) + [_Char] -> {ok, list_to_atom("sigil_" ++ SigilName)}; + _ -> unsafe_to_atom("sigil_" ++ SigilName, Line, TokenColumn, Scope) + end, + case MaybeEncoded of + {ok, Atom} -> + {Final, Modifiers} = collect_modifiers(Rest, []), + Token = {sigil, {Line, TokenColumn, nil}, Atom, Parts, Modifiers, Indentation, Delimiter}, + NewColumnWithModifiers = NewColumn + length(Modifiers), + tokenize(Final, NewLine, NewColumnWithModifiers, Scope, [Token | Tokens]); + + {error, Reason} -> + error(Reason, Rest, Scope, Tokens) end. -%% do is only valid after the end, true, false and nil keywords -do_keyword_valid([{Atom, _}|_]) -> +%% Fail early on invalid do syntax. For example, after +%% most keywords, after comma and so on. +tokenize_keyword_terminator(DoLine, DoColumn, do, [{identifier, {Line, Column, Meta}, Atom} | T]) -> + {ok, add_token_with_eol({do, {DoLine, DoColumn, nil}}, + [{do_identifier, {Line, Column, Meta}, Atom} | T])}; +tokenize_keyword_terminator(_Line, _Column, do, [{'fn', _} | _]) -> + {error, invalid_do_with_fn_error("unexpected reserved word: "), "do"}; +tokenize_keyword_terminator(Line, Column, do, Tokens) -> + case is_valid_do(Tokens) of + true -> {ok, add_token_with_eol({do, {Line, Column, nil}}, Tokens)}; + false -> {error, invalid_do_error("unexpected reserved word: "), "do"} + end; +tokenize_keyword_terminator(Line, Column, Atom, Tokens) -> + {ok, [{Atom, {Line, Column, nil}} | Tokens]}. + +is_valid_do([{Atom, _} | _]) -> case Atom of - 'end' -> true; - nil -> true; - true -> true; - false -> true; - _ -> keyword(Atom) == false + ',' -> false; + ';' -> false; + 'not' -> false; + 'and' -> false; + 'or' -> false; + 'when' -> false; + 'in' -> false; + 'after' -> false; + 'else' -> false; + 'catch' -> false; + 'rescue' -> false; + _ -> true end; -do_keyword_valid(_) -> +is_valid_do(_) -> true. -% Regular keywords -keyword('fn') -> token; -keyword('end') -> token; -keyword('true') -> token; -keyword('false') -> token; -keyword('nil') -> token; - -% Operators keywords -keyword('not') -> unary_op; -keyword('and') -> and_op; -keyword('or') -> or_op; -keyword('xor') -> or_op; -keyword('when') -> when_op; -keyword('in') -> in_op; - -% Block keywords -keyword('after') -> block; -keyword('else') -> block; -keyword('rescue') -> block; -keyword('catch') -> block; - -keyword(_) -> false. +invalid_character_error(What, Char) -> + io_lib:format("invalid character \"~ts\" (code point U+~4.16.0B) in ~ts: ", [[Char], Char, What]). + +invalid_do_error(Prefix) -> + {Prefix, ". In case you wanted to write a \"do\" expression, " + "you must either use do-blocks or separate the keyword argument with comma. " + "For example, you should either write:\n\n" + " if some_condition? do\n" + " :this\n" + " else\n" + " :that\n" + " end\n\n" + "or the equivalent construct:\n\n" + " if(some_condition?, do: :this, else: :that)\n\n" + "where \"some_condition?\" is the first argument and the second argument is a keyword list.\n\n" + "You may see this error if you forget a trailing comma before the \"do\" in a \"do\" block"}. + +invalid_do_with_fn_error(Prefix) -> + {Prefix, ". Anonymous functions are written as:\n\n" + " fn pattern -> expression end\n\nPlease remove the \"do\" keyword"}. + +% TODO: Turn into an error on v2.0 +maybe_warn_too_many_of_same_char([T | _] = Token, [T | _] = _Rest, Line, Column, Scope) -> + Message = io_lib:format( + "found \"~ts\" followed by \"~ts\", please use a space between \"~ts\" and the next \"~ts\"", + [Token, [T], Token, [T]] + ), + prepend_warning(Line, Column, Message, Scope); +maybe_warn_too_many_of_same_char(_Token, _Rest, _Line, _Column, Scope) -> + Scope. + +%% TODO: Turn into an error on v2.0 +maybe_warn_for_ambiguous_bang_before_equals(Kind, Unencoded, [$= | _], Line, Column, Scope) -> + {What, Identifier} = + case Kind of + atom -> {"atom", [$: | Unencoded]}; + identifier -> {"identifier", Unencoded} + end, + + case lists:last(Identifier) of + Last when Last =:= $!; Last =:= $? -> + Msg = io_lib:format("found ~ts \"~ts\", ending with \"~ts\", followed by =. " + "It is unclear if you mean \"~ts ~ts=\" or \"~ts =\". Please add " + "a space before or after ~ts to remove the ambiguity", + [What, Identifier, [Last], lists:droplast(Identifier), [Last], Identifier, [Last]]), + prepend_warning(Line, Column, Msg, Scope); + _ -> + Scope + end; +maybe_warn_for_ambiguous_bang_before_equals(_Kind, _Atom, _Rest, _Line, _Column, Scope) -> + Scope. + +prepend_warning(Line, Column, Msg, #elixir_tokenizer{warnings=Warnings} = Scope) -> + Scope#elixir_tokenizer{warnings = [{{Line, Column}, Msg} | Warnings]}. + +track_ascii(true, Scope) -> Scope; +track_ascii(false, Scope) -> Scope#elixir_tokenizer{ascii_identifiers_only=false}. + +maybe_unicode_lint_warnings(_Ascii=false, Tokens, Warnings) -> + 'Elixir.String.Tokenizer.Security':unicode_lint_warnings(lists:reverse(Tokens)) ++ Warnings; +maybe_unicode_lint_warnings(_Ascii=true, _Tokens, Warnings) -> + Warnings. + +error(Reason, Rest, #elixir_tokenizer{warnings=Warnings}, Tokens) -> + {error, Reason, Rest, Warnings, Tokens}. + +%% Cursor handling + +add_cursor(_Line, Column, noprune, Terminators, Tokens) -> + {Column, Terminators, Tokens}; +add_cursor(Line, Column, prune_and_cursor, Terminators, Tokens) -> + PrePrunedTokens = prune_identifier(Tokens), + PrunedTokens = prune_tokens(PrePrunedTokens, []), + CursorTokens = [ + {')', {Line, Column + 11, nil}}, + {'(', {Line, Column + 10, nil}}, + {paren_identifier, {Line, Column, nil}, '__cursor__'} + | PrunedTokens + ], + {Column + 12, Terminators, CursorTokens}. + +prune_identifier([{identifier, _, _} | Tokens]) -> Tokens; +prune_identifier(Tokens) -> Tokens. + +%%% Any terminator needs to be closed +prune_tokens([{'end', _} | Tokens], Opener) -> + prune_tokens(Tokens, ['end' | Opener]); +prune_tokens([{')', _} | Tokens], Opener) -> + prune_tokens(Tokens, [')' | Opener]); +prune_tokens([{']', _} | Tokens], Opener) -> + prune_tokens(Tokens, [']' | Opener]); +prune_tokens([{'}', _} | Tokens], Opener) -> + prune_tokens(Tokens, ['}' | Opener]); +prune_tokens([{'>>', _} | Tokens], Opener) -> + prune_tokens(Tokens, ['>>' | Opener]); +%%% Close opened terminators +prune_tokens([{'fn', _} | Tokens], ['end' | Opener]) -> + prune_tokens(Tokens, Opener); +prune_tokens([{'do', _} | Tokens], ['end' | Opener]) -> + prune_tokens(Tokens, Opener); +prune_tokens([{'(', _} | Tokens], [')' | Opener]) -> + prune_tokens(Tokens, Opener); +prune_tokens([{'[', _} | Tokens], [']' | Opener]) -> + prune_tokens(Tokens, Opener); +prune_tokens([{'{', _} | Tokens], ['}' | Opener]) -> + prune_tokens(Tokens, Opener); +prune_tokens([{'<<', _} | Tokens], ['>>' | Opener]) -> + prune_tokens(Tokens, Opener); +%%% or it is time to stop... +prune_tokens([{';', _} | _] = Tokens, []) -> + Tokens; +prune_tokens([{'eol', _} | _] = Tokens, []) -> + Tokens; +prune_tokens([{',', _} | _] = Tokens, []) -> + Tokens; +prune_tokens([{'fn', _} | _] = Tokens, []) -> + Tokens; +prune_tokens([{'do', _} | _] = Tokens, []) -> + Tokens; +prune_tokens([{'(', _} | _] = Tokens, []) -> + Tokens; +prune_tokens([{'[', _} | _] = Tokens, []) -> + Tokens; +prune_tokens([{'{', _} | _] = Tokens, []) -> + Tokens; +prune_tokens([{'<<', _} | _] = Tokens, []) -> + Tokens; +prune_tokens([{identifier, _, _} | _] = Tokens, []) -> + Tokens; +prune_tokens([{block_identifier, _, _} | _] = Tokens, []) -> + Tokens; +prune_tokens([{kw_identifier, _, _} | _] = Tokens, []) -> + Tokens; +prune_tokens([{kw_identifier_safe, _, _} | _] = Tokens, []) -> + Tokens; +prune_tokens([{kw_identifier_unsafe, _, _} | _] = Tokens, []) -> + Tokens; +prune_tokens([{OpType, _, _} | _] = Tokens, []) + when OpType =:= comp_op; OpType =:= at_op; OpType =:= unary_op; OpType =:= and_op; + OpType =:= or_op; OpType =:= arrow_op; OpType =:= match_op; OpType =:= in_op; + OpType =:= in_match_op; OpType =:= type_op; OpType =:= dual_op; OpType =:= mult_op; + OpType =:= power_op; OpType =:= concat_op; OpType =:= range_op; OpType =:= xor_op; + OpType =:= pipe_op; OpType =:= stab_op; OpType =:= when_op; OpType =:= assoc_op; + OpType =:= rel_op; OpType =:= ternary_op; OpType =:= capture_op; OpType =:= ellipsis_op -> + Tokens; +%%% or we traverse until the end. +prune_tokens([_ | Tokens], Opener) -> + prune_tokens(Tokens, Opener); +prune_tokens([], _Opener) -> + []. diff --git a/lib/elixir/src/elixir_tokenizer.hrl b/lib/elixir/src/elixir_tokenizer.hrl new file mode 100644 index 00000000000..88325989597 --- /dev/null +++ b/lib/elixir/src/elixir_tokenizer.hrl @@ -0,0 +1,43 @@ +%% SPDX-License-Identifier: Apache-2.0 +%% SPDX-FileCopyrightText: 2021 The Elixir Team + +%% Numbers +-define(is_hex(S), (?is_digit(S) orelse (S >= $A andalso S =< $F) orelse (S >= $a andalso S =< $f))). +-define(is_bin(S), (S >= $0 andalso S =< $1)). +-define(is_octal(S), (S >= $0 andalso S =< $7)). + +%% Digits and letters +-define(is_digit(S), (S >= $0 andalso S =< $9)). +-define(is_upcase(S), (S >= $A andalso S =< $Z)). +-define(is_downcase(S), (S >= $a andalso S =< $z)). + +%% Others +-define(is_quote(S), (S =:= $" orelse S =:= $')). +-define(is_sigil(S), (S =:= $/ orelse S =:= $< orelse S =:= $" orelse S =:= $' orelse + S =:= $[ orelse S =:= $( orelse S =:= ${ orelse S =:= $|)). +-define(LOC(Line, Column), [{line, Line}, {column, Column}]). + +%% Spaces +-define(is_horizontal_space(S), (S =:= $\s orelse S =:= $\t)). +-define(is_vertical_space(S), (S =:= $\r orelse S =:= $\n)). +-define(is_space(S), (?is_horizontal_space(S) orelse ?is_vertical_space(S))). + +%% Bidirectional control +%% Retrieved from https://trojansource.codes/trojan-source.pdf +-define(bidi(C), C =:= 16#202A; + C =:= 16#202B; + C =:= 16#202D; + C =:= 16#202E; + C =:= 16#2066; + C =:= 16#2067; + C =:= 16#2068; + C =:= 16#202C; + C =:= 16#2069). + +%% Unsupported newlines +%% https://www.unicode.org/reports/tr55/ +-define(break(C), C =:= 16#000B; + C =:= 16#000C; + C =:= 16#0085; + C =:= 16#2028; + C =:= 16#2029). \ No newline at end of file diff --git a/lib/elixir/src/elixir_translator.erl b/lib/elixir/src/elixir_translator.erl deleted file mode 100644 index bc0b1389ee9..00000000000 --- a/lib/elixir/src/elixir_translator.erl +++ /dev/null @@ -1,425 +0,0 @@ -%% Translate Elixir quoted expressions to Erlang Abstract Format. -%% Expects the tree to be expanded. --module(elixir_translator). --export([translate/2, translate_arg/3, translate_args/2, translate_block/3]). --import(elixir_scope, [mergev/2, mergec/2]). --import(elixir_errors, [compile_error/3, compile_error/4]). --include("elixir.hrl"). - -%% = - -translate({'=', Meta, [Left, Right]}, S) -> - Return = case Left of - {'_', _, Atom} when is_atom(Atom) -> false; - _ -> true - end, - - {TRight, SR} = translate_block(Right, Return, S), - {TLeft, SL} = elixir_clauses:match(fun translate/2, Left, SR), - {{match, ?line(Meta), TLeft, TRight}, SL}; - -%% Containers - -translate({'{}', Meta, Args}, S) when is_list(Args) -> - {TArgs, SE} = translate_args(Args, S), - {{tuple, ?line(Meta), TArgs}, SE}; - -translate({'%{}', Meta, Args}, S) when is_list(Args) -> - elixir_map:translate_map(Meta, Args, S); - -translate({'%', Meta, [Left, Right]}, S) -> - elixir_map:translate_struct(Meta, Left, Right, S); - -translate({'<<>>', Meta, Args}, S) when is_list(Args) -> - elixir_bitstring:translate(Meta, Args, S); - -%% Blocks - -translate({'__block__', Meta, Args}, #elixir_scope{return=Return} = S) when is_list(Args) -> - {TArgs, SA} = translate_block(Args, [], Return, S#elixir_scope{return=true}), - {{block, ?line(Meta), TArgs}, SA}; - -%% Erlang op - -translate({'__op__', Meta, [Op, Expr]}, S) when is_atom(Op) -> - {TExpr, NS} = translate(Expr, S), - {{op, ?line(Meta), Op, TExpr}, NS}; - -translate({'__op__', Meta, [Op, Left, Right]}, S) when is_atom(Op) -> - {[TLeft, TRight], NS} = translate_args([Left, Right], S), - {{op, ?line(Meta), Op, TLeft, TRight}, NS}; - -%% Lexical - -translate({Lexical, _, [_, _]}, S) when Lexical == import; Lexical == alias; Lexical == require -> - {{atom, 0, nil}, S}; - -%% Pseudo variables - -translate({'__CALLER__', Meta, Atom}, S) when is_atom(Atom) -> - {{var, ?line(Meta), '__CALLER__'}, S#elixir_scope{caller=true}}; - -%% Functions - -translate({'&', Meta, [{'/', [], [{Fun, [], Atom}, Arity]}]}, S) - when is_atom(Fun), is_atom(Atom), is_integer(Arity) -> - {{'fun', ?line(Meta), {function, Fun, Arity}}, S}; -translate({'&', Meta, [Arg]}, S) when is_integer(Arg) -> - compile_error(Meta, S#elixir_scope.file, "unhandled &~B outside of a capture", [Arg]); - -translate({fn, Meta, Clauses}, S) -> - elixir_fn:translate(Meta, Clauses, S); - -%% Cond - -translate({'cond', _Meta, [[{do, Pairs}]]}, S) -> - [{'->', Meta, [[Condition], Body]}|T] = lists:reverse(Pairs), - Case = - case Condition of - {'_', _, Atom} when is_atom(Atom) -> - compile_error(Meta, S#elixir_scope.file, "unbound variable _ inside cond. " - "If you want the last clause to always match, you probably meant to use: true ->"); - X when is_atom(X) and (X /= false) and (X /= nil) -> - build_cond_clauses(T, Body, Meta); - _ -> - {Truthy, Other} = build_truthy_clause(Meta, Condition, Body), - Error = {{'.', Meta, [erlang, error]}, [], [cond_clause]}, - Falsy = {'->', Meta, [[Other], Error]}, - Acc = {'case', Meta, [Condition, [{do, [Truthy, Falsy]}]]}, - build_cond_clauses(T, Acc, Meta) - end, - translate(Case, S); - -%% Case - -translate({'case', Meta, [Expr, KV]}, #elixir_scope{return=Return} = RS) -> - S = RS#elixir_scope{return=true}, - Clauses = elixir_clauses:get_pairs(do, KV, match), - {TExpr, NS} = translate(Expr, S), - {TClauses, TS} = elixir_clauses:clauses(Meta, Clauses, Return, NS), - {{'case', ?line(Meta), TExpr, TClauses}, TS}; - -%% Try - -translate({'try', Meta, [Clauses]}, #elixir_scope{return=Return} = RS) -> - S = RS#elixir_scope{noname=true, return=true}, - Do = proplists:get_value('do', Clauses, nil), - {TDo, SB} = elixir_translator:translate(Do, S), - - Catch = [Tuple || {X, _} = Tuple <- Clauses, X == 'rescue' orelse X == 'catch'], - {TCatch, SC} = elixir_try:clauses(Meta, Catch, Return, mergec(S, SB)), - - case lists:keyfind('after', 1, Clauses) of - {'after', After} -> - {TBlock, SA} = translate(After, mergec(S, SC)), - TAfter = unblock(TBlock); - false -> - {TAfter, SA} = {[], mergec(S, SC)} - end, - - Else = elixir_clauses:get_pairs(else, Clauses, match), - {TElse, SE} = elixir_clauses:clauses(Meta, Else, Return, mergec(S, SA)), - - SF = (mergec(S, SE))#elixir_scope{noname=RS#elixir_scope.noname}, - {{'try', ?line(Meta), unblock(TDo), TElse, TCatch, TAfter}, SF}; - -%% Receive - -translate({'receive', Meta, [KV]}, #elixir_scope{return=Return} = RS) -> - S = RS#elixir_scope{return=true}, - Do = elixir_clauses:get_pairs(do, KV, match, true), - - case lists:keyfind('after', 1, KV) of - false -> - {TClauses, SC} = elixir_clauses:clauses(Meta, Do, Return, S), - {{'receive', ?line(Meta), TClauses}, SC}; - _ -> - After = elixir_clauses:get_pairs('after', KV, expr), - {TClauses, SC} = elixir_clauses:clauses(Meta, Do ++ After, Return, S), - {FClauses, TAfter} = elixir_utils:split_last(TClauses), - {_, _, [FExpr], _, FAfter} = TAfter, - {{'receive', ?line(Meta), FClauses, FExpr, FAfter}, SC} - end; - -%% Comprehensions - -translate({for, Meta, [_|_] = Args}, S) -> - elixir_for:translate(Meta, Args, S); - -%% Super - -translate({super, Meta, Args}, S) when is_list(Args) -> - Module = assert_module_scope(Meta, super, S), - Function = assert_function_scope(Meta, super, S), - elixir_def_overridable:ensure_defined(Meta, Module, Function, S), - - {_, Arity} = Function, - - {TArgs, TS} = if - length(Args) == Arity -> - translate_args(Args, S); - true -> - compile_error(Meta, S#elixir_scope.file, "super must be called with the same number of " - "arguments as the current function") - end, - - Super = elixir_def_overridable:name(Module, Function), - {{call, ?line(Meta), {atom, ?line(Meta), Super}, TArgs}, TS#elixir_scope{super=true}}; - -%% Variables - -translate({'^', Meta, [{Name, VarMeta, Kind}]}, #elixir_scope{context=match} = S) when is_atom(Name), is_atom(Kind) -> - Tuple = {Name, var_kind(VarMeta, Kind)}, - case orddict:find(Tuple, S#elixir_scope.backup_vars) of - {ok, {Value, _Counter}} -> - {{var, ?line(Meta), Value}, S}; - error -> - compile_error(Meta, S#elixir_scope.file, "unbound variable ^~ts", [Name]) - end; - -translate({'_', Meta, Kind}, #elixir_scope{context=match} = S) when is_atom(Kind) -> - {{var, ?line(Meta), '_'}, S}; - -translate({'_', Meta, Kind}, S) when is_atom(Kind) -> - compile_error(Meta, S#elixir_scope.file, "unbound variable _"); - -translate({Name, Meta, Kind}, #elixir_scope{extra=map_key} = S) when is_atom(Name), is_atom(Kind) -> - compile_error(Meta, S#elixir_scope.file, "illegal use of variable ~ts in map key", [Name]); - -translate({Name, Meta, Kind}, S) when is_atom(Name), is_atom(Kind) -> - elixir_scope:translate_var(Meta, Name, var_kind(Meta, Kind), S); - -%% Local calls - -translate({Name, Meta, Args}, S) when is_atom(Name), is_list(Meta), is_list(Args) -> - if - S#elixir_scope.context == match -> - compile_error(Meta, S#elixir_scope.file, - "cannot invoke function ~ts/~B inside match", [Name, length(Args)]); - S#elixir_scope.context == guard -> - Arity = length(Args), - File = S#elixir_scope.file, - case Arity of - 0 -> compile_error(Meta, File, "unknown variable ~ts or cannot invoke " - "function ~ts/~B inside guard", [Name, Name, Arity]); - _ -> compile_error(Meta, File, "cannot invoke local ~ts/~B inside guard", - [Name, Arity]) - end; - S#elixir_scope.function == nil -> - compile_error(Meta, S#elixir_scope.file, "undefined function ~ts/~B", [Name, length(Args)]); - true -> - Line = ?line(Meta), - {TArgs, NS} = translate_args(Args, S), - {{call, Line, {atom, Line, Name}, TArgs}, NS} - end; - -%% Remote calls - -translate({{'.', _, [Left, Right]}, Meta, Args}, S) - when (is_tuple(Left) orelse is_atom(Left)), is_atom(Right), is_list(Meta), is_list(Args) -> - {TLeft, SL} = translate(Left, S), - {TArgs, SA} = translate_args(Args, mergec(S, SL)), - - Line = ?line(Meta), - Arity = length(Args), - TRight = {atom, Line, Right}, - - %% We need to rewrite erlang function calls as operators - %% because erl_eval chokes on them. We can remove this - %% once a fix is merged into Erlang, keeping only the - %% list operators one (since it is required for inlining - %% [1,2,3] ++ Right in matches). - case (Left == erlang) andalso erl_op(Right, Arity) of - true -> - {list_to_tuple([op, Line, Right] ++ TArgs), mergev(SL, SA)}; - false -> - assert_allowed_in_context(Meta, Left, Right, Arity, S), - SC = mergev(SL, SA), - - case not is_atom(Left) andalso (Arity == 0) of - true -> - {Var, _, SV} = elixir_scope:build_var('_', SC), - TVar = {var, Line, Var}, - TMap = {map, Line, [ - {map_field_assoc, Line, - {atom, Line, '__struct__'}, - {atom, Line, 'Elixir.KeyError'}}, - {map_field_assoc, Line, - {atom, Line, '__exception__'}, - {atom, Line, 'true'}}, - {map_field_assoc, Line, - {atom, Line, key}, - {atom, Line, TRight}}, - {map_field_assoc, Line, - {atom, Line, term}, - TVar}]}, - - %% TODO There is a bug in dialyzer that makes it fail on - %% empty maps. We work around the bug below by using - %% the is_map/1 guard instead of matching on map. Hopefully - %% we can use a match on 17.1. - %% - %% http://erlang.org/pipermail/erlang-bugs/2014-April/004338.html - {{'case', -1, TLeft, [ - {clause, -1, - [{map, Line, [{map_field_exact, Line, TRight, TVar}]}], - [], - [TVar]}, - {clause, -1, - [TVar], - [[elixir_utils:erl_call(Line, erlang, is_map, [TVar])]], - [elixir_utils:erl_call(Line, erlang, error, [TMap])]}, - {clause, -1, - [TVar], - [], - [{call, Line, {remote, Line, TVar, TRight}, []}]} - ]}, SV}; - false -> - {{call, Line, {remote, Line, TLeft, TRight}, TArgs}, SC} - end - end; - -%% Anonymous function calls - -translate({{'.', _, [Expr]}, Meta, Args}, S) when is_list(Args) -> - {TExpr, SE} = translate(Expr, S), - {TArgs, SA} = translate_args(Args, mergec(S, SE)), - {{call, ?line(Meta), TExpr, TArgs}, mergev(SE, SA)}; - -%% Literals - -translate(List, S) when is_list(List) -> - Fun = case S#elixir_scope.context of - match -> fun translate/2; - _ -> fun(X, Acc) -> translate_arg(X, Acc, S) end - end, - translate_list(List, Fun, S, []); - -translate({Left, Right}, S) -> - {TArgs, SE} = translate_args([Left, Right], S), - {{tuple, 0, TArgs}, SE}; - -translate(Other, S) -> - {elixir_utils:elixir_to_erl(Other), S}. - -%% Helpers - -erl_op(Op, Arity) -> - erl_internal:list_op(Op, Arity) orelse - erl_internal:comp_op(Op, Arity) orelse - erl_internal:bool_op(Op, Arity) orelse - erl_internal:arith_op(Op, Arity). - -translate_list([{'|', _, [_, _]=Args}], Fun, Acc, List) -> - {[TLeft,TRight], TAcc} = lists:mapfoldl(Fun, Acc, Args), - {build_list([TLeft|List], TRight), TAcc}; -translate_list([H|T], Fun, Acc, List) -> - {TH, TAcc} = Fun(H, Acc), - translate_list(T, Fun, TAcc, [TH|List]); -translate_list([], _Fun, Acc, List) -> - {build_list(List, {nil, 0}), Acc}. - -build_list([H|T], Acc) -> - build_list(T, {cons, 0, H, Acc}); -build_list([], Acc) -> - Acc. - -var_kind(Meta, Kind) -> - case lists:keyfind(counter, 1, Meta) of - {counter, Counter} -> Counter; - false -> Kind - end. - -%% Pack a list of expressions from a block. -unblock({'block', _, Exprs}) -> Exprs; -unblock(Expr) -> [Expr]. - -%% Translate args - -translate_arg(Arg, Acc, S) when is_number(Arg); is_atom(Arg); is_binary(Arg); is_pid(Arg); is_function(Arg) -> - {TArg, _} = translate(Arg, S), - {TArg, Acc}; -translate_arg(Arg, Acc, S) -> - {TArg, TAcc} = translate(Arg, mergec(S, Acc)), - {TArg, mergev(Acc, TAcc)}. - -translate_args(Args, #elixir_scope{context=match} = S) -> - lists:mapfoldl(fun translate/2, S, Args); - -translate_args(Args, S) -> - lists:mapfoldl(fun(X, Acc) -> translate_arg(X, Acc, S) end, S, Args). - -%% Translate blocks - -translate_block([], Acc, _Return, S) -> - {lists:reverse(Acc), S}; -translate_block([H], Acc, Return, S) -> - {TH, TS} = translate_block(H, Return, S), - translate_block([], [TH|Acc], Return, TS); -translate_block([H|T], Acc, Return, S) -> - {TH, TS} = translate_block(H, false, S), - translate_block(T, [TH|Acc], Return, TS). - -translate_block(Expr, Return, S) -> - case (Return == false) andalso handles_no_return(Expr) of - true -> translate(Expr, S#elixir_scope{return=Return}); - false -> translate(Expr, S) - end. - -%% return is typically true, except when we find one -%% of the expressions below, which may handle return=false -%% but must always return return=true. -handles_no_return({'try', _, [_]}) -> true; -handles_no_return({'cond', _, [_]}) -> true; -handles_no_return({'for', _, [_|_]}) -> true; -handles_no_return({'case', _, [_, _]}) -> true; -handles_no_return({'receive', _, [_]}) -> true; -handles_no_return({'__block__', _, [_|_]}) -> true; -handles_no_return(_) -> false. - -%% Cond - -build_cond_clauses([{'->', NewMeta, [[Condition], Body]}|T], Acc, OldMeta) -> - {Truthy, Other} = build_truthy_clause(NewMeta, Condition, Body), - Falsy = {'->', OldMeta, [[Other], Acc]}, - Case = {'case', NewMeta, [Condition, [{do, [Truthy, Falsy]}]]}, - build_cond_clauses(T, Case, NewMeta); -build_cond_clauses([], Acc, _) -> - Acc. - -build_truthy_clause(Meta, Condition, Body) -> - case elixir_utils:returns_boolean(Condition) of - true -> - {{'->', Meta, [[true], Body]}, false}; - false -> - Var = {'cond', [], 'Elixir'}, - Head = {'when', [], [Var, - {'__op__', [], [ - 'andalso', - {{'.', [], [erlang, '/=']}, [], [Var, nil]}, - {{'.', [], [erlang, '/=']}, [], [Var, false]} - ]} - ]}, - {{'->', Meta, [[Head], Body]}, {'_', [], nil}} - end. - -%% Assertions - -assert_module_scope(Meta, Kind, #elixir_scope{module=nil,file=File}) -> - compile_error(Meta, File, "cannot invoke ~ts outside module", [Kind]); -assert_module_scope(_Meta, _Kind, #elixir_scope{module=Module}) -> Module. - -assert_function_scope(Meta, Kind, #elixir_scope{function=nil,file=File}) -> - compile_error(Meta, File, "cannot invoke ~ts outside function", [Kind]); -assert_function_scope(_Meta, _Kind, #elixir_scope{function=Function}) -> Function. - -assert_allowed_in_context(Meta, Left, Right, Arity, #elixir_scope{context=Context} = S) - when (Context == match) orelse (Context == guard) -> - case (Left == erlang) andalso erl_internal:guard_bif(Right, Arity) of - true -> ok; - false -> - compile_error(Meta, S#elixir_scope.file, "cannot invoke remote function ~ts.~ts/~B inside ~ts", - ['Elixir.Macro':to_string(Left), Right, Arity, Context]) - end; -assert_allowed_in_context(_, _, _, _, _) -> - ok. diff --git a/lib/elixir/src/elixir_try.erl b/lib/elixir/src/elixir_try.erl deleted file mode 100644 index 73729a70ec9..00000000000 --- a/lib/elixir/src/elixir_try.erl +++ /dev/null @@ -1,195 +0,0 @@ --module(elixir_try). --export([clauses/4]). --include("elixir.hrl"). - -clauses(_Meta, Clauses, Return, S) -> - Catch = elixir_clauses:get_pairs('catch', Clauses, 'catch'), - Rescue = elixir_clauses:get_pairs(rescue, Clauses, rescue), - reduce_clauses(Rescue ++ Catch, [], S, Return, S). - -reduce_clauses([H|T], Acc, SAcc, Return, S) -> - {TH, TS} = each_clause(H, Return, SAcc), - reduce_clauses(T, TH ++ Acc, elixir_scope:mergec(S, TS), Return, S); -reduce_clauses([], Acc, SAcc, _Return, _S) -> - {lists:reverse(Acc), SAcc}. - -each_clause({'catch', Meta, Raw, Expr}, Return, S) -> - {Args, Guards} = elixir_clauses:extract_splat_guards(Raw), - - Final = case Args of - [X] -> [throw, X, {'_', Meta, nil}]; - [X,Y] -> [X, Y, {'_', Meta, nil}] - end, - - Condition = [{'{}', Meta, Final}], - {TC, TS} = elixir_clauses:clause(?line(Meta), fun elixir_translator:translate_args/2, - Condition, Expr, Guards, Return, S), - {[TC], TS}; - -each_clause({rescue, Meta, [{in, _, [Left, Right]}], Expr}, Return, S) -> - {VarName, _, CS} = elixir_scope:build_var('_', S), - Var = {VarName, Meta, nil}, - {Parts, Safe, FS} = rescue_guards(Meta, Var, Right, CS), - - Body = - case Left of - {'_', _, Atom} when is_atom(Atom) -> - Expr; - _ -> - Normalized = - case Safe of - true -> Var; - false -> {{'.', Meta, ['Elixir.Exception', normalize]}, Meta, [error, Var]} - end, - prepend_to_block(Meta, {'=', Meta, [Left, Normalized]}, Expr) - end, - - build_rescue(Meta, Parts, Body, Return, FS); - -each_clause({rescue, Meta, _, _}, _Return, S) -> - elixir_errors:compile_error(Meta, S#elixir_scope.file, "invalid arguments for rescue in try"); - -each_clause({Key, Meta, _, _}, _Return, S) -> - elixir_errors:compile_error(Meta, S#elixir_scope.file, "invalid key ~ts in try", [Key]). - -%% Helpers - -build_rescue(Meta, Parts, Body, Return, S) -> - Matches = [Match || {Match, _} <- Parts], - - {{clause, Line, TMatches, _, TBody}, TS} = - elixir_clauses:clause(?line(Meta), fun elixir_translator:translate_args/2, - Matches, Body, [], Return, S), - - TClauses = - [begin - TArgs = [{tuple, Line, [{atom, Line, error}, TMatch, {var, Line, '_'}]}], - TGuards = elixir_clauses:guards(Line, Guards, [], TS), - {clause, Line, TArgs, TGuards, TBody} - end || {TMatch, {_, Guards}} <- lists:zip(TMatches, Parts)], - - {TClauses, TS}. - -%% Convert rescue clauses into guards. -rescue_guards(_, Var, {'_', _, _}, S) -> {[{Var, []}], false, S}; - -rescue_guards(Meta, Var, Aliases, S) -> - {Elixir, Erlang} = rescue_each_ref(Meta, Var, Aliases, [], [], S), - - {ElixirParts, ES} = - case Elixir of - [] -> {[], S}; - _ -> - {VarName, _, CS} = elixir_scope:build_var('_', S), - StructVar = {VarName, Meta, nil}, - Map = {'%{}', Meta, [{'__struct__', StructVar}, {'__exception__', true}]}, - Match = {'=', Meta, [Map, Var]}, - Guards = [{erl(Meta, '=='), Meta, [StructVar, Mod]} || Mod <- Elixir], - {[{Match, Guards}], CS} - end, - - ErlangParts = - case Erlang of - [] -> []; - _ -> [{Var, Erlang}] - end, - - {ElixirParts ++ ErlangParts, ErlangParts == [], ES}. - -%% Rescue each atom name considering their Erlang or Elixir matches. -%% Matching of variables is done with Erlang exceptions is done in -%% function for optimization. - -rescue_each_ref(Meta, Var, [H|T], Elixir, Erlang, S) when is_atom(H) -> - case erl_rescue_guard_for(Meta, Var, H) of - false -> rescue_each_ref(Meta, Var, T, [H|Elixir], Erlang, S); - Expr -> rescue_each_ref(Meta, Var, T, [H|Elixir], [Expr|Erlang], S) - end; - -rescue_each_ref(_, _, [], Elixir, Erlang, _) -> - {Elixir, Erlang}. - -%% Handle erlang rescue matches. - -erl_rescue_guard_for(Meta, Var, 'Elixir.UndefinedFunctionError') -> - {erl(Meta, '=='), Meta, [Var, undef]}; - -erl_rescue_guard_for(Meta, Var, 'Elixir.FunctionClauseError') -> - {erl(Meta, '=='), Meta, [Var, function_clause]}; - -erl_rescue_guard_for(Meta, Var, 'Elixir.SystemLimitError') -> - {erl(Meta, '=='), Meta, [Var, system_limit]}; - -erl_rescue_guard_for(Meta, Var, 'Elixir.ArithmeticError') -> - {erl(Meta, '=='), Meta, [Var, badarith]}; - -erl_rescue_guard_for(Meta, Var, 'Elixir.CondClauseError') -> - {erl(Meta, '=='), Meta, [Var, cond_clause]}; - -erl_rescue_guard_for(Meta, Var, 'Elixir.BadArityError') -> - erl_and(Meta, - erl_tuple_size(Meta, Var, 2), - erl_record_compare(Meta, Var, badarity)); - -erl_rescue_guard_for(Meta, Var, 'Elixir.BadFunctionError') -> - erl_and(Meta, - erl_tuple_size(Meta, Var, 2), - erl_record_compare(Meta, Var, badfun)); - -erl_rescue_guard_for(Meta, Var, 'Elixir.MatchError') -> - erl_and(Meta, - erl_tuple_size(Meta, Var, 2), - erl_record_compare(Meta, Var, badmatch)); - -erl_rescue_guard_for(Meta, Var, 'Elixir.CaseClauseError') -> - erl_and(Meta, - erl_tuple_size(Meta, Var, 2), - erl_record_compare(Meta, Var, case_clause)); - -erl_rescue_guard_for(Meta, Var, 'Elixir.TryClauseError') -> - erl_and(Meta, - erl_tuple_size(Meta, Var, 2), - erl_record_compare(Meta, Var, try_clause)); - -erl_rescue_guard_for(Meta, Var, 'Elixir.BadStructError') -> - erl_and(Meta, - erl_tuple_size(Meta, Var, 3), - erl_record_compare(Meta, Var, badstruct)); - -erl_rescue_guard_for(Meta, Var, 'Elixir.ArgumentError') -> - erl_or(Meta, - {erl(Meta, '=='), Meta, [Var, badarg]}, - erl_and(Meta, - erl_tuple_size(Meta, Var, 2), - erl_record_compare(Meta, Var, badarg))); - -erl_rescue_guard_for(Meta, Var, 'Elixir.ErlangError') -> - IsNotTuple = {erl(Meta, 'not'), Meta, [{erl(Meta, is_tuple), Meta, [Var]}]}, - IsException = {erl(Meta, '/='), Meta, [ - {erl(Meta, element), Meta, [2, Var]}, '__exception__' - ]}, - erl_or(Meta, IsNotTuple, IsException); - -erl_rescue_guard_for(_, _, _) -> - false. - -%% Helpers - -erl_tuple_size(Meta, Var, Size) -> - {erl(Meta, '=='), Meta, [{erl(Meta, tuple_size), Meta, [Var]}, Size]}. - -erl_record_compare(Meta, Var, Expr) -> - {erl(Meta, '=='), Meta, [ - {erl(Meta, element), Meta, [1, Var]}, - Expr - ]}. - -prepend_to_block(_Meta, Expr, {'__block__', Meta, Args}) -> - {'__block__', Meta, [Expr|Args]}; - -prepend_to_block(Meta, Expr, Args) -> - {'__block__', Meta, [Expr, Args]}. - -erl(Meta, Op) -> {'.', Meta, [erlang, Op]}. -erl_or(Meta, Left, Right) -> {'__op__', Meta, ['orelse', Left, Right]}. -erl_and(Meta, Left, Right) -> {'__op__', Meta, ['andalso', Left, Right]}. diff --git a/lib/elixir/src/elixir_utils.erl b/lib/elixir/src/elixir_utils.erl index 1268bd6b185..0a6095a9e72 100644 --- a/lib/elixir/src/elixir_utils.erl +++ b/lib/elixir/src/elixir_utils.erl @@ -1,38 +1,110 @@ +%% SPDX-License-Identifier: Apache-2.0 +%% SPDX-FileCopyrightText: 2021 The Elixir Team +%% SPDX-FileCopyrightText: 2012 Plataformatec + %% Convenience functions used throughout elixir source code %% for ast manipulation and querying. -module(elixir_utils). --export([elixir_to_erl/1, get_line/1, split_last/1, - characters_to_list/1, characters_to_binary/1, macro_name/1, - convert_to_boolean/4, returns_boolean/1, atom_concat/1, - read_file_type/1, read_link_type/1, relative_to_cwd/1, erl_call/4]). +-export([get_line/1, get_line/2, get_file/2, generated/1, + split_last/1, split_opts/1, noop/0, var_context/2, + characters_to_list/1, characters_to_binary/1, relative_to_cwd/1, + macro_name/1, returns_boolean/1, caller/4, meta_keep/1, + read_file_type/1, read_file_type/2, read_link_type/1, read_posix_mtime_and_size/1, + change_posix_time/2, change_universal_time/2, var_info/2, + guard_op/2, guard_info/1, extract_splat_guards/1, extract_guards/1, + erlang_comparison_op_to_elixir/1, erl_fa_to_elixir_fa/2, jaro_similarity/2]). -include("elixir.hrl"). -include_lib("kernel/include/file.hrl"). +var_info(Name, Kind) when Kind == nil; is_integer(Kind) -> + io_lib:format("\"~ts\"", [Name]); +var_info(Name, Kind) -> + io_lib:format("\"~ts\" (context ~ts)", [Name, elixir_aliases:inspect(Kind)]). + +guard_info(#elixir_ex{prematch={_, _, {bitsize, _}}}) -> "bitstring size specifier"; +guard_info(_) -> "guard". + macro_name(Macro) -> - list_to_atom(lists:concat(['MACRO-',Macro])). + list_to_atom("MACRO-" ++ atom_to_list(Macro)). -atom_concat(Atoms) -> - list_to_atom(lists:concat(Atoms)). +erl_fa_to_elixir_fa(Name, Arity) -> + case atom_to_list(Name) of + "MACRO-" ++ Rest -> {list_to_atom(Rest), Arity - 1}; + _ -> {Name, Arity} + end. -erl_call(Line, Module, Function, Args) -> - {call, Line, - {remote, Line, {atom, Line, Module}, {atom, Line, Function}}, - Args - }. +guard_op('andalso', 2) -> + true; +guard_op('orelse', 2) -> + true; +guard_op(Op, Arity) -> + try erl_internal:op_type(Op, Arity) of + arith -> true; + comp -> true; + bool -> true; + list -> false; + send -> false + catch + _:_ -> false + end. -get_line(Opts) when is_list(Opts) -> - case lists:keyfind(line, 1, Opts) of - {line, Line} when is_integer(Line) -> Line; - false -> 0 +erlang_comparison_op_to_elixir('/=') -> '!='; +erlang_comparison_op_to_elixir('=<') -> '<='; +erlang_comparison_op_to_elixir('=:=') -> '==='; +erlang_comparison_op_to_elixir('=/=') -> '!=='; +erlang_comparison_op_to_elixir(Other) -> Other. + +var_context(Meta, Kind) -> + case lists:keyfind(counter, 1, Meta) of + {counter, Counter} -> Counter; + false -> Kind end. -split_last([]) -> {[], []}; -split_last(List) -> split_last(List, []). -split_last([H], Acc) -> {lists:reverse(Acc), H}; -split_last([H|T], Acc) -> split_last(T, [H|Acc]). +% Extract guards + +extract_guards({'when', _, [Left, Right]}) -> {Left, extract_or_guards(Right)}; +extract_guards(Else) -> {Else, []}. + +extract_or_guards({'when', _, [Left, Right]}) -> [Left | extract_or_guards(Right)]; +extract_or_guards(Term) -> [Term]. + +% Extract guards when multiple left side args are allowed. + +extract_splat_guards([{'when', _, [_ | _] = Args}]) -> + {Left, Right} = split_last(Args), + {Left, extract_or_guards(Right)}; +extract_splat_guards(Else) -> + {Else, []}. + +%% No-op function that can be used for stuff like preventing tail-call +%% optimization to kick in. +noop() -> + ok. + +split_last([]) -> {[], []}; +split_last(List) -> split_last(List, []). +split_last([H], Acc) -> {lists:reverse(Acc), H}; +split_last([H | T], Acc) -> split_last(T, [H | Acc]). + +%% Useful to handle options similarly in `opts, do ... end` and `opts, do: ...`. +split_opts(Args) -> + case elixir_utils:split_last(Args) of + {OuterCases, OuterOpts} when is_list(OuterOpts) -> + case elixir_utils:split_last(OuterCases) of + {InnerCases, InnerOpts} when is_list(InnerOpts) -> + {InnerCases, InnerOpts ++ OuterOpts}; + _ -> + {OuterCases, OuterOpts} + end; + _ -> + {Args, []} + end. read_file_type(File) -> - case file:read_file_info(File) of + read_file_type(File, []). + +read_file_type(File, Opts) -> + case file:read_file_info(File, [{time, posix} | Opts]) of {ok, #file_info{type=Type}} -> {ok, Type}; {error, _} = Error -> Error end. @@ -43,89 +115,96 @@ read_link_type(File) -> {error, _} = Error -> Error end. +read_posix_mtime_and_size(File) -> + case file:read_file_info(File, [raw, {time, posix}]) of + {ok, #file_info{mtime=Mtime, size=Size}} -> {ok, Mtime, Size}; + {error, _} = Error -> Error + end. + +change_posix_time(Name, Time) when is_integer(Time) -> + file:write_file_info(Name, #file_info{mtime=Time}, [raw, {time, posix}]). + +change_universal_time(Name, {{Y, M, D}, {H, Min, Sec}}=Time) + when is_integer(Y), is_integer(M), is_integer(D), + is_integer(H), is_integer(Min), is_integer(Sec) -> + file:write_file_info(Name, #file_info{mtime=Time}, [raw, {time, universal}]). + relative_to_cwd(Path) -> - case elixir_compiler:get_opt(internal) of - true -> Path; - false -> 'Elixir.String':to_char_list('Elixir.Path':relative_to_cwd(Path)) + try elixir_config:get(relative_paths) of + true -> 'Elixir.Path':relative_to_cwd(Path); + false -> Path + catch + _:_ -> Path end. characters_to_list(Data) when is_list(Data) -> Data; characters_to_list(Data) -> - case elixir_compiler:get_opt(internal) of - true -> unicode:characters_to_list(Data); - false -> 'Elixir.String':to_char_list(Data) + case unicode:characters_to_list(Data) of + Result when is_list(Result) -> Result; + {error, Encoded, Rest} -> conversion_error(invalid, Encoded, Rest); + {incomplete, Encoded, Rest} -> conversion_error(incomplete, Encoded, Rest) end. characters_to_binary(Data) when is_binary(Data) -> Data; characters_to_binary(Data) -> - case elixir_compiler:get_opt(internal) of - true -> unicode:characters_to_binary(Data); - false -> 'Elixir.List':to_string(Data) + case unicode:characters_to_binary(Data) of + Result when is_binary(Result) -> Result; + {error, Encoded, Rest} -> conversion_error(invalid, Encoded, Rest); + {incomplete, Encoded, Rest} -> conversion_error(incomplete, Encoded, Rest) end. -%% elixir to erl. Handles only valid quoted expressions, -%% that's why things like maps and references are not in the list. - -elixir_to_erl(Tree) when is_tuple(Tree) -> - {tuple, 0, [elixir_to_erl(X) || X <- tuple_to_list(Tree)]}; +conversion_error(Kind, Encoded, Rest) -> + error('Elixir.UnicodeConversionError':exception([{encoded, Encoded}, {rest, Rest}, {kind, Kind}])). -elixir_to_erl([]) -> - {nil, 0}; +%% Returns the caller as a stacktrace entry. +caller(Line, File, nil, _) -> + {elixir_compiler_0, '__FILE__', 1, stack_location(Line, File)}; +caller(Line, File, Module, nil) -> + {Module, '__MODULE__', 0, stack_location(Line, File)}; +caller(Line, File, Module, {Name, Arity}) -> + {Module, Name, Arity, stack_location(Line, File)}. -elixir_to_erl(<<>>) -> - {bin, 0, []}; +stack_location(Line, File) -> + [{file, elixir_utils:characters_to_list(elixir_utils:relative_to_cwd(File))}, + {line, Line}]. -elixir_to_erl(Tree) when is_list(Tree) -> - elixir_to_erl_cons_1(Tree, []); - -elixir_to_erl(Tree) when is_atom(Tree) -> - {atom, 0, Tree}; - -elixir_to_erl(Tree) when is_integer(Tree) -> - {integer, 0, Tree}; - -elixir_to_erl(Tree) when is_float(Tree) -> - {float, 0, Tree}; - -elixir_to_erl(Tree) when is_binary(Tree) -> - %% Note that our binaries are utf-8 encoded and we are converting - %% to a list using binary_to_list. The reason for this is that Erlang - %% considers a string in a binary to be encoded in latin1, so the bytes - %% are not changed in any fashion. - {bin, 0, [{bin_element, 0, {string, 0, binary_to_list(Tree)}, default, default}]}; - -elixir_to_erl(Function) when is_function(Function) -> - case (erlang:fun_info(Function, type) == {type, external}) andalso - (erlang:fun_info(Function, env) == {env, []}) of - true -> - {module, Module} = erlang:fun_info(Function, module), - {name, Name} = erlang:fun_info(Function, name), - {arity, Arity} = erlang:fun_info(Function, arity), - - {'fun', 0, {function, - {atom, 0, Module}, - {atom, 0, Name}, - {integer, 0, Arity}}}; - false -> - error(badarg) - end; - -elixir_to_erl(Pid) when is_pid(Pid) -> - elixir_utils:erl_call(0, erlang, binary_to_term, - [elixir_utils:elixir_to_erl(term_to_binary(Pid))]); +get_line(Opts) when is_list(Opts) -> + case lists:keyfind(line, 1, Opts) of + {line, Line} when is_integer(Line) -> Line; + _ -> 0 + end. -elixir_to_erl(_Other) -> - error(badarg). +get_line(Meta, Env) when is_list(Meta) -> + case lists:keyfind(line, 1, Meta) of + {line, LineOpt} when is_integer(LineOpt) -> LineOpt; + false -> ?key(Env, line) + end. -elixir_to_erl_cons_1([H|T], Acc) -> elixir_to_erl_cons_1(T, [H|Acc]); -elixir_to_erl_cons_1(Other, Acc) -> elixir_to_erl_cons_2(Acc, elixir_to_erl(Other)). +get_file(Meta, Env) when is_list(Meta) -> + case lists:keyfind(file, 1, Meta) of + {file, FileOpt} when is_binary(FileOpt) -> FileOpt; + false -> ?key(Env, file) + end. -elixir_to_erl_cons_2([H|T], Acc) -> - elixir_to_erl_cons_2(T, {cons, 0, elixir_to_erl(H), Acc}); -elixir_to_erl_cons_2([], Acc) -> - Acc. +generated([{generated, true} | _] = Meta) -> Meta; +generated(Meta) -> [{generated, true} | Meta]. + +%% Meta location. +%% +%% Macros add a file pair on location keep which we +%% should take into account for error reporting. +%% +%% Returns {binary, integer} on location keep or nil. + +meta_keep(Meta) -> + case lists:keyfind(keep, 1, Meta) of + {keep, {File, Line} = Pair} when is_binary(File), is_integer(Line) -> + Pair; + _ -> + nil + end. %% Boolean checks @@ -138,61 +217,116 @@ returns_boolean({{'.', _, [erlang, Op]}, _, [_, _]}) when Op == '=='; Op == '/='; Op == '=<'; Op == '>='; Op == '<'; Op == '>'; Op == '=:='; Op == '=/=' -> true; -returns_boolean({'__op__', _, [Op, _, Right]}) when Op == 'andalso'; Op == 'orelse' -> +returns_boolean({{'.', _, [erlang, Op]}, _, [_, Right]}) when + Op == 'andalso'; Op == 'orelse' -> returns_boolean(Right); returns_boolean({{'.', _, [erlang, Fun]}, _, [_]}) when Fun == is_atom; Fun == is_binary; Fun == is_bitstring; Fun == is_boolean; Fun == is_float; Fun == is_function; Fun == is_integer; Fun == is_list; Fun == is_number; Fun == is_pid; Fun == is_port; Fun == is_reference; - Fun == is_tuple -> true; + Fun == is_tuple; Fun == is_map; Fun == is_process_alive -> true; returns_boolean({{'.', _, [erlang, Fun]}, _, [_, _]}) when - Fun == is_function -> true; + Fun == is_map_key; Fun == is_function; Fun == is_record -> true; returns_boolean({{'.', _, [erlang, Fun]}, _, [_, _, _]}) when - Fun == function_exported -> true; + Fun == function_exported; Fun == is_record -> true; returns_boolean({'case', _, [_, [{do, Clauses}]]}) -> lists:all(fun - ({'->',_,[_, Expr]}) -> returns_boolean(Expr) + ({'->', _, [_, Expr]}) -> returns_boolean(Expr) end, Clauses); returns_boolean({'cond', _, [[{do, Clauses}]]}) -> lists:all(fun - ({'->',_,[_, Expr]}) -> returns_boolean(Expr) + ({'->', _, [_, Expr]}) -> returns_boolean(Expr) end, Clauses); -returns_boolean({'__block__', [], Exprs}) -> +returns_boolean({'__block__', _, Exprs}) -> returns_boolean(lists:last(Exprs)); returns_boolean(_) -> false. -convert_to_boolean(Line, Expr, Bool, S) when is_integer(Line) -> - case {returns_boolean(Expr), Bool} of - {true, true} -> {Expr, S}; - {true, false} -> {{op, Line, 'not', Expr}, S}; - _ -> do_convert_to_boolean(Line, Expr, Bool, S) - end. -%% Notice we use a temporary var and bundle nil -%% and false checks in the same clause since -%% it makes dialyzer happy. -do_convert_to_boolean(Line, Expr, Bool, S) -> - {Name, _, TS} = elixir_scope:build_var('_', S), - Var = {var, Line, Name}, - Any = {var, Line, '_'}, - OrElse = do_guarded_convert_to_boolean(Line, Var, 'orelse', '=='), - - FalseResult = {atom,Line,not Bool}, - TrueResult = {atom,Line,Bool}, - - {{'case', Line, Expr, [ - {clause, Line, [Var], [[OrElse]], [FalseResult]}, - {clause, Line, [Any], [], [TrueResult]} - ]}, TS}. - -do_guarded_convert_to_boolean(Line, Expr, Op, Comp) -> - Left = {op, Line, Comp, Expr, {atom, Line, false}}, - Right = {op, Line, Comp, Expr, {atom, Line, nil}}, - {op, Line, Op, Left, Right}. +% TODO: Remove me when we require Erlang/OTP 27+ +% This is a polyfill for older versions, copying the code from +% https://github.com/erlang/otp/pull/7879 +-spec jaro_similarity(String1, String2) -> Similarity when + String1 :: unicode:chardata(), + String2 :: unicode:chardata(), + Similarity :: float(). %% Between +0.0 and 1.0 +jaro_similarity(A0, B0) -> + {A, ALen} = str_to_gcl_and_length(A0), + {B, BLen} = str_to_indexmap(B0), + Dist = max(ALen, BLen) div 2, + {AM, BM} = jaro_match(A, B, -Dist, Dist, [], []), + if + ALen =:= 0 andalso BLen =:= 0 -> + 1.0; + ALen =:= 0 orelse BLen =:= 0 -> + 0.0; + AM =:= [] -> + 0.0; + true -> + {M,T} = jaro_calc_mt(AM, BM, 0, 0), + (M/ALen + M/BLen + (M-T/2)/M) / 3 + end. + +jaro_match([A|As], B0, Min, Max, AM, BM) -> + case jaro_detect(maps:get(A, B0, []), Min, Max) of + false -> + jaro_match(As, B0, Min+1, Max+1, AM, BM); + {J, Remain} -> + B = B0#{A => Remain}, + jaro_match(As, B, Min+1, Max+1, [A|AM], add_rsorted({J,A},BM)) + end; +jaro_match(_A, _B, _Min, _Max, AM, BM) -> + {AM, BM}. + +jaro_detect([Idx|Rest], Min, Max) when Min < Idx, Idx < Max -> + {Idx, Rest}; +jaro_detect([Idx|Rest], Min, Max) when Idx < Max -> + jaro_detect(Rest, Min, Max); +jaro_detect(_, _, _) -> + false. + +jaro_calc_mt([CharA|AM], [{_, CharA}|BM], M, T) -> + jaro_calc_mt(AM, BM, M+1, T); +jaro_calc_mt([_|AM], [_|BM], M, T) -> + jaro_calc_mt(AM, BM, M+1, T+1); +jaro_calc_mt([], [], M, T) -> + {M, T}. + + +%% Returns GC list and length +str_to_gcl_and_length(S0) -> + gcl_and_length(unicode_util:gc(S0), [], 0). + +gcl_and_length([C|Str], Acc, N) -> + gcl_and_length(unicode_util:gc(Str), [C|Acc], N+1); +gcl_and_length([], Acc, N) -> + {lists:reverse(Acc), N}; +gcl_and_length({error, Err}, _, _) -> + error({badarg, Err}). + +%% Returns GC map with index and length +str_to_indexmap(S) -> + [M|L] = str_to_map(unicode_util:gc(S), 0), + {M,L}. + +str_to_map([], L) -> [#{}|L]; +str_to_map([G | Gs], I) -> + [M|L] = str_to_map(unicode_util:gc(Gs), I+1), + [maps:put(G, [I | maps:get(G, M, [])], M)| L]; +str_to_map({error,Error}, _) -> + error({badarg, Error}). + +%% Add in decreasing order +add_rsorted(A, [H|_]=BM) when A > H -> + [A|BM]; +add_rsorted(A, [H|BM]) -> + [H|add_rsorted(A,BM)]; +add_rsorted(A, []) -> + [A]. + diff --git a/lib/elixir/src/iex.erl b/lib/elixir/src/iex.erl new file mode 100644 index 00000000000..5a2f9a9da68 --- /dev/null +++ b/lib/elixir/src/iex.erl @@ -0,0 +1,87 @@ +%% SPDX-License-Identifier: Apache-2.0 +%% SPDX-FileCopyrightText: 2021 The Elixir Team + +-module(iex). +-export([start/0, start/2, shell/0, sync_remote/2]). + +%% Manual tests for changing the CLI boot. +%% +%% 1. In some situations, we cannot read inputs as IEx boots: +%% +%% $ iex -e ":io.get_line(:foo)" +%% +%% 2. In some situations, connecting to a remote node via --remsh +%% is not possible. This can be tested by starting two IEx nodes: +%% +%% $ iex --sname foo +%% $ iex --sname bar --remsh foo +%% +%% 3. When still using --remsh, we need to guarantee the arguments +%% are processed on the local node and not the remote one. For such, +%% one can replace the last line above by: +%% +%% $ iex --sname bar --remsh foo -e 'IO.inspect node()' +%% +%% And verify that the local node name is printed. +%% +%% 4. Finally, in some other circumstances, printing messages may become +%% borked. This can be verified with: +%% +%% $ iex -e ":logger.info(~c'foo~nbar', [])" +%% +%% By the time those instructions have been written, all tests above pass. + +start() -> + start([], {elixir_utils, noop, []}). + +start(Opts, MFA) -> + {ok, _} = application:ensure_all_started(elixir), + {ok, _} = application:ensure_all_started(iex), + + spawn(fun() -> + case init:notify_when_started(self()) of + started -> ok; + _ -> init:wait_until_started() + end, + + ok = io:setopts([{binary, true}, {encoding, unicode}]), + 'Elixir.IEx.Server':run_from_shell(Opts, MFA) + end). + +shell() -> + Args = init:get_plain_arguments(), + + case get_remsh(Args) of + nil -> + start_mfa(Args, {elixir, start_cli, []}); + + Remote -> + Ref = make_ref(), + + Parent = + spawn_link(fun() -> + receive + {'begin', Ref, Other} -> + elixir:start_cli(), + Other ! {done, Ref} + end + end), + + {remote, Remote, start_mfa(Args, {?MODULE, sync_remote, [Parent, Ref]})} + end. + +sync_remote(Parent, Ref) -> + Parent ! {'begin', Ref, self()}, + receive {done, Ref} -> ok end. + +start_mfa(Args, MFA) -> + Opts = [{dot_iex, get_dot_iex(Args)}, {on_eof, halt}], + {?MODULE, start, [Opts, MFA]}. + +get_dot_iex(["--dot-iex", H | _]) -> elixir_utils:characters_to_binary(H); +get_dot_iex([_ | T]) -> get_dot_iex(T); +get_dot_iex([]) -> nil. + +get_remsh(["--remsh", H | _]) -> H; +get_remsh([_ | T]) -> get_remsh(T); +get_remsh([]) -> nil. diff --git a/lib/elixir/test/doc_test.exs b/lib/elixir/test/doc_test.exs deleted file mode 100644 index 3713782a0b1..00000000000 --- a/lib/elixir/test/doc_test.exs +++ /dev/null @@ -1,42 +0,0 @@ -ExUnit.start [] - -defmodule KernelTest do - use ExUnit.Case, async: true - - doctest Access - doctest Atom - doctest Base - doctest Bitwise - doctest Code - doctest Collectable - doctest Enum - doctest Exception - doctest Float - doctest Inspect - doctest Inspect.Algebra - doctest Integer - doctest IO - doctest IO.ANSI - doctest Kernel - doctest Kernel.SpecialForms - doctest Keyword - doctest List - doctest Macro - doctest Map - doctest Module - doctest Node - doctest OptionParser - doctest Path - doctest Process - doctest Protocol - doctest Range - doctest Record - doctest Regex - doctest Stream - doctest String - doctest String.Chars - doctest StringIO - doctest Tuple - doctest URI - doctest Version -end diff --git a/lib/elixir/test/elixir/access_test.exs b/lib/elixir/test/elixir/access_test.exs index b28993addf3..7dda561db09 100644 --- a/lib/elixir/test/elixir/access_test.exs +++ b/lib/elixir/test/elixir/access_test.exs @@ -1,8 +1,14 @@ -Code.require_file "test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("test_helper.exs", __DIR__) defmodule AccessTest do use ExUnit.Case, async: true + doctest Access + # Test nil at compilation time does not fail # and that @config[:foo] has proper precedence. @config nil @@ -19,8 +25,12 @@ defmodule AccessTest do test "for nil" do assert nil[:foo] == nil + assert Access.fetch(nil, :foo) == :error assert Access.get(nil, :foo) == nil - assert Access.get_and_update(nil, :foo, fn nil -> {:ok, :bar} end) == {:ok, :bar} + + assert_raise ArgumentError, "could not put/update key :foo on a nil value", fn -> + Access.get_and_update(nil, :foo, fn nil -> {:ok, :bar} end) + end end test "for keywords" do @@ -28,9 +38,23 @@ defmodule AccessTest do assert [foo: [bar: :baz]][:foo][:bar] == :baz assert [foo: [bar: :baz]][:fuu][:bar] == nil + assert Access.fetch([foo: :bar], :foo) == {:ok, :bar} + assert Access.fetch([foo: :bar], :bar) == :error + + msg = ~r/the Access calls for keywords expect the key to be an atom/ + + assert_raise ArgumentError, msg, fn -> + Access.fetch([], "foo") + end + assert Access.get([foo: :bar], :foo) == :bar assert Access.get_and_update([], :foo, fn nil -> {:ok, :baz} end) == {:ok, [foo: :baz]} - assert Access.get_and_update([foo: :bar], :foo, fn :bar -> {:ok, :baz} end) == {:ok, [foo: :baz]} + + assert Access.get_and_update([foo: :bar], :foo, fn :bar -> {:ok, :baz} end) == + {:ok, [foo: :baz]} + + assert Access.pop([foo: :bar], :foo) == {:bar, []} + assert Access.pop([], :foo) == {nil, []} end test "for maps" do @@ -39,18 +63,298 @@ defmodule AccessTest do assert %{1.0 => 1.0}[1.0] == 1.0 assert %{1 => 1}[1.0] == nil + assert Access.fetch(%{foo: :bar}, :foo) == {:ok, :bar} + assert Access.fetch(%{foo: :bar}, :bar) == :error + assert Access.get(%{foo: :bar}, :foo) == :bar assert Access.get_and_update(%{}, :foo, fn nil -> {:ok, :baz} end) == {:ok, %{foo: :baz}} - assert Access.get_and_update(%{foo: :bar}, :foo, fn :bar -> {:ok, :baz} end) == {:ok, %{foo: :baz}} + + assert Access.get_and_update(%{foo: :bar}, :foo, fn :bar -> {:ok, :baz} end) == + {:ok, %{foo: :baz}} + + assert Access.pop(%{foo: :bar}, :foo) == {:bar, %{}} + assert Access.pop(%{}, :foo) == {nil, %{}} end - test "for atoms" do - assert_raise Protocol.UndefinedError, ~r"protocol Access not implemented for :foo", fn -> - Access.get(:foo, :bar) + test "for struct" do + defmodule Sample do + defstruct [:name] + end + + message = + ~r"function AccessTest.Sample.fetch/2 is undefined \(AccessTest.Sample does not implement the Access behaviour" + + assert_raise UndefinedFunctionError, message, fn -> + Access.fetch(struct(Sample, []), :name) end - assert_raise Protocol.UndefinedError, ~r"protocol Access not implemented for :foo", fn -> - Access.get_and_update(:foo, :bar, fn _ -> {:ok, :baz} end) + message = + ~r"function AccessTest.Sample.get_and_update/3 is undefined \(AccessTest.Sample does not implement the Access behaviour" + + assert_raise UndefinedFunctionError, message, fn -> + Access.get_and_update(struct(Sample, []), :name, fn nil -> {:ok, :baz} end) + end + + message = + ~r"function AccessTest.Sample.pop/2 is undefined \(AccessTest.Sample does not implement the Access behaviour" + + assert_raise UndefinedFunctionError, message, fn -> + Access.pop(struct(Sample, []), :name) + end + end + + describe "fetch!/2" do + assert Access.fetch!(%{foo: :bar}, :foo) == :bar + + assert_raise ArgumentError, + ~r/the Access calls for keywords expect the key to be an atom/, + fn -> Access.fetch!([], "foo") end + + assert_raise KeyError, + ~r/key \"foo\" not found/, + fn -> Access.fetch!(nil, "foo") end + end + + describe "filter/1" do + @test_list [1, 2, 3, 4, 5, 6] + + test "filters in get_in" do + assert get_in(@test_list, [Access.filter(&(&1 > 3))]) == [4, 5, 6] + end + + test "retains order in get_and_update_in" do + assert get_and_update_in(@test_list, [Access.filter(&(&1 == 3 || &1 == 2))], &{&1 * 2, &1}) == + {[4, 6], [1, 2, 3, 4, 5, 6]} + end + + test "retains order in pop_in" do + assert pop_in(@test_list, [Access.filter(&(&1 == 3 || &1 == 2))]) == {[2, 3], [1, 4, 5, 6]} + end + + test "chains with other access functions" do + mixed_map_and_list = %{foo: Enum.map(@test_list, &%{value: &1})} + + assert get_in(mixed_map_and_list, [:foo, Access.filter(&(&1.value <= 3)), :value]) == + [1, 2, 3] + end + end + + describe "slice/1" do + @test_list [1, 2, 3, 4, 5, 6, 7] + + test "retrieves a range from the start of the list" do + assert [2, 3] == get_in(@test_list, [Access.slice(1..2)]) + end + + test "retrieves a range from the end of the list" do + assert [6, 7] == get_in(@test_list, [Access.slice(-2..-1)]) + end + + test "retrieves a range from positive first and negative last" do + assert [2, 3, 4, 5, 6] == get_in(@test_list, [Access.slice(1..-2//1)]) + end + + test "retrieves a range from negative first and positive last" do + assert [6, 7] == get_in(@test_list, [Access.slice(-2..7//1)]) + end + + test "retrieves a range with steps" do + assert [1, 3] == get_in(@test_list, [Access.slice(0..2//2)]) + assert [2, 5] == get_in(@test_list, [Access.slice(1..4//3)]) + assert [2] == get_in(@test_list, [Access.slice(1..2//3)]) + assert [1, 3, 5, 7] == get_in(@test_list, [Access.slice(0..6//2)]) + end + + test "pops a range from the start of the list" do + assert {[2, 3], [1, 4, 5, 6, 7]} == pop_in(@test_list, [Access.slice(1..2)]) + end + + test "pops a range from the end of the list" do + assert {[6, 7], [1, 2, 3, 4, 5]} == pop_in(@test_list, [Access.slice(-2..-1)]) + end + + test "pops a range from positive first and negative last" do + assert {[2, 3, 4, 5, 6], [1, 7]} == pop_in(@test_list, [Access.slice(1..-2//1)]) + end + + test "pops a range from negative first and positive last" do + assert {[6, 7], [1, 2, 3, 4, 5]} == pop_in(@test_list, [Access.slice(-2..7//1)]) + end + + test "pops a range with steps" do + assert {[1, 3, 5], [2, 4, 6, 7]} == pop_in(@test_list, [Access.slice(0..4//2)]) + assert {[2], [1, 3, 4, 5, 6, 7]} == pop_in(@test_list, [Access.slice(1..2//2)]) + assert {[1, 4], [1, 2, 5, 6, 7]} == pop_in([1, 2, 1, 4, 5, 6, 7], [Access.slice(2..3)]) + end + + test "updates range from the start of the list" do + assert [-1, 2, 3, 4, 5, 6, 7] == update_in(@test_list, [Access.slice(0..0)], &(&1 * -1)) + + assert [1, -2, -3, 4, 5, 6, 7] == update_in(@test_list, [Access.slice(1..2)], &(&1 * -1)) + end + + test "updates range from the end of the list" do + assert [1, 2, 3, 4, 5, -6, -7] == update_in(@test_list, [Access.slice(-2..-1)], &(&1 * -1)) + + assert [-1, -2, 3, 4, 5, 6, 7] == update_in(@test_list, [Access.slice(-7..-6)], &(&1 * -1)) + end + + test "updates a range from positive first and negative last" do + assert [1, -2, -3, -4, -5, -6, 7] == + update_in(@test_list, [Access.slice(1..-2//1)], &(&1 * -1)) + end + + test "updates a range from negative first and positive last" do + assert [1, 2, 3, 4, 5, -6, -7] == + update_in(@test_list, [Access.slice(-2..7//1)], &(&1 * -1)) + end + + test "updates a range with steps" do + assert [-1, 2, -3, 4, -5, 6, 7] == + update_in(@test_list, [Access.slice(0..4//2)], &(&1 * -1)) + end + + test "returns empty when the start of the range is greater than the end" do + assert [] == get_in(@test_list, [Access.slice(2..1//1)]) + end + end + + describe "at/1" do + @test_list [1, 2, 3, 4, 5, 6] + + test "returns element from the end if index is negative" do + assert get_in(@test_list, [Access.at(-2)]) == 5 + end + + test "returns nil if index is out of bounds counting from the end" do + assert get_in(@test_list, [Access.at(-10)]) == nil + end + + test "updates the element counting from the end if index is negative" do + assert get_and_update_in(@test_list, [Access.at(-2)], fn prev -> + {prev, :foo} + end) == {5, [1, 2, 3, 4, :foo, 6]} + end + + test "returns nil and does not update if index is out of bounds" do + assert get_and_update_in(@test_list, [Access.at(-10)], fn prev -> + {prev, :foo} + end) == {nil, [1, 2, 3, 4, 5, 6]} + end + end + + describe "at!/1" do + @test_list [1, 2, 3, 4, 5, 6] + + test "returns a list element when the index is within bounds, with get_in" do + assert get_in(@test_list, [Access.at!(5)]) == 6 + assert get_in(@test_list, [Access.at!(-6)]) == 1 + end + + test "updates a list element when the index is within bounds, with get_and_update_in" do + assert get_and_update_in(@test_list, [Access.at!(5)], fn prev -> + {prev, :foo} + end) == {6, [1, 2, 3, 4, 5, :foo]} + + assert get_and_update_in(@test_list, [Access.at!(-6)], fn prev -> + {prev, :foo} + end) == {1, [:foo, 2, 3, 4, 5, 6]} + end + + test "raises OutOfBoundsError when out of bounds, with get_in" do + assert_raise Enum.OutOfBoundsError, fn -> + get_in(@test_list, [Access.at!(6)]) + end + + assert_raise Enum.OutOfBoundsError, fn -> + get_in(@test_list, [Access.at!(-7)]) + end + end + + test "raises OutOfBoundsError when out of bounds, with get_and_update_in" do + assert_raise Enum.OutOfBoundsError, fn -> + get_and_update_in(@test_list, [Access.at!(6)], fn prev -> {prev, :foo} end) + end + + assert_raise Enum.OutOfBoundsError, fn -> + get_and_update_in(@test_list, [Access.at!(-7)], fn prev -> {prev, :foo} end) + end + end + + test "raises when not given a list" do + assert_raise RuntimeError, "Access.at!/1 expected a list, got: %{}", fn -> + get_in(%{}, [Access.at!(0)]) + end + end + + test "chains" do + input = %{list: [%{greeting: "hi"}]} + assert get_in(input, [:list, Access.at!(0), :greeting]) == "hi" + end + end + + describe "values/0" do + @test_map %{a: 1, b: 2, c: 3, d: 4} + @test_list [a: 1, b: 2, c: 3, d: 4] + + test "retrieves values in a map" do + assert [1, 2, 3, 4] = get_in(@test_map, [Access.values()]) |> Enum.sort() + end + + test "retrieves values in a keyword list" do + assert [1, 2, 3, 4] = get_in(@test_list, [Access.values()]) + end + + test "gets and updates values in a map" do + assert {gets, %{a: 3, b: 4, c: 5, d: 6}} = + get_and_update_in(@test_map, [Access.values()], fn n -> {n + 1, n + 2} end) + + assert [2, 3, 4, 5] = Enum.sort(gets) + end + + test "gets and updates values in a keyword list" do + assert {[2, 3, 4, 5], [a: 3, b: 4, c: 5, d: 6]} = + get_and_update_in(@test_list, [Access.values()], fn n -> {n + 1, n + 2} end) + end + + test "pops values from a map" do + assert {gets, %{c: 4, d: 5}} = + get_and_update_in(@test_map, [Access.values()], fn n -> + if(n > 2, do: {-n, n + 1}, else: :pop) + end) + + assert [-4, -3, 1, 2] = Enum.sort(gets) + end + + test "pops values from a keyword list" do + assert {[1, 2, -3, -4], [c: 4, d: 5]} = + get_and_update_in(@test_list, [Access.values()], fn n -> + if(n > 2, do: {-n, n + 1}, else: :pop) + end) + end + + test "raises when not given a map or a keyword list" do + message = ~r[^Access.values/0 expected a map or a keyword list, got: .*] + + assert_raise RuntimeError, message, fn -> + get_in(123, [Access.values()]) + end + + assert_raise RuntimeError, message, fn -> + get_and_update_in(:some_atom, [Access.values()], fn x -> {x, x} end) + end + + assert_raise RuntimeError, message, fn -> + get_in([:a, :b, :c], [Access.values()]) + end + + assert_raise RuntimeError, message, fn -> + get_in([{:a, :b, :c}, {:d, :e, :f}], [Access.values()]) + end + + assert_raise RuntimeError, message, fn -> + get_in([{1, 2}, {3, 4}], [Access.values()]) + end end end end diff --git a/lib/elixir/test/elixir/agent_test.exs b/lib/elixir/test/elixir/agent_test.exs index d4d58548806..ec0e5c3c45f 100644 --- a/lib/elixir/test/elixir/agent_test.exs +++ b/lib/elixir/test/elixir/agent_test.exs @@ -1,37 +1,81 @@ -Code.require_file "test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("test_helper.exs", __DIR__) defmodule AgentTest do use ExUnit.Case, async: true - test "start_link/2 workflow with unregistered name" do - {:ok, pid} = Agent.start_link(fn -> %{} end) + doctest Agent + + def identity(state) do + state + end + + test "can be supervised directly" do + assert {:ok, _} = Supervisor.start_link([{Agent, fn -> :ok end}], strategy: :one_for_one) + end + + test "generates child_spec/1" do + defmodule MyAgent do + use Agent + end - {:links, links} = Process.info(self, :links) + assert MyAgent.child_spec([:hello]) == %{ + id: MyAgent, + start: {MyAgent, :start_link, [[:hello]]} + } + + defmodule CustomAgent do + use Agent, id: :id, restart: :temporary, shutdown: :infinity, start: {:foo, :bar, []} + end + + assert CustomAgent.child_spec([:hello]) == %{ + id: :id, + restart: :temporary, + shutdown: :infinity, + start: {:foo, :bar, []} + } + end + + test "start_link/2 workflow with unregistered name and anonymous functions" do + {:ok, pid} = Agent.start_link(&Map.new/0) + + {:links, links} = Process.info(self(), :links) assert pid in links + assert :proc_lib.translate_initial_call(pid) == {Map, :new, 0} + assert Agent.update(pid, &Map.put(&1, :hello, :world)) == :ok assert Agent.get(pid, &Map.get(&1, :hello), 3000) == :world assert Agent.get_and_update(pid, &Map.pop(&1, :hello), 3000) == :world - assert Agent.get(pid, &(&1)) == %{} + assert Agent.get(pid, & &1) == %{} assert Agent.stop(pid) == :ok wait_until_dead(pid) end - test "start/2 workflow with registered name" do - {:ok, pid} = Agent.start(fn -> %{} end, name: :agent) + test "start_link/2 with spawn_opt" do + {:ok, pid} = Agent.start_link(fn -> 0 end, spawn_opt: [priority: :high]) + assert Process.info(pid, :priority) == {:priority, :high} + end + + test "start/2 workflow with registered name and module functions" do + {:ok, pid} = Agent.start(Map, :new, [], name: :agent) assert Process.info(pid, :registered_name) == {:registered_name, :agent} - assert Agent.cast(:agent, &Map.put(&1, :hello, :world)) == :ok - assert Agent.get(:agent, &Map.get(&1, :hello)) == :world - assert Agent.get_and_update(:agent, &Map.pop(&1, :hello)) == :world - assert Agent.get(:agent, &(&1)) == %{} + assert :proc_lib.translate_initial_call(pid) == {Map, :new, 0} + assert Agent.cast(:agent, Map, :put, [:hello, :world]) == :ok + assert Agent.get(:agent, Map, :get, [:hello]) == :world + assert Agent.get_and_update(:agent, Map, :pop, [:hello]) == :world + assert Agent.get(:agent, AgentTest, :identity, []) == %{} assert Agent.stop(:agent) == :ok assert Process.info(pid, :registered_name) == nil end test ":sys.change_code/4 with mfa" do - { :ok, pid } = Agent.start_link(fn -> %{} end) + {:ok, pid} = Agent.start_link(fn -> %{} end) :ok = :sys.suspend(pid) - mfa = { Map, :put, [:hello, :world] } + mfa = {Map, :put, [:hello, :world]} assert :sys.change_code(pid, __MODULE__, "vsn", mfa) == :ok :ok = :sys.resume(pid) assert Agent.get(pid, &Map.get(&1, :hello)) == :world @@ -39,12 +83,12 @@ defmodule AgentTest do end test ":sys.change_code/4 with raising mfa" do - { :ok, pid } = Agent.start_link(fn -> %{} end) + {:ok, pid} = Agent.start_link(fn -> %{} end) :ok = :sys.suspend(pid) - mfa = { :erlang, :error, [] } - assert match?({ :error, _ }, :sys.change_code(pid, __MODULE__, "vsn", mfa)) + mfa = {:erlang, :error, []} + assert match?({:error, _}, :sys.change_code(pid, __MODULE__, "vsn", mfa)) :ok = :sys.resume(pid) - assert Agent.get(pid, &(&1)) == %{} + assert Agent.get(pid, & &1) == %{} assert Agent.stop(pid) == :ok end diff --git a/lib/elixir/test/elixir/application_test.exs b/lib/elixir/test/elixir/application_test.exs index 20258320fb2..7c1b380e8fa 100644 --- a/lib/elixir/test/elixir/application_test.exs +++ b/lib/elixir/test/elixir/application_test.exs @@ -1,31 +1,203 @@ -Code.require_file "test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("test_helper.exs", __DIR__) defmodule ApplicationTest do use ExUnit.Case, async: true + import PathHelpers + import ExUnit.CaptureIO + + @app :elixir + test "application environment" do + assert_raise ArgumentError, ~r/because the application was not loaded nor configured/, fn -> + Application.fetch_env!(:unknown, :unknown) + end + + assert_raise ArgumentError, ~r/because configuration at :unknown was not set/, fn -> + Application.fetch_env!(:elixir, :unknown) + end + assert Application.get_env(:elixir, :unknown) == nil assert Application.get_env(:elixir, :unknown, :default) == :default assert Application.fetch_env(:elixir, :unknown) == :error assert Application.put_env(:elixir, :unknown, :known) == :ok assert Application.fetch_env(:elixir, :unknown) == {:ok, :known} + assert Application.fetch_env!(:elixir, :unknown) == :known assert Application.get_env(:elixir, :unknown, :default) == :known assert {:unknown, :known} in Application.get_all_env(:elixir) assert Application.delete_env(:elixir, :unknown) == :ok assert Application.get_env(:elixir, :unknown, :default) == :default + after + Application.delete_env(:elixir, :unknown) + end + + test "deprecated non-atom keys" do + assert_deprecated(fn -> + Application.put_env(:elixir, [:a, :b], :c) + end) + + assert_deprecated(fn -> + assert Application.get_env(:elixir, [:a, :b]) == :c + end) + + assert_deprecated(fn -> + assert Application.fetch_env!(:elixir, [:a, :b]) == :c + end) + after + assert_deprecated(fn -> + Application.delete_env(:elixir, [:a, :b]) + end) + end + + defp assert_deprecated(fun) do + assert capture_io(:stderr, fun) =~ ~r/passing non-atom as application env key is deprecated/ + end + + describe "compile environment" do + test "invoked at compile time" do + assert_raise ArgumentError, ~r/because the application was not loaded nor configured/, fn -> + compile_env!(:unknown, :unknown) + end + + assert_received {:compile_env, :unknown, [:unknown], :error} + + assert_raise ArgumentError, ~r/because configuration at :unknown was not set/, fn -> + compile_env!(:elixir, :unknown) + end + + assert_received {:compile_env, :elixir, [:unknown], :error} + + assert compile_env(:elixir, :unknown) == nil + assert_received {:compile_env, :elixir, [:unknown], :error} + + assert compile_env(:elixir, :unknown, :default) == :default + assert_received {:compile_env, :elixir, [:unknown], :error} + + assert Application.put_env(:elixir, :unknown, nested: [key: :value]) == :ok + + assert compile_env(@app, :unknown, :default) == [nested: [key: :value]] + assert_received {:compile_env, :elixir, [:unknown], {:ok, [nested: [key: :value]]}} + assert compile_env(:elixir, :unknown, :default) == [nested: [key: :value]] + assert_received {:compile_env, :elixir, [:unknown], {:ok, [nested: [key: :value]]}} + + assert compile_env(:elixir, :unknown) == [nested: [key: :value]] + assert_received {:compile_env, :elixir, [:unknown], {:ok, [nested: [key: :value]]}} + + assert compile_env!(@app, :unknown) == [nested: [key: :value]] + assert_received {:compile_env, :elixir, [:unknown], {:ok, [nested: [key: :value]]}} + assert compile_env!(:elixir, :unknown) == [nested: [key: :value]] + assert_received {:compile_env, :elixir, [:unknown], {:ok, [nested: [key: :value]]}} + + assert compile_env(:elixir, [:unknown, :nested]) == [key: :value] + assert_received {:compile_env, :elixir, [:unknown, :nested], {:ok, [key: :value]}} + + assert compile_env!(:elixir, [:unknown, :nested]) == [key: :value] + assert_received {:compile_env, :elixir, [:unknown, :nested], {:ok, [key: :value]}} + + assert compile_env(:elixir, [:unknown, :nested, :key]) == :value + assert_received {:compile_env, :elixir, [:unknown, :nested, :key], {:ok, :value}} + + assert compile_env!(:elixir, [:unknown, :nested, :key]) == :value + assert_received {:compile_env, :elixir, [:unknown, :nested, :key], {:ok, :value}} + + assert compile_env(:elixir, [:unknown, :unknown, :key], :default) == :default + assert_received {:compile_env, :elixir, [:unknown, :unknown, :key], :error} + + assert compile_env(:elixir, [:unknown, :nested, :unknown], :default) == :default + assert_received {:compile_env, :elixir, [:unknown, :nested, :unknown], :error} + after + Application.delete_env(:elixir, :unknown) + end + + def trace({:compile_env, _, _, _} = msg, %Macro.Env{}) do + send(self(), msg) + :ok + end + + def trace(_, _), do: :ok + + defp compile_env(app, key, default \\ nil) do + code = + quote do + require Application + Application.compile_env(unquote(app), unquote(key), unquote(default)) + end + + {result, _binding} = Code.eval_quoted(code, [], tracers: [__MODULE__]) + result + end + + defp compile_env!(app, key) do + code = + quote do + require Application + Application.compile_env!(unquote(app), unquote(key)) + end + + {result, _binding} = Code.eval_quoted(code, [], tracers: [__MODULE__]) + result + end + end + + test "loaded and started applications" do + started = Application.started_applications() + assert is_list(started) + assert {:elixir, ~c"elixir", _} = List.keyfind(started, :elixir, 0) + + started_timeout = Application.started_applications(7000) + assert is_list(started_timeout) + assert {:elixir, ~c"elixir", _} = List.keyfind(started_timeout, :elixir, 0) + + loaded = Application.loaded_applications() + assert is_list(loaded) + assert {:elixir, ~c"elixir", _} = List.keyfind(loaded, :elixir, 0) + end + + test "application specification" do + assert is_list(Application.spec(:elixir)) + assert Application.spec(:unknown) == nil + assert Application.spec(:unknown, :description) == nil + + assert Application.spec(:elixir, :description) == ~c"elixir" + assert_raise FunctionClauseError, fn -> Application.spec(:elixir, :unknown) end + end + + test "application module" do + assert Application.get_application(String) == :elixir + assert Application.get_application(__MODULE__) == nil + assert Application.get_application(__MODULE__.Unknown) == nil end test "application directory" do root = Path.expand("../../../..", __DIR__) - assert Application.app_dir(:elixir) == - Path.join(root, "bin/../lib/elixir") - assert Application.app_dir(:elixir, "priv") == - Path.join(root, "bin/../lib/elixir/priv") + + assert normalize_app_dir(Application.app_dir(:elixir)) == + normalize_app_dir(Path.join(root, "bin/../lib/elixir")) + + assert normalize_app_dir(Application.app_dir(:elixir, "priv")) == + normalize_app_dir(Path.join(root, "bin/../lib/elixir/priv")) + + assert normalize_app_dir(Application.app_dir(:elixir, ["priv", "foo"])) == + normalize_app_dir(Path.join(root, "bin/../lib/elixir/priv/foo")) assert_raise ArgumentError, fn -> Application.app_dir(:unknown) end end + + if windows?() do + defp normalize_app_dir(path) do + path |> String.downcase() |> Path.expand() + end + else + defp normalize_app_dir(path) do + path |> String.downcase() + end + end end diff --git a/lib/elixir/test/elixir/atom_test.exs b/lib/elixir/test/elixir/atom_test.exs index 7d6d5bcb7ff..e72b7fd9ff7 100644 --- a/lib/elixir/test/elixir/atom_test.exs +++ b/lib/elixir/test/elixir/atom_test.exs @@ -1,13 +1,19 @@ -Code.require_file "test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("test_helper.exs", __DIR__) defmodule AtomTest do use ExUnit.Case, async: true + doctest Atom, except: [:moduledoc] + test "to_string/1" do - assert Atom.to_string(:"héllo") == "héllo" + assert "héllo" |> String.to_atom() |> Atom.to_string() == "héllo" end - test "to_char_list/1" do - assert Atom.to_char_list(:"héllo") == 'héllo' + test "to_charlist/1" do + assert "héllo" |> String.to_atom() |> Atom.to_charlist() == ~c"héllo" end end diff --git a/lib/elixir/test/elixir/base_test.exs b/lib/elixir/test/elixir/base_test.exs index d976bc14173..aeb5f4b5c24 100644 --- a/lib/elixir/test/elixir/base_test.exs +++ b/lib/elixir/test/elixir/base_test.exs @@ -1,10 +1,16 @@ -Code.require_file "test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("test_helper.exs", __DIR__) defmodule BaseTest do use ExUnit.Case, async: true + + doctest Base import Base - test "encode16" do + test "encode16/1" do assert "" == encode16("") assert "66" == encode16("f") assert "666F" == encode16("fo") @@ -14,10 +20,11 @@ defmodule BaseTest do assert "666F6F626172" == encode16("foobar") assert "A1B2C3D4E5F67891" == encode16(<<161, 178, 195, 212, 229, 246, 120, 145>>) - assert "a1b2c3d4e5f67891" == encode16(<<161, 178, 195, 212, 229, 246, 120, 145>>, case: :lower) + assert "a1b2c3d4e5f67891" == + encode16(<<161, 178, 195, 212, 229, 246, 120, 145>>, case: :lower) end - test "decode16" do + test "decode16/1" do assert {:ok, ""} == decode16("") assert {:ok, "f"} == decode16("66") assert {:ok, "fo"} == decode16("666F") @@ -25,13 +32,16 @@ defmodule BaseTest do assert {:ok, "foob"} == decode16("666F6F62") assert {:ok, "fooba"} == decode16("666F6F6261") assert {:ok, "foobar"} == decode16("666F6F626172") - assert {:ok, <<161, 178, 195, 212, 229, 246, 120, 145>>} == decode16("A1B2C3D4E5F67891") + assert {:ok, <<161, 178, 195, 212, 229, 246, 120, 145>>} == decode16("A1B2C3D4E5F67891") + + assert {:ok, <<161, 178, 195, 212, 229, 246, 120, 145>>} == + decode16("a1b2c3d4e5f67891", case: :lower) - assert {:ok, <<161, 178, 195, 212, 229, 246, 120, 145>>} == decode16("a1b2c3d4e5f67891", case: :lower) - assert {:ok, <<161, 178, 195, 212, 229, 246, 120, 145>>} == decode16("a1B2c3D4e5F67891", case: :mixed) + assert {:ok, <<161, 178, 195, 212, 229, 246, 120, 145>>} == + decode16("a1B2c3D4e5F67891", case: :mixed) end - test "decode16!" do + test "decode16!/1" do assert "" == decode16!("") assert "f" == decode16!("66") assert "fo" == decode16!("666F") @@ -41,424 +51,1048 @@ defmodule BaseTest do assert "foobar" == decode16!("666F6F626172") assert <<161, 178, 195, 212, 229, 246, 120, 145>> == decode16!("A1B2C3D4E5F67891") - assert <<161, 178, 195, 212, 229, 246, 120, 145>> == decode16!("a1b2c3d4e5f67891", case: :lower) - assert <<161, 178, 195, 212, 229, 246, 120, 145>> == decode16!("a1B2c3D4e5F67891", case: :mixed) + assert <<161, 178, 195, 212, 229, 246, 120, 145>> == + decode16!("a1b2c3d4e5f67891", case: :lower) + + assert <<161, 178, 195, 212, 229, 246, 120, 145>> == + decode16!("a1B2c3D4e5F67891", case: :mixed) end - test "decode16 non-alphabet digit" do + test "decode16/1 errors on non-alphabet character" do assert :error == decode16("66KF") assert :error == decode16("66ff") assert :error == decode16("66FF", case: :lower) end - test "decode16! non-alphabet digit" do - assert_raise ArgumentError, "non-alphabet digit found: K", fn -> + test "decode16!/1 errors on non-alphabet character" do + assert_raise ArgumentError, "non-alphabet character found: \"K\" (byte 75)", fn -> decode16!("66KF") end - assert_raise ArgumentError, "non-alphabet digit found: f", fn -> + + assert_raise ArgumentError, "non-alphabet character found: \"f\" (byte 102)", fn -> decode16!("66ff") end - assert_raise ArgumentError, "non-alphabet digit found: F", fn -> + + assert_raise ArgumentError, "non-alphabet character found: \"F\" (byte 70)", fn -> decode16!("66FF", case: :lower) end end - test "decode16 odd-length string" do + test "decode16/1 errors on odd-length string" do assert :error == decode16("666") end - test "decode16! odd-length string" do - assert_raise ArgumentError, "odd-length string", fn -> + test "decode16!/1 errors odd-length string" do + assert_raise ArgumentError, ~r/string given to decode has wrong length/, fn -> decode16!("666") end end - test "encode64 empty" do + test "valid16?/1" do + assert valid16?("") + assert valid16?("66") + assert valid16?("666F") + assert valid16?("666F6F") + assert valid16?("666F6F62") + assert valid16?("666F6F6261") + assert valid16?("666F6F626172") + assert valid16?("A1B2C3D4E5F67891") + assert valid16?("a1b2c3d4e5f67891", case: :lower) + assert valid16?("a1B2c3D4e5F67891", case: :mixed) + end + + test "valid16?/1 returns false on non-alphabet character" do + refute valid16?("66KF") + refute valid16?("66ff") + refute valid16?("66FF", case: :lower) + refute valid16?("66fg", case: :mixed) + end + + test "valid16?/1 errors on odd-length string" do + refute valid16?("666") + end + + test "encode64/1 can deal with empty strings" do assert "" == encode64("") end - test "encode64 two pads" do + test "encode64/1 with two pads" do assert "QWxhZGRpbjpvcGVuIHNlc2FtZQ==" == encode64("Aladdin:open sesame") end - test "encode64 one pad" do + test "encode64/1 with one pad" do assert "SGVsbG8gV29ybGQ=" == encode64("Hello World") end - test "encode64 no pad" do + test "encode64/1 with no pad" do assert "QWxhZGRpbjpvcGVuIHNlc2Ft" == encode64("Aladdin:open sesam") - assert "MDEyMzQ1Njc4OSFAIzBeJiooKTs6PD4sLiBbXXt9" == encode64(<<"0123456789!@#0^&*();:<>,. []{}">>) + + assert "MDEyMzQ1Njc4OSFAIzBeJiooKTs6PD4sLiBbXXt9" == + encode64(<<"0123456789!@#0^&*();:<>,. []{}">>) + end + + test "encode64/1 with one pad and ignoring padding" do + assert "SGVsbG8gV29ybGQ" == encode64("Hello World", padding: false) + end + + test "encode64/1 with two pads and ignoring padding" do + assert "QWxhZGRpbjpvcGVuIHNlc2FtZQ" == encode64("Aladdin:open sesame", padding: false) + end + + test "encode64/1 with no pads and ignoring padding" do + assert "QWxhZGRpbjpvcGVuIHNlc2Ft" == encode64("Aladdin:open sesam", padding: false) end - test "decode64 empty" do + test "decode64/1 can deal with empty strings" do assert {:ok, ""} == decode64("") end - test "decode64! empty" do + test "decode64!/1 can deal with empty strings" do assert "" == decode64!("") end - test "decode64 two pads" do + test "decode64/1 with two pads" do assert {:ok, "Aladdin:open sesame"} == decode64("QWxhZGRpbjpvcGVuIHNlc2FtZQ==") end - test "decode64! two pads" do + test "decode64!/1 with two pads" do assert "Aladdin:open sesame" == decode64!("QWxhZGRpbjpvcGVuIHNlc2FtZQ==") end - test "decode64 one pad" do + test "decode64/1 with one pad" do assert {:ok, "Hello World"} == decode64("SGVsbG8gV29ybGQ=") end - test "decode64! one pad" do + test "decode64!/1 with one pad" do assert "Hello World" == decode64!("SGVsbG8gV29ybGQ=") end - test "decode64 no pad" do + test "decode64/1 with no pad" do assert {:ok, "Aladdin:open sesam"} == decode64("QWxhZGRpbjpvcGVuIHNlc2Ft") end - test "decode64! no pad" do + test "decode64!/1 with no pad" do assert "Aladdin:open sesam" == decode64!("QWxhZGRpbjpvcGVuIHNlc2Ft") end - test "decode64 non-alphabet digit" do + test "decode64/1 errors on non-alphabet character" do assert :error == decode64("Zm9)") end - test "decode64! non-alphabet digit" do - assert_raise ArgumentError, "non-alphabet digit found: )", fn -> + test "decode64!/1 errors on non-alphabet character" do + assert_raise ArgumentError, "non-alphabet character found: \")\" (byte 41)", fn -> decode64!("Zm9)") end end - test "decode64 incorrect padding" do + test "decode64/1 errors on whitespace unless there's ignore: :whitespace" do + assert :error == decode64("\nQWxhZGRp bjpvcGVu\sIHNlc2Ft\t") + + assert {:ok, "Aladdin:open sesam"} == + decode64("\nQWxhZGRp bjpvcGVu\sIHNlc2Ft\t", ignore: :whitespace) + end + + test "decode64!/1 errors on whitespace unless there's ignore: :whitespace" do + assert_raise ArgumentError, "non-alphabet character found: \"\\n\" (byte 10)", fn -> + decode64!("\nQWxhZGRp bjpvcGVu\sIHNlc2Ft\t") + end + + assert "Aladdin:open sesam" == + decode64!("\nQWxhZGRp bjpvcGVu\sIHNlc2Ft\t", ignore: :whitespace) + end + + test "decode64/1 errors on incorrect padding" do assert :error == decode64("SGVsbG8gV29ybGQ") end - test "decode64! incorrect padding" do + test "decode64!/1 errors on incorrect padding" do assert_raise ArgumentError, "incorrect padding", fn -> decode64!("SGVsbG8gV29ybGQ") end end - test "url_encode64 empty" do + test "decode64/2 with two pads and ignoring padding" do + assert {:ok, "Aladdin:open sesame"} == decode64("QWxhZGRpbjpvcGVuIHNlc2FtZQ", padding: false) + end + + test "decode64!/2 with two pads and ignoring padding" do + assert "Aladdin:open sesame" == decode64!("QWxhZGRpbjpvcGVuIHNlc2FtZQ", padding: false) + end + + test "decode64/2 with one pad and ignoring padding" do + assert {:ok, "Hello World"} == decode64("SGVsbG8gV29ybGQ", padding: false) + end + + test "decode64!/2 with one pad and ignoring padding" do + assert "Hello World" == decode64!("SGVsbG8gV29ybGQ", padding: false) + end + + test "decode64/2 with no pad and ignoring padding" do + assert {:ok, "Aladdin:open sesam"} == decode64("QWxhZGRpbjpvcGVuIHNlc2Ft", padding: false) + end + + test "decode64!/2 with no pad and ignoring padding" do + assert "Aladdin:open sesam" == decode64!("QWxhZGRpbjpvcGVuIHNlc2Ft", padding: false) + end + + test "decode64/2 with incorrect padding and ignoring padding" do + assert {:ok, "Hello World"} == decode64("SGVsbG8gV29ybGQ", padding: false) + end + + test "decode64!/2 with incorrect padding and ignoring padding" do + assert "Hello World" == decode64!("SGVsbG8gV29ybGQ", padding: false) + end + + test "valid64?/1 can deal with empty strings" do + assert valid64?("") + end + + test "valid64?/1 with two pads" do + assert valid64?("QWxhZGRpbjpvcGVuIHNlc2FtZQ==") + end + + test "valid64?/1 with one pad" do + assert valid64?("SGVsbG8gV29ybGQ=") + end + + test "valid64?/1 with no pad" do + assert valid64?("QWxhZGRpbjpvcGVuIHNlc2Ft") + end + + test "valid64?/1 returns false on non-alphabet character" do + refute valid64?("Zm9)") + end + + test "valid64?/1 returns false on whitespace unless there's ignore: :whitespace" do + refute valid64?("\nQWxhZGRp bjpvcGVu\sIHNlc2Ft\t") + + assert valid64?("\nQWxhZGRp bjpvcGVu\sIHNlc2Ft\t", ignore: :whitespace) + end + + test "valid64?/1 returns false on incorrect padding" do + refute valid64?("SGVsbG8gV29ybGQ") + end + + test "valid64?/2 with two pads and ignoring padding" do + assert valid64?("QWxhZGRpbjpvcGVuIHNlc2FtZQ", padding: false) + end + + test "valid64?/2 with one pad and ignoring padding" do + assert valid64?("SGVsbG8gV29ybGQ", padding: false) + end + + test "valid64?/2 with no pad and ignoring padding" do + assert valid64?("QWxhZGRpbjpvcGVuIHNlc2Ft", padding: false) + end + + test "valid64?/2 with incorrect padding and ignoring padding" do + assert valid64?("SGVsbG8gV29ybGQ", padding: false) + end + + test "url_encode64/1 can deal with empty strings" do assert "" == url_encode64("") end - test "url_encode64 two pads" do + test "url_encode64/1 with two pads" do assert "QWxhZGRpbjpvcGVuIHNlc2FtZQ==" == url_encode64("Aladdin:open sesame") end - test "url_encode64 one pad" do + test "url_encode64/1 with one pad" do assert "SGVsbG8gV29ybGQ=" == url_encode64("Hello World") end - test "url_encode64 no pad" do + test "url_encode64/1 with no pad" do assert "QWxhZGRpbjpvcGVuIHNlc2Ft" == url_encode64("Aladdin:open sesam") - assert "MDEyMzQ1Njc4OSFAIzBeJiooKTs6PD4sLiBbXXt9" == url_encode64(<<"0123456789!@#0^&*();:<>,. []{}">>) + + assert "MDEyMzQ1Njc4OSFAIzBeJiooKTs6PD4sLiBbXXt9" == + url_encode64(<<"0123456789!@#0^&*();:<>,. []{}">>) end - test "url_encode64 no URL unsafe characters" do - refute "/3/+/A==" == url_encode64(<<255,127,254,252>>) - assert "_3_-_A==" == url_encode64(<<255,127,254,252>>) + test "url_encode64/2 with two pads and ignoring padding" do + assert "QWxhZGRpbjpvcGVuIHNlc2FtZQ" == url_encode64("Aladdin:open sesame", padding: false) end - test "url_decode64 empty" do + test "url_encode64/2 with one pad and ignoring padding" do + assert "SGVsbG8gV29ybGQ" == url_encode64("Hello World", padding: false) + end + + test "url_encode64/2 with no pad and ignoring padding" do + assert "QWxhZGRpbjpvcGVuIHNlc2Ft" == url_encode64("Aladdin:open sesam", padding: false) + end + + test "url_encode64/1 doesn't produce URL-unsafe characters" do + refute "/3/+/A==" == url_encode64(<<255, 127, 254, 252>>) + assert "_3_-_A==" == url_encode64(<<255, 127, 254, 252>>) + end + + test "url_decode64/1 can deal with empty strings" do assert {:ok, ""} == url_decode64("") end - test "url_decode64! empty" do + test "url_decode64!/1 can deal with empty strings" do assert "" == url_decode64!("") end - test "url_decode64 two pads" do + test "url_decode64/1 with two pads" do assert {:ok, "Aladdin:open sesame"} == url_decode64("QWxhZGRpbjpvcGVuIHNlc2FtZQ==") end - test "url_decode64! two pads" do + test "url_decode64!/1 with two pads" do assert "Aladdin:open sesame" == url_decode64!("QWxhZGRpbjpvcGVuIHNlc2FtZQ==") end - test "url_decode64 one pad" do + test "url_decode64/1 with one pad" do assert {:ok, "Hello World"} == url_decode64("SGVsbG8gV29ybGQ=") end - test "url_decode64! one pad" do + test "url_decode64!/1 with one pad" do assert "Hello World" == url_decode64!("SGVsbG8gV29ybGQ=") end - test "url_decode64 no pad" do + test "url_decode64/1 with no pad" do assert {:ok, "Aladdin:open sesam"} == url_decode64("QWxhZGRpbjpvcGVuIHNlc2Ft") end - test "url_decode64! no pad" do + test "url_decode64!/1 with no pad" do assert "Aladdin:open sesam" == url_decode64!("QWxhZGRpbjpvcGVuIHNlc2Ft") end - test "url_decode64 non-alphabet digit" do + test "url_decode64/1,2 error on whitespace unless there's ignore: :whitespace" do + assert :error == url_decode64("\nQWxhZGRp bjpvcGVu\sIHNlc2Ft\t") + + assert {:ok, "Aladdin:open sesam"} == + url_decode64("\nQWxhZGRp bjpvcGVu\sIHNlc2Ft\t", ignore: :whitespace) + end + + test "url_decode64!/1,2 error on whitespace unless there's ignore: :whitespace" do + assert_raise ArgumentError, "non-alphabet character found: \"\\n\" (byte 10)", fn -> + url_decode64!("\nQWxhZGRp bjpvcGVu\sIHNlc2Ft\t") + end + + assert "Aladdin:open sesam" == + url_decode64!("\nQWxhZGRp bjpvcGVu\sIHNlc2Ft\t", ignore: :whitespace) + end + + test "url_decode64/1 errors on non-alphabet character" do assert :error == url_decode64("Zm9)") end - test "url_decode64! non-alphabet digit" do - assert_raise ArgumentError, "non-alphabet digit found: )", fn -> + test "url_decode64!/1 errors on non-alphabet character" do + assert_raise ArgumentError, "non-alphabet character found: \")\" (byte 41)", fn -> url_decode64!("Zm9)") end end - test "url_decode64 incorrect padding" do + test "url_decode64/1 errors on incorrect padding" do assert :error == url_decode64("SGVsbG8gV29ybGQ") end - test "url_decode64! incorrect padding" do + test "url_decode64!/1 errors on incorrect padding" do assert_raise ArgumentError, "incorrect padding", fn -> url_decode64!("SGVsbG8gV29ybGQ") end end - test "encode32 empty" do + test "url_decode64/2 with two pads and ignoring padding" do + assert {:ok, "Aladdin:open sesame"} == + url_decode64("QWxhZGRpbjpvcGVuIHNlc2FtZQ", padding: false) + end + + test "url_decode64!/2 with two pads and ignoring padding" do + assert "Aladdin:open sesame" == url_decode64!("QWxhZGRpbjpvcGVuIHNlc2FtZQ", padding: false) + end + + test "url_decode64/2 with one pad and ignoring padding" do + assert {:ok, "Hello World"} == url_decode64("SGVsbG8gV29ybGQ", padding: false) + end + + test "url_decode64!/2 with one pad and ignoring padding" do + assert "Hello World" == url_decode64!("SGVsbG8gV29ybGQ", padding: false) + end + + test "url_decode64/2 with no pad and ignoring padding" do + assert {:ok, "Aladdin:open sesam"} == url_decode64("QWxhZGRpbjpvcGVuIHNlc2Ft", padding: false) + end + + test "url_decode64!/2 with no pad and ignoring padding" do + assert "Aladdin:open sesam" == url_decode64!("QWxhZGRpbjpvcGVuIHNlc2Ft", padding: false) + end + + test "url_decode64/2 ignores incorrect padding when :padding is false" do + assert {:ok, "Hello World"} == url_decode64("SGVsbG8gV29ybGQ", padding: false) + end + + test "url_decode64!/2 ignores incorrect padding when :padding is false" do + assert "Hello World" == url_decode64!("SGVsbG8gV29ybGQ", padding: false) + end + + test "url_valid64?/1 can deal with empty strings" do + assert url_valid64?("") + end + + test "url_valid64?/1 with two pads" do + assert url_valid64?("QWxhZGRpbjpvcGVuIHNlc2FtZQ==") + end + + test "url_valid64?/1 with one pad" do + assert url_valid64?("SGVsbG8gV29ybGQ=") + end + + test "url_valid64?/1 with no pad" do + assert url_valid64?("QWxhZGRpbjpvcGVuIHNlc2Ft") + end + + test "url_valid64?/1 returns false on non-alphabet character" do + refute url_valid64?("Zm9)") + end + + test "url_valid64?/1 returns false on whitespace unless there's ignore: :whitespace" do + refute url_valid64?("\nQWxhZGRp bjpvcGVu\sIHNlc2Ft\t") + + assert url_valid64?("\nQWxhZGRp bjpvcGVu\sIHNlc2Ft\t", ignore: :whitespace) + end + + test "url_valid64?/1 returns false on incorrect padding" do + refute url_valid64?("SGVsbG8gV29ybGQ") + end + + test "url_valid64?/2 with two pads and ignoring padding" do + assert url_valid64?("QWxhZGRpbjpvcGVuIHNlc2FtZQ", padding: false) + end + + test "url_valid64?/2 with one pad and ignoring padding" do + assert url_valid64?("SGVsbG8gV29ybGQ", padding: false) + end + + test "url_valid64?/2 with no pad and ignoring padding" do + assert url_valid64?("QWxhZGRpbjpvcGVuIHNlc2Ft", padding: false) + end + + test "url_valid64?/2 errors on incorrect padding" do + refute url_valid64?("SGVsbG8gV29ybGQ") + end + + test "url_valid64?/2 ignores incorrect padding when :padding is false" do + assert url_valid64?("SGVsbG8gV29ybGQ", padding: false) + end + + test "encode32/1 can deal with empty strings" do assert "" == encode32("") end - test "encode32 one pad" do + test "encode32/1 with one pad" do assert "MZXW6YQ=" == encode32("foob") end - test "encode32 three pads" do + test "encode32/1 with three pads" do assert "MZXW6===" == encode32("foo") end - test "encode32 four pads" do + test "encode32/1 with four pads" do assert "MZXQ====" == encode32("fo") end - test "encode32 six pads" do + test "encode32/1 with six pads" do assert "MZXW6YTBOI======" == encode32("foobar") assert "MY======" == encode32("f") end - test "encode32 no pads" do + test "encode32/1 with no pads" do assert "MZXW6YTB" == encode32("fooba") end - test "encode32 lowercase" do + test "encode32/2 with one pad and ignoring padding" do + assert "MZXW6YQ" == encode32("foob", padding: false) + end + + test "encode32/2 with three pads and ignoring padding" do + assert "MZXW6" == encode32("foo", padding: false) + end + + test "encode32/2 with four pads and ignoring padding" do + assert "MZXQ" == encode32("fo", padding: false) + end + + test "encode32/2 with six pads and ignoring padding" do + assert "MZXW6YTBOI" == encode32("foobar", padding: false) + end + + test "encode32/2 with no pads and ignoring padding" do + assert "MZXW6YTB" == encode32("fooba", padding: false) + end + + test "encode32/2 with lowercase" do assert "mzxw6ytb" == encode32("fooba", case: :lower) end - test "decode32 empty" do + test "decode32/1 can deal with empty strings" do assert {:ok, ""} == decode32("") end - test "decode32! empty" do + test "decode32!/2 can deal with empty strings" do assert "" == decode32!("") end - test "decode32 one pad" do + test "decode32/1 with one pad" do assert {:ok, "foob"} == decode32("MZXW6YQ=") end - test "decode32! one pad" do + test "decode32!/1 with one pad" do assert "foob" == decode32!("MZXW6YQ=") end - test "decode32 three pads" do + test "decode32/1 with three pads" do assert {:ok, "foo"} == decode32("MZXW6===") end - test "decode32! three pads" do + test "decode32!/1 with three pads" do assert "foo" == decode32!("MZXW6===") end - test "decode32 four pads" do + test "decode32/1 with four pads" do assert {:ok, "fo"} == decode32("MZXQ====") end - test "decode32! four pads" do + test "decode32!/1 with four pads" do assert "fo" == decode32!("MZXQ====") end - test "decode32 lowercase" do + test "decode32/2 with lowercase" do assert {:ok, "fo"} == decode32("mzxq====", case: :lower) end - test "decode32! lowercase" do + test "decode32!/2 with lowercase" do assert "fo" == decode32!("mzxq====", case: :lower) end - test "decode32 mixed case" do + test "decode32/2 with mixed case" do assert {:ok, "fo"} == decode32("mZXq====", case: :mixed) end - test "decode32! mixed case" do + test "decode32!/2 with mixed case" do assert "fo" == decode32!("mZXq====", case: :mixed) end - test "decode32 six pads" do + test "decode32/1 with six pads" do assert {:ok, "foobar"} == decode32("MZXW6YTBOI======") assert {:ok, "f"} == decode32("MY======") end - test "decode32! six pads" do + test "decode32!/1 with six pads" do assert "foobar" == decode32!("MZXW6YTBOI======") assert "f" == decode32!("MY======") end - test "decode32 no pads" do + test "decode32/1 with no pads" do assert {:ok, "fooba"} == decode32("MZXW6YTB") end - test "decode32! no pads" do + test "decode32!/1 with no pads" do assert "fooba" == decode32!("MZXW6YTB") end - test "decode32 non-alphabet digit" do + test "decode32/1,2 error on non-alphabet character" do assert :error == decode32("MZX)6YTB") assert :error == decode32("66ff") assert :error == decode32("66FF", case: :lower) end - test "decode32! non-alphabet digit" do - assert_raise ArgumentError, "non-alphabet digit found: )", fn -> + test "decode32!/1,2 argument error on non-alphabet character" do + assert_raise ArgumentError, "non-alphabet character found: \")\" (byte 41)", fn -> decode32!("MZX)6YTB") end - assert_raise ArgumentError, "non-alphabet digit found: m", fn -> + + assert_raise ArgumentError, "non-alphabet character found: \"m\" (byte 109)", fn -> decode32!("mzxw6ytboi======") end - assert_raise ArgumentError, "non-alphabet digit found: M", fn -> + + assert_raise ArgumentError, "non-alphabet character found: \"M\" (byte 77)", fn -> decode32!("MZXW6YTBOI======", case: :lower) end + + assert_raise ArgumentError, "non-alphabet character found: \"0\" (byte 48)", fn -> + decode32!("0ZXW6YTB0I======", case: :mixed) + end end - test "decode32 incorrect padding" do + test "decode32/1 errors on incorrect padding" do assert :error == decode32("MZXW6YQ") end - test "decode32! incorrect padding" do + test "decode32!/1 errors on incorrect padding" do assert_raise ArgumentError, "incorrect padding", fn -> decode32!("MZXW6YQ") end end - test "hex_encode32 empty" do + test "decode32/2 with one pad and :padding to false" do + assert {:ok, "foob"} == decode32("MZXW6YQ", padding: false) + end + + test "decode32!/2 with one pad and :padding to false" do + assert "foob" == decode32!("MZXW6YQ", padding: false) + end + + test "decode32/2 with three pads and ignoring padding" do + assert {:ok, "foo"} == decode32("MZXW6", padding: false) + end + + test "decode32!/2 with three pads and ignoring padding" do + assert "foo" == decode32!("MZXW6", padding: false) + end + + test "decode32/2 with four pads and ignoring padding" do + assert {:ok, "fo"} == decode32("MZXQ", padding: false) + end + + test "decode32!/2 with four pads and ignoring padding" do + assert "fo" == decode32!("MZXQ", padding: false) + end + + test "decode32/2 with :lower case and ignoring padding" do + assert {:ok, "fo"} == decode32("mzxq", case: :lower, padding: false) + end + + test "decode32!/2 with :lower case and ignoring padding" do + assert "fo" == decode32!("mzxq", case: :lower, padding: false) + end + + test "decode32/2 with :mixed case and ignoring padding" do + assert {:ok, "fo"} == decode32("mZXq", case: :mixed, padding: false) + end + + test "decode32!/2 with :mixed case and ignoring padding" do + assert "fo" == decode32!("mZXq", case: :mixed, padding: false) + end + + test "decode32/2 with six pads and ignoring padding" do + assert {:ok, "foobar"} == decode32("MZXW6YTBOI", padding: false) + end + + test "decode32!/2 with six pads and ignoring padding" do + assert "foobar" == decode32!("MZXW6YTBOI", padding: false) + end + + test "decode32/2 with no pads and ignoring padding" do + assert {:ok, "fooba"} == decode32("MZXW6YTB", padding: false) + end + + test "decode32!/2 with no pads and ignoring padding" do + assert "fooba" == decode32!("MZXW6YTB", padding: false) + end + + test "decode32/2 ignores incorrect padding when :padding is false" do + assert {:ok, "foob"} == decode32("MZXW6YQ", padding: false) + end + + test "decode32!/2 ignores incorrect padding when :padding is false" do + "foob" = decode32!("MZXW6YQ", padding: false) + end + + test "valid32?/1 can deal with empty strings" do + assert valid32?("") + end + + test "valid32?/1 with one pad" do + assert valid32?("MZXW6YQ=") + end + + test "valid32?/1 with three pads" do + assert valid32?("MZXW6===") + end + + test "valid32?/1 with four pads" do + assert valid32?("MZXQ====") + end + + test "valid32?/1 with lowercase" do + assert valid32?("mzxq====", case: :lower) + end + + test "valid32?/1 with mixed case" do + assert valid32?("mZXq====", case: :mixed) + end + + test "valid32?/1 with six pads" do + assert valid32?("MZXW6YTBOI======") + end + + test "valid32?/1 with no pads" do + assert valid32?("MZXW6YTB") + end + + test "valid32?/1,2 returns false on non-alphabet character" do + refute valid32?("MZX)6YTB") + refute valid32?("66ff") + refute valid32?("66FF", case: :lower) + refute valid32?("0ZXW6YTB0I======", case: :mixed) + end + + test "valid32?/1 returns false on incorrect padding" do + refute valid32?("MZXW6YQ") + end + + test "valid32?/2 with one pad and :padding to false" do + assert valid32?("MZXW6YQ", padding: false) + end + + test "valid32?/2 with three pads and ignoring padding" do + assert valid32?("MZXW6", padding: false) + end + + test "valid32?/2 with four pads and ignoring padding" do + assert valid32?("MZXQ", padding: false) + end + + test "valid32?/2 with :lower case and ignoring padding" do + assert valid32?("mzxq", case: :lower, padding: false) + end + + test "valid32?/2 with :mixed case and ignoring padding" do + assert valid32?("mZXq", case: :mixed, padding: false) + end + + test "valid32?/2 with six pads and ignoring padding" do + assert valid32?("MZXW6YTBOI", padding: false) + end + + test "valid32?/2 with no pads and ignoring padding" do + assert valid32?("MZXW6YTB", padding: false) + end + + test "valid32?/2 ignores incorrect padding when :padding is false" do + assert valid32?("MZXW6YQ", padding: false) + end + + test "hex_encode32/1 can deal with empty strings" do assert "" == hex_encode32("") end - test "hex_encode32 one pad" do + test "hex_encode32/1 with one pad" do assert "CPNMUOG=" == hex_encode32("foob") end - test "hex_encode32 three pads" do + test "hex_encode32/1 with three pads" do assert "CPNMU===" == hex_encode32("foo") end - test "hex_encode32 four pads" do + test "hex_encode32/1 with four pads" do assert "CPNG====" == hex_encode32("fo") end - test "hex_encode32 six pads" do + test "hex_encode32/1 with six pads" do assert "CPNMUOJ1E8======" == hex_encode32("foobar") assert "CO======" == hex_encode32("f") end - test "hex_encode32 no pads" do + test "hex_encode32/1 with no pads" do assert "CPNMUOJ1" == hex_encode32("fooba") end - test "hex_encode32 lowercase" do + test "hex_encode32/2 with one pad and ignoring padding" do + assert "CPNMUOG" == hex_encode32("foob", padding: false) + end + + test "hex_encode32/2 with three pads and ignoring padding" do + assert "CPNMU" == hex_encode32("foo", padding: false) + end + + test "hex_encode32/2 with four pads and ignoring padding" do + assert "CPNG" == hex_encode32("fo", padding: false) + end + + test "hex_encode32/2 with six pads and ignoring padding" do + assert "CPNMUOJ1E8" == hex_encode32("foobar", padding: false) + end + + test "hex_encode32/2 with no pads and ignoring padding" do + assert "CPNMUOJ1" == hex_encode32("fooba", padding: false) + end + + test "hex_encode32/2 with lowercase" do assert "cpnmuoj1" == hex_encode32("fooba", case: :lower) end - test "hex_decode32 empty" do + test "hex_decode32/1 can deal with empty strings" do assert {:ok, ""} == hex_decode32("") end - test "hex_decode32! empty" do + test "hex_decode32!/1 can deal with empty strings" do assert "" == hex_decode32!("") end - test "hex_decode32 one pad" do + test "hex_decode32/1 with one pad" do assert {:ok, "foob"} == hex_decode32("CPNMUOG=") end - test "hex_decode32! one pad" do + test "hex_decode32!/1 with one pad" do assert "foob" == hex_decode32!("CPNMUOG=") end - test "hex_decode32 three pads" do + test "hex_decode32/1 with three pads" do assert {:ok, "foo"} == hex_decode32("CPNMU===") end - test "hex_decode32! three pads" do + test "hex_decode32!/1 with three pads" do assert "foo" == hex_decode32!("CPNMU===") end - test "hex_decode32 four pads" do + test "hex_decode32/1 with four pads" do assert {:ok, "fo"} == hex_decode32("CPNG====") end - test "hex_decode32! four pads" do + test "hex_decode32!/1 with four pads" do assert "fo" == hex_decode32!("CPNG====") end - test "hex_decode32 six pads" do + test "hex_decode32/1 with six pads" do assert {:ok, "foobar"} == hex_decode32("CPNMUOJ1E8======") assert {:ok, "f"} == hex_decode32("CO======") end - test "hex_decode32! six pads" do + test "hex_decode32!/1 with six pads" do assert "foobar" == hex_decode32!("CPNMUOJ1E8======") assert "f" == hex_decode32!("CO======") end - test "hex_decode32 no pads" do + test "hex_decode32/1 with no pads" do assert {:ok, "fooba"} == hex_decode32("CPNMUOJ1") end - test "hex_decode32! no pads" do + test "hex_decode32!/1 with no pads" do assert "fooba" == hex_decode32!("CPNMUOJ1") end - test "hex_decode32 non-alphabet digit" do + test "hex_decode32/1,2 error on non-alphabet character" do assert :error == hex_decode32("CPN)UOJ1") assert :error == hex_decode32("66f") assert :error == hex_decode32("66F", case: :lower) end - test "hex_decode32! non-alphabet digit" do - assert_raise ArgumentError, "non-alphabet digit found: )", fn -> + test "hex_decode32!/1,2 error non-alphabet character" do + assert_raise ArgumentError, "non-alphabet character found: \")\" (byte 41)", fn -> hex_decode32!("CPN)UOJ1") end - assert_raise ArgumentError, "non-alphabet digit found: c", fn -> + + assert_raise ArgumentError, "non-alphabet character found: \"c\" (byte 99)", fn -> hex_decode32!("cpnmuoj1e8======") end - assert_raise ArgumentError, "non-alphabet digit found: C", fn -> + + assert_raise ArgumentError, "non-alphabet character found: \"C\" (byte 67)", fn -> hex_decode32!("CPNMUOJ1E8======", case: :lower) end end - test "hex_decode32 incorrect padding" do + test "hex_decode32/1 errors on incorrect padding" do assert :error == hex_decode32("CPNMUOG") end - test "hex_decode32! incorrect padding" do + test "hex_decode32!/1 errors on incorrect padding" do assert_raise ArgumentError, "incorrect padding", fn -> hex_decode32!("CPNMUOG") end end - test "hex_decode32 lowercase" do + test "hex_decode32/2 with lowercase" do assert {:ok, "fo"} == hex_decode32("cpng====", case: :lower) end - test "hex_decode32! lowercase" do + test "hex_decode32!/2 with lowercase" do assert "fo" == hex_decode32!("cpng====", case: :lower) end - test "hex_decode32 mixed case" do + test "hex_decode32/2 with mixed case" do assert {:ok, "fo"} == hex_decode32("cPNg====", case: :mixed) end - test "hex_decode32! mixed case" do + test "hex_decode32!/2 with mixed case" do assert "fo" == hex_decode32!("cPNg====", case: :mixed) end + + test "decode16!/1 errors on non-UTF-8 char" do + assert_raise ArgumentError, "non-alphabet character found: \"\\0\" (byte 0)", fn -> + decode16!("012" <> <<0>>) + end + end + + test "hex_decode32/2 with one pad and ignoring padding" do + assert {:ok, "foob"} == hex_decode32("CPNMUOG", padding: false) + end + + test "hex_decode32!/2 with one pad and ignoring padding" do + assert "foob" == hex_decode32!("CPNMUOG", padding: false) + end + + test "hex_decode32/2 with three pads and ignoring padding" do + assert {:ok, "foo"} == hex_decode32("CPNMU", padding: false) + end + + test "hex_decode32!/2 with three pads and ignoring padding" do + assert "foo" == hex_decode32!("CPNMU", padding: false) + end + + test "hex_decode32/2 with four pads and ignoring padding" do + assert {:ok, "fo"} == hex_decode32("CPNG", padding: false) + end + + test "hex_decode32!/2 with four pads and ignoring padding" do + assert "fo" == hex_decode32!("CPNG", padding: false) + end + + test "hex_decode32/2 with six pads and ignoring padding" do + assert {:ok, "foobar"} == hex_decode32("CPNMUOJ1E8", padding: false) + end + + test "hex_decode32!/2 with six pads and ignoring padding" do + assert "foobar" == hex_decode32!("CPNMUOJ1E8", padding: false) + end + + test "hex_decode32/2 with no pads and ignoring padding" do + assert {:ok, "fooba"} == hex_decode32("CPNMUOJ1", padding: false) + end + + test "hex_decode32!/2 with no pads and ignoring padding" do + assert "fooba" == hex_decode32!("CPNMUOJ1", padding: false) + end + + test "hex_decode32/2 ignores incorrect padding when :padding is false" do + assert {:ok, "foob"} == hex_decode32("CPNMUOG", padding: false) + end + + test "hex_decode32!/2 ignores incorrect padding when :padding is false" do + "foob" = hex_decode32!("CPNMUOG", padding: false) + end + + test "hex_decode32/2 with :lower case and ignoring padding" do + assert {:ok, "fo"} == hex_decode32("cpng", case: :lower, padding: false) + end + + test "hex_decode32!/2 with :lower case and ignoring padding" do + assert "fo" == hex_decode32!("cpng", case: :lower, padding: false) + end + + test "hex_decode32/2 with :mixed case and ignoring padding" do + assert {:ok, "fo"} == hex_decode32("cPNg====", case: :mixed, padding: false) + end + + test "hex_decode32!/2 with :mixed case and ignoring padding" do + assert "fo" == hex_decode32!("cPNg", case: :mixed, padding: false) + end + + test "hex_valid32?/1 can deal with empty strings" do + assert hex_valid32?("") + end + + test "hex_valid32?/1 with one pad" do + assert hex_valid32?("CPNMUOG=") + end + + test "hex_valid32?/1 with three pads" do + assert hex_valid32?("CPNMU===") + end + + test "hex_valid32?/1 with four pads" do + assert hex_valid32?("CPNG====") + end + + test "hex_valid32?/1 with six pads" do + assert hex_valid32?("CPNMUOJ1E8======") + assert hex_valid32?("CO======") + end + + test "hex_valid32?/1 with no pads" do + assert hex_valid32?("CPNMUOJ1") + end + + test "hex_valid32?/1,2 returns false on non-alphabet character" do + refute hex_valid32?("CPN)UOJ1") + refute hex_valid32?("66f") + refute hex_valid32?("66F", case: :lower) + end + + test "hex_valid32?/1 returns false on incorrect padding" do + refute hex_valid32?("CPNMUOG") + end + + test "hex_valid32?/2 with lowercase" do + assert hex_valid32?("cpng====", case: :lower) + end + + test "hex_valid32?/2 with mixed case" do + assert hex_valid32?("cPNg====", case: :mixed) + end + + test "hex_valid32?/2 with one pad and ignoring padding" do + assert hex_valid32?("CPNMUOG", padding: false) + end + + test "hex_valid32?/2 with three pads and ignoring padding" do + assert hex_valid32?("CPNMU", padding: false) + end + + test "hex_valid32?/2 with four pads and ignoring padding" do + assert hex_valid32?("CPNG", padding: false) + end + + test "hex_valid32?/2 with six pads and ignoring padding" do + assert hex_valid32?("CPNMUOJ1E8", padding: false) + end + + test "hex_valid32?/2 with no pads and ignoring padding" do + assert hex_valid32?("CPNMUOJ1", padding: false) + end + + test "hex_valid32?/2 ignores incorrect padding when :padding is false" do + assert hex_valid32?("CPNMUOG", padding: false) + end + + test "hex_valid32?/2 with :lower case and ignoring padding" do + assert hex_valid32?("cpng", case: :lower, padding: false) + end + + test "hex_valid32?/2 with :mixed case and ignoring padding" do + assert hex_valid32?("cPNg====", case: :mixed, padding: false) + end + + # TODO: add valid? tests + test "encode then decode is identity" do + for {encode, decode, valid?} <- [ + {&encode16/2, &decode16!/2, &valid16?/2}, + {&encode32/2, &decode32!/2, &valid32?/2}, + {&hex_encode32/2, &hex_decode32!/2, &hex_valid32?/2}, + {&encode64/2, &decode64!/2, &valid64?/2}, + {&url_encode64/2, &url_decode64!/2, &url_valid64?/2} + ], + encode_case <- [:upper, :lower], + decode_case <- [:upper, :lower, :mixed], + encode_case == decode_case or decode_case == :mixed, + pad? <- [true, false], + len <- 0..256 do + data = + 0 + |> :lists.seq(len - 1) + |> Enum.shuffle() + |> IO.iodata_to_binary() + + allowed_opts = + encode + |> Function.info() + |> Keyword.fetch!(:name) + |> case do + :encode16 -> [:case] + :encode64 -> [:padding] + :url_encode64 -> [:padding] + _ -> [:case, :padding] + end + + encoded = encode.(data, Keyword.take([case: encode_case, padding: pad?], allowed_opts)) + + decode_opts = Keyword.take([case: decode_case, padding: pad?], allowed_opts) + assert valid?.(encoded, decode_opts) + expected = decode.(encoded, decode_opts) + + assert data == expected, + "identity did not match for #{inspect(data)} when #{inspect(encode)} (#{encode_case})" + end + end end diff --git a/lib/elixir/test/elixir/behaviour_test.exs b/lib/elixir/test/elixir/behaviour_test.exs deleted file mode 100644 index 3ff575302d5..00000000000 --- a/lib/elixir/test/elixir/behaviour_test.exs +++ /dev/null @@ -1,64 +0,0 @@ -Code.require_file "test_helper.exs", __DIR__ - -defmodule BehaviourTest do - use ExUnit.Case, async: true - - defmodule Sample do - use Behaviour - - @doc "I should be first." - defcallback first(integer) :: integer - - @doc "Foo" - defcallback foo(atom(), binary) :: binary - - @doc "Bar" - defcallback bar(External.hello, my_var :: binary) :: binary - - defcallback guarded(my_var) :: my_var when my_var: binary - - defcallback orr(atom | integer) :: atom - - defcallback literal(123, {atom}, :atom, [integer], true) :: atom - - @doc "I should be last." - defmacrocallback last(integer) :: Macro.t - end - - test :docs do - docs = Sample.__behaviour__(:docs) - assert [ - {{:first, 1}, 10, :def, "I should be first."}, - {{:foo, 2}, 13, :def, "Foo"}, - {{:bar, 2}, 16, :def, "Bar"}, - {{:guarded, 1}, 18, :def, nil}, - {{:orr, 1}, 20, :def, nil}, - {{:literal, 5}, 22, :def, nil}, - {{:last, 1}, 25, :defmacro, "I should be last."} - ] = docs - end - - test :callbacks do - assert Sample.__behaviour__(:callbacks) == [first: 1, guarded: 1, "MACRO-last": 2, literal: 5, orr: 1, foo: 2, bar: 2] - end - - test :specs do - assert length(Keyword.get_values(Sample.module_info[:attributes], :callback)) == 7 - end - - test :default_is_not_supported do - assert_raise ArgumentError, fn -> - defmodule WithDefault do - use Behaviour - defcallback hello(num \\ 0 :: integer) :: integer - end - end - - assert_raise ArgumentError, fn -> - defmodule WithDefault do - use Behaviour - defcallback hello(num :: integer \\ 0) :: integer - end - end - end -end diff --git a/lib/elixir/test/elixir/bitwise_test.exs b/lib/elixir/test/elixir/bitwise_test.exs index 17872979d26..41e13d9d06f 100644 --- a/lib/elixir/test/elixir/bitwise_test.exs +++ b/lib/elixir/test/elixir/bitwise_test.exs @@ -1,59 +1,52 @@ -Code.require_file "test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec -defmodule Bitwise.FunctionsTest do +Code.require_file("test_helper.exs", __DIR__) + +defmodule BitwiseTest do use ExUnit.Case, async: true - use Bitwise, skip_operators: true - test :bnot do + import Bitwise + doctest Bitwise + + test "bnot/1" do assert bnot(1) == -2 end - test :band do + test "band/2" do assert band(1, 1) == 1 end - test :bor do + test "bor/2" do assert bor(0, 1) == 1 end - test :bxor do + test "bxor/2" do assert bxor(1, 1) == 0 end - test :bsl do + test "bsl/2" do assert bsl(1, 1) == 2 end - test :bsr do + test "bsr/2" do assert bsr(1, 1) == 0 end -end - -defmodule Bitwise.OperatorsTest do - use ExUnit.Case, async: true - use Bitwise, only_operators: true - test :bnot do - assert ~~~1 == -2 - end - - test :band do + test "band (&&&)" do assert (1 &&& 1) == 1 end - test :bor do + test "bor (|||)" do assert (0 ||| 1) == 1 end - test :bxor do - assert 1 ^^^ 1 == 0 - end - - test :bsl do - assert (1 <<< 1) == 2 + test "bsl (<<<)" do + assert 1 <<< 1 == 2 end - test :bsr do - assert (1 >>> 1) == 0 + test "bsr (>>>)" do + assert 1 >>> 1 == 0 end end diff --git a/lib/elixir/test/elixir/calendar/date_range_test.exs b/lib/elixir/test/elixir/calendar/date_range_test.exs new file mode 100644 index 00000000000..fe6cab37e59 --- /dev/null +++ b/lib/elixir/test/elixir/calendar/date_range_test.exs @@ -0,0 +1,200 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) +Code.require_file("holocene.exs", __DIR__) + +defmodule Date.RangeTest do + use ExUnit.Case, async: true + + @asc_range Date.range(~D[2000-01-01], ~D[2001-01-01]) + @asc_range_2 Date.range(~D[2000-01-01], ~D[2001-01-01], 2) + @desc_range Date.range(~D[2001-01-01], ~D[2000-01-01], -1) + @desc_range_2 Date.range(~D[2001-01-01], ~D[2000-01-01], -2) + @empty_range Date.range(~D[2001-01-01], ~D[2000-01-01], 1) + + describe "Enum.member?/2" do + test "for ascending range" do + assert Enum.member?(@asc_range, ~D[2000-02-22]) + assert Enum.member?(@asc_range, ~D[2000-01-01]) + assert Enum.member?(@asc_range, ~D[2001-01-01]) + refute Enum.member?(@asc_range, ~D[2002-01-01]) + refute Enum.member?(@asc_range, Calendar.Holocene.date(12000, 1, 1)) + + assert Enum.member?(@asc_range_2, ~D[2000-01-03]) + refute Enum.member?(@asc_range_2, ~D[2000-01-02]) + end + + test "for descending range" do + assert Enum.member?(@desc_range, ~D[2000-02-22]) + assert Enum.member?(@desc_range, ~D[2000-01-01]) + assert Enum.member?(@desc_range, ~D[2001-01-01]) + refute Enum.member?(@desc_range, ~D[1999-01-01]) + refute Enum.member?(@desc_range, Calendar.Holocene.date(12000, 1, 1)) + + assert Enum.member?(@desc_range_2, ~D[2000-12-30]) + refute Enum.member?(@desc_range_2, ~D[2000-12-29]) + end + + test "empty range" do + refute Enum.member?(@empty_range, @empty_range.first) + end + end + + describe "Enum.count/1" do + test "for ascending range" do + assert Enum.count(@asc_range) == 367 + assert Enum.count(@asc_range_2) == 184 + end + + test "for descending range" do + assert Enum.count(@desc_range) == 367 + assert Enum.count(@desc_range_2) == 184 + end + + test "for empty range" do + assert Enum.count(@empty_range) == 0 + end + end + + describe "Enum.slice/3" do + test "for ascending range" do + assert Enum.slice(@asc_range, 3, 3) == [~D[2000-01-04], ~D[2000-01-05], ~D[2000-01-06]] + assert Enum.slice(@asc_range, -3, 3) == [~D[2000-12-30], ~D[2000-12-31], ~D[2001-01-01]] + + assert Enum.slice(@asc_range_2, 3, 3) == [~D[2000-01-07], ~D[2000-01-09], ~D[2000-01-11]] + assert Enum.slice(@asc_range_2, -3, 3) == [~D[2000-12-28], ~D[2000-12-30], ~D[2001-01-01]] + end + + test "for descending range" do + assert Enum.slice(@desc_range, 3, 3) == [~D[2000-12-29], ~D[2000-12-28], ~D[2000-12-27]] + assert Enum.slice(@desc_range, -3, 3) == [~D[2000-01-03], ~D[2000-01-02], ~D[2000-01-01]] + + assert Enum.slice(@desc_range_2, 3, 3) == [~D[2000-12-26], ~D[2000-12-24], ~D[2000-12-22]] + assert Enum.slice(@desc_range_2, -3, 3) == [~D[2000-01-05], ~D[2000-01-03], ~D[2000-01-01]] + end + + test "for empty range" do + assert Enum.slice(@empty_range, 1, 3) == [] + assert Enum.slice(@empty_range, 3, 3) == [] + assert Enum.slice(@empty_range, -1, 3) == [] + assert Enum.slice(@empty_range, -3, 3) == [] + end + end + + describe "Enum.reduce/3" do + test "for ascending range" do + assert Enum.take(@asc_range, 3) == [~D[2000-01-01], ~D[2000-01-02], ~D[2000-01-03]] + + assert Enum.take(@asc_range_2, 3) == [~D[2000-01-01], ~D[2000-01-03], ~D[2000-01-05]] + end + + test "for descending range" do + assert Enum.take(@desc_range, 3) == [~D[2001-01-01], ~D[2000-12-31], ~D[2000-12-30]] + + assert Enum.take(@desc_range_2, 3) == [~D[2001-01-01], ~D[2000-12-30], ~D[2000-12-28]] + end + + test "for empty range" do + assert Enum.take(@empty_range, 3) == [] + end + end + + test "Enum.take/1 for empty range with negative step" do + assert Enum.take(@empty_range, -1) == [] + end + + test "works with date-like structs" do + range = Date.range(~N[2000-01-01 09:00:00], ~U[2000-01-02 09:00:00Z]) + assert range.first == ~D[2000-01-01] + assert range.last == ~D[2000-01-02] + assert Enum.to_list(range) == [~D[2000-01-01], ~D[2000-01-02]] + + range = Date.range(~N[2000-01-01 09:00:00], ~U[2000-01-03 09:00:00Z], 2) + assert range.first == ~D[2000-01-01] + assert range.last == ~D[2000-01-03] + assert Enum.to_list(range) == [~D[2000-01-01], ~D[2000-01-03]] + end + + test "both dates must have matching calendars" do + first = ~D[2000-01-01] + last = Calendar.Holocene.date(12001, 1, 1) + + assert_raise ArgumentError, "both dates must have matching calendars", fn -> + Date.range(first, last) + end + end + + test "accepts equal but non Calendar.ISO calendars" do + first = Calendar.Holocene.date(12000, 1, 1) + last = Calendar.Holocene.date(12001, 1, 1) + range = Date.range(first, last) + assert range + assert first in range + assert last in range + assert Enum.count(range) == 367 + end + + test "step is a non-zero integer" do + step = 1.0 + message = ~r"the step must be a non-zero integer" + + assert_raise ArgumentError, message, fn -> + Date.range(~D[2000-01-01], ~D[2000-01-31], step) + end + + step = 0 + message = ~r"the step must be a non-zero integer" + + assert_raise ArgumentError, message, fn -> + Date.range(~D[2000-01-01], ~D[2000-01-31], step) + end + end + + describe "old date ranges" do + test "enumerable" do + asc = %{ + __struct__: Date.Range, + first: ~D[2021-07-14], + first_in_iso_days: 738_350, + last: ~D[2021-07-17], + last_in_iso_days: 738_353 + } + + desc = %{ + __struct__: Date.Range, + first: ~D[2021-07-17], + first_in_iso_days: 738_353, + last: ~D[2021-07-14], + last_in_iso_days: 738_350 + } + + # member? implementations tests also empty? + assert Enumerable.member?(asc, ~D[2021-07-15]) + assert {:ok, 4, _} = Enumerable.slice(asc) + + assert Enum.reduce(asc, [], fn x, acc -> [x | acc] end) == [ + ~D[2021-07-17], + ~D[2021-07-16], + ~D[2021-07-15], + ~D[2021-07-14] + ] + + assert Enum.count(asc) == 4 + + # member? implementations tests also empty? + assert Enumerable.member?(desc, ~D[2021-07-15]) + assert {:ok, 4, _} = Enumerable.slice(desc) + + assert Enum.reduce(desc, [], fn x, acc -> [x | acc] end) == [ + ~D[2021-07-14], + ~D[2021-07-15], + ~D[2021-07-16], + ~D[2021-07-17] + ] + + assert Enum.count(desc) == 4 + end + end +end diff --git a/lib/elixir/test/elixir/calendar/date_test.exs b/lib/elixir/test/elixir/calendar/date_test.exs new file mode 100644 index 00000000000..c7d33533335 --- /dev/null +++ b/lib/elixir/test/elixir/calendar/date_test.exs @@ -0,0 +1,299 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) +Code.require_file("holocene.exs", __DIR__) +Code.require_file("fakes.exs", __DIR__) + +defmodule DateTest do + use ExUnit.Case, async: true + doctest Date + + test "sigil_D" do + assert ~D[2000-01-01] == + %Date{calendar: Calendar.ISO, year: 2000, month: 1, day: 1} + + assert ~D[20001-01-01 Calendar.Holocene] == + %Date{calendar: Calendar.Holocene, year: 20001, month: 1, day: 1} + + assert_raise ArgumentError, + ~s/cannot parse "2000-50-50" as Date for Calendar.ISO, reason: :invalid_date/, + fn -> Code.eval_string("~D[2000-50-50]") end + + assert_raise ArgumentError, + ~s/cannot parse "2000-04-15 notalias" as Date for Calendar.ISO, reason: :invalid_format/, + fn -> Code.eval_string("~D[2000-04-15 notalias]") end + + assert_raise ArgumentError, + ~s/cannot parse "20010415" as Date for Calendar.ISO, reason: :invalid_format/, + fn -> Code.eval_string(~s{~D[20010415]}) end + + assert_raise ArgumentError, + ~s/cannot parse "20001-50-50" as Date for Calendar.Holocene, reason: :invalid_date/, + fn -> Code.eval_string("~D[20001-50-50 Calendar.Holocene]") end + + assert_raise UndefinedFunctionError, fn -> + Code.eval_string("~D[2000-01-01 UnknownCalendar]") + end + end + + test "to_string/1" do + date = ~D[2000-01-01] + assert to_string(date) == "2000-01-01" + assert Date.to_string(date) == "2000-01-01" + assert Date.to_string(Map.from_struct(date)) == "2000-01-01" + + assert to_string(%{date | calendar: FakeCalendar}) == "1/1/2000" + assert Date.to_string(%{date | calendar: FakeCalendar}) == "1/1/2000" + + date2 = Date.new!(5_874_897, 12, 31) + assert to_string(date2) == "5874897-12-31" + assert Date.to_string(date2) == "5874897-12-31" + assert Date.to_string(Map.from_struct(date2)) == "5874897-12-31" + + assert to_string(%{date2 | calendar: FakeCalendar}) == "31/12/5874897" + assert Date.to_string(%{date2 | calendar: FakeCalendar}) == "31/12/5874897" + end + + test "inspect/1" do + assert inspect(~D[2000-01-01]) == "~D[2000-01-01]" + assert inspect(~D[-0100-12-31]) == "~D[-0100-12-31]" + + date = %{~D[2000-01-01] | calendar: FakeCalendar} + assert inspect(date) == "~D[1/1/2000 FakeCalendar]" + + assert inspect(Date.new!(99999, 12, 31)) == "Date.new!(99999, 12, 31)" + assert inspect(Date.new!(-99999, 1, 1)) == "Date.new!(-99999, 1, 1)" + + date2 = %{Date.new!(99999, 12, 31) | calendar: FakeCalendar} + + assert inspect(%{date2 | calendar: FakeCalendar}) == + "~D[31/12/99999 FakeCalendar]" + end + + test "compare/2" do + date1 = ~D[-0001-12-30] + date2 = ~D[-0001-12-31] + date3 = ~D[0001-01-01] + date4 = Date.new!(5_874_897, 12, 31) + date5 = Date.new!(-4713, 1, 1) + + assert Date.compare(date1, date1) == :eq + assert Date.compare(date1, date2) == :lt + assert Date.compare(date2, date1) == :gt + assert Date.compare(date3, date3) == :eq + assert Date.compare(date2, date3) == :lt + assert Date.compare(date3, date2) == :gt + assert Date.compare(date4, date1) == :gt + assert Date.compare(date1, date4) == :lt + assert Date.compare(date4, date4) == :eq + assert Date.compare(date4, date5) == :gt + assert Date.compare(date5, date4) == :lt + assert Date.compare(date5, date5) == :eq + + assert_raise ArgumentError, + ~r/cannot compare .*\n\n.* their calendars have incompatible day rollover moments/, + fn -> Date.compare(date1, %{date2 | calendar: FakeCalendar}) end + end + + test "before?/2 and after?/2" do + date1 = ~D[2022-11-01] + date2 = ~D[2022-11-02] + date3 = Date.new!(5_874_897, 12, 31) + date4 = Date.new!(-4713, 1, 1) + + assert Date.before?(date1, date2) + assert Date.before?(date1, date3) + assert Date.before?(date4, date1) + assert not Date.before?(date2, date1) + assert not Date.before?(date3, date1) + assert not Date.before?(date1, date4) + + assert Date.after?(date2, date1) + assert Date.after?(date3, date2) + assert Date.after?(date2, date4) + assert not Date.after?(date1, date2) + assert not Date.after?(date2, date3) + assert not Date.after?(date4, date2) + end + + test "compare/2 across calendars" do + date1 = ~D[2000-01-01] + date2 = Calendar.Holocene.date(12000, 01, 01) + assert Date.compare(date1, date2) == :eq + + date2 = Calendar.Holocene.date(12001, 01, 01) + assert Date.compare(date1, date2) == :lt + assert Date.compare(date2, date1) == :gt + end + + test "day_of_week/1" do + assert Date.day_of_week(Calendar.Holocene.date(2016, 10, 31)) == 1 + assert Date.day_of_week(Calendar.Holocene.date(2016, 11, 01)) == 2 + assert Date.day_of_week(Calendar.Holocene.date(2016, 11, 02)) == 3 + assert Date.day_of_week(Calendar.Holocene.date(2016, 11, 03)) == 4 + assert Date.day_of_week(Calendar.Holocene.date(2016, 11, 04)) == 5 + assert Date.day_of_week(Calendar.Holocene.date(2016, 11, 05)) == 6 + assert Date.day_of_week(Calendar.Holocene.date(2016, 11, 06)) == 7 + assert Date.day_of_week(Calendar.Holocene.date(2016, 11, 07)) == 1 + + assert Date.day_of_week(Calendar.Holocene.date(2016, 10, 30), :sunday) == 1 + assert Date.day_of_week(Calendar.Holocene.date(2016, 10, 31), :sunday) == 2 + assert Date.day_of_week(Calendar.Holocene.date(2016, 11, 01), :sunday) == 3 + assert Date.day_of_week(Calendar.Holocene.date(2016, 11, 02), :sunday) == 4 + assert Date.day_of_week(Calendar.Holocene.date(2016, 11, 03), :sunday) == 5 + assert Date.day_of_week(Calendar.Holocene.date(2016, 11, 04), :sunday) == 6 + assert Date.day_of_week(Calendar.Holocene.date(2016, 11, 05), :sunday) == 7 + assert Date.day_of_week(Calendar.Holocene.date(2016, 11, 06), :sunday) == 1 + end + + test "beginning_of_week" do + assert Date.beginning_of_week(Calendar.Holocene.date(2020, 07, 11)) == + Calendar.Holocene.date(2020, 07, 06) + + assert Date.beginning_of_week(Calendar.Holocene.date(2020, 07, 06)) == + Calendar.Holocene.date(2020, 07, 06) + + assert Date.beginning_of_week(Calendar.Holocene.date(2020, 07, 11), :sunday) == + Calendar.Holocene.date(2020, 07, 05) + + assert Date.beginning_of_week(Calendar.Holocene.date(2020, 07, 11), :saturday) == + Calendar.Holocene.date(2020, 07, 11) + end + + test "end_of_week" do + assert Date.end_of_week(Calendar.Holocene.date(2020, 07, 11)) == + Calendar.Holocene.date(2020, 07, 12) + + assert Date.end_of_week(Calendar.Holocene.date(2020, 07, 05)) == + Calendar.Holocene.date(2020, 07, 05) + + assert Date.end_of_week(Calendar.Holocene.date(2020, 07, 05), :sunday) == + Calendar.Holocene.date(2020, 07, 11) + + assert Date.end_of_week(Calendar.Holocene.date(2020, 07, 05), :saturday) == + Calendar.Holocene.date(2020, 07, 10) + end + + test "convert/2" do + assert Date.convert(~D[2000-01-01], Calendar.Holocene) == + {:ok, Calendar.Holocene.date(12000, 01, 01)} + + assert ~D[2000-01-01] + |> Date.convert!(Calendar.Holocene) + |> Date.convert!(Calendar.ISO) == ~D[2000-01-01] + + assert Date.convert(~N[2000-01-01 00:00:00], Calendar.Holocene) == + {:ok, Calendar.Holocene.date(12000, 01, 01)} + + assert Date.convert(~D[2016-02-03], FakeCalendar) == {:error, :incompatible_calendars} + + assert_raise ArgumentError, + "cannot convert ~D[2016-02-03] to target calendar FakeCalendar, reason: :incompatible_calendars", + fn -> Date.convert!(~D[2016-02-03], FakeCalendar) end + end + + test "add/2" do + assert Date.add(~D[0000-01-01], 3_652_424) == ~D[9999-12-31] + assert Date.add(~D[0000-01-01], 3_652_425) == Date.new!(10000, 1, 1) + assert Date.add(~D[0000-01-01], -1) == ~D[-0001-12-31] + assert Date.add(~D[0000-01-01], -365) == ~D[-0001-01-01] + assert Date.add(~D[0000-01-01], -366) == ~D[-0002-12-31] + assert Date.add(~D[0000-01-01], -(365 * 4)) == ~D[-0004-01-02] + assert Date.add(~D[0000-01-01], -(365 * 5)) == ~D[-0005-01-02] + assert Date.add(~D[0000-01-01], -(365 * 100)) == ~D[-0100-01-25] + assert Date.add(~D[0000-01-01], -3_652_059) == ~D[-9999-01-01] + assert Date.add(~D[0000-01-01], -3_652_060) == Date.new!(-10000, 12, 31) + assert Date.add(Date.new!(5_874_897, 12, 31), 1) == Date.new!(5_874_898, 1, 1) + end + + test "diff/2" do + assert Date.diff(~D[2000-01-31], ~D[2000-01-01]) == 30 + assert Date.diff(~D[2000-01-01], ~D[2000-01-31]) == -30 + + assert Date.diff(~D[0000-01-01], ~D[-0001-01-01]) == 365 + assert Date.diff(~D[-0003-01-01], ~D[-0004-01-01]) == 366 + + assert Date.diff(Date.new!(5_874_898, 1, 1), Date.new!(5_874_897, 1, 1)) == 365 + assert Date.diff(Date.new!(5_874_905, 1, 1), Date.new!(5_874_904, 1, 1)) == 366 + + date1 = ~D[2000-01-01] + date2 = Calendar.Holocene.date(12000, 01, 14) + assert Date.diff(date1, date2) == -13 + assert Date.diff(date2, date1) == 13 + + assert_raise ArgumentError, + ~r/cannot calculate the difference between .* because their calendars are not compatible/, + fn -> Date.diff(date1, %{date2 | calendar: FakeCalendar}) end + end + + test "shift/2" do + assert Date.shift(~D[2012-02-29], day: -1) == ~D[2012-02-28] + assert Date.shift(~D[2012-02-29], month: -1) == ~D[2012-01-29] + assert Date.shift(~D[2012-02-29], week: -9) == ~D[2011-12-28] + assert Date.shift(~D[2012-02-29], month: 1) == ~D[2012-03-29] + assert Date.shift(~D[2012-02-29], year: -1) == ~D[2011-02-28] + assert Date.shift(~D[2012-02-29], year: 4) == ~D[2016-02-29] + assert Date.shift(~D[0000-01-01], day: -1) == ~D[-0001-12-31] + assert Date.shift(~D[0000-01-01], month: -1) == ~D[-0001-12-01] + assert Date.shift(~D[0000-01-01], year: -1) == ~D[-0001-01-01] + assert Date.shift(~D[0000-01-01], year: -1) == ~D[-0001-01-01] + assert Date.shift(~D[2000-01-01], month: 12) == ~D[2001-01-01] + assert Date.shift(~D[0000-01-01], day: 2, year: 1, month: 37) == ~D[0004-02-03] + + assert Date.shift(Date.new!(5_874_904, 2, 29), day: -1) == Date.new!(5_874_904, 2, 28) + assert Date.shift(Date.new!(5_874_904, 2, 29), month: -2) == Date.new!(5_874_903, 12, 29) + assert Date.shift(Date.new!(5_874_904, 2, 29), week: -9) == Date.new!(5_874_903, 12, 28) + assert Date.shift(Date.new!(5_874_904, 2, 29), month: 1) == Date.new!(5_874_904, 3, 29) + assert Date.shift(Date.new!(5_874_904, 2, 29), year: -1) == Date.new!(5_874_903, 2, 28) + assert Date.shift(Date.new!(5_874_904, 2, 29), year: -4) == Date.new!(5_874_900, 2, 28) + assert Date.shift(Date.new!(5_874_904, 2, 29), year: 4) == Date.new!(5_874_908, 2, 29) + + assert Date.shift(Date.new!(5_874_904, 2, 29), day: 1, year: 4, month: 2) == + Date.new!(5_874_908, 4, 30) + + assert_raise ArgumentError, + "unsupported unit :second. Expected :year, :month, :week, :day", + fn -> Date.shift(~D[2012-02-29], second: 86400) end + + assert_raise ArgumentError, + "unknown unit :months. Expected :year, :month, :week, :day", + fn -> Date.shift(~D[2012-01-01], months: 12) end + + assert_raise ArgumentError, + "unsupported value nil for :day. Expected an integer", + fn -> Date.shift(~D[2012-02-29], year: 1, day: nil) end + + assert_raise ArgumentError, + "cannot shift date by time scale unit. Expected :year, :month, :week, :day", + fn -> Date.shift(~D[2012-02-29], %Duration{second: 86400}) end + + # Microsecond precision is ignored + assert Date.shift(~D[2012-02-29], Duration.new!(microsecond: {0, 6})) == ~D[2012-02-29] + + assert_raise ArgumentError, + "cannot shift date by time scale unit. Expected :year, :month, :week, :day", + fn -> Date.shift(~D[2012-02-29], %Duration{microsecond: {100, 6}}) end + + # Implements calendar callback + assert_raise RuntimeError, "shift_date/4 not implemented", fn -> + date = Calendar.Holocene.date(10000, 01, 01) + Date.shift(date, month: 1) + end + end + + test "utc_today/1" do + date = Date.utc_today() + assert date.year > 2020 + assert date.calendar == Calendar.ISO + + date = Date.utc_today(Calendar.ISO) + assert date.year > 2020 + assert date.calendar == Calendar.ISO + + date = Date.utc_today(Calendar.Holocene) + assert date.year > 12020 + assert date.calendar == Calendar.Holocene + end +end diff --git a/lib/elixir/test/elixir/calendar/datetime_test.exs b/lib/elixir/test/elixir/calendar/datetime_test.exs new file mode 100644 index 00000000000..712e34633ee --- /dev/null +++ b/lib/elixir/test/elixir/calendar/datetime_test.exs @@ -0,0 +1,1181 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) +Code.require_file("holocene.exs", __DIR__) +Code.require_file("fakes.exs", __DIR__) + +defmodule DateTimeTest do + use ExUnit.Case + doctest DateTime + + test "sigil_U" do + assert ~U[2000-01-01T12:34:56Z] == + %DateTime{ + calendar: Calendar.ISO, + year: 2000, + month: 1, + day: 1, + hour: 12, + minute: 34, + second: 56, + std_offset: 0, + utc_offset: 0, + time_zone: "Etc/UTC", + zone_abbr: "UTC" + } + + assert ~U[2000-01-01T12:34:56+00:00 Calendar.Holocene] == + %DateTime{ + calendar: Calendar.Holocene, + year: 2000, + month: 1, + day: 1, + hour: 12, + minute: 34, + second: 56, + std_offset: 0, + utc_offset: 0, + time_zone: "Etc/UTC", + zone_abbr: "UTC" + } + + assert ~U[2000-01-01 12:34:56+00:00] == + %DateTime{ + calendar: Calendar.ISO, + year: 2000, + month: 1, + day: 1, + hour: 12, + minute: 34, + second: 56, + std_offset: 0, + utc_offset: 0, + time_zone: "Etc/UTC", + zone_abbr: "UTC" + } + + assert ~U[2000-01-01 12:34:56Z Calendar.Holocene] == + %DateTime{ + calendar: Calendar.Holocene, + year: 2000, + month: 1, + day: 1, + hour: 12, + minute: 34, + second: 56, + std_offset: 0, + utc_offset: 0, + time_zone: "Etc/UTC", + zone_abbr: "UTC" + } + + assert_raise ArgumentError, + ~s/cannot parse "2001-50-50T12:34:56Z" as UTC DateTime for Calendar.ISO, reason: :invalid_date/, + fn -> Code.eval_string("~U[2001-50-50T12:34:56Z]") end + + assert_raise ArgumentError, + ~s/cannot parse "2001-01-01T12:34:65Z" as UTC DateTime for Calendar.ISO, reason: :invalid_time/, + fn -> Code.eval_string("~U[2001-01-01T12:34:65Z]") end + + assert_raise ArgumentError, + ~s/cannot parse "2001-01-01T12:34:56\+01:00" as UTC DateTime for Calendar.ISO, reason: :non_utc_offset/, + fn -> Code.eval_string("~U[2001-01-01T12:34:56+01:00]") end + + assert_raise ArgumentError, + ~s/cannot parse "2001-01-01 12:34:56Z notalias" as UTC DateTime for Calendar.ISO, reason: :invalid_format/, + fn -> Code.eval_string("~U[2001-01-01 12:34:56Z notalias]") end + + assert_raise ArgumentError, + ~s/cannot parse "2001-01-01T12:34:56Z notalias" as UTC DateTime for Calendar.ISO, reason: :invalid_format/, + fn -> Code.eval_string("~U[2001-01-01T12:34:56Z notalias]") end + + assert_raise ArgumentError, + ~s/cannot parse "2001-50-50T12:34:56Z" as UTC DateTime for Calendar.Holocene, reason: :invalid_date/, + fn -> Code.eval_string("~U[2001-50-50T12:34:56Z Calendar.Holocene]") end + + assert_raise ArgumentError, + ~s/cannot parse "2001-01-01T12:34:65Z" as UTC DateTime for Calendar.Holocene, reason: :invalid_time/, + fn -> Code.eval_string("~U[2001-01-01T12:34:65Z Calendar.Holocene]") end + + assert_raise ArgumentError, + ~s/cannot parse "2001-01-01T12:34:56+01:00 Calendar.Holocene" as UTC DateTime for Calendar.Holocene, reason: :non_utc_offset/, + fn -> Code.eval_string("~U[2001-01-01T12:34:56+01:00 Calendar.Holocene]") end + + assert_raise UndefinedFunctionError, fn -> + Code.eval_string("~U[2001-01-01 12:34:56 UnknownCalendar]") + end + + assert_raise UndefinedFunctionError, fn -> + Code.eval_string("~U[2001-01-01T12:34:56 UnknownCalendar]") + end + end + + test "to_string/1" do + datetime = %DateTime{ + year: 2000, + month: 2, + day: 29, + zone_abbr: "BRM", + hour: 23, + minute: 0, + second: 7, + microsecond: {0, 0}, + utc_offset: -12600, + std_offset: 3600, + time_zone: "Brazil/Manaus" + } + + assert to_string(datetime) == "2000-02-29 23:00:07-02:30 BRM Brazil/Manaus" + assert DateTime.to_string(datetime) == "2000-02-29 23:00:07-02:30 BRM Brazil/Manaus" + + assert DateTime.to_string(Map.from_struct(datetime)) == + "2000-02-29 23:00:07-02:30 BRM Brazil/Manaus" + + assert to_string(%{datetime | calendar: FakeCalendar}) == + "29/2/2000F23::0::7 Brazil/Manaus BRM -12600 3600" + + assert DateTime.to_string(%{datetime | calendar: FakeCalendar}) == + "29/2/2000F23::0::7 Brazil/Manaus BRM -12600 3600" + end + + test "inspect/1" do + utc_datetime = ~U[2000-01-01 23:00:07.005Z] + assert inspect(utc_datetime) == "~U[2000-01-01 23:00:07.005Z]" + + assert inspect(%{utc_datetime | year: 99999}) == "#DateTime<99999-01-01 23:00:07.005Z>" + + assert inspect(%{utc_datetime | calendar: FakeCalendar}) == + "~U[1/1/2000F23::0::7 Etc/UTC UTC 0 0 FakeCalendar]" + + datetime = %DateTime{ + year: 2000, + month: 2, + day: 29, + zone_abbr: "BRM", + hour: 23, + minute: 0, + second: 7, + microsecond: {0, 0}, + utc_offset: -12600, + std_offset: 3600, + time_zone: "Brazil/Manaus" + } + + assert inspect(datetime) == "#DateTime<2000-02-29 23:00:07-02:30 BRM Brazil/Manaus>" + + assert inspect(%{datetime | calendar: FakeCalendar}) == + "#DateTime<29/2/2000F23::0::7 Brazil/Manaus BRM -12600 3600 FakeCalendar>" + end + + test "from_iso8601/1 handles positive and negative offsets" do + assert DateTime.from_iso8601("2015-01-24T09:50:07-10:00") |> elem(1) == + %DateTime{ + microsecond: {0, 0}, + month: 1, + std_offset: 0, + time_zone: "Etc/UTC", + utc_offset: 0, + year: 2015, + zone_abbr: "UTC", + day: 24, + hour: 19, + minute: 50, + second: 7 + } + + assert DateTime.from_iso8601("2015-01-24T09:50:07+10:00") |> elem(1) == + %DateTime{ + microsecond: {0, 0}, + month: 1, + std_offset: 0, + time_zone: "Etc/UTC", + utc_offset: 0, + year: 2015, + zone_abbr: "UTC", + day: 23, + hour: 23, + minute: 50, + second: 7 + } + + assert DateTime.from_iso8601("0000-01-01T01:22:07+10:30") |> elem(1) == + %DateTime{ + microsecond: {0, 0}, + month: 12, + std_offset: 0, + time_zone: "Etc/UTC", + utc_offset: 0, + year: -1, + zone_abbr: "UTC", + day: 31, + hour: 14, + minute: 52, + second: 7 + } + end + + test "from_iso8601/1 handles negative dates" do + assert DateTime.from_iso8601("-2015-01-24T09:50:07-10:00") |> elem(1) == + %DateTime{ + microsecond: {0, 0}, + month: 1, + std_offset: 0, + time_zone: "Etc/UTC", + utc_offset: 0, + year: -2015, + zone_abbr: "UTC", + day: 24, + hour: 19, + minute: 50, + second: 7 + } + + assert DateTime.from_iso8601("-2015-01-24T09:50:07+10:00") |> elem(1) == + %DateTime{ + microsecond: {0, 0}, + month: 1, + std_offset: 0, + time_zone: "Etc/UTC", + utc_offset: 0, + year: -2015, + zone_abbr: "UTC", + day: 23, + hour: 23, + minute: 50, + second: 7 + } + + assert DateTime.from_iso8601("-0001-01-01T01:22:07+10:30") |> elem(1) == + %DateTime{ + microsecond: {0, 0}, + month: 12, + std_offset: 0, + time_zone: "Etc/UTC", + utc_offset: 0, + year: -2, + zone_abbr: "UTC", + day: 31, + hour: 14, + minute: 52, + second: 7 + } + + assert DateTime.from_iso8601("-0001-01-01T01:22:07-10:30") |> elem(1) == + %DateTime{ + microsecond: {0, 0}, + month: 1, + std_offset: 0, + time_zone: "Etc/UTC", + utc_offset: 0, + year: -1, + zone_abbr: "UTC", + day: 1, + hour: 11, + minute: 52, + second: 7 + } + + assert DateTime.from_iso8601("-0001-12-31T23:22:07-10:30") |> elem(1) == + %DateTime{ + microsecond: {0, 0}, + month: 1, + std_offset: 0, + time_zone: "Etc/UTC", + utc_offset: 0, + year: 0, + zone_abbr: "UTC", + day: 1, + hour: 9, + minute: 52, + second: 7 + } + end + + test "from_iso8601/3 with basic format handles positive and negative offsets" do + assert DateTime.from_iso8601("20150124T095007-1000", Calendar.ISO, :basic) == + DateTime.from_iso8601("2015-01-24T09:50:07-10:00", Calendar.ISO) + + assert DateTime.from_iso8601("20150124T095007+1000", Calendar.ISO, :basic) == + DateTime.from_iso8601("2015-01-24T09:50:07+10:00", Calendar.ISO) + + assert DateTime.from_iso8601("00000101T012207+1030", Calendar.ISO, :basic) == + DateTime.from_iso8601("0000-01-01T01:22:07+10:30", Calendar.ISO) + end + + test "from_iso8601/3 with basic format handles negative dates" do + assert DateTime.from_iso8601("-20150124T095007-1000", Calendar.ISO, :basic) == + DateTime.from_iso8601("-2015-01-24T09:50:07-10:00", Calendar.ISO) + + assert DateTime.from_iso8601("-20150124T095007+1000", Calendar.ISO, :basic) == + DateTime.from_iso8601("-2015-01-24T09:50:07+10:00", Calendar.ISO) + + assert DateTime.from_iso8601("-00010101T012207+1030", Calendar.ISO, :basic) == + DateTime.from_iso8601("-0001-01-01T01:22:07+10:30", Calendar.ISO) + + assert DateTime.from_iso8601("-00010101T012207-1030", Calendar.ISO, :basic) == + DateTime.from_iso8601("-0001-01-01T01:22:07-10:30", Calendar.ISO) + + assert DateTime.from_iso8601("-00011231T232207-1030", Calendar.ISO, :basic) == + DateTime.from_iso8601("-0001-12-31T23:22:07-10:30", Calendar.ISO) + end + + test "from_iso8601/2 handles either a calendar or a format as the second parameter" do + assert DateTime.from_iso8601("20150124T095007-1000", :basic) == + DateTime.from_iso8601("2015-01-24T09:50:07-10:00", Calendar.ISO) + end + + test "from_iso8601 handles invalid date, time, formats correctly" do + assert DateTime.from_iso8601("2015-01-23T23:50:07") == {:error, :missing_offset} + assert DateTime.from_iso8601("2015-01-23 23:50:61") == {:error, :invalid_time} + assert DateTime.from_iso8601("2015-01-32 23:50:07") == {:error, :invalid_date} + assert DateTime.from_iso8601("2015-01-23 23:50:07A") == {:error, :invalid_format} + assert DateTime.from_iso8601("2015-01-23T23:50:07.123-00:60") == {:error, :invalid_format} + + assert DateTime.from_iso8601("20150123T235007", Calendar.ISO, :basic) == + {:error, :missing_offset} + + assert DateTime.from_iso8601("20150123 235061", Calendar.ISO, :basic) == + {:error, :invalid_time} + + assert DateTime.from_iso8601("20150132 235007", Calendar.ISO, :basic) == + {:error, :invalid_date} + + assert DateTime.from_iso8601("20150123 235007A", Calendar.ISO, :basic) == + {:error, :invalid_format} + + assert DateTime.from_iso8601("2015-01-24T09:50:07-10:00", Calendar.ISO, :basic) == + {:error, :invalid_format} + + assert DateTime.from_iso8601("20150123T235007.123-0060", Calendar.ISO, :basic) == + {:error, :invalid_format} + end + + test "from_unix/2" do + min_datetime = %DateTime{ + calendar: Calendar.ISO, + day: 1, + hour: 0, + microsecond: {0, 0}, + minute: 0, + month: 1, + second: 0, + std_offset: 0, + time_zone: "Etc/UTC", + utc_offset: 0, + year: -9999, + zone_abbr: "UTC" + } + + assert DateTime.from_unix(-377_705_116_800) == {:ok, min_datetime} + + assert DateTime.from_unix(-377_705_116_800_000_001, :microsecond) == + {:error, :invalid_unix_time} + + assert DateTime.from_unix(143_256_036_886_856, 1024) == + {:ok, + %DateTime{ + calendar: Calendar.ISO, + day: 17, + hour: 7, + microsecond: {320_312, 6}, + minute: 5, + month: 3, + second: 22, + std_offset: 0, + time_zone: "Etc/UTC", + utc_offset: 0, + year: 6403, + zone_abbr: "UTC" + }} + + max_datetime = %DateTime{ + calendar: Calendar.ISO, + day: 31, + hour: 23, + microsecond: {999_999, 6}, + minute: 59, + month: 12, + second: 59, + std_offset: 0, + time_zone: "Etc/UTC", + utc_offset: 0, + year: 9999, + zone_abbr: "UTC" + } + + assert DateTime.from_unix(253_402_300_799_999_999, :microsecond) == {:ok, max_datetime} + + assert DateTime.from_unix(253_402_300_800) == {:error, :invalid_unix_time} + + minus_datetime = %DateTime{ + calendar: Calendar.ISO, + day: 31, + hour: 23, + microsecond: {999_999, 6}, + minute: 59, + month: 12, + second: 59, + std_offset: 0, + time_zone: "Etc/UTC", + utc_offset: 0, + year: 1969, + zone_abbr: "UTC" + } + + assert DateTime.from_unix(-1, :microsecond) == {:ok, minus_datetime} + + assert_raise ArgumentError, fn -> + DateTime.from_unix(0, :unknown_atom) + end + + assert_raise ArgumentError, fn -> + DateTime.from_unix(0, "invalid type") + end + end + + test "from_unix!/2" do + # with Unix times back to 0 Gregorian seconds + datetime = %DateTime{ + calendar: Calendar.ISO, + day: 1, + hour: 0, + microsecond: {0, 0}, + minute: 0, + month: 1, + second: 0, + std_offset: 0, + time_zone: "Etc/UTC", + utc_offset: 0, + year: 0, + zone_abbr: "UTC" + } + + assert DateTime.from_unix!(-62_167_219_200) == datetime + + assert_raise ArgumentError, fn -> + DateTime.from_unix!(-377_705_116_801) + end + + assert_raise ArgumentError, fn -> + DateTime.from_unix!(0, :unknown_atom) + end + + assert_raise ArgumentError, fn -> + DateTime.from_unix!(0, "invalid type") + end + end + + test "to_unix/2 works with Unix times back to 0 Gregorian seconds" do + # with Unix times back to 0 Gregorian seconds + gregorian_0 = %DateTime{ + calendar: Calendar.ISO, + day: 1, + hour: 0, + microsecond: {0, 0}, + minute: 0, + month: 1, + second: 0, + std_offset: 0, + time_zone: "Etc/UTC", + utc_offset: 0, + year: 0, + zone_abbr: "UTC" + } + + assert DateTime.to_unix(gregorian_0) == -62_167_219_200 + assert DateTime.to_unix(Map.from_struct(gregorian_0)) == -62_167_219_200 + + min_datetime = %{gregorian_0 | year: -9999} + assert DateTime.to_unix(min_datetime) == -377_705_116_800 + end + + test "compare/2" do + datetime1 = %DateTime{ + year: 2000, + month: 2, + day: 29, + zone_abbr: "CET", + hour: 23, + minute: 0, + second: 7, + microsecond: {0, 0}, + utc_offset: 3600, + std_offset: 0, + time_zone: "Europe/Warsaw" + } + + datetime2 = %DateTime{ + year: 2000, + month: 2, + day: 29, + zone_abbr: "AMT", + hour: 23, + minute: 0, + second: 7, + microsecond: {0, 0}, + utc_offset: -14400, + std_offset: 0, + time_zone: "America/Manaus" + } + + datetime3 = %DateTime{ + year: -99, + month: 2, + day: 28, + zone_abbr: "AMT", + hour: 23, + minute: 0, + second: 7, + microsecond: {0, 0}, + utc_offset: -14400, + std_offset: 0, + time_zone: "America/Manaus" + } + + assert DateTime.compare(datetime1, datetime1) == :eq + assert DateTime.compare(datetime1, datetime2) == :lt + assert DateTime.compare(datetime2, datetime1) == :gt + assert DateTime.compare(datetime3, datetime3) == :eq + assert DateTime.compare(datetime2, datetime3) == :gt + assert DateTime.compare(datetime3, datetime1) == :lt + assert DateTime.compare(Map.from_struct(datetime3), Map.from_struct(datetime1)) == :lt + end + + test "before?/2 and after?/2" do + datetime1 = ~U[2015-01-02T12:34:56Z] + datetime2 = ~U[2015-01-02T12:55:55Z] + + assert DateTime.before?(datetime1, datetime2) + assert not DateTime.before?(datetime2, datetime1) + + assert DateTime.after?(datetime2, datetime1) + assert not DateTime.after?(datetime1, datetime2) + end + + test "convert/2" do + datetime_iso = %DateTime{ + year: 2000, + month: 2, + day: 29, + zone_abbr: "CET", + hour: 23, + minute: 0, + second: 7, + microsecond: {0, 0}, + utc_offset: 3600, + std_offset: 0, + time_zone: "Europe/Warsaw" + } + + datetime_hol = %DateTime{ + year: 12000, + month: 2, + day: 29, + zone_abbr: "CET", + hour: 23, + minute: 0, + second: 7, + microsecond: {0, 0}, + utc_offset: 3600, + std_offset: 0, + time_zone: "Europe/Warsaw", + calendar: Calendar.Holocene + } + + assert DateTime.convert(datetime_iso, Calendar.Holocene) == {:ok, datetime_hol} + + assert datetime_iso + |> DateTime.convert!(Calendar.Holocene) + |> DateTime.convert!(Calendar.ISO) == datetime_iso + + assert %{datetime_iso | microsecond: {123, 6}} + |> DateTime.convert!(Calendar.Holocene) + |> DateTime.convert!(Calendar.ISO) == %{datetime_iso | microsecond: {123, 6}} + + assert DateTime.convert(datetime_iso, FakeCalendar) == {:error, :incompatible_calendars} + + # Test passing non-struct map when converting to different calendar returns DateTime struct + assert DateTime.convert(Map.from_struct(datetime_iso), Calendar.Holocene) == + {:ok, datetime_hol} + + # Test passing non-struct map when converting to same calendar returns DateTime struct + assert DateTime.convert(Map.from_struct(datetime_iso), Calendar.ISO) == + {:ok, datetime_iso} + end + + test "from_iso8601/1 with tz offsets" do + assert DateTime.from_iso8601("2017-06-02T14:00:00+01:00") + |> elem(1) == + %DateTime{ + year: 2017, + month: 6, + day: 2, + zone_abbr: "UTC", + hour: 13, + minute: 0, + second: 0, + microsecond: {0, 0}, + utc_offset: 0, + std_offset: 0, + time_zone: "Etc/UTC" + } + + assert DateTime.from_iso8601("2017-06-02T14:00:00-04:00") + |> elem(1) == + %DateTime{ + year: 2017, + month: 6, + day: 2, + zone_abbr: "UTC", + hour: 18, + minute: 0, + second: 0, + microsecond: {0, 0}, + utc_offset: 0, + std_offset: 0, + time_zone: "Etc/UTC" + } + + assert DateTime.from_iso8601("2017-06-02T14:00:00+0100") + |> elem(1) == + %DateTime{ + year: 2017, + month: 6, + day: 2, + zone_abbr: "UTC", + hour: 13, + minute: 0, + second: 0, + microsecond: {0, 0}, + utc_offset: 0, + std_offset: 0, + time_zone: "Etc/UTC" + } + + assert DateTime.from_iso8601("2017-06-02T14:00:00-0400") + |> elem(1) == + %DateTime{ + year: 2017, + month: 6, + day: 2, + zone_abbr: "UTC", + hour: 18, + minute: 0, + second: 0, + microsecond: {0, 0}, + utc_offset: 0, + std_offset: 0, + time_zone: "Etc/UTC" + } + + assert DateTime.from_iso8601("2017-06-02T14:00:00+01") + |> elem(1) == + %DateTime{ + year: 2017, + month: 6, + day: 2, + zone_abbr: "UTC", + hour: 13, + minute: 0, + second: 0, + microsecond: {0, 0}, + utc_offset: 0, + std_offset: 0, + time_zone: "Etc/UTC" + } + + assert DateTime.from_iso8601("2017-06-02T14:00:00-04") + |> elem(1) == + %DateTime{ + year: 2017, + month: 6, + day: 2, + zone_abbr: "UTC", + hour: 18, + minute: 0, + second: 0, + microsecond: {0, 0}, + utc_offset: 0, + std_offset: 0, + time_zone: "Etc/UTC" + } + end + + test "from_iso8601/3 with basic format with tz offsets" do + assert DateTime.from_iso8601("20170602T140000+0100", Calendar.ISO, :basic) == + DateTime.from_iso8601("2017-06-02T14:00:00+01:00", Calendar.ISO) + + assert DateTime.from_iso8601("20170602T140000-0400", Calendar.ISO, :basic) == + DateTime.from_iso8601("2017-06-02T14:00:00-04:00") + + assert DateTime.from_iso8601("20170602T140000+01", Calendar.ISO, :basic) == + DateTime.from_iso8601("2017-06-02T14:00:00+01") + + assert DateTime.from_iso8601("20170602T140000-04", Calendar.ISO, :basic) == + DateTime.from_iso8601("2017-06-02T14:00:00-04") + end + + test "truncate/2" do + datetime = %DateTime{ + year: 2017, + month: 11, + day: 6, + zone_abbr: "CET", + hour: 0, + minute: 6, + second: 23, + microsecond: {0, 0}, + utc_offset: 3600, + std_offset: 0, + time_zone: "Europe/Paris" + } + + datetime_map = Map.from_struct(datetime) + + assert DateTime.truncate(%{datetime | microsecond: {123_456, 6}}, :microsecond) == + %{datetime | microsecond: {123_456, 6}} + + # A struct should be returned when passing a map. + assert DateTime.truncate(%{datetime_map | microsecond: {123_456, 6}}, :microsecond) == + %{datetime | microsecond: {123_456, 6}} + + assert DateTime.truncate(%{datetime | microsecond: {0, 0}}, :millisecond) == + %{datetime | microsecond: {0, 0}} + + assert DateTime.truncate(%{datetime | microsecond: {000_100, 6}}, :millisecond) == + %{datetime | microsecond: {0, 3}} + + assert DateTime.truncate(%{datetime | microsecond: {000_999, 6}}, :millisecond) == + %{datetime | microsecond: {0, 3}} + + assert DateTime.truncate(%{datetime | microsecond: {001_000, 6}}, :millisecond) == + %{datetime | microsecond: {1000, 3}} + + assert DateTime.truncate(%{datetime | microsecond: {001_200, 6}}, :millisecond) == + %{datetime | microsecond: {1000, 3}} + + assert DateTime.truncate(%{datetime | microsecond: {123_456, 6}}, :millisecond) == + %{datetime | microsecond: {123_000, 3}} + + assert DateTime.truncate(%{datetime | microsecond: {123_456, 6}}, :second) == + %{datetime | microsecond: {0, 0}} + end + + describe "diff" do + test "diff with invalid time unit" do + dt = DateTime.utc_now() + + message = + ~r/unsupported time unit\. Expected :day, :hour, :minute, :second, :millisecond, :microsecond, :nanosecond, or a positive integer, got "day"/ + + assert_raise ArgumentError, message, fn -> DateTime.diff(dt, dt, "day") end + end + + test "diff with valid time unit" do + dt1 = %DateTime{ + year: 100, + month: 2, + day: 28, + zone_abbr: "CET", + hour: 23, + minute: 0, + second: 7, + microsecond: {0, 0}, + utc_offset: 3600, + std_offset: 0, + time_zone: "Europe/Warsaw" + } + + dt2 = %DateTime{ + year: -0004, + month: 2, + day: 29, + zone_abbr: "CET", + hour: 23, + minute: 0, + second: 7, + microsecond: {0, 0}, + utc_offset: 3600, + std_offset: 0, + time_zone: "Europe/Warsaw" + } + + assert DateTime.diff(dt1, dt2) == 3_281_904_000 + + # Test with a non-struct map conforming to Calendar.datetime + assert DateTime.diff(Map.from_struct(dt1), Map.from_struct(dt2)) == 3_281_904_000 + end + + test "diff with microseconds" do + datetime = ~U[2023-02-01 10:30:10.123456Z] + + in_almost_7_days = + datetime + |> DateTime.add(7, :day) + |> DateTime.add(-1, :microsecond) + + assert DateTime.diff(in_almost_7_days, datetime, :day) == 6 + end + + test "diff in microseconds" do + datetime1 = ~U[2023-02-01 10:30:10.000000Z] + datetime2 = DateTime.add(datetime1, 1234, :microsecond) + + assert DateTime.diff(datetime1, datetime2, :microsecond) == -1234 + end + end + + describe "from_naive" do + test "uses default time zone database from config" do + Calendar.put_time_zone_database(FakeTimeZoneDatabase) + + assert DateTime.from_naive( + ~N[2018-07-01 12:34:25.123456], + "Europe/Copenhagen", + FakeTimeZoneDatabase + ) == + {:ok, + %DateTime{ + day: 1, + hour: 12, + microsecond: {123_456, 6}, + minute: 34, + month: 7, + second: 25, + std_offset: 3600, + time_zone: "Europe/Copenhagen", + utc_offset: 3600, + year: 2018, + zone_abbr: "CEST" + }} + after + Calendar.put_time_zone_database(Calendar.UTCOnlyTimeZoneDatabase) + end + + test "with compatible calendar on unambiguous wall clock" do + holocene_ndt = %NaiveDateTime{ + calendar: Calendar.Holocene, + year: 12018, + month: 7, + day: 1, + hour: 12, + minute: 34, + second: 25, + microsecond: {123_456, 6} + } + + assert DateTime.from_naive(holocene_ndt, "Europe/Copenhagen", FakeTimeZoneDatabase) == + {:ok, + %DateTime{ + calendar: Calendar.Holocene, + day: 1, + hour: 12, + microsecond: {123_456, 6}, + minute: 34, + month: 7, + second: 25, + std_offset: 3600, + time_zone: "Europe/Copenhagen", + utc_offset: 3600, + year: 12018, + zone_abbr: "CEST" + }} + end + + test "with compatible calendar on ambiguous wall clock" do + holocene_ndt = %NaiveDateTime{ + calendar: Calendar.Holocene, + year: 12018, + month: 10, + day: 28, + hour: 02, + minute: 30, + second: 00, + microsecond: {123_456, 6} + } + + assert {:ambiguous, first_dt, second_dt} = + DateTime.from_naive(holocene_ndt, "Europe/Copenhagen", FakeTimeZoneDatabase) + + assert %DateTime{calendar: Calendar.Holocene, zone_abbr: "CEST"} = first_dt + assert %DateTime{calendar: Calendar.Holocene, zone_abbr: "CET"} = second_dt + end + + test "with compatible calendar on gap" do + holocene_ndt = %NaiveDateTime{ + calendar: Calendar.Holocene, + year: 12019, + month: 03, + day: 31, + hour: 02, + minute: 30, + second: 00, + microsecond: {123_456, 6} + } + + assert {:gap, first_dt, second_dt} = + DateTime.from_naive(holocene_ndt, "Europe/Copenhagen", FakeTimeZoneDatabase) + + assert %DateTime{calendar: Calendar.Holocene, zone_abbr: "CET"} = first_dt + assert %DateTime{calendar: Calendar.Holocene, zone_abbr: "CEST"} = second_dt + end + + test "with incompatible calendar" do + ndt = %{~N[2018-07-20 00:00:00] | calendar: FakeCalendar} + + assert DateTime.from_naive(ndt, "Europe/Copenhagen", FakeTimeZoneDatabase) == + {:error, :incompatible_calendars} + end + end + + describe "from_naive!" do + test "raises on ambiguous wall clock" do + assert_raise ArgumentError, ~r"ambiguous", fn -> + DateTime.from_naive!(~N[2018-10-28 02:30:00], "Europe/Copenhagen", FakeTimeZoneDatabase) + end + end + + test "raises on gap" do + assert_raise ArgumentError, ~r"gap", fn -> + DateTime.from_naive!(~N[2019-03-31 02:30:00], "Europe/Copenhagen", FakeTimeZoneDatabase) + end + end + end + + describe "shift_zone" do + test "with compatible calendar" do + holocene_ndt = %NaiveDateTime{ + calendar: Calendar.Holocene, + year: 12018, + month: 7, + day: 1, + hour: 12, + minute: 34, + second: 25, + microsecond: {123_456, 6} + } + + {:ok, holocene_dt} = + DateTime.from_naive(holocene_ndt, "Europe/Copenhagen", FakeTimeZoneDatabase) + + {:ok, dt} = DateTime.shift_zone(holocene_dt, "America/Los_Angeles", FakeTimeZoneDatabase) + + assert dt == %DateTime{ + calendar: Calendar.Holocene, + day: 1, + hour: 3, + microsecond: {123_456, 6}, + minute: 34, + month: 7, + second: 25, + std_offset: 3600, + time_zone: "America/Los_Angeles", + utc_offset: -28800, + year: 12018, + zone_abbr: "PDT" + } + end + + test "uses default time zone database from config" do + Calendar.put_time_zone_database(FakeTimeZoneDatabase) + + {:ok, dt} = DateTime.from_naive(~N[2018-07-01 12:34:25.123456], "Europe/Copenhagen") + {:ok, dt} = DateTime.shift_zone(dt, "America/Los_Angeles") + + assert dt == %DateTime{ + day: 1, + hour: 3, + microsecond: {123_456, 6}, + minute: 34, + month: 7, + second: 25, + std_offset: 3600, + time_zone: "America/Los_Angeles", + utc_offset: -28800, + year: 2018, + zone_abbr: "PDT" + } + after + Calendar.put_time_zone_database(Calendar.UTCOnlyTimeZoneDatabase) + end + end + + describe "add" do + test "add with invalid time unit" do + dt = DateTime.utc_now() + + message = + ~r/unsupported time unit\. Expected :day, :hour, :minute, :second, :millisecond, :microsecond, :nanosecond, or a positive integer, got "day"/ + + assert_raise ArgumentError, message, fn -> DateTime.add(dt, 1, "day") end + end + + test "add with non-struct map that conforms to Calendar.datetime" do + dt_map = DateTime.from_naive!(~N[2018-08-28 00:00:00], "Etc/UTC") |> Map.from_struct() + + assert DateTime.add(dt_map, 1, :second) == %DateTime{ + calendar: Calendar.ISO, + year: 2018, + month: 8, + day: 28, + hour: 0, + minute: 0, + second: 1, + std_offset: 0, + time_zone: "Etc/UTC", + zone_abbr: "UTC", + utc_offset: 0, + microsecond: {0, 0} + } + end + + test "error with UTC only database and non UTC datetime" do + dt = + DateTime.from_naive!(~N[2018-08-28 00:00:00], "Europe/Copenhagen", FakeTimeZoneDatabase) + + assert_raise ArgumentError, fn -> + DateTime.add(dt, 1, :second) + end + end + + test "add/2 with other calendars" do + assert ~N[2000-01-01 12:34:15.123456] + |> NaiveDateTime.convert!(Calendar.Holocene) + |> DateTime.from_naive!("Etc/UTC") + |> DateTime.add(10, :second) == + %DateTime{ + calendar: Calendar.Holocene, + year: 12000, + month: 1, + day: 1, + hour: 12, + minute: 34, + second: 25, + std_offset: 0, + time_zone: "Etc/UTC", + zone_abbr: "UTC", + utc_offset: 0, + microsecond: {123_456, 6} + } + end + end + + describe "to_iso8601" do + test "to_iso8601/2 with a normal DateTime struct" do + datetime = DateTime.from_naive!(~N[2018-07-01 12:34:25.123456], "Etc/UTC") + + assert DateTime.to_iso8601(datetime) == "2018-07-01T12:34:25.123456Z" + end + + test "to_iso8601/2 with a non-struct map conforming to the Calendar.datetime type" do + datetime_map = + DateTime.from_naive!(~N[2018-07-01 12:34:25.123456], "Etc/UTC") |> Map.from_struct() + + assert DateTime.to_iso8601(datetime_map) == "2018-07-01T12:34:25.123456Z" + end + end + + describe "to_date/1" do + test "upcasting" do + assert catch_error(DateTime.to_date(~N[2000-02-29 12:23:34])) + end + end + + describe "to_time/1" do + test "upcasting" do + assert catch_error(DateTime.to_time(~N[2000-02-29 12:23:34])) + end + end + + describe "to_naive/1" do + test "upcasting" do + assert catch_error(DateTime.to_naive(~N[2000-02-29 12:23:34])) + end + end + + test "shift/2" do + assert DateTime.shift(~U[2000-01-01 00:00:00Z], year: 1) == ~U[2001-01-01 00:00:00Z] + assert DateTime.shift(~U[2000-01-01 00:00:00Z], month: 1) == ~U[2000-02-01 00:00:00Z] + assert DateTime.shift(~U[2000-01-01 00:00:00Z], month: 1, day: 28) == ~U[2000-02-29 00:00:00Z] + assert DateTime.shift(~U[2000-01-01 00:00:00Z], month: 1, day: 30) == ~U[2000-03-02 00:00:00Z] + assert DateTime.shift(~U[2000-01-01 00:00:00Z], month: 2, day: 29) == ~U[2000-03-30 00:00:00Z] + + assert DateTime.shift(~U[2000-01-01 00:00:00Z], microsecond: {4000, 4}) == + ~U[2000-01-01 00:00:00.0040Z] + + assert DateTime.shift(~U[2000-02-29 00:00:00Z], year: -1) == ~U[1999-02-28 00:00:00Z] + assert DateTime.shift(~U[2000-02-29 00:00:00Z], month: -1) == ~U[2000-01-29 00:00:00Z] + + assert DateTime.shift(~U[2000-02-29 00:00:00Z], month: -1, day: -28) == + ~U[2000-01-01 00:00:00Z] + + assert DateTime.shift(~U[2000-02-29 00:00:00Z], month: -1, day: -30) == + ~U[1999-12-30 00:00:00Z] + + assert DateTime.shift(~U[2000-02-29 00:00:00Z], month: -1, day: -29) == + ~U[1999-12-31 00:00:00Z] + + datetime = + DateTime.new!(~D[2018-11-04], ~T[03:00:00], "America/Los_Angeles", FakeTimeZoneDatabase) + + assert DateTime.shift(datetime, [month: -1], FakeTimeZoneDatabase) == + %DateTime{ + calendar: Calendar.ISO, + year: 2018, + month: 10, + day: 4, + hour: 4, + minute: 0, + second: 0, + microsecond: {0, 0}, + time_zone: "America/Los_Angeles", + std_offset: 3600, + utc_offset: -28800, + zone_abbr: "PDT" + } + + datetime = + DateTime.new!(~D[2018-11-04], ~T[00:00:00], "America/Los_Angeles", FakeTimeZoneDatabase) + + assert DateTime.shift(datetime, [hour: 2], FakeTimeZoneDatabase) == + %DateTime{ + calendar: Calendar.ISO, + year: 2018, + month: 11, + day: 4, + hour: 1, + minute: 0, + second: 0, + microsecond: {0, 0}, + time_zone: "America/Los_Angeles", + std_offset: 0, + utc_offset: -28800, + zone_abbr: "PST" + } + + datetime = + DateTime.new!(~D[2019-03-31], ~T[01:00:00], "Europe/Copenhagen", FakeTimeZoneDatabase) + + assert DateTime.shift(datetime, [hour: 1], FakeTimeZoneDatabase) == + %DateTime{ + calendar: Calendar.ISO, + year: 2019, + month: 03, + day: 31, + hour: 3, + minute: 0, + second: 0, + microsecond: {0, 0}, + time_zone: "Europe/Copenhagen", + std_offset: 3600, + utc_offset: 3600, + zone_abbr: "CEST" + } + + assert_raise ArgumentError, + "unknown unit :months. Expected :year, :month, :week, :day, :hour, :minute, :second, :microsecond", + fn -> DateTime.shift(~U[2012-01-01 00:00:00Z], months: 12) end + end +end diff --git a/lib/elixir/test/elixir/calendar/duration_test.exs b/lib/elixir/test/elixir/calendar/duration_test.exs new file mode 100644 index 00000000000..faa4671426d --- /dev/null +++ b/lib/elixir/test/elixir/calendar/duration_test.exs @@ -0,0 +1,439 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team + +Code.require_file("../test_helper.exs", __DIR__) + +defmodule DurationTest do + use ExUnit.Case, async: true + doctest Duration + + test "new!/1" do + assert Duration.new!(year: 2, month: 1, week: 3) == %Duration{year: 2, month: 1, week: 3} + assert Duration.new!(microsecond: {20000, 2}) == %Duration{microsecond: {20000, 2}} + + duration = %Duration{year: 1} + assert ^duration = Duration.new!(duration) + + assert_raise ArgumentError, + "unsupported value nil for :month. Expected an integer", + fn -> Duration.new!(month: nil) end + + assert_raise ArgumentError, + "unknown unit :years. Expected :year, :month, :week, :day, :hour, :minute, :second, :microsecond", + fn -> Duration.new!(years: 1) end + + assert_raise ArgumentError, + "unsupported value {1, 2, 3} for :microsecond. Expected a tuple {ms, precision} where precision is an integer from 0 to 6", + fn -> Duration.new!(microsecond: {1, 2, 3}) end + + assert_raise ArgumentError, + "unsupported value {100, 7} for :microsecond. Expected a tuple {ms, precision} where precision is an integer from 0 to 6", + fn -> Duration.new!(microsecond: {100, 7}) end + end + + test "add/2" do + d1 = %Duration{ + year: 1, + month: 2, + week: 3, + day: 4, + hour: 5, + minute: 6, + second: 7, + microsecond: {8, 6} + } + + d2 = %Duration{ + year: 8, + month: 7, + week: 6, + day: 5, + hour: 4, + minute: 3, + second: 2, + microsecond: {1, 6} + } + + assert Duration.add(d1, d2) == %Duration{ + year: 9, + month: 9, + week: 9, + day: 9, + hour: 9, + minute: 9, + second: 9, + microsecond: {9, 6} + } + + assert Duration.add(d1, d2) == Duration.add(d2, d1) + + d1 = %Duration{month: 2, week: 3, day: 4} + d2 = %Duration{year: 8, day: 2, second: 2} + + assert Duration.add(d1, d2) == %Duration{ + year: 8, + month: 2, + week: 3, + day: 6, + hour: 0, + minute: 0, + second: 2, + microsecond: {0, 0} + } + + d1 = %Duration{microsecond: {1000, 4}} + d2 = %Duration{microsecond: {5, 6}} + assert Duration.add(d1, d2) == %Duration{microsecond: {1005, 6}} + end + + test "subtract/2" do + d1 = %Duration{ + year: 1, + month: 2, + week: 3, + day: 4, + hour: 5, + minute: 6, + second: 7, + microsecond: {8, 6} + } + + d2 = %Duration{ + year: 8, + month: 7, + week: 6, + day: 5, + hour: 4, + minute: 3, + second: 2, + microsecond: {1, 6} + } + + assert Duration.subtract(d1, d2) == %Duration{ + year: -7, + month: -5, + week: -3, + day: -1, + hour: 1, + minute: 3, + second: 5, + microsecond: {7, 6} + } + + assert Duration.subtract(d2, d1) == %Duration{ + year: 7, + month: 5, + week: 3, + day: 1, + hour: -1, + minute: -3, + second: -5, + microsecond: {-7, 6} + } + + assert Duration.subtract(d1, d2) != Duration.subtract(d2, d1) + + d1 = %Duration{year: 10, month: 2, week: 3, day: 4} + d2 = %Duration{year: 8, day: 2, second: 2} + + assert Duration.subtract(d1, d2) == %Duration{ + year: 2, + month: 2, + week: 3, + day: 2, + hour: 0, + minute: 0, + second: -2, + microsecond: {0, 0} + } + + d1 = %Duration{microsecond: {1000, 4}} + d2 = %Duration{microsecond: {5, 6}} + assert Duration.subtract(d1, d2) == %Duration{microsecond: {995, 6}} + end + + test "multiply/2" do + duration = %Duration{ + year: 1, + month: 2, + week: 3, + day: 4, + hour: 5, + minute: 6, + second: 7, + microsecond: {8, 6} + } + + assert Duration.multiply(duration, 3) == %Duration{ + year: 3, + month: 6, + week: 9, + day: 12, + hour: 15, + minute: 18, + second: 21, + microsecond: {24, 6} + } + + assert Duration.multiply(%Duration{year: 2, day: 4, minute: 5}, 4) == + %Duration{ + year: 8, + month: 0, + week: 0, + day: 16, + hour: 0, + minute: 20, + second: 0, + microsecond: {0, 0} + } + end + + test "negate/1" do + duration = %Duration{ + year: 1, + month: 2, + week: 3, + day: 4, + hour: 5, + minute: 6, + second: 7, + microsecond: {8, 6} + } + + assert Duration.negate(duration) == %Duration{ + year: -1, + month: -2, + week: -3, + day: -4, + hour: -5, + minute: -6, + second: -7, + microsecond: {-8, 6} + } + + assert Duration.negate(%Duration{year: 2, day: 4, minute: 5}) == + %Duration{ + year: -2, + month: 0, + week: 0, + day: -4, + hour: 0, + minute: -5, + second: 0, + microsecond: {0, 0} + } + end + + test "from_iso8601/1" do + assert Duration.from_iso8601("P1Y2M3DT4H5M6S") == + {:ok, %Duration{year: 1, month: 2, day: 3, hour: 4, minute: 5, second: 6}} + + assert Duration.from_iso8601("P3WT5H3M") == {:ok, %Duration{week: 3, hour: 5, minute: 3}} + assert Duration.from_iso8601("PT5H3M") == {:ok, %Duration{hour: 5, minute: 3}} + assert Duration.from_iso8601("P1Y2M3D") == {:ok, %Duration{year: 1, month: 2, day: 3}} + assert Duration.from_iso8601("PT4H5M6S") == {:ok, %Duration{hour: 4, minute: 5, second: 6}} + assert Duration.from_iso8601("P1Y2M") == {:ok, %Duration{year: 1, month: 2}} + assert Duration.from_iso8601("P3D") == {:ok, %Duration{day: 3}} + assert Duration.from_iso8601("PT4H5M") == {:ok, %Duration{hour: 4, minute: 5}} + assert Duration.from_iso8601("PT6S") == {:ok, %Duration{second: 6}} + assert Duration.from_iso8601("P2M4Y") == {:error, :invalid_date_component} + assert Duration.from_iso8601("P4Y2W3Y") == {:error, :invalid_date_component} + assert Duration.from_iso8601("P5HT4MT3S") == {:error, :invalid_date_component} + assert Duration.from_iso8601("P5H3HT4M") == {:error, :invalid_date_component} + assert Duration.from_iso8601("P0.5Y") == {:error, :invalid_date_component} + assert Duration.from_iso8601("PT1D") == {:error, :invalid_time_component} + assert Duration.from_iso8601("PT.6S") == {:error, :invalid_time_component} + assert Duration.from_iso8601("PT0.5H") == {:error, :invalid_time_component} + assert Duration.from_iso8601("invalid") == {:error, :invalid_duration} + end + + test "from_iso8601!/1" do + assert Duration.from_iso8601!("P1Y2M3DT4H5M6S") == %Duration{ + year: 1, + month: 2, + day: 3, + hour: 4, + minute: 5, + second: 6 + } + + assert Duration.from_iso8601!("P3WT5H3M") == %Duration{week: 3, hour: 5, minute: 3} + assert Duration.from_iso8601!("PT5H3M") == %Duration{hour: 5, minute: 3} + assert Duration.from_iso8601!("P1Y2M3D") == %Duration{year: 1, month: 2, day: 3} + assert Duration.from_iso8601!("PT4H5M6S") == %Duration{hour: 4, minute: 5, second: 6} + assert Duration.from_iso8601!("P1Y2M") == %Duration{year: 1, month: 2} + assert Duration.from_iso8601!("P3D") == %Duration{day: 3} + assert Duration.from_iso8601!("PT4H5M") == %Duration{hour: 4, minute: 5} + assert Duration.from_iso8601!("PT6S") == %Duration{second: 6} + assert Duration.from_iso8601!("PT1,6S") == %Duration{second: 1, microsecond: {600_000, 1}} + assert Duration.from_iso8601!("PT-1.6S") == %Duration{second: -1, microsecond: {-600_000, 1}} + assert Duration.from_iso8601!("PT0,6S") == %Duration{second: 0, microsecond: {600_000, 1}} + assert Duration.from_iso8601!("PT-0,6S") == %Duration{second: 0, microsecond: {-600_000, 1}} + assert Duration.from_iso8601!("-PT-0,6S") == %Duration{second: 0, microsecond: {600_000, 1}} + assert Duration.from_iso8601!("-P10DT4H") == %Duration{day: -10, hour: -4} + assert Duration.from_iso8601!("-P10DT-4H") == %Duration{day: -10, hour: 4} + assert Duration.from_iso8601!("P-10D") == %Duration{day: -10} + assert Duration.from_iso8601!("+P10DT-4H") == %Duration{day: 10, hour: -4} + assert Duration.from_iso8601!("P+10D") == %Duration{day: 10} + assert Duration.from_iso8601!("-P+10D") == %Duration{day: -10} + + assert Duration.from_iso8601!("PT-1.234567S") == %Duration{ + second: -1, + microsecond: {-234_567, 6} + } + + assert Duration.from_iso8601!("PT1.12345678S") == %Duration{ + second: 1, + microsecond: {123_456, 6} + } + + assert Duration.from_iso8601!("P3Y4W-3DT-6S") == %Duration{ + year: 3, + week: 4, + day: -3, + second: -6 + } + + assert Duration.from_iso8601!("PT-4.23S") == %Duration{second: -4, microsecond: {-230_000, 2}} + + assert_raise ArgumentError, + ~s/failed to parse duration "P5H3HT4M". reason: :invalid_date_component/, + fn -> + Duration.from_iso8601!("P5H3HT4M") + end + + assert_raise ArgumentError, + ~s/failed to parse duration "P4Y2W3Y". reason: :invalid_date_component/, + fn -> + Duration.from_iso8601!("P4Y2W3Y") + end + + assert_raise ArgumentError, + ~s/failed to parse duration "invalid". reason: :invalid_duration/, + fn -> + Duration.from_iso8601!("invalid") + end + + assert_raise ArgumentError, + ~s/failed to parse duration "P4.5YT6S". reason: :invalid_date_component/, + fn -> + Duration.from_iso8601!("P4.5YT6S") + end + end + + test "to_iso8601/1" do + assert %Duration{year: 1, month: 2, day: 3, hour: 4, minute: 5, second: 6} + |> Duration.to_iso8601() == "P1Y2M3DT4H5M6S" + + assert %Duration{week: 3, hour: 5, minute: 3} |> Duration.to_iso8601() == "P3WT5H3M" + assert %Duration{hour: 5, minute: 3} |> Duration.to_iso8601() == "PT5H3M" + assert %Duration{year: 1, month: 2, day: 3} |> Duration.to_iso8601() == "P1Y2M3D" + assert %Duration{hour: 4, minute: 5, second: 6} |> Duration.to_iso8601() == "PT4H5M6S" + assert %Duration{year: 1, month: 2} |> Duration.to_iso8601() == "P1Y2M" + assert %Duration{day: 3} |> Duration.to_iso8601() == "P3D" + assert %Duration{hour: 4, minute: 5} |> Duration.to_iso8601() == "PT4H5M" + assert %Duration{second: 6} |> Duration.to_iso8601() == "PT6S" + assert %Duration{second: 1, microsecond: {600_000, 1}} |> Duration.to_iso8601() == "PT1.6S" + assert %Duration{second: -1, microsecond: {-600_000, 1}} |> Duration.to_iso8601() == "PT-1.6S" + + assert %Duration{second: -1, microsecond: {-234_567, 6}} |> Duration.to_iso8601() == + "PT-1.234567S" + + assert %Duration{second: 1, microsecond: {123_456, 6}} |> Duration.to_iso8601() == + "PT1.123456S" + + assert %Duration{year: 3, week: 4, day: -3, second: -6} |> Duration.to_iso8601() == + "P3Y4W-3DT-6S" + + assert %Duration{second: -4, microsecond: {-230_000, 2}} |> Duration.to_iso8601() == + "PT-4.23S" + + assert %Duration{second: -4, microsecond: {230_000, 2}} |> Duration.to_iso8601() == + "PT-3.77S" + + assert %Duration{second: 2, microsecond: {-1_200_000, 4}} |> Duration.to_iso8601() == + "PT0.8000S" + + assert %Duration{second: 1, microsecond: {-1_200_000, 3}} |> Duration.to_iso8601() == + "PT-0.200S" + + assert %Duration{microsecond: {-800_000, 2}} |> Duration.to_iso8601() == "PT-0.80S" + assert %Duration{microsecond: {-800_000, 0}} |> Duration.to_iso8601() == "PT0S" + assert %Duration{microsecond: {-1_200_000, 2}} |> Duration.to_iso8601() == "PT-1.20S" + end + + test "to_string/1" do + assert Duration.to_string(%Duration{year: 1, month: 2, day: 3, hour: 4, minute: 5, second: 6}) == + "1a 2mo 3d 4h 5min 6s" + + assert Duration.to_string(%Duration{week: 3, hour: 5, minute: 3}) == + "3wk 5h 3min" + + assert Duration.to_string(%Duration{hour: 5, minute: 3}) == + "5h 3min" + + assert Duration.to_string(%Duration{year: 1, month: 2, day: 3}) == + "1a 2mo 3d" + + assert Duration.to_string(%Duration{hour: 4, minute: 5, second: 6}) == + "4h 5min 6s" + + assert Duration.to_string(%Duration{year: 1, month: 2}) == + "1a 2mo" + + assert Duration.to_string(%Duration{day: 3}) == + "3d" + + assert Duration.to_string(%Duration{hour: 4, minute: 5}) == + "4h 5min" + + assert Duration.to_string(%Duration{second: 6}) == + "6s" + + assert Duration.to_string(%Duration{second: 1, microsecond: {600_000, 1}}) == + "1.6s" + + assert Duration.to_string(%Duration{second: -1, microsecond: {-600_000, 1}}) == + "-1.6s" + + assert Duration.to_string(%Duration{second: -1, microsecond: {-234_567, 6}}) == + "-1.234567s" + + assert Duration.to_string(%Duration{second: 1, microsecond: {123_456, 6}}) == + "1.123456s" + + assert Duration.to_string(%Duration{year: 3, week: 4, day: -3, second: -6}) == + "3a 4wk -3d -6s" + + assert Duration.to_string(%Duration{second: -4, microsecond: {-230_000, 2}}) == + "-4.23s" + + assert Duration.to_string(%Duration{second: -4, microsecond: {230_000, 2}}) == + "-3.77s" + + assert Duration.to_string(%Duration{second: 2, microsecond: {-1_200_000, 4}}) == + "0.8000s" + + assert Duration.to_string(%Duration{second: 1, microsecond: {-1_200_000, 3}}) == + "-0.200s" + + assert Duration.to_string(%Duration{microsecond: {-800_000, 2}}) == + "-0.80s" + + assert Duration.to_string(%Duration{microsecond: {-800_000, 0}}) == + "0s" + + assert Duration.to_string(%Duration{microsecond: {-1_200_000, 2}}) == + "-1.20s" + + assert Duration.to_string(%Duration{year: 1, month: 2, day: 3, hour: 4, minute: 5, second: 6}, + units: [year: "year", month: "month", day: "day"], + separator: "-" + ) == + "1year-2month-3day-4h-5min-6s" + end + + test "inspect/1" do + assert inspect(%Duration{hour: 5, minute: 3}) == "%Duration{hour: 5, minute: 3}" + end +end diff --git a/lib/elixir/test/elixir/calendar/fakes.exs b/lib/elixir/test/elixir/calendar/fakes.exs new file mode 100644 index 00000000000..21b3043a7c3 --- /dev/null +++ b/lib/elixir/test/elixir/calendar/fakes.exs @@ -0,0 +1,235 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +defmodule FakeCalendar do + def time_to_string(hour, minute, second, _), do: "#{hour}::#{minute}::#{second}" + def date_to_string(year, month, day), do: "#{day}/#{month}/#{year}" + + def naive_datetime_to_string(year, month, day, hour, minute, second, microsecond) do + date_to_string(year, month, day) <> "F" <> time_to_string(hour, minute, second, microsecond) + end + + def datetime_to_string( + year, + month, + day, + hour, + minute, + second, + microsecond, + time_zone, + abbr, + utc_offset, + std_offset + ) do + naive_datetime_to_string(year, month, day, hour, minute, second, microsecond) <> + " #{time_zone} #{abbr} #{utc_offset} #{std_offset}" + end + + def day_rollover_relative_to_midnight_utc, do: {123_456, 123_457} +end + +defmodule FakeTimeZoneDatabase do + @behaviour Calendar.TimeZoneDatabase + + @time_zone_period_cph_summer_2018 %{ + std_offset: 3600, + utc_offset: 3600, + zone_abbr: "CEST", + from_wall: ~N[2018-03-25 03:00:00], + until_wall: ~N[2018-10-28 03:00:00] + } + + @time_zone_period_cph_winter_2018_2019 %{ + std_offset: 0, + utc_offset: 3600, + zone_abbr: "CET", + from_wall: ~N[2018-10-28 02:00:00], + until_wall: ~N[2019-03-31 02:00:00] + } + + @time_zone_period_cph_summer_2019 %{ + std_offset: 3600, + utc_offset: 3600, + zone_abbr: "CEST", + from_wall: ~N[2019-03-31 03:00:00], + until_wall: ~N[2019-10-27 03:00:00] + } + + @time_zone_period_usla_summer_2018 %{ + std_offset: 3600, + utc_offset: -28800, + zone_abbr: "PDT", + from_wall: ~N[2018-03-11 02:00:00], + until_wall: ~N[2018-11-04 02:00:00] + } + + @time_zone_period_usla_winter_2018_2019 %{ + std_offset: 0, + utc_offset: -28800, + zone_abbr: "PST", + from_wall: ~N[2018-11-04 02:00:00], + until_wall: ~N[2019-03-10 03:00:00] + } + + @spec time_zone_period_from_utc_iso_days(Calendar.iso_days(), Calendar.time_zone()) :: + {:ok, TimeZoneDatabase.time_zone_period()} | {:error, :time_zone_not_found} + @impl true + def time_zone_period_from_utc_iso_days(iso_days, time_zone) do + {:ok, ndt} = naive_datetime_from_iso_days(iso_days) + time_zone_periods_from_utc(time_zone, NaiveDateTime.to_erl(ndt)) + end + + @spec time_zone_periods_from_wall_datetime(Calendar.naive_datetime(), Calendar.time_zone()) :: + {:ok, TimeZoneDatabase.time_zone_period()} + | {:ambiguous, TimeZoneDatabase.time_zone_period(), TimeZoneDatabase.time_zone_period()} + | {:gap, + {TimeZoneDatabase.time_zone_period(), TimeZoneDatabase.time_zone_period_limit()}, + {TimeZoneDatabase.time_zone_period(), TimeZoneDatabase.time_zone_period_limit()}} + | {:error, :time_zone_not_found} + @impl true + def time_zone_periods_from_wall_datetime(naive_datetime, time_zone) do + time_zone_periods_from_wall(time_zone, NaiveDateTime.to_erl(naive_datetime)) + end + + defp time_zone_periods_from_utc("Europe/Copenhagen", erl_datetime) + when erl_datetime >= {{2018, 3, 25}, {1, 0, 0}} and + erl_datetime < {{2018, 10, 28}, {3, 0, 0}} do + {:ok, @time_zone_period_cph_summer_2018} + end + + defp time_zone_periods_from_utc("Europe/Copenhagen", erl_datetime) + when erl_datetime >= {{2018, 10, 28}, {2, 0, 0}} and + erl_datetime < {{2019, 3, 31}, {1, 0, 0}} do + {:ok, @time_zone_period_cph_winter_2018_2019} + end + + defp time_zone_periods_from_utc("Europe/Copenhagen", erl_datetime) + when erl_datetime >= {{2019, 3, 31}, {1, 0, 0}} and + erl_datetime < {{2019, 10, 28}, {3, 0, 0}} do + {:ok, @time_zone_period_cph_summer_2019} + end + + defp time_zone_periods_from_utc("Europe/Copenhagen", erl_datetime) + when erl_datetime >= {{2015, 3, 29}, {1, 0, 0}} and + erl_datetime < {{2015, 10, 25}, {1, 0, 0}} do + {:ok, + %{ + std_offset: 3600, + utc_offset: 3600, + zone_abbr: "CEST" + }} + end + + defp time_zone_periods_from_utc("America/Los_Angeles", erl_datetime) + when erl_datetime >= {{2018, 3, 11}, {2, 0, 0}} and + erl_datetime < {{2018, 11, 4}, {2, 0, 0}} do + {:ok, @time_zone_period_usla_summer_2018} + end + + defp time_zone_periods_from_utc("America/Los_Angeles", erl_datetime) + when erl_datetime >= {{2018, 11, 4}, {2, 0, 0}} and + erl_datetime < {{2019, 3, 10}, {3, 0, 0}} do + {:ok, @time_zone_period_usla_winter_2018_2019} + end + + defp time_zone_periods_from_utc("Etc/UTC", _erl_datetime) do + {:ok, + %{ + std_offset: 0, + utc_offset: 0, + zone_abbr: "UTC" + }} + end + + defp time_zone_periods_from_utc(time_zone, _) when time_zone != "Europe/Copenhagen" do + {:error, :time_zone_not_found} + end + + defp time_zone_periods_from_wall("Europe/Copenhagen", erl_datetime) + when erl_datetime >= {{2019, 3, 31}, {2, 0, 0}} and + erl_datetime < {{2019, 3, 31}, {3, 0, 0}} do + {:gap, + {@time_zone_period_cph_winter_2018_2019, @time_zone_period_cph_winter_2018_2019.until_wall}, + {@time_zone_period_cph_summer_2019, @time_zone_period_cph_summer_2019.from_wall}} + end + + defp time_zone_periods_from_wall("Europe/Copenhagen", erl_datetime) + when erl_datetime < {{2018, 10, 28}, {3, 0, 0}} and + erl_datetime >= {{2018, 10, 28}, {2, 0, 0}} do + {:ambiguous, @time_zone_period_cph_summer_2018, @time_zone_period_cph_winter_2018_2019} + end + + defp time_zone_periods_from_wall("Europe/Copenhagen", erl_datetime) + when erl_datetime >= {{2018, 3, 25}, {3, 0, 0}} and + erl_datetime < {{2018, 10, 28}, {3, 0, 0}} do + {:ok, @time_zone_period_cph_summer_2018} + end + + defp time_zone_periods_from_wall("Europe/Copenhagen", erl_datetime) + when erl_datetime >= {{2018, 10, 28}, {2, 0, 0}} and + erl_datetime < {{2019, 3, 31}, {2, 0, 0}} do + {:ok, @time_zone_period_cph_winter_2018_2019} + end + + defp time_zone_periods_from_wall("Europe/Copenhagen", erl_datetime) + when erl_datetime >= {{2019, 3, 31}, {3, 0, 0}} and + erl_datetime < {{2019, 10, 27}, {3, 0, 0}} do + {:ok, @time_zone_period_cph_summer_2019} + end + + defp time_zone_periods_from_wall("America/Los_Angeles", erl_datetime) + when erl_datetime >= {{2018, 3, 11}, {2, 0, 0}} and + erl_datetime < {{2018, 11, 4}, {2, 0, 0}} do + {:ok, @time_zone_period_usla_summer_2018} + end + + defp time_zone_periods_from_wall("America/Los_Angeles", erl_datetime) + when erl_datetime >= {{2018, 11, 4}, {3, 0, 0}} and + erl_datetime < {{2019, 3, 10}, {3, 0, 0}} do + {:ok, @time_zone_period_usla_winter_2018_2019} + end + + defp time_zone_periods_from_wall("Europe/Copenhagen", erl_datetime) + when erl_datetime >= {{2015, 3, 29}, {3, 0, 0}} and + erl_datetime < {{2015, 10, 25}, {3, 0, 0}} do + {:ok, + %{ + std_offset: 3600, + utc_offset: 3600, + zone_abbr: "CEST" + }} + end + + defp time_zone_periods_from_wall("Europe/Copenhagen", erl_datetime) + when erl_datetime >= {{2090, 3, 26}, {3, 0, 0}} and + erl_datetime < {{2090, 10, 29}, {3, 0, 0}} do + {:ok, + %{ + std_offset: 3600, + utc_offset: 3600, + zone_abbr: "CEST" + }} + end + + defp time_zone_periods_from_wall("Etc/UTC", _erl_datetime) do + {:ok, + %{ + std_offset: 0, + utc_offset: 0, + zone_abbr: "UTC" + }} + end + + defp time_zone_periods_from_wall(time_zone, _) when time_zone != "Europe/Copenhagen" do + {:error, :time_zone_not_found} + end + + defp naive_datetime_from_iso_days(iso_days) do + {year, month, day, hour, minute, second, microsecond} = + Calendar.ISO.naive_datetime_from_iso_days(iso_days) + + NaiveDateTime.new(year, month, day, hour, minute, second, microsecond) + end +end diff --git a/lib/elixir/test/elixir/calendar/holocene.exs b/lib/elixir/test/elixir/calendar/holocene.exs new file mode 100644 index 00000000000..8e9e2a9b3ee --- /dev/null +++ b/lib/elixir/test/elixir/calendar/holocene.exs @@ -0,0 +1,177 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +defmodule Calendar.Holocene do + # This calendar is used to test conversions between calendars. + # It implements the Holocene calendar, which is based on the + # Proleptic Gregorian calendar with every year + 10000. + + @behaviour Calendar + + def date(year, month, day) do + %Date{year: year, month: month, day: day, calendar: __MODULE__} + end + + def naive_datetime(year, month, day, hour, minute, second, microsecond \\ {0, 0}) do + %NaiveDateTime{ + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond, + calendar: __MODULE__ + } + end + + @impl true + def date_to_string(year, month, day) do + "#{year}-#{zero_pad(month, 2)}-#{zero_pad(day, 2)}" + end + + @impl true + def naive_datetime_to_string(year, month, day, hour, minute, second, microsecond) do + "#{year}-#{zero_pad(month, 2)}-#{zero_pad(day, 2)}" <> + Calendar.ISO.time_to_string(hour, minute, second, microsecond) + end + + @impl true + def datetime_to_string( + year, + month, + day, + hour, + minute, + second, + microsecond, + _time_zone, + zone_abbr, + _utc_offset, + _std_offset + ) do + "#{year}-#{zero_pad(month, 2)}-#{zero_pad(day, 2)}" <> + Calendar.ISO.time_to_string(hour, minute, second, microsecond) <> + " #{zone_abbr}" + end + + @impl true + defdelegate time_to_string(hour, minute, second, microsecond), to: Calendar.ISO + + @impl true + def day_rollover_relative_to_midnight_utc(), do: {0, 1} + + @impl true + def naive_datetime_from_iso_days(entry) do + {year, month, day, hour, minute, second, microsecond} = + Calendar.ISO.naive_datetime_from_iso_days(entry) + + {year + 10000, month, day, hour, minute, second, microsecond} + end + + @impl true + def naive_datetime_to_iso_days(year, month, day, hour, minute, second, microsecond) do + Calendar.ISO.naive_datetime_to_iso_days( + year - 10000, + month, + day, + hour, + minute, + second, + microsecond + ) + end + + defp zero_pad(val, count) when val >= 0 do + String.pad_leading("#{val}", count, ["0"]) + end + + defp zero_pad(val, count) do + "-" <> zero_pad(-val, count) + end + + @impl true + def parse_date(string) do + {year, month, day} = + string + |> String.split("-") + |> Enum.map(&String.to_integer/1) + |> List.to_tuple() + + if valid_date?(year, month, day) do + {:ok, {year, month, day}} + else + {:error, :invalid_date} + end + end + + @impl true + def valid_date?(year, month, day) do + :calendar.valid_date(year, month, day) + end + + @impl true + defdelegate parse_time(string), to: Calendar.ISO + + @impl true + defdelegate parse_naive_datetime(string), to: Calendar.ISO + + @impl true + defdelegate parse_utc_datetime(string), to: Calendar.ISO + + @impl true + defdelegate time_from_day_fraction(day_fraction), to: Calendar.ISO + + @impl true + defdelegate time_to_day_fraction(hour, minute, second, microsecond), to: Calendar.ISO + + @impl true + defdelegate leap_year?(year), to: Calendar.ISO + + @impl true + defdelegate days_in_month(year, month), to: Calendar.ISO + + @impl true + defdelegate months_in_year(year), to: Calendar.ISO + + @impl true + defdelegate day_of_week(year, month, day, starting_on), to: Calendar.ISO + + @impl true + defdelegate day_of_year(year, month, day), to: Calendar.ISO + + @impl true + defdelegate quarter_of_year(year, month, day), to: Calendar.ISO + + @impl true + defdelegate year_of_era(year, month, day), to: Calendar.ISO + + @impl true + defdelegate day_of_era(year, month, day), to: Calendar.ISO + + @impl true + defdelegate valid_time?(hour, minute, second, microsecond), to: Calendar.ISO + + @impl true + defdelegate iso_days_to_beginning_of_day(iso_days), to: Calendar.ISO + + @impl true + defdelegate iso_days_to_end_of_day(iso_days), to: Calendar.ISO + + # The Holocene calendar extends most year and day count guards implemented in the ISO calendars. + @impl true + def shift_date(_year, _month, _day, _duration) do + raise "shift_date/4 not implemented" + end + + @impl true + def shift_naive_datetime(_year, _month, _day, _hour, _minute, _second, _microsecond, _duration) do + raise "shift_naive_datetime/8 not implemented" + end + + @impl true + def shift_time(_hour, _minute, _second, _microsecond, _duration) do + raise "shift_time/5 not implemented" + end +end diff --git a/lib/elixir/test/elixir/calendar/iso_test.exs b/lib/elixir/test/elixir/calendar/iso_test.exs new file mode 100644 index 00000000000..8a77e9f5730 --- /dev/null +++ b/lib/elixir/test/elixir/calendar/iso_test.exs @@ -0,0 +1,651 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Calendar.ISOTest do + use ExUnit.Case, async: true + doctest Calendar.ISO + + describe "date_from_iso_days" do + test "with positive dates" do + assert {0, 1, 1} == iso_day_roundtrip(0, 1, 1) + assert {0, 12, 31} == iso_day_roundtrip(0, 12, 31) + assert {1, 12, 31} == iso_day_roundtrip(1, 12, 31) + assert {4, 1, 1} == iso_day_roundtrip(4, 1, 1) + assert {4, 12, 31} == iso_day_roundtrip(4, 12, 31) + assert {9999, 12, 31} == iso_day_roundtrip(9999, 12, 31) + assert {9999, 1, 1} == iso_day_roundtrip(9999, 1, 1) + assert {9996, 12, 31} == iso_day_roundtrip(9996, 12, 31) + assert {9996, 1, 1} == iso_day_roundtrip(9996, 1, 1) + end + + test "with negative dates" do + assert {-1, 1, 1} == iso_day_roundtrip(-1, 1, 1) + assert {-1, 12, 31} == iso_day_roundtrip(-1, 12, 31) + assert {-1, 12, 31} == iso_day_roundtrip(-1, 12, 31) + assert {-2, 1, 1} == iso_day_roundtrip(-2, 1, 1) + assert {-5, 12, 31} == iso_day_roundtrip(-5, 12, 31) + + assert {-4, 1, 1} == iso_day_roundtrip(-4, 1, 1) + assert {-4, 12, 31} == iso_day_roundtrip(-4, 12, 31) + + assert {-9999, 12, 31} == iso_day_roundtrip(-9999, 12, 31) + assert {-9996, 12, 31} == iso_day_roundtrip(-9996, 12, 31) + + assert {-9996, 12, 31} == iso_day_roundtrip(-9996, 12, 31) + assert {-9996, 1, 1} == iso_day_roundtrip(-9996, 1, 1) + end + end + + describe "date_to_string/4" do + test "regular use" do + assert Calendar.ISO.date_to_string(1000, 1, 1, :basic) == "10000101" + assert Calendar.ISO.date_to_string(1000, 1, 1, :extended) == "1000-01-01" + + assert Calendar.ISO.date_to_string(-123, 1, 1, :basic) == "-01230101" + assert Calendar.ISO.date_to_string(-123, 1, 1, :extended) == "-0123-01-01" + end + + test "handles years > 9999" do + assert Calendar.ISO.date_to_string(10000, 1, 1, :basic) == "100000101" + assert Calendar.ISO.date_to_string(10000, 1, 1, :extended) == "10000-01-01" + end + end + + describe "naive_datetime_to_iso_days/7" do + test "raises with invalid dates" do + assert_raise ArgumentError, "invalid date: 2018-02-30", fn -> + Calendar.ISO.naive_datetime_to_iso_days(2018, 2, 30, 0, 0, 0, {0, 0}) + end + + assert_raise ArgumentError, "invalid date: 2017-11--03", fn -> + Calendar.ISO.naive_datetime_to_iso_days(2017, 11, -3, 0, 0, 0, {0, 0}) + end + end + end + + describe "day_of_week/4" do + test "raises with invalid dates" do + assert_raise ArgumentError, "invalid date: 2018-02-30", fn -> + Calendar.ISO.day_of_week(2018, 2, 30, :default) + end + + assert_raise ArgumentError, "invalid date: 2017-11-00", fn -> + Calendar.ISO.day_of_week(2017, 11, 0, :default) + end + end + end + + describe "day_of_era/3" do + test "raises with invalid dates" do + assert_raise ArgumentError, "invalid date: 2018-02-30", fn -> + Calendar.ISO.day_of_era(2018, 2, 30) + end + + assert_raise ArgumentError, "invalid date: 2017-11-00", fn -> + Calendar.ISO.day_of_era(2017, 11, 0) + end + end + end + + describe "day_of_year/3" do + test "raises with invalid dates" do + assert_raise ArgumentError, "invalid date: 2018-02-30", fn -> + Calendar.ISO.day_of_year(2018, 2, 30) + end + + assert_raise ArgumentError, "invalid date: 2017-11-00", fn -> + Calendar.ISO.day_of_year(2017, 11, 0) + end + end + end + + test "year_of_era/3" do + # Compatibility tests for year_of_era/1 + assert Calendar.ISO.year_of_era(-9999) == {10000, 0} + assert Calendar.ISO.year_of_era(-1) == {2, 0} + assert Calendar.ISO.year_of_era(0) == {1, 0} + assert Calendar.ISO.year_of_era(1) == {1, 1} + assert Calendar.ISO.year_of_era(1984) == {1984, 1} + + assert Calendar.ISO.year_of_era(-9999, 1, 1) == {10000, 0} + assert Calendar.ISO.year_of_era(-1, 1, 1) == {2, 0} + assert Calendar.ISO.year_of_era(0, 12, 1) == {1, 0} + assert Calendar.ISO.year_of_era(1, 12, 1) == {1, 1} + assert Calendar.ISO.year_of_era(1984, 12, 1) == {1984, 1} + + random_positive_year = Enum.random(1..9999) + assert Calendar.ISO.year_of_era(random_positive_year, 1, 1) == {random_positive_year, 1} + end + + defp iso_day_roundtrip(year, month, day) do + iso_days = Calendar.ISO.date_to_iso_days(year, month, day) + Calendar.ISO.date_from_iso_days(iso_days) + end + + describe "parse_date/1" do + test "supports both only extended format by default" do + assert Calendar.ISO.parse_date("20150123") == {:error, :invalid_format} + assert Calendar.ISO.parse_date("2015-01-23") == {:ok, {2015, 1, 23}} + end + end + + describe "parse_date/2" do + test "allows enforcing basic formats" do + assert Calendar.ISO.parse_date("20150123", :basic) == {:ok, {2015, 1, 23}} + assert Calendar.ISO.parse_date("2015-01-23", :basic) == {:error, :invalid_format} + end + + test "allows enforcing extended formats" do + assert Calendar.ISO.parse_date("20150123", :extended) == {:error, :invalid_format} + assert Calendar.ISO.parse_date("2015-01-23", :extended) == {:ok, {2015, 1, 23}} + end + + test "errors on other format names" do + assert_raise FunctionClauseError, fn -> + Calendar.ISO.parse_date("20150123", :other) + end + + assert_raise FunctionClauseError, fn -> + Calendar.ISO.parse_date("2015-01-23", :other) + end + end + end + + describe "parse_time/1" do + test "supports only extended format by default" do + assert Calendar.ISO.parse_time("235007") == {:error, :invalid_format} + assert Calendar.ISO.parse_time("23:50:07") == {:ok, {23, 50, 7, {0, 0}}} + end + + test "ignores offset data but requires valid ones" do + assert Calendar.ISO.parse_time("23:50:07Z") == {:ok, {23, 50, 7, {0, 0}}} + assert Calendar.ISO.parse_time("23:50:07+01:00") == {:ok, {23, 50, 7, {0, 0}}} + + assert Calendar.ISO.parse_time("2015-01-23 23:50-00:00") == {:error, :invalid_format} + assert Calendar.ISO.parse_time("2015-01-23 23:50-00:60") == {:error, :invalid_format} + assert Calendar.ISO.parse_time("2015-01-23 23:50-24:00") == {:error, :invalid_format} + end + + test "supports either comma or period millisecond delimiters" do + assert Calendar.ISO.parse_time("23:50:07,012345") == {:ok, {23, 50, 7, {12345, 6}}} + assert Calendar.ISO.parse_time("23:50:07.012345") == {:ok, {23, 50, 7, {12345, 6}}} + end + + test "only supports reduced precision for milliseconds" do + assert Calendar.ISO.parse_time("23:50:07.012345") == {:ok, {23, 50, 7, {12345, 6}}} + assert Calendar.ISO.parse_time("23:50:07") == {:ok, {23, 50, 7, {0, 0}}} + assert Calendar.ISO.parse_time("23:50") == {:error, :invalid_format} + assert Calendar.ISO.parse_time("23") == {:error, :invalid_format} + end + + test "supports various millisecond precisions" do + assert Calendar.ISO.parse_time("23:50:07.012345") == {:ok, {23, 50, 7, {12345, 6}}} + assert Calendar.ISO.parse_time("23:50:07.0123") == {:ok, {23, 50, 7, {12300, 4}}} + assert Calendar.ISO.parse_time("23:50:07.01") == {:ok, {23, 50, 7, {10000, 2}}} + assert Calendar.ISO.parse_time("23:50:07.0") == {:ok, {23, 50, 7, {0, 1}}} + assert Calendar.ISO.parse_time("23:50:07") == {:ok, {23, 50, 7, {0, 0}}} + end + + test "truncates extra millisecond precision" do + assert Calendar.ISO.parse_time("23:50:07.012345") == {:ok, {23, 50, 7, {12345, 6}}} + assert Calendar.ISO.parse_time("23:50:07.0123456") == {:ok, {23, 50, 7, {12345, 6}}} + end + + test "rejects strings with formatting errors" do + assert Calendar.ISO.parse_time("23:50:07A") == {:error, :invalid_format} + assert Calendar.ISO.parse_time("23:50:07.") == {:error, :invalid_format} + end + + test "refuses to parse the wrong thing" do + assert Calendar.ISO.parse_time("2015:01:23 23-50-07") == {:error, :invalid_format} + assert Calendar.ISO.parse_time("2015-01-23 23:50:07") == {:error, :invalid_format} + assert Calendar.ISO.parse_time("2015:01:23T23-50-07") == {:error, :invalid_format} + assert Calendar.ISO.parse_time("2015-01-23T23:50:07") == {:error, :invalid_format} + assert Calendar.ISO.parse_time("2015:01:23") == {:error, :invalid_format} + assert Calendar.ISO.parse_time("2015-01-23") == {:error, :invalid_format} + end + + test "recognizes invalid times" do + assert Calendar.ISO.parse_time("23:59:61") == {:error, :invalid_time} + assert Calendar.ISO.parse_time("23:61:59") == {:error, :invalid_time} + assert Calendar.ISO.parse_time("25:59:59") == {:error, :invalid_time} + end + end + + describe "parse_time/2" do + test "allows enforcing basic formats" do + assert Calendar.ISO.parse_time("235007", :basic) == {:ok, {23, 50, 7, {0, 0}}} + assert Calendar.ISO.parse_time("23:50:07", :basic) == {:error, :invalid_format} + end + + test "allows enforcing extended formats" do + assert Calendar.ISO.parse_time("235007", :extended) == {:error, :invalid_format} + assert Calendar.ISO.parse_time("23:50:07", :extended) == {:ok, {23, 50, 7, {0, 0}}} + end + + test "errors on other format names" do + assert_raise FunctionClauseError, fn -> + Calendar.ISO.parse_time("235007", :other) + end + + assert_raise FunctionClauseError, fn -> + Calendar.ISO.parse_time("23:50:07", :other) + end + end + end + + describe "parse_naive_datetime/1" do + test "rejects strings with formatting errors" do + assert Calendar.ISO.parse_naive_datetime("2015:01:23 23-50-07") == {:error, :invalid_format} + assert Calendar.ISO.parse_naive_datetime("2015-01-23P23:50:07") == {:error, :invalid_format} + + assert Calendar.ISO.parse_naive_datetime("2015-01-23 23:50:07A") == + {:error, :invalid_format} + end + + test "recognizes invalid dates and times" do + assert Calendar.ISO.parse_naive_datetime("2015-01-23 23:50:61") == {:error, :invalid_time} + assert Calendar.ISO.parse_naive_datetime("2015-01-32 23:50:07") == {:error, :invalid_date} + end + + test "ignores offset data but requires valid ones" do + assert Calendar.ISO.parse_naive_datetime("2015-01-23T23:50:07.123+02:30") == + {:ok, {2015, 1, 23, 23, 50, 7, {123_000, 3}}} + + assert Calendar.ISO.parse_naive_datetime("2015-01-23T23:50:07.123+00:00") == + {:ok, {2015, 1, 23, 23, 50, 7, {123_000, 3}}} + + assert Calendar.ISO.parse_naive_datetime("2015-01-23T23:50:07.123-02:30") == + {:ok, {2015, 1, 23, 23, 50, 7, {123_000, 3}}} + + assert Calendar.ISO.parse_naive_datetime("2015-01-23T23:50:07.123-00:00") == + {:error, :invalid_format} + + assert Calendar.ISO.parse_naive_datetime("2015-01-23T23:50:07.123-00:60") == + {:error, :invalid_format} + + assert Calendar.ISO.parse_naive_datetime("2015-01-23T23:50:07.123-24:00") == + {:error, :invalid_format} + end + + test "supports both spaces and 'T' as datetime separators" do + assert Calendar.ISO.parse_naive_datetime("2015-01-23 23:50:07") == + {:ok, {2015, 1, 23, 23, 50, 7, {0, 0}}} + + assert Calendar.ISO.parse_naive_datetime("2015-01-23T23:50:07") == + {:ok, {2015, 1, 23, 23, 50, 7, {0, 0}}} + end + + test "supports only extended format by default" do + assert Calendar.ISO.parse_naive_datetime("20150123 235007.123") == + {:error, :invalid_format} + + assert Calendar.ISO.parse_naive_datetime("2015-01-23 23:50:07.123") == + {:ok, {2015, 1, 23, 23, 50, 7, {123_000, 3}}} + end + + test "errors on mixed basic and extended formats" do + assert Calendar.ISO.parse_naive_datetime("20150123 23:50:07.123") == + {:error, :invalid_format} + + assert Calendar.ISO.parse_naive_datetime("2015-01-23 235007.123") == + {:error, :invalid_format} + end + end + + describe "parse_naive_datetime/2" do + test "allows enforcing basic formats" do + assert Calendar.ISO.parse_naive_datetime("20150123 235007.123", :basic) == + {:ok, {2015, 1, 23, 23, 50, 7, {123_000, 3}}} + + assert Calendar.ISO.parse_naive_datetime("2015-01-23 23:50:07.123", :basic) == + {:error, :invalid_format} + end + + test "allows enforcing extended formats" do + assert Calendar.ISO.parse_naive_datetime("20150123 235007.123", :extended) == + {:error, :invalid_format} + + assert Calendar.ISO.parse_naive_datetime("2015-01-23 23:50:07.123", :extended) == + {:ok, {2015, 1, 23, 23, 50, 7, {123_000, 3}}} + end + + test "errors on other format names" do + assert_raise FunctionClauseError, fn -> + Calendar.ISO.parse_naive_datetime("20150123 235007.123", :other) + end + + assert_raise FunctionClauseError, fn -> + Calendar.ISO.parse_naive_datetime("2015-01-23 23:50:07.123", :other) + end + end + end + + describe "parse_utc_datetime/1" do + test "rejects strings with formatting errors" do + assert Calendar.ISO.parse_utc_datetime("2015:01:23 23-50-07Z") == {:error, :invalid_format} + assert Calendar.ISO.parse_utc_datetime("2015-01-23P23:50:07Z") == {:error, :invalid_format} + + assert Calendar.ISO.parse_utc_datetime("2015-01-23 23:50:07A") == {:error, :invalid_format} + end + + test "recognizes invalid dates, times, and offsets" do + assert Calendar.ISO.parse_utc_datetime("2015-01-23 23:50:07") == {:error, :missing_offset} + assert Calendar.ISO.parse_utc_datetime("2015-01-23 23:50:61Z") == {:error, :invalid_time} + assert Calendar.ISO.parse_utc_datetime("2015-01-32 23:50:07Z") == {:error, :invalid_date} + end + + test "interprets offset data" do + assert Calendar.ISO.parse_utc_datetime("2015-01-23T23:50:07.123+02:30") == + {:ok, {2015, 1, 23, 21, 20, 7, {123_000, 3}}, 9000} + + assert Calendar.ISO.parse_utc_datetime("2015-01-23T23:50:07.123+00:00") == + {:ok, {2015, 1, 23, 23, 50, 7, {123_000, 3}}, 0} + + assert Calendar.ISO.parse_utc_datetime("2015-01-23T23:50:07.123-02:30") == + {:ok, {2015, 1, 24, 2, 20, 7, {123_000, 3}}, -9000} + + assert Calendar.ISO.parse_utc_datetime("2015-01-23T23:50:07.123-00:00") == + {:error, :invalid_format} + + assert Calendar.ISO.parse_utc_datetime("2015-01-23T23:50:07.123-00:60") == + {:error, :invalid_format} + + assert Calendar.ISO.parse_utc_datetime("2015-01-23T23:50:07.123-24:00") == + {:error, :invalid_format} + end + + test "supports both spaces and 'T' as datetime separators" do + assert Calendar.ISO.parse_utc_datetime("2015-01-23 23:50:07Z") == + {:ok, {2015, 1, 23, 23, 50, 7, {0, 0}}, 0} + + assert Calendar.ISO.parse_utc_datetime("2015-01-23T23:50:07Z") == + {:ok, {2015, 1, 23, 23, 50, 7, {0, 0}}, 0} + end + + test "supports only extended format by default" do + assert Calendar.ISO.parse_utc_datetime("20150123 235007.123Z") == + {:error, :invalid_format} + + assert Calendar.ISO.parse_utc_datetime("2015-01-23 23:50:07.123Z") == + {:ok, {2015, 1, 23, 23, 50, 7, {123_000, 3}}, 0} + end + + test "errors on mixed basic and extended formats" do + assert Calendar.ISO.parse_utc_datetime("20150123 23:50:07.123Z") == + {:error, :invalid_format} + + assert Calendar.ISO.parse_utc_datetime("2015-01-23 235007.123Z") == + {:error, :invalid_format} + end + end + + describe "parse_utc_datetime/2" do + test "allows enforcing basic formats" do + assert Calendar.ISO.parse_utc_datetime("20150123 235007.123Z", :basic) == + {:ok, {2015, 1, 23, 23, 50, 7, {123_000, 3}}, 0} + + assert Calendar.ISO.parse_utc_datetime("2015-01-23 23:50:07.123Z", :basic) == + {:error, :invalid_format} + end + + test "allows enforcing extended formats" do + assert Calendar.ISO.parse_utc_datetime("20150123 235007.123Z", :extended) == + {:error, :invalid_format} + + assert Calendar.ISO.parse_utc_datetime("2015-01-23 23:50:07.123Z", :extended) == + {:ok, {2015, 1, 23, 23, 50, 7, {123_000, 3}}, 0} + end + + test "errors on other format names" do + assert_raise FunctionClauseError, fn -> + Calendar.ISO.parse_naive_datetime("20150123 235007.123Z", :other) + end + + assert_raise FunctionClauseError, fn -> + Calendar.ISO.parse_naive_datetime("2015-01-23 23:50:07.123Z", :other) + end + end + + test "errors on mixed basic and extended formats" do + assert Calendar.ISO.parse_utc_datetime("20150123 23:50:07.123Z", :basic) == + {:error, :invalid_format} + + assert Calendar.ISO.parse_utc_datetime("20150123 23:50:07.123Z", :extended) == + {:error, :invalid_format} + + assert Calendar.ISO.parse_utc_datetime("2015-01-23 235007.123Z", :basic) == + {:error, :invalid_format} + + assert Calendar.ISO.parse_utc_datetime("2015-01-23 235007.123Z", :extended) == + {:error, :invalid_format} + end + end + + test "shift_date/2" do + assert Calendar.ISO.shift_date(2024, 3, 2, Duration.new!([])) == {2024, 3, 2} + assert Calendar.ISO.shift_date(2024, 3, 2, Duration.new!(year: 1)) == {2025, 3, 2} + assert Calendar.ISO.shift_date(2024, 3, 2, Duration.new!(month: 2)) == {2024, 5, 2} + assert Calendar.ISO.shift_date(2024, 3, 2, Duration.new!(week: 3)) == {2024, 3, 23} + assert Calendar.ISO.shift_date(2024, 3, 2, Duration.new!(day: 5)) == {2024, 3, 7} + + assert Calendar.ISO.shift_date(0, 1, 1, Duration.new!(month: 1)) == {0, 2, 1} + assert Calendar.ISO.shift_date(0, 1, 1, Duration.new!(year: 1)) == {1, 1, 1} + assert Calendar.ISO.shift_date(0, 1, 1, Duration.new!(year: -2, month: 2)) == {-2, 3, 1} + assert Calendar.ISO.shift_date(-4, 1, 1, Duration.new!(year: -1)) == {-5, 1, 1} + + assert Calendar.ISO.shift_date(2024, 3, 2, Duration.new!(year: 1, month: 2, week: 3, day: 5)) == + {2025, 5, 28} + + assert Calendar.ISO.shift_date(2024, 3, 2, Duration.new!(year: -1, month: -2, week: -3)) == + {2022, 12, 12} + + assert Calendar.ISO.shift_date(2020, 2, 28, Duration.new!(day: 1)) == {2020, 2, 29} + assert Calendar.ISO.shift_date(2020, 2, 29, Duration.new!(year: 1)) == {2021, 2, 28} + assert Calendar.ISO.shift_date(2024, 3, 31, Duration.new!(month: -1)) == {2024, 2, 29} + assert Calendar.ISO.shift_date(2024, 3, 31, Duration.new!(month: -2)) == {2024, 1, 31} + assert Calendar.ISO.shift_date(2024, 1, 31, Duration.new!(month: 1)) == {2024, 2, 29} + assert Calendar.ISO.shift_date(2024, 1, 31, Duration.new!(month: 2)) == {2024, 3, 31} + assert Calendar.ISO.shift_date(2024, 1, 31, Duration.new!(month: 3)) == {2024, 4, 30} + assert Calendar.ISO.shift_date(2024, 1, 31, Duration.new!(month: 4)) == {2024, 5, 31} + assert Calendar.ISO.shift_date(2024, 1, 31, Duration.new!(month: 5)) == {2024, 6, 30} + assert Calendar.ISO.shift_date(2024, 1, 31, Duration.new!(month: 6)) == {2024, 7, 31} + assert Calendar.ISO.shift_date(2024, 1, 31, Duration.new!(month: 7)) == {2024, 8, 31} + assert Calendar.ISO.shift_date(2024, 1, 31, Duration.new!(month: 8)) == {2024, 9, 30} + assert Calendar.ISO.shift_date(2024, 1, 31, Duration.new!(month: 9)) == {2024, 10, 31} + end + + test "shift_naive_datetime/2" do + assert Calendar.ISO.shift_naive_datetime( + 2024, + 3, + 2, + 0, + 0, + 0, + {0, 0}, + Duration.new!([]) + ) == {2024, 3, 2, 0, 0, 0, {0, 0}} + + assert Calendar.ISO.shift_naive_datetime( + 2000, + 1, + 1, + 0, + 0, + 0, + {0, 0}, + Duration.new!(year: 1) + ) == {2001, 1, 1, 0, 0, 0, {0, 0}} + + assert Calendar.ISO.shift_naive_datetime( + 2000, + 1, + 1, + 0, + 0, + 0, + {0, 0}, + Duration.new!(month: 1) + ) == {2000, 2, 1, 0, 0, 0, {0, 0}} + + assert Calendar.ISO.shift_naive_datetime( + 2000, + 1, + 1, + 0, + 0, + 0, + {0, 0}, + Duration.new!(month: 1, day: 28) + ) == {2000, 2, 29, 0, 0, 0, {0, 0}} + + assert Calendar.ISO.shift_naive_datetime( + 2000, + 1, + 1, + 0, + 0, + 0, + {0, 0}, + Duration.new!(month: 1, day: 30) + ) == {2000, 3, 2, 0, 0, 0, {0, 0}} + + assert Calendar.ISO.shift_naive_datetime( + 2000, + 1, + 1, + 0, + 0, + 0, + {0, 0}, + Duration.new!(month: 2, day: 29) + ) == {2000, 3, 30, 0, 0, 0, {0, 0}} + + assert Calendar.ISO.shift_naive_datetime( + 2000, + 2, + 29, + 0, + 0, + 0, + {0, 0}, + Duration.new!(year: -1) + ) == {1999, 2, 28, 0, 0, 0, {0, 0}} + + assert Calendar.ISO.shift_naive_datetime( + 2000, + 2, + 29, + 0, + 0, + 0, + {0, 0}, + Duration.new!(month: -1) + ) == {2000, 1, 29, 0, 0, 0, {0, 0}} + + assert Calendar.ISO.shift_naive_datetime( + 2000, + 2, + 29, + 0, + 0, + 0, + {0, 0}, + Duration.new!(month: -1, day: -28) + ) == {2000, 1, 1, 0, 0, 0, {0, 0}} + + assert Calendar.ISO.shift_naive_datetime( + 2000, + 2, + 29, + 0, + 0, + 0, + {0, 0}, + Duration.new!(month: -1, day: -30) + ) == {1999, 12, 30, 0, 0, 0, {0, 0}} + + assert Calendar.ISO.shift_naive_datetime( + 2000, + 2, + 29, + 0, + 0, + 0, + {0, 0}, + Duration.new!(month: -1, day: -29) + ) == {1999, 12, 31, 0, 0, 0, {0, 0}} + + assert Calendar.ISO.shift_naive_datetime( + 2000, + 1, + 1, + 0, + 0, + 0, + {0, 0}, + Duration.new!(hour: 12) + ) == {2000, 1, 1, 12, 0, 0, {0, 0}} + + assert Calendar.ISO.shift_naive_datetime( + 2000, + 1, + 1, + 0, + 0, + 0, + {0, 0}, + Duration.new!(minute: -65) + ) == {1999, 12, 31, 22, 55, 0, {0, 0}} + end + + test "shift_time/2" do + assert Calendar.ISO.shift_time(0, 0, 0, {0, 0}, Duration.new!(hour: 1)) == {1, 0, 0, {0, 0}} + assert Calendar.ISO.shift_time(0, 0, 0, {0, 0}, Duration.new!(hour: -1)) == {23, 0, 0, {0, 0}} + + assert Calendar.ISO.shift_time(0, 0, 0, {0, 0}, Duration.new!(minute: 30)) == + {0, 30, 0, {0, 0}} + + assert Calendar.ISO.shift_time(0, 0, 0, {0, 0}, Duration.new!(minute: -30)) == + {23, 30, 0, {0, 0}} + + assert Calendar.ISO.shift_time(0, 0, 0, {0, 0}, Duration.new!(second: 30)) == + {0, 0, 30, {0, 0}} + + assert Calendar.ISO.shift_time(0, 0, 0, {0, 0}, Duration.new!(second: -30)) == + {23, 59, 30, {0, 0}} + + assert Calendar.ISO.shift_time(0, 0, 0, {0, 0}, Duration.new!(microsecond: {100, 6})) == + {0, 0, 0, {100, 6}} + + assert Calendar.ISO.shift_time(0, 0, 0, {0, 0}, Duration.new!(microsecond: {-100, 6})) == + {23, 59, 59, {999_900, 6}} + + assert Calendar.ISO.shift_time(0, 0, 0, {0, 0}, Duration.new!(microsecond: {2000, 4})) == + {0, 0, 0, {2000, 4}} + + assert Calendar.ISO.shift_time(0, 0, 0, {0, 0}, Duration.new!(microsecond: {-2000, 4})) == + {23, 59, 59, {998_000, 4}} + + assert Calendar.ISO.shift_time(0, 0, 0, {3500, 6}, Duration.new!(microsecond: {-2000, 4})) == + {0, 0, 0, {1500, 4}} + + assert Calendar.ISO.shift_time(0, 0, 0, {3500, 4}, Duration.new!(minute: 5)) == + {0, 5, 0, {3500, 4}} + + assert Calendar.ISO.shift_time(0, 0, 0, {3500, 6}, Duration.new!(hour: 4)) == + {4, 0, 0, {3500, 6}} + + assert Calendar.ISO.shift_time( + 23, + 59, + 59, + {999_900, 6}, + Duration.new!(hour: 4, microsecond: {100, 6}) + ) == {4, 0, 0, {0, 6}} + end +end diff --git a/lib/elixir/test/elixir/calendar/naive_datetime_test.exs b/lib/elixir/test/elixir/calendar/naive_datetime_test.exs new file mode 100644 index 00000000000..a3796a34a91 --- /dev/null +++ b/lib/elixir/test/elixir/calendar/naive_datetime_test.exs @@ -0,0 +1,461 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) +Code.require_file("holocene.exs", __DIR__) +Code.require_file("fakes.exs", __DIR__) + +defmodule NaiveDateTimeTest do + use ExUnit.Case, async: true + doctest NaiveDateTime + + test "sigil_N" do + assert ~N[2000-01-01T12:34:56] == + %NaiveDateTime{ + calendar: Calendar.ISO, + year: 2000, + month: 1, + day: 1, + hour: 12, + minute: 34, + second: 56 + } + + assert ~N[2000-01-01T12:34:56 Calendar.Holocene] == + %NaiveDateTime{ + calendar: Calendar.Holocene, + year: 2000, + month: 1, + day: 1, + hour: 12, + minute: 34, + second: 56 + } + + assert ~N[2000-01-01 12:34:56] == + %NaiveDateTime{ + calendar: Calendar.ISO, + year: 2000, + month: 1, + day: 1, + hour: 12, + minute: 34, + second: 56 + } + + assert ~N[2000-01-01 12:34:56 Calendar.Holocene] == + %NaiveDateTime{ + calendar: Calendar.Holocene, + year: 2000, + month: 1, + day: 1, + hour: 12, + minute: 34, + second: 56 + } + + assert_raise ArgumentError, + ~s/cannot parse "2001-50-50T12:34:56" as NaiveDateTime for Calendar.ISO, reason: :invalid_date/, + fn -> Code.eval_string("~N[2001-50-50T12:34:56]") end + + assert_raise ArgumentError, + ~s/cannot parse "2001-01-01T12:34:65" as NaiveDateTime for Calendar.ISO, reason: :invalid_time/, + fn -> Code.eval_string("~N[2001-01-01T12:34:65]") end + + assert_raise ArgumentError, + ~s/cannot parse "20010101 123456" as NaiveDateTime for Calendar.ISO, reason: :invalid_format/, + fn -> Code.eval_string(~s{~N[20010101 123456]}) end + + assert_raise ArgumentError, + ~s/cannot parse "2001-01-01 12:34:56 notalias" as NaiveDateTime for Calendar.ISO, reason: :invalid_format/, + fn -> Code.eval_string("~N[2001-01-01 12:34:56 notalias]") end + + assert_raise ArgumentError, + ~s/cannot parse "2001-01-01T12:34:56 notalias" as NaiveDateTime for Calendar.ISO, reason: :invalid_format/, + fn -> Code.eval_string("~N[2001-01-01T12:34:56 notalias]") end + + assert_raise ArgumentError, + ~s/cannot parse "2001-50-50T12:34:56" as NaiveDateTime for Calendar.Holocene, reason: :invalid_date/, + fn -> Code.eval_string("~N[2001-50-50T12:34:56 Calendar.Holocene]") end + + assert_raise ArgumentError, + ~s/cannot parse "2001-01-01T12:34:65" as NaiveDateTime for Calendar.Holocene, reason: :invalid_time/, + fn -> Code.eval_string("~N[2001-01-01T12:34:65 Calendar.Holocene]") end + + assert_raise UndefinedFunctionError, fn -> + Code.eval_string("~N[2001-01-01 12:34:56 UnknownCalendar]") + end + + assert_raise UndefinedFunctionError, fn -> + Code.eval_string("~N[2001-01-01T12:34:56 UnknownCalendar]") + end + end + + test "to_string/1" do + assert to_string(~N[2000-01-01 23:00:07.005]) == "2000-01-01 23:00:07.005" + assert NaiveDateTime.to_string(~N[2000-01-01 23:00:07.005]) == "2000-01-01 23:00:07.005" + + ndt = %{~N[2000-01-01 23:00:07.005] | calendar: FakeCalendar} + assert to_string(ndt) == "1/1/2000F23::0::7" + end + + test "inspect/1" do + assert inspect(~N[2000-01-01 23:00:07.005]) == "~N[2000-01-01 23:00:07.005]" + assert inspect(~N[-0100-12-31 23:00:07.005]) == "~N[-0100-12-31 23:00:07.005]" + + assert inspect(%{~N[2000-01-01 23:00:07.005] | year: 99999}) == + "NaiveDateTime.new!(99999, 1, 1, 23, 0, 7, {5000, 3})" + + ndt = %{~N[2000-01-01 23:00:07.005] | calendar: FakeCalendar} + assert inspect(ndt) == "~N[1/1/2000F23::0::7 FakeCalendar]" + end + + test "compare/2" do + ndt1 = ~N[2000-04-16 13:30:15.0049] + ndt2 = ~N[2000-04-16 13:30:15.0050] + ndt3 = ~N[2001-04-16 13:30:15.0050] + ndt4 = ~N[-0001-04-16 13:30:15.004] + assert NaiveDateTime.compare(ndt1, ndt1) == :eq + assert NaiveDateTime.compare(ndt1, ndt2) == :lt + assert NaiveDateTime.compare(ndt2, ndt1) == :gt + assert NaiveDateTime.compare(ndt3, ndt1) == :gt + assert NaiveDateTime.compare(ndt3, ndt2) == :gt + assert NaiveDateTime.compare(ndt4, ndt4) == :eq + assert NaiveDateTime.compare(ndt1, ndt4) == :gt + assert NaiveDateTime.compare(ndt4, ndt3) == :lt + end + + test "before?/2 and after?/2" do + ndt1 = ~N[2022-01-12 10:10:11.0019] + ndt2 = ~N[2022-02-16 13:20:15.0019] + + assert NaiveDateTime.before?(ndt1, ndt2) + assert not NaiveDateTime.before?(ndt2, ndt1) + + assert NaiveDateTime.after?(ndt2, ndt1) + assert not NaiveDateTime.after?(ndt1, ndt2) + end + + test "to_iso8601/1" do + ndt = ~N[2000-04-16 12:34:15.1234] + ndt = put_in(ndt.calendar, FakeCalendar) + + message = + "cannot convert #{inspect(ndt)} to target calendar Calendar.ISO, " <> + "reason: #{inspect(ndt.calendar)} and Calendar.ISO have different day rollover moments, " <> + "making this conversion ambiguous" + + assert_raise ArgumentError, message, fn -> + NaiveDateTime.to_iso8601(ndt) + end + end + + describe "add" do + test "add with invalid time unit" do + dt = NaiveDateTime.utc_now() + + message = + ~r/unsupported time unit\. Expected :day, :hour, :minute, :second, :millisecond, :microsecond, :nanosecond, or a positive integer, got "day"/ + + assert_raise ArgumentError, message, fn -> NaiveDateTime.add(dt, 1, "day") end + end + + test "add with other calendars" do + assert ~N[2000-01-01 12:34:15.123456] + |> NaiveDateTime.convert!(Calendar.Holocene) + |> NaiveDateTime.add(10, :second) == + %NaiveDateTime{ + calendar: Calendar.Holocene, + year: 12000, + month: 1, + day: 1, + hour: 12, + minute: 34, + second: 25, + microsecond: {123_456, 6} + } + end + + test "add with datetime" do + dt = %DateTime{ + year: 2000, + month: 2, + day: 29, + zone_abbr: "CET", + hour: 23, + minute: 0, + second: 7, + microsecond: {0, 0}, + utc_offset: 3600, + std_offset: 0, + time_zone: "Europe/Warsaw" + } + + assert NaiveDateTime.add(dt, 21, :second) == ~N[2000-02-29 23:00:28] + end + end + + describe "diff" do + test "diff with invalid time unit" do + dt = NaiveDateTime.utc_now() + + message = + ~r/unsupported time unit\. Expected :day, :hour, :minute, :second, :millisecond, :microsecond, :nanosecond, or a positive integer, got "day"/ + + assert_raise ArgumentError, message, fn -> NaiveDateTime.diff(dt, dt, "day") end + end + + test "diff with other calendars" do + assert ~N[2000-01-01 12:34:15.123456] + |> NaiveDateTime.convert!(Calendar.Holocene) + |> NaiveDateTime.add(10, :second) + |> NaiveDateTime.diff(~N[2000-01-01 12:34:15.123456]) == 10 + end + + test "diff with datetime" do + dt = %DateTime{ + year: 2000, + month: 2, + day: 29, + zone_abbr: "CET", + hour: 23, + minute: 0, + second: 7, + microsecond: {0, 0}, + utc_offset: 3600, + std_offset: 0, + time_zone: "Europe/Warsaw" + } + + assert NaiveDateTime.diff(%{dt | second: 57}, dt, :second) == 50 + end + end + + test "convert/2" do + assert NaiveDateTime.convert(~N[2000-01-01 12:34:15.123400], Calendar.Holocene) == + {:ok, Calendar.Holocene.naive_datetime(12000, 1, 1, 12, 34, 15, {123_400, 6})} + + assert ~N[2000-01-01 12:34:15] + |> NaiveDateTime.convert!(Calendar.Holocene) + |> NaiveDateTime.convert!(Calendar.ISO) == ~N[2000-01-01 12:34:15] + + assert ~N[2000-01-01 12:34:15.123456] + |> NaiveDateTime.convert!(Calendar.Holocene) + |> NaiveDateTime.convert!(Calendar.ISO) == ~N[2000-01-01 12:34:15.123456] + + assert NaiveDateTime.convert(~N[2016-02-03 00:00:01], FakeCalendar) == + {:error, :incompatible_calendars} + + assert NaiveDateTime.convert(~N[1970-01-01 00:00:00], Calendar.Holocene) == + {:ok, Calendar.Holocene.naive_datetime(11970, 1, 1, 0, 0, 0, {0, 0})} + + assert NaiveDateTime.convert(DateTime.from_unix!(0, :second), Calendar.Holocene) == + {:ok, Calendar.Holocene.naive_datetime(11970, 1, 1, 0, 0, 0, {0, 0})} + end + + test "truncate/2" do + assert NaiveDateTime.truncate(~N[2017-11-06 00:23:51.123456], :microsecond) == + ~N[2017-11-06 00:23:51.123456] + + assert NaiveDateTime.truncate(~N[2017-11-06 00:23:51.0], :millisecond) == + ~N[2017-11-06 00:23:51.0] + + assert NaiveDateTime.truncate(~N[2017-11-06 00:23:51.999], :millisecond) == + ~N[2017-11-06 00:23:51.999] + + assert NaiveDateTime.truncate(~N[2017-11-06 00:23:51.1009], :millisecond) == + ~N[2017-11-06 00:23:51.100] + + assert NaiveDateTime.truncate(~N[2017-11-06 00:23:51.123456], :millisecond) == + ~N[2017-11-06 00:23:51.123] + + assert NaiveDateTime.truncate(~N[2017-11-06 00:23:51.000456], :millisecond) == + ~N[2017-11-06 00:23:51.000] + + assert NaiveDateTime.truncate(~N[2017-11-06 00:23:51.123456], :second) == + ~N[2017-11-06 00:23:51] + end + + test "truncate/2 with datetime" do + dt = %DateTime{ + year: 2000, + month: 2, + day: 29, + zone_abbr: "CET", + hour: 23, + minute: 0, + second: 7, + microsecond: {3000, 6}, + utc_offset: 3600, + std_offset: 0, + time_zone: "Europe/Warsaw" + } + + assert NaiveDateTime.truncate(dt, :millisecond) == ~N[2000-02-29 23:00:07.003] + assert catch_error(NaiveDateTime.truncate(~T[00:00:00.000000], :millisecond)) + end + + describe "utc_now/1" do + test "utc_now/1 with default calendar (ISO)" do + naive_datetime = NaiveDateTime.utc_now() + assert naive_datetime.year >= 2019 + end + + test "utc_now/1 with alternative calendar" do + naive_datetime = NaiveDateTime.utc_now(Calendar.Holocene) + assert naive_datetime.calendar == Calendar.Holocene + assert naive_datetime.year >= 12019 + end + end + + describe "local_now/1" do + test "local_now/1 with default calendar (ISO)" do + naive_datetime = NaiveDateTime.local_now() + assert naive_datetime.year >= 2018 + end + + test "local_now/1 alternative calendar" do + naive_datetime = NaiveDateTime.local_now(Calendar.Holocene) + assert naive_datetime.calendar == Calendar.Holocene + assert naive_datetime.year >= 12018 + end + + test "local_now/1 incompatible calendar" do + assert_raise ArgumentError, + ~s(cannot get "local now" in target calendar FakeCalendar, reason: cannot convert from Calendar.ISO to FakeCalendar.), + fn -> + NaiveDateTime.local_now(FakeCalendar) + end + end + end + + describe "to_date/2" do + test "downcasting" do + dt = %DateTime{ + year: 2000, + month: 2, + day: 29, + zone_abbr: "CET", + hour: 23, + minute: 0, + second: 7, + microsecond: {3000, 6}, + utc_offset: 3600, + std_offset: 0, + time_zone: "Europe/Warsaw" + } + + assert NaiveDateTime.to_date(dt) == ~D[2000-02-29] + end + + test "upcasting" do + assert catch_error(NaiveDateTime.to_date(~D[2000-02-29])) + end + end + + describe "to_time/2" do + test "downcasting" do + dt = %DateTime{ + year: 2000, + month: 2, + day: 29, + zone_abbr: "CET", + hour: 23, + minute: 0, + second: 7, + microsecond: {3000, 6}, + utc_offset: 3600, + std_offset: 0, + time_zone: "Europe/Warsaw" + } + + assert NaiveDateTime.to_time(dt) == ~T[23:00:07.003000] + end + + test "upcasting" do + assert catch_error(NaiveDateTime.to_time(~T[00:00:00.000000])) + end + end + + describe "beginning_of_day/1" do + test "precision" do + assert NaiveDateTime.beginning_of_day(~N[2000-01-01 23:00:07.123]) == + ~N[2000-01-01 00:00:00.000] + + assert NaiveDateTime.beginning_of_day(~N[2000-01-01 23:00:07]) == ~N[2000-01-01 00:00:00] + end + end + + describe "end_of_day/1" do + test "precision" do + assert NaiveDateTime.end_of_day(~N[2000-01-01 23:00:07.1]) == ~N[2000-01-01 23:59:59.9] + assert NaiveDateTime.end_of_day(~N[2000-01-01 23:00:07.123]) == ~N[2000-01-01 23:59:59.999] + + assert NaiveDateTime.end_of_day(~N[2000-01-01 23:00:07.12345]) == + ~N[2000-01-01 23:59:59.99999] + + assert NaiveDateTime.end_of_day(~N[2000-01-01 23:00:07]) == ~N[2000-01-01 23:59:59] + end + end + + test "shift/2" do + naive_datetime = ~N[2000-01-01 00:00:00] + assert NaiveDateTime.shift(naive_datetime, year: 1) == ~N[2001-01-01 00:00:00] + assert NaiveDateTime.shift(naive_datetime, month: 1) == ~N[2000-02-01 00:00:00] + assert NaiveDateTime.shift(naive_datetime, week: 3) == ~N[2000-01-22 00:00:00] + assert NaiveDateTime.shift(naive_datetime, day: 2) == ~N[2000-01-03 00:00:00] + assert NaiveDateTime.shift(naive_datetime, hour: 6) == ~N[2000-01-01 06:00:00] + assert NaiveDateTime.shift(naive_datetime, minute: 30) == ~N[2000-01-01 00:30:00] + assert NaiveDateTime.shift(naive_datetime, second: 45) == ~N[2000-01-01 00:00:45] + assert NaiveDateTime.shift(naive_datetime, year: -1) == ~N[1999-01-01 00:00:00] + assert NaiveDateTime.shift(naive_datetime, month: -1) == ~N[1999-12-01 00:00:00] + assert NaiveDateTime.shift(naive_datetime, week: -1) == ~N[1999-12-25 00:00:00] + assert NaiveDateTime.shift(naive_datetime, day: -1) == ~N[1999-12-31 00:00:00] + assert NaiveDateTime.shift(naive_datetime, hour: -12) == ~N[1999-12-31 12:00:00] + assert NaiveDateTime.shift(naive_datetime, minute: -45) == ~N[1999-12-31 23:15:00] + assert NaiveDateTime.shift(naive_datetime, second: -30) == ~N[1999-12-31 23:59:30] + assert NaiveDateTime.shift(naive_datetime, year: 1, month: 2) == ~N[2001-03-01 00:00:00] + + assert NaiveDateTime.shift(naive_datetime, microsecond: {-500, 6}) == + ~N[1999-12-31 23:59:59.999500] + + assert NaiveDateTime.shift(naive_datetime, microsecond: {500, 6}) == + ~N[2000-01-01 00:00:00.000500] + + assert NaiveDateTime.shift(naive_datetime, microsecond: {100, 6}) == + ~N[2000-01-01 00:00:00.000100] + + assert NaiveDateTime.shift(naive_datetime, microsecond: {100, 4}) == + ~N[2000-01-01 00:00:00.0001] + + assert NaiveDateTime.shift(naive_datetime, month: 2, day: 3, hour: 6, minute: 15) == + ~N[2000-03-04 06:15:00] + + assert NaiveDateTime.shift(naive_datetime, + year: 1, + month: 2, + week: 3, + day: 4, + hour: 5, + minute: 6, + second: 7, + microsecond: {8, 6} + ) == ~N[2001-03-26 05:06:07.000008] + + assert NaiveDateTime.shift(naive_datetime, + year: -1, + month: -2, + week: -3, + day: -4, + hour: -5, + minute: -6, + second: -7, + microsecond: {-8, 6} + ) == ~N[1998-10-06 18:53:52.999992] + + assert_raise ArgumentError, + "unknown unit :months. Expected :year, :month, :week, :day, :hour, :minute, :second, :microsecond", + fn -> NaiveDateTime.shift(~N[2000-01-01 00:00:00], months: 12) end + end +end diff --git a/lib/elixir/test/elixir/calendar/time_test.exs b/lib/elixir/test/elixir/calendar/time_test.exs new file mode 100644 index 00000000000..2818e755117 --- /dev/null +++ b/lib/elixir/test/elixir/calendar/time_test.exs @@ -0,0 +1,177 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) +Code.require_file("holocene.exs", __DIR__) +Code.require_file("fakes.exs", __DIR__) + +defmodule TimeTest do + use ExUnit.Case, async: true + doctest Time + + test "sigil_T" do + assert ~T[12:34:56] == + %Time{calendar: Calendar.ISO, hour: 12, minute: 34, second: 56} + + assert ~T[12:34:56 Calendar.Holocene] == + %Time{calendar: Calendar.Holocene, hour: 12, minute: 34, second: 56} + + assert_raise ArgumentError, + ~s/cannot parse "12:34:65" as Time for Calendar.ISO, reason: :invalid_time/, + fn -> Code.eval_string("~T[12:34:65]") end + + assert_raise ArgumentError, + ~s/cannot parse "123456" as Time for Calendar.ISO, reason: :invalid_format/, + fn -> Code.eval_string("~T[123456]") end + + assert_raise ArgumentError, + ~s/cannot parse "12:34:56 notalias" as Time for Calendar.ISO, reason: :invalid_format/, + fn -> Code.eval_string("~T[12:34:56 notalias]") end + + assert_raise ArgumentError, + ~s/cannot parse "12:34:65" as Time for Calendar.Holocene, reason: :invalid_time/, + fn -> Code.eval_string("~T[12:34:65 Calendar.Holocene]") end + + assert_raise UndefinedFunctionError, fn -> + Code.eval_string("~T[12:34:56 UnknownCalendar]") + end + end + + test "to_iso8601/2" do + time1 = ~T[23:00:07.005] + assert Time.to_iso8601(time1) == "23:00:07.005" + assert Time.to_iso8601(Map.from_struct(time1)) == "23:00:07.005" + assert Time.to_iso8601(time1, :basic) == "230007.005" + + time2 = ~T[23:00:07.005 Calendar.Holocene] + assert Time.to_iso8601(time2) == "23:00:07.005" + assert Time.to_iso8601(Map.from_struct(time2)) == "23:00:07.005" + assert Time.to_iso8601(time2, :basic) == "230007.005" + end + + test "to_string/1" do + time = ~T[23:00:07.005] + assert to_string(time) == "23:00:07.005" + assert Time.to_string(time) == "23:00:07.005" + assert Time.to_string(Map.from_struct(time)) == "23:00:07.005" + + assert to_string(%{time | calendar: FakeCalendar}) == "23::0::7" + assert Time.to_string(%{time | calendar: FakeCalendar}) == "23::0::7" + end + + test "inspect/1" do + assert inspect(~T[23:00:07.005]) == "~T[23:00:07.005]" + + time = %{~T[23:00:07.005] | calendar: FakeCalendar} + assert inspect(time) == "~T[23::0::7 FakeCalendar]" + end + + test "compare/2" do + time0 = ~T[01:01:01.0] + time1 = ~T[01:01:01.005] + time2 = ~T[01:01:01.0050] + time3 = ~T[23:01:01.0050] + assert Time.compare(time0, time1) == :lt + assert Time.compare(time1, time1) == :eq + assert Time.compare(time1, time2) == :eq + assert Time.compare(time1, time3) == :lt + assert Time.compare(time3, time2) == :gt + + time1_holocene = ~T[01:01:01.005 Calendar.Holocene] + assert Time.compare(time1_holocene, time1) == :eq + assert Time.compare(time1_holocene, time2) == :eq + assert Time.compare(time1_holocene, time3) == :lt + end + + test "before?/2 and after?/2" do + time1 = ~T[05:02:01.234] + time2 = ~T[10:00:04.123] + + assert Time.before?(time1, time2) + assert not Time.before?(time2, time1) + + assert Time.after?(time2, time1) + assert not Time.after?(time1, time2) + end + + test "diff/3" do + time1 = ~T[05:02:01.234] + time2 = ~T[10:00:04.123] + time1_holocene = ~T[05:02:01.234 Calendar.Holocene] + + assert Time.diff(time1, time2) == -17883 + assert Time.diff(time1, time2, :hour) == -4 + assert Time.diff(time1, time2, :minute) == -298 + assert Time.diff(time1, time2, :second) == -17883 + assert Time.diff(time1, time2, :millisecond) == -17_882_889 + assert Time.diff(time1, time2, :microsecond) == -17_882_889_000 + + assert Time.diff(time1_holocene, time2) == -17883 + assert Time.diff(time1_holocene, time2, :hour) == -4 + assert Time.diff(time1_holocene, time2, :minute) == -298 + assert Time.diff(time1_holocene, time2, :second) == -17883 + assert Time.diff(time1_holocene, time2, :millisecond) == -17_882_889 + assert Time.diff(time1_holocene, time2, :microsecond) == -17_882_889_000 + end + + test "truncate/2" do + assert Time.truncate(~T[01:01:01.123456], :microsecond) == ~T[01:01:01.123456] + + assert Time.truncate(~T[01:01:01.0], :millisecond) == ~T[01:01:01.0] + assert Time.truncate(~T[01:01:01.00], :millisecond) == ~T[01:01:01.00] + assert Time.truncate(~T[01:01:01.1], :millisecond) == ~T[01:01:01.1] + assert Time.truncate(~T[01:01:01.100], :millisecond) == ~T[01:01:01.100] + assert Time.truncate(~T[01:01:01.999], :millisecond) == ~T[01:01:01.999] + assert Time.truncate(~T[01:01:01.1000], :millisecond) == ~T[01:01:01.100] + assert Time.truncate(~T[01:01:01.1001], :millisecond) == ~T[01:01:01.100] + assert Time.truncate(~T[01:01:01.123456], :millisecond) == ~T[01:01:01.123] + assert Time.truncate(~T[01:01:01.000123], :millisecond) == ~T[01:01:01.000] + assert Time.truncate(~T[01:01:01.00012], :millisecond) == ~T[01:01:01.000] + + assert Time.truncate(~T[01:01:01.123456], :second) == ~T[01:01:01] + end + + test "add/3" do + time = ~T[00:00:00.0] + + assert Time.add(time, 1, :hour) == ~T[01:00:00.0] + + assert Time.add(time, 1, 10) == ~T[00:00:00.100000] + + assert_raise ArgumentError, ~r/Expected :hour, :minute, :second/, fn -> + Time.add(time, 1, 0) + end + end + + test "shift/2" do + time = ~T[00:00:00.0] + assert Time.shift(time, hour: 1) == ~T[01:00:00.0] + assert Time.shift(time, hour: 25) == ~T[01:00:00.0] + assert Time.shift(time, minute: 25) == ~T[00:25:00.0] + assert Time.shift(time, second: 50) == ~T[00:00:50.0] + assert Time.shift(time, microsecond: {150, 6}) == ~T[00:00:00.000150] + assert Time.shift(time, microsecond: {1000, 4}) == ~T[00:00:00.0010] + assert Time.shift(time, hour: 2, minute: 65, second: 5) == ~T[03:05:05.0] + + assert_raise ArgumentError, + "unsupported unit :day. Expected :hour, :minute, :second, :microsecond", + fn -> Time.shift(time, day: 1) end + + assert_raise ArgumentError, + "unknown unit :hours. Expected :hour, :minute, :second, :microsecond", + fn -> Time.shift(time, hours: 12) end + + assert_raise ArgumentError, + "cannot shift time by date scale unit. Expected :hour, :minute, :second, :microsecond", + fn -> Time.shift(time, %Duration{day: 1}) end + + assert_raise ArgumentError, + "unsupported value nil for :minute. Expected an integer", + fn -> Time.shift(time, minute: nil) end + + assert_raise ArgumentError, + ~r/unsupported value 1 for :microsecond. Expected a tuple \{ms, precision\}/, + fn -> Time.shift(time, microsecond: 1) end + end +end diff --git a/lib/elixir/test/elixir/calendar_test.exs b/lib/elixir/test/elixir/calendar_test.exs new file mode 100644 index 00000000000..c812f311ac1 --- /dev/null +++ b/lib/elixir/test/elixir/calendar_test.exs @@ -0,0 +1,450 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("test_helper.exs", __DIR__) + +defmodule CalendarTest do + use ExUnit.Case, async: true + doctest Calendar + + describe "strftime/3" do + test "returns received string if there is no datetime formatting to be found in it" do + assert Calendar.strftime(~N[2019-08-20 15:47:34.001], "same string") == "same string" + end + + test "formats all time zones blank when receiving a NaiveDateTime" do + assert Calendar.strftime(~N[2019-08-15 17:07:57.001], "%z%Z") == "" + end + + test "raises error when trying to format a date with a map that has no date fields" do + time_without_date = %{hour: 15, minute: 47, second: 34, microsecond: {0, 0}} + + assert_raise KeyError, fn -> Calendar.strftime(time_without_date, "%x") end + end + + test "raises error when trying to format a time with a map that has no time fields" do + date_without_time = %{year: 2019, month: 8, day: 20} + + assert_raise KeyError, fn -> Calendar.strftime(date_without_time, "%X") end + end + + test "raises error when the format is invalid" do + assert_raise ArgumentError, "invalid strftime format: %-", fn -> + Calendar.strftime(~N[2019-08-20 15:47:34.001], "%-2-ç") + end + + assert_raise ArgumentError, "invalid strftime format: %", fn -> + Calendar.strftime(~N[2019-08-20 15:47:34.001], "%") + end + end + + test "raises error when the preferred_datetime calls itself" do + assert_raise ArgumentError, fn -> + Calendar.strftime(~N[2019-08-20 15:47:34.001], "%c", preferred_datetime: "%c") + end + end + + test "raises error when the preferred_date calls itself" do + assert_raise ArgumentError, fn -> + Calendar.strftime(~N[2019-08-20 15:47:34.001], "%x", preferred_date: "%x") + end + end + + test "raises error when the preferred_time calls itself" do + assert_raise ArgumentError, fn -> + Calendar.strftime(~N[2019-08-20 15:47:34.001], "%X", preferred_time: "%X") + end + end + + test "raises error when the preferred formats creates a circular chain" do + assert_raise ArgumentError, fn -> + Calendar.strftime(~N[2019-08-20 15:47:34.001], "%c", + preferred_datetime: "%x", + preferred_date: "%X", + preferred_time: "%c" + ) + end + end + + test "with preferred formats are included multiple times on the same string" do + assert Calendar.strftime(~N[2019-08-15 17:07:57.001], "%c %c %x %x %X %X") == + "2019-08-15 17:07:57 2019-08-15 17:07:57 2019-08-15 2019-08-15 17:07:57 17:07:57" + end + + test "`-` removes padding" do + assert Calendar.strftime(~D[2019-01-01], "%-j") == "1" + assert Calendar.strftime(~T[17:07:57.001], "%-999M") == "7" + end + + test "formats time zones correctly when receiving a DateTime" do + datetime_with_zone = %DateTime{ + year: 2019, + month: 8, + day: 15, + zone_abbr: "EEST", + hour: 17, + minute: 7, + second: 57, + microsecond: {0, 0}, + utc_offset: 7200, + std_offset: 3600, + time_zone: "UK" + } + + assert Calendar.strftime(datetime_with_zone, "%z %Z") == "+0300 EEST" + end + + test "formats AM and PM correctly on the %P and %p options" do + am_time_almost_pm = ~U[2019-08-26 11:59:59.001Z] + pm_time = ~U[2019-08-26 12:00:57.001Z] + pm_time_almost_am = ~U[2019-08-26 23:59:57.001Z] + am_time = ~U[2019-08-26 00:00:01.001Z] + + assert Calendar.strftime(am_time_almost_pm, "%P %p") == "am AM" + assert Calendar.strftime(pm_time, "%P %p") == "pm PM" + assert Calendar.strftime(pm_time_almost_am, "%P %p") == "pm PM" + assert Calendar.strftime(am_time, "%P %p") == "am AM" + end + + test "formats all weekdays correctly with %A and %a formats" do + sunday = ~U[2019-08-25 11:59:59.001Z] + monday = ~U[2019-08-26 11:59:59.001Z] + tuesday = ~U[2019-08-27 11:59:59.001Z] + wednesday = ~U[2019-08-28 11:59:59.001Z] + thursday = ~U[2019-08-29 11:59:59.001Z] + friday = ~U[2019-08-30 11:59:59.001Z] + saturday = ~U[2019-08-31 11:59:59.001Z] + + assert Calendar.strftime(sunday, "%A %a") == "Sunday Sun" + assert Calendar.strftime(monday, "%A %a") == "Monday Mon" + assert Calendar.strftime(tuesday, "%A %a") == "Tuesday Tue" + assert Calendar.strftime(wednesday, "%A %a") == "Wednesday Wed" + assert Calendar.strftime(thursday, "%A %a") == "Thursday Thu" + assert Calendar.strftime(friday, "%A %a") == "Friday Fri" + assert Calendar.strftime(saturday, "%A %a") == "Saturday Sat" + end + + test "formats all months correctly with the %B and %b formats" do + assert Calendar.strftime(%{month: 1}, "%B %b") == "January Jan" + assert Calendar.strftime(%{month: 2}, "%B %b") == "February Feb" + assert Calendar.strftime(%{month: 3}, "%B %b") == "March Mar" + assert Calendar.strftime(%{month: 4}, "%B %b") == "April Apr" + assert Calendar.strftime(%{month: 5}, "%B %b") == "May May" + assert Calendar.strftime(%{month: 6}, "%B %b") == "June Jun" + assert Calendar.strftime(%{month: 7}, "%B %b") == "July Jul" + assert Calendar.strftime(%{month: 8}, "%B %b") == "August Aug" + assert Calendar.strftime(%{month: 9}, "%B %b") == "September Sep" + assert Calendar.strftime(%{month: 10}, "%B %b") == "October Oct" + assert Calendar.strftime(%{month: 11}, "%B %b") == "November Nov" + assert Calendar.strftime(%{month: 12}, "%B %b") == "December Dec" + end + + test "formats all weekdays correctly on %A with day_of_week_names option" do + sunday = ~U[2019-08-25 11:59:59.001Z] + monday = ~U[2019-08-26 11:59:59.001Z] + tuesday = ~U[2019-08-27 11:59:59.001Z] + wednesday = ~U[2019-08-28 11:59:59.001Z] + thursday = ~U[2019-08-29 11:59:59.001Z] + friday = ~U[2019-08-30 11:59:59.001Z] + saturday = ~U[2019-08-31 11:59:59.001Z] + + day_of_week_names = fn day_of_week -> + {"segunda-feira", "terça-feira", "quarta-feira", "quinta-feira", "sexta-feira", "sábado", + "domingo"} + |> elem(day_of_week - 1) + end + + assert Calendar.strftime(sunday, "%A", day_of_week_names: day_of_week_names) == + "domingo" + + assert Calendar.strftime(monday, "%A", day_of_week_names: day_of_week_names) == + "segunda-feira" + + assert Calendar.strftime(tuesday, "%A", day_of_week_names: day_of_week_names) == + "terça-feira" + + assert Calendar.strftime(wednesday, "%A", day_of_week_names: day_of_week_names) == + "quarta-feira" + + assert Calendar.strftime(thursday, "%A", day_of_week_names: day_of_week_names) == + "quinta-feira" + + assert Calendar.strftime(friday, "%A", day_of_week_names: day_of_week_names) == + "sexta-feira" + + assert Calendar.strftime(saturday, "%A", day_of_week_names: day_of_week_names) == + "sábado" + end + + test "formats all months correctly on the %B with month_names option" do + month_names = fn month -> + {"январь", "февраль", "март", "апрель", "май", "июнь", "июль", "август", "сентябрь", + "октябрь", "ноябрь", "декабрь"} + |> elem(month - 1) + end + + assert Calendar.strftime(%{month: 1}, "%B", month_names: month_names) == "январь" + assert Calendar.strftime(%{month: 2}, "%B", month_names: month_names) == "февраль" + assert Calendar.strftime(%{month: 3}, "%B", month_names: month_names) == "март" + assert Calendar.strftime(%{month: 4}, "%B", month_names: month_names) == "апрель" + assert Calendar.strftime(%{month: 5}, "%B", month_names: month_names) == "май" + assert Calendar.strftime(%{month: 6}, "%B", month_names: month_names) == "июнь" + assert Calendar.strftime(%{month: 7}, "%B", month_names: month_names) == "июль" + assert Calendar.strftime(%{month: 8}, "%B", month_names: month_names) == "август" + assert Calendar.strftime(%{month: 9}, "%B", month_names: month_names) == "сентябрь" + assert Calendar.strftime(%{month: 10}, "%B", month_names: month_names) == "октябрь" + assert Calendar.strftime(%{month: 11}, "%B", month_names: month_names) == "ноябрь" + assert Calendar.strftime(%{month: 12}, "%B", month_names: month_names) == "декабрь" + end + + test "formats all weekdays correctly on the %a format with abbreviated_day_of_week_names option" do + sunday = ~U[2019-08-25 11:59:59.001Z] + monday = ~U[2019-08-26 11:59:59.001Z] + tuesday = ~U[2019-08-27 11:59:59.001Z] + wednesday = ~U[2019-08-28 11:59:59.001Z] + thursday = ~U[2019-08-29 11:59:59.001Z] + friday = ~U[2019-08-30 11:59:59.001Z] + saturday = ~U[2019-08-31 11:59:59.001Z] + + abbreviated_day_of_week_names = fn day_of_week -> + {"seg", "ter", "qua", "qui", "sex", "sáb", "dom"} + |> elem(day_of_week - 1) + end + + assert Calendar.strftime(sunday, "%a", + abbreviated_day_of_week_names: abbreviated_day_of_week_names + ) == "dom" + + assert Calendar.strftime(monday, "%a", + abbreviated_day_of_week_names: abbreviated_day_of_week_names + ) == "seg" + + assert Calendar.strftime(tuesday, "%a", + abbreviated_day_of_week_names: abbreviated_day_of_week_names + ) == "ter" + + assert Calendar.strftime(wednesday, "%a", + abbreviated_day_of_week_names: abbreviated_day_of_week_names + ) == "qua" + + assert Calendar.strftime(thursday, "%a", + abbreviated_day_of_week_names: abbreviated_day_of_week_names + ) == "qui" + + assert Calendar.strftime(friday, "%a", + abbreviated_day_of_week_names: abbreviated_day_of_week_names + ) == "sex" + + assert Calendar.strftime(saturday, "%a", + abbreviated_day_of_week_names: abbreviated_day_of_week_names + ) == "sáb" + end + + test "formats all months correctly on the %b format with abbreviated_month_names option" do + abbreviated_month_names = fn month -> + {"янв", "февр", "март", "апр", "май", "июнь", "июль", "авг", "сент", "окт", "нояб", "дек"} + |> elem(month - 1) + end + + assert Calendar.strftime(%{month: 1}, "%b", + abbreviated_month_names: abbreviated_month_names + ) == + "янв" + + assert Calendar.strftime(%{month: 2}, "%b", + abbreviated_month_names: abbreviated_month_names + ) == + "февр" + + assert Calendar.strftime(%{month: 3}, "%b", + abbreviated_month_names: abbreviated_month_names + ) == + "март" + + assert Calendar.strftime(%{month: 4}, "%b", + abbreviated_month_names: abbreviated_month_names + ) == + "апр" + + assert Calendar.strftime(%{month: 5}, "%b", + abbreviated_month_names: abbreviated_month_names + ) == + "май" + + assert Calendar.strftime(%{month: 6}, "%b", + abbreviated_month_names: abbreviated_month_names + ) == + "июнь" + + assert Calendar.strftime(%{month: 7}, "%b", + abbreviated_month_names: abbreviated_month_names + ) == + "июль" + + assert Calendar.strftime(%{month: 8}, "%b", + abbreviated_month_names: abbreviated_month_names + ) == + "авг" + + assert Calendar.strftime(%{month: 9}, "%b", + abbreviated_month_names: abbreviated_month_names + ) == + "сент" + + assert Calendar.strftime(%{month: 10}, "%b", + abbreviated_month_names: abbreviated_month_names + ) == "окт" + + assert Calendar.strftime(%{month: 11}, "%b", + abbreviated_month_names: abbreviated_month_names + ) == "нояб" + + assert Calendar.strftime(%{month: 12}, "%b", + abbreviated_month_names: abbreviated_month_names + ) == "дек" + end + + test "formats ignores padding and width options on microseconds" do + datetime = ~U[2019-08-15 17:07:57.001234Z] + assert Calendar.strftime(datetime, "%f") == "001234" + assert Calendar.strftime(datetime, "%f") == Calendar.strftime(datetime, "%_20f") + assert Calendar.strftime(datetime, "%f") == Calendar.strftime(datetime, "%020f") + assert Calendar.strftime(datetime, "%f") == Calendar.strftime(datetime, "%-f") + end + + test "formats properly dates with different microsecond precisions" do + assert Calendar.strftime(~U[2019-08-15 17:07:57.5Z], "%f") == "5" + assert Calendar.strftime(~U[2019-08-15 17:07:57.45Z], "%f") == "45" + assert Calendar.strftime(~U[2019-08-15 17:07:57.345Z], "%f") == "345" + assert Calendar.strftime(~U[2019-08-15 17:07:57.2345Z], "%f") == "2345" + assert Calendar.strftime(~U[2019-08-15 17:07:57.12345Z], "%f") == "12345" + assert Calendar.strftime(~U[2019-08-15 17:07:57.012345Z], "%f") == "012345" + end + + test "formats properly different microsecond precisions of zero" do + assert Calendar.strftime(~N[2019-08-15 17:07:57.0], "%f") == "0" + assert Calendar.strftime(~N[2019-08-15 17:07:57.00], "%f") == "00" + assert Calendar.strftime(~N[2019-08-15 17:07:57.000], "%f") == "000" + assert Calendar.strftime(~N[2019-08-15 17:07:57.0000], "%f") == "0000" + assert Calendar.strftime(~N[2019-08-15 17:07:57.00000], "%f") == "00000" + assert Calendar.strftime(~N[2019-08-15 17:07:57.000000], "%f") == "000000" + end + + test "returns a single zero if there's no microseconds precision" do + assert Calendar.strftime(~N[2019-08-15 17:07:57], "%f") == "0" + end + + test "handles `0` both as padding and as part of a width" do + assert Calendar.strftime(~N[2019-08-15 17:07:57], "%10A") == " Thursday" + assert Calendar.strftime(~N[2019-08-15 17:07:57], "%010A") == "00Thursday" + end + + test "formats Epoch time with %s" do + assert Calendar.strftime(~N[2019-08-15 17:07:57], "%s") == "1565888877" + + datetime = %DateTime{ + year: 2019, + month: 8, + day: 15, + hour: 17, + minute: 7, + second: 57, + microsecond: {0, 0}, + time_zone: "Europe/Berlin", + zone_abbr: "CET", + utc_offset: 3600, + std_offset: 0 + } + + assert Calendar.strftime(datetime, "%s") == "1565885277" + end + + test "formats datetime with all options and modifiers" do + assert Calendar.strftime( + ~U[2019-08-15 17:07:57.001Z], + "%04% %a %A %b %B %-3c %d %f %H %I %j %m %_5M %p %P %q %S %u %x %X %y %Y %z %Z" + ) == + "000% Thu Thursday Aug August 2019-08-15 17:07:57 15 001 17 05 227 08 7 PM pm 3 57 4 2019-08-15 17:07:57 19 2019 +0000 UTC" + end + + test "formats according to custom configs" do + assert Calendar.strftime( + ~U[2019-08-15 17:07:57.001Z], + "%A %a %p %B %b %c %x %X", + am_pm_names: fn + :am -> "a" + :pm -> "p" + end, + month_names: fn month -> + {"Janeiro", "Fevereiro", "Março", "Abril", "Maio", "Junho", "Julho", "Agosto", + "Setembro", "Outubro", "Novembro", "Dezembro"} + |> elem(month - 1) + end, + day_of_week_names: fn day_of_week -> + {"понедельник", "вторник", "среда", "четверг", "пятница", "суббота", + "воскресенье"} + |> elem(day_of_week - 1) + end, + abbreviated_month_names: fn month -> + {"Jan", "Fev", "Mar", "Abr", "Mai", "Jun", "Jul", "Ago", "Set", "Out", "Nov", + "Dez"} + |> elem(month - 1) + end, + abbreviated_day_of_week_names: fn day_of_week -> + {"ПНД", "ВТР", "СРД", "ЧТВ", "ПТН", "СБТ", "ВСК"} + |> elem(day_of_week - 1) + end, + preferred_date: "%05Y-%m-%d", + preferred_time: "%M:%_3H%S", + preferred_datetime: "%%" + ) == "четверг ЧТВ P Agosto Ago % 02019-08-15 07: 1757" + end + + test "formats according to custom configs with 2-arity functions" do + assert Calendar.strftime( + ~U[2019-08-15 17:07:57.001Z], + "%A %a %p %B %b %c %x %X", + am_pm_names: fn + :am, ~U[2019-08-15 17:07:57.001Z] -> "a" + :pm, ~U[2019-08-15 17:07:57.001Z] -> "p" + end, + month_names: fn month, ~U[2019-08-15 17:07:57.001Z] -> + {"Janeiro", "Fevereiro", "Março", "Abril", "Maio", "Junho", "Julho", "Agosto", + "Setembro", "Outubro", "Novembro", "Dezembro"} + |> elem(month - 1) + end, + day_of_week_names: fn day_of_week, ~U[2019-08-15 17:07:57.001Z] -> + {"понедельник", "вторник", "среда", "четверг", "пятница", "суббота", + "воскресенье"} + |> elem(day_of_week - 1) + end, + abbreviated_month_names: fn month, ~U[2019-08-15 17:07:57.001Z] -> + {"Jan", "Fev", "Mar", "Abr", "Mai", "Jun", "Jul", "Ago", "Set", "Out", "Nov", + "Dez"} + |> elem(month - 1) + end, + abbreviated_day_of_week_names: fn day_of_week, ~U[2019-08-15 17:07:57.001Z] -> + {"ПНД", "ВТР", "СРД", "ЧТВ", "ПТН", "СБТ", "ВСК"} + |> elem(day_of_week - 1) + end, + preferred_date: "%05Y-%m-%d", + preferred_time: "%M:%_3H%S", + preferred_datetime: "%%" + ) == "четверг ЧТВ P Agosto Ago % 02019-08-15 07: 1757" + end + + test "raises on unknown option according to custom configs" do + assert_raise ArgumentError, "unknown option :unknown given to Calendar.strftime/3", fn -> + Calendar.strftime(~D[2019-08-15], "%D", unknown: "option") + end + end + + test "zero padding for negative year" do + assert Calendar.strftime(Date.new!(-1, 1, 1), "%Y") == "-0001" + assert Calendar.strftime(Date.new!(-11, 1, 1), "%Y") == "-0011" + assert Calendar.strftime(Date.new!(-111, 1, 1), "%Y") == "-0111" + assert Calendar.strftime(Date.new!(-1111, 1, 1), "%Y") == "-1111" + end + end +end diff --git a/lib/elixir/test/elixir/changelog_test.exs b/lib/elixir/test/elixir/changelog_test.exs new file mode 100644 index 00000000000..9245d3de06e --- /dev/null +++ b/lib/elixir/test/elixir/changelog_test.exs @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team + +ExUnit.start() + +defmodule ChangelogTest do + use ExUnit.Case, async: true + doctest_file(Path.expand("../../../../CHANGELOG.md", __DIR__)) +end diff --git a/lib/elixir/test/elixir/code_formatter/calls_test.exs b/lib/elixir/test/elixir/code_formatter/calls_test.exs new file mode 100644 index 00000000000..61ef8c9148f --- /dev/null +++ b/lib/elixir/test/elixir/code_formatter/calls_test.exs @@ -0,0 +1,1233 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Code.Formatter.CallsTest do + use ExUnit.Case, async: true + + import CodeFormatterHelpers + + @short_length [line_length: 10] + @medium_length [line_length: 20] + + describe "next break fits" do + test "does not apply to function calls" do + bad = "foo(very_long_call(bar))" + + good = """ + foo( + very_long_call( + bar + ) + ) + """ + + assert_format bad, good, @short_length + end + + test "does not apply to strings" do + bad = "foo(\"very long string\")" + + good = """ + foo( + "very long string" + ) + """ + + assert_format bad, good, @short_length + end + + test "for functions" do + assert_same """ + foo(fn x -> y end) + """ + + assert_same """ + foo(fn + a1 -> :ok + b2 -> :error + end) + """ + + assert_same """ + foo(bar, fn + a1 -> :ok + b2 -> :error + end) + """ + + assert_same """ + foo(fn x -> + :really_long_atom + end) + """, + @medium_length + + assert_same """ + foo(bar, fn + a1 -> + :ok + + b2 -> + :really_long_error + end) + """, + @medium_length + end + + test "for heredocs" do + assert_same """ + foo(~c''' + bar + ''') + """ + + assert_same ~S''' + foo(""" + bar + """) + ''' + + assert_same """ + foo(~S''' + bar + ''') + """ + + assert_same """ + foo(~S''' + very long line does trigger another break + ''') + """, + @short_length + end + + test "for lists" do + bad = "foo([1, 2, 3, 4])" + + good = """ + foo([ + 1, + 2, + 3, + 4 + ]) + """ + + assert_format bad, good, @short_length + end + + test "for {} calls" do + bad = """ + alias Foo.{ + Bar, Baz + } + """ + + good = """ + alias Foo.{ + Bar, + Baz + } + """ + + assert_format bad, good, @medium_length + end + + test "for binaries only on eol" do + bad = "foo(<<1, 2, 3, 4>>)" + + good = """ + foo( + <<1, 2, + 3, 4>> + ) + """ + + assert_format bad, good, @short_length + + bad = """ + foo(<< + # foo + 1, + 2, + 3, + 4>>) + """ + + good = """ + foo(<< + # foo + 1, + 2, + 3, + 4 + >>) + """ + + assert_format bad, good, @short_length + end + + test "for maps" do + assert_same "a(%{x: 1})", @short_length + assert_format "ab(%{x: 1})", "ab(%{\n x: 1\n})", @short_length + end + end + + describe "local calls" do + test "without arguments" do + assert_format "foo( )", "foo()" + end + + test "without arguments doesn't split on line limit" do + assert_same "very_long_function_name()", @short_length + end + + test "removes outer parens except for unquote_splicing/1" do + assert_format "(foo())", "foo()" + assert_same "(unquote_splicing(123))" + end + + test "with arguments" do + assert_format "foo( :one ,:two,\n :three)", "foo(:one, :two, :three)" + end + + test "with arguments splits on line limit" do + bad = """ + fun(x, y, z) + """ + + good = """ + fun( + x, + y, + z + ) + """ + + assert_format bad, good, @short_length + end + + test "with arguments on comma limit" do + bad = """ + import(foo(abc, cde), :next) + """ + + good = """ + import( + foo(abc, cde), + :next + ) + """ + + assert_format bad, good, @medium_length + end + + test "with keyword lists" do + assert_same "foo(foo: 1, bar: 2)" + assert_same "foo(:hello, foo: 1, bar: 2)" + + bad = """ + foo(:hello, foo: 1, bar: 2) + """ + + good = """ + foo( + :hello, + foo: 1, + bar: 2 + ) + """ + + assert_format bad, good, @short_length + + bad = """ + foo(:hello, foo: 1, + bar: 2, baz: 3) + """ + + assert_format bad, """ + foo(:hello, foo: 1, bar: 2, baz: 3) + """ + end + + test "with lists maybe rewritten as keyword lists" do + assert_format "foo([foo: 1, bar: 2])", "foo(foo: 1, bar: 2)" + assert_format "foo(:arg, [foo: 1, bar: 2])", "foo(:arg, foo: 1, bar: 2)" + assert_same "foo(:arg, [:elem, foo: 1, bar: 2])" + end + + test "without parens" do + assert_same "import :foo, :bar" + assert_same "bar = if foo, do: bar, else: baz" + + assert_same """ + for :one, + :two, + :three, + fn -> + :ok + end + """ + + assert_same """ + for :one, fn -> + :ok + end + """ + end + + test "without parens on line limit" do + bad = "import :long_atom, :other_arg" + + good = """ + import :long_atom, + :other_arg + """ + + assert_format bad, good, @short_length + end + + test "without parens on comma limit" do + bad = """ + import foo(abc, cde), :next + """ + + good = """ + import foo( + abc, + cde + ), + :next + """ + + assert_format bad, good, @medium_length + end + + test "without parens and with keyword lists preserves multiline" do + assert_same """ + defstruct foo: 1, + bar: 2 + """ + + assert_same """ + config :app, + foo: 1 + """ + + assert_same """ + config :app, + foo: 1, + bar: 2 + """ + + assert_same """ + config :app, :key, + foo: 1, + bar: 2 + """ + + assert_same """ + config :app, + :key, + foo: 1, + bar: 2 + """ + + bad = """ + config :app, foo: 1, + bar: 2 + """ + + assert_format bad, """ + config :app, + foo: 1, + bar: 2 + """ + end + + test "without parens and with keyword lists on comma limit" do + bad = """ + import foo(abc, cde), opts: :next + """ + + good = """ + import foo( + abc, + cde + ), + opts: :next + """ + + assert_format bad, good, @medium_length + end + + test "without parens and with keyword lists on line limit" do + assert_same "import :atom, opts: [foo: :bar]" + + bad = "import :atom, opts: [foo: :bar]" + + good = """ + import :atom, + opts: [foo: :bar] + """ + + assert_format bad, good, @medium_length + + bad = "import :atom, really_long_key: [foo: :bar]" + + good = """ + import :atom, + really_long_key: [ + foo: :bar + ] + """ + + assert_format bad, good, @medium_length + + assert_same """ + import :foo, + one: two, + three: four, + five: [6, 7, 8, 9] + """, + @medium_length + + assert_same """ + import :really_long_atom_but_no_breaks, + one: two, + three: four + """, + @medium_length + + bad = "with :really_long_atom1, :really_long_atom2, opts: [foo: :bar]" + + good = """ + with :really_long_atom1, + :really_long_atom2, + opts: [ + foo: :bar + ] + """ + + assert_format bad, good, @medium_length + end + + test "without parens from option" do + assert_format "foo bar", "foo(bar)" + assert_same "foo bar", locals_without_parens: [foo: 1] + assert_same "foo(bar)", locals_without_parens: [foo: 1] + assert_same "foo bar", locals_without_parens: [foo: :*] + assert_same "foo(bar)", locals_without_parens: [foo: :*] + end + + test "without parens on unique argument" do + assert_same "foo(for 1, 2, 3)" + assert_same "foo(bar, for(1, 2, 3))" + assert_same "assert for 1, 2, 3" + assert_same "assert foo, for(1, 2, 3)" + + assert_same """ + assert for 1, 2, 3 do + :ok + end + """ + + assert_same """ + assert foo, for(1, 2, 3) do + :ok + end + """ + + assert_same """ + assert for(1, 2, 3) do + :ok + end + """ + + assert_same """ + assert (for 1, 2, 3 do + :ok + end) + """ + end + + test "call on call" do + assert_same "unquote(call)()" + assert_same "unquote(call)(one, two)" + + assert_same """ + unquote(call)() do + :ok + end + """ + + assert_same """ + unquote(call)(one, two) do + :ok + end + """ + end + + test "call on call on line limit" do + bad = "foo(bar)(one, two, three)" + + good = """ + foo(bar)( + one, + two, + three + ) + """ + + assert_format bad, good, @short_length + end + + test "with generators" do + assert_same "foo(bar <- baz, is_bat(bar))" + assert_same "for bar <- baz, is_bat(bar)" + + assert_same """ + foo( + bar <- baz, + is_bat(bar), + bat <- bar + ) + """ + + assert_same """ + for bar <- baz, + is_bat(bar), + bat <- bar + """ + + assert_same """ + for bar <- baz, + is_bat(bar), + bat <- bar do + :ok + end + """ + + assert_same """ + for bar <- baz, + is_bat(bar), + bat <- bar, + into: %{} + """ + end + + test "preserves user choice on parens even when it fits" do + assert_same """ + call( + :hello, + :foo, + :bar + ) + """ + + assert_same """ + call( + :hello, + :foo, + :bar + ) do + 1 + 2 + end + """ + + # Doesn't preserve this because only the ending has a newline + assert_format "call(foo, bar, baz\n)", "call(foo, bar, baz)" + + # Doesn't preserve because there are no args + bad = """ + call() do + 1 + 2 + end + """ + + assert_format bad, """ + call do + 1 + 2 + end + """ + + # Doesn't preserve because we have a single argument with next break fits + bad = """ + call( + %{ + key: :value + } + ) + """ + + assert_format bad, """ + call(%{ + key: :value + }) + """ + end + end + + describe "remote calls" do + test "with no arguments" do + assert_format "Foo . Bar . baz", "Foo.Bar.baz()" + assert_format ":erlang.\nget_stacktrace", ":erlang.get_stacktrace()" + assert_format "@foo.bar", "@foo.bar" + assert_format "@foo.bar()", "@foo.bar()" + assert_format "(@foo).bar()", "@foo.bar()" + assert_format "__MODULE__.start_link", "__MODULE__.start_link()" + assert_format "Foo.bar.baz.bong", "Foo.bar().baz.bong" + assert_format "(1 + 2).foo", "(1 + 2).foo" + assert_format "(1 + 2).foo()", "(1 + 2).foo()" + end + + test "with arguments" do + assert_format "Foo . Bar. baz(1, 2, 3)", "Foo.Bar.baz(1, 2, 3)" + assert_format ":erlang.\nget(\n:some_key)", ":erlang.get(:some_key)" + assert_format ":erlang.\nget(:some_key\n)", ":erlang.get(:some_key)" + assert_same "@foo.bar(1, 2, 3)" + assert_same "__MODULE__.start_link(1, 2, 3)" + assert_same "foo.bar(1).baz(2, 3)" + end + + test "inspects function names correctly" do + assert_same ~S[MyModule."my function"(1, 2)] + assert_same ~S[MyModule."Foo.Bar"(1, 2)] + assert_same ~S[Kernel.+(1, 2)] + assert_same ~S[:erlang.+(1, 2)] + assert_same ~S[foo."bar baz"(1, 2)] + assert_same ~S[foo."bar\nbaz"(1, 2)] + end + + test "splits on arguments and dot on line limit" do + bad = """ + MyModule.Foo.bar(:one, :two, :three) + """ + + good = """ + MyModule.Foo.bar( + :one, + :two, + :three + ) + """ + + assert_format bad, good, @medium_length + + bad = """ + My_function.foo().bar(2, 3).baz(4, 5) + """ + + good = """ + My_function.foo().bar( + 2, + 3 + ).baz(4, 5) + """ + + assert_format bad, good, @medium_length + end + + test "doesn't split on parens on empty arguments" do + assert_same "Mod.func()", @short_length + end + + test "with keyword lists" do + assert_same "mod.foo(foo: 1, bar: 2)" + + assert_same "mod.foo(:hello, foo: 1, bar: 2)" + + assert_same """ + mod.really_long_function_name( + :hello, + foo: 1, + bar: 2 + ) + """, + @short_length + + assert_same """ + really_long_module_name.foo( + :hello, + foo: 1, + bar: 2 + ) + """, + @short_length + end + + test "wraps left side in parens if it is an anonymous function" do + assert_same "(fn -> :ok end).foo" + end + + test "wraps left side in parens if it is a do-end block" do + assert_same """ + (if true do + :ok + end).foo + """ + end + + test "wraps left side in parens if it is a do-end block as an argument" do + assert_same """ + import (if true do + :ok + end).foo + """ + end + + test "call on call" do + assert_same "foo.bar(call)()" + assert_same "foo.bar(call)(one, two)" + + assert_same """ + foo.bar(call)() do + end + """ + + assert_same """ + foo.bar(call)(one, two) do + :ok + end + """ + end + + test "call on call on line limit" do + bad = "a.b(foo)(one, two, three)" + + good = """ + a.b(foo)( + one, + two, + three + ) + """ + + assert_format bad, good, @short_length + end + + test "on vars" do + assert_same "foo.bar" + assert_same "foo.bar()" + end + + test "on vars before blocks" do + assert_same """ + if var.field do + raise "oops" + end + """ + end + + test "on vars before brackets" do + assert_same """ + exception.opts[:foo] + """ + end + + test "preserves user choice on parens even when it fits" do + assert_same """ + Remote.call( + :hello, + :foo, + :bar + ) + """ + + # Doesn't preserve this because only the ending has a newline + assert_format "Remote.call(foo, bar, baz\n)", "Remote.call(foo, bar, baz)" + + assert_same """ + Remote.call( + :hello, + :foo, + fn -> :bar end + ) + """ + end + end + + describe "anonymous function calls" do + test "without arguments" do + assert_format "foo . ()", "foo.()" + assert_format "(foo.()).().()", "foo.().().()" + assert_same "@foo.()" + assert_same "(1 + 1).()" + assert_same ":foo.()" + end + + test "with arguments" do + assert_format "foo . (1, 2 , 3 )", "foo.(1, 2, 3)" + assert_format "foo . (1, 2 ).(3,4)", "foo.(1, 2).(3, 4)" + assert_same "@foo.(:one, :two)" + assert_same "foo.(1 + 1).(hello)" + end + + test "does not split on dot on line limit" do + assert_same "my_function.()", @short_length + end + + test "splits on arguments on line limit" do + bad = """ + my_function.(1, 2, 3) + """ + + good = """ + my_function.( + 1, + 2, + 3 + ) + """ + + assert_format bad, good, @short_length + + bad = """ + my_function.(1, 2).f(3, 4).(5, 6) + """ + + good = """ + my_function.( + 1, + 2 + ).f(3, 4).( + 5, + 6 + ) + """ + + assert_format bad, good, @short_length + end + + test "with keyword lists" do + assert_same "foo.(foo: 1, bar: 2)" + + assert_same "foo.(:hello, foo: 1, bar: 2)" + + assert_same """ + foo.( + :hello, + foo: 1, + bar: 2 + ) + """, + @short_length + end + + test "wraps left side in parens if it is an anonymous function" do + assert_same "(fn -> :ok end).()" + end + + test "wraps left side in parens if it is a do-end block" do + assert_same """ + (if true do + :ok + end).() + """ + end + + test "wraps left side in parens if it is a do-end block as an argument" do + assert_same """ + import (if true do + :ok + end).() + """ + end + + test "preserves user choice on parens even when it fits" do + assert_same """ + call.( + :hello, + :foo, + :bar + ) + """ + + # Doesn't preserve this because only the ending has a newline + assert_format "call.(foo, bar, baz\n)", "call.(foo, bar, baz)" + end + end + + describe "do-end blocks" do + test "with non-block keywords" do + assert_same "foo(do: nil)" + end + + test "with forced block keywords" do + good = """ + foo do + nil + end + """ + + assert_format "foo(do: nil)", good, force_do_end_blocks: true + + # Avoid false positives + assert_same "foo(do: 1, do: 2)", force_do_end_blocks: true + assert_same "foo(do: 1, another: 2)", force_do_end_blocks: true + end + + test "with multiple keywords" do + assert_same """ + foo do + :do + rescue + :rescue + catch + :catch + else + :else + after + :after + end + """ + end + + test "with multiple keywords and arrows" do + assert_same """ + foo do + a1 -> a2 + b1 -> b2 + rescue + a1 -> a2 + b1 -> b2 + catch + a1 -> a2 + b1 -> b2 + else + a1 -> a2 + b1 -> b2 + after + a1 -> a2 + b1 -> b2 + end + """ + end + + test "with no extra arguments" do + assert_same """ + foo do + :ok + end + """ + end + + test "with no extra arguments and line breaks" do + assert_same """ + foo do + a1 -> + really_long_line + + b1 -> + b2 + rescue + c1 + catch + d1 -> d1 + e1 -> e1 + else + f2 + after + g1 -> + really_long_line + + h1 -> + h2 + end + """, + @medium_length + end + + test "with extra arguments" do + assert_same """ + foo bar, baz do + :ok + end + """ + end + + test "with extra arguments and line breaks" do + assert_same """ + foo bar do + a1 -> + really_long_line + + b1 -> + b2 + rescue + c1 + catch + d1 -> d1 + e1 -> e1 + else + f2 + after + g1 -> + really_long_line + + h1 -> + h2 + end + """, + @medium_length + + assert_same """ + foo really, + long, + list, + of, + arguments do + a1 -> + really_long_line + + b1 -> + b2 + rescue + c1 + catch + d1 -> d1 + e1 -> e1 + else + f2 + after + g1 -> + really_long_line + + h1 -> + h2 + end + """, + @medium_length + end + + test "when empty" do + assert_same """ + foo do + end + """ + + assert_same """ + foo do + rescue + catch + else + after + end + """ + end + + test "inside call" do + bad = "foo (bar do :ok end)" + + good = """ + foo( + bar do + :ok + end + ) + """ + + assert_format bad, good + + bad = "import (bar do :ok end)" + + good = """ + import (bar do + :ok + end) + """ + + assert_format bad, good + end + + test "inside operator" do + bad = "foo + bar do :ok end" + + good = """ + foo + + bar do + :ok + end + """ + + assert_format bad, good + end + + test "inside operator inside argument" do + bad = "fun foo + (bar do :ok end)" + + good = """ + fun( + foo + + bar do + :ok + end + ) + """ + + assert_format bad, good + + bad = "if foo + (bar do :ok end) do :ok end" + + good = """ + if foo + + (bar do + :ok + end) do + :ok + end + """ + + assert_format bad, good + end + + test "inside operator inside argument with remote call" do + bad = "if foo + (Bar.baz do :ok end) do :ok end" + + good = """ + if foo + + (Bar.baz do + :ok + end) do + :ok + end + """ + + assert_format bad, good + end + + test "keeps repeated keys" do + assert_same """ + receive do + :ok + after + 0 -> 1 + after + 2 -> 3 + end + """ + end + + test "preserves user choice even when it fits" do + assert_same """ + case do + 1 -> + :ok + + 2 -> + :ok + end + """ + + assert_same """ + case do + 1 -> + :ok + + 2 -> + :ok + + 3 -> + :ok + end + """ + end + end + + describe "tuple calls" do + test "without arguments" do + assert_format "foo . {}", "foo.{}" + end + + test "with arguments" do + assert_format "foo.{bar,baz,bat,}", "foo.{bar, baz, bat}" + end + + test "with arguments on line limit" do + bad = "foo.{bar,baz,bat,}" + + good = """ + foo.{ + bar, + baz, + bat + } + """ + + assert_format bad, good, @short_length + + bad = "really_long_expression.{bar,baz,bat,}" + + good = """ + really_long_expression.{ + bar, + baz, + bat + } + """ + + assert_format bad, good, @short_length + end + + test "with keywords" do + assert_same "expr.{:hello, foo: bar, baz: bat}" + end + + test "preserves user choice on parens even when it fits" do + assert_same """ + call.{ + :hello, + :foo, + :bar + } + """ + + assert_format "call.{foo, bar, baz\n}", "call.{foo, bar, baz}" + end + end + + describe "access" do + test "with one argument" do + assert_format "foo[ bar ]", "foo[bar]" + end + + test "with arguments on line limit" do + bad = "foo[really_long_argument()]" + + good = """ + foo[ + really_long_argument() + ] + """ + + assert_format bad, good, @short_length + + bad = "really_long_expression[really_long_argument()]" + + good = """ + really_long_expression[ + really_long_argument() + ] + """ + + assert_format bad, good, @short_length + end + + test "with do-end blocks" do + assert_same """ + (if true do + false + end)[key] + """ + end + + test "with keywords" do + assert_format "expr[[]]", "expr[[]]" + assert_format "expr[foo: bar, baz: bat]", "expr[foo: bar, baz: bat]" + assert_format "expr[[foo: bar, baz: bat]]", "expr[[foo: bar, baz: bat]]" + end + end +end diff --git a/lib/elixir/test/elixir/code_formatter/comments_test.exs b/lib/elixir/test/elixir/code_formatter/comments_test.exs new file mode 100644 index 00000000000..c5f8c8e8e2c --- /dev/null +++ b/lib/elixir/test/elixir/code_formatter/comments_test.exs @@ -0,0 +1,1553 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Code.Formatter.CommentsTest do + use ExUnit.Case, async: true + + import CodeFormatterHelpers + + @short_length [line_length: 10] + + describe "at the root" do + test "for empty documents" do + assert_same "# hello world" + end + + test "are reformatted" do + assert_format "#oops", "# oops" + assert_format "##oops", "## oops" + assert_same "# ## oops" + end + + test "recognizes hashbangs" do + assert_format "#! /usr/bin/env elixir", "#! /usr/bin/env elixir" + assert_format "#!/usr/bin/env elixir", "#!/usr/bin/env elixir" + assert_same "#!" + end + + test "before and after expressions" do + assert_same """ + # before comment + :hello + """ + + assert_same """ + :hello + # after comment + """ + + assert_same """ + # before comment + :hello + # after comment + """ + end + + test "on expressions" do + bad = """ + :hello # this is hello + :world # this is world + """ + + good = """ + # this is hello + :hello + # this is world + :world + """ + + assert_format bad, good + + bad = """ + foo # this is foo + ++ bar # this is bar + ++ baz # this is baz + """ + + good = """ + # this is foo + # this is bar + # this is baz + foo ++ + bar ++ + baz + """ + + assert_format bad, good, @short_length + end + + test "empty comment" do + assert_same """ + # + :foo + """ + end + + test "before and after expressions with newlines" do + assert_same """ + # before comment + # second line + + :hello + + # middle comment 1 + + # + + # middle comment 2 + + :world + + # after comment + # second line + """ + end + end + + describe "modules attributes" do + test "with comments around" do + assert_same """ + defmodule Sample do + # Comment 0 + @moduledoc false + # Comment 1 + + # Comment 2 + @attr1 1 + # Comment 3 + + # Comment 4 + @doc "Doc" + # Comment 5 + @attr2 2 + # Comment 6 + def sample, do: :sample + end + """ + end + + test "with comments only after" do + assert_same """ + @moduledoc false + # Comment 1 + + @attr 1 + """ + end + + test "with too many new lines" do + bad = """ + defmodule Sample do + + # Comment 0 + + + @moduledoc false + + + # Comment 1 + + + # Comment 2 + + + @attr1 1 + + + # Comment 3 + + + # Comment 4 + + + @doc "Doc" + + + # Comment 5 + + + @attr2 2 + + + # Comment 6 + + + def sample, do: :sample + end + """ + + assert_format bad, """ + defmodule Sample do + # Comment 0 + + @moduledoc false + + # Comment 1 + + # Comment 2 + + @attr1 1 + + # Comment 3 + + # Comment 4 + + @doc "Doc" + + # Comment 5 + + @attr2 2 + + # Comment 6 + + def sample, do: :sample + end + """ + end + end + + describe "interpolation" do + test "with comment outside before, during and after" do + assert_same ~S""" + # comment + IO.puts("Hello #{world}") + """ + + assert_same ~S""" + IO.puts("Hello #{world}") + # comment + """ + end + + test "with trailing comments" do + # This is trailing so we move the comment out + trailing = ~S""" + IO.puts("Hello #{world}") # comment + """ + + assert_format trailing, ~S""" + # comment + IO.puts("Hello #{world}") + """ + + # This is ambiguous so we move the comment out + ambiguous = ~S""" + IO.puts("Hello #{world # comment + }") + """ + + assert_format ambiguous, ~S""" + # comment + IO.puts("Hello #{world}") + """ + end + + test "with comment inside before and after" do + bad = ~S""" + IO.puts( + "Hello #{ + # comment + world + }" + ) + """ + + good = ~S""" + IO.puts( + # comment + "Hello #{world}" + ) + """ + + assert_format bad, good + + bad = ~S""" + IO.puts( + "Hello #{ + world + # comment + }" + ) + """ + + good = ~S""" + IO.puts( + "Hello #{world}" + # comment + ) + """ + + assert_format bad, good + + bad = ~S""" + IO.puts("Hello #{hello + world}") + """ + + good = ~S""" + IO.puts( + "Hello #{hello + world}" + ) + """ + + assert_format bad, good + end + end + + describe "parens blocks" do + test "with comment outside before and after" do + assert_same ~S""" + # comment + assert ( + hello + world + ) + """ + + assert_same ~S""" + assert ( + hello + world + ) + + # comment + """ + end + + test "with trailing comments" do + # This is ambiguous so we move the comment out + ambiguous = ~S""" + assert ( # comment + hello + world + ) + """ + + assert_format ambiguous, ~S""" + # comment + assert ( + hello + world + ) + """ + + # This is ambiguous so we move the comment out + ambiguous = ~S""" + assert ( + hello + world + ) # comment + """ + + assert_format ambiguous, ~S""" + assert ( + hello + world + ) + + # comment + """ + end + + test "with comment inside before and after" do + assert_same ~S""" + assert ( + # comment + hello + world + ) + """ + + assert_same ~S""" + assert ( + hello + world + # comment + ) + """ + end + end + + describe "access" do + test "before and after single arg" do + assert_same ~S""" + foo[ + # bar + baz + # bat + ] + """ + end + + test "before and after keywords" do + assert_same ~S""" + foo[ + # bar + one: :two, + # baz + three: :four + # bat + ] + """ + end + end + + describe "calls" do + test "local with parens inside before and after" do + assert_same ~S""" + call( + # before + hello, + # middle + world + # after + ) + """ + + assert_same ~S""" + call( + # command + ) + """ + end + + test "remote with parens inside before and after" do + assert_same ~S""" + Remote.call( + # before + hello, + # middle + world + # after + ) + """ + + assert_same ~S""" + Remote.call( + # command + ) + """ + end + + test "local with parens and keywords inside before and after" do + assert_same ~S""" + call( + # before + hello, + # middle + world, + # key before + key: hello, + # key middle + key: world + # key after + ) + """ + end + + test "remote with parens and keywords inside before and after" do + assert_same ~S""" + call( + # before + hello, + # middle + world, + # key before + key: hello, + # key middle + key: world + # key after + ) + """ + end + + test "local with no parens inside before and after" do + bad = ~S""" + # before + assert hello, + # middle + world + # after + """ + + assert_format bad, ~S""" + # before + assert hello, + # middle + world + + # after + """ + end + + test "local with no parens and keywords inside before and after" do + bad = ~S""" + config hello, world, + # key before + key: hello, + # key middle + key: world + # key after + """ + + assert_format bad, ~S""" + config hello, world, + # key before + key: hello, + # key middle + key: world + + # key after + """ + + bad = ~S""" + # before + config hello, + # middle + world, + # key before + key: hello, + # key middle + key: world + # key after + """ + + assert_format bad, ~S""" + # before + config hello, + # middle + world, + # key before + key: hello, + # key middle + key: world + + # key after + """ + end + end + + describe "anonymous functions" do + test "with one clause and no args" do + assert_same ~S""" + fn -> + # comment + hello + world + end + """ + + assert_same ~S""" + fn -> + hello + world + # comment + end + """ + end + + test "with one clause and no args and trailing comments" do + bad = ~S""" + fn # comment + -> + hello + world + end + """ + + assert_format bad, ~S""" + # comment + fn -> + hello + world + end + """ + + bad = ~S""" + fn + # comment + -> + hello + world + end + """ + + assert_format bad, ~S""" + # comment + fn -> + hello + world + end + """ + end + + test "with one clause and args" do + assert_same ~S""" + fn hello -> + # before + hello + # middle + world + # after + end + """ + end + + test "with one clause and args and trailing comments" do + bad = ~S""" + fn # fn + # before head + hello # middle head + # after head + -> + # before body + world # middle body + # after body + end + """ + + assert_format bad, ~S""" + # fn + fn + # before head + # middle head + hello -> + # after head + # before body + # middle body + world + # after body + end + """ + end + + test "with multiple clauses and args" do + bad = ~S""" + fn # fn + # before one + one, # middle one + # after one / before two + two # middle two + # after two + -> + # before hello + hello # middle hello + # after hello + + # before three + three # middle three + # after three + -> + # before world + world # middle world + # after world + end + """ + + assert_format bad, ~S""" + # fn + fn + # before one + # middle one + # after one / before two + # middle two + one, two -> + # after two + # before hello + # middle hello + hello + + # after hello + + # before three + # middle three + three -> + # after three + # before world + # middle world + world + # after world + end + """ + end + + test "with commented out clause" do + assert_same """ + fn + arg1 -> + body1 + + # arg2 -> + # body 2 + + arg3 -> + body3 + end + """ + end + end + + describe "do-end blocks" do + test "with comment outside before and after" do + assert_same ~S""" + # comment + assert do + hello + world + end + """ + + assert_same ~S""" + assert do + hello + world + end + + # comment + """ + end + + test "with trailing comments" do + # This is ambiguous so we move the comment out + ambiguous = ~S""" + assert do # comment + hello + world + end + """ + + assert_format ambiguous, ~S""" + # comment + assert do + hello + world + end + """ + + # This is ambiguous so we move the comment out + ambiguous = ~S""" + assert do + hello + world + end # comment + """ + + assert_format ambiguous, ~S""" + assert do + hello + world + end + + # comment + """ + end + + test "with comment inside before and after" do + assert_same ~S""" + assert do + # comment + hello + world + end + """ + + assert_same ~S""" + assert do + hello + world + # comment + end + """ + end + + test "with comment inside before and after and multiple keywords" do + assert_same ~S""" + assert do + # before + hello + world + # after + rescue + # before + hello + world + # after + after + # before + hello + world + # after + catch + # before + hello + world + # after + else + # before + hello + world + # after + end + """ + end + + test "when empty" do + assert_same ~S""" + assert do + # comment + end + """ + + assert_same ~S""" + assert do + # comment + rescue + # comment + after + # comment + catch + # comment + else + # comment + end + """ + end + + test "with one-line clauses" do + bad = ~S""" + assert do # do + # before + one -> two + end + """ + + assert_format bad, ~S""" + # do + assert do + # before + one -> two + end + """ + + bad = ~S""" + assert do # do + # before + one -> two + # after + three -> four + end + """ + + assert_format bad, ~S""" + # do + assert do + # before + one -> two + # after + three -> four + end + """ + end + + test "with multiple clauses and args" do + bad = ~S""" + assert do # do + # before one + one, # middle one + # after one / before two + two # middle two + # after two + -> + # before hello + hello # middle hello + # after hello + + # before three + three # middle three + # after three + -> + # before world + world # middle world + # after world + end + """ + + assert_format bad, ~S""" + # do + assert do + # before one + # middle one + # after one / before two + # middle two + one, two -> + # after two + # before hello + # middle hello + hello + + # after hello + + # before three + # middle three + three -> + # after three + # before world + # middle world + world + # after world + end + """ + end + end + + describe "operators" do + test "with comment before, during and after uniform pipelines" do + assert_same """ + foo + # |> bar + # |> baz + |> bat + """ + + bad = """ + # before + foo # this is foo + |> bar # this is bar + |> baz # this is baz + # after + """ + + good = """ + # before + # this is foo + foo + # this is bar + |> bar + # this is baz + |> baz + + # after + """ + + assert_format bad, good, @short_length + end + + test "with comment before, during and after mixed pipelines" do + assert_same """ + foo + # |> bar + # |> baz + ~> bat + """ + + bad = """ + # before + foo # this is foo + ~> bar # this is bar + <~> baz # this is baz + # after + """ + + good = """ + # before + # this is foo + foo + # this is bar + ~> bar + # this is baz + <~> baz + + # after + """ + + assert_format bad, good, @short_length + end + + test "with comment before, during and after uniform right" do + assert_same """ + foo + # | bar + # | baz + | bat + """ + + bad = """ + # before + foo # this is foo + | bar # this is bar + | baz # this is baz + # after + """ + + good = """ + # before + # this is foo + foo + # this is bar + | bar + # this is baz + | baz + + # after + """ + + assert_format bad, good, @short_length + end + + test "with comment before, during and after mixed right" do + assert_same """ + one + # when two + # when three + when four + # | five + | six + """ + end + + test "handles nodes without meta info" do + assert_same "(a -> b) |> (c -> d)" + assert_same "(a -> b) when c: d" + assert_same "(a -> b) when (c -> d)" + end + end + + describe "containers" do + test "with comment outside before, during and after" do + assert_same ~S""" + # comment + [one, two, three] + """ + + assert_same ~S""" + [one, two, three] + # comment + """ + end + + test "with trailing comments" do + # This is trailing so we move the comment out + trailing = ~S""" + [one, two, three] # comment + """ + + assert_format trailing, ~S""" + # comment + [one, two, three] + """ + + # This is ambiguous so we move the comment out + ambiguous = ~S""" + [# comment + one, two, three] + """ + + assert_format ambiguous, ~S""" + # comment + [ + one, + two, + three + ] + """ + end + + test "when empty" do + assert_same ~S""" + [ + # comment + ] + """ + end + + test "with block" do + assert_same ~S""" + [ + ( + # before + multi + line + # after + ) + ] + """ + end + + test "with comments inside lists before and after" do + bad = ~S""" + [ + # 1. one + + # 1. two + # 1. three + one, + after_one, + after_one do + :ok + end, + + # 2. one + + # 2. two + # 2. three + # two, + + # 3. one + + # 3. two + # 3. three + three # final + + # 4. one + + # 4. two + # 4. three + # four + ] + """ + + good = ~S""" + [ + # 1. one + + # 1. two + # 1. three + one, + after_one, + after_one do + :ok + end, + + # 2. one + + # 2. two + # 2. three + # two, + + # 3. one + + # 3. two + # 3. three + # final + three + + # 4. one + + # 4. two + # 4. three + # four + ] + """ + + assert_format bad, good + end + + test "with comments inside tuples before and after" do + bad = ~S""" + { + # 1. one + + # 1. two + # 1. three + one, + after_one, + after_one do + :ok + end, + + # 2. one + + # 2. two + # 2. three + # two, + + # 3. one + + # 3. two + # 3. three + three # final + + # 4. one + + # 4. two + # 4. three + # four + } + """ + + good = ~S""" + { + # 1. one + + # 1. two + # 1. three + one, + after_one, + after_one do + :ok + end, + + # 2. one + + # 2. two + # 2. three + # two, + + # 3. one + + # 3. two + # 3. three + # final + three + + # 4. one + + # 4. two + # 4. three + # four + } + """ + + assert_format bad, good + end + + test "with comments inside bitstrings before and after" do + bad = ~S""" + << + # 1. one + + # 1. two + # 1. three + one, + after_one, + after_one do + :ok + end, + + # 2. one + + # 2. two + # 2. three + # two, + + # 3. one + + # 3. two + # 3. three + three # final + + # 4. one + + # 4. two + # 4. three + # four + >> + """ + + good = ~S""" + << + # 1. one + + # 1. two + # 1. three + one, + after_one, + after_one do + :ok + end, + + # 2. one + + # 2. two + # 2. three + # two, + + # 3. one + + # 3. two + # 3. three + # final + three + + # 4. one + + # 4. two + # 4. three + # four + >> + """ + + assert_format bad, good + end + + test "with comments inside maps before and after" do + bad = ~S""" + %{ + # 1. one + + # 1. two + # 1. three + one: one, + one: after_one, + one: after_one do + :ok + end, + + # 2. one + + # 2. two + # 2. three + # two, + + # 3. one + + # 3. two + # 3. three + three: three # final + + # 4. one + + # 4. two + # 4. three + # four + } + """ + + good = ~S""" + %{ + # 1. one + + # 1. two + # 1. three + one: one, + one: after_one, + one: + after_one do + :ok + end, + + # 2. one + + # 2. two + # 2. three + # two, + + # 3. one + + # 3. two + # 3. three + # final + three: three + + # 4. one + + # 4. two + # 4. three + # four + } + """ + + assert_format bad, good + end + + test "with comments inside structs before and after" do + bad = ~S""" + %Foo{bar | + # 1. one + + # 1. two + # 1. three + one: one, + one: after_one, + one: after_one do + :ok + end, + + # 2. one + + # 2. two + # 2. three + # two, + + # 3. one + + # 3. two + # 3. three + three: three # final + + # 4. one + + # 4. two + # 4. three + # four + } + """ + + good = ~S""" + %Foo{ + bar + | # 1. one + + # 1. two + # 1. three + one: one, + one: after_one, + one: + after_one do + :ok + end, + + # 2. one + + # 2. two + # 2. three + # two, + + # 3. one + + # 3. two + # 3. three + # final + three: three + + # 4. one + + # 4. two + # 4. three + # four + } + """ + + assert_format bad, good + end + end + + describe "defstruct" do + test "with first field comments" do + bad = ~S""" + defmodule Foo do + # defstruct + defstruct [ # foo + # 1. one + one: 1, # 2. one + # 1. two + # 2. two + two: 2 + ] + end + """ + + good = ~S""" + defmodule Foo do + # defstruct + # foo + defstruct [ + # 1. one + # 2. one + one: 1, + # 1. two + # 2. two + two: 2 + ] + end + """ + + assert_format bad, good + end + + test "with first field comments and defstruct has the parens" do + bad = ~S""" + defmodule Foo do + # defstruct + defstruct([ # foo + # 1. one + one: 1, # 2. one + # 1. two + # 2. two + two: 2 + ]) + end + """ + + good = ~S""" + defmodule Foo do + # defstruct + # foo + defstruct( + # 1. one + # 2. one + one: 1, + # 1. two + # 2. two + two: 2 + ) + end + """ + + assert_format bad, good + end + + test "without first field comments" do + bad = ~S""" + defmodule Foo do + # defstruct + defstruct [ + one: 1, + # 1. two + two: 2 # 2. two + ] + end + """ + + good = ~S""" + defmodule Foo do + # defstruct + defstruct one: 1, + # 1. two + # 2. two + two: 2 + end + """ + + assert_format bad, good + end + + test "without field comments" do + bad = ~S""" + defmodule Foo do + # defstruct + defstruct [ + one: 1, + two: 2 + ] + end + """ + + good = ~S""" + defmodule Foo do + # defstruct + defstruct one: 1, + two: 2 + end + """ + + assert_format bad, good + end + + test "without square brackets" do + assert_same ~S""" + defmodule Foo do + # defstruct + defstruct one: 1, + # 1. two + # 2. two + two: 2 + end + """ + end + end +end diff --git a/lib/elixir/test/elixir/code_formatter/containers_test.exs b/lib/elixir/test/elixir/code_formatter/containers_test.exs new file mode 100644 index 00000000000..a62c1513403 --- /dev/null +++ b/lib/elixir/test/elixir/code_formatter/containers_test.exs @@ -0,0 +1,664 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Code.Formatter.ContainersTest do + use ExUnit.Case, async: true + + import CodeFormatterHelpers + + @short_length [line_length: 10] + @medium_length [line_length: 20] + + describe "tuples" do + test "without arguments" do + assert_format "{ }", "{}" + end + + test "with arguments" do + assert_format "{1,2}", "{1, 2}" + assert_format "{1,2,3}", "{1, 2, 3}" + end + + test "is flex on line limits" do + bad = "{1, 2, 3, 4}" + + good = """ + {1, 2, 3, + 4} + """ + + assert_format bad, good, @short_length + end + + test "removes trailing comma" do + assert_format "{1,}", "{1}" + assert_format "{1, 2, 3,}", "{1, 2, 3}" + end + + test "with keyword lists" do + # The one below is not valid syntax + # assert_same "{foo: 1, bar: 2}" + assert_same "{:hello, foo: 1, bar: 2}" + + tuple = """ + { + :hello, + foo: 1, + bar: 2 + } + """ + + assert_same tuple, @short_length + end + + test "preserves user choice even when it fits" do + assert_same """ + { + :hello, + :foo, + :bar + } + """ + + # Doesn't preserve this because only the ending has a newline + assert_format "{foo, bar, baz\n}", "{foo, bar, baz}" + end + + test "preserves user choice even when it fits with trailing comma" do + bad = """ + { + :hello, + :foo, + :bar, + } + """ + + assert_format bad, """ + { + :hello, + :foo, + :bar + } + """ + end + end + + describe "lists" do + test "empty" do + assert_format "[ ]", "[]" + assert_format "[\n]", "[]" + end + + test "with elements" do + assert_format "[ 1 , 2,3, 4 ]", "[1, 2, 3, 4]" + end + + test "with tail" do + assert_format "[1,2,3|4]", "[1, 2, 3 | 4]" + end + + test "are strict on line limit" do + bad = """ + [11, 22, 33, 44] + """ + + good = """ + [ + 11, + 22, + 33, + 44 + ] + """ + + assert_format bad, good, @short_length + + bad = """ + [11, 22, 33 | 44] + """ + + good = """ + [ + 11, + 22, + 33 | 44 + ] + """ + + assert_format bad, good, @short_length + + bad = """ + [1, 2, 3 | 4] + """ + + good = """ + [ + 1, + 2, + 3 | 4 + ] + """ + + assert_format bad, good, @short_length + + bad = """ + [1, 2, 3 | really_long_expression()] + """ + + good = """ + [ + 1, + 2, + 3 + | really_long_expression() + ] + """ + + assert_format bad, good, @short_length + end + + test "removes trailing comma" do + assert_format "[1,]", "[1]" + assert_format "[1, 2, 3,]", "[1, 2, 3]" + end + + test "with keyword lists" do + assert_same "[foo: 1, bar: 2]" + assert_same "[:hello, foo: 1, bar: 2]" + + # Pseudo keyword lists are kept as is + assert_same "[{:foo, 1}, {:bar, 2}]" + + keyword = """ + [ + foo: 1, + bar: 2 + ] + """ + + assert_same keyword, @short_length + end + + test "with keyword lists on comma line limit" do + bad = """ + [ + foooo: 1, + barrr: 2 + ] + """ + + good = """ + [ + foooo: + 1, + barrr: 2 + ] + """ + + assert_format bad, good, @short_length + end + + test "with quoted keyword lists" do + assert_same ~S(["with spaces": 1]) + assert_same ~S(["one #{two} three": 1]) + assert_same ~S(["\w": 1, "\\w": 2]) + assert_same ~S(["Elixir.Foo": 1, "Elixir.Bar": 2]) + assert_format ~S(["Foo": 1, "Bar": 2]), ~S([Foo: 1, Bar: 2]) + assert_same ~S(["with \"scare quotes\"": 1]) + end + + test "with operators keyword lists" do + assert_same ~S([.: :.]) + assert_same ~S([..: :..]) + assert_same ~S([...: :...]) + end + + test "preserves user choice even when it fits" do + assert_same """ + [ + :hello, + :foo, + :bar + ] + """ + + # Doesn't preserve this because only the ending has a newline + assert_format "[foo, bar, baz\n]", "[foo, bar, baz]" + end + + test "preserves user choice even when it fits with trailing comma" do + bad = """ + [ + :hello, + :foo, + :bar, + ] + """ + + assert_format bad, """ + [ + :hello, + :foo, + :bar + ] + """ + end + end + + describe "bitstrings" do + test "without arguments" do + assert_format "<< >>", "<<>>" + assert_format "<<\n>>", "<<>>" + end + + test "with arguments" do + assert_format "<<1,2,3>>", "<<1, 2, 3>>" + end + + test "add parens on first and last in case of binary ambiguity" do + assert_format "<< <<>> >>", "<<(<<>>)>>" + assert_format "<< <<>> + <<>> >>", "<<(<<>> + <<>>)>>" + assert_format "<< 1 + <<>> >>", "<<(1 + <<>>)>>" + assert_format "<< <<>> + 1 >>", "<<(<<>> + 1)>>" + assert_format "<< <<>>, <<>>, <<>> >>", "<<(<<>>), <<>>, (<<>>)>>" + assert_format "<< <<>>::1, <<>>::2, <<>>::3 >>", "<<(<<>>)::1, <<>>::2, <<>>::3>>" + assert_format "<< <<>>::<<>> >>", "<<(<<>>)::(<<>>)>>" + end + + test "add parens on first in case of operator ambiguity" do + assert_format "<< ~~~1::8 >>", "<<(~~~1)::8>>" + assert_format "<< ~s[foo]::binary >>", "<<(~s[foo])::binary>>" + end + + test "with modifiers" do + assert_format "<< 1 :: 1 >>", "<<1::1>>" + assert_format "<< 1 :: 2 + 3 >>", "<<1::(2 + 3)>>" + assert_format "<< 1 :: 2 - integer >>", "<<1::2-integer>>" + assert_format "<< 1 :: 2 - unit(3) >>", "<<1::2-unit(3)>>" + assert_format "<< 1 :: 2 * 3 - unit(4) >>", "<<1::2*3-unit(4)>>" + assert_format "<< 1 :: 2 - unit(3) - 4 / 5 >>", "<<1::2-unit(3)-(4 / 5)>>" + assert_format "<<0 :: ( x - 1 ) * 5>>", "<<0::(x-1)*5>>" + assert_format "<<0 :: 2 * 3 * 4>>", "<<0::(2*3)*4>>" + end + + test "in comprehensions" do + assert_format "<< 0, 1 :: 1 <- x >>", "<<0, 1::1 <- x>>" + assert_format "<< 0, 1 :: 2 + 3 <- x >>", "<<0, 1::(2 + 3) <- x>>" + assert_format "<< 0, 1 :: 2 - integer <- x >>", "<<0, 1::2-integer <- x>>" + assert_format "<< 0, 1 :: 2 - unit(3) <- x >>", "<<0, 1::2-unit(3) <- x>>" + assert_format "<< 0, 1 :: 2 * 3 - unit(4) <- x >>", "<<0, 1::2*3-unit(4) <- x>>" + assert_format "<< 0, 1 :: 2 - unit(3) - 4 / 5 <- x >>", "<<0, 1::2-unit(3)-(4 / 5) <- x>>" + + assert_same "<<(<> <- <>)>>" + assert_same "<<(y <- <>)>>" + assert_same "<<(<> <- x)>>" + end + + test "keeps parentheses by default" do + assert_same "<>" + assert_same "<>" + + assert_same "<>" + assert_same "<>" + + assert_same "<>" + assert_same "<<0, 1::2-integer() <- x>>" + end + + test "is flex on line limits" do + bad = "<<1, 2, 3, 4>>" + + good = """ + <<1, 2, 3, + 4>> + """ + + assert_format bad, good, @short_length + end + + test "preserves user choice even when it fits" do + assert_same """ + << + :hello, + :foo, + :bar + >> + """ + + # Doesn't preserve this because only the ending has a newline + assert_format "<>", "<>" + end + + test "preserves user choice even when it fits with trailing comma" do + bad = """ + << + :hello, + :foo, + :bar, + >> + """ + + assert_format bad, """ + << + :hello, + :foo, + :bar + >> + """ + end + end + + describe "maps" do + test "without arguments" do + assert_format "%{ }", "%{}" + end + + test "with arguments" do + assert_format "%{1 => 2,3 => 4}", "%{1 => 2, 3 => 4}" + end + + test "is strict on line limits" do + bad = "%{1 => 2, 3 => 4}" + + good = """ + %{ + 1 => 2, + 3 => 4 + } + """ + + assert_format bad, good, @short_length + + map = """ + %{ + a(1, 2) => b, + c(3, 4) => d + } + """ + + assert_same map, @medium_length + + map = """ + %{ + a => fn x -> + y + end, + b => fn y -> + z + end + } + """ + + assert_same map, @medium_length + + map = """ + %{ + a => + for( + y <- x, + z <- y, + do: 123 + ) + } + """ + + assert_same map, @medium_length + + map = """ + %{ + a => + for do + :ok + end + } + """ + + assert_same map, @short_length + end + + test "removes trailing comma" do + assert_format "%{1 => 2,}", "%{1 => 2}" + end + + test "with keyword lists" do + assert_same "%{:foo => :bar, baz: :bat}" + + map = """ + %{ + :foo => :bar, + baz: :bat + } + """ + + assert_same map, @medium_length + end + + test "preserves user choice in regards to =>" do + assert_same "%{:hello => 1, :world => 2}" + assert_format "%{:true => 1, :false => 2}", "%{true => 1, false => 2}" + end + + test "preserves user choice even when it fits" do + assert_same """ + %{ + :hello => 1, + :foo => 2, + :bar => 3 + } + """ + + # Doesn't preserve this because only the ending has a newline + assert_format "%{foo: 1, bar: 2\n}", "%{foo: 1, bar: 2}" + end + + test "preserves user choice even when it fits with trailing comma" do + bad = """ + %{ + hello, + foo, + bar, + } + """ + + assert_format bad, """ + %{ + hello, + foo, + bar + } + """ + end + + test "preserves user choice when a newline is used after keyword" do + good = """ + %{ + hello: + {:ok, :world} + } + """ + + assert_same good, @medium_length + end + + test "preserves user choice when a newline is used after assoc" do + good = """ + %{ + hello => + {:ok, :world} + } + """ + + assert_same good, @medium_length + end + end + + describe "maps with update" do + test "with arguments" do + assert_format "%{foo | 1 => 2,3 => 4}", "%{foo | 1 => 2, 3 => 4}" + end + + test "is strict on line limits" do + bad = "%{foo | 1 => 2, 3 => 4}" + + good = """ + %{ + foo + | 1 => 2, + 3 => 4 + } + """ + + assert_format bad, good, line_length: 11 + end + + test "removes trailing comma" do + assert_format "%{foo | 1 => 2,}", "%{foo | 1 => 2}" + end + + test "with keyword lists" do + assert_same "%{foo | :foo => :bar, baz: :bat}" + + map = """ + %{ + foo + | :foo => :bar, + baz: :bat + } + """ + + assert_same map, @medium_length + end + + test "preserves user choice even when it fits" do + assert_same """ + %{ + foo + | :hello => 1, + :foo => 2, + :bar => 3 + } + """ + end + + test "wraps operators in parens" do + assert_format "%{foo && bar | baz: :bat}", "%{(foo && bar) | baz: :bat}" + assert_same "%{@foo | baz: :bat}" + end + end + + describe "structs" do + test "without arguments" do + assert_format "%struct{ }", "%struct{}" + end + + test "with arguments" do + assert_format "%struct{1 => 2,3 => 4}", "%struct{1 => 2, 3 => 4}" + end + + test "is strict on line limits" do + bad = "%struct{1 => 2, 3 => 4}" + + good = """ + %struct{ + 1 => 2, + 3 => 4 + } + """ + + assert_format bad, good, @short_length + end + + test "removes trailing comma" do + assert_format "%struct{1 => 2,}", "%struct{1 => 2}" + end + + test "with keyword lists" do + assert_same "%struct{:foo => :bar, baz: :bat}" + + struct = """ + %struct{ + :foo => :bar, + baz: :bat + } + """ + + assert_same struct, @medium_length + end + + test "preserves user choice even when it fits" do + assert_same """ + %Foo{ + :hello => 1, + :foo => 2, + :bar => 3 + } + """ + end + end + + describe "struct with update" do + test "with arguments" do + assert_format "%struct{foo | 1 => 2,3 => 4}", "%struct{foo | 1 => 2, 3 => 4}" + end + + test "is strict on line limits" do + bad = "%struct{foo | 1 => 2, 3 => 4}" + + good = """ + %struct{ + foo + | 1 => 2, + 3 => 4 + } + """ + + assert_format bad, good, line_length: 11 + end + + test "removes trailing comma" do + assert_format "%struct{foo | 1 => 2,}", "%struct{foo | 1 => 2}" + end + + test "with keyword lists" do + assert_same "%struct{foo | :foo => :bar, baz: :bat}" + + struct = """ + %struct{ + foo + | :foo => :bar, + baz: :bat + } + """ + + assert_same struct, @medium_length + end + + test "preserves user choice even when it fits" do + assert_same """ + %Foo{ + foo + | :hello => 1, + :foo => 2, + :bar => 3 + } + """ + end + + test "converges" do + bad = "hello_world(%struct{foo | 1 => 2, 3 => 4})" + + good = """ + hello_world(%struct{ + foo + | 1 => 2, + 3 => 4 + }) + """ + + assert_format bad, good, line_length: 30 + end + end +end diff --git a/lib/elixir/test/elixir/code_formatter/general_test.exs b/lib/elixir/test/elixir/code_formatter/general_test.exs new file mode 100644 index 00000000000..3c91e5fba5c --- /dev/null +++ b/lib/elixir/test/elixir/code_formatter/general_test.exs @@ -0,0 +1,947 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Code.Formatter.GeneralTest do + use ExUnit.Case, async: true + + import CodeFormatterHelpers + + @short_length [line_length: 10] + + test "does not emit warnings" do + assert_format "fn -> end", "fn -> nil end" + end + + describe "unicode normalization" do + test "with nfc normalizations" do + assert_format "ç", "ç" + end + + test "with custom normalizations" do + assert_format "µs", "μs" + end + end + + describe "aliases" do + test "with atom-only parts" do + assert_same "Elixir" + assert_same "Elixir.Foo" + assert_same "Foo.Bar.Baz" + end + + test "removes spaces between aliases" do + assert_format "Foo . Bar . Baz", "Foo.Bar.Baz" + end + + test "starting with expression" do + assert_same "__MODULE__.Foo.Bar" + # Syntactically valid, semantically invalid + assert_same ~S[~c"Foo".Bar.Baz] + end + + test "wraps the head in parens if it has an operator" do + assert_format "+(Foo . Bar . Baz)", "+Foo.Bar.Baz" + assert_format "(+Foo) . Bar . Baz", "(+Foo).Bar.Baz" + end + end + + describe "sigils" do + test "without interpolation" do + assert_same ~S[~s(foo)] + assert_same ~S[~s{foo bar}] + assert_same ~S[~r/Bar Baz/] + assert_same ~S[~w<>] + assert_same ~S[~W()] + assert_same ~S[~MAT()] + assert_same ~S[~MAT{1,2,3}] + end + + test "with escapes" do + assert_same ~S[~s(foo \) bar)] + assert_same ~S[~s(f\a\b\ro)] + + assert_same ~S""" + ~S(foo\ + bar) + """ + end + + test "with nested new lines" do + assert_same ~S""" + foo do + ~S(foo\ + bar) + end + """ + + assert_same ~S""" + foo do + ~s(#{bar} + ) + end + """ + end + + test "with interpolation" do + assert_same ~S[~s(one #{2} three)] + end + + test "with modifiers" do + assert_same ~S[~w(one two three)a] + assert_same ~S[~z(one two three)foo] + end + + test "with interpolation on line limit" do + assert_same ~S""" + ~s(one #{"two"} three) + """, + @short_length + end + + test "with heredoc syntax" do + assert_same ~S""" + ~s''' + one\a + #{:two}\r + three\0 + ''' + """ + + assert_same ~S''' + ~s""" + one\a + #{:two}\r + three\0 + """ + ''' + end + + test "with heredoc syntax and modifier" do + assert_same ~S""" + ~s''' + foo + '''rsa + """ + end + + test "with heredoc syntax and interpolation on line limit" do + assert_same ~S""" + ~s''' + one #{"two two"} three + ''' + """, + @short_length + end + + test "with custom formatting" do + bad = """ + ~W/foo bar baz/ + """ + + good = """ + ~W/foo bar baz/ + """ + + formatter = fn content, opts -> + assert opts == [file: nil, line: 1, sigil: :W, modifiers: [], opening_delimiter: "/"] + content |> String.split(~r/ +/) |> Enum.join(" ") + end + + assert_format bad, good, sigils: [W: formatter] + + bad = """ + var = ~Wabc + """ + + good = """ + var = ~Wabc + """ + + formatter = fn content, opts -> + assert opts == [file: nil, line: 1, sigil: :W, modifiers: ~c"abc", opening_delimiter: "<"] + content |> String.split(~r/ +/) |> Enum.intersperse(" ") + end + + assert_format bad, good, sigils: [W: formatter] + + bad = """ + var = ~MAT{foo bar baz}abc + """ + + good = """ + var = ~MAT{foo bar baz}abc + """ + + formatter = fn content, opts -> + assert opts == [ + file: nil, + line: 1, + sigil: :MAT, + modifiers: ~c"abc", + opening_delimiter: "{" + ] + + content |> String.split(~r/ +/) |> Enum.intersperse(" ") + end + + assert_format bad, good, sigils: [MAT: formatter] + end + + test "with custom formatting on heredocs" do + bad = """ + ~W''' + foo bar baz + ''' + """ + + good = """ + ~W''' + foo bar baz + ''' + """ + + formatter = fn content, opts -> + assert opts == [file: nil, line: 1, sigil: :W, modifiers: [], opening_delimiter: "'''"] + content |> String.split(~r/ +/) |> Enum.join(" ") + end + + assert_format bad, good, sigils: [W: formatter] + + bad = ~S''' + if true do + ~W""" + foo + bar + baz + """abc + end + ''' + + good = ~S''' + if true do + ~W""" + foo + bar + baz + """abc + end + ''' + + formatter = fn content, opts -> + assert opts == [ + file: nil, + line: 2, + sigil: :W, + modifiers: ~c"abc", + opening_delimiter: ~S/"""/ + ] + + content |> String.split(~r/ +/) |> Enum.join("\n") + end + + assert_format bad, good, sigils: [W: formatter] + end + end + + describe "anonymous functions" do + test "with a single clause and no arguments" do + assert_format "fn ->:ok end", "fn -> :ok end" + + bad = "fn -> :foo end" + + good = """ + fn -> + :foo + end + """ + + assert_format bad, good, @short_length + + assert_same "fn () when node() == :nonode@nohost -> true end" + end + + test "with a single clause and arguments" do + assert_format "fn x ,y-> x + y end", "fn x, y -> x + y end" + + bad = "fn x -> foo(x) end" + + good = """ + fn x -> + foo(x) + end + """ + + assert_format bad, good, @short_length + + bad = "fn one, two, three -> foo(x) end" + + good = """ + fn one, + two, + three -> + foo(x) + end + """ + + assert_format bad, good, @short_length + end + + test "with a single clause and when" do + code = """ + fn arg + when guard -> + :ok + end + """ + + assert_same code, @short_length + end + + test "keeps parens if argument includes keyword list" do + assert_same """ + fn [] when is_integer(x) -> + x + 42 + end + """ + + bad = """ + fn (input: x) when is_integer(x) -> + x + 42 + end + """ + + good = """ + fn [input: x] when is_integer(x) -> + x + 42 + end + """ + + assert_format bad, good + end + + test "with a single clause, followed by a newline, and can fit in one line" do + assert_same """ + fn + hello -> world + end + """ + end + + test "with a single clause, followed by a newline, and can not fit in one line" do + assert_same """ + SomeModule.long_function_name_that_approaches_max_columns(argument, acc, fn + %SomeStruct{key: key}, acc -> more_code(key, acc) + end) + """ + end + + test "with multiple clauses" do + code = """ + fn + 1 -> :ok + 2 -> :ok + end + """ + + assert_same code, @short_length + + code = """ + fn + 1 -> + :ok + + 2 -> + :error + end + """ + + assert_same code, @short_length + + code = """ + fn + arg11, + arg12 -> + body1 + + arg21, + arg22 -> + body2 + end + """ + + assert_same code, @short_length + + code = """ + fn + arg11, + arg12 -> + body1 + + arg21, + arg22 -> + body2 + + arg31, + arg32 -> + body3 + end + """ + + assert_same code, @short_length + end + + test "with heredocs" do + assert_same ~S''' + fn + arg1 -> + """ + foo + """ + + arg2 -> + """ + bar + """ + end + ''' + end + + test "with multiple empty clauses" do + assert_same """ + fn + () -> :ok1 + () -> :ok2 + end + """ + end + + test "with when in clauses" do + assert_same """ + fn + a1 when a + b -> :ok + b1 when c + d -> :ok + end + """ + + long = """ + fn + a1, a2 when a + b -> :ok + b1, b2 when c + d -> :ok + end + """ + + assert_same long + + good = """ + fn + a1, a2 + when a + + b -> + :ok + + b1, b2 + when c + + d -> + :ok + end + """ + + assert_format long, good, @short_length + end + + test "uses block context for the body of each clause" do + assert_same "fn -> @foo bar end" + end + + test "preserves user choice even when it fits" do + assert_same """ + fn + 1 -> + :ok + + 2 -> + :ok + end + """ + + assert_same """ + fn + 1 -> + :ok + + 2 -> + :ok + + 3 -> + :ok + end + """ + end + + test "with -> on line limit" do + bad = """ + fn ab, cd -> + ab + cd + end + """ + + good = """ + fn ab, + cd -> + ab + cd + end + """ + + assert_format bad, good, @short_length + + bad = """ + fn + ab, cd -> + 1 + xy, zw -> + 2 + end + """ + + good = """ + fn + ab, + cd -> + 1 + + xy, + zw -> + 2 + end + """ + + assert_format bad, good, @short_length + end + end + + describe "anonymous functions types" do + test "with a single clause and no arguments" do + assert_same "(-> :ok)" + assert_format "(->:ok)", "(-> :ok)" + assert_format "( -> :ok)", "(-> :ok)" + assert_format "(() -> :really_long_atom)", "(-> :really_long_atom)", @short_length + assert_same "(() when node() == :nonode@nohost -> true)" + end + + test "with a single clause and arguments" do + assert_format "( x ,y-> x + y )", "(x, y -> x + y)" + + bad = "(x -> :really_long_atom)" + + good = """ + (x -> + :really_long_atom) + """ + + assert_format bad, good, @short_length + + bad = "(one, two, three -> foo(x))" + + good = """ + (one, + two, + three -> + foo(x)) + """ + + assert_format bad, good, @short_length + end + + test "with multiple clauses" do + code = """ + ( + 1 -> :ok + 2 -> :ok + ) + """ + + assert_same code, @short_length + + code = """ + ( + 1 -> + :ok + + 2 -> + :error + ) + """ + + assert_same code, @short_length + + code = """ + ( + arg11, + arg12 -> + body1 + + arg21, + arg22 -> + body2 + ) + """ + + assert_same code, @short_length + + code = """ + ( + arg11, + arg12 -> + body1 + + arg21, + arg22 -> + body2 + + arg31, + arg32 -> + body2 + ) + """ + + assert_same code, @short_length + end + + test "with heredocs" do + assert_same ~S''' + ( + arg1 -> + """ + foo + """ + + arg2 -> + """ + bar + """ + ) + ''' + end + + test "with multiple empty clauses" do + assert_same """ + ( + () -> :ok1 + () -> :ok2 + ) + """ + end + + test "preserves user choice even when it fits" do + assert_same """ + ( + 1 -> + :ok + + 2 -> + :ok + ) + """ + + assert_same """ + ( + 1 -> + :ok + + 2 -> + :ok + + 3 -> + :ok + ) + """ + end + end + + describe "blocks" do + test "empty" do + assert_format "(;)", "" + assert_format "quote do: (;)", "quote do: nil" + assert_format "quote do end", "quote do\nend" + assert_format "quote do ; end", "quote do\nend" + end + + test "with multiple lines" do + assert_same """ + foo = bar + baz = bat + """ + end + + test "with multiple lines with line limit" do + code = """ + foo = + bar(one) + + baz = + bat(two) + + a(b) + """ + + assert_same code, @short_length + + code = """ + foo = + bar(one) + + a(b) + + baz = + bat(two) + """ + + assert_same code, @short_length + + code = """ + a(b) + + foo = + bar(one) + + baz = + bat(two) + """ + + assert_same code, @short_length + + code = """ + foo = + bar(one) + + one = + two(ghi) + + baz = + bat(two) + """ + + assert_same code, @short_length + end + + test "with multiple lines with line limit inside block" do + code = """ + block do + a = + b(foo) + + c = + d(bar) + + e = + f(baz) + end + """ + + assert_same code, @short_length + end + + test "with multiple lines with cancel expressions" do + code = """ + foo(%{ + key: 1 + }) + + bar(%{ + key: 1 + }) + + baz(%{ + key: 1 + }) + """ + + assert_same code, @short_length + end + + test "with heredoc" do + assert_same ~S''' + block do + """ + a + + b + + c + """ + end + ''' + end + + test "keeps user newlines" do + assert_same """ + defmodule Mod do + field(:foo) + field(:bar) + field(:baz) + belongs_to(:one) + belongs_to(:two) + timestamp() + lock() + has_many(:three) + has_many(:four) + :ok + has_one(:five) + has_one(:six) + foo = 1 + bar = 2 + :before + baz = 3 + :after + end + """ + + bad = """ + defmodule Mod do + field(:foo) + + field(:bar) + + field(:baz) + + + belongs_to(:one) + belongs_to(:two) + + + timestamp() + + lock() + + + has_many(:three) + has_many(:four) + + + :ok + + + has_one(:five) + has_one(:six) + + + foo = 1 + bar = 2 + + + :before + baz = 3 + :after + end + """ + + good = """ + defmodule Mod do + field(:foo) + + field(:bar) + + field(:baz) + + belongs_to(:one) + belongs_to(:two) + + timestamp() + + lock() + + has_many(:three) + has_many(:four) + + :ok + + has_one(:five) + has_one(:six) + + foo = 1 + bar = 2 + + :before + baz = 3 + :after + end + """ + + assert_format bad, good + end + + test "with multiple defs" do + assert_same """ + def foo(:one), do: 1 + def foo(:two), do: 2 + def foo(:three), do: 3 + """ + end + + test "with module attributes" do + assert_same ~S''' + defmodule Foo do + @constant 1 + @constant 2 + + @doc """ + foo + """ + def foo do + :ok + end + + @spec bar :: 1 + @spec bar :: 2 + def bar do + :ok + end + + @other_constant 3 + + @spec baz :: 4 + @doc """ + baz + """ + def baz do + :ok + end + + @another_constant 5 + @another_constant 5 + + @doc """ + baz + """ + @spec baz :: 6 + def baz do + :ok + end + end + ''' + end + + test "as function arguments" do + assert_same """ + fun( + ( + foo + bar + ) + ) + """ + + assert_same """ + assert true, + do: + ( + foo + bar + ) + """ + end + end +end diff --git a/lib/elixir/test/elixir/code_formatter/integration_test.exs b/lib/elixir/test/elixir/code_formatter/integration_test.exs new file mode 100644 index 00000000000..7534bcd8823 --- /dev/null +++ b/lib/elixir/test/elixir/code_formatter/integration_test.exs @@ -0,0 +1,720 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Code.Formatter.IntegrationTest do + use ExUnit.Case, async: true + + import CodeFormatterHelpers + + test "empty documents" do + assert_format " ", "" + assert_format "\n", "" + assert_format ";", "" + end + + test "function with multiple calls and case" do + assert_same """ + def equivalent(string1, string2) when is_binary(string1) and is_binary(string2) do + quoted1 = Code.string_to_quoted!(string1) + quoted2 = Code.string_to_quoted!(string2) + + case not_equivalent(quoted1, quoted2) do + {left, right} -> {:error, left, right} + nil -> :ok + end + end + """ + end + + test "function with long pipeline" do + assert_same ~S""" + def to_algebra!(string, opts \\ []) when is_binary(string) and is_list(opts) do + string + |> Code.string_to_quoted!(wrap_literals_in_blocks: true, unescape: false) + |> block_to_algebra(state(opts)) + |> elem(0) + end + """ + end + + test "case with multiple multi-line arrows" do + assert_same ~S""" + case meta[:format] do + :list_heredoc -> + string = list |> List.to_string() |> escape_string(:heredoc) + {@single_heredoc |> line(string) |> concat(@single_heredoc) |> force_unfit(), state} + + :charlist -> + string = list |> List.to_string() |> escape_string(@single_quote) + {@single_quote |> concat(string) |> concat(@single_quote), state} + + _other -> + list_to_algebra(list, state) + end + """ + end + + test "function with long guards" do + assert_same """ + defp module_attribute_read?({:@, _, [{var, _, var_context}]}) + when is_atom(var) and is_atom(var_context) do + Macro.classify_atom(var) == :identifier + end + """ + end + + test "anonymous function with single clause and blocks" do + assert_same """ + {args_doc, state} = + Enum.reduce(args, {[], state}, fn quoted, {acc, state} -> + {doc, state} = quoted_to_algebra(quoted, :block, state) + doc = doc |> concat(nest(break(""), :reset)) |> group() + {[doc | acc], state} + end) + """ + end + + test "anonymous function with long single clause and blocks" do + assert_same """ + {function_count, call_count, total_time} = + Enum.reduce(call_results, {0, 0, 0}, fn {_, {count, time}}, + {function_count, call_count, total_time} -> + {function_count + 1, call_count + count, total_time + time} + end) + """ + end + + test "cond with long clause args" do + assert_same """ + cond do + parent_prec == prec and parent_assoc == side -> + binary_op_to_algebra(op, op_string, left, right, context, state, op_info, nesting) + + parent_op in @required_parens_on_binary_operands or parent_prec > prec or + (parent_prec == prec and parent_assoc != side) -> + {operand, state} = + binary_op_to_algebra(op, op_string, left, right, context, state, op_info, 2) + + {concat(concat("(", nest(operand, 1)), ")"), state} + + true -> + binary_op_to_algebra(op, op_string, left, right, context, state, op_info, 2) + end + """ + end + + test "type with multiple |" do + assert_same """ + @type t :: + binary + | :doc_nil + | :doc_line + | doc_string + | doc_cons + | doc_nest + | doc_break + | doc_group + | doc_color + | doc_force + | doc_cancel + """ + end + + test "spec with when keywords and |" do + assert_same """ + @spec send(dest, msg, [option]) :: :ok | :noconnect | :nosuspend + when dest: pid | port | atom | {atom, node}, + msg: any, + option: :noconnect | :nosuspend + """ + + assert_same """ + @spec send(dest, msg, [option]) :: :ok | :noconnect | :nosuspend + when dest: + pid + | port + | atom + | {atom, node} + | and_a_really_long_type_to_force_a_line_break + | followed_by_another_really_long_type + """ + + assert_same """ + @callback get_and_update(data, key, (value -> {get_value, value} | :pop)) :: {get_value, data} + when get_value: var, data: container + """ + end + + test "spec with multiple keys on type" do + assert_same """ + @spec foo(%{(String.t() | atom) => any}) :: any + """ + end + + test "multiple whens with new lines" do + assert_same """ + def sleep(timeout) + when is_integer(timeout) and timeout >= 0 + when timeout == :infinity do + receive after: (timeout -> :ok) + end + """ + end + + test "function with operator and pipeline" do + assert_same """ + defp apply_next_break_fits?({fun, meta, args}) when is_atom(fun) and is_list(args) do + meta[:terminator] in [@double_heredoc, @single_heredoc] and + fun |> Atom.to_string() |> String.starts_with?("sigil_") + end + """ + end + + test "mixed parens and no parens calls with anonymous function" do + assert_same ~S""" + node interface do + resolve_type(fn + %{__struct__: str}, _ -> + str |> Model.Node.model_to_node_type() + + value, _ -> + Logger.warning("Could not extract node type from value: #{inspect(value)}") + nil + end) + end + """ + end + + test "long defstruct definition" do + assert_same """ + defstruct name: nil, + module: nil, + schema: nil, + alias: nil, + base_module: nil, + web_module: nil, + basename: nil, + file: nil, + test_file: nil + """ + end + + test "mix of operators and arguments" do + assert_same """ + def count(%{path: path, line_or_bytes: bytes}) do + case File.stat(path) do + {:ok, %{size: 0}} -> {:error, __MODULE__} + {:ok, %{size: size}} -> {:ok, div(size, bytes) + if(rem(size, bytes) == 0, do: 0, else: 1)} + {:error, reason} -> raise File.Error, reason: reason, action: "stream", path: path + end + end + """ + end + + test "mix of left and right operands" do + assert_same """ + defp server_get_modules(handlers) do + for(handler(module: module) <- handlers, do: module) + |> :ordsets.from_list() + |> :ordsets.to_list() + end + """ + + assert_same """ + neighbours = for({_, _} = t <- neighbours, do: t) |> :sets.from_list() + """ + end + + test "long expression with single line anonymous function" do + assert_same """ + for_many(uniq_list_of(integer(1..10000)), fn list -> + assert Enum.uniq(list) == list + end) + """ + end + + test "long comprehension" do + assert_same """ + for %{app: app, opts: opts, top_level: true} <- Mix.Dep.cached(), + Keyword.get(opts, :app, true), + Keyword.get(opts, :runtime, true), + not Keyword.get(opts, :optional, false), + app not in included_applications, + app not in included_applications, + do: app + """ + end + + test "short comprehensions" do + assert_same """ + for {protocol, :protocol, _beam} <- removed_metadata, + remove_consolidated(protocol, output), + do: {protocol, true}, + into: %{} + """ + end + + test "comprehensions with when" do + assert_same """ + for {key, value} when is_atom(key) <- Map.to_list(map), + key = Atom.to_string(key), + String.starts_with?(key, hint) do + %{kind: :map_key, name: key, value_is_map: is_map(value)} + end + """ + + assert_same """ + with {_, doc} when unquote(doc_attr?) <- + Module.get_attribute(__MODULE__, unquote(name), unquote(escaped)), + do: doc + """ + end + + test "next break fits followed by inline tuple" do + assert_same """ + assert ExUnit.Filters.eval([line: "1"], [:line], %{line: 3, describe_line: 2}, tests) == + {:error, "due to line filter"} + """ + end + + test "try/catch with clause comment" do + assert_same """ + def format_error(reason) do + try do + do_format_error(reason) + catch + # A user could create an error that looks like a built-in one + # causing an error. + :error, _ -> + inspect(reason) + end + end + """ + end + + test "case with when and clause comment" do + assert_same """ + case decomposition do + # Decomposition + <> when h != ?< -> + decomposition = + decomposition + |> :binary.split(" ", [:global]) + |> Enum.map(&String.to_integer(&1, 16)) + + Map.put(dacc, String.to_integer(codepoint, 16), decomposition) + + _ -> + dacc + end + """ + end + + test "do-end inside binary" do + assert_same """ + <> + """ + end + + test "anonymous function with parens around integer argument" do + bad = """ + fn (1) -> "hello" end + """ + + assert_format bad, """ + fn 1 -> "hello" end + """ + end + + test "no parens keywords at the end of the line" do + bad = """ + defmodule Mod do + def token_list_downcase(<>, acc) when is_whitespace(char) or is_comma(char), do: token_list_downcase(rest, acc) + def token_list_downcase(some_really_long_arg11, some_really_long_arg22, some_really_long_arg33), do: token_list_downcase(rest, acc) + end + """ + + assert_format bad, """ + defmodule Mod do + def token_list_downcase(<>, acc) when is_whitespace(char) or is_comma(char), + do: token_list_downcase(rest, acc) + + def token_list_downcase(some_really_long_arg11, some_really_long_arg22, some_really_long_arg33), + do: token_list_downcase(rest, acc) + end + """ + end + + test "do at the end of the line" do + bad = """ + foo bar, baz, quux do + :ok + end + """ + + good = """ + foo bar, + baz, + quux do + :ok + end + """ + + assert_format bad, good, line_length: 18 + end + + test "keyword lists in last line" do + assert_same """ + content = + config(VeryLongModuleNameThatWillCauseBreak, "new.html", + conn: conn, + changeset: changeset, + categories: categories + ) + """ + + assert_same """ + content = + config VeryLongModuleNameThatWillCauseBreak, "new.html", + conn: conn, + changeset: changeset, + categories: categories + """ + end + + test "keyword list at line limit" do + bad = """ + pre() + config(arg, foo: bar) + post() + """ + + good = """ + pre() + + config(arg, + foo: bar + ) + + post() + """ + + assert_format bad, good, line_length: 20 + end + + test "do at the end of the line with single argument" do + bad = """ + defmodule Location do + def new(line, column) when is_integer(line) and line >= 0 and is_integer(column) and column >= 0 do + %{column: column, line: line} + end + end + """ + + assert_format bad, """ + defmodule Location do + def new(line, column) + when is_integer(line) and line >= 0 and is_integer(column) and column >= 0 do + %{column: column, line: line} + end + end + """ + end + + test "tuples as trees" do + bad = """ + @document Parser.parse( + {"html", [], [ + {"head", [], []}, + {"body", [], [ + {"div", [], [ + {"p", [], ["1"]}, + {"p", [], ["2"]}, + {"div", [], [ + {"p", [], ["3"]}, + {"p", [], ["4"]}]}, + {"p", [], ["5"]}]}]}]}) + """ + + assert_format bad, """ + @document Parser.parse( + {"html", [], + [ + {"head", [], []}, + {"body", [], + [ + {"div", [], + [ + {"p", [], ["1"]}, + {"p", [], ["2"]}, + {"div", [], + [ + {"p", [], ["3"]}, + {"p", [], ["4"]} + ]}, + {"p", [], ["5"]} + ]} + ]} + ]} + ) + """ + end + + test "nested tuples as lines" do + assert_same """ + {:ok, + {1, 2, 3, + 4, 5}} = + call() + """, + line_length: 10 + end + + test "first argument in a call without parens with comments" do + assert_same """ + with bar :: + :ok + | :invalid + # | :unknown + | :other + """ + + assert_same """ + @spec bar :: + :ok + | :invalid + # | :unknown + | :other + """ + end + + test "when with keywords inside call" do + assert_same """ + quote((bar(foo(1)) when bat: foo(1)), []) + """ + + assert_same """ + quote(do: (bar(foo(1)) when bat: foo(1)), line: 1) + """ + + assert_same """ + typespec(quote(do: (bar(foo(1)) when bat: foo(1))), [foo: 1], []) + """ + end + + test "false positive sigil" do + assert_same """ + def sigil_d(<>, calendar) do + ymd(year, month, day, calendar) + end + """ + end + + test "newline after stab" do + assert_same """ + capture_io(":erl. mof*,,l", fn -> + assert :io.scan_erl_form(~c">") == {:ok, [{:":", 1}, {:atom, 1, :erl}, {:dot, 1}], 1} + + expected_tokens = [{:atom, 1, :mof}, {:*, 1}, {:",", 1}, {:",", 1}, {:atom, 1, :l}] + assert :io.scan_erl_form(~c">") == {:ok, expected_tokens, 1} + + assert :io.scan_erl_form(~c">") == {:eof, 1} + end) + """ + end + + test "capture with operators" do + assert_same """ + "this works" |> (&String.upcase/1) |> (&String.downcase/1) + """ + + assert_same """ + "this works" || (&String.upcase/1) || (&String.downcase/1) + """ + + assert_same """ + "this works" == (&String.upcase/1) == (&String.downcase/1) + """ + + bad = """ + "this works" = (&String.upcase/1) = (&String.downcase/1) + """ + + assert_format bad, """ + "this works" = (&String.upcase/1) = &String.downcase/1 + """ + + bad = """ + "this works" ++ (&String.upcase/1) ++ (&String.downcase/1) + """ + + assert_format bad, """ + "this works" ++ (&String.upcase/1) ++ &String.downcase/1 + """ + + bad = """ + "this works" +++ (&String.upcase/1) +++ (&String.downcase/1) + """ + + assert_format bad, """ + "this works" +++ (&String.upcase/1) +++ &String.downcase/1 + """ + + bad = """ + "this works" | (&String.upcase/1) | (&String.downcase/1) + """ + + assert_format bad, """ + "this works" | (&String.upcase/1) | &String.downcase/1 + """ + + bad = ~S""" + "this works" \\ (&String.upcase/1) \\ (&String.downcase/1) + """ + + assert_format bad, ~S""" + "this works" \\ &String.upcase/1 \\ &String.downcase/1 + """ + end + + test "multiline expression inside interpolation" do + bad = ~S""" + Logger.info("Example: #{ + inspect(%{ + a: 1, + b: 2 + }) + }") + """ + + assert_format bad, ~S""" + Logger.info("Example: #{inspect(%{a: 1, b: 2})}") + """ + end + + test "comment inside operator with when" do + bad = """ + raise function(x) :: + # Comment + any + """ + + assert_format bad, """ + # Comment + raise function(x) :: + any + """ + + bad = """ + raise function(x) :: + # Comment + any + when x: any + """ + + assert_format bad, """ + raise function(x) :: + any + # Comment + when x: any + """ + + bad = """ + @spec function(x) :: + # Comment + any + when x: any + """ + + assert_format bad, """ + @spec function(x) :: + any + # Comment + when x: any + """ + + bad = """ + @spec function(x) :: + # Comment + any + when x + when y + """ + + assert_format bad, """ + @spec function(x) :: + any + # Comment + when x + when y + """ + end + + test "nested heredocs with multi-line string in interpolation" do + bad = ~S''' + def foo do + """ + #{(feature_flag(:feature_x) && " + new_field + " || "")} + """ + end + ''' + + good = ~S''' + def foo do + """ + #{(feature_flag(:feature_x) && " + new_field + ") || ""} + """ + end + ''' + + assert_format bad, good + end + + test "functions with infinity line length" do + assert_same ~S""" + x = fn -> + {:ok, pid} = Repl.start_link({self(), opts}) + assert Exception.message(error) =~ msg + end + """, + line_length: :infinity + + assert_same ~S""" + capture_log(fn x -> + {:ok, pid} = Repl.start_link({self(), opts}) + assert Exception.message(error) =~ msg + end) =~ msg + """, + line_length: :infinity + + assert_same ~S""" + capture_log(fn -> + {:ok, pid} = Repl.start_link({self(), opts}) + assert Exception.message(error) =~ msg + end) =~ msg + """, + line_length: :infinity + + assert_same ~S""" + capture_log(fn x -> + {:ok, pid} = Repl.start_link({self(), opts}) + assert Exception.message(error) =~ msg + end) =~ msg + """, + line_length: :infinity + end + + test "functions without parentheses within do: keyword" do + assert_format ~S"defmodule Foo, do: foo bar, baz", + ~S"defmodule Foo, do: foo(bar, baz)" + end +end diff --git a/lib/elixir/test/elixir/code_formatter/literals_test.exs b/lib/elixir/test/elixir/code_formatter/literals_test.exs new file mode 100644 index 00000000000..da252fbf4e0 --- /dev/null +++ b/lib/elixir/test/elixir/code_formatter/literals_test.exs @@ -0,0 +1,412 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Code.Formatter.LiteralsTest do + use ExUnit.Case, async: true + + import CodeFormatterHelpers + + @short_length [line_length: 10] + + describe "integers" do + test "in decimal base" do + assert_same "0" + assert_same "100" + assert_same "007" + assert_same "10000" + assert_same "100_00" + assert_format "100000", "100_000" + assert_format "1000000", "1_000_000" + end + + test "in binary base" do + assert_same "0b0" + assert_same "0b1" + assert_same "0b101" + assert_same "0b01" + assert_same "0b111_111" + end + + test "in octal base" do + assert_same "0o77" + assert_same "0o0" + assert_same "0o01" + assert_same "0o777_777" + end + + test "in hex base" do + assert_same "0x1" + assert_format "0xabcdef", "0xABCDEF" + assert_same "0x01" + assert_format "0xfff_fff", "0xFFF_FFF" + end + + test "as chars" do + assert_same "?a" + assert_same "?1" + assert_same "?è" + assert_same "??" + assert_same "?\\\\" + assert_same "?\\s" + assert_same "?🎾" + end + end + + describe "floats" do + test "with normal notation" do + assert_same "0.0" + assert_same "1.0" + assert_same "123.456" + assert_same "0.0000001" + assert_same "001.100" + assert_same "0_10000_0.000_000" + assert_format "0100000.000000", "0_100_000.000000" + end + + test "with scientific notation" do + assert_same "1.0e1" + assert_same "1.0e-1" + assert_same "1.0e01" + assert_same "1.0e-01" + assert_same "001.100e-010" + assert_same "0_100_0000.100e-010" + assert_format "0100000.0e-5", "0_100_000.0e-5" + + assert_format "1.0E01", "1.0e01" + assert_format "1.0E-01", "1.0e-01" + end + end + + describe "atoms" do + test "true, false, nil" do + assert_same "nil" + assert_same "true" + assert_same "false" + end + + test "without escapes" do + assert_same ~S[:foo] + assert_same ~S[:\\] + end + + test "with escapes" do + assert_same ~S[:"f\a\b\ro"] + assert_format ~S[:'f\a\b\ro'], ~S[:"f\a\b\ro"] + assert_format ~S[:'single \' quote'], ~S[:"single ' quote"] + assert_format ~S[:"double \" quote"], ~S[:"double \" quote"] + assert_same ~S[:"\\"] + end + + test "with unicode" do + assert_same ~S[:ólá] + end + + test "does not reformat aliases" do + assert_same ~S[:"Elixir.String"] + end + + test "removes quotes when they are not necessary" do + assert_format ~S[:"foo"], ~S[:foo] + assert_format ~S[:"++"], ~S[:++] + end + + test "quoted operators" do + assert_same ~S[:"::"] + end + + test "uses double quotes even when single quotes are used" do + assert_format ~S[:'foo bar'], ~S[:"foo bar"] + end + + test "with interpolation" do + assert_same ~S[:"one #{2} three"] + end + + test "with escapes and interpolation" do + assert_same ~S[:"one\n\"#{2}\"\nthree"] + end + + test "with interpolation on line limit" do + assert_same ~S""" + :"one #{"two"} three" + """, + @short_length + end + end + + describe "strings" do + test "without escapes" do + assert_same ~S["foo"] + end + + test "with escapes" do + assert_same ~S["f\a\b\ro"] + assert_same ~S["double \" quote"] + end + + test "keeps literal new lines" do + assert_same """ + "fo + o" + """ + end + + test "with interpolation" do + assert_same ~S["one #{} three"] + assert_same ~S["one #{2} three"] + end + + test "with interpolation uses block content" do + assert_format ~S["one #{@two(three)}"], ~S["one #{@two three}"] + end + + test "with interpolation on line limit" do + assert_same ~S""" + "one #{"two"} three" + """, + @short_length + end + + test "with escaped interpolation" do + assert_same ~S["one\#{two}three"] + end + + test "with escapes and interpolation" do + assert_same ~S["one\n\"#{2}\"\nthree"] + end + + test "is measured in graphemes" do + assert_same ~S""" + "áá#{0}áá" + """, + @short_length + end + + test "literal new lines don't count towards line limit" do + assert_same ~S""" + "one + #{"two"} + three" + """, + @short_length + end + end + + describe "charlists" do + test "without escapes" do + assert_same ~S[''] + assert_same ~S[' '] + assert_same ~S['foo'] + end + + test "with escapes" do + assert_same ~S['f\a\b\ro'] + assert_same ~S['single \' quote'] + assert_same ~S['double " quote'] + assert_same ~S['escaped \" quote'] + assert_same ~S['\\"'] + end + + test "keeps literal new lines" do + assert_same """ + 'fo + o' + """ + end + + test "with interpolation" do + assert_same ~S['one #{2} three'] + assert_same ~S['#{1}\n \\ " \"'] + end + + test "with escape and interpolation" do + assert_same ~S['one\n\'#{2}\'\nthree'] + end + + test "with interpolation on line limit" do + assert_same ~S""" + 'one #{"two"} three' + """, + @short_length + end + + test "literal new lines don't count towards line limit" do + assert_same ~S""" + 'one + #{"two"} + three' + """, + @short_length + end + end + + describe "string heredocs" do + test "without escapes" do + assert_same ~S''' + """ + hello + """ + ''' + end + + test "with escapes" do + assert_same ~S''' + """ + f\a\b\ro + """ + ''' + + assert_same ~S''' + """ + multiple "\"" quotes + """ + ''' + end + + test "with interpolation" do + assert_same ~S''' + """ + one + #{2} + three + """ + ''' + + assert_same ~S''' + """ + one + " + #{2} + " + three + """ + ''' + end + + test "with interpolation on line limit" do + assert_same ~S''' + """ + one #{"two two"} three + """ + ''', + @short_length + end + + test "nested with empty lines" do + assert_same ~S''' + nested do + """ + + foo + + + bar + + """ + end + ''' + end + + test "nested with empty lines and interpolation" do + assert_same ~S''' + nested do + """ + + #{foo} + + + #{bar} + + """ + end + ''' + + assert_same ~S''' + nested do + """ + #{foo} + + + #{bar} + """ + end + ''' + end + + test "literal new lines don't count towards line limit" do + assert_same ~S''' + """ + one + #{"two"} + three + """ + ''', + @short_length + end + + test "with escaped new lines" do + assert_same ~S''' + """ + one\ + #{"two"}\ + three\ + """ + ''' + end + + test "with new lines" do + assert_format ~s|foo do\n """\n foo\n \n bar\n """\nend|, + ~s|foo do\n """\n foo\n\n bar\n """\nend| + + assert_format ~s|foo do\r\n """\r\n foo\r\n \r\n bar\r\n """\r\nend|, + ~s|foo do\n """\n foo\r\n\r\n bar\r\n """\nend| + end + end + + describe "charlist heredocs" do + test "without escapes" do + assert_same ~S""" + ''' + hello + ''' + """ + end + + test "with escapes" do + assert_same ~S""" + ''' + f\a\b\ro + ''' + """ + + assert_same ~S""" + ''' + multiple "\"" quotes + ''' + """ + end + + test "with interpolation" do + assert_same ~S""" + ''' + one + #{2} + three + ''' + """ + + assert_same ~S""" + ''' + one + " + #{2} + " + three + ''' + """ + end + end +end diff --git a/lib/elixir/test/elixir/code_formatter/migration_test.exs b/lib/elixir/test/elixir/code_formatter/migration_test.exs new file mode 100644 index 00000000000..67ebee75798 --- /dev/null +++ b/lib/elixir/test/elixir/code_formatter/migration_test.exs @@ -0,0 +1,328 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team + +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Code.Formatter.MigrationTest do + use ExUnit.Case, async: true + + import CodeFormatterHelpers + + @short_length [line_length: 10] + + describe "migrate_bitstring_modifiers: true" do + @opts [migrate_bitstring_modifiers: true] + + test "normalizes bitstring modifiers" do + assert_format "<>", "<>", @opts + assert_same "<>", @opts + + assert_format "<>", "<>", @opts + assert_same "<>", @opts + + assert_format "<>", "<>", @opts + assert_same "<>", @opts + assert_same "<<0::size*unit, bytes::binary>>", @opts + assert_format "<<0::size*unit, bytes::custom>>", "<<0::size*unit, bytes::custom()>>", @opts + + assert_format "<<0, 1::2-integer() <- x>>", "<<0, 1::2-integer <- x>>", @opts + assert_same "<<0, 1::2-integer <- x>>", @opts + end + end + + describe "migrate_call_parens_on_pipe: true" do + @opts [migrate_call_parens_on_pipe: true] + + test "adds parentheses on the right operand" do + assert_format "x |> y", "x |> y()", @opts + assert_format "x |> y |> z", "x |> y() |> z()", @opts + assert_format "x |> y.z", "x |> y.z()", @opts + assert_format "x |> y.z.t", "x |> y.z.t()", @opts + end + + test "does nothing within defmacro" do + assert_same "defmacro left |> right, do: ...", @opts + end + + test "does nothing without the migrate_unless option" do + assert_same "x |> y" + assert_same "x |> y |> z" + assert_same "x |> y.z" + end + end + + describe "migrate_charlists_as_sigils: true" do + @opts [migrate_charlists_as_sigils: true] + + test "without escapes" do + assert_format ~S[''], ~S[~c""], @opts + assert_format ~S[' '], ~S[~c" "], @opts + assert_format ~S['foo'], ~S[~c"foo"], @opts + end + + test "with escapes" do + assert_format ~S['f\a\b\ro'], ~S[~c"f\a\b\ro"], @opts + assert_format ~S['single \' quote'], ~S[~c"single ' quote"], @opts + assert_format ~S['double " quote'], ~S[~c'double " quote'], @opts + assert_format ~S['escaped \" quote'], ~S[~c'escaped \" quote'], @opts + assert_format ~S['\\"'], ~S[~c'\\"'], @opts + end + + test "keeps literal new lines" do + assert_format """ + 'fo + o' + """, + """ + ~c"fo + o" + """, + @opts + end + + test "with interpolation" do + assert_format ~S['one #{2} three'], ~S[~c"one #{2} three"], @opts + assert_format ~S['#{1}\n \\ " \"'], ~S[~c'#{1}\n \\ " \"'], @opts + end + + test "with escape and interpolation" do + assert_format ~S['one\n\'#{2}\'\nthree'], ~S[~c"one\n'#{2}'\nthree"], @opts + assert_format ~S['one\n"#{2}"\nthree'], ~S[~c'one\n"#{2}"\nthree'], @opts + end + + test "with interpolation on line limit" do + assert_format ~S""" + 'one #{"two"} three' + """, + ~S""" + ~c"one #{"two"} three" + """, + @short_length ++ @opts + end + + test "literal new lines don't count towards line limit" do + assert_format ~S""" + 'one + #{"two"} + three' + """, + ~S""" + ~c"one + #{"two"} + three" + """, + @short_length ++ @opts + end + + test "heredocs without escapes" do + assert_format ~S""" + ''' + hello + ''' + """, + ~S''' + ~c""" + hello + """ + ''', + @opts + end + + test "heredocs with escapes" do + assert_format ~S""" + ''' + f\a\b\ro + ''' + """, + ~S''' + ~c""" + f\a\b\ro + """ + ''', + @opts + + assert_format ~S""" + ''' + multiple "\"" quotes + ''' + """, + ~S''' + ~c""" + multiple "\"" quotes + """ + ''', + @opts + end + + test "heredocs with interpolation" do + assert_format ~S""" + ''' + one + #{2} + three + ''' + """, + ~S''' + ~c""" + one + #{2} + three + """ + ''', + @opts + + assert_format ~S""" + ''' + one + " + #{2} + " + three + ''' + """, + ~S''' + ~c""" + one + " + #{2} + " + three + """ + ''', + @opts + end + + test "heredocs with interpolation on line limit" do + assert_format ~S""" + ''' + one #{"two two"} three + ''' + """, + ~S''' + ~c""" + one #{"two two"} three + """ + ''', + @short_length ++ @opts + end + + test "heredocs literal new lines don't count towards line limit" do + assert_format ~S""" + ''' + one + #{"two"} + three + ''' + """, + ~S''' + ~c""" + one + #{"two"} + three + """ + ''', + @short_length ++ @opts + end + end + + describe "migrate_unless: true" do + @opts [migrate_unless: true] + + test "rewrites unless as an if with negated condition" do + bad = "unless x, do: y" + + good = "if !x, do: y" + + assert_format bad, good, @opts + + bad = """ + unless x do + y + else + z + end + """ + + good = """ + if !x do + y + else + z + end + """ + + assert_format bad, good, @opts + end + + test "rewrites pipelines with negated condition" do + bad = "x |> unless(do: y)" + + good = "!x |> if(do: y)" + + assert_format bad, good, @opts + + bad = "x |> foo() |> unless(do: y)" + + good = "x |> foo() |> Kernel.!() |> if(do: y)" + + assert_format bad, good, @opts + + bad = "unless x |> foo(), do: y" + good = "if !(x |> foo()), do: y" + + assert_format bad, good, @opts + end + + test "rewrites in as not in" do + assert_format "unless x in y, do: 1", "if x not in y, do: 1", @opts + end + + test "rewrites equality operators" do + assert_format "unless x == y, do: 1", "if x != y, do: 1", @opts + assert_format "unless x === y, do: 1", "if x !== y, do: 1", @opts + assert_format "unless x != y, do: 1", "if x == y, do: 1", @opts + assert_format "unless x !== y, do: 1", "if x === y, do: 1", @opts + end + + test "rewrites boolean or is_* conditions with not" do + assert_format "unless x > 0, do: 1", "if not (x > 0), do: 1", @opts + assert_format "unless is_atom(x), do: 1", "if not is_atom(x), do: 1", @opts + end + + test "removes ! or not in condition" do + assert_format "unless not x, do: 1", "if x, do: 1", @opts + assert_format "unless !x, do: 1", "if x, do: 1", @opts + end + + test "does nothing without the migrate_unless option" do + assert_same "unless x, do: y" + assert_same "unless x, do: y, else: z" + end + end + + describe "migrate: true" do + test "enables :migrate_bitstring_modifiers" do + assert_format "<>", "<>", migrate: true + end + + test "enables :migrate_call_parens_on_pipe" do + bad = "x |> y" + + good = "x |> y()" + + assert_format bad, good, migrate: true + end + + test "enables :migrate_charlists_as_sigils" do + assert_format ~S['abc'], ~S[~c"abc"], migrate: true + end + + test "enables :migrate_unless" do + bad = "unless x, do: y" + + good = "if !x, do: y" + + assert_format bad, good, migrate: true + end + end +end diff --git a/lib/elixir/test/elixir/code_formatter/operators_test.exs b/lib/elixir/test/elixir/code_formatter/operators_test.exs new file mode 100644 index 00000000000..e897d5729f4 --- /dev/null +++ b/lib/elixir/test/elixir/code_formatter/operators_test.exs @@ -0,0 +1,986 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Code.Formatter.OperatorsTest do + use ExUnit.Case, async: true + + import CodeFormatterHelpers + + @short_length [line_length: 10] + @medium_length [line_length: 20] + + describe "nullary" do + test "formats symbol operators" do + assert_same ".." + end + + test "combines with unary and binary operators" do + assert_same "not .." + assert_same "left = .." + assert_same ".. = right" + end + + test "is wrapped in parentheses on ambiguous calls" do + assert_same "require (..)" + assert_same "require foo, (..)" + assert_same "require (..), bar" + assert_same "require(..)" + assert_same "require(foo, ..)" + assert_same "require(.., bar)" + + assert_same "assert [.., :ok]" + assert_same "assert {.., :ok}" + assert_same "assert (..) == 0..-1//1" + assert_same "assert 0..-1//1 == (..)" + + assert_same """ + defmacro (..) do + :ok + end\ + """ + + assert_format "Range.range? (..)", "Range.range?(..)" + end + end + + describe "unary" do + test "formats symbol operators without spaces" do + assert_format "+ 1", "+1" + assert_format "- 1", "-1" + assert_format "! 1", "!1" + assert_format "^ 1", "^1" + end + + test "formats word operators with spaces" do + assert_same "not 1" + assert_same "not true" + end + + test "wraps operand if it is a unary or binary operator" do + assert_format "!+1", "!(+1)" + assert_format "+ +1", "+(+1)" + assert_format "not +1", "not (+1)" + assert_format "!not 1", "!(not 1)" + assert_format "not !1", "not (!1)" + assert_format "not(!1)", "not (!1)" + assert_format "not(1 + 1)", "not (1 + 1)" + assert_format "-(2**2)", "-(2 ** 2)" + end + + test "wraps operand in ambiguous calls" do + assert_same "def -(2 ** 2)" + assert_same "def -var" + assert_format "def (-(2 ** 2))", "def -(2 ** 2)" + assert_format "def --var", "def -- var" + assert_format "def -+var", "def -(+var)" + end + + test "does not wrap operand if it is a nestable operator" do + assert_format "! ! var", "!!var" + assert_same "not not var" + end + + test "nests operand" do + bad = "+foo(bar, baz, bat)" + + good = """ + +foo( + bar, + baz, + bat + ) + """ + + assert_format bad, good, @short_length + + operator = """ + +assert foo, + bar + """ + + assert_same operator, @short_length + end + + test "does not nest operand" do + bad = "not foo(bar, baz, bat)" + + good = """ + not foo( + bar, + baz, + bat + ) + """ + + assert_format bad, good, @short_length + + operator = """ + not assert foo, + bar + """ + + assert_same operator, @short_length + end + + test "inside do-end block" do + assert_same """ + if +value do + true + end + """ + end + end + + describe "binary without space" do + test "formats without spaces" do + assert_format "1 .. 2", "1..2" + end + + test "never breaks" do + assert_same "123_456_789..987_654_321", @short_length + end + end + + describe "ternary without space" do + test "formats without spaces" do + assert_format "1 .. 2 // 3", "1..2//3" + assert_same "(1..2//3).step" + end + + test "never breaks" do + assert_same "123_456_789..987_654_321//147_268_369", @short_length + end + end + + describe "binary without newline" do + test "formats without spaces" do + assert_same "1 in 2" + assert_format "1\\\\2", "1 \\\\ 2" + end + + test "never breaks" do + assert_same "123_456_789 in 987_654_321", @short_length + end + + test "not in" do + assert_format "not(foo in bar)", "foo not in bar" + + assert_same "foo not in bar" + assert_same "(not foo) in bar" + assert_same "(!foo) in bar" + end + + test "bitwise precedence" do + assert_format "(crc >>> 8) ||| byte", "crc >>> 8 ||| byte" + assert_same "crc >>> (8 ||| byte)" + end + end + + describe "binary operators with preceding new line" do + test "formats with spaces" do + assert_format "1|>2", "1 |> 2" + end + + test "breaks into new line" do + bad = "123_456_789 |> 987_654_321" + + good = """ + 123_456_789 + |> 987_654_321 + """ + + assert_format bad, good, @short_length + + bad = "123 |> foo(bar, baz)" + + good = """ + 123 + |> foo( + bar, + baz + ) + """ + + assert_format bad, good, @short_length + + bad = "123 |> foo(bar) |> bar(bat)" + + good = """ + 123 + |> foo( + bar + ) + |> bar( + bat + ) + """ + + assert_format bad, good, @short_length + + bad = "foo(bar, 123 |> bar(baz))" + + good = """ + foo( + bar, + 123 + |> bar( + baz + ) + ) + """ + + assert_format bad, good, @short_length + + bad = "foo(bar, baz) |> 123" + + good = """ + foo( + bar, + baz + ) + |> 123 + """ + + assert_format bad, good, @short_length + + bad = "foo(bar, baz) |> 123 |> 456" + + good = """ + foo( + bar, + baz + ) + |> 123 + |> 456 + """ + + assert_format bad, good, @short_length + + bad = "123 |> foo(bar, baz) |> 456" + + good = """ + 123 + |> foo( + bar, + baz + ) + |> 456 + """ + + assert_format bad, good, @short_length + end + + test "with multiple of the different entry and same precedence" do + assert_same "foo <~> bar ~> baz" + + bad = "foo <~> bar ~> baz" + + good = """ + foo + <~> bar + ~> baz + """ + + assert_format bad, good, @short_length + end + + test "with multiple of the different entry, same precedence and right associative" do + assert_format "foo ++ bar ++ baz -- bat", "foo ++ bar ++ (baz -- bat)" + + assert_format "foo +++ bar +++ baz --- bat", "foo +++ bar +++ (baz --- bat)" + end + + test "preserves user choice even when it fits" do + assert_same """ + foo + |> bar + """ + + assert_same """ + foo = + one + |> two() + |> three() + """ + + bad = """ + foo |> + bar + """ + + good = """ + foo + |> bar + """ + + assert_format bad, good + end + end + + describe "binary with following new line" do + test "formats with spaces" do + assert_format "1++2", "1 ++ 2" + + assert_format "1+++2", "1 +++ 2" + end + + test "breaks into new line" do + bad = "123_456_789 ++ 987_654_321" + + good = """ + 123_456_789 ++ + 987_654_321 + """ + + assert_format bad, good, @short_length + + bad = "123 ++ foo(bar)" + + good = """ + 123 ++ + foo(bar) + """ + + assert_format bad, good, @short_length + + bad = "123 ++ foo(bar, baz)" + + good = """ + 123 ++ + foo( + bar, + baz + ) + """ + + assert_format bad, good, @short_length + + bad = "foo(bar, 123 ++ bar(baz))" + + good = """ + foo( + bar, + 123 ++ + bar( + baz + ) + ) + """ + + assert_format bad, good, @short_length + + bad = "foo(bar, baz) ++ 123" + + good = """ + foo( + bar, + baz + ) ++ 123 + """ + + assert_format bad, good, @short_length + end + + test "with multiple of the same entry and left associative" do + assert_same "foo == bar == baz" + + bad = "a == b == c" + + good = """ + a == b == + c + """ + + assert_format bad, good, @short_length + + bad = "(a == (b == c))" + + good = """ + a == + (b == c) + """ + + assert_format bad, good, @short_length + + bad = "foo == bar == baz" + + good = """ + foo == bar == + baz + """ + + assert_format bad, good, @short_length + + bad = "(foo == (bar == baz))" + + good = """ + foo == + (bar == + baz) + """ + + assert_format bad, good, @short_length + end + + test "with multiple of the same entry and right associative" do + assert_same "foo ++ bar ++ baz" + assert_format "foo -- bar -- baz", "foo -- (bar -- baz)" + assert_same "foo +++ bar +++ baz" + assert_format "foo --- bar --- baz", "foo --- (bar --- baz)" + + bad = "a ++ b ++ c" + + good = """ + a ++ + b ++ c + """ + + assert_format bad, good, @short_length + + bad = "((a ++ b) ++ c)" + + good = """ + (a ++ b) ++ + c + """ + + assert_format bad, good, @short_length + + bad = "foo ++ bar ++ baz" + + good = """ + foo ++ + bar ++ + baz + """ + + assert_format bad, good, @short_length + + bad = "((foo ++ bar) ++ baz)" + + good = """ + (foo ++ + bar) ++ + baz + """ + + assert_format bad, good, @short_length + end + + test "with precedence" do + assert_format "(a + b) == (c + d)", "a + b == c + d" + assert_format "a + (b == c) + d", "a + (b == c) + d" + + bad = "(a + b) == (c + d)" + + good = """ + a + b == + c + d + """ + + assert_format bad, good, @short_length + + bad = "a * (b + c) * d" + + good = """ + a * + (b + c) * + d + """ + + assert_format bad, good, @short_length + + bad = "(one + two) == (three + four)" + + good = """ + one + two == + three + four + """ + + assert_format bad, good, @medium_length + + bad = "one * (two + three) * four" + + good = """ + one * (two + three) * + four + """ + + assert_format bad, good, @medium_length + + bad = "one * (two + three + four) * five" + + good = """ + one * + (two + three + + four) * five + """ + + assert_format bad, good, @medium_length + + bad = "var = one * (two + three + four) * five" + + good = """ + var = + one * + (two + three + + four) * five + """ + + assert_format bad, good, @medium_length + end + + test "with required parens" do + assert_same "(a |> b) ++ (c |> d)" + assert_format "a + b |> c + d", "(a + b) |> (c + d)" + assert_format "a ++ b |> c ++ d", "(a ++ b) |> (c ++ d)" + assert_format "a |> b ++ c |> d", "a |> (b ++ c) |> d" + end + + test "with required parens skips on no parens" do + assert_same "1..2 |> 3..4" + end + + test "with logical operators" do + assert_same "a or b or c" + assert_format "a or b and c", "a or (b and c)" + assert_format "a and b or c", "(a and b) or c" + end + + test "mixed before and after lines" do + bad = "var :: a | b and c | d" + + good = """ + var :: + a + | b and + c + | d + """ + + assert_format bad, good, @short_length + + bad = "var :: a | b and c + d + e + f | g" + + good = """ + var :: + a + | b and + c + d + e + f + | g + """ + + assert_format bad, good, @medium_length + + assert_same """ + var :: + { + :one, + :two + } + | :three + """ + end + + test "preserves user choice even when it fits and left associative" do + assert_same """ + foo + bar + + baz + bat + """ + + assert_same """ + foo + + bar + + baz + + bat + """ + end + + test "preserves user choice even when it fits and right associative" do + bad = """ + foo ++ bar ++ + baz ++ bat + """ + + assert_format bad, """ + foo ++ + bar ++ + baz ++ bat + """ + + assert_same """ + foo ++ + bar ++ + baz ++ + bat + """ + end + end + + # Theoretically it fits under binary operators + # but the goal of this section is to test common idioms. + describe "match" do + test "with calls" do + bad = "var = fun(one, two, three)" + + good = """ + var = + fun( + one, + two, + three + ) + """ + + assert_format bad, good, @short_length + + bad = "fun(one, two, three) = var" + + good = """ + fun( + one, + two, + three + ) = var + """ + + assert_format bad, good, @short_length + + bad = "fun(foo, bar) = fun(baz, bat)" + + good = """ + fun( + foo, + bar + ) = + fun( + baz, + bat + ) + """ + + assert_format bad, good, @short_length + + bad = "fun(foo, bar) = fun(baz, bat)" + + good = """ + fun(foo, bar) = + fun(baz, bat) + """ + + assert_format bad, good, @medium_length + end + + test "with containers" do + bad = "var = [one, two, three]" + + good = """ + var = [ + one, + two, + three + ] + """ + + assert_format bad, good, @short_length + + bad = """ + var = + [one, two, three] + """ + + good = """ + var = + [ + one, + two, + three + ] + """ + + assert_format bad, good, @short_length + + bad = "[one, two, three] = var" + + good = """ + [ + one, + two, + three + ] = var + """ + + assert_format bad, good, @short_length + + bad = "[one, two, three] = foo(bar, baz)" + + good = """ + [one, two, three] = + foo(bar, baz) + """ + + assert_format bad, good, @medium_length + end + + test "with heredoc" do + heredoc = ~S''' + var = """ + one + """ + ''' + + assert_same heredoc, @short_length + + heredoc = ~S''' + var = """ + #{one} + """ + ''' + + assert_same heredoc, @short_length + end + + test "with anonymous functions" do + bad = "var = fn arg1 -> body1; arg2 -> body2 end" + + good = """ + var = fn + arg1 -> + body1 + + arg2 -> + body2 + end + """ + + assert_format bad, good, @short_length + + good = """ + var = fn + arg1 -> body1 + arg2 -> body2 + end + """ + + assert_format bad, good, @medium_length + end + + test "with do-end blocks" do + assert_same """ + var = + case true do + foo -> bar + baz -> bat + end + """ + end + end + + describe "module attributes" do + test "when reading" do + assert_format "@ my_attribute", "@my_attribute" + end + + test "when setting" do + assert_format "@ my_attribute(:some_value)", "@my_attribute :some_value" + end + + test "doesn't split when reading on line limit" do + assert_same "@my_long_attribute", @short_length + end + + test "doesn't split when setting on line limit" do + assert_same "@my_long_attribute :some_value", @short_length + end + + test "with do-end block" do + assert_same """ + @attr (for x <- y do + z + end) + """ + end + + test "is parenthesized when setting inside a call" do + assert_same "my_fun(@foo(bar), baz)" + end + + test "fall back to @ as an operator when needed" do + assert_same "@(1 + 1)" + assert_same "@:foo" + assert_same "+@foo" + assert_same "@@foo" + assert_same "@(+foo)" + assert_same "!(@(1 + 1))" + assert_same "(@Foo).Baz" + assert_same "@bar(1, 2)" + + assert_format "@+1", "@(+1)" + assert_format "@Foo.Baz", "(@Foo).Baz" + assert_format "@(Foo.Bar).Baz", "(@(Foo.Bar)).Baz" + end + + test "with next break fits" do + attribute = ~S''' + @doc """ + foo + """ + ''' + + assert_same attribute + + attribute = ~S''' + @doc foo: """ + bar + """ + ''' + + assert_same attribute + end + + test "without next break fits" do + bad = "@really_long_expr foo + bar" + + good = """ + @really_long_expr foo + + bar + """ + + assert_format bad, good, @short_length + end + + test "with do-end blocks" do + attribute = """ + @doc do + :ok + end + """ + + assert_same attribute, @short_length + + attribute = """ + use (@doc do + :end + end) + """ + + assert_same attribute, @short_length + end + + test "do not rewrite lists to keyword lists" do + assert_same """ + @foo [ + bar: baz + ] + """ + end + end + + describe "capture" do + test "with integers" do + assert_same "&1" + assert_format "&(&1)", "& &1" + assert_format "&(&1.foo)", "& &1.foo" + end + + test "with operators inside" do + assert_format "& +1", "&(+1)" + assert_format "& 1[:foo]", "& 1[:foo]" + assert_format "& not &1", "&(not &1)" + assert_format "& a ++ b", "&(a ++ b)" + assert_format "& &1 && &2", "&(&1 && &2)" + assert_same "&(&1 | &2)" + end + + test "with operators outside" do + assert_same "(& &1) == (& &2)" + assert_same "(& &1) and (& &2)" + assert_same "(&foo/1) and (&bar/1)" + assert_same "[(&IO.puts/1) | &IO.puts/2]" + end + + test "with call expressions" do + assert_format "& local(&1, &2)", "&local(&1, &2)" + assert_format "&-local(&1, &2)", "&(-local(&1, &2))" + end + + test "with blocks" do + bad = "&(1; 2)" + + good = """ + &( + 1 + 2 + ) + """ + + assert_format bad, good + end + + test "with no parens" do + capture = """ + &assert foo, + bar + """ + + assert_same capture, @short_length + end + + test "precedence when combined with calls" do + assert_same "(&Foo).Bar" + assert_format "&(Foo).Bar", "&Foo.Bar" + assert_format "&(Foo.Bar).Baz", "&Foo.Bar.Baz" + end + + test "local/arity" do + assert_format "&(foo/1)", "&foo/1" + assert_format "&(foo/bar)", "&(foo / bar)" + end + + test "operator/arity" do + assert_same "&+/2" + assert_same "&and/2" + assert_same "& &&/2" + assert_same "& &/1" + assert_same "&//2" + end + + test "Module.remote/arity" do + assert_format "&(Mod.foo/1)", "&Mod.foo/1" + assert_format "&(Mod.++/1)", "&Mod.++/1" + assert_format ~s[&(Mod."foo bar"/1)], ~s[&Mod."foo bar"/1] + assert_format ~S[&(Mod."foo\nbar"/1)], ~S[&Mod."foo\nbar"/1] + + # Invalid + assert_format "& Mod.foo/bar", "&(Mod.foo() / bar)" + + # This is "invalid" as a special form but we don't + # have enough knowledge to know that, so let's just + # make sure we format it properly with proper wrapping. + assert_same "&(1 + 2).foo/1" + + assert_same "&my_function.foo.bar/3", @short_length + end + end + + describe "when" do + test "with keywords" do + assert_same "foo when bar: :baz" + end + + test "with keywords on line breaks" do + bad = "foo when one: :two, three: :four" + + good = """ + foo + when one: :two, + three: :four + """ + + assert_format bad, good, @medium_length + end + end +end diff --git a/lib/elixir/test/elixir/code_fragment_test.exs b/lib/elixir/test/elixir/code_fragment_test.exs new file mode 100644 index 00000000000..497ca3ae0bf --- /dev/null +++ b/lib/elixir/test/elixir/code_fragment_test.exs @@ -0,0 +1,1507 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team + +Code.require_file("test_helper.exs", __DIR__) + +defmodule CodeFragmentTest do + use ExUnit.Case, async: true + + doctest Code.Fragment + alias Code.Fragment, as: CF + + describe "cursor_context/2" do + test "expressions" do + assert CF.cursor_context([]) == :expr + assert CF.cursor_context(",") == :expr + assert CF.cursor_context("[") == :expr + assert CF.cursor_context("<<") == :expr + assert CF.cursor_context("=>") == :expr + assert CF.cursor_context("->") == :expr + assert CF.cursor_context("foo(<<") == :expr + assert CF.cursor_context("hello: ") == :expr + assert CF.cursor_context("\n") == :expr + assert CF.cursor_context(~c"\n") == :expr + assert CF.cursor_context("\n\n") == :expr + assert CF.cursor_context(~c"\n\n") == :expr + assert CF.cursor_context("\r\n") == :expr + assert CF.cursor_context(~c"\r\n") == :expr + assert CF.cursor_context("\r\n\r\n") == :expr + assert CF.cursor_context(~c"\r\n\r\n") == :expr + end + + test "local_or_var" do + assert CF.cursor_context("hello_wo") == {:local_or_var, ~c"hello_wo"} + assert CF.cursor_context("hello_world?") == {:local_or_var, ~c"hello_world?"} + assert CF.cursor_context("hello_world!") == {:local_or_var, ~c"hello_world!"} + assert CF.cursor_context("hello/wor") == {:local_or_var, ~c"wor"} + assert CF.cursor_context("hello..wor") == {:local_or_var, ~c"wor"} + assert CF.cursor_context("hello::wor") == {:local_or_var, ~c"wor"} + assert CF.cursor_context("[hello_wo") == {:local_or_var, ~c"hello_wo"} + assert CF.cursor_context("'hello_wo") == {:local_or_var, ~c"hello_wo"} + assert CF.cursor_context("hellò_wó") == {:local_or_var, ~c"hellò_wó"} + assert CF.cursor_context("hello? world") == {:local_or_var, ~c"world"} + assert CF.cursor_context("hello! world") == {:local_or_var, ~c"world"} + assert CF.cursor_context("hello: world") == {:local_or_var, ~c"world"} + assert CF.cursor_context("__MODULE__") == {:local_or_var, ~c"__MODULE__"} + end + + test "dot" do + assert CF.cursor_context("hello.") == {:dot, {:var, ~c"hello"}, ~c""} + assert CF.cursor_context(":hello.") == {:dot, {:unquoted_atom, ~c"hello"}, ~c""} + assert CF.cursor_context("nested.map.") == {:dot, {:dot, {:var, ~c"nested"}, ~c"map"}, ~c""} + + assert CF.cursor_context("Hello.") == {:dot, {:alias, ~c"Hello"}, ~c""} + assert CF.cursor_context("Hello.World.") == {:dot, {:alias, ~c"Hello.World"}, ~c""} + assert CF.cursor_context("Hello.wor") == {:dot, {:alias, ~c"Hello"}, ~c"wor"} + assert CF.cursor_context("hello.wor") == {:dot, {:var, ~c"hello"}, ~c"wor"} + assert CF.cursor_context("Hello.++") == {:dot, {:alias, ~c"Hello"}, ~c"++"} + assert CF.cursor_context(":hello.wor") == {:dot, {:unquoted_atom, ~c"hello"}, ~c"wor"} + assert CF.cursor_context(":hell@o.wor") == {:dot, {:unquoted_atom, ~c"hell@o"}, ~c"wor"} + assert CF.cursor_context(":he@ll@o.wor") == {:dot, {:unquoted_atom, ~c"he@ll@o"}, ~c"wor"} + assert CF.cursor_context(":hell@@o.wor") == {:dot, {:unquoted_atom, ~c"hell@@o"}, ~c"wor"} + assert CF.cursor_context("@hello.wor") == {:dot, {:module_attribute, ~c"hello"}, ~c"wor"} + + assert CF.cursor_context("@hello. wor") == {:dot, {:module_attribute, ~c"hello"}, ~c"wor"} + assert CF.cursor_context("@hello .wor") == {:dot, {:module_attribute, ~c"hello"}, ~c"wor"} + assert CF.cursor_context("@hello . wor") == {:dot, {:module_attribute, ~c"hello"}, ~c"wor"} + + assert CF.cursor_context("@hello.\nwor") == {:dot, {:module_attribute, ~c"hello"}, ~c"wor"} + assert CF.cursor_context("@hello. \nwor") == {:dot, {:module_attribute, ~c"hello"}, ~c"wor"} + assert CF.cursor_context("@hello.\n wor") == {:dot, {:module_attribute, ~c"hello"}, ~c"wor"} + + assert CF.cursor_context("@hello.\r\nwor") == + {:dot, {:module_attribute, ~c"hello"}, ~c"wor"} + + assert CF.cursor_context("@hello\n.wor") == {:dot, {:module_attribute, ~c"hello"}, ~c"wor"} + assert CF.cursor_context("@hello \n.wor") == {:dot, {:module_attribute, ~c"hello"}, ~c"wor"} + assert CF.cursor_context("@hello\n .wor") == {:dot, {:module_attribute, ~c"hello"}, ~c"wor"} + + assert CF.cursor_context("@hello. # some comment\nwor") == + {:dot, {:module_attribute, ~c"hello"}, ~c"wor"} + + assert CF.cursor_context("@hello. # some comment\n\nwor") == + {:dot, {:module_attribute, ~c"hello"}, ~c"wor"} + + assert CF.cursor_context("@hello. # some comment\nsub\n.wor") == + {:dot, {:dot, {:module_attribute, ~c"hello"}, ~c"sub"}, ~c"wor"} + + assert CF.cursor_context(~c"@hello.\nwor") == + {:dot, {:module_attribute, ~c"hello"}, ~c"wor"} + + assert CF.cursor_context(~c"@hello.\r\nwor") == + {:dot, {:module_attribute, ~c"hello"}, ~c"wor"} + + assert CF.cursor_context(~c"@hello\n.wor") == + {:dot, {:module_attribute, ~c"hello"}, ~c"wor"} + + assert CF.cursor_context(~c"@hello. # some comment\nwor") == + {:dot, {:module_attribute, ~c"hello"}, ~c"wor"} + + assert CF.cursor_context(~c"@hello. # some comment\n\nwor") == + {:dot, {:module_attribute, ~c"hello"}, ~c"wor"} + + assert CF.cursor_context(~c"@hello. # some comment\nsub\n.wor") == + {:dot, {:dot, {:module_attribute, ~c"hello"}, ~c"sub"}, ~c"wor"} + + assert CF.cursor_context("nested.map.wor") == + {:dot, {:dot, {:var, ~c"nested"}, ~c"map"}, ~c"wor"} + + assert CF.cursor_context("__MODULE__.") == {:dot, {:var, ~c"__MODULE__"}, ~c""} + + assert CF.cursor_context("__MODULE__.Sub.") == + {:dot, {:alias, {:local_or_var, ~c"__MODULE__"}, ~c"Sub"}, ~c""} + + assert CF.cursor_context("@hello.Sub.wor") == + {:dot, {:alias, {:module_attribute, ~c"hello"}, ~c"Sub"}, ~c"wor"} + end + + test "local_arity" do + assert CF.cursor_context("hello/") == {:local_arity, ~c"hello"} + end + + test "local_call" do + assert CF.cursor_context("hello\s") == {:local_call, ~c"hello"} + assert CF.cursor_context("hello\t") == {:local_call, ~c"hello"} + assert CF.cursor_context("hello(") == {:local_call, ~c"hello"} + assert CF.cursor_context("hello(\s") == {:local_call, ~c"hello"} + assert CF.cursor_context("hello(\t") == {:local_call, ~c"hello"} + assert CF.cursor_context("hello(\n") == {:local_call, ~c"hello"} + assert CF.cursor_context("hello(\r\n") == {:local_call, ~c"hello"} + end + + test "dot_arity" do + assert CF.cursor_context("Foo.hello/") == {:dot_arity, {:alias, ~c"Foo"}, ~c"hello"} + assert CF.cursor_context("Foo.+/") == {:dot_arity, {:alias, ~c"Foo"}, ~c"+"} + assert CF.cursor_context("Foo . hello /") == {:dot_arity, {:alias, ~c"Foo"}, ~c"hello"} + assert CF.cursor_context("Foo . + /") == {:dot_arity, {:alias, ~c"Foo"}, ~c"+"} + assert CF.cursor_context("foo.hello/") == {:dot_arity, {:var, ~c"foo"}, ~c"hello"} + + assert CF.cursor_context(":foo.hello/") == + {:dot_arity, {:unquoted_atom, ~c"foo"}, ~c"hello"} + + assert CF.cursor_context("@f.hello/") == {:dot_arity, {:module_attribute, ~c"f"}, ~c"hello"} + end + + test "dot_call" do + assert CF.cursor_context("Foo.hello\s") == {:dot_call, {:alias, ~c"Foo"}, ~c"hello"} + assert CF.cursor_context("Foo.hello\t") == {:dot_call, {:alias, ~c"Foo"}, ~c"hello"} + assert CF.cursor_context("Foo.hello(") == {:dot_call, {:alias, ~c"Foo"}, ~c"hello"} + assert CF.cursor_context("Foo.hello(\s") == {:dot_call, {:alias, ~c"Foo"}, ~c"hello"} + assert CF.cursor_context("Foo.hello(\t") == {:dot_call, {:alias, ~c"Foo"}, ~c"hello"} + assert CF.cursor_context("Foo . hello (") == {:dot_call, {:alias, ~c"Foo"}, ~c"hello"} + assert CF.cursor_context("Foo . hello (\s") == {:dot_call, {:alias, ~c"Foo"}, ~c"hello"} + assert CF.cursor_context("Foo . hello (\t") == {:dot_call, {:alias, ~c"Foo"}, ~c"hello"} + + assert CF.cursor_context(":foo.hello\s") == + {:dot_call, {:unquoted_atom, ~c"foo"}, ~c"hello"} + + assert CF.cursor_context(":foo.hello\t") == + {:dot_call, {:unquoted_atom, ~c"foo"}, ~c"hello"} + + assert CF.cursor_context(":foo.hello(") == {:dot_call, {:unquoted_atom, ~c"foo"}, ~c"hello"} + + assert CF.cursor_context(":foo.hello(\s") == + {:dot_call, {:unquoted_atom, ~c"foo"}, ~c"hello"} + + assert CF.cursor_context(":foo.hello(\t") == + {:dot_call, {:unquoted_atom, ~c"foo"}, ~c"hello"} + + assert CF.cursor_context(":foo.hello\s") == + {:dot_call, {:unquoted_atom, ~c"foo"}, ~c"hello"} + + assert CF.cursor_context("foo.hello\s") == {:dot_call, {:var, ~c"foo"}, ~c"hello"} + assert CF.cursor_context("foo.hello\t") == {:dot_call, {:var, ~c"foo"}, ~c"hello"} + assert CF.cursor_context("foo.hello(") == {:dot_call, {:var, ~c"foo"}, ~c"hello"} + assert CF.cursor_context("foo.hello(\s") == {:dot_call, {:var, ~c"foo"}, ~c"hello"} + assert CF.cursor_context("foo.hello(\t") == {:dot_call, {:var, ~c"foo"}, ~c"hello"} + assert CF.cursor_context("foo.hello(\n") == {:dot_call, {:var, ~c"foo"}, ~c"hello"} + assert CF.cursor_context("foo.hello(\r\n") == {:dot_call, {:var, ~c"foo"}, ~c"hello"} + + assert CF.cursor_context("@f.hello\s") == {:dot_call, {:module_attribute, ~c"f"}, ~c"hello"} + assert CF.cursor_context("@f.hello\t") == {:dot_call, {:module_attribute, ~c"f"}, ~c"hello"} + assert CF.cursor_context("@f.hello(") == {:dot_call, {:module_attribute, ~c"f"}, ~c"hello"} + + assert CF.cursor_context("@f.hello(\s") == + {:dot_call, {:module_attribute, ~c"f"}, ~c"hello"} + + assert CF.cursor_context("@f.hello(\t") == + {:dot_call, {:module_attribute, ~c"f"}, ~c"hello"} + + assert CF.cursor_context("Foo.+\s") == {:dot_call, {:alias, ~c"Foo"}, ~c"+"} + assert CF.cursor_context("Foo.+\t") == {:dot_call, {:alias, ~c"Foo"}, ~c"+"} + assert CF.cursor_context("Foo.+(") == {:dot_call, {:alias, ~c"Foo"}, ~c"+"} + assert CF.cursor_context("Foo.+(\s") == {:dot_call, {:alias, ~c"Foo"}, ~c"+"} + assert CF.cursor_context("Foo.+(\t") == {:dot_call, {:alias, ~c"Foo"}, ~c"+"} + assert CF.cursor_context("Foo . + (") == {:dot_call, {:alias, ~c"Foo"}, ~c"+"} + assert CF.cursor_context("Foo . + (\s") == {:dot_call, {:alias, ~c"Foo"}, ~c"+"} + assert CF.cursor_context("Foo . + (\t") == {:dot_call, {:alias, ~c"Foo"}, ~c"+"} + + assert CF.cursor_context("__MODULE__.Foo.hello(") == + {:dot_call, {:alias, {:local_or_var, ~c"__MODULE__"}, ~c"Foo"}, ~c"hello"} + + assert CF.cursor_context("@foo.Foo.hello(") == + {:dot_call, {:alias, {:module_attribute, ~c"foo"}, ~c"Foo"}, ~c"hello"} + end + + test "anonymous_call" do + assert CF.cursor_context("hello.(") == {:anonymous_call, {:var, ~c"hello"}} + assert CF.cursor_context("hello.(\s") == {:anonymous_call, {:var, ~c"hello"}} + assert CF.cursor_context("hello.(\t") == {:anonymous_call, {:var, ~c"hello"}} + assert CF.cursor_context("hello.(\n") == {:anonymous_call, {:var, ~c"hello"}} + assert CF.cursor_context("hello.(\r\n") == {:anonymous_call, {:var, ~c"hello"}} + + assert CF.cursor_context("hello . (") == {:anonymous_call, {:var, ~c"hello"}} + + assert CF.cursor_context("@hello.(") == {:anonymous_call, {:module_attribute, ~c"hello"}} + assert CF.cursor_context("@hello . (") == {:anonymous_call, {:module_attribute, ~c"hello"}} + end + + test "nested expressions" do + assert CF.cursor_context("Hello.world()") == :none + assert CF.cursor_context("hello().") == {:dot, :expr, ~c""} + assert CF.cursor_context("Foo.hello ('(').") == {:dot, :expr, ~c""} + assert CF.cursor_context("Foo.hello('(', ?), ?().bar") == {:dot, :expr, ~c"bar"} + assert CF.cursor_context("Hello.bar(World.call(42), ?), ?().foo") == {:dot, :expr, ~c"foo"} + assert CF.cursor_context("Foo.hello( ).world") == {:dot, :expr, ~c"world"} + assert CF.cursor_context("hello.dyn_impl().call(42).bar") == {:dot, :expr, ~c"bar"} + + assert CF.cursor_context("Foo.dyn_impl().call(") == {:dot_call, :expr, ~c"call"} + assert CF.cursor_context("hello().call(") == {:dot_call, :expr, ~c"call"} + end + + test "alias" do + assert CF.cursor_context("HelloWor") == {:alias, ~c"HelloWor"} + assert CF.cursor_context("Hello.Wor") == {:alias, ~c"Hello.Wor"} + assert CF.cursor_context("Hello.\nWor") == {:alias, ~c"Hello.Wor"} + assert CF.cursor_context("Hello.\r\nWor") == {:alias, ~c"Hello.Wor"} + assert CF.cursor_context("Hello . Wor") == {:alias, ~c"Hello.Wor"} + assert CF.cursor_context("Hello::Wor") == {:alias, ~c"Wor"} + assert CF.cursor_context("Hello..Wor") == {:alias, ~c"Wor"} + + assert CF.cursor_context("hello.World") == + {:alias, {:local_or_var, ~c"hello"}, ~c"World"} + + assert CF.cursor_context("__MODULE__.Wor") == + {:alias, {:local_or_var, ~c"__MODULE__"}, ~c"Wor"} + + assert CF.cursor_context("@foo.Wor") == {:alias, {:module_attribute, ~c"foo"}, ~c"Wor"} + end + + test "structs" do + assert CF.cursor_context("%") == {:struct, ~c""} + assert CF.cursor_context(":%") == {:unquoted_atom, ~c"%"} + assert CF.cursor_context("::%") == {:struct, ~c""} + + assert CF.cursor_context("%HelloWor") == {:struct, ~c"HelloWor"} + + assert CF.cursor_context("%Hello.") == {:struct, {:dot, {:alias, ~c"Hello"}, ~c""}} + assert CF.cursor_context("%Hello.nam") == {:struct, {:dot, {:alias, ~c"Hello"}, ~c"nam"}} + assert CF.cursor_context("%Hello.Wor") == {:struct, ~c"Hello.Wor"} + assert CF.cursor_context("% Hello . Wor") == {:struct, ~c"Hello.Wor"} + + assert CF.cursor_context("%__MODULE_") == {:struct, {:local_or_var, ~c"__MODULE_"}} + assert CF.cursor_context("%__MODULE__") == {:struct, {:local_or_var, ~c"__MODULE__"}} + + assert CF.cursor_context("%__MODULE__.") == + {:struct, {:dot, {:local_or_var, ~c"__MODULE__"}, ~c""}} + + assert CF.cursor_context("%__MODULE__.Sub.") == + {:struct, {:dot, {:alias, {:local_or_var, ~c"__MODULE__"}, ~c"Sub"}, ~c""}} + + assert CF.cursor_context("%__MODULE__.Wor") == + {:struct, {:alias, {:local_or_var, ~c"__MODULE__"}, ~c"Wor"}} + + assert CF.cursor_context("%@foo") == + {:struct, {:module_attribute, ~c"foo"}} + + assert CF.cursor_context("%@foo.") == + {:struct, {:dot, {:module_attribute, ~c"foo"}, ~c""}} + + assert CF.cursor_context("%@foo.Wor") == + {:struct, {:alias, {:module_attribute, ~c"foo"}, ~c"Wor"}} + end + + test "unquoted atom" do + assert CF.cursor_context(":") == {:unquoted_atom, ~c""} + assert CF.cursor_context(":HelloWor") == {:unquoted_atom, ~c"HelloWor"} + assert CF.cursor_context(":HelloWór") == {:unquoted_atom, ~c"HelloWór"} + assert CF.cursor_context(":hello_wor") == {:unquoted_atom, ~c"hello_wor"} + assert CF.cursor_context(":Óla_mundo") == {:unquoted_atom, ~c"Óla_mundo"} + assert CF.cursor_context(":Ol@_mundo") == {:unquoted_atom, ~c"Ol@_mundo"} + assert CF.cursor_context(":Ol@") == {:unquoted_atom, ~c"Ol@"} + assert CF.cursor_context("foo:hello_wor") == {:unquoted_atom, ~c"hello_wor"} + + # Operators from atoms + assert CF.cursor_context(":+") == {:unquoted_atom, ~c"+"} + assert CF.cursor_context(":or") == {:unquoted_atom, ~c"or"} + assert CF.cursor_context(":<") == {:unquoted_atom, ~c"<"} + assert CF.cursor_context(":.") == {:unquoted_atom, ~c"."} + assert CF.cursor_context(":..") == {:unquoted_atom, ~c".."} + assert CF.cursor_context(":->") == {:unquoted_atom, ~c"->"} + assert CF.cursor_context(":%") == {:unquoted_atom, ~c"%"} + end + + test "operators" do + assert CF.cursor_context("/") == {:operator, ~c"/"} + assert CF.cursor_context("+") == {:operator, ~c"+"} + assert CF.cursor_context("++") == {:operator, ~c"++"} + assert CF.cursor_context("!") == {:operator, ~c"!"} + assert CF.cursor_context("<") == {:operator, ~c"<"} + assert CF.cursor_context("<<<") == {:operator, ~c"<<<"} + assert CF.cursor_context("..") == {:operator, ~c".."} + assert CF.cursor_context("<~") == {:operator, ~c"<~"} + assert CF.cursor_context("=~") == {:operator, ~c"=~"} + assert CF.cursor_context("<~>") == {:operator, ~c"<~>"} + assert CF.cursor_context("::") == {:operator, ~c"::"} + + assert CF.cursor_context("+ ") == {:operator_call, ~c"+"} + assert CF.cursor_context("++ ") == {:operator_call, ~c"++"} + assert CF.cursor_context("! ") == {:operator_call, ~c"!"} + assert CF.cursor_context("< ") == {:operator_call, ~c"<"} + assert CF.cursor_context("<<< ") == {:operator_call, ~c"<<<"} + assert CF.cursor_context(".. ") == {:operator_call, ~c".."} + assert CF.cursor_context("<~ ") == {:operator_call, ~c"<~"} + assert CF.cursor_context("=~ ") == {:operator_call, ~c"=~"} + assert CF.cursor_context("<~> ") == {:operator_call, ~c"<~>"} + assert CF.cursor_context(":: ") == {:operator_call, ~c"::"} + + assert CF.cursor_context("...(") == {:operator_call, ~c"..."} + assert CF.cursor_context("...(\s") == {:operator_call, ~c"..."} + assert CF.cursor_context("+(") == {:operator_call, ~c"+"} + assert CF.cursor_context("++(\s") == {:operator_call, ~c"++"} + + assert CF.cursor_context("+/") == {:operator_arity, ~c"+"} + assert CF.cursor_context("++/") == {:operator_arity, ~c"++"} + assert CF.cursor_context("!/") == {:operator_arity, ~c"!"} + assert CF.cursor_context("/") == {:operator_arity, ~c"<~>"} + assert CF.cursor_context("::/") == {:operator_arity, ~c"::"} + + # Unknown operators altogether + assert CF.cursor_context("***") == :none + + # Textual operators are shown as local_or_var UNLESS there is space + assert CF.cursor_context("when") == {:local_or_var, ~c"when"} + assert CF.cursor_context("when ") == {:operator_call, ~c"when"} + assert CF.cursor_context("when.") == :none + + assert CF.cursor_context("not") == {:local_or_var, ~c"not"} + assert CF.cursor_context("not ") == {:operator_call, ~c"not"} + assert CF.cursor_context("not.") == :none + end + + test "sigil" do + assert CF.cursor_context("~") == {:sigil, ~c""} + assert CF.cursor_context("~ ") == :none + + assert CF.cursor_context("~r") == {:sigil, ~c"r"} + assert CF.cursor_context("~r/") == :none + assert CF.cursor_context("~r<") == :none + + assert CF.cursor_context("~r''") == :none + assert CF.cursor_context("~r' '") == :none + assert CF.cursor_context("~r'foo'") == :none + + # The slash is used in sigils, arities, and operators, so there is ambiguity + assert CF.cursor_context("~r//") == {:operator, ~c"/"} + assert CF.cursor_context("~r/ /") == {:operator, ~c"/"} + assert CF.cursor_context("~r/foo/") == {:local_arity, ~c"foo"} + + assert CF.cursor_context("~R") == {:sigil, ~c"R"} + assert CF.cursor_context("~R/") == :none + assert CF.cursor_context("~R<") == :none + + assert CF.cursor_context("Foo.~") == :none + assert CF.cursor_context("Foo.~ ") == :none + end + + test "module attribute" do + assert CF.cursor_context("@") == {:module_attribute, ~c""} + assert CF.cursor_context("@hello_wo") == {:module_attribute, ~c"hello_wo"} + end + + test "keyword or binary operator" do + # Literals + assert CF.cursor_context("Foo.Bar ") == {:block_keyword_or_binary_operator, ~c""} + assert CF.cursor_context("Foo ") == {:block_keyword_or_binary_operator, ~c""} + assert CF.cursor_context(":foo ") == {:block_keyword_or_binary_operator, ~c""} + assert CF.cursor_context("123 ") == {:block_keyword_or_binary_operator, ~c""} + assert CF.cursor_context("nil ") == {:block_keyword_or_binary_operator, ~c""} + assert CF.cursor_context("true ") == {:block_keyword_or_binary_operator, ~c""} + assert CF.cursor_context("false ") == {:block_keyword_or_binary_operator, ~c""} + assert CF.cursor_context("\"foo\" ") == {:block_keyword_or_binary_operator, ~c""} + assert CF.cursor_context("'foo' ") == {:block_keyword_or_binary_operator, ~c""} + + # Containers + assert CF.cursor_context("(foo) ") == {:block_keyword_or_binary_operator, ~c""} + assert CF.cursor_context("[foo] ") == {:block_keyword_or_binary_operator, ~c""} + assert CF.cursor_context("{foo} ") == {:block_keyword_or_binary_operator, ~c""} + assert CF.cursor_context("<> ") == {:block_keyword_or_binary_operator, ~c""} + + # False positives + assert CF.cursor_context("foo ~>> ") == {:operator_call, ~c"~>>"} + assert CF.cursor_context("foo >>> ") == {:operator_call, ~c">>>"} + end + + test "keyword from keyword or binary operator" do + # Literals + assert CF.cursor_context("Foo.Bar do") == {:block_keyword_or_binary_operator, ~c"do"} + assert CF.cursor_context("Foo.Bar d") == {:block_keyword_or_binary_operator, ~c"d"} + assert CF.cursor_context("Foo d") == {:block_keyword_or_binary_operator, ~c"d"} + assert CF.cursor_context(":foo d") == {:block_keyword_or_binary_operator, ~c"d"} + assert CF.cursor_context("123 d") == {:block_keyword_or_binary_operator, ~c"d"} + assert CF.cursor_context("nil d") == {:block_keyword_or_binary_operator, ~c"d"} + assert CF.cursor_context("true d") == {:block_keyword_or_binary_operator, ~c"d"} + assert CF.cursor_context("false d") == {:block_keyword_or_binary_operator, ~c"d"} + assert CF.cursor_context("\"foo\" d") == {:block_keyword_or_binary_operator, ~c"d"} + assert CF.cursor_context("'foo' d") == {:block_keyword_or_binary_operator, ~c"d"} + + # Containers + assert CF.cursor_context("(foo) d") == {:block_keyword_or_binary_operator, ~c"d"} + assert CF.cursor_context("[foo] d") == {:block_keyword_or_binary_operator, ~c"d"} + assert CF.cursor_context("{foo} d") == {:block_keyword_or_binary_operator, ~c"d"} + assert CF.cursor_context("<> d") == {:block_keyword_or_binary_operator, ~c"d"} + + # False positives + assert CF.cursor_context("foo ~>> d") == {:local_or_var, ~c"d"} + assert CF.cursor_context("foo >>> d") == {:local_or_var, ~c"d"} + end + + test "operator from keyword or binary operator" do + # Literals + assert CF.cursor_context("Foo.Bar +") == {:operator, ~c"+"} + assert CF.cursor_context("Foo +") == {:operator, ~c"+"} + assert CF.cursor_context(":foo +") == {:operator, ~c"+"} + assert CF.cursor_context("123 +") == {:operator, ~c"+"} + assert CF.cursor_context("nil +") == {:operator, ~c"+"} + assert CF.cursor_context("true +") == {:operator, ~c"+"} + assert CF.cursor_context("false +") == {:operator, ~c"+"} + assert CF.cursor_context("\"foo\" +") == {:operator, ~c"+"} + assert CF.cursor_context("'foo' +") == {:operator, ~c"+"} + + # Containers + assert CF.cursor_context("(foo) +") == {:operator, ~c"+"} + assert CF.cursor_context("[foo] +") == {:operator, ~c"+"} + assert CF.cursor_context("{foo} +") == {:operator, ~c"+"} + assert CF.cursor_context("<> +") == {:operator, ~c"+"} + + # False positives + assert CF.cursor_context("foo ~>> +") == {:operator, ~c"+"} + assert CF.cursor_context("foo >>> +") == {:operator, ~c"+"} + end + + test "none" do + # Punctuation + assert CF.cursor_context(")") == :none + assert CF.cursor_context("}") == :none + assert CF.cursor_context(">>") == :none + assert CF.cursor_context("'") == :none + assert CF.cursor_context("\"") == :none + + # Numbers + assert CF.cursor_context("123") == :none + assert CF.cursor_context("123?") == :none + assert CF.cursor_context("123!") == :none + assert CF.cursor_context("123var?") == :none + assert CF.cursor_context("0x") == :none + + # Codepoints + assert CF.cursor_context("?") == :none + assert CF.cursor_context("?a") == :none + assert CF.cursor_context("?foo") == :none + + # Dots + assert CF.cursor_context(".") == :none + assert CF.cursor_context("Mundo.Óla") == :none + assert CF.cursor_context(":hello.World") == :none + + # Aliases + assert CF.cursor_context("Hello::Wór") == :none + assert CF.cursor_context("ÓlaMundo") == :none + assert CF.cursor_context("HelloWór") == :none + assert CF.cursor_context("@Hello") == :none + assert CF.cursor_context("Hello(") == :none + + # Identifier + assert CF.cursor_context("foo@bar") == :none + assert CF.cursor_context("@foo@bar") == :none + end + + test "newlines" do + assert CF.cursor_context("this+does-not*matter\nHello.") == + {:dot, {:alias, ~c"Hello"}, ~c""} + + assert CF.cursor_context(~c"this+does-not*matter\nHello.") == + {:dot, {:alias, ~c"Hello"}, ~c""} + + assert CF.cursor_context("this+does-not*matter\r\nHello.") == + {:dot, {:alias, ~c"Hello"}, ~c""} + + assert CF.cursor_context(~c"this+does-not*matter\r\nHello.") == + {:dot, {:alias, ~c"Hello"}, ~c""} + end + end + + describe "surround_context/2" do + test "newlines" do + for i <- 1..8 do + assert CF.surround_context("\n\nhello_wo\n", {3, i}) == %{ + context: {:local_or_var, ~c"hello_wo"}, + begin: {3, 1}, + end: {3, 9} + } + + assert CF.surround_context("\r\n\r\nhello_wo\r\n", {3, i}) == %{ + context: {:local_or_var, ~c"hello_wo"}, + begin: {3, 1}, + end: {3, 9} + } + + assert CF.surround_context(~c"\r\n\r\nhello_wo\r\n", {3, i}) == %{ + context: {:local_or_var, ~c"hello_wo"}, + begin: {3, 1}, + end: {3, 9} + } + end + end + + test "column out of range" do + assert CF.surround_context("hello", {1, 20}) == :none + end + + test "local_or_var" do + for i <- 1..8 do + assert CF.surround_context("hello_wo", {1, i}) == %{ + context: {:local_or_var, ~c"hello_wo"}, + begin: {1, 1}, + end: {1, 9} + } + end + + assert CF.surround_context("hello_wo", {1, 9}) == :none + + for i <- 2..9 do + assert CF.surround_context(" hello_wo", {1, i}) == %{ + context: {:local_or_var, ~c"hello_wo"}, + begin: {1, 2}, + end: {1, 10} + } + end + + assert CF.surround_context(" hello_wo", {1, 10}) == :none + + for i <- 1..6 do + assert CF.surround_context("hello!", {1, i}) == %{ + context: {:local_or_var, ~c"hello!"}, + begin: {1, 1}, + end: {1, 7} + } + end + + assert CF.surround_context("hello!", {1, 7}) == :none + + for i <- 1..5 do + assert CF.surround_context("안녕_세상", {1, i}) == %{ + context: {:local_or_var, ~c"안녕_세상"}, + begin: {1, 1}, + end: {1, 6} + } + end + + assert CF.surround_context("안녕_세상", {1, 6}) == :none + + # Keywords are not local or var + for keyword <- ~w(do end after catch else rescue fn true false nil)c, + length = length(keyword), + i <- 1..length do + assert CF.surround_context(keyword, {1, i}) == %{ + context: {:keyword, keyword}, + begin: {1, 1}, + end: {1, length + 1} + } + + assert CF.surround_context(~c"Foo " ++ keyword, {1, 4 + i}) == %{ + context: {:keyword, keyword}, + begin: {1, 5}, + end: {1, length + 5} + } + end + end + + test "local call" do + for i <- 1..8 do + assert CF.surround_context("hello_wo(", {1, i}) == %{ + context: {:local_call, ~c"hello_wo"}, + begin: {1, 1}, + end: {1, 9} + } + end + + assert CF.surround_context("hello_wo(", {1, 9}) == :none + + for i <- 1..8 do + assert CF.surround_context("hello_wo (", {1, i}) == %{ + context: {:local_call, ~c"hello_wo"}, + begin: {1, 1}, + end: {1, 9} + } + end + + assert CF.surround_context("hello_wo (", {1, 9}) == :none + + for i <- 1..6 do + assert CF.surround_context("hello!(", {1, i}) == %{ + context: {:local_call, ~c"hello!"}, + begin: {1, 1}, + end: {1, 7} + } + end + + assert CF.surround_context("hello!(", {1, 7}) == :none + + for i <- 1..5 do + assert CF.surround_context("안녕_세상(", {1, i}) == %{ + context: {:local_call, ~c"안녕_세상"}, + begin: {1, 1}, + end: {1, 6} + } + end + + assert CF.surround_context("안녕_세상(", {1, 6}) == :none + end + + test "local arity" do + for i <- 1..8 do + assert CF.surround_context("hello_wo/", {1, i}) == %{ + context: {:local_arity, ~c"hello_wo"}, + begin: {1, 1}, + end: {1, 9} + } + end + + assert CF.surround_context("hello_wo/", {1, 9}) == :none + + for i <- 1..8 do + assert CF.surround_context("hello_wo /", {1, i}) == %{ + context: {:local_arity, ~c"hello_wo"}, + begin: {1, 1}, + end: {1, 9} + } + end + + assert CF.surround_context("hello_wo /", {1, 9}) == :none + + for i <- 1..6 do + assert CF.surround_context("hello!/", {1, i}) == %{ + context: {:local_arity, ~c"hello!"}, + begin: {1, 1}, + end: {1, 7} + } + end + + assert CF.surround_context("hello!/", {1, 7}) == :none + + for i <- 1..5 do + assert CF.surround_context("안녕_세상/", {1, i}) == %{ + context: {:local_arity, ~c"안녕_세상"}, + begin: {1, 1}, + end: {1, 6} + } + end + + assert CF.surround_context("안녕_세상/", {1, 6}) == :none + end + + test "textual operators" do + for op <- ~w(when not or and in), i <- 1..byte_size(op) do + assert CF.surround_context("#{op}", {1, i}) == %{ + context: {:operator, String.to_charlist(op)}, + begin: {1, 1}, + end: {1, byte_size(op) + 1} + } + + assert CF.surround_context("Foo #{op}", {1, 4 + i}) == %{ + context: {:operator, String.to_charlist(op)}, + begin: {1, 5}, + end: {1, byte_size(op) + 5} + } + end + end + + test "dot" do + for i <- 1..5 do + assert CF.surround_context("Hello.wor", {1, i}) == %{ + context: {:alias, ~c"Hello"}, + begin: {1, 1}, + end: {1, 6} + } + end + + for i <- 6..9 do + assert CF.surround_context("Hello.wor", {1, i}) == %{ + context: {:dot, {:alias, ~c"Hello"}, ~c"wor"}, + begin: {1, 1}, + end: {1, 10} + } + end + + assert CF.surround_context("Hello.", {1, 6}) == :none + + for i <- 1..5 do + assert CF.surround_context("Hello . wor", {1, i}) == %{ + context: {:alias, ~c"Hello"}, + begin: {1, 1}, + end: {1, 6} + } + end + + for i <- 6..11 do + assert CF.surround_context("Hello . wor", {1, i}) == %{ + context: {:dot, {:alias, ~c"Hello"}, ~c"wor"}, + begin: {1, 1}, + end: {1, 12} + } + end + + assert CF.surround_context("Hello .", {1, 6}) == :none + + for i <- 1..5 do + assert CF.surround_context("hello.wor", {1, i}) == %{ + context: {:local_or_var, ~c"hello"}, + begin: {1, 1}, + end: {1, 6} + } + end + + for i <- 6..9 do + assert CF.surround_context("hello.wor", {1, i}) == %{ + context: {:dot, {:var, ~c"hello"}, ~c"wor"}, + begin: {1, 1}, + end: {1, 10} + } + end + + assert CF.surround_context("hello # comment\n .wor", {2, 4}) == %{ + context: {:dot, {:var, ~c"hello"}, ~c"wor"}, + begin: {1, 1}, + end: {2, 7} + } + + assert CF.surround_context("123 + hello. # comment\n\n wor", {3, 4}) == %{ + context: {:dot, {:var, ~c"hello"}, ~c"wor"}, + begin: {1, 7}, + end: {3, 6} + } + + assert CF.surround_context("hello. # comment\n\n # wor", {3, 5}) == %{ + context: {:local_or_var, ~c"wor"}, + begin: {3, 4}, + end: {3, 7} + } + end + + test "alias" do + for i <- 1..8 do + assert CF.surround_context("HelloWor", {1, i}) == %{ + context: {:alias, ~c"HelloWor"}, + begin: {1, 1}, + end: {1, 9} + } + end + + assert CF.surround_context("HelloWor", {1, 9}) == :none + + for i <- 2..9 do + assert CF.surround_context(" HelloWor", {1, i}) == %{ + context: {:alias, ~c"HelloWor"}, + begin: {1, 2}, + end: {1, 10} + } + end + + assert CF.surround_context(" HelloWor", {1, 10}) == :none + + for i <- 1..9 do + assert CF.surround_context("Hello.Wor", {1, i}) == %{ + context: {:alias, ~c"Hello.Wor"}, + begin: {1, 1}, + end: {1, 10} + } + end + + assert CF.surround_context("Hello.Wor", {1, 10}) == :none + + for i <- 1..11 do + assert CF.surround_context("Hello . Wor", {1, i}) == %{ + context: {:alias, ~c"Hello.Wor"}, + begin: {1, 1}, + end: {1, 12} + } + end + + assert CF.surround_context("Hello . Wor", {1, 12}) == :none + + for i <- 1..15 do + assert CF.surround_context("Foo . Bar . Baz", {1, i}) == %{ + context: {:alias, ~c"Foo.Bar.Baz"}, + begin: {1, 1}, + end: {1, 16} + } + end + + for i <- 1..3 do + assert CF.surround_context("Foo # dc\n. Bar .\n Baz", {i, 1}) == %{ + context: {:alias, ~c"Foo.Bar.Baz"}, + begin: {1, 1}, + end: {3, 5} + } + end + + for i <- 1..11 do + assert CF.surround_context("Foo.Bar.Baz.foo(bar)", {1, i}) == %{ + context: {:alias, ~c"Foo.Bar.Baz"}, + begin: {1, 1}, + end: {1, 12} + } + end + end + + test "underscored special forms" do + assert CF.surround_context("__MODULE__", {1, 1}) == %{ + context: {:local_or_var, ~c"__MODULE__"}, + begin: {1, 1}, + end: {1, 11} + } + + for i <- 1..14 do + assert CF.surround_context("__MODULE__.Foo", {1, i}) == %{ + context: {:alias, {:local_or_var, ~c"__MODULE__"}, ~c"Foo"}, + begin: {1, 1}, + end: {1, 15} + } + end + + for i <- 1..18 do + assert CF.surround_context("__MODULE__.Foo.Sub", {1, i}) == %{ + context: {:alias, {:local_or_var, ~c"__MODULE__"}, ~c"Foo.Sub"}, + begin: {1, 1}, + end: {1, 19} + } + end + + assert CF.surround_context("%__MODULE__{}", {1, 5}) == %{ + context: {:struct, {:local_or_var, ~c"__MODULE__"}}, + begin: {1, 1}, + end: {1, 12} + } + + assert CF.surround_context("%__MODULE__.Foo{}", {1, 13}) == %{ + context: {:struct, {:alias, {:local_or_var, ~c"__MODULE__"}, ~c"Foo"}}, + begin: {1, 1}, + end: {1, 16} + } + + assert CF.surround_context("%__MODULE__.Foo.Sub{}", {1, 17}) == %{ + context: {:struct, {:alias, {:local_or_var, ~c"__MODULE__"}, ~c"Foo.Sub"}}, + begin: {1, 1}, + end: {1, 20} + } + + assert CF.surround_context("__MODULE__.call()", {1, 13}) == %{ + context: {:dot, {:var, ~c"__MODULE__"}, ~c"call"}, + begin: {1, 1}, + end: {1, 16} + } + + assert CF.surround_context("__MODULE__.Foo.call()", {1, 17}) == %{ + context: {:dot, {:alias, {:local_or_var, ~c"__MODULE__"}, ~c"Foo"}, ~c"call"}, + begin: {1, 1}, + end: {1, 20} + } + + assert CF.surround_context("__MODULE__.Foo.Sub.call()", {1, 21}) == %{ + context: {:dot, {:alias, {:local_or_var, ~c"__MODULE__"}, ~c"Foo.Sub"}, ~c"call"}, + begin: {1, 1}, + end: {1, 24} + } + + assert CF.surround_context("__ENV__.module.call()", {1, 17}) == %{ + context: {:dot, {:dot, {:var, ~c"__ENV__"}, ~c"module"}, ~c"call"}, + begin: {1, 1}, + end: {1, 20} + } + end + + test "attribute submodules" do + for i <- 1..9 do + assert CF.surround_context("@some.Foo", {1, i}) == %{ + context: {:alias, {:module_attribute, ~c"some"}, ~c"Foo"}, + begin: {1, 1}, + end: {1, 10} + } + end + + for i <- 1..13 do + assert CF.surround_context("@some.Foo.Sub", {1, i}) == %{ + context: {:alias, {:module_attribute, ~c"some"}, ~c"Foo.Sub"}, + begin: {1, 1}, + end: {1, 14} + } + end + + assert CF.surround_context("%@some{}", {1, 5}) == %{ + context: {:struct, {:module_attribute, ~c"some"}}, + begin: {1, 1}, + end: {1, 7} + } + + assert CF.surround_context("%@some.Foo{}", {1, 10}) == %{ + context: {:struct, {:alias, {:module_attribute, ~c"some"}, ~c"Foo"}}, + begin: {1, 1}, + end: {1, 11} + } + + assert CF.surround_context("%@some.Foo.Sub{}", {1, 14}) == %{ + context: {:struct, {:alias, {:module_attribute, ~c"some"}, ~c"Foo.Sub"}}, + begin: {1, 1}, + end: {1, 15} + } + + assert CF.surround_context("@some.call()", {1, 8}) == %{ + context: {:dot, {:module_attribute, ~c"some"}, ~c"call"}, + begin: {1, 1}, + end: {1, 11} + } + + assert CF.surround_context("@some.Foo.call()", {1, 12}) == %{ + context: {:dot, {:alias, {:module_attribute, ~c"some"}, ~c"Foo"}, ~c"call"}, + begin: {1, 1}, + end: {1, 15} + } + + assert CF.surround_context("@some.Foo.Sub.call()", {1, 16}) == %{ + context: {:dot, {:alias, {:module_attribute, ~c"some"}, ~c"Foo.Sub"}, ~c"call"}, + begin: {1, 1}, + end: {1, 19} + } + end + + test "struct" do + assert CF.surround_context("%", {1, 1}) == :none + assert CF.surround_context("::%", {1, 1}) == :none + assert CF.surround_context("::%", {1, 2}) == :none + assert CF.surround_context("::%Hello", {1, 1}) == :none + assert CF.surround_context("::%Hello", {1, 2}) == :none + + assert CF.surround_context("::%Hello", {1, 3}) == %{ + context: {:struct, ~c"Hello"}, + begin: {1, 3}, + end: {1, 9} + } + + assert CF.surround_context("::% Hello", {1, 3}) == %{ + context: {:struct, ~c"Hello"}, + begin: {1, 3}, + end: {1, 10} + } + + assert CF.surround_context("::% Hello", {1, 4}) == %{ + context: {:struct, ~c"Hello"}, + begin: {1, 3}, + end: {1, 10} + } + + # Alias + assert CF.surround_context("%HelloWor", {1, 1}) == %{ + context: {:struct, ~c"HelloWor"}, + begin: {1, 1}, + end: {1, 10} + } + + assert CF.surround_context("%HelloWor.some", {1, 12}) == %{ + context: {:struct, {:dot, {:alias, ~c"HelloWor"}, ~c"some"}}, + begin: {1, 1}, + end: {1, 15} + } + + for i <- 2..9 do + assert CF.surround_context("%HelloWor", {1, i}) == %{ + context: {:struct, ~c"HelloWor"}, + begin: {1, 1}, + end: {1, 10} + } + end + + assert CF.surround_context("%HelloWor", {1, 10}) == :none + + # With dot + assert CF.surround_context("%Hello.Wor", {1, 1}) == %{ + context: {:struct, ~c"Hello.Wor"}, + begin: {1, 1}, + end: {1, 11} + } + + for i <- 2..10 do + assert CF.surround_context("%Hello.Wor", {1, i}) == %{ + context: {:struct, ~c"Hello.Wor"}, + begin: {1, 1}, + end: {1, 11} + } + end + + assert CF.surround_context("%Hello.Wor", {1, 11}) == :none + + # With spaces + assert CF.surround_context("% Hello . Wor", {1, 1}) == %{ + context: {:struct, ~c"Hello.Wor"}, + begin: {1, 1}, + end: {1, 14} + } + + for i <- 2..13 do + assert CF.surround_context("% Hello . Wor", {1, i}) == %{ + context: {:struct, ~c"Hello.Wor"}, + begin: {1, 1}, + end: {1, 14} + } + end + + assert CF.surround_context("% Hello . Wor", {1, 14}) == :none + end + + test "module attributes" do + for i <- 1..10 do + assert CF.surround_context("@hello_wor", {1, i}) == %{ + context: {:module_attribute, ~c"hello_wor"}, + begin: {1, 1}, + end: {1, 11} + } + end + + assert CF.surround_context("@Hello", {1, 1}) == :none + end + + test "operators" do + for i <- 2..4 do + assert CF.surround_context("1<<<3", {1, i}) == %{ + context: {:operator, ~c"<<<"}, + begin: {1, 2}, + end: {1, 5} + } + end + + for i <- 3..5 do + assert CF.surround_context("1 <<< 3", {1, i}) == %{ + context: {:operator, ~c"<<<"}, + begin: {1, 3}, + end: {1, 6} + } + end + + for i <- 2..3 do + assert CF.surround_context("1::3", {1, i}) == %{ + context: {:operator, ~c"::"}, + begin: {1, 2}, + end: {1, 4} + } + end + + for i <- 3..4 do + assert CF.surround_context("1 :: 3", {1, i}) == %{ + context: {:operator, ~c"::"}, + begin: {1, 3}, + end: {1, 5} + } + end + + for i <- 2..3 do + assert CF.surround_context("x..y", {1, i}) == %{ + context: {:operator, ~c".."}, + begin: {1, 2}, + end: {1, 4} + } + end + + for i <- 3..4 do + assert CF.surround_context("x .. y", {1, i}) == %{ + context: {:operator, ~c".."}, + begin: {1, 3}, + end: {1, 5} + } + end + + assert CF.surround_context("@", {1, 1}) == %{ + context: {:operator, ~c"@"}, + begin: {1, 1}, + end: {1, 2} + } + + assert CF.surround_context("!", {1, 1}) == %{ + context: {:operator, ~c"!"}, + begin: {1, 1}, + end: {1, 2} + } + + assert CF.surround_context("!foo", {1, 1}) == %{ + context: {:operator, ~c"!"}, + begin: {1, 1}, + end: {1, 2} + } + + assert CF.surround_context("foo !bar", {1, 5}) == %{ + context: {:operator, ~c"!"}, + begin: {1, 5}, + end: {1, 6} + } + + # invalid + assert CF.surround_context("->", {1, 2}) == :none + end + + test "sigil" do + assert CF.surround_context("~", {1, 1}) == :none + assert CF.surround_context("~~r", {1, 1}) == :none + assert CF.surround_context("~~r", {1, 2}) == :none + + assert CF.surround_context("~r/foo/", {1, 1}) == %{ + begin: {1, 1}, + context: {:sigil, ~c"r"}, + end: {1, 3} + } + + assert CF.surround_context("~r/foo/", {1, 2}) == %{ + begin: {1, 1}, + context: {:sigil, ~c"r"}, + end: {1, 3} + } + + assert CF.surround_context("~r/foo/", {1, 3}) == :none + + assert CF.surround_context("~R", {1, 1}) == %{ + begin: {1, 1}, + context: {:sigil, ~c"R"}, + end: {1, 3} + } + + assert CF.surround_context("~R", {1, 2}) == %{ + begin: {1, 1}, + context: {:sigil, ~c"R"}, + end: {1, 3} + } + + assert CF.surround_context("~R", {1, 3}) == :none + end + + test "dot operator" do + for i <- 4..7 do + assert CF.surround_context("Foo.<<<", {1, i}) == %{ + context: {:dot, {:alias, ~c"Foo"}, ~c"<<<"}, + begin: {1, 1}, + end: {1, 8} + } + end + + for i <- 4..9 do + assert CF.surround_context("Foo . <<<", {1, i}) == %{ + context: {:dot, {:alias, ~c"Foo"}, ~c"<<<"}, + begin: {1, 1}, + end: {1, 10} + } + end + + for i <- 4..6 do + assert CF.surround_context("Foo.::", {1, i}) == %{ + context: {:dot, {:alias, ~c"Foo"}, ~c"::"}, + begin: {1, 1}, + end: {1, 7} + } + end + + for i <- 4..8 do + assert CF.surround_context("Foo . ::", {1, i}) == %{ + context: {:dot, {:alias, ~c"Foo"}, ~c"::"}, + begin: {1, 1}, + end: {1, 9} + } + end + end + + test "capture operator" do + assert CF.surround_context("& &123 + 1", {1, 1}) == %{ + context: {:operator, ~c"&"}, + begin: {1, 1}, + end: {1, 2} + } + + for i <- 3..6 do + assert CF.surround_context("& &123 + 1", {1, i}) == %{ + context: {:capture_arg, ~c"&123"}, + begin: {1, 3}, + end: {1, 7} + } + end + end + + test "capture operator false positive" do + assert CF.surround_context("1&&2", {1, 3}) == %{ + context: {:operator, ~c"&&"}, + begin: {1, 2}, + end: {1, 4} + } + + assert CF.surround_context("1&&2", {1, 4}) == :none + + assert CF.surround_context("&a", {1, 2}) == %{ + context: {:local_or_var, ~c"a"}, + begin: {1, 2}, + end: {1, 3} + } + end + + test "unquoted atom" do + for i <- 1..10 do + assert CF.surround_context(":hello_wor", {1, i}) == %{ + context: {:unquoted_atom, ~c"hello_wor"}, + begin: {1, 1}, + end: {1, 11} + } + end + + for i <- 1..10 do + assert CF.surround_context(":Hello@Wor", {1, i}) == %{ + context: {:unquoted_atom, ~c"Hello@Wor"}, + begin: {1, 1}, + end: {1, 11} + } + end + + assert CF.surround_context(":", {1, 1}) == :none + end + + test "keyword keys" do + for i <- 2..4 do + assert CF.surround_context("[foo:", {1, i}) == %{ + context: {:key, ~c"foo"}, + begin: {1, 2}, + end: {1, 5} + } + end + + for i <- 10..12 do + assert CF.surround_context("[foo: 1, bar: 2]", {1, i}) == %{ + context: {:key, ~c"bar"}, + begin: {1, 10}, + end: {1, 13} + } + end + + assert CF.surround_context("if foo?, do: bar()", {1, 10}) == %{ + context: {:key, ~c"do"}, + begin: {1, 10}, + end: {1, 12} + } + end + + test "keyword false positives" do + assert CF.surround_context("<>") + assert cc2q!("foo do") == s2q!("foo do __cursor__() end") + assert cc2q!("foo do true else") == s2q!("foo do true else __cursor__() end") + end + + test "inside interpolation" do + assert cc2q!(~S|"foo #{(|) == s2q!(~S|"foo #{(__cursor__())}"|) + assert cc2q!(~S|"foo #{"bar #{{|) == s2q!(~S|"foo #{"bar #{{__cursor__()}}"}"|) + end + + test "keeps operators" do + assert cc2q!("1 + 2") == s2q!("1 + __cursor__()") + assert cc2q!("&foo") == s2q!("&__cursor__()") + assert cc2q!("&foo/") == s2q!("&foo/__cursor__()") + end + + test "keeps function calls without parens" do + assert cc2q!("alias") == s2q!("__cursor__()") + assert cc2q!("alias ") == s2q!("alias __cursor__()") + assert cc2q!("alias foo") == s2q!("alias __cursor__()") + assert cc2q!("alias Foo.Bar") == s2q!("alias __cursor__()") + assert cc2q!("alias Foo.Bar,") == s2q!("alias Foo.Bar, __cursor__()") + assert cc2q!("alias Foo.Bar, as: ") == s2q!("alias Foo.Bar, as: __cursor__()") + end + + test "do-end blocks" do + assert cc2q!("foo do baz") == s2q!("foo do __cursor__() end") + assert cc2q!("foo do bar; baz") == s2q!("foo do bar; __cursor__() end") + assert cc2q!("foo do bar\nbaz") == s2q!("foo do bar\n__cursor__() end") + + assert cc2q!("foo(bar do baz") == s2q!("foo(bar do __cursor__() end)") + assert cc2q!("foo(bar do baz ") == s2q!("foo(bar do baz(__cursor__()) end)") + assert cc2q!("foo(bar do baz(") == s2q!("foo(bar do baz(__cursor__()) end)") + assert cc2q!("foo(bar do baz bat,") == s2q!("foo(bar do baz(bat, __cursor__()) end)") + + assert cc2q!("foo(bar do baz, bat", trailing_fragment: " -> :ok end") == + s2q!("foo(bar do baz, __cursor__() -> :ok end)") + end + + test "keyword lists" do + assert cc2q!("[bar: ") == s2q!("[bar: __cursor__()]") + assert cc2q!("[bar: baz,") == s2q!("[bar: baz, __cursor__()]") + assert cc2q!("[arg, bar: baz,") == s2q!("[arg, bar: baz, __cursor__()]") + assert cc2q!("[arg: val, bar: baz,") == s2q!("[arg: val, bar: baz, __cursor__()]") + + assert cc2q!("{arg, bar: ") == s2q!("{arg, bar: __cursor__()}") + assert cc2q!("{arg, bar: baz,") == s2q!("{arg, bar: baz, __cursor__()}") + + assert cc2q!("foo(bar: ") == s2q!("foo(bar: __cursor__())") + assert cc2q!("foo(bar: baz,") == s2q!("foo([bar: baz, __cursor__()])") + assert cc2q!("foo(arg, bar: ") == s2q!("foo(arg, bar: __cursor__())") + assert cc2q!("foo(arg, bar: baz,") == s2q!("foo(arg, [bar: baz, __cursor__()])") + assert cc2q!("foo(arg: val, bar: ") == s2q!("foo(arg: val, bar: __cursor__())") + assert cc2q!("foo(arg: val, bar: baz,") == s2q!("foo([arg: val, bar: baz, __cursor__()])") + + assert cc2q!("foo bar: ") == s2q!("foo(bar: __cursor__())") + assert cc2q!("foo bar: baz,") == s2q!("foo([bar: baz, __cursor__()])") + assert cc2q!("foo arg, bar: ") == s2q!("foo(arg, bar: __cursor__())") + assert cc2q!("foo arg, bar: baz,") == s2q!("foo(arg, [bar: baz, __cursor__()])") + assert cc2q!("foo arg: val, bar: ") == s2q!("foo(arg: val, bar: __cursor__())") + assert cc2q!("foo arg: val, bar: baz,") == s2q!("foo([arg: val, bar: baz, __cursor__()])") + end + + test "maps and structs" do + assert cc2q!("%") == s2q!("__cursor__()") + assert cc2q!("%{") == s2q!("%{__cursor__()}") + assert cc2q!("%{bar:") == s2q!("%{__cursor__()}") + assert cc2q!("%{bar: ") == s2q!("%{bar: __cursor__()}") + assert cc2q!("%{bar: baz,") == s2q!("%{bar: baz, __cursor__()}") + assert cc2q!("%{foo | ") == s2q!("%{foo | __cursor__()}") + assert cc2q!("%{foo | bar:") == s2q!("%{foo | __cursor__()}") + assert cc2q!("%{foo | bar: ") == s2q!("%{foo | bar: __cursor__()}") + assert cc2q!("%{foo | bar: baz,") == s2q!("%{foo | bar: baz, __cursor__()}") + + assert cc2q!("%Foo") == s2q!("__cursor__()") + assert cc2q!("%Foo{") == s2q!("%Foo{__cursor__()}") + assert cc2q!("%Foo{bar: ") == s2q!("%Foo{bar: __cursor__()}") + assert cc2q!("%Foo{bar: baz,") == s2q!("%Foo{bar: baz, __cursor__()}") + assert cc2q!("%Foo{foo | ") == s2q!("%Foo{foo | __cursor__()}") + assert cc2q!("%Foo{foo | bar:") == s2q!("%Foo{foo | __cursor__()}") + assert cc2q!("%Foo{foo | bar: ") == s2q!("%Foo{foo | bar: __cursor__()}") + assert cc2q!("%Foo{foo | bar: baz,") == s2q!("%Foo{foo | bar: baz, __cursor__()}") + end + + test "binaries" do + assert cc2q!("<<") == s2q!("<<__cursor__()>>") + assert cc2q!("<>") + assert cc2q!("<>") + assert cc2q!("<>") + end + + test "anonymous functions" do + assert cc2q!("(fn", trailing_fragment: "-> end)") == s2q!("(fn __cursor__() -> nil end)") + + assert cc2q!("(fn", trailing_fragment: "-> 1 + 2 end)") == + s2q!("(fn __cursor__() -> 1 + 2 end)") + + assert cc2q!("(fn x", trailing_fragment: "-> :ok end)") == + s2q!("(fn __cursor__() -> :ok end)") + + assert cc2q!("(fn x", trailing_fragment: ", y -> :ok end)") == + s2q!("(fn __cursor__(), y -> :ok end)") + + assert cc2q!("(fn x,", trailing_fragment: "y -> :ok end)") == + s2q!("(fn x, __cursor__() -> :ok end)") + + assert cc2q!("(fn x,", trailing_fragment: "\ny -> :ok end)") == + s2q!("(fn x, __cursor__()\n -> :ok end)") + + assert cc2q!("(fn x, {", trailing_fragment: "y, z} -> :ok end)") == + s2q!("(fn x, {__cursor__(), z} -> :ok end)") + + assert cc2q!("(fn x, {y", trailing_fragment: ", z} -> :ok end)") == + s2q!("(fn x, {__cursor__(), z} -> :ok end)") + + assert cc2q!("(fn x, {y, ", trailing_fragment: "z} -> :ok end)") == + s2q!("(fn x, {y, __cursor__()} -> :ok end)") + + assert cc2q!("(fn x ->", trailing_fragment: ":ok end)") == + s2q!("(fn x -> __cursor__() end)") + + assert cc2q!("(fn x ->", trailing_fragment: "\n:ok end)") == + s2q!("(fn x -> __cursor__() end)") + + assert cc2q!("(fn x when ", trailing_fragment: "-> :ok end)") == + s2q!("(fn x when __cursor__() -> :ok end)") + + assert cc2q!("(fn x when ", trailing_fragment: "->\n:ok end)") == + s2q!("(fn x when __cursor__() -> :ok end)") + + assert cc2q!("(fn") == s2q!("(__cursor__())") + assert cc2q!("(fn x") == s2q!("(__cursor__())") + assert cc2q!("(fn x,") == s2q!("(__cursor__())") + assert cc2q!("(fn x ->") == s2q!("(fn x -> __cursor__() end)") + assert cc2q!("(fn x -> x") == s2q!("(fn x -> __cursor__() end)") + assert cc2q!("(fn x, y -> x + y") == s2q!("(fn x, y -> x + __cursor__() end)") + assert cc2q!("(fn x, y -> x + y end") == s2q!("(__cursor__())") + end + + test "do -> end" do + assert cc2q!("if do\nx ->\n", trailing_fragment: "y\nz ->\nw\nend") == + s2q!("if do\nx ->\n__cursor__()\nz -> \nw\nend") + + assert cc2q!("if do\nx ->\ny", trailing_fragment: "\nz ->\nw\nend") == + s2q!("if do\nx ->\n__cursor__()\nz -> \nw\nend") + + assert cc2q!("if do\nx ->\ny\n", trailing_fragment: "\nz ->\nw\nend") == + s2q!("if do\nx ->\ny\n__cursor__()\nz -> \nw\nend") + + assert cc2q!("for x <- [], reduce: %{} do\ny, ", trailing_fragment: "-> :ok\nend") == + s2q!("for x <- [], reduce: %{} do\ny, __cursor__() -> :ok\nend") + + assert cc2q!("for x <- [], reduce: %{} do\ny, z when ", trailing_fragment: "-> :ok\nend") == + s2q!("for x <- [], reduce: %{} do\ny, z when __cursor__() -> :ok\nend") + + assert cc2q!("case do\na -> a\nb = ", trailing_fragment: "c -> c\nend") == + s2q!("case do\na -> a\nb = __cursor__() -> c\nend") + end + + test "removes tokens until opening" do + assert cc2q!("(123") == s2q!("(__cursor__())") + assert cc2q!("[foo") == s2q!("[__cursor__()]") + assert cc2q!("{'foo'") == s2q!("{__cursor__()}") + assert cc2q!("foo do :atom") == s2q!("foo do __cursor__() end") + assert cc2q!("foo(:atom") == s2q!("foo(__cursor__())") + end + + test "removes tokens until comma" do + assert cc2q!("[bar, 123") == s2q!("[bar, __cursor__()]") + assert cc2q!("{bar, 'foo'") == s2q!("{bar, __cursor__()}") + assert cc2q!("<>") + assert cc2q!("foo(bar, :atom") == s2q!("foo(bar, __cursor__())") + assert cc2q!("foo bar, :atom") == s2q!("foo(bar, __cursor__())") + end + + test "removes closed terminators" do + assert cc2q!("foo([1, 2, 3]") == s2q!("foo(__cursor__())") + assert cc2q!("foo({1, 2, 3}") == s2q!("foo(__cursor__())") + assert cc2q!("foo((1, 2, 3)") == s2q!("foo(__cursor__())") + assert cc2q!("foo(<<1, 2, 3>>") == s2q!("foo(__cursor__())") + assert cc2q!("foo(bar do :done end") == s2q!("foo(__cursor__())") + end + + test "incomplete expressions" do + assert cc2q!("foo(123, :") == s2q!("foo(123, __cursor__())") + assert cc2q!("foo(123, %") == s2q!("foo(123, __cursor__())") + assert cc2q!("foo(123, 0x") == s2q!("foo(123, __cursor__())") + assert cc2q!("foo(123, ~") == s2q!("foo(123, __cursor__())") + assert cc2q!("foo(123, ~r") == s2q!("foo(123, __cursor__())") + assert cc2q!("foo(123, ~r/") == s2q!("foo(123, __cursor__())") + end + + test "no warnings" do + assert cc2q!(~s"?\\ ") == s2q!("__cursor__()") + assert cc2q!(~s"{fn -> end, ") == s2q!("{fn -> nil end, __cursor__()}") + end + + test "options" do + opts = [columns: true] + assert cc2q!("foo(", opts) == s2q!("foo(__cursor__())", opts) + assert cc2q!("foo(123,", opts) == s2q!("foo(123,__cursor__())", opts) + + opts = [token_metadata: true] + assert cc2q!("foo(", opts) == s2q!("foo(__cursor__())", opts) + assert cc2q!("foo(123,", opts) == s2q!("foo(123,__cursor__())", opts) + + opts = [literal_encoder: fn ast, _ -> {:ok, {:literal, ast}} end] + assert cc2q!("foo(", opts) == s2q!("foo(__cursor__())", opts) + assert cc2q!("foo(123,", opts) == s2q!("foo({:literal, 123},__cursor__())", []) + end + end +end diff --git a/lib/elixir/test/elixir/code_identifier_test.exs b/lib/elixir/test/elixir/code_identifier_test.exs new file mode 100644 index 00000000000..769359b802b --- /dev/null +++ b/lib/elixir/test/elixir/code_identifier_test.exs @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("test_helper.exs", __DIR__) + +defmodule Code.IdentifierTest do + use ExUnit.Case, async: true + doctest Code.Identifier +end diff --git a/lib/elixir/test/elixir/code_normalizer/formatted_ast_test.exs b/lib/elixir/test/elixir/code_normalizer/formatted_ast_test.exs new file mode 100644 index 00000000000..0061feed9db --- /dev/null +++ b/lib/elixir/test/elixir/code_normalizer/formatted_ast_test.exs @@ -0,0 +1,603 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Code.Normalizer.FormatterASTTest do + use ExUnit.Case, async: true + + defmacro assert_same(good, opts \\ []) do + quote bind_quoted: [good: good, opts: opts], location: :keep do + assert IO.iodata_to_binary(Code.format_string!(good, opts)) == + string_to_string(good, opts) + end + end + + def string_to_string(good, opts) do + line_length = Keyword.get(opts, :line_length, 98) + good = String.trim(good) + + to_quoted_opts = + Keyword.merge( + [ + literal_encoder: &{:ok, {:__block__, &2, [&1]}}, + token_metadata: true, + unescape: false + ], + opts + ) + + {quoted, comments} = Code.string_to_quoted_with_comments!(good, to_quoted_opts) + + to_algebra_opts = [comments: comments, escape: false] ++ opts + + quoted + |> Code.quoted_to_algebra(to_algebra_opts) + |> Inspect.Algebra.format(line_length) + |> IO.iodata_to_binary() + end + + describe "integers" do + test "in decimal base" do + assert_same "0" + assert_same "100" + assert_same "007" + assert_same "10000" + assert_same "100_00" + end + + test "in binary base" do + assert_same "0b0" + assert_same "0b1" + assert_same "0b101" + assert_same "0b01" + assert_same "0b111_111" + end + + test "in octal base" do + assert_same "0o77" + assert_same "0o0" + assert_same "0o01" + assert_same "0o777_777" + end + + test "in hex base" do + assert_same "0x1" + assert_same "0x01" + end + + test "as chars" do + assert_same "?a" + assert_same "?1" + assert_same "?è" + assert_same "??" + assert_same "?\\\\" + assert_same "?\\s" + assert_same "?🎾" + end + end + + describe "floats" do + test "with normal notation" do + assert_same "0.0" + assert_same "1.0" + assert_same "123.456" + assert_same "0.0000001" + assert_same "001.100" + assert_same "0_10000_0.000_000" + end + + test "with scientific notation" do + assert_same "1.0e1" + assert_same "1.0e-1" + assert_same "1.0e01" + assert_same "1.0e-01" + assert_same "001.100e-010" + assert_same "0_100_0000.100e-010" + end + end + + describe "atoms" do + test "true, false, nil" do + assert_same "nil" + assert_same "true" + assert_same "false" + end + + test "without escapes" do + assert_same ~S[:foo] + end + + test "with escapes" do + assert_same ~S[:"f\a\b\ro"] + end + + test "with unicode" do + assert_same ~S[:ólá] + end + + test "does not reformat aliases" do + assert_same ~S[:"Elixir.String"] + assert_same ~S[:"Elixir"] + end + + test "quoted operators" do + assert_same ~S[:"::"] + assert_same ~S[:"..//"] + assert_same ~S{["..//": 1]} + end + + test "with interpolation" do + assert_same ~S[:"one #{2} three"] + end + + test "with escapes and interpolation" do + assert_same ~S[:"one\n\"#{2}\"\nthree"] + end + end + + describe "strings" do + test "without escapes" do + assert_same ~S["foo"] + end + + test "with escapes" do + assert_same ~S["\x0A"] + assert_same ~S["f\a\b\ro"] + assert_same ~S["double \" quote"] + end + + test "keeps literal new lines" do + assert_same """ + "fo + o" + """ + end + + test "with interpolation" do + assert_same ~S["one #{} three"] + assert_same ~S["one #{2} three"] + end + + test "with escaped interpolation" do + assert_same ~S["one\#{two}three"] + end + + test "with escapes and interpolation" do + assert_same ~S["one\n\"#{2}\"\nthree"] + end + end + + describe "lists" do + test "on module attribute" do + assert_same ~S"@foo [1]" + end + end + + describe "charlists" do + test "without escapes" do + assert_same ~S[~c""] + assert_same ~S[~c" "] + assert_same ~S[~c"foo"] + end + + test "with escapes" do + assert_same ~S[~c"f\a\b\ro"] + assert_same ~S[~c'single \' quote'] + assert_same ~S[~c"double \" quote"] + end + + test "keeps literal new lines" do + assert_same """ + ~c"fo + o" + """ + end + + test "with interpolation" do + assert_same ~S[~c"one #{2} three"] + end + + test "with escape and interpolation" do + assert_same ~S[~c'one\n\'#{2}\'\nthree'] + end + end + + describe "string heredocs" do + test "without escapes" do + assert_same ~S''' + """ + hello + """ + ''' + end + + test "with escapes" do + assert_same ~S''' + """ + f\a\b\ro + """ + ''' + + assert_same ~S''' + """ + multiple "\"" quotes + """ + ''' + end + + test "with interpolation" do + assert_same ~S''' + """ + one + #{2} + three + """ + ''' + + assert_same ~S''' + """ + one + " + #{2} + " + three + """ + ''' + end + + test "nested with empty lines" do + assert_same ~S''' + nested do + """ + + foo + + + bar + + """ + end + ''' + end + + test "nested with empty lines and interpolation" do + assert_same ~S''' + nested do + """ + + #{foo} + + + #{bar} + + """ + end + ''' + + assert_same ~S''' + nested do + """ + #{foo} + + + #{bar} + """ + end + ''' + end + + test "with escaped new lines" do + assert_same ~S''' + """ + one\ + #{"two"}\ + three\ + """ + ''' + end + end + + describe "charlist heredocs" do + test "without escapes" do + assert_same ~S""" + ~c''' + hello + ''' + """ + end + + test "with escapes" do + assert_same ~S""" + ~c''' + f\a\b\ro + ''' + """ + + assert_same ~S""" + ~c''' + multiple "\"" quotes + ''' + """ + end + + test "with interpolation" do + assert_same ~S""" + ~c''' + one + #{2} + three + ''' + """ + + assert_same ~S""" + ~c''' + one + " + #{2} + " + three + ''' + """ + end + end + + describe "keyword list" do + test "blocks" do + assert_same ~S""" + defmodule Example do + def sample, do: :ok + end + """ + + assert_same ~S""" + with true, do: :ok, else: (_ -> :ok) + """ + end + + test "omitting brackets" do + assert_same ~S""" + @type foo :: a when b: :c + """ + end + + test "last tuple element as keyword list keeps its format" do + assert_same ~S"{:wrapped, [opt1: true, opt2: false]}" + assert_same ~S"{:unwrapped, opt1: true, opt2: false}" + assert_same ~S"{:wrapped, 1, [opt1: true, opt2: false]}" + assert_same ~S"{:unwrapped, 1, opt1: true, opt2: false}" + end + + test "on module attribute" do + assert_same ~S""" + @foo a: b, + c: d + """ + + assert_same ~S"@foo [ + a: b, + c: d + ]" + end + end + + describe "preserves user choice on parenthesis" do + test "in functions with do blocks" do + assert_same(~S""" + foo Bar do + :ok + end + """) + + assert_same(~S""" + foo(Bar) do + :ok + end + """) + end + end + + describe "preserves formatting for sigils" do + test "without interpolation" do + assert_same ~S[~s(foo)] + assert_same ~S[~s{foo bar}] + assert_same ~S[~r/Bar Baz/] + assert_same ~S[~w<>] + assert_same ~S[~W()] + end + + test "with escapes" do + assert_same ~S[~s(foo \) bar)] + assert_same ~S[~s(f\a\b\ro)] + + assert_same ~S""" + ~S(foo\ + bar) + """ + end + + test "with nested new lines" do + assert_same ~S""" + foo do + ~S(foo\ + bar) + end + """ + + assert_same ~S""" + foo do + ~s(#{bar} + ) + end + """ + end + + test "with interpolation" do + assert_same ~S[~s(one #{2} three)] + end + + test "with modifiers" do + assert_same ~S[~w(one two three)a] + assert_same ~S[~z(one two three)foo] + end + + test "with heredoc syntax" do + assert_same ~S""" + ~s''' + one\a + #{:two}\r + three\0 + ''' + """ + + assert_same ~S''' + ~s""" + one\a + #{:two}\r + three\0 + """ + ''' + end + + test "with heredoc syntax and modifier" do + assert_same ~S""" + ~s''' + foo + '''rsa + """ + end + end + + describe "preserves comments formatting" do + test "before and after expressions" do + assert_same """ + # before comment + :hello + """ + + assert_same """ + :hello + # after comment + """ + + assert_same """ + # before comment + :hello + # after comment + """ + end + + test "empty comment" do + assert_same """ + # + :foo + """ + end + + test "handles comments with unescaped literal" do + assert_same """ + # before + Mix.install([:foo]) + # after + """, + literal_encoder: fn literal, _ -> {:ok, literal} end + + assert_same """ + # before + Mix.install([1 + 2, :foo]) + # after + """, + literal_encoder: fn literal, _ -> {:ok, literal} end + + assert_same """ + # before + Mix.install([:foo, 1 + 2]) + # after + """, + literal_encoder: fn literal, _ -> {:ok, literal} end + + assert_same """ + block do + # before 1 + 1 + 1 + + # before 2 + 2 + 2 + end + """, + literal_encoder: fn literal, _ -> {:ok, literal} end + + assert_same """ + block do + # before 1 + Mix.install([1 + 1]) + + # before 2 + Mix.install([2 + 2]) + end + """, + literal_encoder: fn literal, _ -> {:ok, literal} end + end + + test "before and after expressions with newlines" do + assert_same """ + # before comment + # second line + + :hello + + # middle comment 1 + + # + + # middle comment 2 + + :world + + # after comment + # second line + """ + end + + test "interpolation with comment outside before and after" do + assert_same ~S""" + # comment + IO.puts("Hello #{world}") + """ + + assert_same ~S""" + IO.puts("Hello #{world}") + # comment + """ + end + + test "blocks with keyword list" do + assert_same ~S""" + defp sample do + [ + # comment + {:a, "~> 1.2"} + ] + end + """ + + assert_same ~S""" + defp sample do + [ + # comment + {:a, "~> 1.2"}, + {:b, "~> 1.2"} + ] + end + """ + end + + test "keyword literals with variable values" do + assert_same(~S""" + foo = foo() + [foo: foo] + """) + end + end +end diff --git a/lib/elixir/test/elixir/code_normalizer/quoted_ast_test.exs b/lib/elixir/test/elixir/code_normalizer/quoted_ast_test.exs new file mode 100644 index 00000000000..c10fdf041a2 --- /dev/null +++ b/lib/elixir/test/elixir/code_normalizer/quoted_ast_test.exs @@ -0,0 +1,823 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team + +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Code.Normalizer.QuotedASTTest do + use ExUnit.Case, async: true + + describe "quoted_to_algebra/2" do + test "variable" do + assert quoted_to_string(quote(do: foo)) == "foo" + assert quoted_to_string({:{}, [], nil}) == "{}" + end + + test "variable with colors" do + opts = [syntax_colors: [variable: :blue]] + assert quoted_to_string(quote(do: foo), opts) == "\e[34mfoo\e[0m" + end + + test "local call" do + assert quoted_to_string(quote(do: foo(1, 2, 3))) == "foo(1, 2, 3)" + assert quoted_to_string(quote(do: foo([1, 2, 3]))) == "foo([1, 2, 3])" + + assert quoted_to_string(quote(do: foo(1, 2, 3)), locals_without_parens: [foo: 3]) == + "foo 1, 2, 3" + + # Mixing literals and non-literals + assert quoted_to_string(quote(do: foo(a, 2))) == "foo(a, 2)" + assert quoted_to_string(quote(do: foo(1, b))) == "foo(1, b)" + + # Mixing literals and non-literals with line + assert quoted_to_string(quote(line: __ENV__.line, do: foo(a, 2))) == "foo(a, 2)" + assert quoted_to_string(quote(line: __ENV__.line, do: foo(1, b))) == "foo(1, b)" + end + + test "local call with colors" do + opts = [syntax_colors: [call: :blue, number: :yellow, variable: :red]] + + assert quoted_to_string(quote(do: foo(1, a)), opts) == + "\e[34mfoo\e[0m(\e[33m1\e[0m, \e[31ma\e[0m)" + end + + test "local call no parens" do + assert quoted_to_string({:def, [], [1, 2]}) == "def 1, 2" + assert quoted_to_string({:def, [closing: []], [1, 2]}) == "def(1, 2)" + end + + test "remote call" do + assert quoted_to_string(quote(do: foo.bar(1, 2, 3))) == "foo.bar(1, 2, 3)" + assert quoted_to_string(quote(do: foo.bar([1, 2, 3]))) == "foo.bar([1, 2, 3])" + + quoted = + quote do + (foo do + :ok + end).bar([1, 2, 3]) + end + + assert quoted_to_string(quoted) == "(foo do\n :ok\n end).bar([1, 2, 3])" + end + + test "nullary remote call" do + assert quoted_to_string(quote do: foo.bar) == "foo.bar" + assert quoted_to_string(quote do: foo.bar()) == "foo.bar()" + end + + test "atom remote call" do + assert quoted_to_string(quote(do: :foo.bar(1, 2, 3))) == ":foo.bar(1, 2, 3)" + end + + test "remote and fun call" do + assert quoted_to_string(quote(do: foo.bar().(1, 2, 3))) == "foo.bar().(1, 2, 3)" + assert quoted_to_string(quote(do: foo.bar().([1, 2, 3]))) == "foo.bar().([1, 2, 3])" + end + + test "unusual remote atom fun call" do + assert quoted_to_string(quote(do: Foo."42"())) == ~s/Foo."42"()/ + assert quoted_to_string(quote(do: Foo."Bar"())) == ~s/Foo."Bar"()/ + assert quoted_to_string(quote(do: Foo."bar baz"().""())) == ~s/Foo."bar baz"().""()/ + assert quoted_to_string(quote(do: Foo."%{}"())) == ~s/Foo."%{}"()/ + assert quoted_to_string(quote(do: Foo."..."())) == ~s/Foo."..."()/ + end + + test "atom fun call" do + assert quoted_to_string(quote(do: :foo.(1, 2, 3))) == ":foo.(1, 2, 3)" + end + + test "aliases call" do + assert quoted_to_string(quote(do: Foo.Bar.baz(1, 2, 3))) == "Foo.Bar.baz(1, 2, 3)" + assert quoted_to_string(quote(do: Foo.Bar.baz([1, 2, 3]))) == "Foo.Bar.baz([1, 2, 3])" + assert quoted_to_string(quote(do: ?0.Bar.baz([1, 2, 3]))) == "48.Bar.baz([1, 2, 3])" + assert quoted_to_string(quote(do: Foo.bar(<<>>, []))) == "Foo.bar(<<>>, [])" + end + + test "remote call with colors" do + opts = [syntax_colors: [call: :blue, number: :yellow, variable: :red, atom: :green]] + + assert quoted_to_string(quote(do: foo.bar(1, 2)), opts) == + "\e[31mfoo\e[0m.\e[34mbar\e[0m(\e[33m1\e[0m, \e[33m2\e[0m)" + + assert quoted_to_string(quote(do: :foo.bar(1, 2)), opts) == + "\e[32m:foo\e[0m.\e[34mbar\e[0m(\e[33m1\e[0m, \e[33m2\e[0m)" + + assert quoted_to_string(quote(do: Foo.Bar.bar(1, 2)), opts) == + "\e[32mFoo.Bar\e[0m.\e[34mbar\e[0m(\e[33m1\e[0m, \e[33m2\e[0m)" + end + + test "keyword call" do + assert quoted_to_string(quote(do: Foo.bar(foo: :bar))) == "Foo.bar(foo: :bar)" + assert quoted_to_string(quote(do: Foo.bar("Elixir.Foo": :bar))) == "Foo.bar([{Foo, :bar}])" + end + + test "sigil call" do + assert quoted_to_string(quote(do: ~r"123")) == ~S/~r"123"/ + assert quoted_to_string(quote(do: ~r"\n123")) == ~S/~r"\n123"/ + assert quoted_to_string(quote(do: ~r"12\"3")) == ~S/~r"12\"3"/ + assert quoted_to_string(quote(do: ~r/12\/3/u)) == ~S"~r/12\/3/u" + assert quoted_to_string(quote(do: ~r{\n123})) == ~S/~r{\n123}/ + assert quoted_to_string(quote(do: ~r((1\)(2\)3))) == ~S/~r((1\)(2\)3)/ + assert quoted_to_string(quote(do: ~r{\n1{1\}23})) == ~S/~r{\n1{1\}23}/ + assert quoted_to_string(quote(do: ~r|12\|3|)) == ~S"~r|12\|3|" + + assert quoted_to_string(quote(do: ~r[1#{two}3])) == ~S/~r[1#{two}3]/ + assert quoted_to_string(quote(do: ~r|1[#{two}]3|)) == ~S/~r|1[#{two}]3|/ + assert quoted_to_string(quote(do: ~r'1#{two}3'u)) == ~S/~r'1#{two}3'u/ + + assert quoted_to_string(quote(do: ~R"123")) == ~S/~R"123"/ + assert quoted_to_string(quote(do: ~R"123"u)) == ~S/~R"123"u/ + assert quoted_to_string(quote(do: ~R"\n123")) == ~S/~R"\n123"/ + + assert quoted_to_string(quote(do: ~S["'(123)'"])) == ~S/~S["'(123)'"]/ + assert quoted_to_string(quote(do: ~s"#{"foo"}")) == ~S/~s"#{"foo"}"/ + + assert quoted_to_string(quote(do: ~S["'(123)'"]) |> strip_metadata()) == ~S/~S"\"'(123)'\""/ + assert quoted_to_string(quote(do: ~s"#{"foo"}") |> strip_metadata()) == ~S/~s"#{"foo"}"/ + + assert quoted_to_string( + quote do + ~s""" + "\""foo"\"" + """ + end + ) == ~s[~s"""\n"\\""foo"\\""\n"""] + + assert quoted_to_string( + quote do + ~s''' + '\''foo'\'' + ''' + end + ) == ~s[~s'''\n'\\''foo'\\''\n'''] + + assert quoted_to_string( + quote do + ~s""" + "\"foo\"" + """ + end + ) == ~s[~s"""\n"\\"foo\\""\n"""] + + assert quoted_to_string( + quote do + ~s''' + '\"foo\"' + ''' + end + ) == ~s[~s'''\n'\\"foo\\"'\n'''] + + assert quoted_to_string( + quote do + ~S""" + "123" + """ + end + ) == ~s[~S"""\n"123"\n"""] + end + + test "regression: invalid sigil calls" do + assert quoted_to_string(quote do: sigil_r(<<"foo", 123>>, [])) == + "sigil_r(<<\"foo\", 123>>, [])" + + assert quoted_to_string(quote do: sigil_r(<<"foo">>, :invalid_modifiers)) == + "sigil_r(\"foo\", :invalid_modifiers)" + + assert quoted_to_string(quote do: sigil_r(<<"foo">>, [:invalid_modifier])) == + "sigil_r(\"foo\", [:invalid_modifier])" + + assert quoted_to_string(quote do: sigil_r(<<"foo">>, [])) == "~r\"foo\"" + assert quoted_to_string(quote do: sigil_r(<<"foo">>, [?a, ?b, ?c])) == "~r\"foo\"abc" + end + + test "tuple" do + assert quoted_to_string(quote do: {1, 2}) == "{1, 2}" + assert quoted_to_string(quote do: {1}) == "{1}" + assert quoted_to_string(quote do: {1, 2, 3}) == "{1, 2, 3}" + assert quoted_to_string(quote do: {1, 2, 3, foo: :bar}) == "{1, 2, 3, foo: :bar}" + end + + test "tuple with colors" do + opts = [syntax_colors: [tuple: :blue, number: :yellow]] + + assert quoted_to_string(quote(do: {1, 2, 3}), opts) == + "\e[34m{\e[0m\e[33m1\e[0m, \e[33m2\e[0m, \e[33m3\e[0m\e[34m}\e[0m" + end + + test "tuple call" do + assert quoted_to_string(quote(do: alias(Foo.{Bar, Baz, Bong}))) == + "alias Foo.{Bar, Baz, Bong}" + + assert quoted_to_string(quote(do: foo(Foo.{}))) == "foo(Foo.{})" + end + + test "arrow" do + assert quoted_to_string(quote(do: foo(1, (2 -> 3)))) == "foo(1, (2 -> 3))" + end + + test "block" do + quoted = + quote do + 1 + 2 + + ( + :foo + :bar + ) + + 3 + end + + expected = """ + 1 + 2 + + ( + :foo + :bar + ) + + 3 + """ + + assert quoted_to_string(quoted) <> "\n" == expected + end + + test "not in" do + assert quoted_to_string(quote(do: false not in [])) == "false not in []" + end + + test "if else" do + expected = """ + if foo do + bar + else + baz + end + """ + + assert quoted_to_string(quote(do: if(foo, do: bar, else: baz))) <> "\n" == expected + end + + test "case" do + quoted = + quote do + case foo do + true -> + 0 + + false -> + 1 + 2 + end + end + + expected = """ + case foo do + true -> + 0 + + false -> + 1 + 2 + end + """ + + assert quoted_to_string(quoted) <> "\n" == expected + end + + test "case if else" do + expected = """ + case (if foo do + bar + else + baz + end) do + end + """ + + assert quoted_to_string( + quote( + do: + case if(foo, do: bar, else: baz) do + end + ) + ) <> "\n" == expected + end + + test "try" do + quoted = + quote do + try do + foo + catch + _, _ -> + 2 + rescue + ArgumentError -> + 1 + after + 4 + else + _ -> + 3 + end + end + + expected = """ + try do + foo + catch + _, _ -> 2 + rescue + ArgumentError -> 1 + after + 4 + else + _ -> 3 + end + """ + + assert quoted_to_string(quoted) <> "\n" == expected + end + + test "fn" do + assert quoted_to_string(quote(do: fn -> 1 + 2 end)) == "fn -> 1 + 2 end" + assert quoted_to_string(quote(do: fn x -> x + 1 end)) == "fn x -> x + 1 end" + + quoted = + quote do + fn x -> + y = x + 1 + y + end + end + + expected = """ + fn x -> + y = x + 1 + y + end + """ + + assert quoted_to_string(quoted) <> "\n" == expected + + quoted = + quote do + fn + x -> + y = x + 1 + y + + z -> + z + end + end + + expected = """ + fn + x -> + y = x + 1 + y + + z -> + z + end + """ + + assert quoted_to_string(quoted) <> "\n" == expected + + assert quoted_to_string(quote(do: (fn x -> x end).(1))) == "(fn x -> x end).(1)" + + quoted = + quote do + (fn + %{} -> :map + _ -> :other + end).(1) + end + + expected = """ + (fn + %{} -> :map + _ -> :other + end).(1) + """ + + assert quoted_to_string(quoted) <> "\n" == expected + end + + test "range" do + assert quoted_to_string(quote(do: -1..+2)) == "-1..+2" + assert quoted_to_string(quote(do: Foo.integer()..3)) == "Foo.integer()..3" + assert quoted_to_string(quote(do: -1..+2//-3)) == "-1..+2//-3" + + assert quoted_to_string(quote(do: Foo.integer()..3//Bar.bat())) == + "Foo.integer()..3//Bar.bat()" + + # invalid AST + assert quoted_to_string(-1..+2) == "-1..2" + assert quoted_to_string(-1..+2//-3) == "-1..2//-3" + end + + test "when" do + assert quoted_to_string(quote(do: (-> x))) == "(-> x)" + assert quoted_to_string(quote(do: (x when y -> z))) == "(x when y -> z)" + assert quoted_to_string(quote(do: (x, y when z -> w))) == "(x, y when z -> w)" + assert quoted_to_string(quote(do: (x, y when z -> w))) == "(x, y when z -> w)" + assert quoted_to_string(quote(do: (x, y when z -> w))) == "(x, y when z -> w)" + assert quoted_to_string(quote(do: (x when y: z))) == "x when y: z" + assert quoted_to_string(quote(do: (x when y: z, z: w))) == "x when y: z, z: w" + end + + test "nested" do + quoted = + quote do + defmodule Foo do + def foo do + 1 + 1 + end + end + end + + expected = """ + defmodule Foo do + def foo do + 1 + 1 + end + end + """ + + assert quoted_to_string(quoted) <> "\n" == expected + end + + test "operator precedence" do + assert quoted_to_string(quote(do: (1 + 2) * (3 - 4))) == "(1 + 2) * (3 - 4)" + assert quoted_to_string(quote(do: (1 + 2) * 3 - 4)) == "(1 + 2) * 3 - 4" + assert quoted_to_string(quote(do: 1 + 2 + 3)) == "1 + 2 + 3" + assert quoted_to_string(quote(do: 1 + 2 - 3)) == "1 + 2 - 3" + end + + test "capture operator" do + assert quoted_to_string(quote(do: &foo/0)) == "&foo/0" + assert quoted_to_string(quote(do: &Foo.foo/0)) == "&Foo.foo/0" + assert quoted_to_string(quote(do: &(&1 + &2))) == "&(&1 + &2)" + assert quoted_to_string(quote(do: & &1)) == "& &1" + assert quoted_to_string(quote(do: & &1.(:x))) == "& &1.(:x)" + assert quoted_to_string(quote(do: (& &1).(:x))) == "(& &1).(:x)" + end + + test "operators" do + assert quoted_to_string(quote(do: foo |> {1, 2})) == "foo |> {1, 2}" + assert quoted_to_string(quote(do: foo |> {:map, arg})) == "foo |> {:map, arg}" + end + + test "containers" do + assert quoted_to_string(quote(do: {})) == "{}" + assert quoted_to_string(quote(do: [])) == "[]" + assert quoted_to_string(quote(do: {1, 2, 3})) == "{1, 2, 3}" + assert quoted_to_string(quote(do: [1, 2, 3])) == "[1, 2, 3]" + assert quoted_to_string(quote(do: ["Elixir.Foo": :bar])) == "[{Foo, :bar}]" + assert quoted_to_string(quote(do: %{})) == "%{}" + assert quoted_to_string(quote(do: %{:foo => :bar})) == "%{foo: :bar}" + assert quoted_to_string(quote(do: %{:"Elixir.Foo" => :bar})) == "%{Foo => :bar}" + assert quoted_to_string(quote(do: %{{1, 2} => [1, 2, 3]})) == "%{{1, 2} => [1, 2, 3]}" + assert quoted_to_string(quote(do: %{map | "a" => "b"})) == "%{map | \"a\" => \"b\"}" + assert quoted_to_string(quote(do: [1, 2, 3])) == "[1, 2, 3]" + end + + test "false positive containers" do + assert quoted_to_string({:%{}, [], nil}) == "%{}" + end + + test "struct" do + assert quoted_to_string(quote(do: %Test{})) == "%Test{}" + assert quoted_to_string(quote(do: %Test{foo: 1, bar: 1})) == "%Test{foo: 1, bar: 1}" + assert quoted_to_string(quote(do: %Test{struct | foo: 2})) == "%Test{struct | foo: 2}" + assert quoted_to_string(quote(do: %Test{} + 1)) == "%Test{} + 1" + assert quoted_to_string(quote(do: %Test{foo(1)} + 2)) == "%Test{foo(1)} + 2" + end + + test "binary operators" do + assert quoted_to_string(quote(do: 1 + 2)) == "1 + 2" + assert quoted_to_string(quote(do: [1, 2 | 3])) == "[1, 2 | 3]" + assert quoted_to_string(quote(do: [h | t] = [1, 2, 3])) == "[h | t] = [1, 2, 3]" + assert quoted_to_string(quote(do: (x ++ y) ++ z)) == "(x ++ y) ++ z" + assert quoted_to_string(quote(do: (x +++ y) +++ z)) == "(x +++ y) +++ z" + end + + test "unary operators" do + assert quoted_to_string(quote(do: not 1)) == "not 1" + assert quoted_to_string(quote(do: not foo)) == "not foo" + assert quoted_to_string(quote(do: -1)) == "-1" + assert quoted_to_string(quote(do: +(+1))) == "+(+1)" + assert quoted_to_string(quote(do: !(foo > bar))) == "!(foo > bar)" + assert quoted_to_string(quote(do: @foo(bar))) == "@foo bar" + assert quoted_to_string(quote(do: identity(&1))) == "identity(&1)" + end + + test "operators with colors" do + opts = [syntax_colors: [operator: :blue, number: :yellow]] + assert quoted_to_string(quote(do: !!1), opts) == "\e[34m!\e[0m\e[34m!\e[0m\e[33m1\e[0m" + assert quoted_to_string(quote(do: 1 + 2), opts) == "\e[33m1\e[0m\e[34m +\e[0m \e[33m2\e[0m" + end + + test "access" do + assert quoted_to_string(quote(do: a[b])) == "a[b]" + assert quoted_to_string(quote(do: a[1 + 2])) == "a[1 + 2]" + assert quoted_to_string(quote(do: (a || [a: 1])[:a])) == "(a || [a: 1])[:a]" + assert quoted_to_string(quote(do: Map.put(%{}, :a, 1)[:a])) == "Map.put(%{}, :a, 1)[:a]" + end + + test "keyword list" do + assert quoted_to_string(quote(do: [a: a, b: b])) == "[a: a, b: b]" + assert quoted_to_string(quote(do: [a: 1, b: 1 + 2])) == "[a: 1, b: 1 + 2]" + assert quoted_to_string(quote(do: ["a.b": 1, c: 1 + 2])) == "[\"a.b\": 1, c: 1 + 2]" + + tuple = {{:__block__, [format: :keyword], [:a]}, {:b, [], nil}} + assert quoted_to_string([tuple, :foo, tuple]) == "[{:a, b}, :foo, a: b]" + assert quoted_to_string([tuple, :foo, {:c, :d}, tuple]) == "[{:a, b}, :foo, c: :d, a: b]" + + # Not keyword lists + assert quoted_to_string(quote(do: [{binary(), integer()}])) == "[{binary(), integer()}]" + end + + test "keyword list with colors" do + opts = [syntax_colors: [list: :blue, atom: :green, number: :yellow]] + + assert quoted_to_string(quote(do: [a: 1, b: 2]), opts) == + "\e[34m[\e[0m\e[32ma:\e[0m \e[33m1\e[0m, \e[32mb:\e[0m \e[33m2\e[0m\e[34m]\e[0m" + end + + test "keyword list with :do as operand" do + assert quoted_to_string(quote(do: a = [do: 1])) == "a = [do: 1]" + end + + test "interpolation" do + assert quoted_to_string(quote(do: "foo#{bar}baz")) == ~S["foo#{bar}baz"] + end + + test "bit syntax" do + ast = quote(do: <<1::8*4>>) + assert quoted_to_string(ast) == "<<1::8*4>>" + + ast = quote(do: @type(foo :: <<_::8, _::_*4>>)) + assert quoted_to_string(ast) == "@type foo :: <<_::8, _::_*4>>" + + ast = quote(do: <<69 - 4::bits-size(8 - 4)-unit(1), 65>>) + assert quoted_to_string(ast) == "<<69 - 4::bits-size(8 - 4)-unit(1), 65>>" + + ast = quote(do: <<(<<65>>), 65>>) + assert quoted_to_string(ast) == "<<(<<65>>), 65>>" + + ast = quote(do: <<65, (<<65>>)>>) + assert quoted_to_string(ast) == "<<65, (<<65>>)>>" + + ast = quote(do: for(<<(a::4 <- <<1, 2>>)>>, do: a)) + assert quoted_to_string(ast) == "for <<(a::4 <- <<1, 2>>)>> do\n a\nend" + end + + test "integer/float" do + assert quoted_to_string(1) == "1" + assert quoted_to_string({:__block__, [], [1]}) == "1" + assert quoted_to_string(1.23) == "1.23" + end + + test "integer/float with colors" do + opts = [syntax_colors: [number: :yellow]] + assert quoted_to_string(1, opts) == "\e[33m1\e[0m" + assert quoted_to_string(1.23, opts) == "\e[33m1.23\e[0m" + end + + test "charlist" do + assert quoted_to_string(quote(do: [])) == "[]" + assert quoted_to_string(quote(do: ~c"abc")) == ~S/~c"abc"/ + + # False positive + assert quoted_to_string( + quote do + :"Elixir.List".to_charlist([ + case var do + var -> var + end + ]) + end + ) =~ "List.to_charlist([\n case var do\n var -> var\n end\n])" + end + + test "string" do + assert quoted_to_string(quote(do: "")) == ~S/""/ + assert quoted_to_string(quote(do: "abc")) == ~S/"abc"/ + assert quoted_to_string(quote(do: "#{"abc"}")) == ~S/"#{"abc"}"/ + end + + test "string with colors" do + opts = [syntax_colors: [string: :green]] + assert quoted_to_string(quote(do: "abc"), opts) == "\e[32m\"abc\"\e[0m" + end + + test "catch-all" do + assert quoted_to_string(quote do: {unquote(self())}) == "{#{inspect(self())}}" + assert quoted_to_string(quote do: foo(unquote(self()))) == "foo(#{inspect(self())})" + end + + test "last arg keyword list" do + assert quoted_to_string(quote(do: foo([]))) == "foo([])" + assert quoted_to_string(quote(do: foo(x: y))) == "foo(x: y)" + assert quoted_to_string(quote(do: foo(x: 1 + 2))) == "foo(x: 1 + 2)" + assert quoted_to_string(quote(do: foo(x: y, p: q))) == "foo(x: y, p: q)" + assert quoted_to_string(quote(do: foo(a, x: y, p: q))) == "foo(a, x: y, p: q)" + + assert quoted_to_string(quote(do: {[]})) == "{[]}" + assert quoted_to_string(quote(do: {[a: b]})) == "{[a: b]}" + assert quoted_to_string(quote(do: {x, a: b})) == "{x, a: b}" + assert quoted_to_string(quote(do: foo(else: a))) == "foo(else: a)" + assert quoted_to_string(quote(do: foo(catch: a))) == "foo(catch: a)" + assert quoted_to_string(quote(do: foo |> [bar: :baz])) == "foo |> [bar: :baz]" + end + + test "keyword arg with cursor" do + input = "def foo, do: :bar, __cursor__()" + expected = "def foo, [{:do, :bar}, __cursor__()]" + + ast = Code.string_to_quoted!(input) + assert quoted_to_string(ast) == expected + + encoder = &{:ok, {:__block__, &2, [&1]}} + ast = Code.string_to_quoted!(input, literal_encoder: encoder) + assert quoted_to_string(ast) == expected + + ast = Code.string_to_quoted!(input, token_metadata: true) + assert quoted_to_string(ast) == expected + + ast = Code.string_to_quoted!(input, literal_encoder: encoder, token_metadata: true) + assert quoted_to_string(ast) == expected + end + + test "keyword arg with literal encoder and no metadata" do + input = """ + foo(Bar) do + :ok + end + """ + + encoder = &{:ok, {:__block__, &2, [&1]}} + ast = Code.string_to_quoted!(input, literal_encoder: encoder) + assert quoted_to_string(ast) == "foo(Bar, do: :ok)" + end + + test "list in module attribute" do + assert quoted_to_string( + quote do + @foo [] + end + ) == "@foo []" + + assert quoted_to_string( + quote do + @foo [1] + end + ) == "@foo [1]" + + assert quoted_to_string( + quote do + @foo [foo: :bar] + end + ) == "@foo foo: :bar" + + assert quoted_to_string( + quote do + @foo [1, foo: :bar] + end + ) == "@foo [1, foo: :bar]" + end + end + + describe "quoted_to_algebra/2 escapes" do + test "strings with slash escapes" do + assert quoted_to_string(quote(do: "\a\b\d\e\f\n\r\t\v"), escape: false) == + ~s/"\a\b\d\e\f\n\r\t\v"/ + + assert quoted_to_string(quote(do: "\a\b\d\e\f\n\r\t\v")) == + ~s/"\\a\\b\\d\\e\\f\\n\\r\\t\\v"/ + + assert quoted_to_string({:__block__, [], ["\a\b\d\e\f\n\r\t\v"]}, escape: false) == + ~s/"\a\b\d\e\f\n\r\t\v"/ + + assert quoted_to_string({:__block__, [], ["\a\b\d\e\f\n\r\t\v"]}) == + ~s/"\\a\\b\\d\\e\\f\\n\\r\\t\\v"/ + end + + test "strings with non printable characters" do + assert quoted_to_string(quote(do: "\x00\x01\x10"), escape: false) == ~s/"\x00\x01\x10"/ + assert quoted_to_string(quote(do: "\x00\x01\x10")) == ~S/"\0\x01\x10"/ + end + + test "charlists with slash escapes" do + assert quoted_to_string(~c"\a\b\e\n\r\t\v", escape: false) == + ~s/~c"\a\b\e\n\r\t\v"/ + + assert quoted_to_string(~c"\a\b\e\n\r\t\v") == + ~s/~c"\\a\\b\\e\\n\\r\\t\\v"/ + + assert quoted_to_string({:__block__, [], [~c"\a\b\e\n\r\t\v"]}, escape: false) == + ~s/~c"\a\b\e\n\r\t\v"/ + + assert quoted_to_string({:__block__, [], [~c"\a\b\e\n\r\t\v"]}) == + ~s/~c"\\a\\b\\e\\n\\r\\t\\v"/ + end + + test "charlists with non printable characters" do + assert quoted_to_string(~c"\x00\x01\x10", escape: false) == ~S/[0, 1, 16]/ + assert quoted_to_string(~c"\x00\x01\x10") == ~S/[0, 1, 16]/ + end + + test "atoms" do + assert quoted_to_string(quote(do: :"a\nb\tc"), escape: false) == ~s/:"a\nb\tc"/ + assert quoted_to_string(quote(do: :"a\nb\tc")) == ~S/:"a\nb\tc"/ + + assert quoted_to_string({:__block__, [], [:"a\nb\tc"]}, escape: false) == ~s/:"a\nb\tc"/ + assert quoted_to_string({:__block__, [], [:"a\nb\tc"]}) == ~S/:"a\nb\tc"/ + + assert quoted_to_string(quote(do: :"Elixir")) == "Elixir" + assert quoted_to_string(quote(do: :"Elixir.Foo")) == "Foo" + assert quoted_to_string(quote(do: :"Elixir.Foo.Bar")) == "Foo.Bar" + assert quoted_to_string(quote(do: :"Elixir.foobar")) == ~S/:"Elixir.foobar"/ + end + + test "atoms with non printable characters" do + assert quoted_to_string(quote(do: :"\x00\x01\x10"), escape: false) == ~s/:"\0\x01\x10"/ + assert quoted_to_string(quote(do: :"\x00\x01\x10")) == ~S/:"\0\x01\x10"/ + end + + test "atoms with interpolations" do + assert quoted_to_string(quote(do: :"foo\n#{bar}\tbaz"), escape: false) == + ~s[:"foo\n\#{bar}\tbaz"] + + assert quoted_to_string(quote(do: :"foo\n#{bar}\tbaz")) == ~S[:"foo\n#{bar}\tbaz"] + + assert quoted_to_string(quote(do: :"foo\"bar"), escape: false) == ~S[:"foo\"bar"] + assert quoted_to_string(quote(do: :"foo\"bar")) == ~S[:"foo\"bar"] + + assert quoted_to_string(quote(do: :"foo#{~s/\n/}bar"), escape: false) == + ~S[:"foo#{~s/\n/}bar"] + + assert quoted_to_string(quote(do: :"foo#{~s/\n/}bar")) == ~S[:"foo#{~s/\n/}bar"] + + assert quoted_to_string(quote(do: :"one\n\"#{2}\"\nthree"), escape: false) == + ~s[:"one\n\\"\#{2}\\"\nthree"] + + assert quoted_to_string(quote(do: :"one\n\"#{2}\"\nthree")) == ~S[:"one\n\"#{2}\"\nthree"] + end + + test ":erlang.binary_to_atom/2 edge cases" do + assert quoted_to_string(quote(do: :erlang.binary_to_atom(<<>>, :utf8))) == ~S[:""] + + assert quoted_to_string(quote(do: :erlang.binary_to_atom(<<1>>, :utf8))) == + ~S":erlang.binary_to_atom(<<1>>, :utf8)" + end + end + + describe "quoted_to_algebra/2 with invalid" do + test "block" do + assert quoted_to_string({:__block__, [], {:bar, [], []}}) == + "{:__block__, [], {:bar, [], []}}" + + assert quoted_to_string({:foo, [], [{:do, :ok}, :not_keyword]}) == + "foo({:do, :ok}, :not_keyword)" + + assert quoted_to_string({:foo, [], [[{:do, :ok}, :not_keyword]]}) == + "foo([{:do, :ok}, :not_keyword])" + end + + test "ode" do + assert quoted_to_string(1..3) == "1..3" + end + end + + describe "quoted_to_algebra/2 does not escape" do + test "sigils" do + assert quoted_to_string(quote(do: ~s/a\nb\tc/), escape: false) == ~S"~s/a\nb\tc/" + assert quoted_to_string(quote(do: ~s/a\nb\tc/)) == ~S"~s/a\nb\tc/" + + assert quoted_to_string(quote(do: ~s/\a\b\d\e\f\n\r\t\v/), escape: false) == + ~S"~s/\a\b\d\e\f\n\r\t\v/" + + assert quoted_to_string(quote(do: ~s/\a\b\d\e\f\n\r\t\v/)) == ~S"~s/\a\b\d\e\f\n\r\t\v/" + + assert quoted_to_string(quote(do: ~s/\x00\x01\x10/), escape: false) == ~S"~s/\x00\x01\x10/" + assert quoted_to_string(quote(do: ~s/\x00\x01\x10/)) == ~S"~s/\x00\x01\x10/" + end + end + + defp strip_metadata(ast) do + Macro.prewalk(ast, &Macro.update_meta(&1, fn _ -> [] end)) + end + + defp quoted_to_string(quoted, opts \\ []) do + doc = Code.quoted_to_algebra(quoted, opts) + + Inspect.Algebra.format(doc, 98) + |> IO.iodata_to_binary() + end +end diff --git a/lib/elixir/test/elixir/code_test.exs b/lib/elixir/test/elixir/code_test.exs index 645f784548c..2a71e824dd3 100644 --- a/lib/elixir/test/elixir/code_test.exs +++ b/lib/elixir/test/elixir/code_test.exs @@ -1,74 +1,268 @@ -Code.require_file "test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("test_helper.exs", __DIR__) defmodule CodeTest do use ExUnit.Case, async: true + + doctest Code import PathHelpers - def one, do: 1 def genmodule(name) do defmodule name do - Kernel.LexicalTracker.remotes(__MODULE__) + Kernel.LexicalTracker.references(__ENV__.lexical_tracker) end end - contents = quote do - defmodule CodeTest.Sample do - def eval_quoted_info, do: {__MODULE__, __ENV__.file, __ENV__.line} + contents = + quote do + defmodule CodeTest.Sample do + def eval_quoted_info, do: {__MODULE__, __ENV__.file, __ENV__.line} + end end - end - Code.eval_quoted contents, [], file: "sample.ex", line: 13 + Code.eval_quoted(contents, [], file: "sample.ex", line: 13) - test :eval_string do - assert Code.eval_string("1 + 2") == {3, []} - assert {3, _} = Code.eval_string("a + b", [a: 1, b: 2], Macro.Env.location(__ENV__)) - end + describe "with_diagnostics/2" do + test "captures warnings" do + assert {:warn, [%{message: "hello"}]} = + Code.with_diagnostics(fn -> + IO.warn("hello") + :warn + end) + end - test :eval_string_with_other_context do - assert Code.eval_string("var!(a, Sample) = 1") == {1, [{{:a,Sample},1}]} - end + test "captures and logs warnings" do + assert ExUnit.CaptureIO.capture_io(:stderr, fn -> + assert {:warn, [%{message: "hello"}]} = + Code.with_diagnostics([log: true], fn -> + IO.warn("hello") + :warn + end) + end) =~ "hello" + end - test :eval_with_unnamed_scopes do - assert {%RuntimeError{}, [a: %RuntimeError{}]} = - Code.eval_string("a = (try do (raise \"hello\") rescue e -> e end)") - end + test "can be nested" do + assert {:warn, [%{message: "hello"}]} = + Code.with_diagnostics(fn -> + IO.warn("hello") - test :eval_with_scope do - assert Code.eval_string("one", [], delegate_locals_to: __MODULE__) == {1, []} - end + assert {:nested, [%{message: "world"}]} = + Code.with_diagnostics(fn -> + IO.warn("world") + :nested + end) - test :eval_options do - assert Code.eval_string("is_atom(:foo) and K.is_list([])", [], - functions: [{Kernel, [is_atom: 1]}], - macros: [{Kernel, [..: 2, and: 2]}], - aliases: [{K, Kernel}], - requires: [Kernel]) == {true, []} - end + :warn + end) + end - test :eval_stacktrace do - try do - Code.eval_string("<>", a: :a, b: :b) - rescue - _ -> - assert System.stacktrace |> Enum.any?(&(elem(&1, 0) == __MODULE__)) + test "includes column information on unused variables" do + assert {_, [%{position: {1, 12}}]} = + Code.with_diagnostics(fn -> + quoted = Code.string_to_quoted!("if true do var = :foo end", columns: true) + Code.eval_quoted(quoted, []) + end) end - end - test :eval_with_requires do - assert Code.eval_string("Kernel.if true, do: :ok", [], requires: [Z, Kernel]) == {:ok, []} + test "includes column information on unused aliases" do + sample = """ + defmodule CodeTest.UnusedAlias do + alias String.Chars + end + """ + + assert {_, [%{position: {2, 3}}]} = + Code.with_diagnostics(fn -> + quoted = Code.string_to_quoted!(sample, columns: true) + Code.eval_quoted(quoted, []) + end) + end + + test "includes column information on unused imports" do + sample = """ + defmodule CodeTest.UnusedImport do + import URI + end + """ + + assert {_, [%{position: {2, 3}}]} = + Code.with_diagnostics(fn -> + quoted = Code.string_to_quoted!(sample, columns: true) + Code.eval_quoted(quoted, []) + end) + end + + test "includes column information on unknown remote function calls" do + sample = """ + defmodule CodeTest.UnknownRemoteCall do + def perform do + UnknownModule.foo() + end + end + """ + + assert {_, [%{position: {3, 19}}]} = + Code.with_diagnostics(fn -> + quoted = Code.string_to_quoted!(sample, columns: true) + Code.eval_quoted(quoted, []) + end) + end + + test "captures unknown local calls" do + sample = """ + defmodule CodeTest.UnknownLocalCall do + def perform do + foo() + end + end + """ + + assert {:rescued, [%{message: message}]} = + Code.with_diagnostics(fn -> + try do + quoted = Code.string_to_quoted!(sample, columns: true) + Code.eval_quoted(quoted, []) + rescue + _ -> :rescued + end + end) + + assert message =~ "undefined function foo/0" + end end - test :eval_quoted do - assert Code.eval_quoted(quote(do: 1 + 2)) == {3, []} - assert CodeTest.Sample.eval_quoted_info() == {CodeTest.Sample, "sample.ex", 13} + describe "eval_string/1,2,3" do + test "correctly evaluates a string of code" do + assert Code.eval_string("1 + 2") == {3, []} + assert Code.eval_string("two = 1 + 1") == {2, [two: 2]} + end + + test "keeps bindings on optimized evals" do + assert Code.eval_string("import Enum", x: 1) == {Enum, [x: 1]} + end + + test "supports a %Macro.Env{} struct as the third argument" do + assert {3, _} = Code.eval_string("a + b", [a: 1, b: 2], __ENV__) + end + + test "supports unnamed scopes" do + assert {%RuntimeError{}, [a: %RuntimeError{}]} = + Code.eval_string("a = (try do (raise \"hello\") rescue e -> e end)") + end + + test "returns bindings from a different context" do + assert Code.eval_string("var!(a, Sample) = 1") == {1, [{{:a, Sample}, 1}]} + end + + defmacro hygiene_var do + quote do + a = 1 + end + end + + test "does not return bindings from macro hygiene" do + assert Code.eval_string("require CodeTest; CodeTest.hygiene_var()") == {1, []} + end + + test "does not raise on duplicate bindings" do + # The order of which values win is not guaranteed, but it should evaluate successfully. + assert Code.eval_string("b = String.Chars.to_string(a)", a: 0, a: 1) == + {"1", [{:b, "1"}, {:a, 1}]} + + assert Code.eval_string("b = String.Chars.to_string(a)", a: 0, a: 1, c: 2) == + {"1", [{:c, 2}, {:b, "1"}, {:a, 1}]} + end + + test "keeps caller in stacktrace" do + try do + Code.eval_string("<>", [a: :a, b: :b], file: "myfile") + rescue + _ -> + assert Enum.any?(__STACKTRACE__, &(elem(&1, 0) == __MODULE__)) + end + end + + test "includes eval file in stacktrace" do + try do + Code.eval_string("<>", [a: :a, b: :b], file: "myfile") + rescue + _ -> + assert Exception.format_stacktrace(__STACKTRACE__) =~ "myfile:1" + end + + try do + Code.eval_string( + "Enum.map([a: :a, b: :b], fn {a, b} -> <> end)", + [], + file: "myfile" + ) + rescue + _ -> + assert Exception.format_stacktrace(__STACKTRACE__) =~ "myfile:1" + end + end + + test "warns when lexical tracker process is dead" do + {pid, ref} = spawn_monitor(fn -> :ok end) + assert_receive {:DOWN, ^ref, _, _, _} + env = %{__ENV__ | lexical_tracker: pid} + + assert ExUnit.CaptureIO.capture_io(:stderr, fn -> + assert Code.eval_string("1 + 2", [], env) == {3, []} + end) =~ "an __ENV__ with outdated compilation information was given to eval" + end + + test "formats diagnostic file paths as relatives" do + {_, diagnostics} = + Code.with_diagnostics(fn -> + try do + Code.eval_string("x", []) + rescue + e -> e + end + end) + + assert [ + %{ + message: "undefined variable \"x\"", + position: 1, + file: "nofile", + source: "nofile", + stacktrace: [], + severity: :error + } + ] = diagnostics + end end - test :eval_quoted_with_env do - alias :lists, as: MyList - assert Code.eval_quoted(quote(do: MyList.flatten [[1, 2, 3]]), [], __ENV__) == {[1, 2, 3],[]} + describe "eval_quoted/1" do + test "evaluates expression" do + assert Code.eval_quoted(quote(do: 1 + 2)) == {3, []} + assert CodeTest.Sample.eval_quoted_info() == {CodeTest.Sample, "sample.ex", 13} + end + + test "with %Macro.Env{} at runtime" do + alias :lists, as: MyList + quoted = quote(do: MyList.flatten([[1, 2, 3]])) + + assert Code.eval_quoted(quoted, [], __ENV__) == {[1, 2, 3], []} + + # Let's check it discards tracers since the lexical tracker is explicitly nil + assert Code.eval_quoted(quoted, [], %{__ENV__ | tracers: [:bad]}) == {[1, 2, 3], []} + end + + test "with %Macro.Env{} at compile time" do + defmodule CompileTimeEnv do + alias String.Chars + {"foo", []} = Code.eval_string("Chars.to_string(:foo)", [], __ENV__) + end + end end - test :eval_file do + test "eval_file/1" do assert Code.eval_file(fixture_path("code_sample.exs")) == {3, [var: 3]} assert_raise Code.LoadError, fn -> @@ -76,94 +270,577 @@ defmodule CodeTest do end end - test :require do - Code.require_file fixture_path("code_sample.exs") - assert fixture_path("code_sample.exs") in Code.loaded_files + describe "eval_quoted_with_env/3" do + test "returns results, bindings, and env" do + alias :lists, as: MyList + quoted = quote(do: MyList.flatten([[1, 2, 3]])) + env = Code.env_for_eval(__ENV__) + assert Code.eval_quoted_with_env(quoted, [], env) == {[1, 2, 3], [], env} + + quoted = quote(do: alias(:dict, as: MyDict)) + {:dict, [], env} = Code.eval_quoted_with_env(quoted, [], env) + assert Keyword.fetch(env.aliases, Elixir.MyDict) == {:ok, :dict} + end + + test "manages env vars" do + env = Code.env_for_eval(__ENV__) + {1, [x: 1], env} = Code.eval_quoted_with_env(quote(do: var!(x) = 1), [], env) + assert Macro.Env.vars(env) == [{:x, nil}] + end + + test "prunes vars" do + env = Code.env_for_eval(__ENV__) + + fun = fn quoted, binding -> + {_, binding, env} = Code.eval_quoted_with_env(quoted, binding, env, prune_binding: true) + {binding, Macro.Env.vars(env)} + end + + assert fun.(quote(do: 123), []) == {[], []} + assert fun.(quote(do: 123), x: 2, y: 3) == {[], []} + + assert fun.(quote(do: var!(x) = 1), []) == {[x: 1], [x: nil]} + assert fun.(quote(do: var!(x) = 1), x: 2, y: 3) == {[x: 1], [x: nil]} + + assert fun.(quote(do: var!(x, :foo) = 1), []) == {[{{:x, :foo}, 1}], [x: :foo]} + assert fun.(quote(do: var!(x, :foo) = 1), x: 2, y: 3) == {[{{:x, :foo}, 1}], [x: :foo]} + + assert fun.(quote(do: var!(x, :foo) = 1), [{{:x, :foo}, 2}, {{:y, :foo}, 3}]) == + {[{{:x, :foo}, 1}], [x: :foo]} + + assert fun.(quote(do: fn -> var!(x, :foo) = 1 end), []) == {[], []} + assert fun.(quote(do: fn -> var!(x, :foo) = 1 end), x: 1, y: 2) == {[], []} + + assert fun.(quote(do: fn -> var!(x) end), x: 2, y: 3) == {[x: 2], [x: nil]} + + assert fun.(quote(do: fn -> var!(x, :foo) end), [{{:x, :foo}, 2}, {{:y, :foo}, 3}]) == + {[{{:x, :foo}, 2}], [x: :foo]} + end + + test "undefined function" do + env = Code.env_for_eval(__ENV__) + quoted = quote do: foo() + + assert_exception( + UndefinedFunctionError, + ["** (UndefinedFunctionError) function foo/0 is undefined (there is no such import)"], + fn -> + Code.eval_quoted_with_env(quoted, [], env) + end + ) + end + + defmodule Tracer do + def trace(event, env) do + send(self(), {:trace, event, env}) + :ok + end + end + + test "with tracing and pruning" do + env = %{Code.env_for_eval(__ENV__) | tracers: [Tracer], function: nil} + binding = [x: 1, y: 2, z: 3] + + quoted = + quote do + defmodule Elixir.CodeTest.TracingPruning do + var!(y) = :updated + var!(y) + var!(x) + end + end + + {_, binding, env} = Code.eval_quoted_with_env(quoted, binding, env, prune_binding: true) + assert Enum.sort(binding) == [] + assert env.versioned_vars == %{} + + assert_receive {:trace, {:on_module, _, _}, %{module: CodeTest.TracingPruning} = trace_env} + + assert trace_env.versioned_vars == %{ + {:result, :elixir_compiler} => 5, + {:x, nil} => 1, + {:y, nil} => 4 + } + end + + test "with defguard" do + require Integer, warn: false + env = Code.env_for_eval(__ENV__) + quoted = quote do: Integer.is_even(1) + {false, binding, env} = Code.eval_quoted_with_env(quoted, [], env, prune_binding: true) + assert binding == [] + assert Macro.Env.vars(env) == [] + end + end + + describe "compile_file/1" do + test "compiles the given path" do + assert Code.compile_file(fixture_path("code_sample.exs")) == [] + refute fixture_path("code_sample.exs") in Code.required_files() + end + end + + test "require_file/1" do + assert Code.require_file(fixture_path("code_sample.exs")) == [] + assert fixture_path("code_sample.exs") in Code.required_files() assert Code.require_file(fixture_path("code_sample.exs")) == nil - Code.unload_files [fixture_path("code_sample.exs")] - refute fixture_path("code_sample.exs") in Code.loaded_files + Code.unrequire_files([fixture_path("code_sample.exs")]) + refute fixture_path("code_sample.exs") in Code.required_files() assert Code.require_file(fixture_path("code_sample.exs")) != nil + after + Code.unrequire_files([fixture_path("code_sample.exs")]) + end + + test "string_to_quoted!/2 errors take lines/columns/indentation into account" do + assert_exception( + SyntaxError, + ["nofile:1:5:", "syntax error before:", "1 + * 3", "^"], + fn -> + Code.string_to_quoted!("1 + * 3") + end + ) + + assert_exception( + SyntaxError, + ["nofile:10:5:", "syntax error before:", "1 + * 3", "^"], + fn -> + Code.string_to_quoted!("1 + * 3", line: 10) + end + ) + + assert_exception( + SyntaxError, + ["nofile:10:7:", "syntax error before:", "1 + * 3", "^"], + fn -> + Code.string_to_quoted!("1 + * 3", line: 10, column: 3) + end + ) + + assert_exception( + SyntaxError, + ["nofile:11:15:", "syntax error before:", "1 + * 3", "^"], + fn -> + Code.string_to_quoted!(":ok\n1 + * 3", line: 10, column: 3, indentation: 10) + end + ) end - test :string_to_quoted do - assert Code.string_to_quoted("1 + 2") == {:ok, {:+, [line: 1], [1, 2]}} - assert Code.string_to_quoted!("1 + 2") == {:+, [line: 1], [1, 2]} + test "string_to_quoted only requires the List.Chars protocol implementation to work" do + assert {:ok, 1.23} = Code.string_to_quoted(1.23) + assert 1.23 = Code.string_to_quoted!(1.23) + assert {:ok, 1.23, []} = Code.string_to_quoted_with_comments(1.23) + assert {1.23, []} = Code.string_to_quoted_with_comments!(1.23) + end + + test "string_to_quoted returns error on incomplete escaped string" do + assert {:error, {meta, "missing terminator: \" (for string starting at line 1)", ""}} = + Code.string_to_quoted("\"\\") + + assert meta[:line] == 1 + assert meta[:column] == 1 + assert meta[:end_line] == 1 + assert meta[:end_column] == 3 + end + + test "string_to_quoted with comments" do + assert Code.string_to_quoted_with_comments(""" + # top + [ + # before + + # right-before + expr, # middle + # right-after + + # after + ] + # bottom + """) == + { + :ok, + [{:expr, [line: 6], nil}], + [ + %{ + column: 1, + line: 1, + next_eol_count: 1, + previous_eol_count: 1, + text: "# top" + }, + %{ + column: 3, + line: 3, + next_eol_count: 2, + previous_eol_count: 1, + text: "# before" + }, + %{ + column: 3, + line: 5, + next_eol_count: 1, + previous_eol_count: 2, + text: "# right-before" + }, + %{ + column: 9, + line: 6, + next_eol_count: 1, + previous_eol_count: 0, + text: "# middle" + }, + %{ + column: 3, + line: 7, + next_eol_count: 2, + previous_eol_count: 1, + text: "# right-after" + }, + %{ + column: 3, + line: 9, + next_eol_count: 1, + previous_eol_count: 2, + text: "# after" + }, + %{ + column: 1, + line: 11, + next_eol_count: 1, + previous_eol_count: 1, + text: "# bottom" + } + ] + } + end + + test "string_to_quoted handles unescape errors properly" do + # Test invalid hex escape character + assert {:error, {meta, message, token}} = Code.string_to_quoted("a.'\\xg'") + + assert meta[:line] == 1 + assert meta[:column] == 3 - assert Code.string_to_quoted("a.1") == - {:error, {1, "syntax error before: ", "1"}} + assert message == + "invalid hex escape character, expected \\xHH where H is a hexadecimal digit. Syntax error after: " - assert_raise SyntaxError, fn -> - Code.string_to_quoted!("a.1") + assert token == "\\x" + + # Test invalid Unicode escape character + assert {:error, {meta2, message2, token2}} = Code.string_to_quoted("a.'\\ug'") + + assert meta2[:line] == 1 + assert meta2[:column] == 3 + + assert message2 == + "invalid Unicode escape character, expected \\uHHHH or \\u{H*} where H is a hexadecimal digit. Syntax error after: " + + assert token2 == "\\u" + + # Test invalid Unicode code point (surrogate pair) + assert {:error, {meta3, message3, token3}} = Code.string_to_quoted("a.'\\u{D800}'") + + assert meta3[:line] == 1 + assert meta3[:column] == 3 + + assert message3 == "invalid or reserved Unicode code point \\u{D800}. Syntax error after: " + assert token3 == "\\u" + + # Test Unicode code point beyond valid range + assert {:error, {meta4, message4, token4}} = Code.string_to_quoted("a.'\\u{110000}'") + + assert meta4[:line] == 1 + assert meta4[:column] == 3 + + assert message4 == "invalid or reserved Unicode code point \\u{110000}. Syntax error after: " + assert token4 == "\\u" + end + + test "string_to_quoted returns error for invalid UTF-8 in strings" do + invalid_utf8_cases = [ + # charlist + "'\\xFF'", + # charlist heredoc + "'''\n\\xFF\\\n'''" + ] + + for code <- invalid_utf8_cases do + assert {:error, {_, message, _}} = Code.string_to_quoted(code) + assert message =~ "invalid encoding starting at <<255>>" + end + end + + test "string_to_quoted returns error for invalid UTF-8 in quoted atoms and function calls" do + invalid_utf8_cases = [ + # charlist + # ~S{'\xFF'}, + # charlist heredoc + # ~s{'''\n\xFF\n'''}, + # Quoted atom + ~S{:"\xFF"}, + ~S{:'\xFF'}, + # Quoted keyword identifier + ~S{["\xFF": 1]}, + ~S{['\xFF': 1]}, + # Quoted function call + ~S{foo."\xFF"()}, + ~S{foo.'\xFF'()} + ] + + for code <- invalid_utf8_cases do + assert {:error, {_, message, detail}} = Code.string_to_quoted(code) + assert message =~ "invalid encoding in atom: " + assert detail =~ "invalid encoding starting at <<255>>" + + assert {:error, {_, message, detail}} = + Code.string_to_quoted(code, existing_atoms_only: true) + + assert message =~ "invalid encoding in atom: " + assert detail =~ "invalid encoding starting at <<255>>" end end - test :string_to_quoted_existing_atoms_only do - assert :badarg = catch_error(Code.string_to_quoted!(":thereisnosuchatom", existing_atoms_only: true)) + @tag :requires_source + test "compile source" do + assert __MODULE__.__info__(:compile)[:source] == String.to_charlist(__ENV__.file) end - test :string_to_quoted! do - assert Code.string_to_quoted!("1 + 2") == {:+, [line: 1], [1, 2]} + describe "compile_string/1" do + test "compiles the given string" do + assert [{CompileStringSample, _}] = + Code.compile_string("defmodule CompileStringSample, do: :ok") + after + :code.purge(CompileSimpleSample) + :code.delete(CompileSimpleSample) + end - assert_raise SyntaxError, fn -> - Code.string_to_quoted!("a.1") + test "works across lexical scopes" do + assert [{CompileCrossSample, _}] = + Code.compile_string("CodeTest.genmodule CompileCrossSample") + after + :code.purge(CompileCrossSample) + :code.delete(CompileCrossSample) end - assert_raise TokenMissingError, fn -> - Code.string_to_quoted!("1 +") + test "disables tail call optimization at the root" do + try do + Code.compile_string("List.flatten(123)") + rescue + _ -> assert Enum.any?(__STACKTRACE__, &match?({_, :__FILE__, 1, _}, &1)) + end end end - test :compile_source do - assert __MODULE__.__info__(:compile)[:source] == String.to_char_list(__ENV__.file) + test "format_string/2 returns empty iodata for empty string" do + assert Code.format_string!("") == "" end - test :compile_info_returned_with_source_accessible_through_keyword_module do - compile = __MODULE__.__info__(:compile) - assert Keyword.get(compile, :source) != nil + test "ensure_loaded?/1" do + assert Code.ensure_loaded?(__MODULE__) + refute Code.ensure_loaded?(Code.NoFile) end - test :compile_string_works_accross_lexical_scopes do - assert [{CompileCrossSample, _}] = Code.compile_string("CodeTest.genmodule CompileCrossSample") - after - :code.purge CompileCrossSample - :code.delete CompileCrossSample + test "ensure_loaded!/1" do + assert Code.ensure_loaded!(__MODULE__) == __MODULE__ + + assert_raise ArgumentError, "could not load module Code.NoFile due to reason :nofile", fn -> + Code.ensure_loaded!(Code.NoFile) + end end - test :compile_string do - assert [{CompileStringSample, _}] = Code.compile_string("defmodule CompileStringSample, do: :ok") - after - :code.purge CompileSimpleSample - :code.delete CompileSimpleSample + test "ensure_all_loaded/1" do + assert Code.ensure_all_loaded([__MODULE__]) == :ok + assert Code.ensure_all_loaded([__MODULE__, Kernel]) == :ok + + assert {:error, [error]} = Code.ensure_all_loaded([__MODULE__, Code.NoFile, __MODULE__]) + assert error == {Code.NoFile, :nofile} end - test :compile_quoted do - assert [{CompileQuotedSample, _}] = Code.compile_string("defmodule CompileQuotedSample, do: :ok") - after - :code.purge CompileQuotedSample - :code.delete CompileQuotedSample + test "ensure_all_loaded!/1" do + assert Code.ensure_all_loaded!([__MODULE__]) == :ok + assert Code.ensure_all_loaded!([__MODULE__, Kernel]) == :ok + + message = """ + could not load the following modules: + + * Code.NoFile due to reason :nofile + * Code.OtherNoFile due to reason :nofile\ + """ + + assert_raise ArgumentError, message, fn -> + Code.ensure_all_loaded!([__MODULE__, Code.NoFile, Code.OtherNoFile]) + end end - test :ensure_loaded? do - assert Code.ensure_loaded?(__MODULE__) - refute Code.ensure_loaded?(Unknown.Module) + test "ensure_compiled/1" do + assert Code.ensure_compiled(__MODULE__) == {:module, __MODULE__} + assert Code.ensure_compiled(Code.NoFile) == {:error, :nofile} + end + + test "ensure_compiled!/1" do + assert Code.ensure_compiled!(__MODULE__) == __MODULE__ + + assert_raise ArgumentError, "could not load module Code.NoFile due to reason :nofile", fn -> + Code.ensure_compiled!(Code.NoFile) + end + end + + test "put_compiler_option/2 validates options" do + message = "unknown compiler option: :not_a_valid_option" + + assert_raise RuntimeError, message, fn -> + Code.put_compiler_option(:not_a_valid_option, :foo) + end + + message = "compiler option :debug_info should be a boolean, got: :not_a_boolean" + + assert_raise RuntimeError, message, fn -> + Code.put_compiler_option(:debug_info, :not_a_boolean) + end + end + + describe "fetch_docs/1" do + test "is case sensitive" do + assert {:docs_v1, _, :elixir, _, %{"en" => module_doc}, _, _} = Code.fetch_docs(IO) + + assert "Functions handling input/output (IO)." = + module_doc |> String.split("\n") |> Enum.at(0) + + assert Code.fetch_docs(Io) == {:error, :module_not_found} + end end - test :ensure_compiled? do - assert Code.ensure_compiled?(__MODULE__) - refute Code.ensure_compiled?(Unknown.Module) + defp assert_exception(ex, messages, callback) do + e = + assert_raise ex, fn -> + callback.() + end + + error_msg = Exception.format(:error, e, []) + + for msg <- messages do + assert error_msg =~ msg + end end end defmodule Code.SyncTest do use ExUnit.Case - test :path_manipulation do + import PathHelpers + + if System.otp_release() >= "26" do + defp assert_cached(path) do + assert find_path(path) != :nocache + end + + defp refute_cached(path) do + assert find_path(path) == :nocache + end + + defp find_path(path) do + {:status, _, {:module, :code_server}, [_, :running, _, _, state]} = + :sys.get_status(:code_server) + + [:state, _, _otp_root, paths | _] = Tuple.to_list(state) + {_, value} = List.keyfind(paths, to_charlist(path), 0) + value + end + else + defp assert_cached(_path), do: :ok + defp refute_cached(_path), do: :ok + end + + test "prepend_path" do + path = Path.join(__DIR__, "fixtures") + true = Code.prepend_path(path) + assert to_charlist(path) in :code.get_path() + refute_cached(path) + + true = Code.prepend_path(path, cache: true) + assert_cached(path) + + Code.delete_path(path) + refute to_charlist(path) in :code.get_path() + end + + test "append_path" do + path = Path.join(__DIR__, "fixtures") + true = Code.append_path(path) + assert to_charlist(path) in :code.get_path() + refute_cached(path) + + true = Code.append_path(path, cache: true) + assert_cached(path) + + Code.delete_path(path) + refute to_charlist(path) in :code.get_path() + end + + test "prepend_paths" do + path = Path.join(__DIR__, "fixtures") + :ok = Code.prepend_paths([path]) + assert to_charlist(path) in :code.get_path() + refute_cached(path) + + :ok = Code.prepend_paths([path], cache: true) + assert_cached(path) + + Code.delete_paths([path]) + refute to_charlist(path) in :code.get_path() + end + + test "append_paths" do path = Path.join(__DIR__, "fixtures") - Code.prepend_path path - assert to_char_list(path) in :code.get_path + :ok = Code.append_paths([path]) + assert to_charlist(path) in :code.get_path() + refute_cached(path) + + :ok = Code.append_paths([path], cache: true) + assert_cached(path) + + Code.delete_paths([path]) + refute to_charlist(path) in :code.get_path() + end + + test "purges compiler modules" do + quoted = quote(do: :ok) + Code.compile_quoted(quoted) + + {:ok, claimed} = Code.purge_compiler_modules() + assert claimed > 0 - Code.delete_path path - refute to_char_list(path) in :code.get_path + {:ok, claimed} = Code.purge_compiler_modules() + assert claimed == 0 + + quoted = quote(do: Agent.start_link(fn -> :ok end)) + Code.compile_quoted(quoted) + + {:ok, claimed} = Code.purge_compiler_modules() + assert claimed == 0 + end + + test "returns previous options when setting compiler options" do + Code.compiler_options(debug_info: false) + assert Code.compiler_options(debug_info: true) == %{debug_info: false} + after + Code.compiler_options(debug_info: true) + end + + test "compile_file/1 return value" do + assert [{CompileSample, binary}] = Code.compile_file(fixture_path("compile_sample.ex")) + assert is_binary(binary) + after + :code.purge(CompileSample) + :code.delete(CompileSample) + end + + test "require_file/1 return value" do + assert [{CompileSample, binary}] = Code.require_file(fixture_path("compile_sample.ex")) + assert is_binary(binary) + after + Code.unrequire_files([fixture_path("compile_sample.ex")]) + :code.purge(CompileSample) + :code.delete(CompileSample) end end diff --git a/lib/elixir/test/elixir/collectable_test.exs b/lib/elixir/test/elixir/collectable_test.exs new file mode 100644 index 00000000000..edd0fd0c250 --- /dev/null +++ b/lib/elixir/test/elixir/collectable_test.exs @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("test_helper.exs", __DIR__) + +defmodule CollectableTest do + use ExUnit.Case, async: true + + doctest Collectable +end diff --git a/lib/elixir/test/elixir/config/provider_test.exs b/lib/elixir/test/elixir/config/provider_test.exs new file mode 100644 index 00000000000..9ab0ca12f35 --- /dev/null +++ b/lib/elixir/test/elixir/config/provider_test.exs @@ -0,0 +1,202 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Config.ProviderTest do + use ExUnit.Case + + doctest Config.Provider + alias Config.Provider + import PathHelpers + import ExUnit.CaptureIO + + @tmp_path tmp_path("config_provider") + @env_var "ELIXIR_CONFIG_PROVIDER_BOOTED" + @sys_config Path.join(@tmp_path, "sys.config") + + setup context do + File.rm_rf(@tmp_path) + File.mkdir_p!(@tmp_path) + write_sys_config!(context[:sys_config] || []) + + on_exit(fn -> + Application.delete_env(:elixir, :config_provider_init) + Application.delete_env(:elixir, :config_provider_booted) + System.delete_env(@env_var) + end) + end + + test "validate_compile_env" do + assert Config.Provider.validate_compile_env([{:elixir, [:unknown], :error}]) == :ok + + Application.put_env(:elixir, :unknown, nested: [key: :value]) + + assert Config.Provider.validate_compile_env([ + {:elixir, [:unknown], {:ok, [nested: [key: :value]]}} + ]) == :ok + + assert Config.Provider.validate_compile_env([ + {:elixir, [:unknown, :nested], {:ok, [key: :value]}} + ]) == :ok + + assert Config.Provider.validate_compile_env([ + {:elixir, [:unknown, :nested, :key], {:ok, :value}} + ]) == :ok + + assert Config.Provider.validate_compile_env([ + {:elixir, [:unknown, :nested, :unknown], :error} + ]) == :ok + + assert {:error, msg} = + Config.Provider.validate_compile_env([{:elixir, [:unknown, :nested], :error}]) + + assert msg =~ "Compile time value was not set" + + assert {:error, msg} = + Config.Provider.validate_compile_env([ + {:elixir, [:unknown, :nested], {:ok, :another}} + ]) + + assert msg =~ "Compile time value was set to: :another" + + keys = [:unknown, :nested, :key, :too_deep] + + assert {:error, msg} = + Config.Provider.validate_compile_env([{:elixir, keys, :error}]) + + assert msg =~ + "application :elixir failed reading its compile environment for path [:nested, :key, :too_deep] inside key :unknown" + after + Application.delete_env(:elixir, :unknown) + end + + describe "config_path" do + test "validate!" do + assert Provider.validate_config_path!("/foo") == :ok + assert Provider.validate_config_path!({:system, "foo", "bar"}) == :ok + + assert_raise ArgumentError, fn -> Provider.validate_config_path!({:system, 1, 2}) end + assert_raise ArgumentError, fn -> Provider.validate_config_path!(~c"baz") end + end + + test "resolve!" do + env_var = "ELIXIR_CONFIG_PROVIDER_PATH" + + try do + System.put_env(env_var, @tmp_path) + assert Provider.resolve_config_path!("/foo") == "/foo" + assert Provider.resolve_config_path!({:system, env_var, "/bar"}) == @tmp_path <> "/bar" + after + System.delete_env(env_var) + end + end + end + + describe "boot" do + test "runs providers" do + init_and_assert_boot() + config = consult(@sys_config) + assert config[:my_app] == [key: :value] + assert config[:elixir] == [config_provider_booted: {:booted, nil}] + end + + @tag sys_config: [my_app: [encoding: {:_μ, :"£", "£", ~c"£"}]] + test "writes sys_config with encoding" do + init_and_assert_boot() + config = consult(@sys_config) + assert config[:my_app][:encoding] == {:_μ, :"£", "£", ~c"£"} + end + + @tag sys_config: [my_app: [key: :old_value, sys_key: :sys_value, extra_config: :old_value]] + test "writes extra config with overrides" do + init_and_assert_boot(extra_config: [my_app: [key: :old_extra_value, extra_config: :value]]) + + assert consult(@sys_config)[:my_app] == + [sys_key: :sys_value, extra_config: :value, key: :value] + end + + test "returns :booted if already booted and keeps config file" do + init_and_assert_boot() + Application.put_all_env(Keyword.take(consult(@sys_config), [:elixir])) + assert boot() == :booted + refute_received :restart + assert File.exists?(@sys_config) + end + + test "returns :booted if already booted and prunes config file" do + init_and_assert_boot(prune_runtime_sys_config_after_boot: true) + Application.put_all_env(Keyword.take(consult(@sys_config), [:elixir])) + assert boot() == :booted + refute_received :restart + refute File.exists?(@sys_config) + end + + test "returns :booted if already booted and runs validate_compile_env" do + init_and_assert_boot( + prune_runtime_sys_config_after_boot: true, + validate_compile_env: [{:elixir, [:unknown], {:ok, :value}}] + ) + + Application.put_all_env(Keyword.take(consult(@sys_config), [:elixir])) + + assert capture_abort(fn -> boot() end) =~ + "the application :elixir has a different value set for key :unknown" + end + + test "returns without rebooting" do + reader = {Config.Reader, fixture_path("configs/kernel.exs")} + init = Config.Provider.init([reader], @sys_config, reboot_system_after_config: false) + Application.put_all_env(init) + + assert capture_abort(fn -> + Provider.boot(fn -> + raise "should not be called" + end) + end) =~ + "Cannot configure :kernel because :reboot_system_after_config has been set to false" + + # Make sure values before and after match + write_sys_config!(kernel: [elixir_reboot: true]) + Application.put_all_env(init) + System.delete_env(@env_var) + + Provider.boot(fn -> raise "should not be called" end) + assert Application.get_env(:kernel, :elixir_reboot) == true + assert Application.get_env(:elixir_reboot, :key) == :value + end + end + + defp init(opts) do + reader = {Config.Reader, fixture_path("configs/good_config.exs")} + init = Config.Provider.init([reader], Keyword.get(opts, :path, @sys_config), opts) + Application.put_all_env(init) + init + end + + defp boot do + Provider.boot(fn -> send(self(), :restart) end) + end + + defp init_and_assert_boot(opts \\ []) do + init(opts ++ [reboot_system_after_config: true]) + boot() + assert_received :restart + end + + defp consult(file) do + {:ok, [config]} = :file.consult(file) + config + end + + defp capture_abort(fun) do + capture_io(fn -> + assert_raise ErlangError, fun + end) + end + + defp write_sys_config!(data) do + File.write!(@sys_config, IO.chardata_to_string(:io_lib.format("~tw.~n", [data]))) + end +end diff --git a/lib/elixir/test/elixir/config/reader_test.exs b/lib/elixir/test/elixir/config/reader_test.exs new file mode 100644 index 00000000000..4d9b36616f7 --- /dev/null +++ b/lib/elixir/test/elixir/config/reader_test.exs @@ -0,0 +1,95 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Config.ReaderTest do + use ExUnit.Case, async: true + + doctest Config.Reader + import PathHelpers + + test "read_imports!/2" do + assert Config.Reader.read_imports!(fixture_path("configs/good_kw.exs")) == + {[my_app: [key: :value]], [fixture_path("configs/good_kw.exs")]} + + assert Config.Reader.read_imports!(fixture_path("configs/good_config.exs")) == + {[my_app: [key: :value]], [fixture_path("configs/good_config.exs")]} + + assert Config.Reader.read_imports!(fixture_path("configs/good_import.exs")) == + {[my_app: [key: :value]], + [fixture_path("configs/good_config.exs"), fixture_path("configs/good_import.exs")]} + + assert_raise ArgumentError, + ":imports must be a list of paths", + fn -> Config.Reader.read_imports!("config", imports: :disabled) end + + assert_raise File.Error, + fn -> Config.Reader.read_imports!(fixture_path("configs/bad_root.exs")) end + + assert_raise File.Error, + fn -> Config.Reader.read_imports!(fixture_path("configs/bad_import.exs")) end + end + + test "read!/2" do + assert Config.Reader.read!(fixture_path("configs/good_kw.exs")) == + [my_app: [key: :value]] + + assert Config.Reader.read!(fixture_path("configs/good_config.exs")) == + [my_app: [key: :value]] + + assert Config.Reader.read!(fixture_path("configs/good_import.exs")) == + [my_app: [key: :value]] + + assert Config.Reader.read!(fixture_path("configs/env.exs"), env: :dev, target: :host) == + [my_app: [env: :dev, target: :host]] + + assert Config.Reader.read!(fixture_path("configs/env.exs"), env: :prod, target: :embedded) == + [my_app: [env: :prod, target: :embedded]] + + assert_raise ArgumentError, + ~r"expected config for app :sample in .*/bad_app.exs to return keyword list", + fn -> Config.Reader.read!(fixture_path("configs/bad_app.exs")) end + + assert_raise RuntimeError, "no :env key was given to this configuration file", fn -> + Config.Reader.read!(fixture_path("configs/env.exs")) + end + + assert_raise RuntimeError, "no :target key was given to this configuration file", fn -> + Config.Reader.read!(fixture_path("configs/env.exs"), env: :prod) + end + + assert_raise RuntimeError, + ~r"import_config/1 is not enabled for this configuration file", + fn -> + Config.Reader.read!(fixture_path("configs/good_import.exs"), + imports: :disabled + ) + end + end + + test "eval!/3" do + files = ["configs/good_kw.exs", "configs/good_config.exs", "configs/good_import.exs"] + + for file <- files do + file = fixture_path(file) + assert Config.Reader.read!(file) == Config.Reader.eval!(file, File.read!(file)) + end + + file = fixture_path("configs/env.exs") + + assert Config.Reader.read!(file, env: :dev, target: :host) == + Config.Reader.eval!(file, File.read!(file), env: :dev, target: :host) + end + + test "as a provider" do + state = Config.Reader.init(fixture_path("configs/good_config.exs")) + assert Config.Reader.load([my_app: [key: :old_value]], state) == [my_app: [key: :value]] + + state = Config.Reader.init(path: fixture_path("configs/env.exs"), env: :prod, target: :host) + + assert Config.Reader.load([my_app: [env: :dev]], state) == + [my_app: [env: :prod, target: :host]] + end +end diff --git a/lib/elixir/test/elixir/config_test.exs b/lib/elixir/test/elixir/config_test.exs new file mode 100644 index 00000000000..29488e092b7 --- /dev/null +++ b/lib/elixir/test/elixir/config_test.exs @@ -0,0 +1,133 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("test_helper.exs", __DIR__) + +defmodule ConfigTest do + use ExUnit.Case, async: true + + doctest Config + import Config + import PathHelpers + + setup config do + Process.put({Config, :opts}, {config[:env], config[:target]}) + Process.put({Config, :config}, []) + Process.put({Config, :imports}, config[:imports] || []) + :ok + end + + defp config do + Process.get({Config, :config}) + end + + defp files do + Process.get({Config, :imports}) + end + + test "config/2" do + assert config() == [] + + config :lager, key: :value + assert config() == [lager: [key: :value]] + + config :lager, other: :value + assert config() == [lager: [key: :value, other: :value]] + + config :lager, key: :other + assert config() == [lager: [other: :value, key: :other]] + + # Works inside functions too... + f = fn -> config(:lager, key: :fn) end + f.() + assert config() == [lager: [other: :value, key: :fn]] + + # ...and in for comprehensions. + for _ <- 0..0, do: config(:lager, key: :for) + assert config() == [lager: [other: :value, key: :for]] + end + + test "config/3" do + config :app, Repo, key: :value + assert config() == [app: [{Repo, key: :value}]] + + config :app, Repo, other: :value + assert config() == [app: [{Repo, key: :value, other: :value}]] + + config :app, Repo, key: :other + assert config() == [app: [{Repo, other: :value, key: :other}]] + + config :app, Repo, key: [nested: false] + assert config() == [app: [{Repo, other: :value, key: [nested: false]}]] + + config :app, Repo, key: [nested: true] + assert config() == [app: [{Repo, other: :value, key: [nested: true]}]] + + config :app, Repo, key: :other + assert config() == [app: [{Repo, other: :value, key: :other}]] + end + + test "read_config/1" do + assert read_config(:lager) == nil + + config :lager, key: :value + assert read_config(:lager) == [key: :value] + + config :lager, other: :value + assert read_config(:lager) == [key: :value, other: :value] + end + + @tag env: :dev + test "config_env/0" do + assert config_env() == :dev + end + + test "config_env/0 raises if no env is set" do + assert_raise RuntimeError, "no :env key was given to this configuration file", fn -> + config_env() + end + end + + @tag target: :host + test "config_target/0" do + assert config_target() == :host + end + + test "config_target/0 raises if no env is set" do + assert_raise RuntimeError, "no :target key was given to this configuration file", fn -> + config_target() + end + end + + test "import_config/1" do + import_config fixture_path("configs/good_config.exs") + assert config() == [my_app: [key: :value]] + assert files() == [fixture_path("configs/good_config.exs")] + end + + @tag imports: :disabled + test "import_config/1 raises when disabled" do + assert_raise RuntimeError, + ~r"import_config/1 is not enabled for this configuration file", + fn -> import_config fixture_path("configs/good_config.exs") end + end + + test "import_config/1 raises for recursive import" do + assert_raise ArgumentError, + ~r"attempting to load configuration .*/imports_recursive.exs recursively", + fn -> import_config fixture_path("configs/imports_recursive.exs") end + end + + test "import_config/1 with nested" do + config :app, Repo, key: [nested: false, other: true] + import_config fixture_path("configs/nested.exs") + assert config() == [app: [{Repo, key: [other: true, nested: true]}]] + end + + test "import_config/1 with bad path" do + assert_raise File.Error, ~r"could not read file .*/configs/unknown.exs", fn -> + import_config fixture_path("configs/unknown.exs") + end + end +end diff --git a/lib/elixir/test/elixir/dict_test.exs b/lib/elixir/test/elixir/dict_test.exs deleted file mode 100644 index 31e15b447c0..00000000000 --- a/lib/elixir/test/elixir/dict_test.exs +++ /dev/null @@ -1,415 +0,0 @@ -Code.require_file "test_helper.exs", __DIR__ - -# A TestDict implementation used only for testing. -defmodule TestDict do - defstruct list: [] - - def new(list \\ []) when is_list(list) do - %TestDict{list: list} - end - - def size(%TestDict{list: list}) do - length(list) - end - - def update(%TestDict{list: list} = map, key, initial, fun) do - %{map | list: update(list, key, initial, fun)} - end - - def update([{key, value}|list], key, _initial, fun) do - [{key, fun.(value)}|list] - end - - def update([{_, _} = e|list], key, initial, fun) do - [e|update(list, key, initial, fun)] - end - - def update([], key, initial, _fun) do - [{key, initial}] - end - - defimpl Enumerable do - def reduce(%{list: list}, acc, fun), do: Enumerable.List.reduce(list, acc, fun) - def member?(%{list: list}, other), do: Enumerable.List.member(list, other) - def count(%{list: list}), do: Enumerable.List.count(list) - end -end - -defmodule DictTest.Common do - defmacro __using__(_) do - quote location: :keep do - import Enum, only: [sort: 1] - - defp new_dict(list \\ [{"first_key", 1}, {"second_key", 2}]) do - Enum.into list, dict_impl.new - end - - defp new_dict(list, transform) do - Enum.into list, dict_impl.new, transform - end - - defp int_dict do - Enum.into [{1,1}], dict_impl.new - end - - test "access" do - dict = new_dict() - assert dict["first_key"] == 1 - assert dict["other_key"] == nil - end - - test "access uses match operation" do - dict = int_dict() - assert dict[1] == 1 - assert dict[1.0] == nil - end - - test "get/2 and get/3" do - dict = new_dict() - assert Dict.get(dict, "first_key") == 1 - assert Dict.get(dict, "second_key") == 2 - assert Dict.get(dict, "other_key") == nil - assert Dict.get(dict, "other_key", 3) == 3 - end - - test "get/2 with match" do - assert Dict.get(int_dict, 1) == 1 - assert Dict.get(int_dict, 1.0) == nil - end - - test "fetch/2" do - dict = new_dict() - assert Dict.fetch(dict, "first_key") == {:ok, 1} - assert Dict.fetch(dict, "second_key") == {:ok, 2} - assert Dict.fetch(dict, "other_key") == :error - end - - test "fetch/2 with match" do - assert Dict.fetch(int_dict, 1) == {:ok, 1} - assert Dict.fetch(int_dict, 1.0) == :error - end - - test "fetch!/2" do - dict = new_dict() - assert Dict.fetch!(dict, "first_key") == 1 - assert Dict.fetch!(dict, "second_key") == 2 - assert_raise KeyError, fn -> - Dict.fetch!(dict, "other_key") - end - end - - test "put/3" do - dict = new_dict() |> Dict.put("first_key", {1}) - assert Dict.get(dict, "first_key") == {1} - assert Dict.get(dict, "second_key") == 2 - end - - test "put/3 with_match" do - dict = int_dict() - assert Dict.get(Dict.put(dict, 1, :other), 1) == :other - assert Dict.get(Dict.put(dict, 1.0, :other), 1) == 1 - assert Dict.get(Dict.put(dict, 1, :other), 1.0) == nil - assert Dict.get(Dict.put(dict, 1.0, :other), 1.0) == :other - end - - test "put_new/3" do - dict = Dict.put_new(new_dict(), "first_key", {1}) - assert Dict.get(dict, "first_key") == 1 - end - - test "put_new/3 with_match" do - assert Dict.get(Dict.put_new(int_dict, 1, :other), 1) == 1 - assert Dict.get(Dict.put_new(int_dict, 1.0, :other), 1) == 1 - assert Dict.get(Dict.put_new(int_dict, 1, :other), 1.0) == nil - assert Dict.get(Dict.put_new(int_dict, 1.0, :other), 1.0) == :other - end - - test "keys/1" do - assert Enum.sort(Dict.keys(new_dict())) == ["first_key", "second_key"] - assert Dict.keys(new_dict([])) == [] - end - - test "values/1" do - assert Enum.sort(Dict.values(new_dict())) == [1, 2] - assert Dict.values(new_dict([])) == [] - end - - test "delete/2" do - dict = Dict.delete(new_dict(), "second_key") - assert Dict.size(dict) == 1 - assert Dict.has_key?(dict, "first_key") - refute Dict.has_key?(dict, "second_key") - - dict = Dict.delete(new_dict(), "other_key") - assert dict == new_dict() - assert Dict.size(dict) == 2 - end - - test "delete/2 with match" do - assert Dict.get(Dict.delete(int_dict, 1), 1) == nil - assert Dict.get(Dict.delete(int_dict, 1.0), 1) == 1 - end - - test "merge/2" do - dict = new_dict() - assert Dict.merge(new_dict([]), dict) == dict - assert Dict.merge(dict, new_dict([])) == dict - assert Dict.merge(dict, dict) == dict - assert Dict.merge(new_dict([]), new_dict([])) == new_dict([]) - - dict1 = new_dict [{"a", 1}, {"b", 2}, {"c", 3}] - dict2 = new_dict [{"a", 3}, {"c", :a}, {"d", 0}] - assert Dict.merge(dict1, dict2) |> Enum.sort == - [{"a", 3}, {"b", 2}, {"c", :a}, {"d", 0}] - end - - test "merge/2 with other dict" do - dict1 = new_dict [{"a", 1}, {"b", 2}, {"c", 3}] - dict2 = TestDict.new [{"a",3}, {"c",:a}, {"d",0}] - actual = Dict.merge(dict1, dict2) - assert Dict.merge(dict1, dict2) |> Enum.sort == - [{"a", 3}, {"b", 2}, {"c", :a}, {"d", 0}] - assert Dict.merge(dict2, dict1) |> Enum.sort == - [{"a", 1}, {"b", 2}, {"c", 3}, {"d", 0}] - end - - test "merge/3" do - dict1 = new_dict [{"a", 1}, {"b", 2}] - dict2 = new_dict [{"a", 3}, {"d", 4}] - actual = Dict.merge dict1, dict2, fn _k, v1, v2 -> v1 + v2 end - assert Enum.sort(actual) == [{"a", 4}, {"b", 2}, {"d", 4}] - end - - test "has_key?/2" do - dict = new_dict() - assert Dict.has_key?(dict, "first_key") - refute Dict.has_key?(dict, "other_key") - end - - test "has_key?/2 with match" do - assert Dict.has_key?(int_dict, 1) - refute Dict.has_key?(int_dict, 1.0) - end - - test "size/1" do - assert Dict.size(new_dict()) == 2 - assert Dict.size(new_dict([])) == 0 - end - - test "update!/3" do - dict = Dict.update!(new_dict(), "first_key", fn val -> -val end) - assert Dict.get(dict, "first_key") == -1 - - assert_raise KeyError, fn -> - Dict.update!(new_dict(), "non-existent", fn val -> -val end) - end - end - - test "update!/3 with match" do - assert Dict.get(Dict.update!(int_dict(), 1, &(&1 + 1)), 1) == 2 - end - - test "update/4" do - dict = Dict.update(new_dict(), "first_key", 0, fn val -> -val end) - assert Dict.get(dict, "first_key") == -1 - - dict = Dict.update(new_dict(), "non-existent", "...", fn val -> -val end) - assert Dict.get(dict, "non-existent") == "..." - end - - test "update/4 with match" do - dict = int_dict() - assert Dict.get(Dict.update(dict, 1.0, 2, &(&1 + 1)), 1) == 1 - assert Dict.get(Dict.update(dict, 1.0, 2, &(&1 + 1)), 1.0) == 2 - end - - test "pop/2 and pop/3" do - dict = new_dict() - - {v, actual} = Dict.pop(dict, "first_key") - assert v == 1 - assert actual == new_dict([{"second_key", 2}]) - - {v, actual} = Dict.pop(dict, "other_key") - assert v == nil - assert dict == actual - - {v, actual} = Dict.pop(dict, "other_key", "default") - assert v == "default" - assert dict == actual - end - - test "pop/2 and pop/3 with match" do - dict = int_dict() - - {v, actual} = Dict.pop(dict, 1) - assert v == 1 - assert Enum.sort(actual) == [] - - {v, actual} = Dict.pop(dict, 1.0) - assert v == nil - assert actual == dict - end - - test "split/2" do - dict = new_dict() - - {take, drop} = Dict.split(dict, []) - assert take == new_dict([]) - assert drop == dict - - {take, drop} = Dict.split(dict, ["unknown_key"]) - assert take == new_dict([]) - assert drop == dict - - split_keys = ["first_key", "second_key", "unknown_key"] - {take, drop} = Dict.split(dict, split_keys) - - take_expected = new_dict([]) - |> Dict.put("first_key", 1) - |> Dict.put("second_key", 2) - - drop_expected = new_dict([]) - |> Dict.delete("first_key") - |> Dict.delete("second_key") - - assert Enum.sort(take) == Enum.sort(take_expected) - assert Enum.sort(drop) == Enum.sort(drop_expected) - end - - test "split/2 with match" do - dict = int_dict() - {take, drop} = Dict.split(dict, [1]) - assert take == dict - assert drop == new_dict([]) - - {take, drop} = Dict.split(dict, [1.0]) - assert take == new_dict([]) - assert drop == dict - end - - test "split/2 with enum" do - dict = int_dict() - {take, drop} = Dict.split(dict, 1..3) - assert take == dict - assert drop == new_dict([]) - end - - test "take/2" do - dict = new_dict() - take = Dict.take(dict, ["unknown_key"]) - assert take == new_dict([]) - - take = Dict.take(dict, ["first_key"]) - assert take == new_dict([{"first_key", 1}]) - end - - test "take/2 with match" do - dict = int_dict() - assert Dict.take(dict, [1]) == dict - assert Dict.take(dict, [1.0]) == new_dict([]) - end - - test "take/2 with enum" do - dict = int_dict() - assert Dict.take(dict, 1..3) == dict - end - - test "drop/2" do - dict = new_dict() - drop = Dict.drop(dict, ["unknown_key"]) - assert drop == dict - - drop = Dict.drop(dict, ["first_key"]) - assert drop == new_dict([{"second_key", 2}]) - end - - test "drop/2 with match" do - dict = int_dict() - assert Dict.drop(dict, [1]) == new_dict([]) - assert Dict.drop(dict, [1.0]) == dict - end - - test "drop/2 with enum" do - dict = int_dict() - assert Dict.drop(dict, 1..3) == new_dict([]) - end - - test "equal?/2" do - dict1 = new_dict(a: 2, b: 3, f: 5, c: 123) - dict2 = new_dict(a: 2, b: 3, f: 5, c: 123) - assert dict_impl.equal?(dict1, dict2) - assert Dict.equal?(dict1, dict2) - - dict2 = Dict.put(dict2, :a, 3) - refute dict_impl.equal?(dict1, dict2) - refute Dict.equal?(dict1, dict2) - - dict3 = [a: 2, b: 3, f: 5, c: 123, z: 666] - refute Dict.equal?(dict1, dict3) - refute Dict.equal?(dict3, dict1) - end - - test "equal?/2 with match" do - dict1 = new_dict([{1,1}]) - dict2 = new_dict([{1.0,1}]) - assert Dict.equal?(dict1, dict1) - refute Dict.equal?(dict1, dict2) - end - - test "equal?/2 with other dict" do - dict = new_dict([{1,1}]) - assert Dict.equal?(dict, TestDict.new([{1,1}])) - refute Dict.equal?(dict, TestDict.new([{1.0,1}])) - end - - test "is enumerable" do - dict = new_dict() - assert Enum.empty?(new_dict([])) - refute Enum.empty?(dict) - assert Enum.member?(dict, {"first_key", 1}) - refute Enum.member?(dict, {"first_key", 2}) - assert Enum.count(dict) == 2 - assert Enum.reduce(dict, 0, fn({k, v}, acc) -> v + acc end) == 3 - end - - test "is collectable" do - dict = new_dict() - assert Dict.size(dict) == 2 - assert Enum.sort(dict) == [{"first_key", 1}, {"second_key", 2}] - - dict = new_dict([{1}, {2}, {3}], fn {x} -> {<>, x} end) - assert Dict.size(dict) == 3 - assert Enum.sort(dict) == [{"A", 1}, {"B", 2}, {"C", 3}] - - assert Collectable.empty(new_dict) == new_dict([]) - end - - test "is zippable" do - dict = new_dict() - list = Dict.to_list(dict) - assert Enum.zip(list, list) == Enum.zip(dict, dict) - - dict = new_dict(1..120, fn i -> {i, i} end) - list = Dict.to_list(dict) - assert Enum.zip(list, list) == Enum.zip(dict, dict) - end - end - end -end - -defmodule Dict.HashDictTest do - use ExUnit.Case, async: true - use DictTest.Common - - doctest Dict - defp dict_impl, do: HashDict -end - -defmodule Dict.MapDictTest do - use ExUnit.Case, async: true - use DictTest.Common - - doctest Dict - defp dict_impl, do: Map -end diff --git a/lib/elixir/test/elixir/dynamic_supervisor_test.exs b/lib/elixir/test/elixir/dynamic_supervisor_test.exs new file mode 100644 index 00000000000..ea6e1a16c8b --- /dev/null +++ b/lib/elixir/test/elixir/dynamic_supervisor_test.exs @@ -0,0 +1,774 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("test_helper.exs", __DIR__) + +defmodule DynamicSupervisorTest do + use ExUnit.Case, async: true + + defmodule Simple do + use DynamicSupervisor + + def init(args), do: args + end + + test "can be supervised directly" do + children = [{DynamicSupervisor, name: :dyn_sup_spec_test}] + assert {:ok, _} = Supervisor.start_link(children, strategy: :one_for_one) + assert DynamicSupervisor.which_children(:dyn_sup_spec_test) == [] + end + + test "multiple supervisors can be supervised and identified with simple child spec" do + {:ok, _} = Registry.start_link(keys: :unique, name: DynSup.Registry) + + children = [ + {DynamicSupervisor, name: :simple_name}, + {DynamicSupervisor, name: {:global, :global_name}}, + {DynamicSupervisor, name: {:via, Registry, {DynSup.Registry, "via_name"}}} + ] + + assert {:ok, supsup} = Supervisor.start_link(children, strategy: :one_for_one) + + assert {:ok, no_name_dynsup} = + Supervisor.start_child(supsup, {DynamicSupervisor, strategy: :one_for_one}) + + assert DynamicSupervisor.which_children(:simple_name) == [] + assert DynamicSupervisor.which_children({:global, :global_name}) == [] + assert DynamicSupervisor.which_children({:via, Registry, {DynSup.Registry, "via_name"}}) == [] + assert DynamicSupervisor.which_children(no_name_dynsup) == [] + + assert Supervisor.start_child(supsup, {DynamicSupervisor, strategy: :one_for_one}) == + {:error, {:already_started, no_name_dynsup}} + end + + describe "use/2" do + test "generates child_spec/1" do + assert Simple.child_spec([:hello]) == %{ + id: Simple, + start: {Simple, :start_link, [[:hello]]}, + type: :supervisor + } + + defmodule Custom do + use DynamicSupervisor, + id: :id, + restart: :temporary, + shutdown: :infinity, + start: {:foo, :bar, []} + + def init(arg), do: {:producer, arg} + end + + assert Custom.child_spec([:hello]) == %{ + id: :id, + restart: :temporary, + shutdown: :infinity, + start: {:foo, :bar, []}, + type: :supervisor + } + end + end + + describe "init/1" do + test "set default options" do + assert DynamicSupervisor.init([]) == + {:ok, + %{ + strategy: :one_for_one, + intensity: 3, + period: 5, + max_children: :infinity, + extra_arguments: [] + }} + end + end + + describe "start_link/3" do + test "with non-ok init" do + Process.flag(:trap_exit, true) + + assert DynamicSupervisor.start_link(Simple, {:ok, %{strategy: :unknown}}) == + {:error, {:supervisor_data, {:invalid_strategy, :unknown}}} + + assert DynamicSupervisor.start_link(Simple, {:ok, %{intensity: -1}}) == + {:error, {:supervisor_data, {:invalid_intensity, -1}}} + + assert DynamicSupervisor.start_link(Simple, {:ok, %{period: 0}}) == + {:error, {:supervisor_data, {:invalid_period, 0}}} + + assert DynamicSupervisor.start_link(Simple, {:ok, %{max_children: -1}}) == + {:error, {:supervisor_data, {:invalid_max_children, -1}}} + + assert DynamicSupervisor.start_link(Simple, {:ok, %{extra_arguments: -1}}) == + {:error, {:supervisor_data, {:invalid_extra_arguments, -1}}} + + assert DynamicSupervisor.start_link(Simple, {:ok, %{auto_shutdown: :any_significant}}) == + {:error, {:supervisor_data, {:invalid_auto_shutdown, :any_significant}}} + + assert DynamicSupervisor.start_link(Simple, :unknown) == + {:error, {:bad_return, {Simple, :init, :unknown}}} + + assert DynamicSupervisor.start_link(Simple, :ignore) == :ignore + end + + test "with registered process" do + {:ok, pid} = DynamicSupervisor.start_link(Simple, {:ok, %{}}, name: __MODULE__) + + # Sets up a link + {:links, links} = Process.info(self(), :links) + assert pid in links + + # A name + assert Process.whereis(__MODULE__) == pid + + # And the initial call + assert {:supervisor, DynamicSupervisorTest.Simple, 1} = + :proc_lib.translate_initial_call(pid) + + # And shuts down + assert DynamicSupervisor.stop(__MODULE__) == :ok + end + + test "with spawn_opt" do + {:ok, pid} = + DynamicSupervisor.start_link(strategy: :one_for_one, spawn_opt: [priority: :high]) + + assert Process.info(pid, :priority) == {:priority, :high} + end + + test "sets initial call to the same as a regular supervisor" do + {:ok, pid} = Supervisor.start_link([], strategy: :one_for_one) + assert :proc_lib.initial_call(pid) == {:supervisor, Supervisor.Default, [:Argument__1]} + + {:ok, pid} = DynamicSupervisor.start_link(strategy: :one_for_one) + assert :proc_lib.initial_call(pid) == {:supervisor, Supervisor.Default, [:Argument__1]} + end + end + + ## Code change + + describe "code_change/3" do + test "with non-ok init" do + {:ok, pid} = DynamicSupervisor.start_link(Simple, {:ok, %{}}) + + assert fake_upgrade(pid, {:ok, %{strategy: :unknown}}) == + {:error, {:error, {:supervisor_data, {:invalid_strategy, :unknown}}}} + + assert fake_upgrade(pid, {:ok, %{intensity: -1}}) == + {:error, {:error, {:supervisor_data, {:invalid_intensity, -1}}}} + + assert fake_upgrade(pid, {:ok, %{period: 0}}) == + {:error, {:error, {:supervisor_data, {:invalid_period, 0}}}} + + assert fake_upgrade(pid, {:ok, %{max_children: -1}}) == + {:error, {:error, {:supervisor_data, {:invalid_max_children, -1}}}} + + assert fake_upgrade(pid, :unknown) == {:error, :unknown} + assert fake_upgrade(pid, :ignore) == :ok + end + + test "with ok init" do + {:ok, pid} = DynamicSupervisor.start_link(Simple, {:ok, %{}}) + {:ok, _} = DynamicSupervisor.start_child(pid, sleepy_worker()) + assert %{active: 1} = DynamicSupervisor.count_children(pid) + + assert fake_upgrade(pid, {:ok, %{max_children: 1}}) == :ok + assert %{active: 1} = DynamicSupervisor.count_children(pid) + assert DynamicSupervisor.start_child(pid, {Task, fn -> :ok end}) == {:error, :max_children} + end + + defp fake_upgrade(pid, init_arg) do + :ok = :sys.suspend(pid) + :sys.replace_state(pid, fn state -> %{state | args: init_arg} end) + res = :sys.change_code(pid, :gen_server, 123, :extra) + :ok = :sys.resume(pid) + res + end + end + + describe "start_child/2" do + test "supports old child spec" do + {:ok, pid} = DynamicSupervisor.start_link(strategy: :one_for_one) + child = {Task, {Task, :start_link, [fn -> :ok end]}, :temporary, 5000, :worker, [Task]} + assert {:ok, pid} = DynamicSupervisor.start_child(pid, child) + assert is_pid(pid) + end + + test "supports new child spec as tuple" do + {:ok, pid} = DynamicSupervisor.start_link(strategy: :one_for_one) + child = %{id: Task, restart: :temporary, start: {Task, :start_link, [fn -> :ok end]}} + assert {:ok, pid} = DynamicSupervisor.start_child(pid, child) + assert is_pid(pid) + end + + test "supports new child spec" do + {:ok, pid} = DynamicSupervisor.start_link(strategy: :one_for_one) + child = {Task, fn -> Process.sleep(:infinity) end} + assert {:ok, pid} = DynamicSupervisor.start_child(pid, child) + assert is_pid(pid) + end + + test "supports extra arguments" do + parent = self() + fun = fn -> send(parent, :from_child) end + + {:ok, pid} = DynamicSupervisor.start_link(strategy: :one_for_one, extra_arguments: [fun]) + child = %{id: Task, restart: :temporary, start: {Task, :start_link, []}} + assert {:ok, pid} = DynamicSupervisor.start_child(pid, child) + assert is_pid(pid) + assert_receive :from_child + end + + test "with invalid child spec" do + assert DynamicSupervisor.start_child(:not_used, %{}) == {:error, {:invalid_child_spec, %{}}} + + assert DynamicSupervisor.start_child(:not_used, {1, 2, 3, 4, 5, 6}) == + {:error, {:invalid_mfa, 2}} + + assert DynamicSupervisor.start_child(:not_used, %{id: 1, start: {Task, :foo, :bar}}) == + {:error, {:invalid_mfa, {Task, :foo, :bar}}} + + assert DynamicSupervisor.start_child(:not_used, %{ + id: 1, + start: {Task, :foo, [:bar]}, + shutdown: -1 + }) == + {:error, {:invalid_shutdown, -1}} + + assert DynamicSupervisor.start_child(:not_used, %{ + id: 1, + start: {Task, :foo, [:bar]}, + significant: true + }) == + {:error, {:invalid_significant, true}} + end + + test "with different returns" do + {:ok, pid} = DynamicSupervisor.start_link(strategy: :one_for_one) + + assert {:ok, _, :extra} = DynamicSupervisor.start_child(pid, current_module_worker([:ok3])) + assert {:ok, _} = DynamicSupervisor.start_child(pid, current_module_worker([:ok2])) + assert :ignore = DynamicSupervisor.start_child(pid, current_module_worker([:ignore])) + + assert {:error, :found} = + DynamicSupervisor.start_child(pid, current_module_worker([:error])) + + assert {:error, :unknown} = + DynamicSupervisor.start_child(pid, current_module_worker([:unknown])) + end + + test "with throw/error/exit" do + {:ok, pid} = DynamicSupervisor.start_link(strategy: :one_for_one) + + assert {:error, {{:nocatch, :oops}, [_ | _]}} = + DynamicSupervisor.start_child(pid, current_module_worker([:non_local, :throw])) + + assert {:error, {%RuntimeError{}, [_ | _]}} = + DynamicSupervisor.start_child(pid, current_module_worker([:non_local, :error])) + + assert {:error, :oops} = + DynamicSupervisor.start_child(pid, current_module_worker([:non_local, :exit])) + end + + test "with max_children" do + {:ok, pid} = DynamicSupervisor.start_link(strategy: :one_for_one, max_children: 0) + + assert {:error, :max_children} = + DynamicSupervisor.start_child(pid, current_module_worker([:ok2])) + end + + test "temporary child is not restarted regardless of reason" do + child = current_module_worker([:ok2], restart: :temporary) + {:ok, pid} = DynamicSupervisor.start_link(strategy: :one_for_one) + + assert {:ok, child_pid} = DynamicSupervisor.start_child(pid, child) + assert_kill(child_pid, :shutdown) + assert %{workers: 0, active: 0} = DynamicSupervisor.count_children(pid) + + assert {:ok, child_pid} = DynamicSupervisor.start_child(pid, child) + assert_kill(child_pid, :whatever) + assert %{workers: 0, active: 0} = DynamicSupervisor.count_children(pid) + end + + test "transient child is restarted unless normal/shutdown/{shutdown, _}" do + child = current_module_worker([:ok2], restart: :transient) + {:ok, pid} = DynamicSupervisor.start_link(strategy: :one_for_one) + + assert {:ok, child_pid} = DynamicSupervisor.start_child(pid, child) + assert_kill(child_pid, :shutdown) + assert %{workers: 0, active: 0} = DynamicSupervisor.count_children(pid) + + assert {:ok, child_pid} = DynamicSupervisor.start_child(pid, child) + assert_kill(child_pid, {:shutdown, :signal}) + assert %{workers: 0, active: 0} = DynamicSupervisor.count_children(pid) + + assert {:ok, child_pid} = DynamicSupervisor.start_child(pid, child) + assert_kill(child_pid, :whatever) + assert %{workers: 1, active: 1} = DynamicSupervisor.count_children(pid) + end + + test "permanent child is restarted regardless of reason" do + child = current_module_worker([:ok2], restart: :permanent) + {:ok, pid} = DynamicSupervisor.start_link(strategy: :one_for_one, max_restarts: 100_000) + + assert {:ok, child_pid} = DynamicSupervisor.start_child(pid, child) + assert_kill(child_pid, :shutdown) + assert %{workers: 1, active: 1} = DynamicSupervisor.count_children(pid) + + assert {:ok, child_pid} = DynamicSupervisor.start_child(pid, child) + assert_kill(child_pid, {:shutdown, :signal}) + assert %{workers: 2, active: 2} = DynamicSupervisor.count_children(pid) + + assert {:ok, child_pid} = DynamicSupervisor.start_child(pid, child) + assert_kill(child_pid, :whatever) + assert %{workers: 3, active: 3} = DynamicSupervisor.count_children(pid) + end + + test "child is restarted with different values" do + {:ok, pid} = DynamicSupervisor.start_link(strategy: :one_for_one, max_restarts: 100_000) + + assert {:ok, child1} = + DynamicSupervisor.start_child(pid, current_module_worker([:restart, :ok2])) + + assert [{:undefined, ^child1, :worker, [DynamicSupervisorTest]}] = + DynamicSupervisor.which_children(pid) + + assert_kill(child1, :shutdown) + assert %{workers: 1, active: 1} = DynamicSupervisor.count_children(pid) + + assert {:ok, child2} = + DynamicSupervisor.start_child(pid, current_module_worker([:restart, :ok3])) + + assert [ + {:undefined, _, :worker, [DynamicSupervisorTest]}, + {:undefined, ^child2, :worker, [DynamicSupervisorTest]} + ] = DynamicSupervisor.which_children(pid) + + assert_kill(child2, :shutdown) + assert %{workers: 2, active: 2} = DynamicSupervisor.count_children(pid) + + assert {:ok, child3} = + DynamicSupervisor.start_child(pid, current_module_worker([:restart, :ignore])) + + assert [ + {:undefined, _, :worker, [DynamicSupervisorTest]}, + {:undefined, _, :worker, [DynamicSupervisorTest]}, + {:undefined, _, :worker, [DynamicSupervisorTest]} + ] = DynamicSupervisor.which_children(pid) + + assert_kill(child3, :shutdown) + assert %{workers: 2, active: 2} = DynamicSupervisor.count_children(pid) + + assert {:ok, child4} = + DynamicSupervisor.start_child(pid, current_module_worker([:restart, :error])) + + assert [ + {:undefined, _, :worker, [DynamicSupervisorTest]}, + {:undefined, _, :worker, [DynamicSupervisorTest]}, + {:undefined, _, :worker, [DynamicSupervisorTest]} + ] = DynamicSupervisor.which_children(pid) + + assert_kill(child4, :shutdown) + assert %{workers: 3, active: 2} = DynamicSupervisor.count_children(pid) + + assert {:ok, child5} = + DynamicSupervisor.start_child(pid, current_module_worker([:restart, :unknown])) + + assert [ + {:undefined, _, :worker, [DynamicSupervisorTest]}, + {:undefined, _, :worker, [DynamicSupervisorTest]}, + {:undefined, :restarting, :worker, [DynamicSupervisorTest]}, + {:undefined, _, :worker, [DynamicSupervisorTest]} + ] = DynamicSupervisor.which_children(pid) + + assert_kill(child5, :shutdown) + assert %{workers: 4, active: 2} = DynamicSupervisor.count_children(pid) + end + + test "restarting on init children counted in max_children" do + child = current_module_worker([:restart, :error], restart: :permanent) + opts = [strategy: :one_for_one, max_children: 1, max_restarts: 100_000] + {:ok, pid} = DynamicSupervisor.start_link(opts) + + assert {:ok, child_pid} = DynamicSupervisor.start_child(pid, child) + assert_kill(child_pid, :shutdown) + assert %{workers: 1, active: 0} = DynamicSupervisor.count_children(pid) + + child = current_module_worker([:restart, :ok2], restart: :permanent) + assert {:error, :max_children} = DynamicSupervisor.start_child(pid, child) + end + + test "restarting on exit children counted in max_children" do + child = current_module_worker([:ok2], restart: :permanent) + opts = [strategy: :one_for_one, max_children: 1, max_restarts: 100_000] + {:ok, pid} = DynamicSupervisor.start_link(opts) + + assert {:ok, child_pid} = DynamicSupervisor.start_child(pid, child) + assert_kill(child_pid, :shutdown) + assert %{workers: 1, active: 1} = DynamicSupervisor.count_children(pid) + + child = current_module_worker([:ok2], restart: :permanent) + assert {:error, :max_children} = DynamicSupervisor.start_child(pid, child) + end + + test "restarting a child with extra_arguments successfully restarts child" do + parent = self() + + fun = fn -> + send(parent, :from_child) + Process.sleep(:infinity) + end + + {:ok, sup} = DynamicSupervisor.start_link(strategy: :one_for_one, extra_arguments: [fun]) + child = %{id: Task, restart: :transient, start: {Task, :start_link, []}} + + assert {:ok, child} = DynamicSupervisor.start_child(sup, child) + assert is_pid(child) + assert_receive :from_child + assert %{active: 1, workers: 1} = DynamicSupervisor.count_children(sup) + assert_kill(child, :oops) + assert_receive :from_child + assert %{workers: 1, active: 1} = DynamicSupervisor.count_children(sup) + end + + test "child is restarted when trying again" do + child = current_module_worker([:try_again, self()], restart: :permanent) + {:ok, pid} = DynamicSupervisor.start_link(strategy: :one_for_one, max_restarts: 2) + + assert {:ok, child_pid} = DynamicSupervisor.start_child(pid, child) + assert_received {:try_again, true} + assert_kill(child_pid, :shutdown) + assert_receive {:try_again, false} + assert_receive {:try_again, true} + assert %{workers: 1, active: 1} = DynamicSupervisor.count_children(pid) + end + + test "child triggers maximum restarts" do + Process.flag(:trap_exit, true) + child = current_module_worker([:restart, :error], restart: :permanent) + {:ok, pid} = DynamicSupervisor.start_link(strategy: :one_for_one, max_restarts: 1) + + assert {:ok, child_pid} = DynamicSupervisor.start_child(pid, child) + assert_kill(child_pid, :shutdown) + assert_receive {:EXIT, ^pid, :shutdown} + end + + test "child triggers maximum intensity when trying again" do + Process.flag(:trap_exit, true) + child = current_module_worker([:restart, :error], restart: :permanent) + {:ok, pid} = DynamicSupervisor.start_link(strategy: :one_for_one, max_restarts: 10) + + assert {:ok, child_pid} = DynamicSupervisor.start_child(pid, child) + assert_kill(child_pid, :shutdown) + assert_receive {:EXIT, ^pid, :shutdown} + end + + test "with valid shutdown" do + Process.flag(:trap_exit, true) + + {:ok, pid} = DynamicSupervisor.start_link(strategy: :one_for_one) + + for n <- 0..1 do + assert {:ok, child_pid} = + DynamicSupervisor.start_child(pid, %{ + id: n, + start: {Task, :start_link, [fn -> Process.sleep(:infinity) end]}, + shutdown: n + }) + + assert_kill(child_pid, :shutdown) + end + end + + test "with invalid valid shutdown" do + assert DynamicSupervisor.start_child(:not_used, %{ + id: 1, + start: {Task, :start_link, [fn -> :ok end]}, + shutdown: -1 + }) == {:error, {:invalid_shutdown, -1}} + end + + def start_link(:ok3), do: {:ok, spawn_link(fn -> Process.sleep(:infinity) end), :extra} + def start_link(:ok2), do: {:ok, spawn_link(fn -> Process.sleep(:infinity) end)} + def start_link(:error), do: {:error, :found} + def start_link(:ignore), do: :ignore + def start_link(:unknown), do: :unknown + + def start_link(:non_local, :throw), do: throw(:oops) + def start_link(:non_local, :error), do: raise("oops") + def start_link(:non_local, :exit), do: exit(:oops) + + def start_link(:try_again, notify) do + if Process.get(:try_again) do + Process.put(:try_again, false) + send(notify, {:try_again, false}) + {:error, :try_again} + else + Process.put(:try_again, true) + send(notify, {:try_again, true}) + start_link(:ok2) + end + end + + def start_link(:restart, value) do + if Process.get({:restart, value}) do + start_link(value) + else + Process.put({:restart, value}, true) + start_link(:ok2) + end + end + end + + describe "terminate/2" do + test "terminates children with brutal kill" do + Process.flag(:trap_exit, true) + {:ok, sup} = DynamicSupervisor.start_link(strategy: :one_for_one) + + child = sleepy_worker(shutdown: :brutal_kill) + assert {:ok, child1} = DynamicSupervisor.start_child(sup, child) + assert {:ok, child2} = DynamicSupervisor.start_child(sup, child) + assert {:ok, child3} = DynamicSupervisor.start_child(sup, child) + + Process.monitor(child1) + Process.monitor(child2) + Process.monitor(child3) + assert_kill(sup, :shutdown) + assert_receive {:DOWN, _, :process, ^child1, :killed} + assert_receive {:DOWN, _, :process, ^child2, :killed} + assert_receive {:DOWN, _, :process, ^child3, :killed} + end + + test "terminates children with infinity shutdown" do + Process.flag(:trap_exit, true) + {:ok, sup} = DynamicSupervisor.start_link(strategy: :one_for_one) + + child = sleepy_worker(shutdown: :infinity) + assert {:ok, child1} = DynamicSupervisor.start_child(sup, child) + assert {:ok, child2} = DynamicSupervisor.start_child(sup, child) + assert {:ok, child3} = DynamicSupervisor.start_child(sup, child) + + Process.monitor(child1) + Process.monitor(child2) + Process.monitor(child3) + assert_kill(sup, :shutdown) + assert_receive {:DOWN, _, :process, ^child1, :shutdown} + assert_receive {:DOWN, _, :process, ^child2, :shutdown} + assert_receive {:DOWN, _, :process, ^child3, :shutdown} + end + + test "terminates children with infinity shutdown and abnormal reason" do + Process.flag(:trap_exit, true) + {:ok, sup} = DynamicSupervisor.start_link(strategy: :one_for_one) + parent = self() + + fun = fn -> + Process.flag(:trap_exit, true) + send(parent, :ready) + receive(do: (_ -> exit({:shutdown, :oops}))) + end + + child = Supervisor.child_spec({Task, fun}, shutdown: :infinity) + assert {:ok, child1} = DynamicSupervisor.start_child(sup, child) + assert {:ok, child2} = DynamicSupervisor.start_child(sup, child) + assert {:ok, child3} = DynamicSupervisor.start_child(sup, child) + + assert_receive :ready + assert_receive :ready + assert_receive :ready + + Process.monitor(child1) + Process.monitor(child2) + Process.monitor(child3) + assert_kill(sup, :shutdown) + + assert_receive {:DOWN, _, :process, ^child1, {:shutdown, :oops}} + assert_receive {:DOWN, _, :process, ^child2, {:shutdown, :oops}} + assert_receive {:DOWN, _, :process, ^child3, {:shutdown, :oops}} + end + + test "terminates children with integer shutdown" do + Process.flag(:trap_exit, true) + {:ok, sup} = DynamicSupervisor.start_link(strategy: :one_for_one) + + child = sleepy_worker(shutdown: 1000) + assert {:ok, child1} = DynamicSupervisor.start_child(sup, child) + assert {:ok, child2} = DynamicSupervisor.start_child(sup, child) + assert {:ok, child3} = DynamicSupervisor.start_child(sup, child) + + Process.monitor(child1) + Process.monitor(child2) + Process.monitor(child3) + assert_kill(sup, :shutdown) + assert_receive {:DOWN, _, :process, ^child1, :shutdown} + assert_receive {:DOWN, _, :process, ^child2, :shutdown} + assert_receive {:DOWN, _, :process, ^child3, :shutdown} + end + + test "terminates children with integer shutdown and abnormal reason" do + Process.flag(:trap_exit, true) + {:ok, sup} = DynamicSupervisor.start_link(strategy: :one_for_one) + parent = self() + + fun = fn -> + Process.flag(:trap_exit, true) + send(parent, :ready) + receive(do: (_ -> exit({:shutdown, :oops}))) + end + + child = Supervisor.child_spec({Task, fun}, shutdown: 1000) + assert {:ok, child1} = DynamicSupervisor.start_child(sup, child) + assert {:ok, child2} = DynamicSupervisor.start_child(sup, child) + assert {:ok, child3} = DynamicSupervisor.start_child(sup, child) + + assert_receive :ready + assert_receive :ready + assert_receive :ready + + Process.monitor(child1) + Process.monitor(child2) + Process.monitor(child3) + assert_kill(sup, :shutdown) + + assert_receive {:DOWN, _, :process, ^child1, {:shutdown, :oops}} + assert_receive {:DOWN, _, :process, ^child2, {:shutdown, :oops}} + assert_receive {:DOWN, _, :process, ^child3, {:shutdown, :oops}} + end + + test "terminates children with expired integer shutdown" do + Process.flag(:trap_exit, true) + {:ok, sup} = DynamicSupervisor.start_link(strategy: :one_for_one) + parent = self() + + fun = fn -> + Process.sleep(:infinity) + end + + tmt = fn -> + Process.flag(:trap_exit, true) + send(parent, :ready) + Process.sleep(:infinity) + end + + child_fun = Supervisor.child_spec({Task, fun}, shutdown: 1) + child_tmt = Supervisor.child_spec({Task, tmt}, shutdown: 1) + assert {:ok, child1} = DynamicSupervisor.start_child(sup, child_fun) + assert {:ok, child2} = DynamicSupervisor.start_child(sup, child_tmt) + assert {:ok, child3} = DynamicSupervisor.start_child(sup, child_fun) + + assert_receive :ready + Process.monitor(child1) + Process.monitor(child2) + Process.monitor(child3) + assert_kill(sup, :shutdown) + + assert_receive {:DOWN, _, :process, ^child1, :shutdown} + assert_receive {:DOWN, _, :process, ^child2, :killed} + assert_receive {:DOWN, _, :process, ^child3, :shutdown} + end + + test "terminates children with permanent restart and normal reason" do + Process.flag(:trap_exit, true) + {:ok, sup} = DynamicSupervisor.start_link(strategy: :one_for_one) + parent = self() + + fun = fn -> + Process.flag(:trap_exit, true) + send(parent, :ready) + receive(do: (_ -> exit(:normal))) + end + + child = Supervisor.child_spec({Task, fun}, shutdown: :infinity, restart: :permanent) + assert {:ok, child1} = DynamicSupervisor.start_child(sup, child) + assert {:ok, child2} = DynamicSupervisor.start_child(sup, child) + assert {:ok, child3} = DynamicSupervisor.start_child(sup, child) + + assert_receive :ready + assert_receive :ready + assert_receive :ready + + Process.monitor(child1) + Process.monitor(child2) + Process.monitor(child3) + assert_kill(sup, :shutdown) + assert_receive {:DOWN, _, :process, ^child1, :normal} + assert_receive {:DOWN, _, :process, ^child2, :normal} + assert_receive {:DOWN, _, :process, ^child3, :normal} + end + + test "terminates with mixed children" do + Process.flag(:trap_exit, true) + {:ok, sup} = DynamicSupervisor.start_link(strategy: :one_for_one) + + assert {:ok, child1} = + DynamicSupervisor.start_child(sup, sleepy_worker(shutdown: :infinity)) + + assert {:ok, child2} = + DynamicSupervisor.start_child(sup, sleepy_worker(shutdown: :brutal_kill)) + + Process.monitor(child1) + Process.monitor(child2) + assert_kill(sup, :shutdown) + assert_receive {:DOWN, _, :process, ^child1, :shutdown} + assert_receive {:DOWN, _, :process, ^child2, :killed} + end + end + + describe "terminate_child/2" do + test "terminates child with brutal kill" do + {:ok, sup} = DynamicSupervisor.start_link(strategy: :one_for_one) + + child = sleepy_worker(shutdown: :brutal_kill) + assert {:ok, child_pid} = DynamicSupervisor.start_child(sup, child) + + Process.monitor(child_pid) + assert :ok = DynamicSupervisor.terminate_child(sup, child_pid) + assert_receive {:DOWN, _, :process, ^child_pid, :killed} + + assert {:error, :not_found} = DynamicSupervisor.terminate_child(sup, child_pid) + assert %{workers: 0, active: 0} = DynamicSupervisor.count_children(sup) + end + + test "terminates child with integer shutdown" do + {:ok, sup} = DynamicSupervisor.start_link(strategy: :one_for_one) + + child = sleepy_worker(shutdown: 1000) + assert {:ok, child_pid} = DynamicSupervisor.start_child(sup, child) + + Process.monitor(child_pid) + assert :ok = DynamicSupervisor.terminate_child(sup, child_pid) + assert_receive {:DOWN, _, :process, ^child_pid, :shutdown} + + assert {:error, :not_found} = DynamicSupervisor.terminate_child(sup, child_pid) + assert %{workers: 0, active: 0} = DynamicSupervisor.count_children(sup) + end + + test "terminates restarting child" do + {:ok, sup} = DynamicSupervisor.start_link(strategy: :one_for_one, max_restarts: 100_000) + + child = current_module_worker([:restart, :error], restart: :permanent) + assert {:ok, child_pid} = DynamicSupervisor.start_child(sup, child) + assert_kill(child_pid, :shutdown) + assert :ok = DynamicSupervisor.terminate_child(sup, child_pid) + + assert {:error, :not_found} = DynamicSupervisor.terminate_child(sup, child_pid) + assert %{workers: 0, active: 0} = DynamicSupervisor.count_children(sup) + end + end + + defp sleepy_worker(opts \\ []) do + mfa = {Task, :start_link, [Process, :sleep, [:infinity]]} + Supervisor.child_spec(%{id: Task, start: mfa}, opts) + end + + defp current_module_worker(args, opts \\ []) do + Supervisor.child_spec(%{id: __MODULE__, start: {__MODULE__, :start_link, args}}, opts) + end + + defp assert_kill(pid, reason) do + ref = Process.monitor(pid) + Process.exit(pid, reason) + assert_receive {:DOWN, ^ref, _, _, _} + end +end diff --git a/lib/elixir/test/elixir/enum_test.exs b/lib/elixir/test/elixir/enum_test.exs index 779d73605fa..7396f04663e 100644 --- a/lib/elixir/test/elixir/enum_test.exs +++ b/lib/elixir/test/elixir/enum_test.exs @@ -1,53 +1,94 @@ -Code.require_file "test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec -defmodule EnumTest.List do +Code.require_file("test_helper.exs", __DIR__) + +defmodule EnumTest do use ExUnit.Case, async: true + doctest Enum - test :empty? do - assert Enum.empty?([]) - refute Enum.empty?([1, 2, 3]) - refute Enum.empty?(1..3) - end + defp assert_runs_enumeration_only_once(enum_fun) do + enumerator = + Stream.map([:element], fn element -> + send(self(), element) + element + end) - test :member? do - assert Enum.member?([1, 2, 3], 2) - refute Enum.member?([], 0) - refute Enum.member?([1, 2, 3], 0) - assert Enum.member?(1..3, 2) - refute Enum.member?(1..3, 0) + enum_fun.(enumerator) + assert_received :element + refute_received :element end - test :count do - assert Enum.count([1, 2, 3]) == 3 - assert Enum.count([]) == 0 - end + describe "zip_reduce/4" do + test "two non lists" do + left = %{a: 1} + right = %{b: 2} + reducer = fn {_, x}, {_, y}, acc -> [x + y | acc] end + assert Enum.zip_reduce(left, right, [], reducer) == [3] + + # Empty Left + assert Enum.zip_reduce(%{}, right, [], reducer) == [] + + # Empty Right + assert Enum.zip_reduce(left, %{}, [], reducer) == [] + end - test :count_fun do - assert Enum.count([1, 2, 3], fn(x) -> rem(x, 2) == 0 end) == 1 - assert Enum.count([], fn(x) -> rem(x, 2) == 0 end) == 0 + test "lists" do + assert Enum.zip_reduce([1, 2], [3, 4], 0, fn x, y, acc -> x + y + acc end) == 10 + assert Enum.zip_reduce([1, 2], [3, 4], [], fn x, y, acc -> [x + y | acc] end) == [6, 4] + end + + test "when left empty" do + assert Enum.zip_reduce([], [1, 2], 0, fn x, y, acc -> x + y + acc end) == 0 + end + + test "when right empty" do + assert Enum.zip_reduce([1, 2], [], 0, fn x, y, acc -> x + y + acc end) == 0 + end end - test :all? do - assert Enum.all?([2, 4, 6], fn(x) -> rem(x, 2) == 0 end) - refute Enum.all?([2, 3, 4], fn(x) -> rem(x, 2) == 0 end) + describe "zip_reduce/3" do + test "when enums empty" do + assert Enum.zip_reduce([], 0, fn _, acc -> acc end) == 0 + end + + test "lists work" do + enums = [[1, 1], [2, 2], [3, 3]] + result = Enum.zip_reduce(enums, [], fn elements, acc -> [List.to_tuple(elements) | acc] end) + assert result == [{1, 2, 3}, {1, 2, 3}] + end + test "mix and match" do + enums = [[1, 2], 3..4, [5, 6]] + result = Enum.zip_reduce(enums, [], fn elements, acc -> [List.to_tuple(elements) | acc] end) + assert result == [{2, 4, 6}, {1, 3, 5}] + end + end + + test "all?/2" do assert Enum.all?([2, 4, 6]) refute Enum.all?([2, nil, 4]) - assert Enum.all?([]) + + assert Enum.all?([2, 4, 6], fn x -> rem(x, 2) == 0 end) + refute Enum.all?([2, 3, 4], fn x -> rem(x, 2) == 0 end) end - test :any? do - refute Enum.any?([2, 4, 6], fn(x) -> rem(x, 2) == 1 end) - assert Enum.any?([2, 3, 4], fn(x) -> rem(x, 2) == 1 end) + test "any?/2" do + refute Enum.any?([2, 4, 6], fn x -> rem(x, 2) == 1 end) + assert Enum.any?([2, 3, 4], fn x -> rem(x, 2) == 1 end) refute Enum.any?([false, false, false]) assert Enum.any?([false, true, false]) + assert Enum.any?([:foo, false, false]) + refute Enum.any?([false, nil, false]) + refute Enum.any?([]) end - test :at do + test "at/3" do assert Enum.at([2, 4, 6], 0) == 2 assert Enum.at([2, 4, 6], 2) == 6 assert Enum.at([2, 4, 6], 4) == nil @@ -56,42 +97,182 @@ defmodule EnumTest.List do assert Enum.at([2, 4, 6], -4) == nil end - test :concat_1 do + test "chunk/3" do + enum = String.to_atom("Elixir.Enum") + assert enum.chunk(1..5, 2, 1) == Enum.chunk_every(1..5, 2, 1, :discard) + end + + test "chunk/4" do + enum = String.to_atom("Elixir.Enum") + assert enum.chunk(1..5, 2, 1, nil) == Enum.chunk_every(1..5, 2, 1, :discard) + end + + test "chunk_every/2" do + assert Enum.chunk_every([1, 2, 3, 4, 5], 2) == [[1, 2], [3, 4], [5]] + end + + test "chunk_every/4" do + assert Enum.chunk_every([1, 2, 3, 4, 5], 2, 2, [6]) == [[1, 2], [3, 4], [5, 6]] + assert Enum.chunk_every([1, 2, 3, 4, 5, 6], 3, 2, :discard) == [[1, 2, 3], [3, 4, 5]] + assert Enum.chunk_every([1, 2, 3, 4, 5, 6], 2, 3, :discard) == [[1, 2], [4, 5]] + assert Enum.chunk_every([1, 2, 3, 4, 5, 6], 3, 2, []) == [[1, 2, 3], [3, 4, 5], [5, 6]] + assert Enum.chunk_every([1, 2, 3, 4, 5, 6], 3, 3, []) == [[1, 2, 3], [4, 5, 6]] + assert Enum.chunk_every([1, 2, 3, 4, 5], 4, 4, 6..10) == [[1, 2, 3, 4], [5, 6, 7, 8]] + assert Enum.chunk_every([1, 2, 3, 4, 5], 2, 3, []) == [[1, 2], [4, 5]] + assert Enum.chunk_every([1, 2, 3, 4, 5, 6], 2, 3, []) == [[1, 2], [4, 5]] + assert Enum.chunk_every([1, 2, 3, 4, 5, 6, 7], 2, 3, []) == [[1, 2], [4, 5], [7]] + assert Enum.chunk_every([1, 2, 3, 4, 5, 6, 7], 2, 3, [8]) == [[1, 2], [4, 5], [7, 8]] + assert Enum.chunk_every([1, 2, 3, 4, 5, 6, 7], 2, 4, []) == [[1, 2], [5, 6]] + end + + test "chunk_by/2" do + assert Enum.chunk_by([1, 2, 2, 3, 4, 4, 6, 7, 7], &(rem(&1, 2) == 1)) == + [[1], [2, 2], [3], [4, 4, 6], [7, 7]] + + assert Enum.chunk_by([1, 2, 3, 4], fn _ -> true end) == [[1, 2, 3, 4]] + assert Enum.chunk_by([], fn _ -> true end) == [] + assert Enum.chunk_by([1], fn _ -> true end) == [[1]] + end + + test "chunk_while/4" do + chunk_fun = fn i, acc -> + cond do + i > 10 -> + {:halt, acc} + + rem(i, 2) == 0 -> + {:cont, Enum.reverse([i | acc]), []} + + true -> + {:cont, [i | acc]} + end + end + + after_fun = fn + [] -> {:cont, []} + acc -> {:cont, Enum.reverse(acc), []} + end + + assert Enum.chunk_while([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], [], chunk_fun, after_fun) == + [[1, 2], [3, 4], [5, 6], [7, 8], [9, 10]] + + assert Enum.chunk_while(0..9, [], chunk_fun, after_fun) == + [[0], [1, 2], [3, 4], [5, 6], [7, 8], [9]] + + assert Enum.chunk_while(0..10, [], chunk_fun, after_fun) == + [[0], [1, 2], [3, 4], [5, 6], [7, 8], [9, 10]] + + assert Enum.chunk_while(0..11, [], chunk_fun, after_fun) == + [[0], [1, 2], [3, 4], [5, 6], [7, 8], [9, 10]] + + assert Enum.chunk_while([5, 7, 9, 11], [], chunk_fun, after_fun) == [[5, 7, 9]] + + assert Enum.chunk_while([1, 2, 3, 5, 7], [], chunk_fun, after_fun) == [[1, 2], [3, 5, 7]] + + chunk_fn2 = fn + -1, acc -> {:cont, acc, 0} + i, acc -> {:cont, acc + i} + end + + after_fn2 = fn acc -> {:cont, acc, 0} end + + assert Enum.chunk_while([1, -1, 2, 3, -1, 4, 5, 6], 0, chunk_fn2, after_fn2) == [1, 5, 15] + end + + test "concat/1" do assert Enum.concat([[1, [2], 3], [4], [5, 6]]) == [1, [2], 3, 4, 5, 6] - assert Enum.concat(1..3, []) == [1,2,3] assert Enum.concat([[], []]) == [] - assert Enum.concat([[]]) == [] - assert Enum.concat([]) == [] - - assert Enum.concat([1..5, fn acc, _ -> acc end, [1]]) == [1,2,3,4,5,1] + assert Enum.concat([[]]) == [] + assert Enum.concat([]) == [] end - test :concat_2 do + test "concat/2" do assert Enum.concat([], [1]) == [1] assert Enum.concat([1, [2], 3], [4, 5]) == [1, [2], 3, 4, 5] - assert Enum.concat(1..3, []) == [1,2,3] + + assert Enum.concat([1, 2], 3..5) == [1, 2, 3, 4, 5] assert Enum.concat([], []) == [] + assert Enum.concat([], 1..3) == [1, 2, 3] assert Enum.concat(fn acc, _ -> acc end, [1]) == [1] end - test :fetch! do - assert Enum.fetch!([2, 4, 6], 0) == 2 - assert Enum.fetch!([2, 4, 6], 2) == 6 - assert Enum.fetch!([2, 4, 6], -2) == 4 + test "count/1" do + assert Enum.count([1, 2, 3]) == 3 + assert Enum.count([]) == 0 + assert Enum.count([1, true, false, nil]) == 4 + end - assert_raise Enum.OutOfBoundsError, fn -> - Enum.fetch!([2, 4, 6], 4) - end + test "count/2" do + assert Enum.count([1, 2, 3], fn x -> rem(x, 2) == 0 end) == 1 + assert Enum.count([], fn x -> rem(x, 2) == 0 end) == 0 + assert Enum.count([1, true, false, nil], & &1) == 2 + end - assert_raise Enum.OutOfBoundsError, fn -> - Enum.fetch!([2, 4, 6], -4) + test "count_until/2" do + assert Enum.count_until([1, 2, 3], 2) == 2 + assert Enum.count_until([], 2) == 0 + assert Enum.count_until([1, 2], 2) == 2 + end + + test "count_until/2 with streams" do + count_until_stream = fn list, limit -> list |> Stream.map(& &1) |> Enum.count_until(limit) end + + assert count_until_stream.([1, 2, 3], 2) == 2 + assert count_until_stream.([], 2) == 0 + assert count_until_stream.([1, 2], 2) == 2 + end + + test "count_until/3" do + assert Enum.count_until([1, 2, 3, 4, 5, 6], fn x -> rem(x, 2) == 0 end, 2) == 2 + assert Enum.count_until([1, 2], fn x -> rem(x, 2) == 0 end, 2) == 1 + assert Enum.count_until([1, 2, 3, 4], fn x -> rem(x, 2) == 0 end, 2) == 2 + assert Enum.count_until([], fn x -> rem(x, 2) == 0 end, 2) == 0 + end + + test "count_until/3 with streams" do + count_until_stream = fn list, fun, limit -> + list |> Stream.map(& &1) |> Enum.count_until(fun, limit) end + + assert count_until_stream.([1, 2, 3, 4, 5, 6], fn x -> rem(x, 2) == 0 end, 2) == 2 + assert count_until_stream.([1, 2], fn x -> rem(x, 2) == 0 end, 2) == 1 + assert count_until_stream.([1, 2, 3, 4], fn x -> rem(x, 2) == 0 end, 2) == 2 + assert count_until_stream.([], fn x -> rem(x, 2) == 0 end, 2) == 0 + end + + test "dedup/1" do + assert Enum.dedup([1, 1, 2, 1, 1, 2, 1]) == [1, 2, 1, 2, 1] + assert Enum.dedup([2, 1, 1, 2, 1]) == [2, 1, 2, 1] + assert Enum.dedup([1, 2, 3, 4]) == [1, 2, 3, 4] + assert Enum.dedup([1, 1.0, 2.0, 2]) == [1, 1.0, 2.0, 2] + assert Enum.dedup([]) == [] + assert Enum.dedup([nil, nil, true, {:value, true}]) == [nil, true, {:value, true}] + assert Enum.dedup([nil]) == [nil] + end + + test "dedup/1 with streams" do + dedup_stream = fn list -> list |> Stream.map(& &1) |> Enum.dedup() end + + assert dedup_stream.([1, 1, 2, 1, 1, 2, 1]) == [1, 2, 1, 2, 1] + assert dedup_stream.([2, 1, 1, 2, 1]) == [2, 1, 2, 1] + assert dedup_stream.([1, 2, 3, 4]) == [1, 2, 3, 4] + assert dedup_stream.([1, 1.0, 2.0, 2]) == [1, 1.0, 2.0, 2] + assert dedup_stream.([]) == [] + assert dedup_stream.([nil, nil, true, {:value, true}]) == [nil, true, {:value, true}] + assert dedup_stream.([nil]) == [nil] end - test :drop do + test "dedup_by/2" do + assert Enum.dedup_by([{1, :x}, {2, :y}, {2, :z}, {1, :x}], fn {x, _} -> x end) == + [{1, :x}, {2, :y}, {1, :x}] + + assert Enum.dedup_by([5, 1, 2, 3, 2, 1], fn x -> x > 2 end) == [5, 1, 3, 2] + end + + test "drop/2" do assert Enum.drop([1, 2, 3], 0) == [1, 2, 3] assert Enum.drop([1, 2, 3], 1) == [2, 3] assert Enum.drop([1, 2, 3], 2) == [3] @@ -101,221 +282,1232 @@ defmodule EnumTest.List do assert Enum.drop([1, 2, 3], -2) == [1] assert Enum.drop([1, 2, 3], -4) == [] assert Enum.drop([], 3) == [] + + assert_raise FunctionClauseError, fn -> + Enum.drop([1, 2, 3], 0.0) + end end - test :drop_while do - assert Enum.drop_while([1, 2, 3, 4, 3, 2, 1], fn(x) -> x <= 3 end) == [4, 3, 2, 1] - assert Enum.drop_while([1, 2, 3], fn(_) -> false end) == [1, 2, 3] - assert Enum.drop_while([1, 2, 3], fn(x) -> x <= 3 end) == [] - assert Enum.drop_while([], fn(_) -> false end) == [] + test "drop/2 with streams" do + drop_stream = fn list, count -> list |> Stream.map(& &1) |> Enum.drop(count) end + + assert drop_stream.([1, 2, 3], 0) == [1, 2, 3] + assert drop_stream.([1, 2, 3], 1) == [2, 3] + assert drop_stream.([1, 2, 3], 2) == [3] + assert drop_stream.([1, 2, 3], 3) == [] + assert drop_stream.([1, 2, 3], 4) == [] + assert drop_stream.([1, 2, 3], -1) == [1, 2] + assert drop_stream.([1, 2, 3], -2) == [1] + assert drop_stream.([1, 2, 3], -4) == [] + assert drop_stream.([], 3) == [] end - test :find do - assert Enum.find([2, 4, 6], fn(x) -> rem(x, 2) == 1 end) == nil - assert Enum.find([2, 4, 6], 0, fn(x) -> rem(x, 2) == 1 end) == 0 - assert Enum.find([2, 3, 4], fn(x) -> rem(x, 2) == 1 end) == 3 + test "drop_every/2" do + assert Enum.drop_every([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 2) == [2, 4, 6, 8, 10] + assert Enum.drop_every([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 3) == [2, 3, 5, 6, 8, 9] + assert Enum.drop_every([], 2) == [] + assert Enum.drop_every([1, 2], 2) == [2] + assert Enum.drop_every([1, 2, 3], 0) == [1, 2, 3] + + assert_raise FunctionClauseError, fn -> + Enum.drop_every([1, 2, 3], -1) + end end - test :find_value do - assert Enum.find_value([2, 4, 6], fn(x) -> rem(x, 2) == 1 end) == nil - assert Enum.find_value([2, 4, 6], 0, fn(x) -> rem(x, 2) == 1 end) == 0 - assert Enum.find_value([2, 3, 4], fn(x) -> rem(x, 2) == 1 end) + test "drop_while/2" do + assert Enum.drop_while([1, 2, 3, 4, 3, 2, 1], fn x -> x <= 3 end) == [4, 3, 2, 1] + assert Enum.drop_while([1, 2, 3], fn _ -> false end) == [1, 2, 3] + assert Enum.drop_while([1, 2, 3], fn x -> x <= 3 end) == [] + assert Enum.drop_while([], fn _ -> false end) == [] end - test :find_index do - assert Enum.find_index([2, 4, 6], fn(x) -> rem(x, 2) == 1 end) == nil - assert Enum.find_index([2, 3, 4], fn(x) -> rem(x, 2) == 1 end) == 1 + test "each/2" do + try do + assert Enum.each([], fn x -> x end) == :ok + assert Enum.each([1, 2, 3], fn x -> Process.put(:enum_test_each, x * 2) end) == :ok + assert Process.get(:enum_test_each) == 6 + after + Process.delete(:enum_test_each) + end end - test :each do - assert Enum.each([], fn(x) -> x end) == :ok - assert Enum.each([1, 2, 3], fn(x) -> Process.put(:enum_test_each, x * 2) end) == :ok - assert Process.get(:enum_test_each) == 6 - after - Process.delete(:enum_test_each) + test "empty?/1" do + assert Enum.empty?([]) + assert Enum.empty?(%{}) + refute Enum.empty?([1, 2, 3]) + refute Enum.empty?(%{one: 1}) + refute Enum.empty?(1..3) + + assert Stream.take([1], 0) |> Enum.empty?() + refute Stream.take([1], 1) |> Enum.empty?() end - test :fetch do + test "fetch/2" do + assert Enum.fetch([66], 0) == {:ok, 66} + assert Enum.fetch([66], -1) == {:ok, 66} + assert Enum.fetch([66], 1) == :error + assert Enum.fetch([66], -2) == :error + assert Enum.fetch([2, 4, 6], 0) == {:ok, 2} + assert Enum.fetch([2, 4, 6], -1) == {:ok, 6} assert Enum.fetch([2, 4, 6], 2) == {:ok, 6} assert Enum.fetch([2, 4, 6], 4) == :error assert Enum.fetch([2, 4, 6], -2) == {:ok, 4} assert Enum.fetch([2, 4, 6], -4) == :error + + assert Enum.fetch([], 0) == :error + assert Enum.fetch([], 1) == :error end - test :filter do - assert Enum.filter([1, 2, 3], fn(x) -> rem(x, 2) == 0 end) == [2] - assert Enum.filter([2, 4, 6], fn(x) -> rem(x, 2) == 0 end) == [2, 4, 6] + test "fetch!/2" do + assert Enum.fetch!([2, 4, 6], 0) == 2 + assert Enum.fetch!([2, 4, 6], 2) == 6 + assert Enum.fetch!([2, 4, 6], -2) == 4 + + assert_raise Enum.OutOfBoundsError, fn -> + Enum.fetch!([2, 4, 6], 4) + end + + assert_raise Enum.OutOfBoundsError, fn -> + Enum.fetch!([2, 4, 6], -4) + end end - test :filter_with_match do + test "filter/2" do + assert Enum.filter([1, 2, 3], fn x -> rem(x, 2) == 0 end) == [2] + assert Enum.filter([2, 4, 6], fn x -> rem(x, 2) == 0 end) == [2, 4, 6] + + assert Enum.filter([1, 2, false, 3, nil], & &1) == [1, 2, 3] assert Enum.filter([1, 2, 3], &match?(1, &1)) == [1] assert Enum.filter([1, 2, 3], &match?(x when x < 3, &1)) == [1, 2] - assert Enum.filter([1, 2, 3], &match?(_, &1)) == [1, 2, 3] + assert Enum.filter([1, 2, 3], fn _ -> true end) == [1, 2, 3] end - test :filter_map do - assert Enum.filter_map([1, 2, 3], fn(x) -> rem(x, 2) == 0 end, &(&1 * 2)) == [4] - assert Enum.filter_map([2, 4, 6], fn(x) -> rem(x, 2) == 0 end, &(&1 * 2)) == [4, 8, 12] + test "find/3" do + assert Enum.find([2, 4, 6], fn x -> rem(x, 2) == 1 end) == nil + assert Enum.find([2, 4, 6], 0, fn x -> rem(x, 2) == 1 end) == 0 + assert Enum.find([2, 3, 4], fn x -> rem(x, 2) == 1 end) == 3 end - test :flat_map do - assert Enum.flat_map([], fn(x) -> [x, x] end) == [] - assert Enum.flat_map([1, 2, 3], fn(x) -> [x, x] end) == [1, 1, 2, 2, 3, 3] - assert Enum.flat_map([1, 2, 3], fn(x) -> x..x+1 end) == [1, 2, 2, 3, 3, 4] + test "find_index/2" do + assert Enum.find_index([2, 4, 6], fn x -> rem(x, 2) == 1 end) == nil + assert Enum.find_index([2, 3, 4], fn x -> rem(x, 2) == 1 end) == 1 + assert Stream.take(1..3, 3) |> Enum.find_index(fn _ -> false end) == nil + assert Stream.take(1..6, 6) |> Enum.find_index(fn x -> x == 5 end) == 4 end - test :flat_map_reduce do - assert Enum.flat_map_reduce([1, 2, 3], 0, &{[&1, &2], &1 + &2}) == - {[1, 0, 2, 1, 3, 3], 6} + test "find_value/2" do + assert Enum.find_value([2, 4, 6], fn x -> rem(x, 2) == 1 end) == nil + assert Enum.find_value([2, 4, 6], 0, fn x -> rem(x, 2) == 1 end) == 0 + assert Enum.find_value([2, 3, 4], fn x -> rem(x, 2) == 1 end) + end - assert Enum.flat_map_reduce(1..100, 0, fn i, acc -> - if acc < 3, do: {[i], acc + 1}, else: {:halt, acc} - end) == {[1,2,3], 3} + test "flat_map/2" do + assert Enum.flat_map([], fn x -> [x, x] end) == [] + assert Enum.flat_map([1, 2, 3], fn x -> [x, x] end) == [1, 1, 2, 2, 3, 3] + assert Enum.flat_map([1, 2, 3], fn x -> x..(x + 1) end) == [1, 2, 2, 3, 3, 4] + assert Enum.flat_map([1, 2, 3], fn x -> Stream.duplicate(x, 2) end) == [1, 1, 2, 2, 3, 3] end - test :group_by do - assert Enum.group_by([], fn -> nil end) == %{} - assert Enum.group_by(1..6, &rem(&1, 3)) == - %{0 => [6, 3], 1 => [4, 1], 2 => [5, 2]} + test "flat_map/2 with streams" do + flat_map_stream = fn list, fun -> list |> Stream.map(& &1) |> Enum.flat_map(fun) end - result = Enum.group_by(1..6, %{3 => :default}, &rem(&1, 3)) - assert result[0] == [6, 3] - assert result[3] == :default + assert flat_map_stream.([], fn x -> [x, x] end) == [] + assert flat_map_stream.([1, 2, 3], fn x -> [x, x] end) == [1, 1, 2, 2, 3, 3] + assert flat_map_stream.([1, 2, 3], fn x -> x..(x + 1) end) == [1, 2, 2, 3, 3, 4] + assert flat_map_stream.([1, 2, 3], fn x -> Stream.duplicate(x, 2) end) == [1, 1, 2, 2, 3, 3] end - test :into do - assert Enum.into([a: 1, b: 2], %{}) == %{a: 1, b: 2} - assert Enum.into([a: 1, b: 2], %{c: 3}) == %{a: 1, b: 2, c: 3} - assert Enum.into(%{a: 1, b: 2}, []) == [a: 1, b: 2] - assert Enum.into([1, 2, 3], "numbers: ", &to_string/1) == "numbers: 123" - assert Enum.into([1, 2, 3], fn - func, {:cont, x} when is_function(func) -> [x] - list, {:cont, x} -> [x|list] - list, _ -> list - end) == [3, 2, 1] + test "flat_map_reduce/3" do + assert Enum.flat_map_reduce([1, 2, 3], 0, &{[&1, &2], &1 + &2}) == {[1, 0, 2, 1, 3, 3], 6} + end + + test "frequencies/1" do + assert Enum.frequencies([]) == %{} + assert Enum.frequencies(~w{a c a a c b}) == %{"a" => 3, "b" => 1, "c" => 2} + end + + test "frequencies_by/2" do + assert Enum.frequencies_by([], fn _ -> raise "oops" end) == %{} + assert Enum.frequencies_by([12, 7, 6, 5, 1], &Integer.mod(&1, 2)) == %{0 => 2, 1 => 3} end - test :intersperse do + test "group_by/3" do + assert Enum.group_by([], fn _ -> raise "oops" end) == %{} + assert Enum.group_by([1, 2, 3], &rem(&1, 2)) == %{0 => [2], 1 => [1, 3]} + end + + test "intersperse/2" do assert Enum.intersperse([], true) == [] assert Enum.intersperse([1], true) == [1] - assert Enum.intersperse([1,2,3], true) == [1, true, 2, true, 3] + assert Enum.intersperse([1, 2, 3], true) == [1, true, 2, true, 3] + + assert Enum.intersperse(.., true) == [] + assert Enum.intersperse(1..1, true) == [1] + assert Enum.intersperse(1..3, true) == [1, true, 2, true, 3] end - test :join do + test "into/2" do + assert Enum.into([a: 1, b: 2], %{}) == %{a: 1, b: 2} + assert Enum.into([a: 1, b: 2], %{c: 3}) == %{a: 1, b: 2, c: 3} + assert Enum.into(MapSet.new(a: 1, b: 2), %{}) == %{a: 1, b: 2} + assert Enum.into(MapSet.new(a: 1, b: 2), %{c: 3}) == %{a: 1, b: 2, c: 3} + assert Enum.into(%{a: 1, b: 2}, []) |> Enum.sort() == [a: 1, b: 2] + assert Enum.into(1..3, []) == [1, 2, 3] + assert Enum.into(["H", "i"], "") == "Hi" + + assert Enum.into([a: 1, b: 2], MapSet.new()) == MapSet.new(a: 1, b: 2) + assert Enum.into(%{a: 1, b: 2}, MapSet.new()) == MapSet.new(a: 1, b: 2) + assert Enum.into([a: 1, b: 2], MapSet.new(a: 1, c: 3)) == MapSet.new(a: 1, b: 2, c: 3) + end + + test "into/2 exceptions" do + assert_raise ArgumentError, + "collecting into a map requires {key, value} tuples, got: 1", + fn -> Enum.into(1..10, %{}) end + + assert_raise ArgumentError, "collecting into a binary requires a bitstring, got: 1", fn -> + Enum.into(1..10, <<>>) + end + + assert_raise ArgumentError, "collecting into a bitstring requires a bitstring, got: 1", fn -> + Enum.into(1..10, <<1::1>>) + end + end + + test "into/3" do + assert Enum.into([1, 2, 3], [], fn x -> x * 2 end) == [2, 4, 6] + assert Enum.into([1, 2, 3], "numbers: ", &to_string/1) == "numbers: 123" + + assert Enum.into([1, 2, 3], MapSet.new(), &(&1 * 2)) == MapSet.new([2, 4, 6]) + assert Enum.into([1, 2, 3], MapSet.new([0, 2]), &(&1 * 2)) == MapSet.new([0, 2, 4, 6]) + + assert_raise MatchError, fn -> + Enum.into([2, 3], %{a: 1}, & &1) + end + end + + test "join/2" do assert Enum.join([], " = ") == "" assert Enum.join([1, 2, 3], " = ") == "1 = 2 = 3" assert Enum.join([1, "2", 3], " = ") == "1 = 2 = 3" assert Enum.join([1, 2, 3]) == "123" assert Enum.join(["", "", 1, 2, "", 3, "", "\n"], ";") == ";;1;2;;3;;\n" assert Enum.join([""]) == "" + + assert Enum.join(fn acc, _ -> acc end, ".") == "" + end + + test "map/2" do + assert Enum.map([], fn x -> x * 2 end) == [] + assert Enum.map([1, 2, 3], fn x -> x * 2 end) == [2, 4, 6] end - test :map_join do + test "map_every/3" do + assert Enum.map_every([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 2, fn x -> x * 2 end) == + [2, 2, 6, 4, 10, 6, 14, 8, 18, 10] + + assert Enum.map_every([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 3, fn x -> x * 2 end) == + [2, 2, 3, 8, 5, 6, 14, 8, 9, 20] + + assert Enum.map_every([], 2, fn x -> x * 2 end) == [] + assert Enum.map_every([1, 2], 2, fn x -> x * 2 end) == [2, 2] + + assert Enum.map_every([1, 2, 3], 0, fn _x -> raise "should not be invoked" end) == + [1, 2, 3] + + assert Enum.map_every(1..3, 1, fn x -> x * 2 end) == [2, 4, 6] + + assert_raise FunctionClauseError, fn -> + Enum.map_every([1, 2, 3], -1, fn x -> x * 2 end) + end + + assert_raise FunctionClauseError, fn -> + Enum.map_every(1..10, 3.33, fn x -> x * 2 end) + end + + assert Enum.map_every([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 9, fn x -> x + 1000 end) == + [1001, 2, 3, 4, 5, 6, 7, 8, 9, 1010] + + assert Enum.map_every([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 10, fn x -> x + 1000 end) == + [1001, 2, 3, 4, 5, 6, 7, 8, 9, 10] + + assert Enum.map_every([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 100, fn x -> x + 1000 end) == + [1001, 2, 3, 4, 5, 6, 7, 8, 9, 10] + end + + test "map_intersperse/3" do + assert Enum.map_intersperse([], :a, &(&1 * 2)) == [] + assert Enum.map_intersperse([1], :a, &(&1 * 2)) == [2] + assert Enum.map_intersperse([1, 2, 3], :a, &(&1 * 2)) == [2, :a, 4, :a, 6] + end + + test "map_join/3" do assert Enum.map_join([], " = ", &(&1 * 2)) == "" assert Enum.map_join([1, 2, 3], " = ", &(&1 * 2)) == "2 = 4 = 6" assert Enum.map_join([1, 2, 3], &(&1 * 2)) == "246" - assert Enum.map_join(["", "", 1, 2, "", 3, "", "\n"], ";", &(&1)) == ";;1;2;;3;;\n" - assert Enum.map_join([""], "", &(&1)) == "" + assert Enum.map_join(["", "", 1, 2, "", 3, "", "\n"], ";", & &1) == ";;1;2;;3;;\n" + assert Enum.map_join([""], "", & &1) == "" + assert Enum.map_join(fn acc, _ -> acc end, ".", &(&1 + 0)) == "" end - test :join_empty do - fun = fn (acc, _) -> acc end - assert Enum.join(fun, ".") == "" - assert Enum.map_join(fun, ".", &(&1 + 0)) == "" + test "map_reduce/3" do + assert Enum.map_reduce([], 1, fn x, acc -> {x * 2, x + acc} end) == {[], 1} + assert Enum.map_reduce([1, 2, 3], 1, fn x, acc -> {x * 2, x + acc} end) == {[2, 4, 6], 7} end - test :map do - assert Enum.map([], fn x -> x * 2 end) == [] - assert Enum.map([1, 2, 3], fn x -> x * 2 end) == [2, 4, 6] + test "max/1" do + assert Enum.max([1]) == 1 + assert Enum.max([1, 2, 3]) == 3 + assert Enum.max([1, [], :a, {}]) == [] + + assert Enum.max([1, 1.0]) === 1 + assert Enum.max([1.0, 1]) === 1.0 + + assert_raise Enum.EmptyError, fn -> + Enum.max([]) + end + end + + test "max/2 with empty fallback" do + assert Enum.max([], fn -> 0 end) === 0 + assert Enum.max([1, 2], fn -> 0 end) === 2 + end + + test "max/2 with stable sorting" do + assert Enum.max([1, 1.0], &>=/2) === 1 + assert Enum.max([1.0, 1], &>=/2) === 1.0 + assert Enum.max([1, 1.0], &>/2) === 1.0 + assert Enum.max([1.0, 1], &>/2) === 1 + end + + test "max/2 with module" do + assert Enum.max([~D[2019-01-01], ~D[2020-01-01]], Date) === ~D[2020-01-01] + end + + test "max/3" do + assert Enum.max([1], &>=/2, fn -> nil end) == 1 + assert Enum.max([1, 2, 3], &>=/2, fn -> nil end) == 3 + assert Enum.max([1, [], :a, {}], &>=/2, fn -> nil end) == [] + assert Enum.max([], &>=/2, fn -> :empty_value end) == :empty_value + assert Enum.max(%{}, &>=/2, fn -> :empty_value end) == :empty_value + assert_runs_enumeration_only_once(&Enum.max(&1, fn a, b -> a >= b end, fn -> nil end)) + end + + test "max_by/2" do + assert Enum.max_by(["a", "aa", "aaa"], fn x -> String.length(x) end) == "aaa" + + assert Enum.max_by([1, 1.0], & &1) === 1 + assert Enum.max_by([1.0, 1], & &1) === 1.0 + + assert_raise Enum.EmptyError, fn -> + Enum.max_by([], fn x -> String.length(x) end) + end + + assert_raise Enum.EmptyError, fn -> + Enum.max_by(%{}, & &1) + end + end + + test "max_by/3 with stable sorting" do + assert Enum.max_by([1, 1.0], & &1, &>=/2) === 1 + assert Enum.max_by([1.0, 1], & &1, &>=/2) === 1.0 + assert Enum.max_by([1, 1.0], & &1, &>/2) === 1.0 + assert Enum.max_by([1.0, 1], & &1, &>/2) === 1 + end + + test "max_by/3 with module" do + users = [%{id: 1, date: ~D[2019-01-01]}, %{id: 2, date: ~D[2020-01-01]}] + assert Enum.max_by(users, & &1.date, Date).id == 2 + + users = [%{id: 1, date: ~D[2020-01-01]}, %{id: 2, date: ~D[2020-01-01]}] + assert Enum.max_by(users, & &1.date, Date).id == 1 end - test :map_reduce do - assert Enum.map_reduce([], 1, fn(x, acc) -> {x * 2, x + acc} end) == {[], 1} - assert Enum.map_reduce([1, 2, 3], 1, fn(x, acc) -> {x * 2, x + acc} end) == {[2, 4, 6], 7} + test "max_by/4" do + assert Enum.max_by(["a", "aa", "aaa"], fn x -> String.length(x) end, &>=/2, fn -> nil end) == + "aaa" + + assert Enum.max_by([], fn x -> String.length(x) end, &>=/2, fn -> :empty_value end) == + :empty_value + + assert Enum.max_by(%{}, & &1, &>=/2, fn -> :empty_value end) == :empty_value + assert Enum.max_by(%{}, & &1, &>=/2, fn -> {:a, :tuple} end) == {:a, :tuple} + + assert_runs_enumeration_only_once( + &Enum.max_by(&1, fn e -> e end, fn a, b -> a >= b end, fn -> nil end) + ) end - test :partition do - assert Enum.partition([1, 2, 3], fn(x) -> rem(x, 2) == 0 end) == {[2], [1, 3]} - assert Enum.partition([2, 4, 6], fn(x) -> rem(x, 2) == 0 end) == {[2, 4, 6], []} + test "member?/2" do + assert Enum.member?([1, 2, 3], 2) + refute Enum.member?([], 0) + refute Enum.member?([1, 2, 3], 0) end - test :reduce do - assert Enum.reduce([], 1, fn(x, acc) -> x + acc end) == 1 - assert Enum.reduce([1, 2, 3], 1, fn(x, acc) -> x + acc end) == 7 + test "min/1" do + assert Enum.min([1]) == 1 + assert Enum.min([1, 2, 3]) == 1 + assert Enum.min([[], :a, {}]) == :a + + assert Enum.min([1, 1.0]) === 1 + assert Enum.min([1.0, 1]) === 1.0 - assert Enum.reduce([1, 2, 3], fn(x, acc) -> x + acc end) == 6 assert_raise Enum.EmptyError, fn -> - Enum.reduce([], fn(x, acc) -> x + acc end) + Enum.min([]) end end - test :reject do - assert Enum.reject([1, 2, 3], fn(x) -> rem(x, 2) == 0 end) == [1, 3] - assert Enum.reject([2, 4, 6], fn(x) -> rem(x, 2) == 0 end) == [] + test "min/2 with empty fallback" do + assert Enum.min([], fn -> 0 end) === 0 + assert Enum.min([1, 2], fn -> 0 end) === 1 + end + + test "min/2 with stable sorting" do + assert Enum.min([1, 1.0], &<=/2) === 1 + assert Enum.min([1.0, 1], &<=/2) === 1.0 + assert Enum.min([1, 1.0], & nil end) == 1 + assert Enum.min([1, 2, 3], &<=/2, fn -> nil end) == 1 + assert Enum.min([[], :a, {}], &<=/2, fn -> nil end) == :a + assert Enum.min([], &<=/2, fn -> :empty_value end) == :empty_value + assert Enum.min(%{}, &<=/2, fn -> :empty_value end) == :empty_value + assert_runs_enumeration_only_once(&Enum.min(&1, fn a, b -> a <= b end, fn -> nil end)) + end + + test "min_by/2" do + assert Enum.min_by(["a", "aa", "aaa"], fn x -> String.length(x) end) == "a" + + assert Enum.min_by([1, 1.0], & &1) === 1 + assert Enum.min_by([1.0, 1], & &1) === 1.0 + + assert_raise Enum.EmptyError, fn -> + Enum.min_by([], fn x -> String.length(x) end) + end + + assert_raise Enum.EmptyError, fn -> + Enum.min_by(%{}, & &1) + end end - test :reverse do + test "min_by/3 with stable sorting" do + assert Enum.min_by([1, 1.0], & &1, &<=/2) === 1 + assert Enum.min_by([1.0, 1], & &1, &<=/2) === 1.0 + assert Enum.min_by([1, 1.0], & &1, & String.length(x) end, &<=/2, fn -> nil end) == + "a" + + assert Enum.min_by([], fn x -> String.length(x) end, &<=/2, fn -> :empty_value end) == + :empty_value + + assert Enum.min_by(%{}, & &1, &<=/2, fn -> :empty_value end) == :empty_value + assert Enum.min_by(%{}, & &1, &<=/2, fn -> {:a, :tuple} end) == {:a, :tuple} + + assert_runs_enumeration_only_once( + &Enum.min_by(&1, fn e -> e end, fn a, b -> a <= b end, fn -> nil end) + ) + end + + test "min_max/1" do + assert Enum.min_max([1]) == {1, 1} + assert Enum.min_max([2, 3, 1]) == {1, 3} + assert Enum.min_max([[], :a, {}]) == {:a, []} + + assert Enum.min_max([1, 1.0]) === {1, 1} + assert Enum.min_max([1.0, 1]) === {1.0, 1.0} + + assert_raise Enum.EmptyError, fn -> + Enum.min_max([]) + end + end + + test "min_max/2" do + assert Enum.min_max([1], fn -> nil end) == {1, 1} + assert Enum.min_max([2, 3, 1], fn -> nil end) == {1, 3} + assert Enum.min_max([[], :a, {}], fn -> nil end) == {:a, []} + assert Enum.min_max([], fn -> {:empty_min, :empty_max} end) == {:empty_min, :empty_max} + assert Enum.min_max(%{}, fn -> {:empty_min, :empty_max} end) == {:empty_min, :empty_max} + assert_runs_enumeration_only_once(&Enum.min_max(&1, fn -> nil end)) + end + + test "min_max/3" do + dates = [~D[2020-01-01], ~D[2019-01-01]] + + assert Enum.min_max(dates, Date) == + {~D[2019-01-01], ~D[2020-01-01]} + + assert Enum.min_max([~D[2000-01-01]], Date) == + {~D[2000-01-01], ~D[2000-01-01]} + + assert Enum.min_max([3, 1, 2], &>/2, fn -> nil end) == + {3, 1} + + assert Enum.min_max([], &>/2, fn -> {:no_min, :no_max} end) == + {:no_min, :no_max} + + assert Enum.min_max(%{}, &>/2, fn -> {:no_min, :no_max} end) == + {:no_min, :no_max} + + assert Enum.min_max(1..5, &>/2, fn -> {:no_min, :no_max} end) == + {5, 1} + + assert_runs_enumeration_only_once(&Enum.min_max(&1, fn a, b -> a > b end, fn -> nil end)) + end + + test "min_max_by/2" do + assert Enum.min_max_by(["aaa", "a", "aa"], fn x -> String.length(x) end) == {"a", "aaa"} + + assert Enum.min_max_by([1, 1.0], & &1) === {1, 1} + assert Enum.min_max_by([1.0, 1], & &1) === {1.0, 1.0} + + assert_raise Enum.EmptyError, fn -> + Enum.min_max_by([], fn x -> String.length(x) end) + end + end + + test "min_max_by/3" do + assert Enum.min_max_by(["aaa", "a", "aa"], fn x -> String.length(x) end, fn -> nil end) == + {"a", "aaa"} + + assert Enum.min_max_by([], fn x -> String.length(x) end, fn -> {:no_min, :no_max} end) == + {:no_min, :no_max} + + assert Enum.min_max_by(%{}, fn x -> String.length(x) end, fn -> {:no_min, :no_max} end) == + {:no_min, :no_max} + + assert Enum.min_max_by(["aaa", "a", "aa"], fn x -> String.length(x) end, &>/2) == {"aaa", "a"} + + assert_runs_enumeration_only_once(&Enum.min_max_by(&1, fn x -> x end, fn -> nil end)) + end + + test "min_max_by/4" do + users = [%{id: 1, date: ~D[2019-01-01]}, %{id: 2, date: ~D[2020-01-01]}] + + assert Enum.min_max_by(users, & &1.date, Date) == + {%{id: 1, date: ~D[2019-01-01]}, %{id: 2, date: ~D[2020-01-01]}} + + assert Enum.min_max_by(["aaa", "a", "aa"], fn x -> String.length(x) end, &>/2, fn -> nil end) == + {"aaa", "a"} + + assert Enum.min_max_by([], fn x -> String.length(x) end, &>/2, fn -> {:no_min, :no_max} end) == + {:no_min, :no_max} + + assert Enum.min_max_by(%{}, fn x -> String.length(x) end, &>/2, fn -> {:no_min, :no_max} end) == + {:no_min, :no_max} + + assert_runs_enumeration_only_once( + &Enum.min_max_by(&1, fn x -> x end, fn a, b -> a > b end, fn -> nil end) + ) + end + + test "split_with/2" do + assert Enum.split_with([], fn x -> rem(x, 2) == 0 end) == {[], []} + assert Enum.split_with([1, 2, 3], fn x -> rem(x, 2) == 0 end) == {[2], [1, 3]} + assert Enum.split_with([2, 4, 6], fn x -> rem(x, 2) == 0 end) == {[2, 4, 6], []} + + assert Enum.split_with(1..5, fn x -> rem(x, 2) == 0 end) == {[2, 4], [1, 3, 5]} + assert Enum.split_with(-3..0, fn x -> x > 0 end) == {[], [-3, -2, -1, 0]} + + assert Enum.split_with(%{}, fn x -> rem(x, 2) == 0 end) == {[], []} + + assert Enum.split_with(%{a: 1, b: 2}, fn {_k, v} -> rem(v, 2) == 0 end) == + {[b: 2], [a: 1]} + + assert Enum.split_with(%{b: 2, d: 4, f: 6}, fn {_k, v} -> rem(v, 2) == 0 end) == + {Map.to_list(%{b: 2, d: 4, f: 6}), []} + end + + test "random/1" do + # corner cases, independent of the seed + assert_raise Enum.EmptyError, fn -> Enum.random([]) end + assert Enum.random([1]) == 1 + + # set a fixed seed so the test can be deterministic + # please note the order of following assertions is important + seed1 = {1406, 407_414, 139_258} + seed2 = {1306, 421_106, 567_597} + :rand.seed(:exsss, seed1) + assert Enum.random([1, 2]) == 1 + assert Enum.random([1, 2]) == 2 + :rand.seed(:exsss, seed1) + assert Enum.random([1, 2]) == 1 + assert Enum.random([1, 2, 3]) == 1 + assert Enum.random([1, 2, 3, 4]) == 2 + assert Enum.random([1, 2, 3, 4, 5]) == 3 + :rand.seed(:exsss, seed2) + assert Enum.random([1, 2]) == 1 + assert Enum.random([1, 2, 3]) == 2 + assert Enum.random([1, 2, 3, 4]) == 4 + assert Enum.random([1, 2, 3, 4, 5]) == 3 + end + + test "random/1 with streams" do + random_stream = fn list -> list |> Stream.map(& &1) |> Enum.random() end + + assert_raise Enum.EmptyError, fn -> random_stream.([]) end + assert random_stream.([1]) == 1 + + seed = {1406, 407_414, 139_258} + :rand.seed(:exsss, seed) + + assert random_stream.([1, 2]) == 2 + assert random_stream.([1, 2, 3]) == 3 + assert random_stream.([1, 2, 3, 4]) == 1 + assert random_stream.([1, 2, 3, 4, 5]) == 3 + end + + test "reduce/2" do + assert Enum.reduce([1, 2, 3], fn x, acc -> x + acc end) == 6 + + assert_raise Enum.EmptyError, fn -> + Enum.reduce([], fn x, acc -> x + acc end) + end + + assert_raise Enum.EmptyError, fn -> + Enum.reduce(%{}, fn _, acc -> acc end) + end + end + + test "reduce/3" do + assert Enum.reduce([], 1, fn x, acc -> x + acc end) == 1 + assert Enum.reduce([1, 2, 3], 1, fn x, acc -> x + acc end) == 7 + end + + test "reduce/3 with streams" do + reduce_stream = fn list, acc, fun -> list |> Stream.map(& &1) |> Enum.reduce(acc, fun) end + + assert reduce_stream.([], 1, fn x, acc -> x + acc end) == 1 + assert reduce_stream.([1, 2, 3], 1, fn x, acc -> x + acc end) == 7 + end + + test "reduce_while/3" do + assert Enum.reduce_while([1, 2, 3], 1, fn i, acc -> {:cont, acc + i} end) == 7 + assert Enum.reduce_while([1, 2, 3], 1, fn _i, acc -> {:halt, acc} end) == 1 + assert Enum.reduce_while([], 0, fn _i, acc -> {:cont, acc} end) == 0 + end + + test "reject/2" do + assert Enum.reject([1, 2, 3], fn x -> rem(x, 2) == 0 end) == [1, 3] + assert Enum.reject([2, 4, 6], fn x -> rem(x, 2) == 0 end) == [] + assert Enum.reject([1, true, nil, false, 2], & &1) == [nil, false] + end + + test "reverse/1" do assert Enum.reverse([]) == [] assert Enum.reverse([1, 2, 3]) == [3, 2, 1] + assert Enum.reverse([5..5]) == [5..5] + end + + test "reverse/2" do assert Enum.reverse([1, 2, 3], [4, 5, 6]) == [3, 2, 1, 4, 5, 6] + assert Enum.reverse([1, 2, 3], []) == [3, 2, 1] + assert Enum.reverse([5..5], [5]) == [5..5, 5] + end + + test "reverse_slice/3" do + assert Enum.reverse_slice([], 1, 2) == [] + assert Enum.reverse_slice([1, 2, 3], 0, 0) == [1, 2, 3] + assert Enum.reverse_slice([1, 2, 3], 0, 1) == [1, 2, 3] + assert Enum.reverse_slice([1, 2, 3], 0, 2) == [2, 1, 3] + assert Enum.reverse_slice([1, 2, 3], 0, 20_000_000) == [3, 2, 1] + assert Enum.reverse_slice([1, 2, 3], 100, 2) == [1, 2, 3] + assert Enum.reverse_slice([1, 2, 3], 10, 10) == [1, 2, 3] + end + + describe "slide/3" do + test "on an empty enum produces an empty list" do + for enum <- [[], %{}, 0..-1//1, MapSet.new()] do + assert Enum.slide(enum, 0..0, 0) == [] + assert Enum.slide(enum, 1..1, 2) == [] + end + end + + test "on a single-element enumerable is the same as transforming to list" do + for enum <- [["foo"], [1], [%{foo: "bar"}], %{foo: :bar}, MapSet.new(["foo"]), 1..1] do + assert Enum.slide(enum, 0..0, 0) == Enum.to_list(enum) + end + end + + test "moves a single element" do + for zero_to_20 <- [0..20, Enum.to_list(0..20)] do + expected_numbers = Enum.flat_map([0..7, [14], 8..13, 15..20], &Enum.to_list/1) + assert Enum.slide(zero_to_20, 14..14, 8) == expected_numbers + end + + assert Enum.slide([:a, :b, :c, :d, :e, :f], 3..3, 2) == [:a, :b, :d, :c, :e, :f] + assert Enum.slide([:a, :b, :c, :d, :e, :f], 3, 3) == [:a, :b, :c, :d, :e, :f] + end + + test "on a subsection of a list reorders the range correctly" do + for zero_to_20 <- [0..20, Enum.to_list(0..20)] do + expected_numbers = Enum.flat_map([0..7, 14..18, 8..13, 19..20], &Enum.to_list/1) + assert Enum.slide(zero_to_20, 14..18, 8) == expected_numbers + end + + assert Enum.slide([:a, :b, :c, :d, :e, :f], 3..4, 2) == [:a, :b, :d, :e, :c, :f] + end + + test "handles negative indices" do + make_negative_range = fn first..last//1, length -> + (first - length)..(last - length)//1 + end + + test_specs = [ + {[], 0..0, 0}, + {[1], 0..0, 0}, + {[-2, 1], 1..1, 1}, + {[4, -3, 2, -1], 3..3, 2}, + {[-5, -3, 4, 4, 5], 0..2, 3}, + {[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], 4..7, 9}, + {[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], 4..7, 0} + ] + + for {list, range, insertion_point} <- test_specs do + negative_range = make_negative_range.(range, length(list)) + + assert Enum.slide(list, negative_range, insertion_point) == + Enum.slide(list, range, insertion_point) + end + end + + test "handles mixed positive and negative indices" do + for zero_to_20 <- [0..20, Enum.to_list(0..20)] do + assert Enum.slide(zero_to_20, -6..-1, 8) == + Enum.slide(zero_to_20, 15..20, 8) + + assert Enum.slide(zero_to_20, 15..-1//1, 8) == + Enum.slide(zero_to_20, 15..20, 8) + + assert Enum.slide(zero_to_20, -6..20, 8) == + Enum.slide(zero_to_20, 15..20, 8) + + assert Enum.slide(zero_to_20, -100..5, 8) == + Enum.slide(zero_to_20, 0..5, 8) + end + end + + test "raises an error when the step is not exactly 1" do + slide_ranges_that_should_fail = [2..10//2, 8..-1//-1, 10..2//-1, 10..4//-2, -1..-8//-1] + + for zero_to_20 <- [0..20, Enum.to_list(0..20)], + range_that_should_fail <- slide_ranges_that_should_fail do + assert_raise(ArgumentError, fn -> + Enum.slide(zero_to_20, range_that_should_fail, 1) + end) + end + end + + test "doesn't change the order when the first and middle indices match" do + for zero_to_20 <- [0..20, Enum.to_list(0..20)] do + assert Enum.slide(zero_to_20, 8..18, 8) == Enum.to_list(0..20) + end + + assert Enum.slide([:a, :b, :c, :d, :e, :f], 1..3, 1) == [:a, :b, :c, :d, :e, :f] + end + + test "on the whole of an enumerable reorders it correctly" do + for zero_to_20 <- [0..20, Enum.to_list(0..20)] do + expected_numbers = Enum.flat_map([10..20, 0..9], &Enum.to_list/1) + assert Enum.slide(zero_to_20, 10..20, 0) == expected_numbers + end + + assert Enum.slide([:a, :b, :c, :d, :e, :f], 4..5, 0) == [:e, :f, :a, :b, :c, :d] + end + + test "raises when the insertion point is inside the range" do + for zero_to_20 <- [0..20, Enum.to_list(0..20)] do + assert_raise ArgumentError, fn -> + Enum.slide(zero_to_20, 10..18, 14) + end + end + end + + test "accepts range starts that are off the end of the enum, returning the input list" do + assert Enum.slide([], 1..5, 0) == [] + + for zero_to_20 <- [0..20, Enum.to_list(0..20)] do + assert Enum.slide(zero_to_20, 21..25, 3) == Enum.to_list(0..20) + end + end + + test "accepts range ends that are off the end of the enum, truncating the moved range" do + for zero_to_10 <- [0..10, Enum.to_list(0..10)] do + assert Enum.slide(zero_to_10, 8..15, 4) == Enum.slide(zero_to_10, 8..10, 4) + end + end + + test "matches behavior for lists vs. ranges" do + range = 0..20 + list = Enum.to_list(range) + # Below 32 elements, the map implementation currently sticks values in order. + # If ever the MapSet implementation changes, this will fail (not affecting the correctness + # of slide). I figured it'd be worth testing this for the time being just to have + # another enumerable (aside from range) testing the generic implementation. + set = MapSet.new(list) + + test_specs = [ + {0..0, 0}, + {0..0, 20}, + {11..11, 14}, + {11..11, 3}, + {4..8, 19}, + {4..8, 0}, + {4..8, 2}, + {10..20, 0}, + {2..1//1, -20} + ] + + for {slide_range, insertion_point} <- test_specs do + slide = &Enum.slide(&1, slide_range, insertion_point) + assert slide.(list) == slide.(set) + assert slide.(list) == slide.(range) + end + end + + test "inserts at negative indices" do + for zero_to_5 <- [0..5, Enum.to_list(0..5)] do + assert Enum.slide(zero_to_5, 0, -1) == [1, 2, 3, 4, 5, 0] + assert Enum.slide(zero_to_5, 1, -1) == [0, 2, 3, 4, 5, 1] + assert Enum.slide(zero_to_5, 1..2, -2) == [0, 3, 4, 1, 2, 5] + assert Enum.slide(zero_to_5, -5..-4//1, -2) == [0, 3, 4, 1, 2, 5] + end + + assert Enum.slide([:a, :b, :c, :d, :e, :f], -5..-3//1, -2) == + Enum.slide([:a, :b, :c, :d, :e, :f], 1..3, 4) + end + + test "raises when insertion index would fall inside the range" do + for zero_to_5 <- [0..5, Enum.to_list(0..5)] do + assert_raise ArgumentError, fn -> + Enum.slide(zero_to_5, 2..3, -3) + end + end + + for zero_to_10 <- [0..10, Enum.to_list(0..10)], + insertion_idx <- 3..5 do + assert_raise ArgumentError, fn -> + assert Enum.slide(zero_to_10, 2..5, insertion_idx) + end + end + end end - test :scan do - assert Enum.scan([1,2,3,4,5], &(&1 + &2)) == [1,3,6,10,15] + test "scan/2" do + assert Enum.scan([1, 2, 3, 4, 5], &(&1 + &2)) == [1, 3, 6, 10, 15] assert Enum.scan([], &(&1 + &2)) == [] + end - assert Enum.scan([1,2,3,4,5], 0, &(&1 + &2)) == [1,3,6,10,15] + test "scan/3" do + assert Enum.scan([1, 2, 3, 4, 5], 0, &(&1 + &2)) == [1, 3, 6, 10, 15] assert Enum.scan([], 0, &(&1 + &2)) == [] end - test :shuffle do + test "shuffle/1" do # set a fixed seed so the test can be deterministic - :random.seed(1374, 347975, 449264) - assert Enum.shuffle([1, 2, 3, 4, 5]) == [2, 4, 1, 5, 3] + :rand.seed(:exsss, {1374, 347_975, 449_264}) + assert Enum.shuffle([1, 2, 3, 4, 5]) == [2, 5, 4, 3, 1] + end + + test "slice/2" do + list = [1, 2, 3, 4, 5] + assert Enum.slice(list, 0..0) == [1] + assert Enum.slice(list, 0..1) == [1, 2] + assert Enum.slice(list, 0..2) == [1, 2, 3] + + assert Enum.slice(list, 0..10//2) == [1, 3, 5] + assert Enum.slice(list, 0..10//3) == [1, 4] + assert Enum.slice(list, 0..10//4) == [1, 5] + assert Enum.slice(list, 0..10//5) == [1] + assert Enum.slice(list, 0..10//6) == [1] + + assert Enum.slice(list, 0..2//2) == [1, 3] + assert Enum.slice(list, 0..2//3) == [1] + + assert Enum.slice(list, 0..-1//2) == [1, 3, 5] + assert Enum.slice(list, 0..-1//3) == [1, 4] + assert Enum.slice(list, 0..-1//4) == [1, 5] + assert Enum.slice(list, 0..-1//5) == [1] + assert Enum.slice(list, 0..-1//6) == [1] + + assert Enum.slice(list, 1..-1//2) == [2, 4] + assert Enum.slice(list, 1..-1//3) == [2, 5] + assert Enum.slice(list, 1..-1//4) == [2] + assert Enum.slice(list, 1..-1//5) == [2] + + assert Enum.slice(list, -4..-1//2) == [2, 4] + assert Enum.slice(list, -4..-1//3) == [2, 5] + assert Enum.slice(list, -4..-1//4) == [2] + assert Enum.slice(list, -4..-1//5) == [2] + end + + test "slice/3" do + list = [1, 2, 3, 4, 5] + assert Enum.slice(list, 0, 0) == [] + assert Enum.slice(list, 0, 1) == [1] + assert Enum.slice(list, 0, 2) == [1, 2] + assert Enum.slice(list, 1, 2) == [2, 3] + assert Enum.slice(list, 1, 0) == [] + assert Enum.slice(list, 2, 5) == [3, 4, 5] + assert Enum.slice(list, 2, 6) == [3, 4, 5] + assert Enum.slice(list, 5, 5) == [] + assert Enum.slice(list, 6, 5) == [] + assert Enum.slice(list, 6, 0) == [] + assert Enum.slice(list, -6, 0) == [] + assert Enum.slice(list, -6, 5) == [1, 2, 3, 4, 5] + assert Enum.slice(list, -2, 5) == [4, 5] + assert Enum.slice(list, -3, 1) == [3] + + assert_raise FunctionClauseError, fn -> + Enum.slice(list, 0, -1) + end + + assert_raise FunctionClauseError, fn -> + Enum.slice(list, 0.99, 0) + end + + assert_raise FunctionClauseError, fn -> + Enum.slice(list, 0, 0.99) + end + end + + test "slice on infinite streams" do + assert [1, 2, 3] |> Stream.cycle() |> Enum.slice(0, 2) == [1, 2] + assert [1, 2, 3] |> Stream.cycle() |> Enum.slice(0, 5) == [1, 2, 3, 1, 2] + assert [1, 2, 3] |> Stream.cycle() |> Enum.slice(0..1) == [1, 2] + assert [1, 2, 3] |> Stream.cycle() |> Enum.slice(0..4) == [1, 2, 3, 1, 2] + assert [1, 2, 3] |> Stream.cycle() |> Enum.slice(0..4//2) == [1, 3, 2] + assert [1, 2, 3] |> Stream.cycle() |> Enum.slice(0..5//2) == [1, 3, 2] + assert [1, 2, 3] |> Stream.cycle() |> Enum.slice(1..6//2) == [2, 1, 3] + end + + test "slice on pruned infinite streams" do + assert [1, 2, 3] |> Stream.cycle() |> Stream.take(10) |> Enum.slice(0, 2) == [1, 2] + assert [1, 2, 3] |> Stream.cycle() |> Stream.take(10) |> Enum.slice(0, 5) == [1, 2, 3, 1, 2] + assert [1, 2, 3] |> Stream.cycle() |> Stream.take(10) |> Enum.slice(0..1) == [1, 2] + assert [1, 2, 3] |> Stream.cycle() |> Stream.take(10) |> Enum.slice(0..4) == [1, 2, 3, 1, 2] + assert [1, 2, 3] |> Stream.cycle() |> Stream.take(10) |> Enum.slice(0..4//2) == [1, 3, 2] + + assert [1, 2, 3] |> Stream.cycle() |> Stream.take(10) |> Enum.slice(-10..-9//1) == + [1, 2] + + assert [1, 2, 3] |> Stream.cycle() |> Stream.take(10) |> Enum.slice(-10..-6//1) == + [1, 2, 3, 1, 2] + + assert [1, 2, 3] |> Stream.cycle() |> Stream.take(10) |> Enum.slice(-10..-6//2) == + [1, 3, 2] + + assert [1, 2, 3] |> Stream.cycle() |> Stream.take(10) |> Enum.slice(-9..-5//2) == + [2, 1, 3] + end + + test "slice on MapSets" do + assert MapSet.new(1..10) |> Enum.slice(0, 2) |> Enum.count() == 2 + assert MapSet.new(1..3) |> Enum.slice(0, 10) |> Enum.count() == 3 + assert MapSet.new(1..10) |> Enum.slice(0..1) |> Enum.count() == 2 + assert MapSet.new(1..3) |> Enum.slice(0..10) |> Enum.count() == 3 + + assert MapSet.new(1..10) |> Enum.slice(0..4//2) |> Enum.count() == 3 + assert MapSet.new(1..10) |> Enum.slice(0..5//2) |> Enum.count() == 3 + end + + test "sort/1" do + assert Enum.sort([5, 3, 2, 4, 1]) == [1, 2, 3, 4, 5] + end + + test "sort/2" do + assert Enum.sort([5, 3, 2, 4, 1], &(&1 >= &2)) == [5, 4, 3, 2, 1] + assert Enum.sort([5, 3, 2, 4, 1], :asc) == [1, 2, 3, 4, 5] + assert Enum.sort([5, 3, 2, 4, 1], :desc) == [5, 4, 3, 2, 1] + + assert Enum.sort([3, 2, 1, 3, 2, 3], :asc) == [1, 2, 2, 3, 3, 3] + assert Enum.sort([3, 2, 1, 3, 2, 3], :desc) == [3, 3, 3, 2, 2, 1] + + shuffled = Enum.shuffle(1..100) + assert Enum.sort(shuffled, :asc) == Enum.to_list(1..100) + assert Enum.sort(shuffled, :desc) == Enum.reverse(1..100) + end + + test "sort/2 with module" do + assert Enum.sort([~D[2020-01-01], ~D[2018-01-01], ~D[2019-01-01]], Date) == + [~D[2018-01-01], ~D[2019-01-01], ~D[2020-01-01]] + + assert Enum.sort([~D[2020-01-01], ~D[2018-01-01], ~D[2019-01-01]], {:asc, Date}) == + [~D[2018-01-01], ~D[2019-01-01], ~D[2020-01-01]] + + assert Enum.sort([~D[2020-01-01], ~D[2018-01-01], ~D[2019-01-01]], {:desc, Date}) == + [~D[2020-01-01], ~D[2019-01-01], ~D[2018-01-01]] + end + + test "sort/2 with streams" do + sort_stream = fn list, sorter -> list |> Stream.map(& &1) |> Enum.sort(sorter) end + + assert sort_stream.([5, 3, 2, 4, 1], &(&1 >= &2)) == [5, 4, 3, 2, 1] + assert sort_stream.([5, 3, 2, 4, 1], :asc) == [1, 2, 3, 4, 5] + assert sort_stream.([5, 3, 2, 4, 1], :desc) == [5, 4, 3, 2, 1] + + assert sort_stream.([3, 2, 1, 3, 2, 3], :asc) == [1, 2, 2, 3, 3, 3] + assert sort_stream.([3, 2, 1, 3, 2, 3], :desc) == [3, 3, 3, 2, 2, 1] + + shuffled = Enum.shuffle(1..100) + assert sort_stream.(shuffled, :asc) == Enum.to_list(1..100) + assert sort_stream.(shuffled, :desc) == Enum.reverse(1..100) + end + + test "sort_by/3" do + collection = [ + [sorted_data: 4], + [sorted_data: 5], + [sorted_data: 2], + [sorted_data: 1], + [sorted_data: 3] + ] + + asc = [ + [sorted_data: 1], + [sorted_data: 2], + [sorted_data: 3], + [sorted_data: 4], + [sorted_data: 5] + ] + + desc = [ + [sorted_data: 5], + [sorted_data: 4], + [sorted_data: 3], + [sorted_data: 2], + [sorted_data: 1] + ] + + assert Enum.sort_by(collection, & &1[:sorted_data]) == asc + assert Enum.sort_by(collection, & &1[:sorted_data], :asc) == asc + assert Enum.sort_by(collection, & &1[:sorted_data], &>=/2) == desc + assert Enum.sort_by(collection, & &1[:sorted_data], :desc) == desc + end + + test "sort_by/3 with stable sorting" do + collection = [ + [other_data: 2, sorted_data: 4], + [other_data: 1, sorted_data: 5], + [other_data: 2, sorted_data: 2], + [other_data: 3, sorted_data: 1], + [other_data: 4, sorted_data: 3] + ] + + # Stable sorting + assert Enum.sort_by(collection, & &1[:other_data]) == [ + [other_data: 1, sorted_data: 5], + [other_data: 2, sorted_data: 4], + [other_data: 2, sorted_data: 2], + [other_data: 3, sorted_data: 1], + [other_data: 4, sorted_data: 3] + ] + + assert Enum.sort_by(collection, & &1[:other_data]) == + Enum.sort_by(collection, & &1[:other_data], :asc) + + assert Enum.sort_by(collection, & &1[:other_data], & + Enum.split([1, 2, 3], 0.0) + end + end + + test "split_while/2" do + assert Enum.split_while([1, 2, 3], fn _ -> false end) == {[], [1, 2, 3]} + assert Enum.split_while([1, 2, 3], fn _ -> true end) == {[1, 2, 3], []} + assert Enum.split_while([1, 2, 3], fn x -> x > 2 end) == {[], [1, 2, 3]} + assert Enum.split_while([1, 2, 3], fn x -> x > 3 end) == {[], [1, 2, 3]} + assert Enum.split_while([1, 2, 3], fn x -> x < 3 end) == {[1, 2], [3]} + assert Enum.split_while([], fn _ -> true end) == {[], []} + end + + test "sum/1" do + assert Enum.sum([]) == 0 + assert Enum.sum([1]) == 1 + assert Enum.sum([1, 2, 3]) == 6 + assert Enum.sum([1.1, 2.2, 3.3]) == 6.6 + assert Enum.sum([-3, -2, -1, 0, 1, 2, 3]) == 0 + assert Enum.sum(42..42) == 42 + assert Enum.sum(11..17) == 98 + assert Enum.sum(17..11//-1) == 98 + assert Enum.sum(11..-17//-1) == Enum.sum(-17..11) + + assert_raise ArithmeticError, fn -> + Enum.sum([{}]) + end + + assert_raise ArithmeticError, fn -> + Enum.sum([1, {}]) + end end - test :sort do - assert Enum.sort([5, 3, 2, 4, 1]) == [1, 2, 3, 4, 5] - assert Enum.sort([5, 3, 2, 4, 1], &(&1 > &2)) == [5, 4, 3, 2, 1] + test "sum_by/2" do + assert Enum.sum_by([], &hd/1) == 0 + assert Enum.sum_by([[1]], &hd/1) == 1 + assert Enum.sum_by([[1], [2], [3]], &hd/1) == 6 + assert Enum.sum_by([[1.1], [2.2], [3.3]], &hd/1) == 6.6 + assert Enum.sum_by([[-3], [-2], [-1], [0], [1], [2], [3]], &hd/1) == 0 + + assert Enum.sum_by(1..3, &(&1 ** 2)) == 14 + + assert_raise ArithmeticError, fn -> + Enum.sum_by([[{}]], &hd/1) + end + + assert_raise ArithmeticError, fn -> + Enum.sum_by([[1], [{}]], &hd/1) + end + end + + test "product/1" do + assert Enum.product([]) == 1 + assert Enum.product([1]) == 1 + assert Enum.product([1, 2, 3, 4, 5]) == 120 + assert Enum.product([1, -2, 3, 4, 5]) == -120 + assert Enum.product(1..5) == 120 + assert Enum.product(11..-17//-1) == Enum.product(-17..11) + + assert_raise ArithmeticError, fn -> + Enum.product([{}]) + end + + assert_raise ArithmeticError, fn -> + Enum.product([1, {}]) + end + + assert_raise ArithmeticError, fn -> + Enum.product(%{a: 1, b: 2}) + end end - test :split do - assert Enum.split([1, 2, 3], 0) == {[], [1, 2, 3]} - assert Enum.split([1, 2, 3], 1) == {[1], [2, 3]} - assert Enum.split([1, 2, 3], 2) == {[1, 2], [3]} - assert Enum.split([1, 2, 3], 3) == {[1, 2, 3], []} - assert Enum.split([1, 2, 3], 4) == {[1, 2, 3], []} - assert Enum.split([], 3) == {[], []} - assert Enum.split([1, 2, 3], -1) == {[1, 2], [3]} - assert Enum.split([1, 2, 3], -2) == {[1], [2, 3]} - assert Enum.split([1, 2, 3], -3) == {[], [1, 2, 3]} - assert Enum.split([1, 2, 3], -10) == {[], [1, 2, 3]} - end + test "product_by/2" do + assert Enum.product_by([], &hd/1) == 1 + assert Enum.product_by([[1]], &hd/1) == 1 + assert Enum.product_by([[1], [2], [3], [4], [5]], &hd/1) == 120 + assert Enum.product_by([[1], [-2], [3], [4], [5]], &hd/1) == -120 + assert Enum.product_by(1..5, & &1) == 120 + assert Enum.product_by(11..-17//-1, & &1) == 0 - test :split_while do - assert Enum.split_while([1, 2, 3], fn(_) -> false end) == {[], [1, 2, 3]} - assert Enum.split_while([1, 2, 3], fn(_) -> true end) == {[1, 2, 3], []} - assert Enum.split_while([1, 2, 3], fn(x) -> x > 2 end) == {[], [1, 2, 3]} - assert Enum.split_while([1, 2, 3], fn(x) -> x > 3 end) == {[], [1, 2, 3]} - assert Enum.split_while([1, 2, 3], fn(x) -> x < 3 end) == {[1, 2], [3]} - assert Enum.split_while([], fn(_) -> true end) == {[], []} - end + assert_raise ArithmeticError, fn -> + Enum.product_by([[{}]], &hd/1) + end - test :sum do - assert Enum.sum([]) == 0 - assert Enum.sum([1]) == 1 - assert Enum.sum([1, 2, 3]) == 6 - assert Enum.sum([1.1, 2.2, 3.3]) == 6.6 assert_raise ArithmeticError, fn -> - Enum.sum([{}]) + Enum.product_by([[1], [{}]], &hd/1) end + assert_raise ArithmeticError, fn -> - Enum.sum([1,{}]) + Enum.product_by(%{a: 1, b: 2}, & &1) end end - test :take do + test "take/2" do assert Enum.take([1, 2, 3], 0) == [] assert Enum.take([1, 2, 3], 1) == [1] assert Enum.take([1, 2, 3], 2) == [1, 2] @@ -325,178 +1517,480 @@ defmodule EnumTest.List do assert Enum.take([1, 2, 3], -2) == [2, 3] assert Enum.take([1, 2, 3], -4) == [1, 2, 3] assert Enum.take([], 3) == [] + + assert_raise FunctionClauseError, fn -> + Enum.take([1, 2, 3], 0.0) + end end - test :take_every do + test "take_every/2" do assert Enum.take_every([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 2) == [1, 3, 5, 7, 9] + assert Enum.take_every([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 3) == [1, 4, 7, 10] assert Enum.take_every([], 2) == [] assert Enum.take_every([1, 2], 2) == [1] assert Enum.take_every([1, 2, 3], 0) == [] + assert Enum.take_every(1..3, 1) == [1, 2, 3] + + assert_raise FunctionClauseError, fn -> + Enum.take_every([1, 2, 3], -1) + end + + assert_raise FunctionClauseError, fn -> + Enum.take_every(1..10, 3.33) + end + end + + test "take_random/2" do + assert Enum.take_random(-42..-42, 1) == [-42] + + # corner cases, independent of the seed + assert_raise FunctionClauseError, fn -> Enum.take_random([1, 2], -1) end + assert Enum.take_random([], 0) == [] + assert Enum.take_random([], 3) == [] + assert Enum.take_random([1], 0) == [] + assert Enum.take_random([1], 2) == [1] + assert Enum.take_random([1, 2], 0) == [] + + # set a fixed seed so the test can be deterministic + # please note the order of following assertions is important + seed1 = {1406, 407_414, 139_258} + seed2 = {1406, 421_106, 567_597} + :rand.seed(:exsss, seed1) + assert Enum.take_random([1, 2, 3], 1) == [2] + assert Enum.take_random([1, 2, 3], 2) == [2, 3] + assert Enum.take_random([1, 2, 3], 3) == [3, 1, 2] + assert Enum.take_random([1, 2, 3], 4) == [2, 3, 1] + :rand.seed(:exsss, seed2) + assert Enum.take_random([1, 2, 3], 1) == [1] + assert Enum.take_random([1, 2, 3], 2) == [3, 1] + assert Enum.take_random([1, 2, 3], 3) == [2, 3, 1] + assert Enum.take_random([1, 2, 3], 4) == [3, 2, 1] + assert Enum.take_random([1, 2, 3], 129) == [2, 1, 3] + + # assert that every item in the sample comes from the input list + list = for _ <- 1..100, do: make_ref() + + for x <- Enum.take_random(list, 50) do + assert x in list + end + + assert_raise FunctionClauseError, fn -> + Enum.take_random(1..10, -1) + end + + assert_raise FunctionClauseError, fn -> + Enum.take_random(1..10, 10.0) + end + + assert_raise FunctionClauseError, fn -> + Enum.take_random(1..10, 128.1) + end end - test :take_while do - assert Enum.take_while([1, 2, 3], fn(x) -> x > 3 end) == [] - assert Enum.take_while([1, 2, 3], fn(x) -> x <= 1 end) == [1] - assert Enum.take_while([1, 2, 3], fn(x) -> x <= 3 end) == [1, 2, 3] - assert Enum.take_while([], fn(_) -> true end) == [] + test "take_while/2" do + assert Enum.take_while([1, 2, 3], fn x -> x > 3 end) == [] + assert Enum.take_while([1, 2, 3], fn x -> x <= 1 end) == [1] + assert Enum.take_while([1, 2, 3], fn x -> x <= 3 end) == [1, 2, 3] + assert Enum.take_while([], fn _ -> true end) == [] end - test :to_list do + test "to_list/1" do assert Enum.to_list([]) == [] - assert Enum.to_list(1 .. 3) == [1, 2, 3] end - test :traverse do - assert Enum.traverse([1, 2, 3], &(&1 * &1)) == [1, 4, 9] - assert Enum.traverse(%{a: 1, b: 2}, fn {k, v} -> {k, v*2} end) == %{a: 2, b: 4} + test "uniq/1" do + assert Enum.uniq([5, 1, 2, 3, 2, 1]) == [5, 1, 2, 3] + end + + test "uniq_by/2" do + assert Enum.uniq_by([1, 2, 3, 2, 1], fn x -> x end) == [1, 2, 3] + end + + test "unzip/1" do + assert Enum.unzip([{:a, 1}, {:b, 2}, {:c, 3}]) == {[:a, :b, :c], [1, 2, 3]} + assert Enum.unzip([]) == {[], []} + assert Enum.unzip(%{a: 1}) == {[:a], [1]} + assert Enum.unzip(foo: "a", bar: "b") == {[:foo, :bar], ["a", "b"]} + + assert_raise FunctionClauseError, fn -> Enum.unzip([{:a, 1}, {:b, 2, "foo"}]) end + assert_raise FunctionClauseError, fn -> Enum.unzip([{1, 2, {3, {4, 5}}}]) end + assert_raise FunctionClauseError, fn -> Enum.unzip([1, 2, 3]) end end - test :uniq do - assert Enum.uniq([1, 2, 3, 2, 1]) == [1, 2, 3] - assert Enum.uniq([1, 2, 3, 2, 1], fn x -> x end) == [1, 2, 3] + test "with_index/2" do + assert Enum.with_index([]) == [] + assert Enum.with_index([1, 2, 3]) == [{1, 0}, {2, 1}, {3, 2}] + assert Enum.with_index([1, 2, 3], 10) == [{1, 10}, {2, 11}, {3, 12}] + + assert Enum.with_index([1, 2, 3], fn element, index -> {index, element} end) == + [{0, 1}, {1, 2}, {2, 3}] + + assert Enum.with_index(1..0//1) == [] + assert Enum.with_index(1..3) == [{1, 0}, {2, 1}, {3, 2}] + assert Enum.with_index(1..3, 10) == [{1, 10}, {2, 11}, {3, 12}] + + assert Enum.with_index(1..3, fn element, index -> {index, element} end) == + [{0, 1}, {1, 2}, {2, 3}] end - test :zip do + test "zip/2" do assert Enum.zip([:a, :b], [1, 2]) == [{:a, 1}, {:b, 2}] assert Enum.zip([:a, :b], [1, 2, 3, 4]) == [{:a, 1}, {:b, 2}] assert Enum.zip([:a, :b, :c, :d], [1, 2]) == [{:a, 1}, {:b, 2}] + assert Enum.zip([], [1]) == [] assert Enum.zip([1], []) == [] - assert Enum.zip([], []) == [] + assert Enum.zip([], []) == [] end - test :with_index do - assert Enum.with_index([]) == [] - assert Enum.with_index([1,2,3]) == [{1,0},{2,1},{3,2}] - end + test "zip/2 with infinite streams" do + assert Enum.zip([], Stream.cycle([1, 2])) == [] + assert Enum.zip([], Stream.cycle(1..2)) == [] + assert Enum.zip(.., Stream.cycle([1, 2])) == [] + assert Enum.zip(.., Stream.cycle(1..2)) == [] - test :max do - assert Enum.max([1]) == 1 - assert Enum.max([1, 2, 3]) == 3 - assert Enum.max([1, [], :a, {}]) == [] - assert_raise Enum.EmptyError, fn -> - Enum.max([]) - end + assert Enum.zip(Stream.cycle([1, 2]), ..) == [] + assert Enum.zip(Stream.cycle(1..2), ..) == [] + assert Enum.zip(Stream.cycle([1, 2]), ..) == [] + assert Enum.zip(Stream.cycle(1..2), ..) == [] end - test :max_by do - assert Enum.max_by(["a", "aa", "aaa"], fn(x) -> String.length(x) end) == "aaa" - assert_raise Enum.EmptyError, fn -> - Enum.max_by([], fn(x) -> String.length(x) end) - end - end + test "zip/1" do + assert Enum.zip([[:a, :b], [1, 2], ["foo", "bar"]]) == [{:a, 1, "foo"}, {:b, 2, "bar"}] - test :min do - assert Enum.min([1]) == 1 - assert Enum.min([1, 2, 3]) == 1 - assert Enum.min([[], :a, {}]) == :a - assert_raise Enum.EmptyError, fn -> - Enum.min([]) - end - end + assert Enum.zip([[:a, :b], [1, 2, 3, 4], ["foo", "bar", "baz", "qux"]]) == + [{:a, 1, "foo"}, {:b, 2, "bar"}] - test :min_by do - assert Enum.min_by(["a", "aa", "aaa"], fn(x) -> String.length(x) end) == "a" - assert_raise Enum.EmptyError, fn -> - Enum.min_by([], fn(x) -> String.length(x) end) - end - end + assert Enum.zip([[:a, :b, :c, :d], [1, 2], ["foo", "bar", "baz", "qux"]]) == + [{:a, 1, "foo"}, {:b, 2, "bar"}] + + assert Enum.zip([[:a, :b, :c, :d], [1, 2, 3, 4], ["foo", "bar"]]) == + [{:a, 1, "foo"}, {:b, 2, "bar"}] + + assert Enum.zip([1..10, ["foo", "bar"]]) == [{1, "foo"}, {2, "bar"}] - test :chunk do - assert Enum.chunk([1, 2, 3, 4, 5], 2) == [[1, 2], [3, 4]] - assert Enum.chunk([1, 2, 3, 4, 5], 2, 2, [6]) == [[1, 2], [3, 4], [5, 6]] - assert Enum.chunk([1, 2, 3, 4, 5, 6], 3, 2) == [[1, 2, 3], [3, 4, 5]] - assert Enum.chunk([1, 2, 3, 4, 5, 6], 2, 3) == [[1, 2], [4, 5]] - assert Enum.chunk([1, 2, 3, 4, 5, 6], 3, 2, []) == [[1, 2, 3], [3, 4, 5], [5, 6]] - assert Enum.chunk([1, 2, 3, 4, 5, 6], 3, 3, []) == [[1, 2, 3], [4, 5, 6]] - assert Enum.chunk([1, 2, 3, 4, 5], 4, 4, 6..10) == [[1, 2, 3, 4], [5, 6, 7, 8]] + assert Enum.zip([]) == [] + assert Enum.zip([[]]) == [] + assert Enum.zip([[1]]) == [{1}] + + assert Enum.zip([[], [], [], []]) == [] + assert Enum.zip(%{}) == [] end - test :chunk_by do - assert Enum.chunk_by([1, 2, 2, 3, 4, 4, 6, 7, 7], &(rem(&1, 2) == 1)) == [[1], [2, 2], [3], [4, 4, 6], [7, 7]] - assert Enum.chunk_by([1, 2, 3, 4], fn _ -> true end) == [[1, 2, 3, 4]] - assert Enum.chunk_by([], fn _ -> true end) == [] - assert Enum.chunk_by([1], fn _ -> true end) == [[1]] + test "zip_with/3" do + assert Enum.zip_with([1, 2], [3, 4], fn a, b -> a * b end) == [3, 8] + assert Enum.zip_with([:a, :b], [1, 2], &{&1, &2}) == [{:a, 1}, {:b, 2}] + assert Enum.zip_with([:a, :b], [1, 2, 3, 4], &{&1, &2}) == [{:a, 1}, {:b, 2}] + assert Enum.zip_with([:a, :b, :c, :d], [1, 2], &{&1, &2}) == [{:a, 1}, {:b, 2}] + assert Enum.zip_with([], [1], &{&1, &2}) == [] + assert Enum.zip_with([1], [], &{&1, &2}) == [] + assert Enum.zip_with([], [], &{&1, &2}) == [] + + # Ranges + assert Enum.zip_with(1..6, 3..4, fn a, b -> a + b end) == [4, 6] + assert Enum.zip_with([1, 2, 5, 6], 3..4, fn a, b -> a + b end) == [4, 6] + assert Enum.zip_with(fn _, _ -> {:cont, [1, 2]} end, 3..4, fn a, b -> a + b end) == [4, 6] + assert Enum.zip_with(1..1, 0..0, fn a, b -> a + b end) == [1] + + # Date.range + week_1 = Date.range(~D[2020-10-12], ~D[2020-10-16]) + week_2 = Date.range(~D[2020-10-19], ~D[2020-10-23]) + + result = + Enum.zip_with(week_1, week_2, fn a, b -> + Date.day_of_week(a) + Date.day_of_week(b) + end) + + assert result == [2, 4, 6, 8, 10] + + # Maps + result = Enum.zip_with(%{a: 7}, 3..4, fn {key, value}, b -> {key, value + b} end) + assert result == [a: 10] + + result = Enum.zip_with(3..4, %{a: 7}, fn a, {key, value} -> {key, value + a} end) + assert result == [a: 10] end - test :slice do - assert Enum.slice([1,2,3,4,5], 0, 0) == [] - assert Enum.slice([1,2,3,4,5], 0, 1) == [1] - assert Enum.slice([1,2,3,4,5], 0, 2) == [1, 2] - assert Enum.slice([1,2,3,4,5], 1, 2) == [2, 3] - assert Enum.slice([1,2,3,4,5], 1, 0) == [] - assert Enum.slice([1,2,3,4,5], 2, 5) == [3, 4, 5] - assert Enum.slice([1,2,3,4,5], 2, 6) == [3, 4, 5] - assert Enum.slice([1,2,3,4,5], 5, 5) == [] - assert Enum.slice([1,2,3,4,5], 6, 5) == [] - assert Enum.slice([1,2,3,4,5], 6, 0) == [] - assert Enum.slice([1,2,3,4,5], -6, 0) == [] - assert Enum.slice([1,2,3,4,5], -6, 5) == [] - assert Enum.slice([1,2,3,4,5], -2, 5) == [4, 5] - assert Enum.slice([1,2,3,4,5], -3, 1) == [3] - end - - test :slice_range do - assert Enum.slice([1,2,3,4,5], 0..0) == [1] - assert Enum.slice([1,2,3,4,5], 0..1) == [1, 2] - assert Enum.slice([1,2,3,4,5], 0..2) == [1, 2, 3] - assert Enum.slice([1,2,3,4,5], 1..2) == [2, 3] - assert Enum.slice([1,2,3,4,5], 1..0) == [] - assert Enum.slice([1,2,3,4,5], 2..5) == [3, 4, 5] - assert Enum.slice([1,2,3,4,5], 2..6) == [3, 4, 5] - assert Enum.slice([1,2,3,4,5], 4..4) == [5] - assert Enum.slice([1,2,3,4,5], 5..5) == [] - assert Enum.slice([1,2,3,4,5], 6..5) == [] - assert Enum.slice([1,2,3,4,5], 6..0) == [] - assert Enum.slice([1,2,3,4,5], -6..0) == [] - assert Enum.slice([1,2,3,4,5], -6..5) == [] - assert Enum.slice([1,2,3,4,5], -5..-1) == [1, 2, 3, 4, 5] - assert Enum.slice([1,2,3,4,5], -5..-3) == [1, 2, 3] - assert Enum.slice([1,2,3,4,5], -6..-1) == [] - assert Enum.slice([1,2,3,4,5], -6..-3) == [] + test "zip_with/2" do + zip_fun = fn items -> List.to_tuple(items) end + result = Enum.zip_with([[:a, :b], [1, 2], ["foo", "bar"]], zip_fun) + assert result == [{:a, 1, "foo"}, {:b, 2, "bar"}] + + map = %{a: :b, c: :d} + [x1, x2] = Map.to_list(map) + lots = Enum.zip_with([[:a, :b], [1, 2], ["foo", "bar"], map], zip_fun) + assert lots == [{:a, 1, "foo", x1}, {:b, 2, "bar", x2}] + + assert Enum.zip_with([[:a, :b], [1, 2, 3, 4], ["foo", "bar", "baz", "qux"]], zip_fun) == + [{:a, 1, "foo"}, {:b, 2, "bar"}] + + assert Enum.zip_with([[:a, :b, :c, :d], [1, 2], ["foo", "bar", "baz", "qux"]], zip_fun) == + [{:a, 1, "foo"}, {:b, 2, "bar"}] + + assert Enum.zip_with([[:a, :b, :c, :d], [1, 2, 3, 4], ["foo", "bar"]], zip_fun) == + [{:a, 1, "foo"}, {:b, 2, "bar"}] + + assert Enum.zip_with([1..10, ["foo", "bar"]], zip_fun) == [{1, "foo"}, {2, "bar"}] + assert Enum.zip_with([], zip_fun) == [] + assert Enum.zip_with([[]], zip_fun) == [] + assert Enum.zip_with([[1]], zip_fun) == [{1}] + assert Enum.zip_with([[], [], [], []], zip_fun) == [] + assert Enum.zip_with(%{}, zip_fun) == [] + assert Enum.zip_with([[1, 2, 5, 6], 3..4], fn [x, y] -> x + y end) == [4, 6] + + # Ranges + assert Enum.zip_with([1..6, 3..4], fn [a, b] -> a + b end) == [4, 6] + assert Enum.zip_with([[1, 2, 5, 6], 3..4], fn [a, b] -> a + b end) == [4, 6] + assert Enum.zip_with([fn _, _ -> {:cont, [1, 2]} end, 3..4], fn [a, b] -> a + b end) == [4, 6] + assert Enum.zip_with([1..1, 0..0], fn [a, b] -> a + b end) == [1] + + # Date.range + week_1 = Date.range(~D[2020-10-12], ~D[2020-10-16]) + week_2 = Date.range(~D[2020-10-19], ~D[2020-10-23]) + + result = + Enum.zip_with([week_1, week_2], fn [a, b] -> + Date.day_of_week(a) + Date.day_of_week(b) + end) + + assert result == [2, 4, 6, 8, 10] + + # Maps + result = Enum.zip_with([%{a: 7}, 3..4], fn [{key, value}, b] -> {key, value + b} end) + assert result == [a: 10] + + result = Enum.zip_with([%{a: 7}, 3..4], fn [{key, value}, b] -> {key, value + b} end) + assert result == [a: 10] end end defmodule EnumTest.Range do + # Ranges use custom callbacks for protocols in many operations. use ExUnit.Case, async: true - test :all? do - range = 0..5 - refute Enum.all?(range, fn(x) -> rem(x, 2) == 0 end) + test "all?/2" do + assert Enum.all?(0..1) + assert Enum.all?(1..0//-1) + refute Enum.all?(0..5, fn x -> rem(x, 2) == 0 end) + assert Enum.all?(0..1, fn x -> x < 2 end) + + assert Enum.all?(0..1//-1) + assert Enum.all?(0..5//2, fn x -> rem(x, 2) == 0 end) + refute Enum.all?(1..5//2, fn x -> rem(x, 2) == 0 end) + end + + test "any?/2" do + assert Enum.any?(1..0//-1) + refute Enum.any?(0..5, &(&1 > 10)) + assert Enum.any?(0..5, &(&1 > 3)) + + refute Enum.any?(0..1//-1) + assert Enum.any?(0..5//2, fn x -> rem(x, 2) == 0 end) + refute Enum.any?(1..5//2, fn x -> rem(x, 2) == 0 end) + end + + test "at/3" do + assert Enum.at(2..6, 0) == 2 + assert Enum.at(2..6, 4) == 6 + assert Enum.at(2..6, 6) == nil + assert Enum.at(2..6, 6, :none) == :none + assert Enum.at(2..6, -2) == 5 + assert Enum.at(2..6, -8) == nil + + assert Enum.at(0..1//-1, 0) == nil + assert Enum.at(1..1//5, 0) == 1 + assert Enum.at(1..3//2, 0) == 1 + assert Enum.at(1..3//2, 1) == 3 + assert Enum.at(1..3//2, 2) == nil + assert Enum.at(1..3//2, -1) == 3 + assert Enum.at(1..3//2, -2) == 1 + assert Enum.at(1..3//2, -3) == nil + end + + test "chunk_every/2" do + assert Enum.chunk_every(1..5, 2) == [[1, 2], [3, 4], [5]] + assert Enum.chunk_every(1..10//2, 2) == [[1, 3], [5, 7], [9]] + end - range = 0..1 - assert Enum.all?(range, fn(x) -> x < 2 end) - assert Enum.all?(range) + test "chunk_every/4" do + assert Enum.chunk_every(1..5, 2, 2) == [[1, 2], [3, 4], [5]] + assert Enum.chunk_every(1..6, 3, 2, :discard) == [[1, 2, 3], [3, 4, 5]] + assert Enum.chunk_every(1..6, 2, 3, :discard) == [[1, 2], [4, 5]] + assert Enum.chunk_every(1..6, 3, 2, []) == [[1, 2, 3], [3, 4, 5], [5, 6]] + assert Enum.chunk_every(1..5, 4, 4, 6..10) == [[1, 2, 3, 4], [5, 6, 7, 8]] + assert Enum.chunk_every(1..10//2, 4, 4, 11..20) == [[1, 3, 5, 7], [9, 11, 12, 13]] + end - range = 1..0 - assert Enum.all?(range) + test "chunk_by/2" do + assert Enum.chunk_by(1..4, fn _ -> true end) == [[1, 2, 3, 4]] + assert Enum.chunk_by(1..4, &(rem(&1, 2) == 1)) == [[1], [2], [3], [4]] + assert Enum.chunk_by(1..20//3, &(rem(&1, 2) == 1)) == [[1], [4], [7], [10], [13], [16], [19]] + end + + test "concat/1" do + assert Enum.concat([1..2, 4..6]) == [1, 2, 4, 5, 6] + assert Enum.concat([1..5, fn acc, _ -> acc end, [1]]) == [1, 2, 3, 4, 5, 1] + assert Enum.concat([1..5, 6..10//2]) == [1, 2, 3, 4, 5, 6, 8, 10] + end + + test "concat/2" do + assert Enum.concat(1..3, 4..5) == [1, 2, 3, 4, 5] + assert Enum.concat(1..3, [4, 5]) == [1, 2, 3, 4, 5] + assert Enum.concat(1..3, []) == [1, 2, 3] + assert Enum.concat(1..3, 0..0) == [1, 2, 3, 0] + assert Enum.concat(1..5, 6..10//2) == [1, 2, 3, 4, 5, 6, 8, 10] + assert Enum.concat(1..5, 0..1//-1) == [1, 2, 3, 4, 5] + assert Enum.concat(1..5, 1..0//1) == [1, 2, 3, 4, 5] + end + + test "count/1" do + assert Enum.count(1..5) == 5 + assert Enum.count(1..1) == 1 + assert Enum.count(1..9//2) == 5 + assert Enum.count(1..10//2) == 5 + assert Enum.count(1..11//2) == 6 + assert Enum.count(1..11//-2) == 0 + assert Enum.count(11..1//-2) == 6 + assert Enum.count(10..1//-2) == 5 + assert Enum.count(9..1//-2) == 5 + assert Enum.count(9..1//2) == 0 + end + + test "count/2" do + assert Enum.count(1..5, fn x -> rem(x, 2) == 0 end) == 2 + assert Enum.count(1..1, fn x -> rem(x, 2) == 0 end) == 0 + assert Enum.count(0..5//2, fn x -> rem(x, 2) == 0 end) == 3 + assert Enum.count(1..5//2, fn x -> rem(x, 2) == 0 end) == 0 + end + + test "dedup/1" do + assert Enum.dedup(1..3) == [1, 2, 3] + assert Enum.dedup(1..3//2) == [1, 3] + end + + test "dedup_by/2" do + assert Enum.dedup_by(1..3, fn _ -> 1 end) == [1] + assert Enum.dedup_by(1..3//2, fn _ -> 1 end) == [1] + end + + test "drop/2" do + assert Enum.drop(1..3, 0) == [1, 2, 3] + assert Enum.drop(1..3, 1) == [2, 3] + assert Enum.drop(1..3, 2) == [3] + assert Enum.drop(1..3, 3) == [] + assert Enum.drop(1..3, 4) == [] + assert Enum.drop(1..3, -1) == [1, 2] + assert Enum.drop(1..3, -2) == [1] + assert Enum.drop(1..3, -4) == [] + assert Enum.drop(1..0//-1, 3) == [] + + assert Enum.drop(1..9//2, 2) == [5, 7, 9] + assert Enum.drop(1..9//2, -2) == [1, 3, 5] + assert Enum.drop(9..1//-2, 2) == [5, 3, 1] + assert Enum.drop(9..1//-2, -2) == [9, 7, 5] + end + + test "drop_every/2" do + assert Enum.drop_every(1..10, 2) == [2, 4, 6, 8, 10] + assert Enum.drop_every(1..10, 3) == [2, 3, 5, 6, 8, 9] + assert Enum.drop_every(0..0, 2) == [] + assert Enum.drop_every(1..2, 2) == [2] + assert Enum.drop_every(1..3, 0) == [1, 2, 3] + assert Enum.drop_every(1..3, 1) == [] + + assert Enum.drop_every(1..5//2, 0) == [1, 3, 5] + assert Enum.drop_every(1..5//2, 1) == [] + assert Enum.drop_every(1..5//2, 2) == [3] + + assert_raise FunctionClauseError, fn -> + Enum.drop_every(1..10, 3.33) + end end - test :any? do - range = 0..5 - refute Enum.any?(range, &(&1 > 10)) + test "drop_while/2" do + assert Enum.drop_while(0..6, fn x -> x <= 3 end) == [4, 5, 6] + assert Enum.drop_while(0..6, fn _ -> false end) == [0, 1, 2, 3, 4, 5, 6] + assert Enum.drop_while(0..3, fn x -> x <= 3 end) == [] + assert Enum.drop_while(1..0//-1, fn _ -> nil end) == [1, 0] + end - range = 0..5 - assert Enum.any?(range, &(&1 > 3)) + test "each/2" do + try do + assert Enum.each(1..0//-1, fn x -> x end) == :ok + assert Enum.each(1..3, fn x -> Process.put(:enum_test_each, x * 2) end) == :ok + assert Process.get(:enum_test_each) == 6 + after + Process.delete(:enum_test_each) + end - range = 1..0 - assert Enum.any?(range) + try do + assert Enum.each(-1..-3//-1, fn x -> Process.put(:enum_test_each, x * 2) end) == :ok + assert Process.get(:enum_test_each) == -6 + after + Process.delete(:enum_test_each) + end end - test :fetch! do + test "empty?/1" do + refute Enum.empty?(1..0//-1) + refute Enum.empty?(1..2) + refute Enum.empty?(1..2//2) + assert Enum.empty?(1..2//-2) + end + + test "fetch/2" do + # ascending order + assert Enum.fetch(-10..20, 4) == {:ok, -6} + assert Enum.fetch(-10..20, -4) == {:ok, 17} + # ascending order, first + assert Enum.fetch(-10..20, 0) == {:ok, -10} + assert Enum.fetch(-10..20, -31) == {:ok, -10} + # ascending order, last + assert Enum.fetch(-10..20, -1) == {:ok, 20} + assert Enum.fetch(-10..20, 30) == {:ok, 20} + # ascending order, out of bound + assert Enum.fetch(-10..20, 31) == :error + assert Enum.fetch(-10..20, -32) == :error + + # descending order + assert Enum.fetch(20..-10//-1, 4) == {:ok, 16} + assert Enum.fetch(20..-10//-1, -4) == {:ok, -7} + # descending order, first + assert Enum.fetch(20..-10//-1, 0) == {:ok, 20} + assert Enum.fetch(20..-10//-1, -31) == {:ok, 20} + # descending order, last + assert Enum.fetch(20..-10//-1, -1) == {:ok, -10} + assert Enum.fetch(20..-10//-1, 30) == {:ok, -10} + # descending order, out of bound + assert Enum.fetch(20..-10//-1, 31) == :error + assert Enum.fetch(20..-10//-1, -32) == :error + + # edge cases + assert Enum.fetch(42..42, 0) == {:ok, 42} + assert Enum.fetch(42..42, -1) == {:ok, 42} + assert Enum.fetch(42..42, 2) == :error + assert Enum.fetch(42..42, -2) == :error + + assert Enum.fetch(42..42//2, 0) == {:ok, 42} + assert Enum.fetch(42..42//2, -1) == {:ok, 42} + assert Enum.fetch(42..42//2, 2) == :error + assert Enum.fetch(42..42//2, -2) == :error + end + + test "fetch!/2" do assert Enum.fetch!(2..6, 0) == 2 assert Enum.fetch!(2..6, 4) == 6 assert Enum.fetch!(2..6, -1) == 6 assert Enum.fetch!(2..6, -2) == 5 - assert Enum.fetch!(-2..-6, 0) == -2 - assert Enum.fetch!(-2..-6, 4) == -6 + assert Enum.fetch!(-2..-6//-1, 0) == -2 + assert Enum.fetch!(-2..-6//-1, 4) == -6 assert_raise Enum.OutOfBoundsError, fn -> Enum.fetch!(2..6, 8) end assert_raise Enum.OutOfBoundsError, fn -> - Enum.fetch!(-2..-6, 8) + Enum.fetch!(-2..-6//-1, 8) end assert_raise Enum.OutOfBoundsError, fn -> @@ -504,371 +1998,585 @@ defmodule EnumTest.Range do end end - test :count do - range = 1..5 - assert Enum.count(range) == 5 - range = 1..1 - assert Enum.count(range) == 1 - end + test "filter/2" do + assert Enum.filter(1..3, fn x -> rem(x, 2) == 0 end) == [2] + assert Enum.filter(1..6, fn x -> rem(x, 2) == 0 end) == [2, 4, 6] - test :count_fun do - range = 1..5 - assert Enum.count(range, fn(x) -> rem(x, 2) == 0 end) == 2 - range = 1..1 - assert Enum.count(range, fn(x) -> rem(x, 2) == 0 end) == 0 + assert Enum.filter(1..3, &match?(1, &1)) == [1] + assert Enum.filter(1..3, &match?(x when x < 3, &1)) == [1, 2] + assert Enum.filter(1..3, fn _ -> true end) == [1, 2, 3] end - test :chunk do - assert Enum.chunk(1..5, 2) == [[1, 2], [3, 4]] - assert Enum.chunk(1..5, 2, 2, [6]) == [[1, 2], [3, 4], [5, 6]] - assert Enum.chunk(1..6, 3, 2) == [[1, 2, 3], [3, 4, 5]] - assert Enum.chunk(1..6, 2, 3) == [[1, 2], [4, 5]] - assert Enum.chunk(1..6, 3, 2, []) == [[1, 2, 3], [3, 4, 5], [5, 6]] - assert Enum.chunk(1..5, 4, 4, 6..10) == [[1, 2, 3, 4], [5, 6, 7, 8]] + test "find/3" do + assert Enum.find(2..6, fn x -> rem(x, 2) == 0 end) == 2 + assert Enum.find(2..6, fn x -> rem(x, 2) == 1 end) == 3 + assert Enum.find(2..6, fn _ -> false end) == nil + assert Enum.find(2..6, 0, fn _ -> false end) == 0 end - test :chunk_by do - assert Enum.chunk_by(1..4, fn _ -> true end) == [[1, 2, 3, 4]] - assert Enum.chunk_by(1..4, &(rem(&1, 2) == 1)) == [[1], [2], [3], [4]] + test "find_index/2" do + assert Enum.find_index(2..6, fn x -> rem(x, 2) == 1 end) == 1 end - test :drop do - range = 1..3 - assert Enum.drop(range, 0) == [1, 2, 3] - assert Enum.drop(range, 1) == [2, 3] - assert Enum.drop(range, 2) == [3] - assert Enum.drop(range, 3) == [] - assert Enum.drop(range, 4) == [] - assert Enum.drop(range, -1) == [1, 2] - assert Enum.drop(range, -2) == [1] - assert Enum.drop(range, -4) == [] + test "find_value/3" do + assert Enum.find_value(2..6, fn x -> rem(x, 2) == 1 end) + end - range = 1..0 - assert Enum.drop(range, 3) == [] + test "flat_map/2" do + assert Enum.flat_map(1..3, fn x -> [x, x] end) == [1, 1, 2, 2, 3, 3] end - test :drop_while do - range = 0..6 - assert Enum.drop_while(range, fn(x) -> x <= 3 end) == [4, 5, 6] - assert Enum.drop_while(range, fn(_) -> false end) == [0, 1, 2, 3, 4, 5, 6] + test "flat_map_reduce/3" do + assert Enum.flat_map_reduce(1..100, 0, fn i, acc -> + if acc < 3, do: {[i], acc + 1}, else: {:halt, acc} + end) == {[1, 2, 3], 3} + end - range = 0..3 - assert Enum.drop_while(range, fn(x) -> x <= 3 end) == [] + test "group_by/3" do + assert Enum.group_by(1..6, &rem(&1, 3)) == %{0 => [3, 6], 1 => [1, 4], 2 => [2, 5]} - range = 1..0 - assert Enum.drop_while(range, fn(_) -> false end) == [1, 0] + assert Enum.group_by(1..6, &rem(&1, 3), &(&1 * 2)) == + %{0 => [6, 12], 1 => [2, 8], 2 => [4, 10]} end - test :find do - range = 2..6 - assert Enum.find(range, fn(x) -> rem(x, 2) == 0 end) == 2 - assert Enum.find(range, fn(x) -> rem(x, 2) == 1 end) == 3 - assert Enum.find(range, fn _ -> false end) == nil - assert Enum.find(range, 0, fn _ -> false end) == 0 + test "intersperse/2" do + assert Enum.intersperse(1..0//-1, true) == [1, true, 0] + assert Enum.intersperse(1..3, false) == [1, false, 2, false, 3] end - test :find_value do - range = 2..6 - assert Enum.find_value(range, fn(x) -> rem(x, 2) == 1 end) + test "into/2" do + assert Enum.into(1..5, []) == [1, 2, 3, 4, 5] + assert Enum.into(1..5, MapSet.new()) == MapSet.new([1, 2, 3, 4, 5]) end - test :find_index do - range = 2..6 - assert Enum.find_index(range, fn(x) -> rem(x, 2) == 1 end) == 1 + test "into/3" do + assert Enum.into(1..5, [], fn x -> x * 2 end) == [2, 4, 6, 8, 10] + assert Enum.into(1..3, "numbers: ", &to_string/1) == "numbers: 123" + assert Enum.into(1..3, MapSet.new(), &(&1 * 2)) == MapSet.new([2, 4, 6]) end - test :empty? do - range = 1..0 - refute Enum.empty?(range) + test "join/2" do + assert Enum.join(1..0//-1, " = ") == "1 = 0" + assert Enum.join(1..3, " = ") == "1 = 2 = 3" + assert Enum.join(1..3) == "123" + end - range = 1..2 - refute Enum.empty?(range) + test "map/2" do + assert Enum.map(1..3, fn x -> x * 2 end) == [2, 4, 6] + assert Enum.map(-1..-3//-1, fn x -> x * 2 end) == [-2, -4, -6] + assert Enum.map(1..10//2, fn x -> x * 2 end) == [2, 6, 10, 14, 18] + assert Enum.map(3..1//-2, fn x -> x * 2 end) == [6, 2] + assert Enum.map(0..1//-1, fn x -> x * 2 end) == [] end - test :each do - try do - range = 1..0 - assert Enum.each(range, fn(x) -> x end) == :ok + test "map_every/3" do + assert Enum.map_every(1..10, 2, fn x -> x * 2 end) == [2, 2, 6, 4, 10, 6, 14, 8, 18, 10] - range = 1..3 - assert Enum.each(range, fn(x) -> Process.put(:enum_test_each, x * 2) end) == :ok - assert Process.get(:enum_test_each) == 6 - after - Process.delete(:enum_test_each) - end + assert Enum.map_every(-1..-10//-1, 2, fn x -> x * 2 end) == + [-2, -2, -6, -4, -10, -6, -14, -8, -18, -10] - try do - range = -1..-3 - assert Enum.each(range, fn(x) -> Process.put(:enum_test_each, x * 2) end) == :ok - assert Process.get(:enum_test_each) == -6 - after - Process.delete(:enum_test_each) + assert Enum.map_every(1..2, 2, fn x -> x * 2 end) == [2, 2] + assert Enum.map_every(1..3, 0, fn x -> x * 2 end) == [1, 2, 3] + + assert_raise FunctionClauseError, fn -> + Enum.map_every(1..3, -1, fn x -> x * 2 end) end end - test :filter do - range = 1..3 - assert Enum.filter(range, fn(x) -> rem(x, 2) == 0 end) == [2] - - range = 1..6 - assert Enum.filter(range, fn(x) -> rem(x, 2) == 0 end) == [2, 4, 6] + test "map_intersperse/3" do + assert Enum.map_intersperse(1..1, :a, &(&1 * 2)) == [2] + assert Enum.map_intersperse(1..3, :a, &(&1 * 2)) == [2, :a, 4, :a, 6] end - test :filter_with_match do - range = 1..3 - assert Enum.filter(range, &match?(1, &1)) == [1] - assert Enum.filter(range, &match?(x when x < 3, &1)) == [1, 2] - assert Enum.filter(range, &match?(_, &1)) == [1, 2, 3] + test "map_join/3" do + assert Enum.map_join(1..0//-1, " = ", &(&1 * 2)) == "2 = 0" + assert Enum.map_join(1..3, " = ", &(&1 * 2)) == "2 = 4 = 6" + assert Enum.map_join(1..3, &(&1 * 2)) == "246" end - test :filter_map do - range = 1..3 - assert Enum.filter_map(range, fn(x) -> rem(x, 2) == 0 end, &(&1 * 2)) == [4] - - range = 2..6 - assert Enum.filter_map(range, fn(x) -> rem(x, 2) == 0 end, &(&1 * 2)) == [4, 8, 12] + test "map_reduce/3" do + assert Enum.map_reduce(1..0//-1, 1, fn x, acc -> {x * 2, x + acc} end) == {[2, 0], 2} + assert Enum.map_reduce(1..3, 1, fn x, acc -> {x * 2, x + acc} end) == {[2, 4, 6], 7} end - test :flat_map do - range = 1..3 - assert Enum.flat_map(range, fn(x) -> [x, x] end) == [1, 1, 2, 2, 3, 3] - end + test "max/1" do + assert Enum.max(1..1) == 1 + assert Enum.max(1..3) == 3 + assert Enum.max(3..1//-1) == 3 - test :intersperse do - range = 1..0 - assert Enum.intersperse(range, true) == [1, true, 0] + assert Enum.max(1..9//2) == 9 + assert Enum.max(1..10//2) == 9 + assert Enum.max(-1..-9//-2) == -1 - range = 1..3 - assert Enum.intersperse(range, false) == [1, false, 2, false, 3] + assert_raise Enum.EmptyError, fn -> Enum.max(1..0//1) end end - test :into do - assert Enum.into([a: 1, b: 2], %{}) == %{a: 1, b: 2} - assert Enum.into(%{a: 1, b: 2}, []) == [a: 1, b: 2] - assert Enum.into(3..5, [1, 2]) == [1, 2, 3, 4, 5] - assert Enum.into(1..5, []) == [1, 2, 3, 4, 5] - assert Enum.into(1..5, [], fn x -> x * 2 end) == [2, 4, 6, 8, 10] - assert Enum.into(1..3, "numbers: ", &to_string/1) == "numbers: 123" + test "max/2 with empty fallback" do + assert Enum.max(.., fn -> 0 end) === 0 + assert Enum.max(1..2, fn -> 0 end) === 2 end - test :join do - range = 1..0 - assert Enum.join(range, " = ") == "1 = 0" + test "max_by/2" do + assert Enum.max_by(1..1, fn x -> :math.pow(-2, x) end) == 1 + assert Enum.max_by(1..3, fn x -> :math.pow(-2, x) end) == 2 - range = 1..3 - assert Enum.join(range, " = ") == "1 = 2 = 3" - assert Enum.join(range) == "123" + assert Enum.max_by(1..8//3, fn x -> :math.pow(-2, x) end) == 4 + assert_raise Enum.EmptyError, fn -> Enum.max_by(1..0//1, & &1) end end - test :map_join do - range = 1..0 - assert Enum.map_join(range, " = ", &(&1 * 2)) == "2 = 0" + test "member?/2" do + assert Enum.member?(1..3, 2) + refute Enum.member?(1..3, 0) - range = 1..3 - assert Enum.map_join(range, " = ", &(&1 * 2)) == "2 = 4 = 6" - assert Enum.map_join(range, &(&1 * 2)) == "246" - end + assert Enum.member?(1..9//2, 1) + assert Enum.member?(1..9//2, 9) + refute Enum.member?(1..9//2, 10) + refute Enum.member?(1..10//2, 10) + assert Enum.member?(1..2//2, 1) + refute Enum.member?(1..2//2, 2) - test :map do - range = 1..3 - assert Enum.map(range, fn x -> x * 2 end) == [2, 4, 6] + assert Enum.member?(-1..-9//-2, -1) + assert Enum.member?(-1..-9//-2, -9) + refute Enum.member?(-1..-9//-2, -8) - range = -1..-3 - assert Enum.map(range, fn x -> x * 2 end) == [-2, -4, -6] + refute Enum.member?(1..0//1, 1) + refute Enum.member?(0..1//-1, 1) end - test :map_reduce do - range = 1..0 - assert Enum.map_reduce(range, 1, fn(x, acc) -> {x * 2, x + acc} end) == {[2, 0], 2} + test "min/1" do + assert Enum.min(1..1) == 1 + assert Enum.min(1..3) == 1 - range = 1..3 - assert Enum.map_reduce(range, 1, fn(x, acc) -> {x * 2, x + acc} end) == {[2, 4, 6], 7} + assert Enum.min(1..9//2) == 1 + assert Enum.min(1..10//2) == 1 + assert Enum.min(-1..-9//-2) == -9 + + assert_raise Enum.EmptyError, fn -> Enum.min(1..0//1) end end - test :max do - assert Enum.max(1..1) == 1 - assert Enum.max(1..3) == 3 - assert Enum.max(3..1) == 3 + test "min/2 with empty fallback" do + assert Enum.min(.., fn -> 0 end) === 0 + assert Enum.min(1..2, fn -> 0 end) === 1 end - test :max_by do - assert Enum.max_by(1..1, fn(x) -> :math.pow(-2, x) end) == 1 - assert Enum.max_by(1..3, fn(x) -> :math.pow(-2, x) end) == 2 + test "min_by/2" do + assert Enum.min_by(1..1, fn x -> :math.pow(-2, x) end) == 1 + assert Enum.min_by(1..3, fn x -> :math.pow(-2, x) end) == 3 + + assert Enum.min_by(1..8//3, fn x -> :math.pow(-2, x) end) == 7 + assert_raise Enum.EmptyError, fn -> Enum.min_by(1..0//1, & &1) end end - test :min do - assert Enum.min([1]) == 1 - assert Enum.min([1, 2, 3]) == 1 - assert Enum.min([[], :a, {}]) == :a + test "min_max/1" do + assert Enum.min_max(1..1) == {1, 1} + assert Enum.min_max(1..3) == {1, 3} + assert Enum.min_max(3..1//-1) == {1, 3} + + assert Enum.min_max(1..9//2) == {1, 9} + assert Enum.min_max(1..10//2) == {1, 9} + assert Enum.min_max(-1..-9//-2) == {-9, -1} + + assert_raise Enum.EmptyError, fn -> Enum.min_max(1..0//1) end end - test :min_by do - assert Enum.min_by(1..1, fn(x) -> :math.pow(-2, x) end) == 1 - assert Enum.min_by(1..3, fn(x) -> :math.pow(-2, x) end) == 3 + test "min_max_by/2" do + assert Enum.min_max_by(1..1, fn x -> x end) == {1, 1} + assert Enum.min_max_by(1..3, fn x -> x end) == {1, 3} + + assert Enum.min_max_by(1..8//3, fn x -> :math.pow(-2, x) end) == {7, 4} + assert_raise Enum.EmptyError, fn -> Enum.min_max_by(1..0//1, & &1) end end - test :partition do - range = 1..3 - assert Enum.partition(range, fn(x) -> rem(x, 2) == 0 end) == {[2], [1, 3]} + test "split_with/2" do + assert Enum.split_with(1..3, fn x -> rem(x, 2) == 0 end) == {[2], [1, 3]} end - test :reduce do - range = 1..0 - assert Enum.reduce(range, 1, fn(x, acc) -> x + acc end) == 2 + test "random/1" do + # corner cases, independent of the seed + assert Enum.random(1..1) == 1 + + # set a fixed seed so the test can be deterministic + # please note the order of following assertions is important + seed1 = {1406, 407_414, 139_258} + seed2 = {1306, 421_106, 567_597} + :rand.seed(:exsss, seed1) + assert Enum.random(1..2) == 1 + assert Enum.random(1..3) == 1 + assert Enum.random(3..1//-1) == 2 + + :rand.seed(:exsss, seed2) + assert Enum.random(1..2) == 1 + assert Enum.random(1..3) == 2 - range = 1..3 - assert Enum.reduce(range, 1, fn(x, acc) -> x + acc end) == 7 + assert Enum.random(1..10//2) == 7 + assert Enum.random(1..10//2) == 5 - range = 1..3 - assert Enum.reduce(range, fn(x, acc) -> x + acc end) == 6 + assert_raise Enum.EmptyError, fn -> Enum.random(..) end end - test :reject do - range = 1..3 - assert Enum.reject(range, fn(x) -> rem(x, 2) == 0 end) == [1, 3] + test "reduce/2" do + assert Enum.reduce(1..3, fn x, acc -> x + acc end) == 6 + assert Enum.reduce(1..10//2, fn x, acc -> x + acc end) == 25 + assert_raise Enum.EmptyError, fn -> Enum.reduce(0..1//-1, &+/2) end + end - range = 1..6 - assert Enum.reject(range, fn(x) -> rem(x, 2) == 0 end) == [1, 3, 5] + test "reduce/3" do + assert Enum.reduce(1..0//-1, 1, fn x, acc -> x + acc end) == 2 + assert Enum.reduce(1..3, 1, fn x, acc -> x + acc end) == 7 + assert Enum.reduce(1..10//2, 1, fn x, acc -> x + acc end) == 26 + assert Enum.reduce(0..1//-1, 1, fn x, acc -> x + acc end) == 1 end - test :reverse do - assert Enum.reverse([]) == [] - assert Enum.reverse([1, 2, 3]) == [3, 2, 1] - assert Enum.reverse([1, 2, 3], [4, 5, 6]) == [3, 2, 1, 4, 5, 6] + test "reduce_while/3" do + assert Enum.reduce_while(1..100, 0, fn i, acc -> + if i <= 3, do: {:cont, acc + i}, else: {:halt, acc} + end) == 6 + end + + test "reject/2" do + assert Enum.reject(1..3, fn x -> rem(x, 2) == 0 end) == [1, 3] + assert Enum.reject(1..6, fn x -> rem(x, 2) == 0 end) == [1, 3, 5] + end + test "reverse/1" do assert Enum.reverse(0..0) == [0] assert Enum.reverse(1..3) == [3, 2, 1] + assert Enum.reverse(-3..5) == [5, 4, 3, 2, 1, 0, -1, -2, -3] + assert Enum.reverse(5..5) == [5] + + assert Enum.reverse(0..1//-1) == [] + assert Enum.reverse(1..10//2) == [9, 7, 5, 3, 1] + end + + test "reverse/2" do assert Enum.reverse(1..3, 4..6) == [3, 2, 1, 4, 5, 6] assert Enum.reverse([1, 2, 3], 4..6) == [3, 2, 1, 4, 5, 6] assert Enum.reverse(1..3, [4, 5, 6]) == [3, 2, 1, 4, 5, 6] + assert Enum.reverse(-3..5, MapSet.new([-3, -2])) == [5, 4, 3, 2, 1, 0, -1, -2, -3, -3, -2] + assert Enum.reverse(5..5, [5]) == [5, 5] + end + + test "reverse_slice/3" do + assert Enum.reverse_slice(1..6, 2, 0) == [1, 2, 3, 4, 5, 6] + assert Enum.reverse_slice(1..6, 2, 2) == [1, 2, 4, 3, 5, 6] + assert Enum.reverse_slice(1..6, 2, 4) == [1, 2, 6, 5, 4, 3] + assert Enum.reverse_slice(1..6, 2, 10_000_000) == [1, 2, 6, 5, 4, 3] + assert Enum.reverse_slice(1..6, 10_000_000, 4) == [1, 2, 3, 4, 5, 6] + assert Enum.reverse_slice(1..6, 50, 50) == [1, 2, 3, 4, 5, 6] + end + + test "scan/2" do + assert Enum.scan(1..5, &(&1 + &2)) == [1, 3, 6, 10, 15] end - test :scan do - assert Enum.scan(1..5, &(&1 + &2)) == [1,3,6,10,15] - assert Enum.scan(1..5, 0, &(&1 + &2)) == [1,3,6,10,15] + test "scan/3" do + assert Enum.scan(1..5, 0, &(&1 + &2)) == [1, 3, 6, 10, 15] end - test :shuffle do + test "shuffle/1" do # set a fixed seed so the test can be deterministic - :random.seed(1374, 347975, 449264) - assert Enum.shuffle(1..5) == [2, 4, 1, 5, 3] + :rand.seed(:exsss, {1374, 347_975, 449_264}) + assert Enum.shuffle(1..5) == [2, 5, 4, 3, 1] + assert Enum.shuffle(1..10//2) == [5, 1, 7, 9, 3] end - test :slice do + test "slice/2" do + assert Enum.slice(1..5, 0..0) == [1] + assert Enum.slice(1..5, 0..1) == [1, 2] + assert Enum.slice(1..5, 0..2) == [1, 2, 3] + assert Enum.slice(1..5, 1..2) == [2, 3] + assert Enum.slice(1..5, 1..0//1) == [] + assert Enum.slice(1..5, 2..5) == [3, 4, 5] + assert Enum.slice(1..5, 2..6) == [3, 4, 5] + assert Enum.slice(1..5, 4..4) == [5] + assert Enum.slice(1..5, 5..5) == [] + assert Enum.slice(1..5, 6..5//1) == [] + assert Enum.slice(1..5, 6..0//1) == [] + assert Enum.slice(1..5, -3..0) == [] + assert Enum.slice(1..5, -3..1) == [] + assert Enum.slice(1..5, -6..0) == [1] + assert Enum.slice(1..5, -6..5) == [1, 2, 3, 4, 5] + assert Enum.slice(1..5, -6..-1) == [1, 2, 3, 4, 5] + assert Enum.slice(1..5, -5..-1) == [1, 2, 3, 4, 5] + assert Enum.slice(1..5, -5..-3) == [1, 2, 3] + + assert Enum.slice(1..5, 0..10//2) == [1, 3, 5] + assert Enum.slice(1..5, 0..10//3) == [1, 4] + assert Enum.slice(1..5, 0..10//4) == [1, 5] + assert Enum.slice(1..5, 0..10//5) == [1] + assert Enum.slice(1..5, 0..10//6) == [1] + + assert Enum.slice(1..5, 0..2//2) == [1, 3] + assert Enum.slice(1..5, 0..2//3) == [1] + + assert Enum.slice(1..5, 0..-1//2) == [1, 3, 5] + assert Enum.slice(1..5, 0..-1//3) == [1, 4] + assert Enum.slice(1..5, 0..-1//4) == [1, 5] + assert Enum.slice(1..5, 0..-1//5) == [1] + assert Enum.slice(1..5, 0..-1//6) == [1] + + assert Enum.slice(1..5, 1..-1//2) == [2, 4] + assert Enum.slice(1..5, 1..-1//3) == [2, 5] + assert Enum.slice(1..5, 1..-1//4) == [2] + assert Enum.slice(1..5, 1..-1//5) == [2] + + assert Enum.slice(1..5, -4..-1//2) == [2, 4] + assert Enum.slice(1..5, -4..-1//3) == [2, 5] + assert Enum.slice(1..5, -4..-1//4) == [2] + assert Enum.slice(1..5, -4..-1//5) == [2] + + assert Enum.slice(5..1//-1, 0..0) == [5] + assert Enum.slice(5..1//-1, 0..1) == [5, 4] + assert Enum.slice(5..1//-1, 0..2) == [5, 4, 3] + assert Enum.slice(5..1//-1, 1..2) == [4, 3] + assert Enum.slice(5..1//-1, 1..0//1) == [] + assert Enum.slice(5..1//-1, 2..5) == [3, 2, 1] + assert Enum.slice(5..1//-1, 2..6) == [3, 2, 1] + assert Enum.slice(5..1//-1, 4..4) == [1] + assert Enum.slice(5..1//-1, 5..5) == [] + assert Enum.slice(5..1//-1, 6..5//1) == [] + assert Enum.slice(5..1//-1, 6..0//1) == [] + assert Enum.slice(5..1//-1, -6..0) == [5] + assert Enum.slice(5..1//-1, -6..5) == [5, 4, 3, 2, 1] + assert Enum.slice(5..1//-1, -6..-1) == [5, 4, 3, 2, 1] + assert Enum.slice(5..1//-1, -5..-1) == [5, 4, 3, 2, 1] + assert Enum.slice(5..1//-1, -5..-3) == [5, 4, 3] + + assert Enum.slice(1..10//2, 0..0) == [1] + assert Enum.slice(1..10//2, 0..1) == [1, 3] + assert Enum.slice(1..10//2, 0..2) == [1, 3, 5] + assert Enum.slice(1..10//2, 1..2) == [3, 5] + assert Enum.slice(1..10//2, 1..0//1) == [] + assert Enum.slice(1..10//2, 2..5) == [5, 7, 9] + assert Enum.slice(1..10//2, 2..6) == [5, 7, 9] + assert Enum.slice(1..10//2, 4..4) == [9] + assert Enum.slice(1..10//2, 5..5) == [] + assert Enum.slice(1..10//2, 6..5//1) == [] + assert Enum.slice(1..10//2, 6..0//1) == [] + assert Enum.slice(1..10//2, -3..0) == [] + assert Enum.slice(1..10//2, -3..1) == [] + assert Enum.slice(1..10//2, -6..0) == [1] + assert Enum.slice(1..10//2, -6..5) == [1, 3, 5, 7, 9] + assert Enum.slice(1..10//2, -6..-1) == [1, 3, 5, 7, 9] + assert Enum.slice(1..10//2, -5..-1) == [1, 3, 5, 7, 9] + assert Enum.slice(1..10//2, -5..-3) == [1, 3, 5] + + assert_raise ArgumentError, + "Enum.slice/2 does not accept ranges with negative steps, got: 1..3//-2", + fn -> Enum.slice(1..5, 1..3//-2) end + end + + test "slice/3" do assert Enum.slice(1..5, 0, 0) == [] assert Enum.slice(1..5, 0, 1) == [1] assert Enum.slice(1..5, 0, 2) == [1, 2] assert Enum.slice(1..5, 1, 2) == [2, 3] assert Enum.slice(1..5, 1, 0) == [] - assert Enum.slice(1..5, 2, 5) == [3, 4, 5] + assert Enum.slice(1..5, 2, 3) == [3, 4, 5] assert Enum.slice(1..5, 2, 6) == [3, 4, 5] assert Enum.slice(1..5, 5, 5) == [] assert Enum.slice(1..5, 6, 5) == [] assert Enum.slice(1..5, 6, 0) == [] assert Enum.slice(1..5, -6, 0) == [] - assert Enum.slice(1..5, -6, 5) == [] + assert Enum.slice(1..5, -6, 5) == [1, 2, 3, 4, 5] assert Enum.slice(1..5, -2, 5) == [4, 5] assert Enum.slice(1..5, -3, 1) == [3] - end - test :slice_range do - assert Enum.slice(1..5, 0..0) == [1] - assert Enum.slice(1..5, 0..1) == [1, 2] - assert Enum.slice(1..5, 0..2) == [1, 2, 3] - assert Enum.slice(1..5, 1..2) == [2, 3] - assert Enum.slice(1..5, 1..0) == [] - assert Enum.slice(1..5, 2..5) == [3, 4, 5] - assert Enum.slice(1..5, 2..6) == [3, 4, 5] - assert Enum.slice(1..5, 4..4) == [5] - assert Enum.slice(1..5, 5..5) == [] - assert Enum.slice(1..5, 6..5) == [] - assert Enum.slice(1..5, 6..0) == [] - assert Enum.slice(1..5, -6..0) == [] - assert Enum.slice(1..5, -6..5) == [] - assert Enum.slice(1..5, -5..-1) == [1, 2, 3, 4, 5] - assert Enum.slice(1..5, -5..-3) == [1, 2, 3] - assert Enum.slice(1..5, -6..-1) == [] - assert Enum.slice(1..5, -6..-3) == [] - end + assert_raise FunctionClauseError, fn -> + Enum.slice(1..5, 0, -1) + end + + assert_raise FunctionClauseError, fn -> + Enum.slice(1..5, 0.99, 0) + end + + assert_raise FunctionClauseError, fn -> + Enum.slice(1..5, 0, 0.99) + end - test :sort do - assert Enum.sort(3..1) == [1, 2, 3] - assert Enum.sort(2..1) == [1, 2] + assert Enum.slice(5..1//-1, 0, 0) == [] + assert Enum.slice(5..1//-1, 0, 1) == [5] + assert Enum.slice(5..1//-1, 0, 2) == [5, 4] + assert Enum.slice(5..1//-1, 1, 2) == [4, 3] + assert Enum.slice(5..1//-1, 1, 0) == [] + assert Enum.slice(5..1//-1, 2, 3) == [3, 2, 1] + assert Enum.slice(5..1//-1, 2, 6) == [3, 2, 1] + assert Enum.slice(5..1//-1, 4, 4) == [1] + assert Enum.slice(5..1//-1, 5, 5) == [] + assert Enum.slice(5..1//-1, 6, 5) == [] + assert Enum.slice(5..1//-1, 6, 0) == [] + assert Enum.slice(5..1//-1, -6, 0) == [] + assert Enum.slice(5..1//-1, -6, 5) == [5, 4, 3, 2, 1] + + assert Enum.slice(1..10//2, 0, 0) == [] + assert Enum.slice(1..10//2, 0, 1) == [1] + assert Enum.slice(1..10//2, 0, 2) == [1, 3] + assert Enum.slice(1..10//2, 1, 2) == [3, 5] + assert Enum.slice(1..10//2, 1, 0) == [] + assert Enum.slice(1..10//2, 2, 3) == [5, 7, 9] + assert Enum.slice(1..10//2, 2, 6) == [5, 7, 9] + assert Enum.slice(1..10//2, 5, 5) == [] + assert Enum.slice(1..10//2, 6, 5) == [] + assert Enum.slice(1..10//2, 6, 0) == [] + assert Enum.slice(1..10//2, -6, 0) == [] + assert Enum.slice(1..10//2, -6, 5) == [1, 3, 5, 7, 9] + assert Enum.slice(1..10//2, -2, 5) == [7, 9] + assert Enum.slice(1..10//2, -3, 1) == [5] + end + + test "sort/1" do + assert Enum.sort(3..1//-1) == [1, 2, 3] + assert Enum.sort(2..1//-1) == [1, 2] assert Enum.sort(1..1) == [1] + end - assert Enum.sort(3..1, &(&1 > &2)) == [3, 2, 1] - assert Enum.sort(2..1, &(&1 > &2)) == [2, 1] + test "sort/2" do + assert Enum.sort(3..1//-1, &(&1 > &2)) == [3, 2, 1] + assert Enum.sort(2..1//-1, &(&1 > &2)) == [2, 1] assert Enum.sort(1..1, &(&1 > &2)) == [1] - end - test :split do - range = 1..3 - assert Enum.split(range, 0) == {[], [1, 2, 3]} - assert Enum.split(range, 1) == {[1], [2, 3]} - assert Enum.split(range, 2) == {[1, 2], [3]} - assert Enum.split(range, 3) == {[1, 2, 3], []} - assert Enum.split(range, 4) == {[1, 2, 3], []} - assert Enum.split(range, -1) == {[1, 2], [3]} - assert Enum.split(range, -2) == {[1], [2, 3]} - assert Enum.split(range, -3) == {[], [1, 2, 3]} - assert Enum.split(range, -10) == {[], [1, 2, 3]} + assert Enum.sort(3..1//-1, :asc) == [1, 2, 3] + assert Enum.sort(3..1//-1, :desc) == [3, 2, 1] + end - range = 1..0 - assert Enum.split(range, 3) == {[1, 0], []} + test "sort_by/2" do + assert Enum.sort_by(3..1//-1, & &1) == [1, 2, 3] + assert Enum.sort_by(3..1//-1, & &1, :asc) == [1, 2, 3] + assert Enum.sort_by(3..1//-1, & &1, :desc) == [3, 2, 1] end - test :split_while do - range = 1..3 - assert Enum.split_while(range, fn(_) -> false end) == {[], [1, 2, 3]} - assert Enum.split_while(range, fn(_) -> true end) == {[1, 2, 3], []} - assert Enum.split_while(range, fn(x) -> x > 2 end) == {[], [1, 2, 3]} - assert Enum.split_while(range, fn(x) -> x > 3 end) == {[], [1, 2, 3]} - assert Enum.split_while(range, fn(x) -> x < 3 end) == {[1, 2], [3]} + test "split/2" do + assert Enum.split(1..3, 0) == {[], [1, 2, 3]} + assert Enum.split(1..3, 1) == {[1], [2, 3]} + assert Enum.split(1..3, 2) == {[1, 2], [3]} + assert Enum.split(1..3, 3) == {[1, 2, 3], []} + assert Enum.split(1..3, 4) == {[1, 2, 3], []} + assert Enum.split(1..3, -1) == {[1, 2], [3]} + assert Enum.split(1..3, -2) == {[1], [2, 3]} + assert Enum.split(1..3, -3) == {[], [1, 2, 3]} + assert Enum.split(1..3, -10) == {[], [1, 2, 3]} + assert Enum.split(1..0//-1, 3) == {[1, 0], []} + end - range = 1..0 - assert Enum.split_while(range, fn(_) -> true end) == {[1, 0], []} + test "split_while/2" do + assert Enum.split_while(1..3, fn _ -> false end) == {[], [1, 2, 3]} + assert Enum.split_while(1..3, fn _ -> nil end) == {[], [1, 2, 3]} + assert Enum.split_while(1..3, fn _ -> true end) == {[1, 2, 3], []} + assert Enum.split_while(1..3, fn x -> x > 2 end) == {[], [1, 2, 3]} + assert Enum.split_while(1..3, fn x -> x > 3 end) == {[], [1, 2, 3]} + assert Enum.split_while(1..3, fn x -> x < 3 end) == {[1, 2], [3]} + assert Enum.split_while(1..3, fn x -> x end) == {[1, 2, 3], []} + assert Enum.split_while(1..0//-1, fn _ -> true end) == {[1, 0], []} end - test :sum do + test "sum/1" do + assert Enum.sum(0..0) == 0 assert Enum.sum(1..1) == 1 assert Enum.sum(1..3) == 6 + assert Enum.sum(0..100) == 5050 + assert Enum.sum(10..100) == 5005 + assert Enum.sum(100..10//-1) == 5005 + assert Enum.sum(-10..-20//-1) == -165 + assert Enum.sum(-10..2) == -52 + + assert Enum.sum(0..1//-1) == 0 + assert Enum.sum(1..9//2) == 25 + assert Enum.sum(1..10//2) == 25 + assert Enum.sum(9..1//-2) == 25 + end + + test "take/2" do + assert Enum.take(1..3, 0) == [] + assert Enum.take(1..3, 1) == [1] + assert Enum.take(1..3, 2) == [1, 2] + assert Enum.take(1..3, 3) == [1, 2, 3] + assert Enum.take(1..3, 4) == [1, 2, 3] + assert Enum.take(1..3, -1) == [3] + assert Enum.take(1..3, -2) == [2, 3] + assert Enum.take(1..3, -4) == [1, 2, 3] + assert Enum.take(1..0//-1, 3) == [1, 0] + assert Enum.take(1..0//1, -3) == [] + end + + test "take_every/2" do + assert Enum.take_every(1..10, 2) == [1, 3, 5, 7, 9] + assert Enum.take_every(1..2, 2) == [1] + assert Enum.take_every(1..3, 0) == [] + + assert_raise FunctionClauseError, fn -> + Enum.take_every(1..3, -1) + end end - test :take do - range = 1..3 - assert Enum.take(range, 0) == [] - assert Enum.take(range, 1) == [1] - assert Enum.take(range, 2) == [1, 2] - assert Enum.take(range, 3) == [1, 2, 3] - assert Enum.take(range, 4) == [1, 2, 3] - assert Enum.take(range, -1) == [3] - assert Enum.take(range, -2) == [2, 3] - assert Enum.take(range, -4) == [1, 2, 3] + test "take_random/2" do + # corner cases, independent of the seed + assert_raise FunctionClauseError, fn -> Enum.take_random(1..2, -1) end + assert Enum.take_random(1..1, 0) == [] + assert Enum.take_random(1..1, 1) == [1] + assert Enum.take_random(1..1, 2) == [1] + assert Enum.take_random(1..2, 0) == [] - range = 1..0 - assert Enum.take(range, 3) == [1, 0] + # set a fixed seed so the test can be deterministic + # please note the order of following assertions is important + seed1 = {1406, 407_414, 139_258} + seed2 = {1406, 421_106, 567_597} + :rand.seed(:exsss, seed1) + assert Enum.take_random(1..3, 1) == [2] + :rand.seed(:exsss, seed1) + assert Enum.take_random(1..3, 2) == [3, 1] + :rand.seed(:exsss, seed1) + assert Enum.take_random(1..3, 3) == [3, 1, 2] + :rand.seed(:exsss, seed1) + assert Enum.take_random(1..3, 4) == [3, 1, 2] + :rand.seed(:exsss, seed1) + assert Enum.take_random(1..3, 5) == [3, 1, 2] + :rand.seed(:exsss, seed1) + assert Enum.take_random(3..1//-1, 1) == [2] + :rand.seed(:exsss, seed2) + assert Enum.take_random(1..3, 1) == [1] + :rand.seed(:exsss, seed2) + assert Enum.take_random(1..3, 2) == [3, 2] + :rand.seed(:exsss, seed2) + assert Enum.take_random(1..3, 3) == [1, 3, 2] + :rand.seed(:exsss, seed2) + assert Enum.take_random(1..3, 4) == [1, 3, 2] + :rand.seed(:exsss, seed2) + assert Enum.take_random(1..3, 5) == [1, 3, 2] + end + + test "take_while/2" do + assert Enum.take_while(1..3, fn x -> x > 3 end) == [] + assert Enum.take_while(1..3, fn x -> x <= 1 end) == [1] + assert Enum.take_while(1..3, fn x -> x <= 3 end) == [1, 2, 3] + assert Enum.take_while(1..3, fn x -> x end) == [1, 2, 3] + assert Enum.take_while(1..3, fn _ -> nil end) == [] + end + + test "to_list/1" do + assert Enum.to_list(1..3) == [1, 2, 3] + assert Enum.to_list(1..3//2) == [1, 3] + assert Enum.to_list(3..1//-2) == [3, 1] + assert Enum.to_list(0..1//-1) == [] + end + + test "uniq/1" do + assert Enum.uniq(1..3) == [1, 2, 3] end - test :take_every do - assert Enum.take_every(1..10, 2) == [1, 3, 5, 7, 9] - assert Enum.take_every(1..2, 2) == [1] - assert Enum.take_every(1..3, 0) == [] + test "uniq_by/2" do + assert Enum.uniq_by(1..3, fn x -> x end) == [1, 2, 3] end - test :take_while do - range = 1..3 - assert Enum.take_while(range, fn(x) -> x > 3 end) == [] - assert Enum.take_while(range, fn(x) -> x <= 1 end) == [1] - assert Enum.take_while(range, fn(x) -> x <= 3 end) == [1, 2, 3] - assert Enum.take_while([], fn(_) -> true end) == [] + test "unzip/1" do + assert_raise FunctionClauseError, fn -> Enum.unzip(1..3) end end - test :uniq do - assert Enum.uniq(1..3) == [1, 2, 3] - assert Enum.uniq(1..3, fn x -> x end) == [1, 2, 3] + test "with_index/2" do + assert Enum.with_index(1..3) == [{1, 0}, {2, 1}, {3, 2}] + assert Enum.with_index(1..3, 3) == [{1, 3}, {2, 4}, {3, 5}] end - test :zip do + test "zip/2" do assert Enum.zip([:a, :b], 1..2) == [{:a, 1}, {:b, 2}] assert Enum.zip([:a, :b], 1..4) == [{:a, 1}, {:b, 2}] assert Enum.zip([:a, :b, :c, :d], 1..2) == [{:a, 1}, {:b, 2}] @@ -880,10 +2588,168 @@ defmodule EnumTest.Range do assert Enum.zip(1..2, 1..2) == [{1, 1}, {2, 2}] assert Enum.zip(1..4, 1..2) == [{1, 1}, {2, 2}] assert Enum.zip(1..2, 1..4) == [{1, 1}, {2, 2}] + + assert Enum.zip(1..10//2, 1..10//3) == [{1, 1}, {3, 4}, {5, 7}, {7, 10}] + assert Enum.zip(0..1//-1, 1..10//3) == [] + end +end + +defmodule EnumTest.Map do + # Maps use different protocols path than lists and ranges in the cases below. + use ExUnit.Case, async: true + + test "random/1" do + map = %{a: 1, b: 2, c: 3} + [x1, x2, x3] = Map.to_list(map) + seed1 = {1406, 407_414, 139_258} + seed2 = {1406, 421_106, 567_597} + :rand.seed(:exsss, seed1) + assert Enum.random(map) == x3 + assert Enum.random(map) == x1 + assert Enum.random(map) == x2 + + :rand.seed(:exsss, seed2) + assert Enum.random(map) == x3 + assert Enum.random(map) == x2 + end + + test "take_random/2" do + # corner cases, independent of the seed + assert_raise FunctionClauseError, fn -> Enum.take_random(1..2, -1) end + assert Enum.take_random(%{a: 1}, 0) == [] + assert Enum.take_random(%{a: 1}, 2) == [a: 1] + assert Enum.take_random(%{a: 1, b: 2}, 0) == [] + + # set a fixed seed so the test can be deterministic + # please note the order of following assertions is important + map = %{a: 1, b: 2, c: 3} + [x1, x2, x3] = Map.to_list(map) + seed1 = {1406, 407_414, 139_258} + seed2 = {1406, 421_106, 567_597} + :rand.seed(:exsss, seed1) + assert Enum.take_random(map, 1) == [x2] + :rand.seed(:exsss, seed1) + assert Enum.take_random(map, 2) == [x3, x1] + :rand.seed(:exsss, seed1) + assert Enum.take_random(map, 3) == [x3, x1, x2] + :rand.seed(:exsss, seed1) + assert Enum.take_random(map, 4) == [x3, x1, x2] + :rand.seed(:exsss, seed2) + assert Enum.take_random(map, 1) == [x1] + :rand.seed(:exsss, seed2) + assert Enum.take_random(map, 2) == [x3, x2] + :rand.seed(:exsss, seed2) + assert Enum.take_random(map, 3) == [x1, x3, x2] + :rand.seed(:exsss, seed2) + assert Enum.take_random(map, 4) == [x1, x3, x2] + end + + test "reverse/1" do + assert Enum.reverse(%{}) == [] + assert Enum.reverse(MapSet.new()) == [] + + map = %{a: 1, b: 2, c: 3} + assert Enum.reverse(map) == Map.to_list(map) |> Enum.reverse() + end + + test "reverse/2" do + assert Enum.reverse([a: 1, b: 2, c: 3, a: 1], %{x: 1}) == [a: 1, c: 3, b: 2, a: 1, x: 1] + + assert Enum.reverse([], %{a: 1}) == [a: 1] + assert Enum.reverse([], %{}) == [] + assert Enum.reverse(%{a: 1}, []) == [a: 1] + assert Enum.reverse(MapSet.new(), %{}) == [] + end + + test "fetch/2" do + map = %{a: 1, b: 2, c: 3, d: 4, e: 5} + [x1, _x2, _x3, x4, x5] = Map.to_list(map) + assert Enum.fetch(map, 0) == {:ok, x1} + assert Enum.fetch(map, -2) == {:ok, x4} + assert Enum.fetch(map, -6) == :error + assert Enum.fetch(map, 5) == :error + assert Enum.fetch(%{}, 0) == :error + + assert Stream.take(map, 3) |> Enum.fetch(3) == :error + assert Stream.take(map, 5) |> Enum.fetch(4) == {:ok, x5} + end + + test "map_intersperse/3" do + assert Enum.map_intersperse(%{}, :a, & &1) == [] + assert Enum.map_intersperse(%{foo: :bar}, :a, & &1) == [{:foo, :bar}] + + map = %{foo: :bar, baz: :bat} + [x1, x2] = Map.to_list(map) + + assert Enum.map_intersperse(map, :a, & &1) == [x1, :a, x2] + end + + test "slice/2" do + map = %{a: 1, b: 2, c: 3, d: 4, e: 5} + [x1, x2, x3 | _] = Map.to_list(map) + assert Enum.slice(map, 0..0) == [x1] + assert Enum.slice(map, 0..1) == [x1, x2] + assert Enum.slice(map, 0..2) == [x1, x2, x3] + end + + test "slice/3" do + map = %{a: 1, b: 2, c: 3, d: 4, e: 5} + [x1, x2, x3, x4, x5] = Map.to_list(map) + assert Enum.slice(map, 1, 2) == [x2, x3] + assert Enum.slice(map, 1, 0) == [] + assert Enum.slice(map, 2, 5) == [x3, x4, x5] + assert Enum.slice(map, 2, 6) == [x3, x4, x5] + assert Enum.slice(map, 5, 5) == [] + assert Enum.slice(map, 6, 5) == [] + assert Enum.slice(map, 6, 0) == [] + assert Enum.slice(map, -6, 0) == [] + assert Enum.slice(map, -6, 5) == [x1, x2, x3, x4, x5] + assert Enum.slice(map, -2, 5) == [x4, x5] + assert Enum.slice(map, -3, 1) == [x3] + + assert_raise FunctionClauseError, fn -> + Enum.slice(map, 0, -1) + end + + assert_raise FunctionClauseError, fn -> + Enum.slice(map, 0.99, 0) + end + + assert_raise FunctionClauseError, fn -> + Enum.slice(map, 0, 0.99) + end + + assert Enum.slice(map, 0, 0) == [] + assert Enum.slice(map, 0, 1) == [x1] + assert Enum.slice(map, 0, 2) == [x1, x2] + assert Enum.slice(map, 1, 2) == [x2, x3] + assert Enum.slice(map, 1, 0) == [] + assert Enum.slice(map, 2, 5) == [x3, x4, x5] + assert Enum.slice(map, 2, 6) == [x3, x4, x5] + assert Enum.slice(map, 5, 5) == [] + assert Enum.slice(map, 6, 5) == [] + assert Enum.slice(map, 6, 0) == [] + assert Enum.slice(map, -6, 0) == [] + assert Enum.slice(map, -6, 5) == [x1, x2, x3, x4, x5] + assert Enum.slice(map, -2, 5) == [x4, x5] + assert Enum.slice(map, -3, 1) == [x3] + + assert_raise FunctionClauseError, fn -> + Enum.slice(map, 0, -1) + end + + assert_raise FunctionClauseError, fn -> + Enum.slice(map, 0.99, 0) + end + + assert_raise FunctionClauseError, fn -> + Enum.slice(map, 0, 0.99) + end end - test :with_index do - assert Enum.with_index(1..3) == [{1,0},{2,1},{3,2}] + test "reduce/3" do + assert Enum.reduce(%{}, 1, fn x, acc -> x + acc end) == 1 + assert Enum.reduce(%{a: 1, b: 2}, 1, fn {_, x}, acc -> x + acc end) == 4 end end @@ -891,17 +2757,22 @@ defmodule EnumTest.SideEffects do use ExUnit.Case, async: true import ExUnit.CaptureIO - import PathHelpers - test "take with side effects" do - stream = Stream.unfold(1, fn x -> IO.puts x; {x, x + 1} end) + test "take/2 with side effects" do + stream = + Stream.unfold(1, fn x -> + IO.puts(x) + {x, x + 1} + end) + assert capture_io(fn -> - Enum.take(stream, 1) - end) == "1\n" + Enum.take(stream, 1) + end) == "1\n" end - test "take does not consume next without a need" do - path = tmp_path("oneliner.txt") + @tag :tmp_dir + test "take/2 does not consume next without a need", config do + path = Path.join(config.tmp_dir, "oneliner.txt") File.mkdir(Path.dirname(path)) try do @@ -917,8 +2788,8 @@ defmodule EnumTest.SideEffects do end end - test "take with no item works as no-op" do - iterator = File.stream!(fixture_path("unknown.txt")) + test "take/2 with no elements works as no-op" do + iterator = File.stream!(PathHelpers.fixture_path("unknown.txt")) assert Enum.take(iterator, 0) == [] assert Enum.take(iterator, 0) == [] @@ -926,3 +2797,13 @@ defmodule EnumTest.SideEffects do assert Enum.take(iterator, 0) == [] end end + +defmodule EnumTest.Function do + use ExUnit.Case, async: true + + test "raises Protocol.UndefinedError for funs of wrong arity" do + assert_raise Protocol.UndefinedError, fn -> + Enum.to_list(fn -> nil end) + end + end +end diff --git a/lib/elixir/test/elixir/exception_test.exs b/lib/elixir/test/elixir/exception_test.exs index d474028aae7..cc8c8040666 100644 --- a/lib/elixir/test/elixir/exception_test.exs +++ b/lib/elixir/test/elixir/exception_test.exs @@ -1,328 +1,1114 @@ -Code.require_file "test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec -defmodule Kernel.ExceptionTest do +Code.require_file("test_helper.exs", __DIR__) + +defmodule ExceptionTest do use ExUnit.Case, async: true - test "raise preserves the stacktrace" do - stacktrace = - try do - raise "a" - rescue _ -> - [top|_] = System.stacktrace - top - end - file = __ENV__.file |> Path.relative_to_cwd |> String.to_char_list - assert {Kernel.ExceptionTest, :"test raise preserves the stacktrace", _, - [file: ^file, line: 9]} = stacktrace + defp capture_err(fun) do + ExUnit.CaptureIO.capture_io(:stderr, fun) end - test "exception?" do - assert Exception.exception?(%RuntimeError{}) - refute Exception.exception?(%Regex{}) - refute Exception.exception?({}) - end + doctest Exception - test "message" do + doctest RuntimeError + doctest SystemLimitError + doctest MismatchedDelimiterError + doctest SyntaxError + doctest TokenMissingError + doctest BadBooleanError + doctest UndefinedFunctionError + doctest FunctionClauseError + doctest Protocol.UndefinedError + doctest UnicodeConversionError + doctest Enum.OutOfBoundsError + doctest Enum.EmptyError + doctest File.Error + doctest File.CopyError + doctest File.RenameError + doctest File.LinkError + doctest ErlangError + + test "message/1" do defmodule BadException do - def message(_) do - raise "oops" + def message(exception) do + if exception.raise do + raise "oops" + end end end - message = ~r/Got RuntimeError with message \"oops\" while retrieving message for/ + assert "got RuntimeError with message \"oops\" while retrieving Exception.message/1 for %{" <> + inspected = + Exception.message(%{__struct__: BadException, __exception__: true, raise: true}) - assert_raise ArgumentError, message, fn -> - Exception.message(%{__struct__: BadException, __exception__: true}) - end - end + assert inspected =~ "raise: true" + assert inspected =~ "__exception__: true" + assert inspected =~ "__struct__: ExceptionTest.BadException" - require Record + assert "got nil while retrieving Exception.message/1 for %{" <> inspected = + Exception.message(%{__struct__: BadException, __exception__: true, raise: false}) - test "normalize" do - assert Exception.normalize(:throw, :badarg) == :badarg - assert Exception.normalize(:exit, :badarg) == :badarg - assert Exception.normalize({:EXIT, self}, :badarg) == :badarg - assert Exception.normalize(:error, :badarg).__struct__ == ArgumentError - assert Exception.normalize(:error, %ArgumentError{}).__struct__ == ArgumentError + assert inspected =~ "raise: false" + assert inspected =~ "__exception__: true" + assert inspected =~ "__struct__: ExceptionTest.BadException" end - test "format_banner" do - assert Exception.format_banner(:error, :badarg) == "** (ArgumentError) argument error" - assert Exception.format_banner(:throw, :badarg) == "** (throw) :badarg" - assert Exception.format_banner(:exit, :badarg) == "** (exit) :badarg" - assert Exception.format_banner({:EXIT, self}, :badarg) == "** (EXIT from #{inspect self}) :badarg" - end + test "normalize/2" do + assert Exception.normalize(:throw, :badarg, []) == :badarg + assert Exception.normalize(:exit, :badarg, []) == :badarg + assert Exception.normalize({:EXIT, self()}, :badarg, []) == :badarg + assert Exception.normalize(:error, :badarg, []).__struct__ == ArgumentError + assert Exception.normalize(:error, %ArgumentError{}, []).__struct__ == ArgumentError - test "format without stacktrace" do - stacktrace = try do throw(:stack) catch :stack -> System.stacktrace() end - assert Exception.format(:error, :badarg) == "** (ArgumentError) argument error" <> - "\n" <> Exception.format_stacktrace(stacktrace) - end + assert %ErlangError{original: :no_translation, reason: ": foo"} = + Exception.normalize(:error, :no_translation, [ + {:io, :put_chars, [self(), <<222>>], + [error_info: %{module: __MODULE__, function: :dummy_error_extras}]} + ]) - test "format with empty stacktrace" do - assert Exception.format(:error, :badarg, []) == "** (ArgumentError) argument error" + assert %ErlangError{original: {:failed_load_cacerts, :enoent}, reason: ": this is chardata"} = + Exception.normalize(:error, {:failed_load_cacerts, :enoent}, [ + {:pubkey_os_cacerts, :get, 0, + [error_info: %{module: __MODULE__, function: :dummy_error_chardata}]} + ]) end - test "format with EXIT has no stacktrace" do - try do throw(:stack) catch :stack -> System.stacktrace() end - assert Exception.format({:EXIT, self}, :badarg) == "** (EXIT from #{inspect self}) :badarg" - end + test "format/2 without stacktrace" do + stacktrace = + try do + throw(:stack) + catch + :stack -> __STACKTRACE__ + end - test "format_exit" do - assert Exception.format_exit(:bye) == ":bye" - assert Exception.format_exit(:noconnection) == "no connection" - assert Exception.format_exit({:nodedown, :"node@host"}) == "no connection to node@host" - assert Exception.format_exit(:timeout) == "time out" - assert Exception.format_exit(:noproc) == "no process" - assert Exception.format_exit(:killed) == "killed" - assert Exception.format_exit(:normal) == "normal" - assert Exception.format_exit(:shutdown) == "shutdown" - assert Exception.format_exit({:shutdown, :bye}) == "shutdown: :bye" - assert Exception.format_exit({:badarg,[{:not_a_real_module, :function, 0, []}]}) == - "an exception was raised:\n ** (ArgumentError) argument error\n :not_a_real_module.function/0" - assert Exception.format_exit({:bad_call, :request}) == "bad call: :request" - assert Exception.format_exit({:bad_cast, :request}) == "bad cast: :request" - assert Exception.format_exit({:start_spec, :unexpected}) == - "bad start spec: :unexpected" - assert Exception.format_exit({:supervisor_data, :unexpected}) == - "bad supervisor data: :unexpected" + assert Exception.format(:error, :badarg, stacktrace) == + "** (ArgumentError) argument error\n" <> Exception.format_stacktrace(stacktrace) end - defmodule Sup do - def start_link(fun), do: :supervisor.start_link(__MODULE__, fun) - - def init(fun), do: fun.() + test "format/2 with empty stacktrace" do + assert Exception.format(:error, :badarg, []) == "** (ArgumentError) argument error" end - test "format_exit with supervisor errors" do - trap = Process.flag(:trap_exit, true) - - {:error, reason} = __MODULE__.Sup.start_link(fn() -> :foo end) - assert Exception.format_exit(reason) == - "#{inspect(__MODULE__.Sup)}.init/1 returned a bad value: :foo" - - return = {:ok, {:foo, []}} - {:error, reason} = __MODULE__.Sup.start_link(fn() -> return end) - assert Exception.format_exit(reason) == - "bad supervisor data: invalid type: :foo" - - return = {:ok, {{:foo, 1, 1}, []}} - {:error, reason} = __MODULE__.Sup.start_link(fn() -> return end) - assert Exception.format_exit(reason) == - "bad supervisor data: invalid strategy: :foo" - - return = {:ok, {{:one_for_one, :foo, 1}, []}} - {:error, reason} = __MODULE__.Sup.start_link(fn() -> return end) - assert Exception.format_exit(reason) == - "bad supervisor data: invalid intensity: :foo" - - return = {:ok, {{:one_for_one, 1, :foo}, []}} - {:error, reason} = __MODULE__.Sup.start_link(fn() -> return end) - assert Exception.format_exit(reason) == - "bad supervisor data: invalid period: :foo" - - return = {:ok, {{:simple_one_for_one, 1, 1}, :foo}} - {:error, reason} = __MODULE__.Sup.start_link(fn() -> return end) - assert Exception.format_exit(reason) == - "bad start spec: invalid children: :foo" - - return = {:ok, {{:one_for_one, 1, 1}, [:foo]}} - {:error, reason} = __MODULE__.Sup.start_link(fn() -> return end) - assert Exception.format_exit(reason) == - "bad start spec: invalid child spec: :foo" - - return = {:ok, {{:one_for_one, 1, 1}, - [{:child, :foo, :temporary, 1, :worker, []}]}} - {:error, reason} = __MODULE__.Sup.start_link(fn() -> return end) - assert Exception.format_exit(reason) == - "bad start spec: invalid mfa: :foo" - - return = {:ok, {{:one_for_one, 1, 1}, - [{:child, {:m, :f, []}, :foo, 1, :worker, []}]}} - {:error, reason} = __MODULE__.Sup.start_link(fn() -> return end) - assert Exception.format_exit(reason) == - "bad start spec: invalid restart type: :foo" - - return = {:ok, {{:one_for_one, 1, 1}, - [{:child, {:m, :f, []}, :temporary, :foo, :worker, []}]}} - {:error, reason} = __MODULE__.Sup.start_link(fn() -> return end) - assert Exception.format_exit(reason) == - "bad start spec: invalid shutdown: :foo" - - return = {:ok, {{:one_for_one, 1, 1}, - [{:child, {:m, :f, []}, :temporary, 1, :foo, []}]}} - {:error, reason} = __MODULE__.Sup.start_link(fn() -> return end) - assert Exception.format_exit(reason) == - "bad start spec: invalid child type: :foo" - - return = {:ok, {{:one_for_one, 1, 1}, - [{:child, {:m, :f, []}, :temporary, 1, :worker, :foo}]}} - {:error, reason} = __MODULE__.Sup.start_link(fn() -> return end) - assert Exception.format_exit(reason) == - "bad start spec: invalid modules: :foo" - - return = {:ok, {{:one_for_one, 1, 1}, - [{:child, {:m, :f, []}, :temporary, 1, :worker, [{:foo}]}]}} - {:error, reason} = __MODULE__.Sup.start_link(fn() -> return end) - assert Exception.format_exit(reason) == - "bad start spec: invalid module: {:foo}" - - return = {:ok, {{:one_for_one, 1, 1}, - [{:child, {Kernel, :exit, [:foo]}, :temporary, 1, :worker, []}]}} - {:error, reason} = __MODULE__.Sup.start_link(fn() -> return end) - assert Exception.format_exit(reason) == - "shutdown: failed to start child: :child\n ** (EXIT) :foo" - - return = {:ok, {{:one_for_one, 1, 1}, - [{:child, {Kernel, :apply, [fn() -> {:error, :foo} end, []]}, :temporary, 1, :worker, []}]}} - {:error, reason} = __MODULE__.Sup.start_link(fn() -> return end) - assert Exception.format_exit(reason) == - "shutdown: failed to start child: :child\n ** (EXIT) :foo" - - Process.flag(:trap_exit, trap) + test "format/2 with EXIT (has no stacktrace)" do + assert Exception.format({:EXIT, self()}, :badarg, []) == + "** (EXIT from #{inspect(self())}) :badarg" end - test "format_exit with call" do - reason = try do - :gen_server.call(:does_not_exist, :hello) - catch - :exit, reason -> reason - end - - assert Exception.format_exit(reason) == - "exited in: :gen_server.call(:does_not_exist, :hello)\n ** (EXIT) no process" - end + test "format_banner/2" do + assert Exception.format_banner(:error, :badarg) == "** (ArgumentError) argument error" + assert Exception.format_banner(:throw, :badarg) == "** (throw) :badarg" + assert Exception.format_banner(:exit, :badarg) == "** (exit) :badarg" - test "format_exit with call with exception" do - # Fake reason to prevent error_logger printing to stdout - fsm_reason = {%ArgumentError{}, [{:not_a_real_module, :function, 0, []}]} - reason = try do - :gen_fsm.sync_send_event(spawn(fn() -> - :timer.sleep(200) ; exit(fsm_reason) - end), :hello) - catch - :exit, reason -> reason - end - - formatted = Exception.format_exit(reason) - assert formatted =~ ~r"exited in: :gen_fsm\.sync_send_event\(#PID<\d+\.\d+\.\d+>, :hello\)" - assert formatted =~ ~r"\s{4}\*\* \(EXIT\) an exception was raised:\n" - assert formatted =~ ~r"\s{8}\*\* \(ArgumentError\) argument error\n" - assert formatted =~ ~r"\s{12}:not_a_real_module\.function/0" + assert Exception.format_banner({:EXIT, self()}, :badarg) == + "** (EXIT from #{inspect(self())}) :badarg" end - test "format_exit with nested calls" do - # Fake reason to prevent error_logger printing to stdout - event_fun = fn() -> :timer.sleep(200) ; exit(:normal) end - server_pid = spawn(fn()-> :gen_event.call(spawn(event_fun), :handler, :hello) end) - reason = try do - :gen_server.call(server_pid, :hi) - catch - :exit, reason -> reason + test "format_stacktrace/1 from file" do + try do + Code.eval_string("def foo do end", [], file: "my_file") + rescue + ArgumentError -> + assert Exception.format_stacktrace(__STACKTRACE__) =~ "my_file:1: (file)" + else + _ -> flunk("expected failure") end - - formatted = Exception.format_exit(reason) - assert formatted =~ ~r"exited in: :gen_server\.call\(#PID<\d+\.\d+\.\d+>, :hi\)\n" - assert formatted =~ ~r"\s{4}\*\* \(EXIT\) exited in: :gen_event\.call\(#PID<\d+\.\d+\.\d+>, :handler, :hello\)\n" - assert formatted =~ ~r"\s{8}\*\* \(EXIT\) normal" end - test "format_exit with nested calls and exception" do - # Fake reason to prevent error_logger printing to stdout - event_reason = {%ArgumentError{}, [{:not_a_real_module, :function, 0, []}]} - event_fun = fn() -> :timer.sleep(200) ; exit(event_reason) end - server_pid = spawn(fn()-> :gen_event.call(spawn(event_fun), :handler, :hello) end) - reason = try do - :gen_server.call(server_pid, :hi) - catch - :exit, reason -> reason - end - - formatted = Exception.format_exit(reason) - assert formatted =~ ~r"exited in: :gen_server\.call\(#PID<\d+\.\d+\.\d+>, :hi\)\n" - assert formatted =~ ~r"\s{4}\*\* \(EXIT\) exited in: :gen_event\.call\(#PID<\d+\.\d+\.\d+>, :handler, :hello\)\n" - assert formatted =~ ~r"\s{8}\*\* \(EXIT\) an exception was raised:\n" - assert formatted =~ ~r"\s{12}\*\* \(ArgumentError\) argument error\n" - assert formatted =~ ~r"\s{16}:not_a_real_module\.function/0" + test "format_stacktrace/1 from module" do + try do + Code.eval_string( + "defmodule FmtStack do raise ArgumentError, ~s(oops) end", + [], + file: "my_file" + ) + rescue + ArgumentError -> + assert Exception.format_stacktrace(__STACKTRACE__) =~ "my_file:1: (module)" + else + _ -> flunk("expected failure") + end end - test "format_stacktrace_entry with no file or line" do + test "format_stacktrace_entry/1 with no file or line" do assert Exception.format_stacktrace_entry({Foo, :bar, [1, 2, 3], []}) == "Foo.bar(1, 2, 3)" assert Exception.format_stacktrace_entry({Foo, :bar, [], []}) == "Foo.bar()" assert Exception.format_stacktrace_entry({Foo, :bar, 1, []}) == "Foo.bar/1" end - test "format_stacktrace_entry with file and line" do - assert Exception.format_stacktrace_entry({Foo, :bar, [], [file: 'file.ex', line: 10]}) == "file.ex:10: Foo.bar()" - assert Exception.format_stacktrace_entry({Foo, :bar, [1, 2, 3], [file: 'file.ex', line: 10]}) == "file.ex:10: Foo.bar(1, 2, 3)" - assert Exception.format_stacktrace_entry({Foo, :bar, 1, [file: 'file.ex', line: 10]}) == "file.ex:10: Foo.bar/1" + test "format_stacktrace_entry/1 with file and line" do + assert Exception.format_stacktrace_entry({Foo, :bar, [], [file: ~c"file.ex", line: 10]}) == + "file.ex:10: Foo.bar()" + + assert Exception.format_stacktrace_entry( + {Foo, :bar, [1, 2, 3], [file: ~c"file.ex", line: 10]} + ) == + "file.ex:10: Foo.bar(1, 2, 3)" + + assert Exception.format_stacktrace_entry({Foo, :bar, 1, [file: ~c"file.ex", line: 10]}) == + "file.ex:10: Foo.bar/1" end - test "format_stacktrace_entry with file no line" do - assert Exception.format_stacktrace_entry({Foo, :bar, [], [file: 'file.ex']}) == "file.ex: Foo.bar()" - assert Exception.format_stacktrace_entry({Foo, :bar, [], [file: 'file.ex', line: 0]}) == "file.ex: Foo.bar()" - assert Exception.format_stacktrace_entry({Foo, :bar, [1, 2, 3], [file: 'file.ex']}) == "file.ex: Foo.bar(1, 2, 3)" - assert Exception.format_stacktrace_entry({Foo, :bar, 1, [file: 'file.ex']}) == "file.ex: Foo.bar/1" + test "format_stacktrace_entry/1 with file no line" do + assert Exception.format_stacktrace_entry({Foo, :bar, [], [file: ~c"file.ex"]}) == + "file.ex: Foo.bar()" + + assert Exception.format_stacktrace_entry({Foo, :bar, [], [file: ~c"file.ex", line: 0]}) == + "file.ex: Foo.bar()" + + assert Exception.format_stacktrace_entry({Foo, :bar, [1, 2, 3], [file: ~c"file.ex"]}) == + "file.ex: Foo.bar(1, 2, 3)" + + assert Exception.format_stacktrace_entry({Foo, :bar, 1, [file: ~c"file.ex"]}) == + "file.ex: Foo.bar/1" end - test "format_stacktrace_entry with application" do - assert Exception.format_stacktrace_entry({Exception, :bar, [], [file: 'file.ex']}) == - "(elixir) file.ex: Exception.bar()" - assert Exception.format_stacktrace_entry({Exception, :bar, [], [file: 'file.ex', line: 10]}) == - "(elixir) file.ex:10: Exception.bar()" - assert Exception.format_stacktrace_entry({:lists, :bar, [1, 2, 3], []}) == - "(stdlib) :lists.bar(1, 2, 3)" + test "format_stacktrace_entry/1 with application" do + assert Exception.format_stacktrace_entry({Exception, :bar, [], [file: ~c"file.ex"]}) == + "(elixir #{System.version()}) file.ex: Exception.bar()" + + assert Exception.format_stacktrace_entry({Exception, :bar, [], [file: ~c"file.ex", line: 10]}) == + "(elixir #{System.version()}) file.ex:10: Exception.bar()" end - test "format_stacktrace_entry with fun" do - assert Exception.format_stacktrace_entry({fn(x) -> x end, [1], []}) =~ ~r/#Function<.+>\(1\)/ - assert Exception.format_stacktrace_entry({fn(x, y) -> {x, y} end, 2, []}) =~ ~r"#Function<.+>/2" + test "format_stacktrace_entry/1 with fun" do + assert Exception.format_stacktrace_entry({fn x -> x end, [1], []}) =~ ~r/#Function<.+>\(1\)/ + + assert Exception.format_stacktrace_entry({fn x, y -> {x, y} end, 2, []}) =~ + ~r"#Function<.+>/2" end - test "format_mfa" do + test "format_mfa/3" do + # Let's create this atom so that String.to_existing_atom/1 inside + # format_mfa/3 doesn't raise. + _ = :"some function" + assert Exception.format_mfa(Foo, nil, 1) == "Foo.nil/1" assert Exception.format_mfa(Foo, :bar, 1) == "Foo.bar/1" assert Exception.format_mfa(Foo, :bar, []) == "Foo.bar()" assert Exception.format_mfa(nil, :bar, []) == "nil.bar()" assert Exception.format_mfa(:foo, :bar, [1, 2]) == ":foo.bar(1, 2)" + assert Exception.format_mfa(Foo, :b@r, 1) == "Foo.\"b@r\"/1" assert Exception.format_mfa(Foo, :"bar baz", 1) == "Foo.\"bar baz\"/1" assert Exception.format_mfa(Foo, :"-func/2-fun-0-", 4) == "anonymous fn/4 in Foo.func/2" + + assert Exception.format_mfa(Foo, :"-some function/2-fun-0-", 4) == + "anonymous fn/4 in Foo.\"some function\"/2" + + assert Exception.format_mfa(Foo, :"42", 1) == "Foo.\"42\"/1" + assert Exception.format_mfa(Foo, :Bar, [1, 2]) == "Foo.\"Bar\"(1, 2)" + assert Exception.format_mfa(Foo, :%{}, [1, 2]) == "Foo.\"%{}\"(1, 2)" + assert Exception.format_mfa(Foo, :..., 1) == "Foo.\"...\"/1" end - test "format_fa" do - assert Exception.format_fa(fn -> end, 1) =~ - ~r"#Function<\d\.\d+/0 in Kernel\.ExceptionTest\.test format_fa/1>/1" + test "format_mfa/3 with Unicode" do + assert Exception.format_mfa(Foo, :olá, [1, 2]) == "Foo.olá(1, 2)" + assert Exception.format_mfa(Foo, :Olá, [1, 2]) == "Foo.\"Olá\"(1, 2)" + assert Exception.format_mfa(Foo, :Ólá, [1, 2]) == "Foo.\"Ólá\"(1, 2)" + assert Exception.format_mfa(Foo, :こんにちは世界, [1, 2]) == "Foo.こんにちは世界(1, 2)" + + nfd = :unicode.characters_to_nfd_binary("olá") + assert Exception.format_mfa(Foo, String.to_atom(nfd), [1, 2]) == "Foo.\"#{nfd}\"(1, 2)" end - import Exception, only: [message: 1] + test "format_fa/2" do + assert Exception.format_fa(fn -> nil end, 1) =~ + ~r"#Function<\d+\.\d+/0 in ExceptionTest\.\"test format_fa/2\"/1>/1" + end + + describe "format_exit/1" do + test "with atom/tuples" do + assert Exception.format_exit(:bye) == ":bye" + assert Exception.format_exit(:noconnection) == "no connection" + assert Exception.format_exit({:nodedown, :node@host}) == "no connection to node@host" + assert Exception.format_exit(:timeout) == "time out" + assert Exception.format_exit(:noproc) |> String.starts_with?("no process:") + assert Exception.format_exit(:killed) == "killed" + assert Exception.format_exit(:normal) == "normal" + assert Exception.format_exit(:shutdown) == "shutdown" + assert Exception.format_exit(:calling_self) == "process attempted to call itself" + assert Exception.format_exit({:shutdown, :bye}) == "shutdown: :bye" + + assert Exception.format_exit({:badarg, [{:not_a_real_module, :function, 0, []}]}) == + "an exception was raised:\n ** (ArgumentError) argument error\n :not_a_real_module.function/0" + + assert Exception.format_exit({:bad_call, :request}) == "bad call: :request" + assert Exception.format_exit({:bad_cast, :request}) == "bad cast: :request" + + assert Exception.format_exit({:start_spec, :unexpected}) == + "bad child specification, got: :unexpected" + + assert Exception.format_exit({:supervisor_data, :unexpected}) == + "bad supervisor configuration, got: :unexpected" + end + + defmodule Sup do + def start_link(fun), do: :supervisor.start_link(__MODULE__, fun) + + def init(fun), do: fun.() + end + + test "with supervisor errors" do + Process.flag(:trap_exit, true) + + {:error, reason} = __MODULE__.Sup.start_link(fn -> :foo end) + + assert Exception.format_exit(reason) == + "#{inspect(__MODULE__.Sup)}.init/1 returned a bad value: :foo" + + return = {:ok, {:foo, []}} + {:error, reason} = __MODULE__.Sup.start_link(fn -> return end) + assert Exception.format_exit(reason) == "bad supervisor configuration, invalid type: :foo" + + return = {:ok, {{:foo, 1, 1}, []}} + {:error, reason} = __MODULE__.Sup.start_link(fn -> return end) + + assert Exception.format_exit(reason) == + "bad supervisor configuration, invalid strategy: :foo" + + return = {:ok, {{:one_for_one, :foo, 1}, []}} + {:error, reason} = __MODULE__.Sup.start_link(fn -> return end) + + assert Exception.format_exit(reason) == + "bad supervisor configuration, invalid max_restarts (intensity): :foo" + + return = {:ok, {{:one_for_one, 1, :foo}, []}} + {:error, reason} = __MODULE__.Sup.start_link(fn -> return end) + + assert Exception.format_exit(reason) == + "bad supervisor configuration, invalid max_seconds (period): :foo" + + return = {:ok, {{:simple_one_for_one, 1, 1}, :foo}} + {:error, reason} = __MODULE__.Sup.start_link(fn -> return end) + assert Exception.format_exit(reason) == "bad child specification, invalid children: :foo" + + return = {:ok, {{:one_for_one, 1, 1}, [:foo]}} + {:error, reason} = __MODULE__.Sup.start_link(fn -> return end) + + assert Exception.format_exit(reason) == + "bad child specification, invalid child specification: :foo" + + return = {:ok, {{:one_for_one, 1, 1}, [{:child, :foo, :temporary, 1, :worker, []}]}} + {:error, reason} = __MODULE__.Sup.start_link(fn -> return end) + assert Exception.format_exit(reason) == "bad child specification, invalid mfa: :foo" + + return = {:ok, {{:one_for_one, 1, 1}, [{:child, {:m, :f, []}, :foo, 1, :worker, []}]}} + {:error, reason} = __MODULE__.Sup.start_link(fn -> return end) + + assert Exception.format_exit(reason) =~ + "bad child specification, invalid restart type: :foo" + + return = { + :ok, + {{:one_for_one, 1, 1}, [{:child, {:m, :f, []}, :temporary, :foo, :worker, []}]} + } + + {:error, reason} = __MODULE__.Sup.start_link(fn -> return end) + assert Exception.format_exit(reason) =~ "bad child specification, invalid shutdown: :foo" + + return = {:ok, {{:one_for_one, 1, 1}, [{:child, {:m, :f, []}, :temporary, 1, :foo, []}]}} + {:error, reason} = __MODULE__.Sup.start_link(fn -> return end) + assert Exception.format_exit(reason) =~ "bad child specification, invalid child type: :foo" + + return = + {:ok, {{:one_for_one, 1, 1}, [{:child, {:m, :f, []}, :temporary, 1, :worker, :foo}]}} + + {:error, reason} = __MODULE__.Sup.start_link(fn -> return end) + assert Exception.format_exit(reason) =~ "bad child specification, invalid modules: :foo" + + return = { + :ok, + {{:one_for_one, 1, 1}, [{:child, {:m, :f, []}, :temporary, 1, :worker, [{:foo}]}]} + } + + {:error, reason} = __MODULE__.Sup.start_link(fn -> return end) + assert Exception.format_exit(reason) =~ "bad child specification, invalid module: {:foo}" + + return = { + :ok, + { + {:one_for_one, 1, 1}, + [ + {:child, {:m, :f, []}, :permanent, 1, :worker, []}, + {:child, {:m, :f, []}, :permanent, 1, :worker, []} + ] + } + } + + {:error, reason} = __MODULE__.Sup.start_link(fn -> return end) + + assert Exception.format_exit(reason) =~ + "bad child specification, more than one child specification has the id: :child" + + return = { + :ok, + {{:one_for_one, 1, 1}, [{:child, {Kernel, :exit, [:foo]}, :temporary, 1, :worker, []}]} + } + + {:error, reason} = __MODULE__.Sup.start_link(fn -> return end) + + assert Exception.format_exit(reason) == + "shutdown: failed to start child: :child\n ** (EXIT) :foo" + + return = { + :ok, + { + {:one_for_one, 1, 1}, + [{:child, {Kernel, :apply, [fn -> {:error, :foo} end, []]}, :temporary, 1, :worker, []}] + } + } - test "runtime error message" do - assert %RuntimeError{} |> message == "runtime error" - assert %RuntimeError{message: "exception"} |> message == "exception" + {:error, reason} = __MODULE__.Sup.start_link(fn -> return end) + + assert Exception.format_exit(reason) == + "shutdown: failed to start child: :child\n ** (EXIT) :foo" + end + + test "with call" do + reason = + try do + :gen_server.call(:does_not_exist, :hello) + catch + :exit, reason -> reason + end + + expected_to_start_with = + "exited in: :gen_server.call(:does_not_exist, :hello)\n ** (EXIT) no process:" + + assert Exception.format_exit(reason) |> String.starts_with?(expected_to_start_with) + end + + test "with nested calls" do + Process.flag(:trap_exit, true) + # Fake reason to prevent error_logger printing to stdout + exit_fun = fn -> receive do: (_ -> exit(:normal)) end + + outer_pid = + spawn_link(fn -> + Process.flag(:trap_exit, true) + + receive do + _ -> + :gen_event.call(spawn_link(exit_fun), :handler, :hello) + end + end) + + reason = + try do + :gen_server.call(outer_pid, :hi) + catch + :exit, reason -> reason + end + + formatted = Exception.format_exit(reason) + assert formatted =~ ~r"exited in: :gen_server\.call\(#PID<\d+\.\d+\.\d+>, :hi\)\n" + + assert formatted =~ + ~r"\s{4}\*\* \(EXIT\) exited in: :gen_event\.call\(#PID<\d+\.\d+\.\d+>, :handler, :hello\)\n" + + assert formatted =~ ~r"\s{8}\*\* \(EXIT\) normal" + end + + test "format_exit/1 with nested calls and exception" do + Process.flag(:trap_exit, true) + # Fake reason to prevent error_logger printing to stdout + exit_reason = {%ArgumentError{}, [{:not_a_real_module, :function, 0, []}]} + exit_fun = fn -> receive do: (_ -> exit(exit_reason)) end + + outer_pid = + spawn_link(fn -> + Process.flag(:trap_exit, true) + :gen_event.call(spawn_link(exit_fun), :handler, :hello) + end) + + reason = + try do + :gen_server.call(outer_pid, :hi) + catch + :exit, reason -> reason + end + + formatted = Exception.format_exit(reason) + assert formatted =~ ~r"exited in: :gen_server\.call\(#PID<\d+\.\d+\.\d+>, :hi\)\n" + + assert formatted =~ + ~r"\s{4}\*\* \(EXIT\) exited in: :gen_event\.call\(#PID<\d+\.\d+\.\d+>, :handler, :hello\)\n" + + assert formatted =~ ~r"\s{8}\*\* \(EXIT\) an exception was raised:\n" + assert formatted =~ ~r"\s{12}\*\* \(ArgumentError\) argument error\n" + assert formatted =~ ~r"\s{16}:not_a_real_module\.function/0" + end end - test "argument error message" do - assert %ArgumentError{} |> message == "argument error" - assert %ArgumentError{message: "exception"} |> message == "exception" + describe "blaming" do + test "does not annotate throws/exits" do + stack = [{Keyword, :pop, [%{}, :key, nil], [line: 13]}] + assert Exception.blame(:throw, :function_clause, stack) == {:function_clause, stack} + assert Exception.blame(:exit, :function_clause, stack) == {:function_clause, stack} + end + + test "handles operators precedence" do + import PathHelpers + + write_beam( + defmodule OperatorPrecedence do + def test!(x, y) when x in [1, 2, 3] and y >= 4, do: :ok + end + ) + + :code.purge(OperatorPrecedence) + :code.delete(OperatorPrecedence) + + assert blame_message(OperatorPrecedence, & &1.test!(1, 2)) =~ """ + no function clause matching in ExceptionTest.OperatorPrecedence.test!/2 + + The following arguments were given to ExceptionTest.OperatorPrecedence.test!/2: + + # 1 + 1 + + # 2 + 2 + + Attempted function clauses (showing 1 out of 1): + + def test!(x, y) when (x === 1 or -x === 2- or -x === 3-) and -y >= 4- + """ + end + + test "reverts is_struct macro on guards for blaming" do + import PathHelpers + + write_beam( + defmodule Req do + def get!(url) + when is_binary(url) or (is_struct(url) and is_struct(url, URI) and false) do + url + end + + def get!(url, url_module) + when is_binary(url) or (is_struct(url) and is_struct(url, url_module) and false) do + url + end + + def sub_get!(url) when is_struct(url.sub, URI), do: url.sub + end + ) + + :code.purge(Req) + :code.delete(Req) + + assert blame_message(Req, & &1.get!(url: "https://elixir-lang.org")) =~ """ + no function clause matching in ExceptionTest.Req.get!/1 + + The following arguments were given to ExceptionTest.Req.get!/1: + + # 1 + [url: "https://elixir-lang.org"] + + Attempted function clauses (showing 1 out of 1): + + def get!(url) when -is_binary(url)- or -is_struct(url)- and -is_struct(url, URI)- and -false- + """ + + elixir_uri = %URI{} = URI.parse("https://elixir-lang.org") + + assert blame_message(Req, & &1.get!(elixir_uri, URI)) =~ """ + no function clause matching in ExceptionTest.Req.get!/2 + + The following arguments were given to ExceptionTest.Req.get!/2: + + # 1 + %URI{scheme: \"https\", authority: \"elixir-lang.org\", userinfo: nil, host: \"elixir-lang.org\", port: 443, path: nil, query: nil, fragment: nil} + + # 2 + URI + + Attempted function clauses (showing 1 out of 1): + + def get!(url, url_module) when -is_binary(url)- or is_struct(url) and is_struct(url, url_module) and -false- + """ + + assert blame_message(Req, & &1.get!(elixir_uri)) =~ """ + no function clause matching in ExceptionTest.Req.get!/1 + + The following arguments were given to ExceptionTest.Req.get!/1: + + # 1 + %URI{scheme: \"https\", authority: \"elixir-lang.org\", userinfo: nil, host: \"elixir-lang.org\", port: 443, path: nil, query: nil, fragment: nil} + + Attempted function clauses (showing 1 out of 1): + + def get!(url) when -is_binary(url)- or is_struct(url) and is_struct(url, URI) and -false- + """ + + assert blame_message(Req, & &1.sub_get!(%{})) =~ """ + no function clause matching in ExceptionTest.Req.sub_get!/1 + + The following arguments were given to ExceptionTest.Req.sub_get!/1: + + # 1 + %{} + + Attempted function clauses (showing 1 out of 1): + + def sub_get!(url) when -is_struct(url.sub, URI)- + """ + end + + test "annotates badarg on apply" do + assert blame_message([], & &1.foo()) == + "you attempted to apply a function named :foo on []. If you are using Kernel.apply/3, make sure " <> + "the module is an atom. If you are using the dot syntax, such as " <> + "module.function(), make sure the left-hand side of the dot is an atom representing a module" + + assert blame_message([], &apply(&1, :foo, [])) == + "you attempted to apply a function named :foo on []. If you are using Kernel.apply/3, make sure " <> + "the module is an atom. If you are using the dot syntax, such as " <> + "module.function(), make sure the left-hand side of the dot is an atom representing a module" + + assert blame_message([], &apply(&1, :foo, [1, 2])) == + "you attempted to apply a function on []. Modules (the first argument of apply) must always be an atom" + end + + test "annotates function clause errors" do + import PathHelpers + + write_beam( + defmodule ExampleModule do + def fun(arg1, arg2) + def fun(:one, :one), do: :ok + def fun(:two, :two), do: :ok + end + ) + + message = blame_message(ExceptionTest.ExampleModule, & &1.fun(:three, :four)) + + assert message =~ """ + no function clause matching in ExceptionTest.ExampleModule.fun/2 + + The following arguments were given to ExceptionTest.ExampleModule.fun/2: + + # 1 + :three + + # 2 + :four + + Attempted function clauses (showing 2 out of 2): + + def fun(-:one-, -:one-) + def fun(-:two-, -:two-) + """ + end + + test "annotates undefined function error with suggestions" do + assert blame_message(Enum, & &1.map(:ok)) == """ + function Enum.map/1 is undefined or private. Did you mean: + + * map/2 + """ + + assert blame_message(Enum, & &1.man(:ok)) == """ + function Enum.man/1 is undefined or private. Did you mean: + + * map/2 + * max/1 + * max/2 + * max/3 + * min/1 + """ + + message = blame_message(:erlang, & &1.gt_cookie()) + assert message =~ "function :erlang.gt_cookie/0 is undefined or private. Did you mean:" + assert message =~ "* get_cookie/0" + assert message =~ "* set_cookie/2" + end + + test "annotates undefined function error with module suggestions" do + import PathHelpers + + modules = [ + Namespace.A.One, + Namespace.A.Two, + Namespace.A.Three, + Namespace.B.One, + Namespace.B.Two, + Namespace.B.Three + ] + + for module <- modules do + write_beam( + defmodule module do + def foo, do: :bar + end + ) + end + + assert blame_message(ENUM, & &1.map(&1, 1)) == """ + function ENUM.map/2 is undefined (module ENUM is not available). Did you mean: + + * Enum.map/2 + """ + + assert blame_message(ENUM, & &1.not_a_function(&1, 1)) == + "function ENUM.not_a_function/2 is undefined (module ENUM is not available). " <> + "Make sure the module name is correct and has been specified in full (or that an alias has been defined)" + + assert blame_message(One, & &1.foo()) == """ + function One.foo/0 is undefined (module One is not available). Did you mean: + + * Namespace.A.One.foo/0 + * Namespace.B.One.foo/0 + """ + + for module <- modules do + :code.purge(module) + :code.delete(module) + end + end + + test "annotates undefined function clause error with macro hints" do + assert blame_message(Integer, & &1.is_odd(1)) == + "function Integer.is_odd/1 is undefined or private. However, there is " <> + "a macro with the same name and arity. Be sure to require Integer if " <> + "you intend to invoke this macro" + end + + test "annotates undefined function clause error with callback hints" do + capture_err(fn -> + Code.eval_string(""" + defmodule Behaviour do + @callback callback() :: :ok + end + + defmodule Implementation do + @behaviour Behaviour + end + """) + end) + + assert blame_message(Implementation, & &1.callback()) == + "function Implementation.callback/0 is undefined or private" <> + ", but the behaviour Behaviour expects it to be present" + end + + test "does not annotate undefined function clause error with callback hints when callback is optional" do + defmodule BehaviourWithOptional do + @callback callback() :: :ok + @callback optional() :: :ok + @optional_callbacks callback: 0, optional: 0 + end + + defmodule ImplementationWithOptional do + @behaviour BehaviourWithOptional + def callback(), do: :ok + end + + assert blame_message(ImplementationWithOptional, & &1.optional()) == + "function ExceptionTest.ImplementationWithOptional.optional/0 is undefined or private" + end + + test "annotates undefined function clause error with otp obsolete hints" do + assert blame_message(:erlang, & &1.hash(1, 2)) == + "function :erlang.hash/2 is undefined or private, use erlang:phash2/2 instead" + end + + test "annotates undefined function clause error with nil hints" do + assert blame_message(nil, & &1.foo()) == + "function nil.foo/0 is undefined. If you are using the dot syntax, " <> + "such as module.function(), make sure the left-hand side of " <> + "the dot is a module atom" + + assert blame_message("nil.foo()", &Code.eval_string/1) == + "function nil.foo/0 is undefined. If you are using the dot syntax, " <> + "such as module.function(), make sure the left-hand side of " <> + "the dot is a module atom" + end + + test "annotates key error with suggestions if keys are atoms" do + message = blame_message(%{first: nil, second: nil}, fn map -> map.firts end) + + assert message == + """ + key :firts not found in: + + %{first: nil, second: nil} + + Did you mean: + + * :first + """ + + message = blame_message(%{"first" => nil, "second" => nil}, fn map -> map.firts end) + + assert message == """ + key :firts not found in: + + %{"first" => nil, "second" => nil} + """ + + message = + blame_message(%{"first" => nil, "second" => nil}, fn map -> Map.fetch!(map, "firts") end) + + assert message == + """ + key "firts" not found in: + + %{"first" => nil, "second" => nil} + """ + + message = + blame_message( + [ + created_at: nil, + updated_at: nil, + deleted_at: nil, + started_at: nil, + finished_at: nil + ], + fn kwlist -> + Keyword.fetch!(kwlist, :inserted_at) + end + ) + + assert message == """ + key :inserted_at not found in: + + [ + created_at: nil, + updated_at: nil, + deleted_at: nil, + started_at: nil, + finished_at: nil + ] + + Did you mean: + + * :created_at + * :finished_at + * :started_at + """ + end + + test "annotates key error with suggestions for structs" do + message = blame_message(%URI{}, fn map -> map.schema end) + assert message =~ "key :schema not found in:\n\n %URI{" + assert message =~ "Did you mean:" + assert message =~ "* :scheme" + end + + test "annotates +/1 arithmetic errors" do + assert blame_message(:foo, &(+&1)) == "bad argument in arithmetic expression: +(:foo)" + end + + test "annotates -/1 arithmetic errors" do + assert blame_message(:foo, &(-&1)) == "bad argument in arithmetic expression: -(:foo)" + end + + test "annotates div arithmetic errors" do + assert blame_message(0, &div(10, &1)) == + "bad argument in arithmetic expression: div(10, 0)" + end + + test "annotates rem arithmetic errors" do + assert blame_message(0, &rem(10, &1)) == + "bad argument in arithmetic expression: rem(10, 0)" + end + + test "annotates band arithmetic errors" do + import Bitwise + + assert blame_message(:foo, &band(&1, 10)) == + "bad argument in arithmetic expression: Bitwise.band(:foo, 10)" + + assert blame_message(:foo, &(&1 &&& 10)) == + "bad argument in arithmetic expression: Bitwise.band(:foo, 10)" + end + + test "annotates bor arithmetic errors" do + import Bitwise + + assert blame_message(:foo, &bor(&1, 10)) == + "bad argument in arithmetic expression: Bitwise.bor(:foo, 10)" + + assert blame_message(:foo, &(&1 ||| 10)) == + "bad argument in arithmetic expression: Bitwise.bor(:foo, 10)" + end + + test "annotates bxor arithmetic errors" do + import Bitwise + + assert blame_message(:foo, &bxor(&1, 10)) == + "bad argument in arithmetic expression: Bitwise.bxor(:foo, 10)" + end + + test "annotates bsl arithmetic errors" do + import Bitwise + + assert blame_message(:foo, &bsl(10, &1)) == + "bad argument in arithmetic expression: Bitwise.bsl(10, :foo)" + + assert blame_message(:foo, &(10 <<< &1)) == + "bad argument in arithmetic expression: Bitwise.bsl(10, :foo)" + end + + test "annotates bsr arithmetic errors" do + import Bitwise + + assert blame_message(:foo, &bsr(10, &1)) == + "bad argument in arithmetic expression: Bitwise.bsr(10, :foo)" + + assert blame_message(:foo, &(10 >>> &1)) == + "bad argument in arithmetic expression: Bitwise.bsr(10, :foo)" + end + + test "annotates bnot arithmetic errors" do + import Bitwise + + assert blame_message(:foo, &bnot(&1)) == + "bad argument in arithmetic expression: Bitwise.bnot(:foo)" + end + + defp blame_message(arg, fun) do + try do + fun.(arg) + rescue + e -> + Exception.blame(:error, e, __STACKTRACE__) |> elem(0) |> Exception.message() + end + end + end + + describe "blaming unit tests" do + test "annotates clauses errors" do + import PathHelpers + + write_beam( + defmodule BlameModule do + def fun(arg), do: arg + end + ) + + args = [nil] + + {exception, stack} = + Exception.blame(:error, :function_clause, [{BlameModule, :fun, args, [line: 13]}]) + + assert %FunctionClauseError{kind: :def, args: ^args, clauses: [_]} = exception + assert stack == [{BlameModule, :fun, 1, [line: 13]}] + end + + @tag :require_ast + test "annotates args and clauses from mfa" do + import PathHelpers + + write_beam( + defmodule Blaming do + def with_elem(x, y) when elem(x, 1) == 0 and elem(x, y) == 1 do + {x, y} + end + + def fetch(%module{} = container, key), do: {module, container, key} + def fetch(map, key) when is_map(map), do: {map, key} + def fetch(list, key) when is_list(list) and is_atom(key), do: {list, key} + def fetch(nil, _key), do: nil + + require Integer + def even_and_odd(foo, bar) when Integer.is_even(foo) and Integer.is_odd(bar), do: :ok + end + ) + + :code.purge(Blaming) + :code.delete(Blaming) + + {:ok, :def, clauses} = Exception.blame_mfa(Blaming, :with_elem, [1, 2]) + + assert annotated_clauses_to_string(clauses) == [ + "{[+x+, +y+], [-elem(x, 1) == 0- and -elem(x, y) == 1-]}" + ] + + {:ok, :def, clauses} = Exception.blame_mfa(Blaming, :fetch, [self(), "oops"]) + + assert annotated_clauses_to_string(clauses) == [ + "{[-%module{} = container-, +key+], []}", + "{[+map+, +key+], [-is_map(map)-]}", + "{[+list+, +key+], [-is_list(list)- and -is_atom(key)-]}", + "{[-nil-, +_key+], []}" + ] + + {:ok, :def, clauses} = Exception.blame_mfa(Blaming, :even_and_odd, [1, 1]) + + assert annotated_clauses_to_string(clauses) == [ + "{[+foo+, +bar+], [+is_integer(foo)+ and -Bitwise.band(foo, 1) == 0- and +is_integer(bar)+ and +Bitwise.band(bar, 1) == 1+]}" + ] + + {:ok, :defmacro, clauses} = Exception.blame_mfa(Kernel, :!, [true]) + + assert annotated_clauses_to_string(clauses) == [ + "{[-{:!, _, [value]}-], []}", + "{[+value+], []}" + ] + end + + defp annotated_clauses_to_string(clauses) do + Enum.map(clauses, fn {args, clauses} -> + args = Enum.map_join(args, ", ", &arg_to_string/1) + clauses = Enum.map_join(clauses, ", ", &clause_to_string/1) + "{[#{args}], [#{clauses}]}" + end) + end + + defp arg_to_string(%{match?: true, node: node}), do: "+" <> Macro.to_string(node) <> "+" + defp arg_to_string(%{match?: false, node: node}), do: "-" <> Macro.to_string(node) <> "-" + + defp clause_to_string({op, _, [left, right]}), + do: clause_to_string(left) <> " #{op} " <> clause_to_string(right) + + defp clause_to_string(other), + do: arg_to_string(other) + end + + describe "exception messages" do + import Exception, only: [message: 1] + + test "RuntimeError" do + assert %RuntimeError{} |> message() == "runtime error" + assert %RuntimeError{message: "unexpected roquefort"} |> message() == "unexpected roquefort" + end + + test "ArithmeticError" do + assert %ArithmeticError{} |> message() == "bad argument in arithmetic expression" + + assert %ArithmeticError{message: "unexpected camembert"} + |> message() == "unexpected camembert" + end + + test "ArgumentError" do + assert %ArgumentError{} |> message() == "argument error" + assert %ArgumentError{message: "unexpected comté"} |> message() == "unexpected comté" + end + + test "KeyError" do + assert %KeyError{} |> message() == "key nil not found" + assert %KeyError{message: "key missed"} |> message() == "key missed" + end + + test "Enum.OutOfBoundsError" do + assert %Enum.OutOfBoundsError{} |> message() == "out of bounds error" + + assert %Enum.OutOfBoundsError{message: "the brie is not on the table"} + |> message() == "the brie is not on the table" + end + + test "Enum.EmptyError" do + assert %Enum.EmptyError{} |> message() == "empty error" + + assert %Enum.EmptyError{message: "there is no saint-nectaire left!"} + |> message() == "there is no saint-nectaire left!" + end + + test "UndefinedFunctionError" do + assert %UndefinedFunctionError{} |> message() == "undefined function" + + assert %UndefinedFunctionError{module: Kernel, function: :bar, arity: 1} + |> message() == "function Kernel.bar/1 is undefined or private" + + assert %UndefinedFunctionError{module: Foo, function: :bar, arity: 1} + |> message() == + "function Foo.bar/1 is undefined (module Foo is not available). " <> + "Make sure the module name is correct and has been specified in full (or that an alias has been defined)" + + assert %UndefinedFunctionError{module: nil, function: :bar, arity: 3} + |> message() == "function nil.bar/3 is undefined" + + assert %UndefinedFunctionError{module: nil, function: :bar, arity: 0} + |> message() == "function nil.bar/0 is undefined" + end + + test "FunctionClauseError" do + assert %FunctionClauseError{} |> message() == "no function clause matches" + + assert %FunctionClauseError{module: Foo, function: :bar, arity: 1} + |> message() == "no function clause matching in Foo.bar/1" + end + + test "ErlangError" do + assert %ErlangError{original: :sample} |> message() == "Erlang error: :sample" + end + + test "MissingApplicationsError" do + assert %MissingApplicationsError{ + apps: [{:logger, "~> 1.18"}, {:ex_unit, Version.parse_requirement!(">= 0.0.0")}], + description: "applications are required" + } + |> message() == """ + applications are required + + To address this, include these applications as your dependencies: + + {:logger, "~> 1.18"} + {:ex_unit, ">= 0.0.0"}\ + """ + end end - test "undefined function message" do - assert %UndefinedFunctionError{} |> message == "undefined function" - assert %UndefinedFunctionError{module: Foo, function: :bar, arity: 1} |> message == - "undefined function: Foo.bar/1" - assert %UndefinedFunctionError{module: nil, function: :bar, arity: 0} |> message == - "undefined function: nil.bar/0" + describe "error_info" do + test "badarg on erlang" do + assert message(:erlang, & &1.element("foo", "bar")) == """ + errors were found at the given arguments: + + * 1st argument: not an integer + * 2nd argument: not a tuple + """ + end + + test "badarg on ets" do + ets = :ets.new(:foo, []) + :ets.delete(ets) + + assert message(:ets, & &1.insert(ets, 1)) == """ + errors were found at the given arguments: + + * 1st argument: the table identifier does not refer to an existing ETS table + * 2nd argument: not a tuple + """ + end + + test "system_limit on counters" do + assert message(:counters, & &1.new(123_456_789_123_456_789_123_456_789, [])) == """ + a system limit has been reached due to errors at the given arguments: + + * 1st argument: counters array size reached a system limit + """ + end end - test "function clause message" do - assert %FunctionClauseError{} |> message == - "no function clause matches" - assert %FunctionClauseError{module: Foo, function: :bar, arity: 1} |> message == - "no function clause matching in Foo.bar/1" + describe "binary constructor error info" do + defp concat(a, b), do: a <> b + + test "on binary concatenation" do + assert message(123, &concat(&1, "bar")) == + "construction of binary failed: segment 1 of type 'binary': expected a binary but got: 123" + + assert message(~D[0001-02-03], &concat(&1, "bar")) == + "construction of binary failed: segment 1 of type 'binary': expected a binary but got: ~D[0001-02-03]" + end end - test "erlang error message" do - assert %ErlangError{original: :sample} |> message == - "erlang error: :sample" + defp message(arg, fun) do + try do + fun.(arg) + rescue + e -> Exception.message(e) + end + end + + def dummy_error_extras(_exception, _stacktrace), do: %{general: "foo"} + + def dummy_error_chardata(_exception, _stacktrace) do + %{general: ~c"this is " ++ [~c"chardata"], reason: ~c"this " ++ [~c"too"]} end end diff --git a/lib/elixir/test/elixir/file/stream_test.exs b/lib/elixir/test/elixir/file/stream_test.exs new file mode 100644 index 00000000000..cade78f0c85 --- /dev/null +++ b/lib/elixir/test/elixir/file/stream_test.exs @@ -0,0 +1,310 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team + +Code.require_file("../test_helper.exs", __DIR__) + +defmodule File.StreamTest do + use ExUnit.Case + import PathHelpers + + setup do + File.mkdir_p!(tmp_path()) + on_exit(fn -> File.rm_rf(tmp_path()) end) + :ok + end + + defp stream!(node, src, lines_or_bytes_or_modes \\ []) do + :erpc.call(node, File, :stream!, [src, lines_or_bytes_or_modes]) + end + + defp stream!(node, src, modes, lines_or_bytes) do + :erpc.call(node, File, :stream!, [src, modes, lines_or_bytes]) + end + + distributed_node = :"secondary@#{node() |> Atom.to_string() |> :binary.split("@") |> tl()}" + + for {type, node} <- [local: node(), distributed: distributed_node] do + describe "#{type} node" do + @describetag type + @node node + + test "returns a struct" do + src = fixture_path("file.txt") + stream = stream!(@node, src) + assert %File.Stream{} = stream + assert stream.modes == [:raw, :read_ahead, :binary] + assert stream.raw + assert stream.line_or_bytes == :line + + stream = stream!(@node, src, read_ahead: false) + assert %File.Stream{} = stream + assert stream.modes == [:raw, :binary] + assert stream.raw + + stream = stream!(@node, src, read_ahead: 5000) + assert %File.Stream{} = stream + assert stream.modes == [:raw, {:read_ahead, 5000}, :binary] + assert stream.raw + + stream = stream!(@node, src, 10, [:utf8]) + assert %File.Stream{} = stream + assert stream.modes == [{:encoding, :utf8}, :binary] + refute stream.raw + assert stream.line_or_bytes == 10 + end + + test "counts bytes/characters" do + src = fixture_path("file.txt") + stream = stream!(@node, src) + assert Enum.count(stream) == 1 + + stream = stream!(@node, src, [:utf8]) + assert Enum.count(stream) == 1 + + stream = stream!(@node, src, 2) + assert Enum.count(stream) == 2 + end + + test "reads and writes lines" do + src = fixture_path("file.txt") + dest = tmp_path("tmp_test.txt") + + try do + stream = stream!(@node, src) + + File.open(dest, [:write], fn target -> + Enum.each(stream, fn line -> + IO.write(target, String.replace(line, "O", "A")) + end) + end) + + assert File.read(dest) == {:ok, "FAA\n"} + after + File.rm(dest) + end + end + + test "reads and writes bytes" do + src = fixture_path("file.txt") + dest = tmp_path("tmp_test.txt") + + try do + stream = stream!(@node, src, 1) + + File.open(dest, [:write], fn target -> + Enum.each(stream, fn <> -> + IO.write(target, <>) + end) + end) + + assert File.read(dest) == {:ok, "GPP\v"} + after + File.rm(dest) + end + end + + test "is collectable" do + src = fixture_path("file.txt") + dest = tmp_path("tmp_test.txt") + + try do + refute File.exists?(dest) + original = stream!(@node, dest) + + stream = + stream!(@node, src) + |> Stream.map(&String.replace(&1, "O", "A")) + |> Enum.into(original) + + assert stream == original + assert File.read(dest) == {:ok, "FAA\n"} + after + File.rm(dest) + end + end + + test "is collectable with append" do + src = fixture_path("file.txt") + dest = tmp_path("tmp_test.txt") + + try do + refute File.exists?(dest) + original = stream!(@node, dest, [:append]) + + stream!(@node, src, [:append]) + |> Stream.map(&String.replace(&1, "O", "A")) + |> Enum.into(original) + + stream!(@node, src, [:append]) + |> Enum.into(original) + + assert File.read(dest) == {:ok, "FAA\nFOO\n"} + after + File.rm(dest) + end + end + + test "supports byte offset" do + src = fixture_path("file.txt") + + assert @node + |> stream!(src, read_offset: 0) + |> Enum.take(1) == ["FOO\n"] + + assert @node + |> stream!(src, read_offset: 1) + |> Enum.take(1) == ["OO\n"] + + assert @node + |> stream!(src, read_offset: 4) + |> Enum.take(1) == [] + + assert @node |> stream!(src, 1, read_offset: 1) |> Enum.count() == 3 + assert @node |> stream!(src, 1, read_offset: 4) |> Enum.count() == 0 + end + + test "applies offset after trimming BOM" do + src = fixture_path("utf8_bom.txt") + + assert @node + |> stream!(src, [:trim_bom, read_offset: 4]) + |> Enum.take(1) == ["сский\n"] + + assert @node |> stream!(src, 1, [:trim_bom, read_offset: 4]) |> Enum.count() == 15 + end + + test "keeps BOM when raw" do + src = fixture_path("utf8_bom.txt") + + assert @node + |> stream!(src, []) + |> Enum.take(1) == [<<239, 187, 191>> <> "Русский\n"] + + assert @node + |> stream!(src, 1) + |> Enum.take(5) == [<<239>>, <<187>>, <<191>>, <<208>>, <<160>>] + + assert @node |> stream!(src, []) |> Enum.count() == 2 + assert @node |> stream!(src, 1) |> Enum.count() == 22 + end + + test "trims BOM via option when raw" do + src = fixture_path("utf8_bom.txt") + + assert @node + |> stream!(src, [:trim_bom]) + |> Enum.take(1) == ["Русский\n"] + + assert @node + |> stream!(src, 1, [:trim_bom]) + |> Enum.take(5) == [<<208>>, <<160>>, <<209>>, <<131>>, <<209>>] + + assert @node |> stream!(src, [:trim_bom]) |> Enum.count() == 2 + assert @node |> stream!(src, 1, [:trim_bom]) |> Enum.count() == 19 + assert @node |> stream!(src, 2, [:trim_bom]) |> Enum.count() == 10 + end + + test "keeps BOM with utf8 encoding" do + src = fixture_path("utf8_bom.txt") + + assert @node + |> stream!(src, encoding: :utf8) + |> Enum.take(1) == [<<239, 187, 191>> <> "Русский\n"] + + assert @node + |> stream!(src, 1, encoding: :utf8) + |> Enum.take(9) == ["\uFEFF", "Р", "у", "с", "с", "к", "и", "й", "\n"] + end + + test "trims BOM via option with utf8 encoding" do + src = fixture_path("utf8_bom.txt") + + assert @node + |> stream!(src, [{:encoding, :utf8}, :trim_bom]) + |> Enum.take(1) == ["Русский\n"] + + assert @node + |> stream!(src, 1, [{:encoding, :utf8}, :trim_bom]) + |> Enum.take(8) == ["Р", "у", "с", "с", "к", "и", "й", "\n"] + end + + test "keeps BOM with UTF16 BE" do + src = fixture_path("utf16_be_bom.txt") + + assert @node + |> stream!(src, [{:encoding, {:utf16, :big}}]) + |> Enum.take(1) == ["\uFEFFРусский\n"] + end + + test "keeps BOM with UTF16 LE" do + src = fixture_path("utf16_le_bom.txt") + + assert @node + |> stream!(src, [{:encoding, {:utf16, :little}}]) + |> Enum.take(1) == ["\uFEFFРусский\n"] + end + + test "trims BOM via option with utf16 BE encoding" do + src = fixture_path("utf16_be_bom.txt") + + assert @node + |> stream!(src, [{:encoding, {:utf16, :big}}, :trim_bom]) + |> Enum.take(1) == ["Русский\n"] + + assert @node + |> stream!(src, 1, [{:encoding, {:utf16, :big}}, :trim_bom]) + |> Enum.take(8) == ["Р", "у", "с", "с", "к", "и", "й", "\n"] + end + + test "trims BOM via option with utf16 LE encoding" do + src = fixture_path("utf16_le_bom.txt") + + assert @node + |> stream!(src, [{:encoding, {:utf16, :little}}, :trim_bom]) + |> Enum.take(1) == ["Русский\n"] + + assert @node + |> stream!(src, 1, [{:encoding, {:utf16, :little}}, :trim_bom]) + |> Enum.take(8) == ["Р", "у", "с", "с", "к", "и", "й", "\n"] + end + + test "reads and writes line by line in UTF-8" do + src = fixture_path("file.txt") + dest = tmp_path("tmp_test.txt") + + try do + stream = stream!(@node, src) + + File.open(dest, [:write, :utf8], fn target -> + Enum.each(stream, fn line -> + IO.write(target, String.replace(line, "O", "A")) + end) + end) + + assert File.read(dest) == {:ok, "FAA\n"} + after + File.rm(dest) + end + end + + test "reads and writes character in UTF-8" do + src = fixture_path("file.txt") + dest = tmp_path("tmp_test.txt") + + try do + stream = stream!(@node, src, 1, [:utf8]) + + File.open(dest, [:write], fn target -> + Enum.each(stream, fn <> -> + IO.write(target, <>) + end) + end) + + assert File.read(dest) == {:ok, "GPP\v"} + after + File.rm(dest) + end + end + end + end +end diff --git a/lib/elixir/test/elixir/file_test.exs b/lib/elixir/test/elixir/file_test.exs index 3c43287e693..cdfca308578 100644 --- a/lib/elixir/test/elixir/file_test.exs +++ b/lib/elixir/test/elixir/file_test.exs @@ -1,31 +1,371 @@ -Code.require_file "test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec -defmodule Elixir.FileCase do - use ExUnit.CaseTemplate - import PathHelpers +Code.require_file("test_helper.exs", __DIR__) - using do - quote do - import PathHelpers - end - end +defmodule FileTest do + use ExUnit.Case + import PathHelpers setup do - File.mkdir_p!(tmp_path) - on_exit(fn -> File.rm_rf(tmp_path) end) + File.mkdir_p!(tmp_path()) + on_exit(fn -> File.rm_rf(tmp_path()) end) :ok end -end -defmodule FileTest do - use Elixir.FileCase - import Regex, only: [escape: 1] + describe "rename" do + # Following Erlang's underlying implementation + # + # Renaming files + # :ok -> rename file to existing file default behavior + # {:error, :eisdir} -> rename file to existing empty dir + # {:error, :eisdir} -> rename file to existing non-empty dir + # :ok -> rename file to non-existing location + # {:error, :eexist} -> rename file to existing file + # :ok -> rename file to itself + + # Renaming dirs + # {:error, :enotdir} -> rename dir to existing file + # :ok -> rename dir to non-existing leaf location + # {:error, ??} -> rename dir to non-existing parent location + # :ok -> rename dir to itself + # :ok -> rename dir to existing empty dir default behavior + # {:error, :eexist} -> rename dir to existing empty dir + # {:error, :einval} -> rename parent dir to existing sub dir + # {:error, :einval} -> rename parent dir to non-existing sub dir + # {:error, :eexist} -> rename dir to existing non-empty dir + + # other tests + # {:error, :enoent} -> rename unknown source + # :ok -> rename preserves mode + test "rename file to existing file default behavior" do + src = tmp_fixture_path("file.txt") + dest = tmp_path("tmp.file") + + File.write!(dest, "hello") + + try do + assert File.exists?(dest) + assert File.rename(src, dest) == :ok + refute File.exists?(src) + assert File.read!(dest) == "FOO\n" + after + File.rm_rf(src) + File.rm_rf(dest) + end + end + + test "rename file to existing empty dir" do + src = tmp_fixture_path("file.txt") + dest = tmp_path("tmp") + + try do + File.mkdir(dest) + assert File.rename(src, dest) == {:error, :eisdir} + assert File.exists?(src) + refute File.exists?(tmp_path("tmp/file.txt")) + after + File.rm_rf(src) + File.rm_rf(dest) + end + end + + test "rename file to existing non-empty dir" do + src = tmp_fixture_path("file.txt") + dest = tmp_path("tmp") + + try do + File.mkdir_p(Path.join(dest, "a")) + assert File.rename(src, dest) in [{:error, :eisdir}, {:error, :eexist}] + assert File.exists?(src) + refute File.exists?(Path.join(dest, "file.txt")) + after + File.rm_rf(src) + File.rm_rf(dest) + end + end + + test "rename file to non-existing location" do + src = tmp_fixture_path("file.txt") + dest = tmp_path("tmp.file") + + try do + refute File.exists?(dest) + assert File.rename(src, dest) == :ok + assert File.exists?(dest) + refute File.exists?(src) + after + File.rm_rf(src) + File.rm_rf(dest) + end + end + + test "rename file to existing file" do + src = tmp_fixture_path("file.txt") + dest = tmp_path("tmp.file") + + File.write!(dest, "hello") + + try do + assert File.exists?(dest) + assert File.rename(src, dest) == :ok + refute File.exists?(src) + assert File.read!(dest) == "FOO\n" + after + File.rm_rf(src) + File.rm_rf(dest) + end + end + + test "rename file to itself" do + src = tmp_fixture_path("file.txt") + dest = src + + try do + assert File.exists?(src) + assert File.rename(src, dest) == :ok + assert File.exists?(src) + after + File.rm_rf(src) + File.rm_rf(dest) + end + end + + test "rename! file to existing file default behavior" do + src = tmp_fixture_path("file.txt") + dest = tmp_path("tmp.file") + + File.write!(dest, "hello") + + try do + assert File.exists?(dest) + assert File.rename!(src, dest) == :ok + refute File.exists?(src) + assert File.read!(dest) == "FOO\n" + after + File.rm_rf(src) + File.rm_rf(dest) + end + end + + test "rename! with invalid file" do + src = tmp_fixture_path("invalid.txt") + dest = tmp_path("tmp.file") + + message = + "could not rename from #{inspect(src)} to #{inspect(dest)}: no such file or directory" + + assert_raise File.RenameError, message, fn -> + File.rename!(src, dest) + end + end + + test "rename dir to existing file" do + src = tmp_fixture_path("cp_r") + dest = tmp_path("tmp.file") + + try do + File.touch(dest) + assert File.rename(src, dest) == {:error, :enotdir} + after + File.rm_rf(src) + File.rm_rf(dest) + end + end + + test "rename dir to non-existing leaf location" do + src = tmp_fixture_path("cp_r") + dest = tmp_path("tmp") + + try do + refute File.exists?(tmp_path("tmp/a/1.txt")) + refute File.exists?(tmp_path("tmp/a/a/2.txt")) + refute File.exists?(tmp_path("tmp/b/3.txt")) + + assert File.rename(src, dest) == :ok + {:ok, files} = File.ls(dest) + assert length(files) == 2 + assert "a" in files + + {:ok, files} = File.ls(tmp_path("tmp/a")) + assert length(files) == 2 + assert "1.txt" in files + + assert File.exists?(tmp_path("tmp/a/1.txt")) + assert File.exists?(tmp_path("tmp/a/a/2.txt")) + assert File.exists?(tmp_path("tmp/b/3.txt")) + + refute File.exists?(src) + after + File.rm_rf(src) + File.rm_rf(dest) + end + end + + test "rename dir to non-existing parent location" do + src = tmp_fixture_path("cp_r") + dest = tmp_path("tmp/a/b") + + try do + assert File.rename(src, dest) == {:error, :enoent} + assert File.exists?(src) + refute File.exists?(dest) + after + File.rm_rf(src) + File.rm_rf(dest) + end + end + + test "rename dir to itself" do + src = tmp_fixture_path("cp_r") + dest = src + + try do + assert File.exists?(src) + assert File.rename(src, dest) == :ok + assert File.exists?(src) + after + File.rm_rf(src) + File.rm_rf(dest) + end + end + + test "rename parent dir to existing sub dir" do + src = tmp_fixture_path("cp_r") + dest = tmp_path("cp_r/a") + + try do + assert File.exists?(src) + assert File.rename(src, dest) in [{:error, :einval}, {:error, :eexist}] + assert File.exists?(src) + after + File.rm_rf(src) + File.rm_rf(dest) + end + end + + test "rename parent dir to non-existing sub dir" do + src = tmp_fixture_path("cp_r") + dest = tmp_path("cp_r/x") + + try do + assert File.exists?(src) + assert File.rename(src, dest) == {:error, :einval} + assert File.exists?(src) + refute File.exists?(dest) + after + File.rm_rf(src) + File.rm_rf(dest) + end + end + + test "rename dir to existing empty dir default behavior" do + src = tmp_fixture_path("cp_r") + dest = tmp_path("tmp") + + File.mkdir(dest) + + try do + refute File.exists?(tmp_path("tmp/a")) + + assert File.rename(src, dest) == :ok + {:ok, files} = File.ls(dest) + assert length(files) == 2 + assert "a" in files + + {:ok, files} = File.ls(tmp_path("tmp/a")) + assert length(files) == 2 + assert "1.txt" in files + + assert File.exists?(tmp_path("tmp/a/1.txt")) + assert File.exists?(tmp_path("tmp/a/a/2.txt")) + assert File.exists?(tmp_path("tmp/b/3.txt")) + + refute File.exists?(src) + after + File.rm_rf(src) + File.rm_rf(dest) + end + end + + test "rename dir to existing empty dir" do + src = tmp_fixture_path("cp_r") + dest = tmp_path("tmp") + + File.mkdir(dest) + + try do + assert File.exists?(dest) + assert File.rename(src, dest) == :ok + refute File.exists?(src) + assert File.exists?(tmp_path("tmp/a")) + after + File.rm_rf(src) + File.rm_rf(dest) + end + end + + test "rename dir to existing non-empty dir" do + src = tmp_fixture_path("cp_r") + dest = tmp_path("tmp") + + File.mkdir_p(tmp_path("tmp/x")) + + try do + assert File.exists?(tmp_path("tmp/x")) + assert File.exists?(src) + refute File.exists?(tmp_path("tmp/a")) + + assert File.rename(src, dest) == {:error, :eexist} + + assert File.exists?(tmp_path("tmp/x")) + assert File.exists?(src) + refute File.exists?(tmp_path("tmp/a")) + after + File.rm_rf(src) + File.rm_rf(dest) + end + end + + test "rename unknown source" do + src = fixture_path("unknown") + dest = tmp_path("tmp") + + try do + assert File.rename(src, dest) == {:error, :enoent} + after + File.rm_rf(dest) + end + end - defmodule Cp do - use Elixir.FileCase + test "rename preserves mode" do + File.mkdir_p!(tmp_path("tmp")) + src = tmp_fixture_path("cp_mode") + dest = tmp_path("tmp/cp_mode") + + try do + %File.Stat{mode: src_mode} = File.stat!(src) + File.rename(src, dest) + %File.Stat{mode: dest_mode} = File.stat!(dest) + assert src_mode == dest_mode + after + File.rm_rf(src) + File.rm_rf(dest) + end + end + + def tmp_fixture_path(extra) do + src = fixture_path(extra) + dest = tmp_path(extra) + File.cp_r(src, dest) + dest + end + end - test :cp_with_src_file_and_dest_file do - src = fixture_path("file.txt") + describe "cp" do + test "cp with src file and dest file" do + src = fixture_path("file.txt") dest = tmp_path("sample.txt") File.touch(dest) @@ -39,8 +379,8 @@ defmodule FileTest do end end - test :cp_with_src_file_and_dest_dir do - src = fixture_path("file.txt") + test "cp with src file and dest dir" do + src = fixture_path("file.txt") dest = tmp_path("tmp") File.mkdir(dest) @@ -48,32 +388,32 @@ defmodule FileTest do try do assert File.cp(src, dest) == {:error, :eisdir} after - File.rm_rf dest + File.rm_rf(dest) end end - test :cp_with_src_file_and_dest_unknown do - src = fixture_path("file.txt") - dest = tmp_path("tmp.file") + test "cp with src file and dest unknown" do + src = fixture_path("file.txt") + dest = tmp_path("tmp.file") try do refute File.exists?(dest) assert File.cp(src, dest) == :ok assert File.exists?(dest) after - File.rm_rf dest + File.rm_rf(dest) end end - test :cp_with_src_dir do - src = fixture_path("cp_r") - dest = tmp_path("tmp.file") + test "cp with src dir" do + src = fixture_path("cp_r") + dest = tmp_path("tmp.file") assert File.cp(src, dest) == {:error, :eisdir} end - test :cp_with_conflict do - src = fixture_path("file.txt") - dest = tmp_path("tmp.file") + test "cp with conflict" do + src = fixture_path("file.txt") + dest = tmp_path("tmp.file") File.write!(dest, "hello") @@ -82,31 +422,35 @@ defmodule FileTest do assert File.cp(src, dest) == :ok assert File.read!(dest) == "FOO\n" after - File.rm_rf dest + File.rm_rf(dest) end end - test :cp_with_conflict_with_function do - src = fixture_path("file.txt") - dest = tmp_path("tmp.file") + test "cp with conflict with function" do + src = fixture_path("file.txt") + dest = tmp_path("tmp.file") File.write!(dest, "hello") try do assert File.exists?(dest) - assert File.cp(src, dest, fn(src_file, dest_file) -> - assert src_file == src - assert dest_file == dest - false - end) == :ok + + assert File.cp(src, dest, + on_conflict: fn src_file, dest_file -> + assert src_file == src + assert dest_file == dest + false + end + ) == :ok + assert File.read!(dest) == "hello" after - File.rm_rf dest + File.rm_rf(dest) end end - test :cp_with_src_file_and_dest_file! do - src = fixture_path("file.txt") + test "cp! with src file and dest file" do + src = fixture_path("file.txt") dest = tmp_path("sample.txt") File.touch(dest) @@ -120,17 +464,39 @@ defmodule FileTest do end end - test :cp_with_src_dir! do - src = fixture_path("cp_r") - dest = tmp_path("tmp.file") - assert_raise File.CopyError, "could not copy recursively from #{src} to #{dest}: " <> - "illegal operation on a directory", fn -> + test "cp! with src dir" do + src = fixture_path("cp_r") + dest = tmp_path("tmp.file") + + message = + "could not copy from #{inspect(src)} to #{inspect(dest)}: illegal operation on a directory" + + assert_raise File.CopyError, message, fn -> File.cp!(src, dest) end end - test :cp_r_with_src_file_and_dest_file do - src = fixture_path("file.txt") + test "cp itself" do + src = dest = tmp_path("tmp.file") + + File.write!(src, "here") + + try do + assert File.cp(src, dest) == :ok + assert File.read!(dest) == "here" + assert File.cp_r(src, dest) == {:ok, []} + after + File.rm(dest) + end + end + + test "cp_r raises on path with null byte" do + assert_raise ArgumentError, ~r/null byte/, fn -> File.cp_r("source", "foo\0bar") end + assert_raise ArgumentError, ~r/null byte/, fn -> File.cp_r("foo\0bar", "dest") end + end + + test "cp_r with src file and dest file" do + src = fixture_path("file.txt") dest = tmp_path("sample.txt") File.touch(dest) @@ -144,34 +510,34 @@ defmodule FileTest do end end - test :cp_r_with_src_file_and_dest_dir do - src = fixture_path("file.txt") - dest = tmp_path("tmp") + test "cp_r with src file and dest dir" do + src = fixture_path("file.txt") + dest = tmp_path("tmp") File.mkdir(dest) try do - assert io_error? File.cp_r(src, dest) + assert io_error?(File.cp_r(src, dest)) after - File.rm_rf dest + File.rm_rf(dest) end end - test :cp_r_with_src_file_and_dest_unknown do - src = fixture_path("file.txt") - dest = tmp_path("tmp.file") + test "cp_r with src file and dest unknown" do + src = fixture_path("file.txt") + dest = tmp_path("tmp.file") try do refute File.exists?(dest) assert File.cp_r(src, dest) == {:ok, [dest]} assert File.exists?(dest) after - File.rm_rf dest + File.rm_rf(dest) end end - test :cp_r_with_src_dir_and_dest_dir do - src = fixture_path("cp_r") + test "cp_r with src dir and dest dir" do + src = fixture_path("cp_r") dest = tmp_path("tmp") File.mkdir(dest) @@ -190,24 +556,24 @@ defmodule FileTest do assert File.exists?(tmp_path("tmp/a/a/2.txt")) assert File.exists?(tmp_path("tmp/b/3.txt")) after - File.rm_rf dest + File.rm_rf(dest) end end - test :cp_r_with_src_dir_and_dest_file do - src = fixture_path("cp_r") + test "cp_r with src dir and dest file" do + src = fixture_path("cp_r") dest = tmp_path("tmp.file") try do File.touch!(dest) - assert (File.cp_r(src, dest) |> io_error?) + assert File.cp_r(src, dest) |> io_error?() after - File.rm_rf dest + File.rm_rf(dest) end end - test :cp_r_with_src_dir_and_dest_unknown do - src = fixture_path("cp_r") + test "cp_r with src dir and dest unknown" do + src = fixture_path("cp_r") dest = tmp_path("tmp") try do @@ -222,32 +588,112 @@ defmodule FileTest do assert File.exists?(tmp_path("tmp/a/a/2.txt")) assert File.exists?(tmp_path("tmp/b/3.txt")) after - File.rm_rf dest + File.rm_rf(dest) end end - test :cp_r_with_src_unknown do - src = fixture_path("unknown") + test "cp_r with src unknown" do + src = fixture_path("unknown") dest = tmp_path("tmp") assert File.cp_r(src, dest) == {:error, :enoent, src} end - test :cp_r_with_dir_and_file_conflict do - src = fixture_path("cp_r") + test "cp_r with absolute symlink" do + linked_src = fixture_path("cp_r") + src = tmp_path("tmp/src") + dest = tmp_path("tmp/dest") + + File.mkdir_p!(src) + :ok = :file.make_symlink(Path.join(linked_src, "a"), Path.join(src, "sym")) + + try do + {:ok, files} = File.cp_r(src, dest) + assert length(files) == 2 + + assert File.exists?(tmp_path("tmp/dest/sym/1.txt")) + assert File.exists?(tmp_path("tmp/dest/sym/a/2.txt")) + after + File.rm_rf(src) + File.rm_rf(dest) + end + end + + test "cp_r with dereference absolute symlink" do + linked_src = fixture_path("cp_r") + src = tmp_path("tmp/src") + dest = tmp_path("tmp/dest") + + File.mkdir_p!(src) + :ok = :file.make_symlink(Path.join(linked_src, "a"), Path.join(src, "sym")) + + try do + {:ok, files} = File.cp_r(src, dest, dereference_symlinks: true) + assert length(files) == 5 + + assert File.exists?(tmp_path("tmp/dest/sym/1.txt")) + assert File.exists?(tmp_path("tmp/dest/sym/a/2.txt")) + after + File.rm_rf(src) + File.rm_rf(dest) + end + end + + @tag :unix + test "cp_r with relative symlink" do + doc = tmp_path("tmp/doc") + src = tmp_path("tmp/src") + dest = tmp_path("tmp/dest") + + File.mkdir_p!(src) + File.write!(doc, "hello") + :ok = :file.make_symlink("../doc", Path.join(src, "sym")) + + try do + {:ok, files} = File.cp_r(src, dest) + assert length(files) == 2 + assert File.lstat!(tmp_path("tmp/dest/sym")).type == :symlink + after + File.rm_rf(src) + File.rm_rf(dest) + end + end + + @tag :unix + test "cp_r with dereference relative symlink" do + doc = tmp_path("tmp/doc") + src = tmp_path("tmp/src") + dest = tmp_path("tmp/dest") + + File.mkdir_p!(src) + File.write!(doc, "hello") + :ok = :file.make_symlink("../doc", Path.join(src, "sym")) + + try do + {:ok, files} = File.cp_r(src, dest, dereference_symlinks: true) + assert length(files) == 2 + assert File.lstat!(tmp_path("tmp/dest/sym")).type == :regular + after + File.rm_rf(src) + File.rm_rf(dest) + end + end + + test "cp_r with dir and file conflict" do + src = fixture_path("cp_r") dest = tmp_path("tmp") try do File.mkdir(dest) File.write!(Path.join(dest, "a"), "hello") - assert io_error? File.cp_r(src, dest) + assert io_error?(File.cp_r(src, dest)) after - File.rm_rf dest + File.rm_rf(dest) end end - test :cp_r_with_src_dir_and_dest_dir_using_lists do - src = fixture_path("cp_r") |> to_char_list - dest = tmp_path("tmp") |> to_char_list + test "cp_r with src dir and dest dir using lists" do + src = fixture_path("cp_r") |> to_charlist() + dest = tmp_path("tmp") |> to_charlist() File.mkdir(dest) @@ -264,48 +710,52 @@ defmodule FileTest do assert File.exists?(tmp_path("tmp/a/a/2.txt")) assert File.exists?(tmp_path("tmp/b/3.txt")) after - File.rm_rf dest + File.rm_rf(dest) end end - test :cp_r_with_src_with_file_conflict do - src = fixture_path("cp_r") + test "cp_r with src with file conflict" do + src = fixture_path("cp_r") dest = tmp_path("tmp") - File.mkdir_p tmp_path("tmp/a") - File.write! tmp_path("tmp/a/1.txt"), "hello" + File.mkdir_p(tmp_path("tmp/a")) + File.write!(tmp_path("tmp/a/1.txt"), "hello") try do assert File.exists?(tmp_path("tmp/a/1.txt")) File.cp_r(src, dest) assert File.read!(tmp_path("tmp/a/1.txt")) == "" after - File.rm_rf dest + File.rm_rf(dest) end end - test :cp_r_with_src_with_file_conflict_callback do - src = fixture_path("cp_r") + test "cp_r with src with file conflict callback" do + src = fixture_path("cp_r") dest = tmp_path("tmp") - File.mkdir_p tmp_path("tmp/a") - File.write! tmp_path("tmp/a/1.txt"), "hello" + File.mkdir_p(tmp_path("tmp/a")) + File.write!(tmp_path("tmp/a/1.txt"), "hello") try do assert File.exists?(tmp_path("tmp/a/1.txt")) - File.cp_r(src, dest, fn(src_file, dest_file) -> - assert src_file == fixture_path("cp_r/a/1.txt") - assert dest_file == tmp_path("tmp/a/1.txt") - false - end) + + File.cp_r(src, dest, + on_conflict: fn src_file, dest_file -> + assert src_file == fixture_path("cp_r/a/1.txt") + assert dest_file == tmp_path("tmp/a/1.txt") + false + end + ) + assert File.read!(tmp_path("tmp/a/1.txt")) == "hello" after - File.rm_rf dest + File.rm_rf(dest) end end - test :cp_r! do - src = fixture_path("cp_r") + test "cp_r!" do + src = fixture_path("cp_r") dest = tmp_path("tmp") File.mkdir(dest) @@ -321,69 +771,93 @@ defmodule FileTest do assert File.exists?(tmp_path("tmp/a/a/2.txt")) assert File.exists?(tmp_path("tmp/b/3.txt")) after - File.rm_rf dest + File.rm_rf(dest) end end - test :cp_r_with_src_unknown! do - src = fixture_path("unknown") + test "cp_r! with src unknown" do + src = fixture_path("unknown") dest = tmp_path("tmp") - assert_raise File.CopyError, "could not copy recursively from #{src} to #{dest}. #{src}: no such file or directory", fn -> + + message = + "could not copy recursively from #{inspect(src)} to #{inspect(dest)}. #{src}: no such file or directory" + + assert_raise File.CopyError, message, fn -> File.cp_r!(src, dest) end end - test :cp_preserves_mode do - File.mkdir_p!(tmp_path("tmp")) - src = fixture_path("cp_mode") - dest = tmp_path("tmp/cp_mode") + test "cp_r! with file src and dest unknown" do + src = fixture_path("cp_r/a/1.txt") + dest = tmp_path("tmp/unknown/") - File.cp!(src, dest) - %File.Stat{mode: src_mode} = File.stat! src - %File.Stat{mode: dest_mode} = File.stat! dest - assert src_mode == dest_mode + message = + "could not copy recursively from #{inspect(src)} to #{inspect(dest)}. #{dest}: no such file or directory" - # On overwrite - File.cp! src, dest, fn(_, _) -> true end - %File.Stat{mode: src_mode} = File.stat! src - %File.Stat{mode: dest_mode} = File.stat! dest - assert src_mode == dest_mode + assert_raise File.CopyError, message, fn -> + File.cp_r!(src, dest) + end end - defp io_error?(result) do - elem(result, 1) in [:enotdir, :eio, :enoent, :eisdir] + test "cp preserves mode" do + File.mkdir_p!(tmp_path("tmp")) + src = fixture_path("cp_mode") + dest = tmp_path("tmp/cp_mode") + + File.cp!(src, dest) + %File.Stat{mode: src_mode} = File.stat!(src) + %File.Stat{mode: dest_mode} = File.stat!(dest) + assert src_mode == dest_mode + + # On overwrite + File.cp!(src, dest, on_conflict: fn _, _ -> true end) + %File.Stat{mode: src_mode} = File.stat!(src) + %File.Stat{mode: dest_mode} = File.stat!(dest) + assert src_mode == dest_mode end end defmodule Queries do use ExUnit.Case - test :regular do + test "regular" do assert File.regular?(__ENV__.file) - assert File.regular?(String.to_char_list(__ENV__.file)) + assert File.regular?(String.to_charlist(__ENV__.file)) refute File.regular?("#{__ENV__.file}.unknown") end - test :exists do + test "exists" do assert File.exists?(__ENV__.file) - assert File.exists?(fixture_path) + assert File.exists?(fixture_path()) assert File.exists?(fixture_path("file.txt")) refute File.exists?(fixture_path("missing.txt")) refute File.exists?("_missing.txt") end + + test "exists with dangling symlink" do + invalid_file = tmp_path("invalid_file") + dest = tmp_path("dangling_symlink") + File.ln_s(invalid_file, dest) + + try do + refute File.exists?(dest) + after + File.rm(dest) + end + end end - test :ls do - {:ok, value} = File.ls(fixture_path) + test "ls" do + {:ok, value} = File.ls(fixture_path()) assert "code_sample.exs" in value assert "file.txt" in value {:error, :enoent} = File.ls(fixture_path("non-existent-subdirectory")) end - test :ls! do - value = File.ls!(fixture_path) + test "ls!" do + value = File.ls!(fixture_path()) assert "code_sample.exs" in value assert "file.txt" in value @@ -392,45 +866,45 @@ defmodule FileTest do end end - defmodule OpenReadWrite do - use Elixir.FileCase - - test :read_with_binary do + describe "open-read-write" do + test "read with binary" do assert {:ok, "FOO\n"} = File.read(fixture_path("file.txt")) assert {:error, :enoent} = File.read(fixture_path("missing.txt")) end - test :read_with_list do - assert {:ok, "FOO\n"} = File.read(Path.expand('fixtures/file.txt', __DIR__)) - assert {:error, :enoent} = File.read(Path.expand('fixtures/missing.txt', __DIR__)) + test "read with list" do + assert {:ok, "FOO\n"} = File.read(Path.expand(~c"fixtures/file.txt", __DIR__)) + assert {:error, :enoent} = File.read(Path.expand(~c"fixtures/missing.txt", __DIR__)) end - test :read_with_utf8 do - assert {:ok, "Русский\n日\n"} = File.read(Path.expand('fixtures/utf8.txt', __DIR__)) + test "read with UTF-8" do + assert {:ok, "Русский\n日\n"} = File.read(Path.expand(~c"fixtures/utf8.txt", __DIR__)) end - test :read! do + test "read!" do assert File.read!(fixture_path("file.txt")) == "FOO\n" - expected_message = "could not read file fixtures/missing.txt: no such file or directory" + expected_message = "could not read file \"fixtures/missing.txt\": no such file or directory" assert_raise File.Error, expected_message, fn -> File.read!("fixtures/missing.txt") end end - test :write_ascii_content do + test "write ASCII content" do fixture = tmp_path("tmp_test.txt") + try do refute File.exists?(fixture) - assert File.write(fixture, 'test text') == :ok + assert File.write(fixture, ~c"test text") == :ok assert File.read(fixture) == {:ok, "test text"} after File.rm(fixture) end end - test :write_utf8 do + test "write UTF-8" do fixture = tmp_path("tmp_test.txt") + try do refute File.exists?(fixture) assert File.write(fixture, "Русский\n日\n") == :ok @@ -440,8 +914,9 @@ defmodule FileTest do end end - test :write_with_options do + test "write with options" do fixture = tmp_path("tmp_test.txt") + try do refute File.exists?(fixture) assert File.write(fixture, "Русский\n日\n") == :ok @@ -452,32 +927,33 @@ defmodule FileTest do end end - test :open_file_without_modes do + test "open file without modes" do {:ok, file} = File.open(fixture_path("file.txt")) assert IO.gets(file, "") == "FOO\n" assert File.close(file) == :ok end - test :open_file_with_char_list do - {:ok, file} = File.open(fixture_path("file.txt"), [:char_list]) - assert IO.gets(file, "") == 'FOO\n' + test "open file with charlist" do + {:ok, file} = File.open(fixture_path("file.txt"), [:charlist]) + assert IO.gets(file, "") == ~c"FOO\n" assert File.close(file) == :ok end - test :open_utf8_by_default do + test "open UTF-8 by default" do {:ok, file} = File.open(fixture_path("utf8.txt"), [:utf8]) assert IO.gets(file, "") == "Русский\n" assert File.close(file) == :ok end - test :open_readonly_by_default do + test "open readonly by default" do {:ok, file} = File.open(fixture_path("file.txt")) assert_raise ArgumentError, fn -> IO.write(file, "foo") end assert File.close(file) == :ok end - test :open_with_write_permission do + test "open with write permission" do fixture = tmp_path("tmp_text.txt") + try do {:ok, file} = File.open(fixture, [:write]) assert IO.write(file, "foo") == :ok @@ -488,118 +964,132 @@ defmodule FileTest do end end - test :open_with_binwrite_permission do + test "open with binwrite permission" do fixture = tmp_path("tmp_text.txt") + try do {:ok, file} = File.open(fixture, [:write]) assert IO.binwrite(file, "Русский") == :ok assert File.close(file) == :ok + assert_raise ErlangError, fn -> IO.binwrite(file, "Русский") end assert File.read(fixture) == {:ok, "Русский"} after File.rm(fixture) end end - test :open_utf8_and_charlist do - {:ok, file} = File.open(fixture_path("utf8.txt"), [:char_list, :utf8]) + test "open UTF-8 and charlist" do + {:ok, file} = File.open(fixture_path("utf8.txt"), [:charlist, :utf8]) assert IO.gets(file, "") == [1056, 1091, 1089, 1089, 1082, 1080, 1081, 10] assert File.close(file) == :ok end - test :open_respects_encoding do + test "open respects encoding" do {:ok, file} = File.open(fixture_path("utf8.txt"), [{:encoding, :latin1}]) - assert IO.gets(file, "") == <<195, 144, 194, 160, 195, 145, 194, 131, 195, 145, 194, 129, 195, 145, 194, 129, 195, 144, 194, 186, 195, 144, 194, 184, 195, 144, 194, 185, 10>> + + data = + <<195, 144, 194, 160, 195, 145, 194, 131, 195, 145, 194, 129, 195, 145>> <> + <<194, 129, 195, 144, 194, 186, 195, 144, 194, 184, 195, 144, 194, 185, 10>> + + assert IO.gets(file, "") == data assert File.close(file) == :ok end - test :open_a_missing_file do - assert File.open('missing.txt') == {:error, :enoent} + test "open a missing file" do + assert File.open(~c"missing.txt") == {:error, :enoent} end - test :open_a_file_with_function do + test "open a file with function" do file = fixture_path("file.txt") assert File.open(file, &IO.read(&1, :line)) == {:ok, "FOO\n"} end - test :open_a_missing_file! do - message = "could not open missing.txt: no such file or directory" + test "open! a missing file" do + message = "could not open \"missing.txt\": no such file or directory" + assert_raise File.Error, message, fn -> - File.open!('missing.txt') + File.open!(~c"missing.txt") end end - test :open_a_file_with_function! do + test "open! a file with function" do file = fixture_path("file.txt") assert File.open!(file, &IO.read(&1, :line)) == "FOO\n" end end - defmodule Mkdir do - use Elixir.FileCase - - test :mkdir_with_binary do + describe "mkdir" do + test "mkdir with binary" do fixture = tmp_path("tmp_test") + try do refute File.exists?(fixture) assert File.mkdir(fixture) == :ok assert File.exists?(fixture) after - File.rmdir fixture + File.rmdir(fixture) end end - test :mkdir_with_list do - fixture = tmp_path("tmp_test") |> to_char_list + test "mkdir with list" do + fixture = tmp_path("tmp_test") |> to_charlist() + try do refute File.exists?(fixture) assert File.mkdir(fixture) == :ok assert File.exists?(fixture) after - File.rmdir fixture + File.rmdir(fixture) end end - test :mkdir_with_invalid_path do + test "mkdir with invalid path" do fixture = fixture_path("file.txt") - invalid = Path.join fixture, "test" + invalid = Path.join(fixture, "test") assert File.exists?(fixture) - assert io_error? File.mkdir(invalid) + assert io_error?(File.mkdir(invalid)) refute File.exists?(invalid) end - test :mkdir! do + test "mkdir!" do fixture = tmp_path("tmp_test") + try do refute File.exists?(fixture) assert File.mkdir!(fixture) == :ok assert File.exists?(fixture) after - File.rmdir fixture + File.rmdir(fixture) end end - test :mkdir_with_invalid_path! do + test "mkdir! with invalid path" do fixture = fixture_path("file.txt") - invalid = Path.join fixture, "test" + invalid = Path.join(fixture, "test") assert File.exists?(fixture) - assert_raise File.Error, ~r"^could not make directory #{escape invalid}: (not a directory|no such file or directory)", fn -> + + message = + ~r"\Acould not make directory #{inspect(invalid)}: (not a directory|no such file or directory)" + + assert_raise File.Error, message, fn -> File.mkdir!(invalid) end end - test :mkdir_p_with_one_directory do + test "mkdir_p with one directory" do fixture = tmp_path("tmp_test") + try do refute File.exists?(fixture) assert File.mkdir_p(fixture) == :ok assert File.exists?(fixture) after - File.rm_rf fixture + File.rm_rf(fixture) end end - test :mkdir_p_with_nested_directory_and_binary do - base = tmp_path("tmp_test") + test "mkdir_p with nested directory and binary" do + base = tmp_path("tmp_test") fixture = Path.join(base, "test") refute File.exists?(base) @@ -608,12 +1098,12 @@ defmodule FileTest do assert File.exists?(base) assert File.exists?(fixture) after - File.rm_rf base + File.rm_rf(base) end end - test :mkdir_p_with_nested_directory_and_list do - base = tmp_path("tmp_test") |> to_char_list + test "mkdir_p with nested directory and list" do + base = tmp_path("tmp_test") |> to_charlist() fixture = Path.join(base, "test") refute File.exists?(base) @@ -622,12 +1112,12 @@ defmodule FileTest do assert File.exists?(base) assert File.exists?(fixture) after - File.rm_rf base + File.rm_rf(base) end end - test :mkdir_p_with_nested_directory_and_existing_parent do - base = tmp_path("tmp_test") + test "mkdir_p with nested directory and existing parent" do + base = tmp_path("tmp_test") fixture = Path.join(base, "test") File.mkdir(base) @@ -637,72 +1127,92 @@ defmodule FileTest do assert File.exists?(base) assert File.exists?(fixture) after - File.rm_rf base + File.rm_rf(base) end end - test :mkdir_p_with_invalid_path do + test "mkdir_p with invalid path" do assert File.exists?(fixture_path("file.txt")) - invalid = Path.join fixture_path("file.txt"), "test/foo" - assert io_error? File.mkdir(invalid) + invalid = Path.join(fixture_path("file.txt"), "test/foo") + assert io_error?(File.mkdir(invalid)) refute File.exists?(invalid) end - test :mkdir_p! do + test "mkdir_p!" do fixture = tmp_path("tmp_test") + try do refute File.exists?(fixture) assert File.mkdir_p!(fixture) == :ok assert File.exists?(fixture) after - File.rm_rf fixture + File.rm_rf(fixture) end end - test :mkdir_p_with_invalid_path! do + test "mkdir_p! with invalid path" do fixture = fixture_path("file.txt") - invalid = Path.join fixture, "test" + invalid = Path.join(fixture, "test") assert File.exists?(fixture) - assert_raise File.Error, ~r"^could not make directory \(with -p\) #{escape invalid}: (not a directory|no such file or directory)", fn -> + + message = + ~r"\Acould not make directory \(with -p\) #{inspect(invalid)}: (not a directory|no such file or directory)" + + assert_raise File.Error, message, fn -> File.mkdir_p!(invalid) end end - defp io_error?(result) do - {:error, errorcode} = result - errorcode in [:enotdir, :eio, :enoent, :eisdir] + @tag :unix + test "mkdir_p with non-accessible parent directory" do + fixture = tmp_path("tmp_test_parent") + + try do + refute File.exists?(fixture) + assert File.mkdir_p!(fixture) == :ok + %File.Stat{mode: orig_mode} = File.stat!(fixture) + assert File.chmod!(fixture, 0o000) == :ok + + child = Path.join(fixture, "child") + refute File.exists?(child) + + assert File.mkdir_p(child) == {:error, :eacces} + refute File.exists?(child) + + assert File.chmod!(fixture, orig_mode) == :ok + after + File.rm_rf(fixture) + end end end - defmodule Rm do - use Elixir.FileCase - - test :rm_file do + describe "rm" do + test "rm file" do fixture = tmp_path("tmp_test.txt") File.write(fixture, "test") assert File.exists?(fixture) assert File.rm(fixture) == :ok refute File.exists?(fixture) end - - test :rm_read_only_file do + + test "rm read only file" do fixture = tmp_path("tmp_test.txt") File.write(fixture, "test") assert File.exists?(fixture) - File.chmod(fixture, 0100444) + File.chmod(fixture, 0o100444) assert File.rm(fixture) == :ok refute File.exists?(fixture) end - test :rm_file_with_dir do - assert File.rm(fixture_path) == {:error, :eperm} + test "rm file with dir" do + assert File.rm(fixture_path()) == {:error, :eperm} end - test :rm_nonexistent_file do - assert File.rm('missing.txt') == {:error, :enoent} + test "rm nonexistent file" do + assert File.rm(~c"missing.txt") == {:error, :enoent} end - test :rm! do + test "rm!" do fixture = tmp_path("tmp_test.txt") File.write(fixture, "test") assert File.exists?(fixture) @@ -710,13 +1220,15 @@ defmodule FileTest do refute File.exists?(fixture) end - test :rm_with_invalid_file! do - assert_raise File.Error, "could not remove file missing.file: no such file or directory", fn -> + test "rm! with invalid file" do + message = "could not remove file \"missing.file\": no such file or directory" + + assert_raise File.Error, message, fn -> File.rm!("missing.file") end end - test :rmdir do + test "rmdir" do fixture = tmp_path("tmp_test") File.mkdir_p(fixture) assert File.dir?(fixture) @@ -724,11 +1236,11 @@ defmodule FileTest do refute File.exists?(fixture) end - test :rmdir_with_file do - assert io_error? File.rmdir(fixture_path("file.txt")) + test "rmdir with file" do + assert io_error?(File.rmdir(fixture_path("file.txt"))) end - test :rmdir! do + test "rmdir!" do fixture = tmp_path("tmp_test") File.mkdir_p(fixture) assert File.dir?(fixture) @@ -736,14 +1248,42 @@ defmodule FileTest do refute File.exists?(fixture) end - test :rmdir_with_file! do + test "rmdir! with file" do fixture = fixture_path("file.txt") - assert_raise File.Error, ~r"^could not remove directory #{escape fixture}: (not a directory|I/O error)", fn -> + message = ~r"\Acould not remove directory #{inspect(fixture)}: (not a directory|I/O error)" + + assert_raise File.Error, message, fn -> File.rmdir!(fixture) end end - test :rm_rf do + test "rmdir! error messages" do + fixture = tmp_path("tmp_test") + File.mkdir_p(fixture) + File.touch(fixture <> "/file") + + # directory is not empty + dir_not_empty_message = + "could not remove directory #{inspect(fixture)}: directory is not empty" + + assert_raise File.Error, dir_not_empty_message, fn -> + File.rmdir!(fixture) + end + + # directory does not exist + non_existent_dir = fixture <> "/non_existent_dir" + + non_existent_dir_message = + ~r"\Acould not remove directory #{inspect(non_existent_dir)}: (not a directory|no such file or directory)" + + assert_raise File.Error, non_existent_dir_message, fn -> + File.rmdir!(non_existent_dir) + end + + File.rm_rf(fixture) + end + + test "rm_rf" do fixture = tmp_path("tmp") File.mkdir(fixture) File.cp_r!(fixture_path("cp_r"), fixture) @@ -763,15 +1303,19 @@ defmodule FileTest do refute File.exists?(fixture) end - test :rm_rf_with_symlink do + test "rm_rf raises on path with null byte" do + assert_raise ArgumentError, ~r/null byte/, fn -> File.rm_rf("foo\0bar") end + end + + test "rm_rf with symlink" do from = tmp_path("tmp/from") - to = tmp_path("tmp/to") + to = tmp_path("tmp/to") File.mkdir_p!(to) File.write!(Path.join(to, "hello"), "world") :file.make_symlink(to, from) - if File.exists?(from) or not is_win? do + if File.exists?(from) or not windows?() do assert File.exists?(from) {:ok, files} = File.rm_rf(from) @@ -784,8 +1328,8 @@ defmodule FileTest do File.rm(tmp_path("tmp/from")) end - test :rm_rf_with_char_list do - fixture = tmp_path("tmp") |> to_char_list + test "rm_rf with charlist" do + fixture = tmp_path("tmp") |> to_charlist() File.mkdir(fixture) File.cp_r!(fixture_path("cp_r"), fixture) @@ -804,23 +1348,31 @@ defmodule FileTest do refute File.exists?(fixture) end - test :rm_rf_with_file do + test "rm_rf with file" do fixture = tmp_path("tmp") File.write(fixture, "hello") assert File.rm_rf(fixture) == {:ok, [fixture]} end - test :rm_rf_with_unknown do + test "rm_rf with write-only subdir" do + dir = tmp_path("tmp") + subdir = Path.join(dir, "write-only") + File.mkdir_p!(subdir) + File.chmod!(subdir, 0o222) + assert File.rm_rf(dir) == {:ok, [dir, subdir]} + end + + test "rm_rf with unknown" do fixture = tmp_path("tmp.unknown") assert File.rm_rf(fixture) == {:ok, []} end - test :rm_rf_with_invalid do - fixture = fixture_path "file.txt/path" + test "rm_rf with invalid" do + fixture = fixture_path("file.txt/path") assert File.rm_rf(fixture) == {:ok, []} end - test :rm_rf! do + test "rm_rf!" do fixture = tmp_path("tmp") File.mkdir(fixture) File.cp_r!(fixture_path("cp_r"), fixture) @@ -840,455 +1392,540 @@ defmodule FileTest do refute File.exists?(fixture) end - test :rm_rf_with_invalid! do - fixture = fixture_path "file.txt/path" + test "rm_rf! with invalid path" do + fixture = fixture_path("file.txt/path") assert File.rm_rf!(fixture) == [] end + end - defp io_error?(result) do - elem(result, 1) in [:enotdir, :eio, :enoent, :eisdir] + describe "stat" do + test "stat" do + {:ok, info} = File.stat(__ENV__.file) + assert info.mtime end - end - test :stat do - {:ok, info} = File.stat(__ENV__.file) - assert info.mtime - end + test "stat!" do + assert File.stat!(__ENV__.file).mtime + end - test :stat! do - assert File.stat!(__ENV__.file).mtime - end + test "stat with invalid file" do + assert {:error, _} = File.stat("./invalid_file") + end - test :stat_with_invalid_file do - assert {:error, _} = File.stat("./invalid_file") - end + test "stat! with invalid_file" do + assert_raise File.Error, fn -> + File.stat!("./invalid_file") + end + end - test :stat_with_invalid_file! do - assert_raise File.Error, fn -> - File.stat!("./invalid_file") + test "lstat" do + {:ok, info} = File.lstat(__ENV__.file) + assert info.mtime end - end - test :io_stream_utf8 do - src = File.open! fixture_path("file.txt"), [:utf8] - dest = tmp_path("tmp_test.txt") + test "lstat!" do + assert File.lstat!(__ENV__.file).mtime + end + + test "lstat with invalid file" do + invalid_file = tmp_path("invalid_file") + assert {:error, _} = File.lstat(invalid_file) + end - try do - stream = IO.stream(src, :line) - File.open dest, [:write], fn(target) -> - Enum.into stream, IO.stream(target, :line), &String.replace(&1, "O", "A") + test "lstat! with invalid file" do + invalid_file = tmp_path("invalid_file") + + assert_raise File.Error, fn -> + File.lstat!(invalid_file) end - assert File.read(dest) == {:ok, "FAA\n"} - after - File.rm(dest) end - end - test :io_stream do - src = File.open! fixture_path("file.txt") - dest = tmp_path("tmp_test.txt") + test "lstat with dangling symlink" do + invalid_file = tmp_path("invalid_file") + dest = tmp_path("dangling_symlink") + File.ln_s(invalid_file, dest) - try do - stream = IO.binstream(src, :line) - File.open dest, [:write], fn(target) -> - Enum.into stream, IO.binstream(target, :line), &String.replace(&1, "O", "A") + try do + assert {:ok, info} = File.lstat(dest) + assert info.type == :symlink + after + File.rm(dest) end - assert File.read(dest) == {:ok, "FAA\n"} - after - File.rm(dest) end - end - test :stream_map do - src = fixture_path("file.txt") - stream = File.stream!(src) - assert %File.Stream{} = stream - assert stream.modes == [:raw, :read_ahead, :binary] - assert stream.raw - assert stream.line_or_bytes == :line - - src = fixture_path("file.txt") - stream = File.stream!(src, [:utf8], 10) - assert %File.Stream{} = stream - assert stream.modes == [{:encoding, :utf8}, :binary] - refute stream.raw - assert stream.line_or_bytes == 10 + test "lstat! with dangling symlink" do + invalid_file = tmp_path("invalid_file") + dest = tmp_path("dangling_symlink") + File.ln_s(invalid_file, dest) + + try do + assert File.lstat!(dest).type == :symlink + after + File.rm(dest) + end + end end - test :stream_line_utf8 do - src = fixture_path("file.txt") - dest = tmp_path("tmp_test.txt") + describe "IO stream" do + test "IO stream UTF-8" do + src = File.open!(fixture_path("file.txt"), [:utf8]) + dest = tmp_path("tmp_test.txt") - try do - stream = File.stream!(src) - File.open dest, [:write, :utf8], fn(target) -> - Enum.each stream, fn(line) -> - IO.write target, String.replace(line, "O", "A") - end + try do + stream = IO.stream(src, :line) + + File.open(dest, [:write], fn target -> + Enum.into(stream, IO.stream(target, :line), &String.replace(&1, "O", "A")) + end) + + assert File.read(dest) == {:ok, "FAA\n"} + after + File.rm(dest) end - assert File.read(dest) == {:ok, "FAA\n"} - after - File.rm(dest) end - end - test :stream_bytes_utf8 do - src = fixture_path("file.txt") - dest = tmp_path("tmp_test.txt") + test "IO stream" do + src = File.open!(fixture_path("file.txt")) + dest = tmp_path("tmp_test.txt") - try do - stream = File.stream!(src, [:utf8], 1) - File.open dest, [:write], fn(target) -> - Enum.each stream, fn(line) -> - IO.write target, String.replace(line, "OO", "AA") - end + try do + stream = IO.binstream(src, :line) + + File.open(dest, [:write], fn target -> + Enum.into(stream, IO.binstream(target, :line), &String.replace(&1, "O", "A")) + end) + + assert File.read(dest) == {:ok, "FAA\n"} + after + File.rm(dest) end - assert File.read(dest) == {:ok, "FOO\n"} - after - File.rm(dest) end end - test :stream_line do - src = fixture_path("file.txt") - dest = tmp_path("tmp_test.txt") + describe "links" do + test "read_link with regular file" do + dest = tmp_path("symlink") + File.touch(dest) - try do - stream = File.stream!(src) - File.open dest, [:write], fn(target) -> - Enum.each stream, fn(line) -> - IO.write target, String.replace(line, "O", "A") - end + try do + assert File.read_link(dest) == {:error, :einval} + after + File.rm(dest) end - assert File.read(dest) == {:ok, "FAA\n"} - after - File.rm(dest) end - end - test :stream_bytes do - src = fixture_path("file.txt") - dest = tmp_path("tmp_test.txt") + test "read_link with nonexistent file" do + dest = tmp_path("does_not_exist") + assert File.read_link(dest) == {:error, :enoent} + end - try do - stream = File.stream!(src, [], 1) - File.open dest, [:write], fn(target) -> - Enum.each stream, fn(line) -> - IO.write target, String.replace(line, "OO", "AA") - end + test "read_link! with nonexistent file" do + dest = tmp_path("does_not_exist") + assert_raise File.Error, fn -> File.read_link!(dest) end + end + + @tag :unix + test "read_link with symlink" do + target = tmp_path("does_not_need_to_exist") + dest = tmp_path("symlink") + File.ln_s(target, dest) + + try do + assert File.read_link(dest) == {:ok, target} + after + File.rm(dest) end - assert File.read(dest) == {:ok, "FOO\n"} - after - File.rm(dest) end - end - test :stream_into do - src = fixture_path("file.txt") - dest = tmp_path("tmp_test.txt") + @tag :unix + test "read_link! with symlink" do + target = tmp_path("does_not_need_to_exist") + dest = tmp_path("symlink") + File.ln_s(target, dest) - try do - refute File.exists?(dest) + try do + assert File.read_link!(dest) == target + after + File.rm(dest) + end + end - original = File.stream!(dest) - stream = File.stream!(src) - |> Stream.map(&String.replace(&1, "O", "A")) - |> Enum.into(original) + test "ln" do + existing = fixture_path("file.txt") + new = tmp_path("tmp_test.txt") - assert stream == original - assert File.read(dest) == {:ok, "FAA\n"} - after - File.rm(dest) + try do + refute File.exists?(new) + assert File.ln(existing, new) == :ok + assert File.read(new) == {:ok, "FOO\n"} + after + File.rm(new) + end end - end - test :stream_into_append do - src = fixture_path("file.txt") - dest = tmp_path("tmp_test.txt") + test "ln with existing destination" do + existing = fixture_path("file.txt") + assert File.ln(existing, existing) == {:error, :eexist} + end - try do - refute File.exists?(dest) - original = File.stream!(dest, [:append]) + test "ln! with existing destination" do + assert_raise File.LinkError, fn -> + existing = fixture_path("file.txt") + File.ln!(existing, existing) + end + end - File.stream!(src, [:append]) - |> Stream.map(&String.replace(&1, "O", "A")) - |> Enum.into(original) + test "ln_s" do + existing = fixture_path("file.txt") + new = tmp_path("tmp_test.txt") - File.stream!(src, [:append]) - |> Enum.into(original) + try do + refute File.exists?(new) + assert File.ln_s(existing, new) == :ok + assert File.read(new) == {:ok, "FOO\n"} + after + File.rm(new) + end + end - assert File.read(dest) == {:ok, "FAA\nFOO\n"} - after - File.rm(dest) + test "ln_s with existing destination" do + existing = fixture_path("file.txt") + assert File.ln_s(existing, existing) == {:error, :eexist} end - end - test :ln_s do - existing = fixture_path("file.txt") - new = tmp_path("tmp_test.txt") - try do - refute File.exists?(new) - assert File.ln_s(existing, new) == :ok - assert File.read(new) == {:ok, "FOO\n"} - after - File.rm(new) + test "ln_s! with existing destination" do + existing = fixture_path("file.txt") + + assert_raise File.LinkError, fn -> + File.ln_s!(existing, existing) + end end end - test :ln_s_with_existing_destination do - existing = fixture_path("file.txt") - assert File.ln_s(existing, existing) == {:error, :eexist} - end + describe "copy" do + test "copy" do + src = fixture_path("file.txt") + dest = tmp_path("tmp_test.txt") - test :copy do - src = fixture_path("file.txt") - dest = tmp_path("tmp_test.txt") - try do - refute File.exists?(dest) - assert File.copy(src, dest) == {:ok, 4} - assert File.read(dest) == {:ok, "FOO\n"} - after - File.rm(dest) + try do + refute File.exists?(dest) + assert File.copy(src, dest) == {:ok, 4} + assert File.read(dest) == {:ok, "FOO\n"} + after + File.rm(dest) + end end - end - test :copy_with_bytes_count do - src = fixture_path("file.txt") - dest = tmp_path("tmp_test.txt") - try do - refute File.exists?(dest) - assert File.copy(src, dest, 2) == {:ok, 2} - assert {:ok, "FO"} == File.read(dest) - after - File.rm(dest) + test "copy with an io_device" do + {:ok, src} = File.open(fixture_path("file.txt")) + dest = tmp_path("tmp_test.txt") + + try do + refute File.exists?(dest) + assert File.copy(src, dest) == {:ok, 4} + assert File.read(dest) == {:ok, "FOO\n"} + after + File.close(src) + File.rm(dest) + end end - end - test :copy_with_invalid_file do - src = fixture_path("invalid.txt") - dest = tmp_path("tmp_test.txt") - assert File.copy(src, dest, 2) == {:error, :enoent} - end + test "copy with raw io_device" do + {:ok, src} = File.open(fixture_path("file.txt"), [:raw]) + dest = tmp_path("tmp_test.txt") - test :copy! do - src = fixture_path("file.txt") - dest = tmp_path("tmp_test.txt") - try do - refute File.exists?(dest) - assert File.copy!(src, dest) == 4 - assert {:ok, "FOO\n"} == File.read(dest) - after - File.rm(dest) + try do + refute File.exists?(dest) + assert File.copy(src, dest) == {:ok, 4} + assert File.read(dest) == {:ok, "FOO\n"} + after + File.close(src) + File.rm(dest) + end end - end - test :copy_with_bytes_count! do - src = fixture_path("file.txt") - dest = tmp_path("tmp_test.txt") - try do - refute File.exists?(dest) - assert File.copy!(src, dest, 2) == 2 - assert {:ok, "FO"} == File.read(dest) - after - File.rm(dest) + test "copy with ram io_device" do + {:ok, src} = File.open("FOO\n", [:ram]) + dest = tmp_path("tmp_test.txt") + + try do + refute File.exists?(dest) + assert File.copy(src, dest) == {:ok, 4} + assert File.read(dest) == {:ok, "FOO\n"} + after + File.close(src) + File.rm(dest) + end end - end - test :copy_with_invalid_file! do - src = fixture_path("invalid.txt") - dest = tmp_path("tmp_test.txt") - assert_raise File.CopyError, "could not copy from #{src} to #{dest}: no such file or directory", fn -> - File.copy!(src, dest, 2) + test "copy with bytes count" do + src = fixture_path("file.txt") + dest = tmp_path("tmp_test.txt") + + try do + refute File.exists?(dest) + assert File.copy(src, dest, 2) == {:ok, 2} + assert {:ok, "FO"} == File.read(dest) + after + File.rm(dest) + end end - end - test :cwd_and_cd do - {:ok, current} = File.cwd - try do - assert File.cd(fixture_path) == :ok - assert File.exists?("file.txt") - after - File.cd!(current) + test "copy with invalid file" do + src = fixture_path("invalid.txt") + dest = tmp_path("tmp_test.txt") + assert File.copy(src, dest, 2) == {:error, :enoent} end - end - if :file.native_name_encoding == :utf8 do - test :cwd_and_cd_with_utf8 do - File.mkdir_p(tmp_path("héllò")) + test "copy!" do + src = fixture_path("file.txt") + dest = tmp_path("tmp_test.txt") - File.cd!(tmp_path("héllò"), fn -> - assert Path.basename(File.cwd!) == "héllò" - end) - after - File.rm_rf tmp_path("héllò") + try do + refute File.exists?(dest) + assert File.copy!(src, dest) == 4 + assert {:ok, "FOO\n"} == File.read(dest) + after + File.rm(dest) + end end - end - test :invalid_cd do - assert io_error? File.cd(fixture_path("file.txt")) - end + test "copy! with bytes count" do + src = fixture_path("file.txt") + dest = tmp_path("tmp_test.txt") - test :invalid_cd! do - message = ~r"^could not set current working directory to #{escape fixture_path("file.txt")}: (not a directory|no such file or directory)" - assert_raise File.Error, message, fn -> - File.cd!(fixture_path("file.txt")) + try do + refute File.exists?(dest) + assert File.copy!(src, dest, 2) == 2 + assert {:ok, "FO"} == File.read(dest) + after + File.rm(dest) + end end - end - test :cd_with_function do - assert File.cd!(fixture_path, fn -> - assert File.exists?("file.txt") - :cd_result - end) == :cd_result - end + test "copy! with invalid file" do + src = fixture_path("invalid.txt") + dest = tmp_path("tmp_test.txt") - test :touch_with_no_file do - fixture = tmp_path("tmp_test.txt") - time = {{2010, 4, 17}, {14, 0, 0}} + message = + "could not copy from #{inspect(src)} to #{inspect(dest)}: no such file or directory" - try do - refute File.exists?(fixture) - assert File.touch(fixture, time) == :ok - assert {:ok, ""} == File.read(fixture) - assert File.stat!(fixture).mtime == time - after - File.rm(fixture) + assert_raise File.CopyError, message, fn -> + File.copy!(src, dest, 2) + end end end - test :touch_with_timestamp do - fixture = tmp_path("tmp_test.txt") + describe "cwd and cd" do + test "cwd and cd" do + {:ok, current} = File.cwd() + + try do + assert File.cd(fixture_path()) == :ok + assert File.exists?("file.txt") + after + File.cd!(current) + end + end - try do - assert File.touch!(fixture) == :ok - stat = File.stat!(fixture) + if :file.native_name_encoding() == :utf8 do + test "cwd and cd with UTF-8" do + File.mkdir_p(tmp_path("héllò")) - assert File.touch!(fixture, last_year) == :ok - assert stat.mtime > File.stat!(fixture).mtime - after - File.rm(fixture) + File.cd!(tmp_path("héllò"), fn -> + assert Path.basename(File.cwd!()) == "héllò" + end) + after + File.rm_rf(tmp_path("héllò")) + end end - end - test :touch_with_dir do - assert File.touch(fixture_path) == :ok - end + test "invalid cd" do + assert io_error?(File.cd(fixture_path("file.txt"))) + end - test :touch_with_failure do - fixture = fixture_path("file.txt/bar") - assert io_error? File.touch(fixture) - end + test "invalid_cd!" do + message = + ~r"\Acould not set current working directory to #{inspect(fixture_path("file.txt"))}: (not a directory|no such file or directory|I/O error)" - test :touch_with_success! do - assert File.touch!(fixture_path) == :ok - end + assert_raise File.Error, message, fn -> + File.cd!(fixture_path("file.txt")) + end + end - test :touch_with_failure! do - fixture = fixture_path("file.txt/bar") - assert_raise File.Error, ~r"could not touch #{escape fixture}: (not a directory|no such file or directory)", fn -> - File.touch!(fixture) + test "cd with function" do + assert File.cd!(fixture_path(), fn -> + assert File.exists?("file.txt") + :cd_result + end) == :cd_result end end - test :chmod_with_success do - fixture = tmp_path("tmp_test.txt") + describe "touch" do + test "touch with no file" do + fixture = tmp_path("tmp_test.txt") + time = {{2010, 4, 17}, {14, 0, 0}} + + try do + refute File.exists?(fixture) + assert File.touch(fixture, time) == :ok + assert {:ok, ""} == File.read(fixture) + assert File.stat!(fixture).mtime == time + after + File.rm(fixture) + end + end - File.touch(fixture) - try do - assert File.chmod(fixture, 0100666) == :ok - stat = File.stat!(fixture) - assert stat.mode == 0100666 + test "touch with Erlang timestamp" do + fixture = tmp_path("tmp_erlang_touch.txt") - unless is_win? do - assert File.chmod(fixture, 0100777) == :ok + try do + assert File.touch!(fixture, :erlang.universaltime()) == :ok stat = File.stat!(fixture) - assert stat.mode == 0100777 + + assert File.touch!(fixture, last_year()) == :ok + assert stat.mtime > File.stat!(fixture).mtime + after + File.rm(fixture) + end + end + + test "touch with posix timestamp" do + fixture = tmp_path("tmp_posix_touch.txt") + + try do + assert File.touch!(fixture, System.os_time(:second)) == :ok + stat = File.stat!(fixture) + + assert File.touch!(fixture, last_year()) == :ok + assert stat.mtime > File.stat!(fixture).mtime + after + File.rm(fixture) + end + end + + test "touch with dir" do + assert File.touch(fixture_path()) == :ok + end + + test "touch with failure" do + fixture = fixture_path("file.txt/bar") + assert io_error?(File.touch(fixture)) + end + + test "touch! raises" do + fixture = fixture_path("file.txt/bar") + + message = + ~r"\Acould not touch #{inspect(fixture)}: (not a directory|no such file or directory)" + + assert_raise File.Error, message, fn -> + File.touch!(fixture) end - after - File.rm(fixture) end end - test :chmod_with_success! do - fixture = tmp_path("tmp_test.txt") + describe "ch*" do + test "chmod with success" do + fixture = tmp_path("tmp_test.txt") - File.touch(fixture) - try do - assert File.chmod!(fixture, 0100666) == :ok - stat = File.stat!(fixture) - assert stat.mode == 0100666 + File.touch(fixture) - unless is_win? do - assert File.chmod!(fixture, 0100777) == :ok + try do + assert File.chmod(fixture, 0o100666) == :ok stat = File.stat!(fixture) - assert stat.mode == 0100777 + assert stat.mode == 0o100666 + + if not windows?() do + assert File.chmod(fixture, 0o100777) == :ok + stat = File.stat!(fixture) + assert stat.mode == 0o100777 + end + after + File.rm(fixture) end - after - File.rm(fixture) end - end - test :chmod_with_failure do - fixture = tmp_path("tmp_test.txt") - File.rm(fixture) + test "chmod! with success" do + fixture = tmp_path("tmp_test.txt") - assert File.chmod(fixture, 0100777) == {:error,:enoent} - end + File.touch(fixture) + + try do + assert File.chmod!(fixture, 0o100666) == :ok + stat = File.stat!(fixture) + assert stat.mode == 0o100666 + + if not windows?() do + assert File.chmod!(fixture, 0o100777) == :ok + stat = File.stat!(fixture) + assert stat.mode == 0o100777 + end + after + File.rm(fixture) + end + end - test :chmod_with_failure! do - fixture = tmp_path("tmp_test.txt") - File.rm(fixture) + test "chmod with failure" do + fixture = tmp_path("tmp_test.txt") + File.rm(fixture) - message = ~r"could not change mode for #{escape fixture}: no such file or directory" - assert_raise File.Error, message, fn -> - File.chmod!(fixture, 0100777) + assert File.chmod(fixture, 0o100777) == {:error, :enoent} end - end - test :chgrp_with_failure do - fixture = tmp_path("tmp_test.txt") - File.rm(fixture) + test "chmod! with failure" do + fixture = tmp_path("tmp_test.txt") + File.rm(fixture) + + message = ~r"could not change mode for #{inspect(fixture)}: no such file or directory" - assert File.chgrp(fixture, 1) == {:error,:enoent} - end + assert_raise File.Error, message, fn -> + File.chmod!(fixture, 0o100777) + end + end - test :chgrp_with_failure! do - fixture = tmp_path("tmp_test.txt") - File.rm(fixture) + test "chgrp with failure" do + fixture = tmp_path("tmp_test.txt") + File.rm(fixture) - message = ~r"could not change group for #{escape fixture}: no such file or directory" - assert_raise File.Error, message, fn -> - File.chgrp!(fixture, 1) + assert File.chgrp(fixture, 1) == {:error, :enoent} end - end - test :chown_with_failure do - fixture = tmp_path("tmp_test.txt") - File.rm(fixture) + test "chgrp! with failure" do + fixture = tmp_path("tmp_test.txt") + File.rm(fixture) + + message = ~r"could not change group for #{inspect(fixture)}: no such file or directory" - assert File.chown(fixture, 1) == {:error,:enoent} - end + assert_raise File.Error, message, fn -> + File.chgrp!(fixture, 1) + end + end - test :chown_with_failure! do - fixture = tmp_path("tmp_test.txt") - File.rm(fixture) + test "chown with failure" do + fixture = tmp_path("tmp_test.txt") + File.rm(fixture) - message = ~r"could not change owner for #{escape fixture}: no such file or directory" - assert_raise File.Error, message, fn -> - File.chown!(fixture, 1) + assert File.chown(fixture, 1) == {:error, :enoent} end - end - defp last_year do - last_year :calendar.local_time + test "chown! with failure" do + fixture = tmp_path("tmp_test.txt") + File.rm(fixture) + + message = ~r"could not change owner for #{inspect(fixture)}: no such file or directory" + + assert_raise File.Error, message, fn -> + File.chown!(fixture, 1) + end + end end - defp last_year({{year, month, day}, time}) do - {{year - 1, month, day}, time} + defp last_year do + System.os_time(:second) - 365 * 24 * 60 * 60 end defp io_error?(result) do - {:error, errorcode} = result - errorcode in [:enotdir, :eio, :enoent, :eisdir] + elem(result, 1) in [:enotdir, :eio, :enoent, :eisdir] end end diff --git a/lib/elixir/test/elixir/fixtures/at_exit.exs b/lib/elixir/test/elixir/fixtures/at_exit.exs index 78460b222ba..f0150a4333b 100644 --- a/lib/elixir/test/elixir/fixtures/at_exit.exs +++ b/lib/elixir/test/elixir/fixtures/at_exit.exs @@ -1,8 +1,9 @@ defmodule AtExit do def at_exit(str) do - System.at_exit fn(_) -> IO.write(str) end + System.at_exit(fn _ -> IO.write(str) end) end end -System.at_exit fn(status) -> IO.puts "cruel world with status #{status}" end + +System.at_exit(fn status -> IO.puts("cruel world with status #{status}") end) AtExit.at_exit("goodbye ") -exit(0) \ No newline at end of file +exit({:shutdown, 1}) diff --git a/lib/elixir/test/elixir/fixtures/code_sample.exs b/lib/elixir/test/elixir/fixtures/code_sample.exs index f1895089889..511623e3a7c 100644 --- a/lib/elixir/test/elixir/fixtures/code_sample.exs +++ b/lib/elixir/test/elixir/fixtures/code_sample.exs @@ -1,3 +1,3 @@ # Some Comments var = 1 + 2 -var \ No newline at end of file +var diff --git a/lib/elixir/test/elixir/fixtures/compile_sample.ex b/lib/elixir/test/elixir/fixtures/compile_sample.ex index 7cbd92b5a29..78761880f11 100644 --- a/lib/elixir/test/elixir/fixtures/compile_sample.ex +++ b/lib/elixir/test/elixir/fixtures/compile_sample.ex @@ -1 +1 @@ -defmodule CompileSample, do: nil \ No newline at end of file +defmodule(CompileSample, do: nil) diff --git a/lib/mix/test/fixtures/configs/bad_app.exs b/lib/elixir/test/elixir/fixtures/configs/bad_app.exs similarity index 100% rename from lib/mix/test/fixtures/configs/bad_app.exs rename to lib/elixir/test/elixir/fixtures/configs/bad_app.exs diff --git a/lib/elixir/test/elixir/fixtures/configs/bad_import.exs b/lib/elixir/test/elixir/fixtures/configs/bad_import.exs new file mode 100644 index 00000000000..d03906ea1e9 --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/configs/bad_import.exs @@ -0,0 +1,2 @@ +import Config +import_config "bad_root.exs" diff --git a/lib/elixir/test/elixir/fixtures/configs/env.exs b/lib/elixir/test/elixir/fixtures/configs/env.exs new file mode 100644 index 00000000000..739e6236941 --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/configs/env.exs @@ -0,0 +1,2 @@ +import Config +config :my_app, env: config_env(), target: config_target() diff --git a/lib/elixir/test/elixir/fixtures/configs/good_config.exs b/lib/elixir/test/elixir/fixtures/configs/good_config.exs new file mode 100644 index 00000000000..dd2bb471195 --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/configs/good_config.exs @@ -0,0 +1,2 @@ +import Config +config :my_app, :key, :value diff --git a/lib/mix/test/fixtures/configs/good_import.exs b/lib/elixir/test/elixir/fixtures/configs/good_import.exs similarity index 71% rename from lib/mix/test/fixtures/configs/good_import.exs rename to lib/elixir/test/elixir/fixtures/configs/good_import.exs index 759edb0108a..549312a2db6 100644 --- a/lib/mix/test/fixtures/configs/good_import.exs +++ b/lib/elixir/test/elixir/fixtures/configs/good_import.exs @@ -1,3 +1,3 @@ -use Mix.Config +import Config import_config "good_config.exs" :done diff --git a/lib/elixir/test/elixir/fixtures/configs/good_kw.exs b/lib/elixir/test/elixir/fixtures/configs/good_kw.exs new file mode 100644 index 00000000000..0c13b63b986 --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/configs/good_kw.exs @@ -0,0 +1 @@ +[my_app: [key: :value]] diff --git a/lib/elixir/test/elixir/fixtures/configs/imports_recursive.exs b/lib/elixir/test/elixir/fixtures/configs/imports_recursive.exs new file mode 100644 index 00000000000..c71c4418ec6 --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/configs/imports_recursive.exs @@ -0,0 +1,2 @@ +import Config +import_config "recursive.exs" diff --git a/lib/elixir/test/elixir/fixtures/configs/kernel.exs b/lib/elixir/test/elixir/fixtures/configs/kernel.exs new file mode 100644 index 00000000000..5020372a5d8 --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/configs/kernel.exs @@ -0,0 +1,3 @@ +import Config +config :kernel, :elixir_reboot, true +config :elixir_reboot, :key, :value diff --git a/lib/elixir/test/elixir/fixtures/configs/nested.exs b/lib/elixir/test/elixir/fixtures/configs/nested.exs new file mode 100644 index 00000000000..5570ef976d9 --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/configs/nested.exs @@ -0,0 +1,2 @@ +import Config +config :app, Repo, key: [nested: true] diff --git a/lib/elixir/test/elixir/fixtures/configs/recursive.exs b/lib/elixir/test/elixir/fixtures/configs/recursive.exs new file mode 100644 index 00000000000..eb25bb4076d --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/configs/recursive.exs @@ -0,0 +1,2 @@ +import Config +import_config "imports_recursive.exs" diff --git a/lib/elixir/test/elixir/fixtures/consolidation/no_impl.ex b/lib/elixir/test/elixir/fixtures/consolidation/no_impl.ex new file mode 100644 index 00000000000..7cd9f90db4c --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/consolidation/no_impl.ex @@ -0,0 +1,3 @@ +defprotocol Protocol.ConsolidationTest.NoImpl do + def ok(term) +end diff --git a/lib/elixir/test/elixir/fixtures/consolidation/sample.ex b/lib/elixir/test/elixir/fixtures/consolidation/sample.ex new file mode 100644 index 00000000000..ee7edea3fbc --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/consolidation/sample.ex @@ -0,0 +1,7 @@ +defprotocol Protocol.ConsolidationTest.Sample do + @type t :: any + @doc "Ok" + @deprecated "Reason" + @spec ok(t) :: boolean + def ok(term) +end diff --git a/lib/elixir/test/elixir/fixtures/consolidation/with_any.ex b/lib/elixir/test/elixir/fixtures/consolidation/with_any.ex new file mode 100644 index 00000000000..513257ec5d2 --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/consolidation/with_any.ex @@ -0,0 +1,5 @@ +defprotocol Protocol.ConsolidationTest.WithAny do + @fallback_to_any true + @doc "Ok" + def ok(term, opts) +end diff --git a/lib/elixir/test/elixir/fixtures/dialyzer/assertions.ex b/lib/elixir/test/elixir/fixtures/dialyzer/assertions.ex new file mode 100644 index 00000000000..a81ceedcd2c --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/dialyzer/assertions.ex @@ -0,0 +1,35 @@ +defmodule Dialyzer.Assertions do + import ExUnit.Assertions + + def assert_with_truthy_match do + assert :ok = known_type_truthy() + end + + def assert_with_truthy_value do + assert known_type_truthy() + end + + def assert_with_unknown_type do + assert unknown_type_truthy() + end + + def refute_with_falsy_value do + refute known_type_falsy() + end + + def refute_with_unknown_type do + refute unknown_type_falsy() + end + + def refute_with_operator(log) do + refute log == "failure" + end + + defp known_type_truthy, do: :ok + defp known_type_falsy, do: nil + + @spec unknown_type_truthy :: any + defp unknown_type_truthy, do: Enum.random([1, true, :ok]) + @spec unknown_type_falsy :: any + defp unknown_type_falsy, do: Enum.random([false, nil]) +end diff --git a/lib/elixir/test/elixir/fixtures/dialyzer/boolean_check.ex b/lib/elixir/test/elixir/fixtures/dialyzer/boolean_check.ex new file mode 100644 index 00000000000..e8123315370 --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/dialyzer/boolean_check.ex @@ -0,0 +1,9 @@ +defmodule Dialyzer.BooleanCheck do + def and_check(arg) when is_boolean(arg) do + arg and arg + end + + def or_check(arg) when is_boolean(arg) do + arg or arg + end +end diff --git a/lib/elixir/test/elixir/fixtures/dialyzer/callback.ex b/lib/elixir/test/elixir/fixtures/dialyzer/callback.ex new file mode 100644 index 00000000000..0eeff5f5c3d --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/dialyzer/callback.ex @@ -0,0 +1,14 @@ +defmodule Dialyzer.Callback do + @callback required(atom) :: atom + @callback required(list) :: list +end + +defmodule Dialyzer.Callback.ImplAtom do + @behaviour Dialyzer.Callback + def required(:ok), do: :ok +end + +defmodule Dialyzer.Callback.ImplList do + @behaviour Dialyzer.Callback + def required([a, b]), do: [b, a] +end diff --git a/lib/elixir/test/elixir/fixtures/dialyzer/cond.ex b/lib/elixir/test/elixir/fixtures/dialyzer/cond.ex new file mode 100644 index 00000000000..9419b9b8e36 --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/dialyzer/cond.ex @@ -0,0 +1,27 @@ +defmodule Dialyzer.Cond do + def one_boolean do + cond do + true -> :ok + end + end + + def two_boolean do + cond do + List.flatten([]) == [] -> :ok + true -> :ok + end + end + + def one_otherwise do + cond do + :otherwise -> :ok + end + end + + def two_otherwise do + cond do + List.flatten([]) == [] -> :ok + :otherwise -> :ok + end + end +end diff --git a/lib/elixir/test/elixir/fixtures/dialyzer/defmacrop.ex b/lib/elixir/test/elixir/fixtures/dialyzer/defmacrop.ex new file mode 100644 index 00000000000..e8d0a42ce7d --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/dialyzer/defmacrop.ex @@ -0,0 +1,11 @@ +defmodule Dialyzer.Defmacrop do + defmacrop good_macro(id) do + quote do + {:good, {:good_macro, unquote(id)}} + end + end + + def run() do + good_macro("Not So Bad") + end +end diff --git a/lib/elixir/test/elixir/fixtures/dialyzer/for_bitstring.ex b/lib/elixir/test/elixir/fixtures/dialyzer/for_bitstring.ex new file mode 100644 index 00000000000..1d22de458e6 --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/dialyzer/for_bitstring.ex @@ -0,0 +1,5 @@ +defmodule Dialyzer.ForBitstring do + def foo() do + for a <- 1..3, into: "", do: <> + end +end diff --git a/lib/elixir/test/elixir/fixtures/dialyzer/for_boolean_check.ex b/lib/elixir/test/elixir/fixtures/dialyzer/for_boolean_check.ex new file mode 100644 index 00000000000..968efe7873e --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/dialyzer/for_boolean_check.ex @@ -0,0 +1,7 @@ +defmodule Dialyzer.ForBooleanCheck do + def foo(enum, potential) when is_binary(potential) do + for element <- enum, string = Atom.to_string(element), string == potential do + element + end + end +end diff --git a/lib/elixir/test/elixir/fixtures/dialyzer/is_struct.ex b/lib/elixir/test/elixir/fixtures/dialyzer/is_struct.ex new file mode 100644 index 00000000000..c61a6a1b1b3 --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/dialyzer/is_struct.ex @@ -0,0 +1,9 @@ +defmodule Dialyzer.IsStruct do + def map_literal_atom_literal() do + is_struct(%Macro.Env{}, Macro.Env) + end + + def arg_atom_literal(arg) do + is_struct(arg, Macro.Env) + end +end diff --git a/lib/elixir/test/elixir/fixtures/dialyzer/macrocallback.ex b/lib/elixir/test/elixir/fixtures/dialyzer/macrocallback.ex new file mode 100644 index 00000000000..42bd2bc9bc7 --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/dialyzer/macrocallback.ex @@ -0,0 +1,11 @@ +defmodule Dialyzer.Macrocallback do + @macrocallback required(atom) :: Macro.t() + @macrocallback optional(atom) :: Macro.t() + @optional_callbacks [optional: 1] +end + +defmodule Dialyzer.Macrocallback.Impl do + @behaviour Dialyzer.Macrocallback + defmacro required(var), do: Macro.expand(var, __CALLER__) + defmacro optional(var), do: Macro.expand(var, __CALLER__) +end diff --git a/lib/elixir/test/elixir/fixtures/dialyzer/opaqueness.ex b/lib/elixir/test/elixir/fixtures/dialyzer/opaqueness.ex new file mode 100644 index 00000000000..d8371c386e5 --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/dialyzer/opaqueness.ex @@ -0,0 +1,22 @@ +defmodule Dialyzer.Opaqueness do + @spec bar(MapSet.t()) :: term() + def bar(set) do + set + end + + def inlined do + # inlining of literals should not violate opaqueness check + bar(MapSet.new([1, 2, 3])) + end + + @my_set MapSet.new([1, 2, 3]) + def module_attr do + bar(@my_set) + end + + # Task.Supervisor returns a Task.t() containing an opaque Task.ref() + @spec run_task() :: Task.t() + def run_task do + Task.Supervisor.async(SupervisorName, fn -> :ok end) + end +end diff --git a/lib/elixir/test/elixir/fixtures/dialyzer/raise.ex b/lib/elixir/test/elixir/fixtures/dialyzer/raise.ex new file mode 100644 index 00000000000..808ba3b0233 --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/dialyzer/raise.ex @@ -0,0 +1,21 @@ +defmodule Dialyzer.Raise do + defexception [:message] + + def exception_var() do + ex = %Dialyzer.Raise{} + raise ex + end + + def exception_var(ex = %Dialyzer.Raise{}) do + raise ex + end + + def string_var() do + string = "hello" + raise string + end + + def string_var(string) when is_binary(string) do + raise string + end +end diff --git a/lib/elixir/test/elixir/fixtures/dialyzer/regressions.ex b/lib/elixir/test/elixir/fixtures/dialyzer/regressions.ex new file mode 100644 index 00000000000..8cb04f394ae --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/dialyzer/regressions.ex @@ -0,0 +1,18 @@ +defmodule Dialyzer.Regressions do + def io_inspect_opts do + IO.inspect(123, label: "foo", limit: :infinity) + end + + def format_opts do + Code.format_string!("", + line_length: 120, + force_do_end_blocks: true, + locals_without_parens: true, + migrate: true + ) + end + + def eex_eval_opts do + EEx.eval_string("foo <%= bar %>", [bar: "baz"], trim: true) + end +end diff --git a/lib/elixir/test/elixir/fixtures/dialyzer/remote_call.ex b/lib/elixir/test/elixir/fixtures/dialyzer/remote_call.ex new file mode 100644 index 00000000000..a63654b4c8d --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/dialyzer/remote_call.ex @@ -0,0 +1,29 @@ +defmodule Dialyzer.RemoteCall do + _ = Application.load(:dialyzer) + + case Application.spec(:dialyzer, :vsn) do + ~c(2.) ++ _ -> + @dialyzer {:no_fail_call, [map_var: 0]} + + three when three < ~c(3.0.2) -> + # regression introduced in 3.0 for map warnings fixed in 3.0.2 + @dialyzer {:no_match, [map_var: 0, mod_var: 0]} + + _ -> + :ok + end + + def map_var() do + map = %{key: 1} + map.key + end + + def map_var(map) when is_map(map) do + map.key + end + + def mod_var() do + module = String.to_atom("Elixir.Hello") + module.fun() + end +end diff --git a/lib/elixir/test/elixir/fixtures/dialyzer/rewrite.ex b/lib/elixir/test/elixir/fixtures/dialyzer/rewrite.ex new file mode 100644 index 00000000000..129f0bbea68 --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/dialyzer/rewrite.ex @@ -0,0 +1,9 @@ +defmodule Dialyzer.Rewrite do + def interpolation do + "foo #{:a}" + end + + def reverse do + Enum.reverse(1..3) + end +end diff --git a/lib/elixir/test/elixir/fixtures/dialyzer/try.ex b/lib/elixir/test/elixir/fixtures/dialyzer/try.ex new file mode 100644 index 00000000000..69f9ca4b45f --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/dialyzer/try.ex @@ -0,0 +1,9 @@ +defmodule Dialyzer.Try do + def rescue_error do + try do + :erlang.error(:badarg) + rescue + e in ErlangError -> {:ok, e} + end + end +end diff --git a/lib/elixir/test/elixir/fixtures/dialyzer/with.ex b/lib/elixir/test/elixir/fixtures/dialyzer/with.ex new file mode 100644 index 00000000000..e4a9a0eb87b --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/dialyzer/with.ex @@ -0,0 +1,45 @@ +defmodule Dialyzer.With do + def with_else do + with :ok <- ok_or_error(), + :ok <- ok_or_other_error(), + :ok <- ok_or_tuple_error(), + :ok <- ok_or_tuple_list_error() do + :ok + else + :error -> + :error + + :other_error -> + :other_error + + {:error, msg} when is_list(msg) or is_tuple(msg) -> + :error + + {:error, msg} when is_list(msg) when is_tuple(msg) -> + :error + + {:error, _msg} -> + :error + end + end + + @spec ok_or_error() :: :ok | :error + defp ok_or_error do + Enum.random([:ok, :error]) + end + + @spec ok_or_other_error() :: :ok | :other_error + defp ok_or_other_error do + Enum.random([:ok, :other_error]) + end + + @spec ok_or_tuple_error() :: :ok | {:error, :err} + defp ok_or_tuple_error do + Enum.random([:ok, {:error, :err}]) + end + + @spec ok_or_tuple_list_error() :: :ok | {:error, [:err]} + defp ok_or_tuple_list_error do + Enum.random([:ok, {:error, [:err]}]) + end +end diff --git a/lib/elixir/test/elixir/fixtures/dialyzer/with_no_return.ex b/lib/elixir/test/elixir/fixtures/dialyzer/with_no_return.ex new file mode 100644 index 00000000000..264f58c8ff8 --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/dialyzer/with_no_return.ex @@ -0,0 +1,13 @@ +defmodule Dialyzer.WithNoReturn do + def with_no_return(list) do + no_return = fn -> throw(:no_return) end + + with [] <- list do + :ok + else + # note: throwing here directly wouldn't be caught in the first place, + # calling a no_return function is what could cause an issue. + _ -> no_return.() + end + end +end diff --git a/lib/elixir/test/elixir/fixtures/dialyzer/with_throwing_else.ex b/lib/elixir/test/elixir/fixtures/dialyzer/with_throwing_else.ex new file mode 100644 index 00000000000..1ea99134e37 --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/dialyzer/with_throwing_else.ex @@ -0,0 +1,12 @@ +defmodule Dialyzer.WithThrowingElse do + def with_throwing_else(map) do + with {:ok, foo} <- Map.fetch(map, :foo), + false <- Enum.empty?(foo) do + foo + else + # several clauses but one is a no_return + :error -> throw(:empty_map) + true -> nil + end + end +end diff --git a/lib/elixir/test/elixir/fixtures/file.bin b/lib/elixir/test/elixir/fixtures/file.bin new file mode 100644 index 00000000000..491c0980ddb --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/file.bin @@ -0,0 +1,4 @@ +LF +CR CRLF +LFCR + \ No newline at end of file diff --git a/lib/elixir/test/elixir/fixtures/init_sample.exs b/lib/elixir/test/elixir/fixtures/init_sample.exs deleted file mode 100644 index 4a1775c5027..00000000000 --- a/lib/elixir/test/elixir/fixtures/init_sample.exs +++ /dev/null @@ -1 +0,0 @@ -IO.puts to_string(1 + 2) \ No newline at end of file diff --git a/lib/elixir/test/elixir/fixtures/multiline_file.txt b/lib/elixir/test/elixir/fixtures/multiline_file.txt new file mode 100644 index 00000000000..9899a767b6c --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/multiline_file.txt @@ -0,0 +1,2 @@ +this is the first line +this is the second line diff --git a/lib/elixir/test/elixir/fixtures/parallel_compiler/bar.ex b/lib/elixir/test/elixir/fixtures/parallel_compiler/bar.ex deleted file mode 100644 index d6134b4c825..00000000000 --- a/lib/elixir/test/elixir/fixtures/parallel_compiler/bar.ex +++ /dev/null @@ -1,5 +0,0 @@ -defmodule Bar do -end - -require Foo -IO.puts Foo.message \ No newline at end of file diff --git a/lib/elixir/test/elixir/fixtures/parallel_compiler/bat.ex b/lib/elixir/test/elixir/fixtures/parallel_compiler/bat.ex deleted file mode 100644 index 87cfccc56c9..00000000000 --- a/lib/elixir/test/elixir/fixtures/parallel_compiler/bat.ex +++ /dev/null @@ -1,3 +0,0 @@ -defmodule Bat do - ThisModuleWillNeverBeAvailable[] -end \ No newline at end of file diff --git a/lib/elixir/test/elixir/fixtures/parallel_compiler/foo.ex b/lib/elixir/test/elixir/fixtures/parallel_compiler/foo.ex deleted file mode 100644 index ea9a7cd6d50..00000000000 --- a/lib/elixir/test/elixir/fixtures/parallel_compiler/foo.ex +++ /dev/null @@ -1,3 +0,0 @@ -defmodule Foo do - def message, do: "message_from_foo" -end \ No newline at end of file diff --git a/lib/elixir/test/elixir/fixtures/parallel_deadlock/bar.ex b/lib/elixir/test/elixir/fixtures/parallel_deadlock/bar.ex deleted file mode 100644 index 4cb9fca2c7b..00000000000 --- a/lib/elixir/test/elixir/fixtures/parallel_deadlock/bar.ex +++ /dev/null @@ -1,3 +0,0 @@ -defmodule Bar do - Foo.__info__(:macros) -end \ No newline at end of file diff --git a/lib/elixir/test/elixir/fixtures/parallel_deadlock/foo.ex b/lib/elixir/test/elixir/fixtures/parallel_deadlock/foo.ex deleted file mode 100644 index d50072d11af..00000000000 --- a/lib/elixir/test/elixir/fixtures/parallel_deadlock/foo.ex +++ /dev/null @@ -1,3 +0,0 @@ -defmodule Foo do - Bar.__info__(:macros) -end \ No newline at end of file diff --git a/lib/elixir/test/elixir/fixtures/parallel_struct/bar.ex b/lib/elixir/test/elixir/fixtures/parallel_struct/bar.ex deleted file mode 100644 index 6e697731403..00000000000 --- a/lib/elixir/test/elixir/fixtures/parallel_struct/bar.ex +++ /dev/null @@ -1,4 +0,0 @@ -defmodule Bar do - defstruct name: "" - def foo?(%Foo{}), do: true -end diff --git a/lib/elixir/test/elixir/fixtures/parallel_struct/foo.ex b/lib/elixir/test/elixir/fixtures/parallel_struct/foo.ex deleted file mode 100644 index 0e7153e01de..00000000000 --- a/lib/elixir/test/elixir/fixtures/parallel_struct/foo.ex +++ /dev/null @@ -1,4 +0,0 @@ -defmodule Foo do - defstruct name: "" - def bar?(%Bar{}), do: true -end diff --git a/lib/elixir/test/elixir/fixtures/utf16_be_bom.txt b/lib/elixir/test/elixir/fixtures/utf16_be_bom.txt new file mode 100644 index 00000000000..2499c9df2b1 Binary files /dev/null and b/lib/elixir/test/elixir/fixtures/utf16_be_bom.txt differ diff --git a/lib/elixir/test/elixir/fixtures/utf16_le_bom.txt b/lib/elixir/test/elixir/fixtures/utf16_le_bom.txt new file mode 100644 index 00000000000..5e16b0d385a Binary files /dev/null and b/lib/elixir/test/elixir/fixtures/utf16_le_bom.txt differ diff --git a/lib/elixir/test/elixir/fixtures/utf8_bom.txt b/lib/elixir/test/elixir/fixtures/utf8_bom.txt new file mode 100644 index 00000000000..bed73fc4ca1 --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/utf8_bom.txt @@ -0,0 +1,2 @@ +Русский +日 diff --git a/lib/elixir/test/elixir/fixtures/warnings_sample.ex b/lib/elixir/test/elixir/fixtures/warnings_sample.ex deleted file mode 100644 index 4a0bfef0ffd..00000000000 --- a/lib/elixir/test/elixir/fixtures/warnings_sample.ex +++ /dev/null @@ -1,4 +0,0 @@ -defmodule WarningsSample do - def hello(a), do: a - def hello(b), do: b -end diff --git a/lib/elixir/test/elixir/float_test.exs b/lib/elixir/test/elixir/float_test.exs index a1a1471095e..fbc9230a490 100644 --- a/lib/elixir/test/elixir/float_test.exs +++ b/lib/elixir/test/elixir/float_test.exs @@ -1,17 +1,36 @@ -Code.require_file "test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("test_helper.exs", __DIR__) defmodule FloatTest do use ExUnit.Case, async: true - test :parse do + doctest Float + + # TODO remove and replace by assert once we require Erlang/OTP 27+ + # We can't easily distinguish between -0.0 and +0.0 on previous version + defmacrop float_assert({:===, _, [left, right]}) do + quote do + # note: these are pure functions so no need to use bind_quoted + # we favor a useful error message instead + assert unquote(left) === unquote(right) + assert to_string(unquote(left)) === to_string(unquote(right)) + end + end + + test "parse/1" do assert Float.parse("12") === {12.0, ""} assert Float.parse("-12") === {-12.0, ""} assert Float.parse("-0.1") === {-0.1, ""} - assert Float.parse("123456789") === {123456789.0, ""} + assert Float.parse("123456789") === {123_456_789.0, ""} assert Float.parse("12.5") === {12.5, ""} assert Float.parse("12.524235") === {12.524235, ""} assert Float.parse("-12.5") === {-12.5, ""} assert Float.parse("-12.524235") === {-12.524235, ""} + assert Float.parse("0.3534091") === {0.3534091, ""} + assert Float.parse("0.3534091elixir") === {0.3534091, "elixir"} assert Float.parse("7.5e3") === {7.5e3, ""} assert Float.parse("7.5e-3") === {7.5e-3, ""} assert Float.parse("12x") === {12.0, "x"} @@ -20,51 +39,201 @@ defmodule FloatTest do assert Float.parse("-12.32453e-10") === {-1.232453e-9, ""} assert Float.parse("0.32453e-10") === {3.2453e-11, ""} assert Float.parse("1.32453e-10") === {1.32453e-10, ""} + assert Float.parse("1.7976931348623159e-99999foo") === {0.0, "foo"} assert Float.parse("1.32.45") === {1.32, ".45"} assert Float.parse("1.o") === {1.0, ".o"} + assert Float.parse("+12.3E+4") === {1.23e5, ""} + assert Float.parse("+12.3E-4x") === {0.00123, "x"} + assert Float.parse("-1.23e-0xFF") === {-1.23, "xFF"} + assert Float.parse("-1.e2") === {-1.0, ".e2"} + assert Float.parse(".12") === :error assert Float.parse("--1.2") === :error assert Float.parse("++1.2") === :error assert Float.parse("pi") === :error + assert Float.parse("1.7976931348623157e308") === {1.7976931348623157e308, ""} + assert Float.parse("1.7976931348623157e308foo") === {1.7976931348623157e308, "foo"} + assert Float.parse("1.7976931348623157e+308foo") === {1.7976931348623157e308, "foo"} + assert Float.parse("1.7976931348623157e-308foo") === {1.7976931348623155e-308, "foo"} + assert Float.parse("1.7976931348623159e308") === :error + assert Float.parse("1.7976931348623159e+308") === :error + assert Float.parse("9e8363") === :error + end + + test "floor/1" do + float_assert Float.floor(12.524235) === 12.0 + float_assert Float.floor(-12.5) === -13.0 + float_assert Float.floor(-12.524235) === -13.0 + float_assert Float.floor(7.5e3) === 7500.0 + float_assert Float.floor(7.5432e3) === 7543.0 + float_assert Float.floor(7.5e-3) === 0.0 + float_assert Float.floor(-12.32453e4) === -123_246.0 + float_assert Float.floor(-12.32453e-10) === -1.0 + float_assert Float.floor(0.32453e-10) === 0.0 + float_assert Float.floor(-0.32453e-10) === -1.0 + float_assert Float.floor(1.32453e-10) === 0.0 + end + + describe "floor/2" do + test "with 0.0" do + for precision <- 0..15 do + float_assert Float.floor(0.0, precision) === 0.0 + float_assert Float.floor(-0.0, precision) === -0.0 + end + end + + test "floor/2 with precision" do + float_assert Float.floor(12.524235, 0) === 12.0 + float_assert Float.floor(-12.524235, 0) === -13.0 + + float_assert Float.floor(12.52, 2) === 12.51 + float_assert Float.floor(-12.52, 2) === -12.52 + + float_assert Float.floor(12.524235, 2) === 12.52 + float_assert Float.floor(-12.524235, 3) === -12.525 + + float_assert Float.floor(12.32453e-20, 2) === 0.0 + float_assert Float.floor(-12.32453e-20, 2) === -0.01 + + assert_raise ArgumentError, "precision 16 is out of valid range of 0..15", fn -> + Float.floor(1.1, 16) + end + end + + test "with subnormal floats" do + float_assert Float.floor(-5.0e-324, 0) === -1.0 + float_assert Float.floor(-5.0e-324, 1) === -0.1 + float_assert Float.floor(-5.0e-324, 2) === -0.01 + float_assert Float.floor(-5.0e-324, 15) === -0.000000000000001 + + for precision <- 0..15 do + float_assert Float.floor(5.0e-324, precision) === 0.0 + end + end + end + + test "ceil/1" do + float_assert Float.ceil(12.524235) === 13.0 + float_assert Float.ceil(-12.5) === -12.0 + float_assert Float.ceil(-12.524235) === -12.0 + float_assert Float.ceil(7.5e3) === 7500.0 + float_assert Float.ceil(7.5432e3) === 7544.0 + float_assert Float.ceil(7.5e-3) === 1.0 + float_assert Float.ceil(-12.32453e4) === -123_245.0 + float_assert Float.ceil(-12.32453e-10) === -0.0 + float_assert Float.ceil(0.32453e-10) === 1.0 + float_assert Float.ceil(-0.32453e-10) === -0.0 + float_assert Float.ceil(1.32453e-10) === 1.0 + float_assert Float.ceil(0.0) === 0.0 end - test :floor do - assert Float.floor(12) === 12 - assert Float.floor(-12) === -12 - assert Float.floor(12.524235) === 12 - assert Float.floor(-12.5) === -13 - assert Float.floor(-12.524235) === -13 - assert Float.floor(7.5e3) === 7500 - assert Float.floor(7.5432e3) === 7543 - assert Float.floor(7.5e-3) === 0 - assert Float.floor(-12.32453e4) === -123246 - assert Float.floor(-12.32453e-10) === -1 - assert Float.floor(0.32453e-10) === 0 - assert Float.floor(-0.32453e-10) === -1 - assert Float.floor(1.32453e-10) === 0 + describe "ceil/2" do + test "with 0.0" do + for precision <- 0..15 do + float_assert Float.ceil(0.0, precision) === 0.0 + float_assert Float.ceil(-0.0, precision) === -0.0 + end + end + + test "with regular floats" do + float_assert Float.ceil(12.524235, 0) === 13.0 + float_assert Float.ceil(-12.524235, 0) === -12.0 + + float_assert Float.ceil(12.52, 2) === 12.52 + float_assert Float.ceil(-12.52, 2) === -12.51 + + float_assert Float.ceil(12.524235, 2) === 12.53 + float_assert Float.ceil(-12.524235, 3) === -12.524 + + float_assert Float.ceil(12.32453e-20, 2) === 0.01 + float_assert Float.ceil(-12.32453e-20, 2) === -0.0 + + float_assert Float.ceil(0.0, 2) === 0.0 + + assert_raise ArgumentError, "precision 16 is out of valid range of 0..15", fn -> + Float.ceil(1.1, 16) + end + end + + test "with small floats rounded up to -0.0" do + float_assert Float.ceil(-0.1, 0) === -0.0 + float_assert Float.ceil(-0.01, 1) === -0.0 + end + + test "with subnormal floats" do + float_assert Float.ceil(5.0e-324, 0) === 1.0 + float_assert Float.ceil(5.0e-324, 1) === 0.1 + float_assert Float.ceil(5.0e-324, 2) === 0.01 + float_assert Float.ceil(5.0e-324, 15) === 0.000000000000001 + + for precision <- 0..15 do + float_assert Float.ceil(-5.0e-324, precision) === -0.0 + end + end end - test :ceil do - assert Float.ceil(12) === 12 - assert Float.ceil(-12) === -12 - assert Float.ceil(12.524235) === 13 - assert Float.ceil(-12.5) === -12 - assert Float.ceil(-12.524235) === -12 - assert Float.ceil(7.5e3) === 7500 - assert Float.ceil(7.5432e3) === 7544 - assert Float.ceil(7.5e-3) === 1 - assert Float.ceil(-12.32453e4) === -123245 - assert Float.ceil(-12.32453e-10) === 0 - assert Float.ceil(0.32453e-10) === 1 - assert Float.ceil(-0.32453e-10) === 0 - assert Float.ceil(1.32453e-10) === 1 + describe "round/2" do + test "with 0.0" do + for precision <- 0..15 do + float_assert Float.round(0.0, precision) === 0.0 + float_assert Float.round(-0.0, precision) === -0.0 + end + end + + test "with regular floats" do + float_assert Float.round(5.5675, 3) === 5.567 + float_assert Float.round(-5.5674, 3) === -5.567 + float_assert Float.round(5.5, 3) === 5.5 + float_assert Float.round(5.5e-10, 10) === 5.0e-10 + float_assert Float.round(5.5e-10, 8) === 0.0 + float_assert Float.round(5.0, 0) === 5.0 + + assert_raise ArgumentError, "precision 16 is out of valid range of 0..15", fn -> + Float.round(1.1, 16) + end + end + + test "with small floats rounded to +0.0 / -0.0" do + float_assert Float.round(0.01, 0) === 0.0 + float_assert Float.round(0.01, 1) === 0.0 + + float_assert Float.round(-0.01, 0) === -0.0 + float_assert Float.round(-0.01, 1) === -0.0 + + float_assert Float.round(-0.49999, 0) === -0.0 + float_assert Float.round(-0.049999, 1) === -0.0 + end + + test "with subnormal floats" do + for precision <- 0..15 do + float_assert Float.round(5.0e-324, precision) === 0.0 + float_assert Float.round(-5.0e-324, precision) === -0.0 + end + end end - test :round do - assert Float.round(5.5675, 3) === 5.568 - assert Float.round(-5.5674, 3) === -5.567 - assert Float.round(5.5, 3) === 5.5 - assert Float.round(5.5e-10, 10) === 6.0e-10 - assert Float.round(5.5e-10, 8) === 0.0 - assert Float.round(5.0, 0) === 5.0 + describe "ratio/1" do + test "with 0.0" do + assert Float.ratio(0.0) == {0, 1} + end + + test "with regular floats" do + assert Float.ratio(3.14) == {7_070_651_414_971_679, 2_251_799_813_685_248} + assert Float.ratio(-3.14) == {-7_070_651_414_971_679, 2_251_799_813_685_248} + assert Float.ratio(1.5) == {3, 2} + end + + test "with subnormal floats" do + assert Float.ratio(5.0e-324) == + {1, + 202_402_253_307_310_618_352_495_346_718_917_307_049_556_649_764_142_118_356_901_358_027_430_339_567_995_346_891_960_383_701_437_124_495_187_077_864_316_811_911_389_808_737_385_793_476_867_013_399_940_738_509_921_517_424_276_566_361_364_466_907_742_093_216_341_239_767_678_472_745_068_562_007_483_424_692_698_618_103_355_649_159_556_340_810_056_512_358_769_552_333_414_615_230_502_532_186_327_508_646_006_263_307_707_741_093_494_784} + + assert Float.ratio(1.0e-323) == + {1, + 101_201_126_653_655_309_176_247_673_359_458_653_524_778_324_882_071_059_178_450_679_013_715_169_783_997_673_445_980_191_850_718_562_247_593_538_932_158_405_955_694_904_368_692_896_738_433_506_699_970_369_254_960_758_712_138_283_180_682_233_453_871_046_608_170_619_883_839_236_372_534_281_003_741_712_346_349_309_051_677_824_579_778_170_405_028_256_179_384_776_166_707_307_615_251_266_093_163_754_323_003_131_653_853_870_546_747_392} + + assert Float.ratio(2.225073858507201e-308) == + {4_503_599_627_370_495, + 202_402_253_307_310_618_352_495_346_718_917_307_049_556_649_764_142_118_356_901_358_027_430_339_567_995_346_891_960_383_701_437_124_495_187_077_864_316_811_911_389_808_737_385_793_476_867_013_399_940_738_509_921_517_424_276_566_361_364_466_907_742_093_216_341_239_767_678_472_745_068_562_007_483_424_692_698_618_103_355_649_159_556_340_810_056_512_358_769_552_333_414_615_230_502_532_186_327_508_646_006_263_307_707_741_093_494_784} + end end end diff --git a/lib/elixir/test/elixir/function_test.exs b/lib/elixir/test/elixir/function_test.exs new file mode 100644 index 00000000000..f99a25f7155 --- /dev/null +++ b/lib/elixir/test/elixir/function_test.exs @@ -0,0 +1,87 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("test_helper.exs", __DIR__) + +defmodule DummyFunction do + def function_with_arity_0 do + true + end + + def zero?(0), do: true + def zero?(_), do: false +end + +defmodule FunctionTest do + use ExUnit.Case, async: true + + doctest Function + import Function + + @information_keys_for_named [:type, :module, :arity, :name, :env] + @information_keys_for_anonymous @information_keys_for_named ++ + [:pid, :index, :new_index, :new_uniq, :uniq] + + describe "capture/3" do + test "captures module functions with arity 0" do + f = capture(DummyFunction, :function_with_arity_0, 0) + + assert is_function(f) + end + + test "captures module functions with any arity" do + f = capture(DummyFunction, :zero?, 1) + + assert is_function(f) + assert f.(0) + end + end + + describe "info/1" do + test "returns info for named captured functions" do + f = &DummyFunction.zero?/1 + expected = [module: DummyFunction, name: :zero?, arity: 1, env: [], type: :external] + + result = info(f) + + assert expected == result + end + + test "returns info for anonymous functions" do + f = fn x -> x end + + result = info(f) + + for {key, _value} <- result do + assert key in @information_keys_for_anonymous + end + end + end + + describe "info/2" do + test "returns info for every possible information key for named functions" do + f = &DummyFunction.zero?/1 + + for x <- @information_keys_for_named do + assert {^x, _} = info(f, x) + end + end + + test "returns info for every possible information key for anonymous functions" do + f = &DummyFunction.zero?/1 + + for x <- @information_keys_for_anonymous do + assert {^x, _} = info(f, x) + end + + assert {:arity, 1} = info(f, :arity) + end + end + + describe "identity/1" do + test "returns whatever it gets passed" do + assert :hello = Function.identity(:hello) + end + end +end diff --git a/lib/elixir/test/elixir/gen_event_test.exs b/lib/elixir/test/elixir/gen_event_test.exs deleted file mode 100644 index 005cb715b21..00000000000 --- a/lib/elixir/test/elixir/gen_event_test.exs +++ /dev/null @@ -1,392 +0,0 @@ -Code.require_file "test_helper.exs", __DIR__ - -defmodule GenEventTest do - use ExUnit.Case, async: true - - defmodule LoggerHandler do - use GenEvent - - def handle_event({:log, x}, messages) do - {:ok, [x|messages]} - end - - def handle_call(:messages, messages) do - {:ok, Enum.reverse(messages), []} - end - - def handle_call(call, state) do - super(call, state) - end - end - - defmodule SlowHandler do - use GenEvent - - def handle_event(_event, _state) do - :timer.sleep(100) - :remove_handler - end - end - - @receive_timeout 1000 - - test "start_link/2 and handler workflow" do - {:ok, pid} = GenEvent.start_link() - - {:links, links} = Process.info(self, :links) - assert pid in links - - assert GenEvent.notify(pid, {:log, 0}) == :ok - assert GenEvent.add_handler(pid, LoggerHandler, []) == :ok - assert GenEvent.notify(pid, {:log, 1}) == :ok - assert GenEvent.notify(pid, {:log, 2}) == :ok - - assert GenEvent.call(pid, LoggerHandler, :messages) == [1, 2] - assert GenEvent.call(pid, LoggerHandler, :messages) == [] - - assert GenEvent.call(pid, LoggerHandler, :whatever) == {:error, :bad_call} - assert GenEvent.call(pid, UnknownHandler, :messages) == {:error, :bad_module} - - assert GenEvent.remove_handler(pid, LoggerHandler, []) == :ok - assert GenEvent.stop(pid) == :ok - end - - test "start/2 with linked handler" do - {:ok, pid} = GenEvent.start() - - {:links, links} = Process.info(self, :links) - refute pid in links - - assert GenEvent.add_handler(pid, LoggerHandler, [], link: true) == :ok - - {:links, links} = Process.info(self, :links) - assert pid in links - - assert GenEvent.notify(pid, {:log, 1}) == :ok - assert GenEvent.sync_notify(pid, {:log, 2}) == :ok - - assert GenEvent.call(pid, LoggerHandler, :messages) == [1, 2] - assert GenEvent.stop(pid) == :ok - end - - test "start/2 with linked swap" do - {:ok, pid} = GenEvent.start() - - assert GenEvent.add_handler(pid, LoggerHandler, []) == :ok - - {:links, links} = Process.info(self, :links) - refute pid in links - - assert GenEvent.swap_handler(pid, LoggerHandler, [], LoggerHandler, [], link: true) == :ok - - {:links, links} = Process.info(self, :links) - assert pid in links - - assert GenEvent.stop(pid) == :ok - end - - test "start/2 with registered name" do - {:ok, _} = GenEvent.start(name: :logger) - assert GenEvent.stop(:logger) == :ok - end - - test "sync stream/2" do - {:ok, pid} = GenEvent.start_link() - parent = self() - - spawn_link fn -> - send parent, Enum.take(GenEvent.stream(pid, mode: :sync), 3) - end - - wait_for_handlers(pid, 1) - - for i <- 1..3 do - GenEvent.sync_notify(pid, i) - end - - # Receive one of the results - assert_receive [1, 2, 3], @receive_timeout - wait_for_handlers(pid, 0) - - spawn_link fn -> - Enum.each(GenEvent.stream(pid, mode: :sync), fn _ -> - :timer.sleep(:infinity) - end) - end - - wait_for_handlers(pid, 1) - - for i <- 1..6 do - GenEvent.notify(pid, i) - end - - wait_for_queue_length(pid, 5) - end - - test "async stream/2" do - {:ok, pid} = GenEvent.start_link() - parent = self() - - spawn_link fn -> - Enum.each(GenEvent.stream(pid, mode: :async), fn _ -> - :timer.sleep(:infinity) - end) - end - - spawn_link fn -> - send parent, Enum.take(GenEvent.stream(pid, mode: :async), 3) - end - - wait_for_handlers(pid, 2) - - for i <- 1..3 do - GenEvent.sync_notify(pid, i) - end - - # Receive one of the results - assert_receive [1, 2, 3], @receive_timeout - - # One of the subscriptions are gone - wait_for_handlers(pid, 1) - end - - Enum.each [:sync, :async], fn mode -> - test "#{mode} stream/2 with parallel use (and first finishing first)" do - {:ok, pid} = GenEvent.start_link() - stream = GenEvent.stream(pid, duration: 200, mode: unquote(mode)) - - parent = self() - spawn_link fn -> send parent, {:take, Enum.take(stream, 3)} end - wait_for_handlers(pid, 1) - spawn_link fn -> send parent, {:to_list, Enum.to_list(stream)} end - wait_for_handlers(pid, 2) - - # Notify the events for both handlers - for i <- 1..3 do - GenEvent.sync_notify(pid, i) - end - assert_receive {:take, [1, 2, 3]}, @receive_timeout - - # Notify the events for to_list stream handler - for i <- 4..5 do - GenEvent.sync_notify(pid, i) - end - - assert_receive {:to_list, [1, 2, 3, 4, 5]}, @receive_timeout - end - - test "#{mode} stream/2 with timeout" do - # Start a manager - {:ok, pid} = GenEvent.start_link() - Process.flag(:trap_exit, true) - - pid = spawn_link fn -> - Enum.take(GenEvent.stream(pid, timeout: 50, mode: unquote(mode)), 5) - end - - assert_receive {:EXIT, ^pid, - {:timeout, {Enumerable.GenEvent, :next, [_, _]}}}, @receive_timeout - end - - test "#{mode} stream/2 with error/timeout on subscription" do - # Start a manager - {:ok, pid} = GenEvent.start_link() - - # Start a subscriber with timeout - child = spawn fn -> Enum.to_list(GenEvent.stream(pid, mode: unquote(mode))) end - wait_for_handlers(pid, 1) - - # Kill and wait until we have 0 handlers - Process.exit(child, :kill) - wait_for_handlers(pid, 0) - GenEvent.stop(pid) - end - - test "#{mode} stream/2 with manager stop" do - # Start a manager and subscribers - {:ok, pid} = GenEvent.start_link() - - parent = self() - stream_pid = spawn_link fn -> - send parent, Enum.take(GenEvent.stream(pid, mode: unquote(mode)), 5) - end - wait_for_handlers(pid, 1) - - # Notify the events - for i <- 1..3 do - GenEvent.sync_notify(pid, i) - end - - Process.flag(:trap_exit, true) - GenEvent.stop(pid) - assert_receive {:EXIT, ^stream_pid, - {:shutdown, {Enumerable.GenEvent, :next, [_, _]}}}, @receive_timeout - end - - test "#{mode} stream/2 with cancel streams" do - # Start a manager and subscribers - {:ok, pid} = GenEvent.start_link() - stream = GenEvent.stream(pid, id: make_ref(), mode: unquote(mode)) - - parent = self() - spawn_link fn -> send parent, Enum.take(stream, 5) end - wait_for_handlers(pid, 1) - - # Notify the events - for i <- 1..3 do - GenEvent.sync_notify(pid, i) - end - - GenEvent.cancel_streams(stream) - assert_receive [1, 2, 3], @receive_timeout - GenEvent.stop(pid) - end - - test "#{mode} stream/2 with swap_handler" do - # Start a manager and subscribers - {:ok, pid} = GenEvent.start_link() - stream = GenEvent.stream(pid, id: make_ref(), mode: unquote(mode)) - - parent = self() - stream_pid = spawn_link fn -> send parent, Enum.take(stream, 5) end - wait_for_handlers(pid, 1) - - # Notify the events - for i <- 1..3 do - GenEvent.sync_notify(pid, i) - end - - [handler] = GenEvent.which_handlers(pid) - Process.flag(:trap_exit, true) - GenEvent.swap_handler(pid, handler, :swap_handler, LogHandler, []) - assert_receive {:EXIT, ^stream_pid, - {{:swapped, LogHandler, _}, - {Enumerable.GenEvent, :next, [_, _]}}}, @receive_timeout - end - - test "#{mode} stream/2 with duration" do - # Start a manager and subscribers - {:ok, pid} = GenEvent.start_link() - stream = GenEvent.stream(pid, duration: 200, mode: unquote(mode)) - - parent = self() - spawn_link fn -> send parent, {:duration, Enum.take(stream, 10)} end - wait_for_handlers(pid, 1) - - # Notify the events - for i <- 1..5 do - GenEvent.sync_notify(pid, i) - end - - # Wait until the handler is gone - wait_for_handlers(pid, 0) - - # The stream is not complete but terminated anyway due to duration - assert_receive {:duration, [1, 2, 3, 4, 5]}, @receive_timeout - - GenEvent.stop(pid) - end - - test "#{mode} stream/2 with manager killed and trap_exit" do - # Start a manager and subscribers - {:ok, pid} = GenEvent.start_link() - stream = GenEvent.stream(pid, mode: unquote(mode)) - - parent = self() - stream_pid = spawn_link fn -> - send parent, Enum.to_list(stream) - end - wait_for_handlers(pid, 1) - - Process.flag(:trap_exit, true) - Process.exit(pid, :kill) - assert_receive {:EXIT, ^pid, :killed}, @receive_timeout - assert_receive {:EXIT, ^stream_pid, - {:killed, {Enumerable.GenEvent, :next, [_,_]}}}, @receive_timeout - end - - test "#{mode} stream/2 with manager not alive" do - # Start a manager and subscribers - stream = GenEvent.stream(:does_not_exit, mode: unquote(mode)) - - parent = self() - stream_pid = spawn_link fn -> - send parent, Enum.to_list(stream) - end - - Process.flag(:trap_exit, true) - assert_receive {:EXIT, ^stream_pid, - {:noproc, {Enumerable.GenEvent, :start, [_]}}}, @receive_timeout - end - - test "#{mode} stream/2 with manager unregistered" do - # Start a manager and subscribers - {:ok, pid} = GenEvent.start_link(name: :unreg) - stream = GenEvent.stream(:unreg, mode: unquote(mode)) - - parent = self() - spawn_link fn -> - send parent, Enum.take(stream, 5) - end - wait_for_handlers(pid, 1) - - # Notify the events - for i <- 1..3 do - GenEvent.sync_notify(pid, i) - end - - # Unregister the process - Process.unregister(:unreg) - - # Notify the remaining events - for i <- 4..5 do - GenEvent.sync_notify(pid, i) - end - - # We should have gotten the message and all handlers were removed - assert_receive [1, 2, 3, 4, 5], @receive_timeout - wait_for_handlers(pid, 0) - end - - test "#{mode} stream/2 flushes events on abort" do - # Start a manager and subscribers - {:ok, pid} = GenEvent.start_link() - - spawn_link fn -> - wait_for_handlers(pid, 2) - GenEvent.notify(pid, 1) - GenEvent.notify(pid, 2) - GenEvent.notify(pid, 3) - end - - GenEvent.add_handler(pid, SlowHandler, []) - stream = GenEvent.stream(pid, mode: unquote(mode)) - - try do - Enum.each stream, fn _ -> throw :done end - catch - :done -> :ok - end - - # Wait for the slow handler to be removed - # so all events have been handled - wait_for_handlers(pid, 0) - - # Check no messages leaked. - refute_received _any - end - end - - defp wait_for_handlers(pid, count) do - unless length(GenEvent.which_handlers(pid)) == count do - wait_for_handlers(pid, count) - end - end - - defp wait_for_queue_length(pid, count) do - {:message_queue_len, n} = Process.info(pid, :message_queue_len) - unless n == count do - wait_for_queue_length(pid, count) - end - end -end diff --git a/lib/elixir/test/elixir/gen_server_test.exs b/lib/elixir/test/elixir/gen_server_test.exs index 880d7d0f864..31ef3dbf802 100644 --- a/lib/elixir/test/elixir/gen_server_test.exs +++ b/lib/elixir/test/elixir/gen_server_test.exs @@ -1,4 +1,8 @@ -Code.require_file "test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("test_helper.exs", __DIR__) defmodule GenServerTest do use ExUnit.Case, async: true @@ -6,49 +10,143 @@ defmodule GenServerTest do defmodule Stack do use GenServer - def handle_call(:stop, from, state) do - GenServer.reply(from, :ok) - {:stop, :normal, state} + def init(args) do + {:ok, args} end - def handle_call(:pop, _from, [h|t]) do + def handle_call(:pop, _from, [h | t]) do {:reply, h, t} end - def handle_call(request, from, state) do - super(request, from, state) + def handle_call(:noreply, _from, h) do + {:noreply, h} end - def handle_cast({:push, item}, state) do - {:noreply, [item|state]} + def handle_call(:stop_self, _from, state) do + reason = catch_exit(GenServer.stop(self())) + {:reply, reason, state} end - def handle_cast(request, state) do - super(request, state) + def handle_cast({:push, element}, state) do + {:noreply, [element | state]} end def terminate(_reason, _state) do # There is a race condition if the agent is # restarted too fast and it is registered. try do - self |> Process.info(:registered_name) |> elem(1) |> Process.unregister + self() |> Process.info(:registered_name) |> elem(1) |> Process.unregister() rescue _ -> :ok end + :ok end end + test "generates child_spec/1" do + assert Stack.child_spec([:hello]) == %{ + id: Stack, + start: {Stack, :start_link, [[:hello]]} + } + + defmodule CustomStack do + use GenServer, id: :id, restart: :temporary, shutdown: :infinity, start: {:foo, :bar, []} + + def init(args) do + {:ok, args} + end + end + + assert CustomStack.child_spec([:hello]) == %{ + id: :id, + restart: :temporary, + shutdown: :infinity, + start: {:foo, :bar, []} + } + end + + test "start_link/3" do + assert_raise ArgumentError, ~r"expected :name option to be one of the following:", fn -> + GenServer.start_link(Stack, [:hello], name: "my_gen_server_name") + end + + assert_raise ArgumentError, ~r"expected :name option to be one of the following:", fn -> + GenServer.start_link(Stack, [:hello], name: {:invalid_tuple, "my_gen_server_name"}) + end + + assert_raise ArgumentError, ~r"expected :name option to be one of the following:", fn -> + GenServer.start_link(Stack, [:hello], name: {:via, "Via", "my_gen_server_name"}) + end + + assert_raise ArgumentError, ~r/Got: "my_gen_server_name"/, fn -> + GenServer.start_link(Stack, [:hello], name: "my_gen_server_name") + end + end + + test "start_link/3 with via" do + GenServer.start_link(Stack, [:hello], name: {:via, :global, :via_stack}) + assert GenServer.call({:via, :global, :via_stack}, :pop) == :hello + end + + test "start_link/3 with global" do + GenServer.start_link(Stack, [:hello], name: {:global, :global_stack}) + assert GenServer.call({:global, :global_stack}, :pop) == :hello + end + + test "start_link/3 with local" do + GenServer.start_link(Stack, [:hello], name: :stack) + assert GenServer.call(:stack, :pop) == :hello + end + test "start_link/2, call/2 and cast/2" do {:ok, pid} = GenServer.start_link(Stack, [:hello]) - {:links, links} = Process.info(self, :links) + {:links, links} = Process.info(self(), :links) assert pid in links assert GenServer.call(pid, :pop) == :hello assert GenServer.cast(pid, {:push, :world}) == :ok assert GenServer.call(pid, :pop) == :world - assert GenServer.call(pid, :stop) == :ok + assert GenServer.stop(pid) == :ok + + assert GenServer.cast({:global, :foo}, {:push, :world}) == :ok + assert GenServer.cast({:via, :foo, :bar}, {:push, :world}) == :ok + assert GenServer.cast(:foo, {:push, :world}) == :ok + end + + @tag capture_log: true + test "call/3 exit messages" do + name = :self + Process.register(self(), name) + :global.register_name(name, self()) + {:ok, pid} = GenServer.start_link(Stack, [:hello]) + {:ok, stopped_pid} = GenServer.start(Stack, [:hello]) + GenServer.stop(stopped_pid) + + assert catch_exit(GenServer.call(name, :pop, 5000)) == + {:calling_self, {GenServer, :call, [name, :pop, 5000]}} + + assert catch_exit(GenServer.call({:global, name}, :pop, 5000)) == + {:calling_self, {GenServer, :call, [{:global, name}, :pop, 5000]}} + + assert catch_exit(GenServer.call({:via, :global, name}, :pop, 5000)) == + {:calling_self, {GenServer, :call, [{:via, :global, name}, :pop, 5000]}} + + assert catch_exit(GenServer.call(self(), :pop, 5000)) == + {:calling_self, {GenServer, :call, [self(), :pop, 5000]}} + + assert catch_exit(GenServer.call(pid, :noreply, 1)) == + {:timeout, {GenServer, :call, [pid, :noreply, 1]}} + + assert catch_exit(GenServer.call(nil, :pop, 5000)) == + {:noproc, {GenServer, :call, [nil, :pop, 5000]}} + + assert catch_exit(GenServer.call(stopped_pid, :pop, 5000)) == + {:noproc, {GenServer, :call, [stopped_pid, :pop, 5000]}} + + assert catch_exit(GenServer.call({:stack, :bogus_node}, :pop, 5000)) == + {{:nodedown, :bogus_node}, {GenServer, :call, [{:stack, :bogus_node}, :pop, 5000]}} end test "nil name" do @@ -58,31 +156,64 @@ defmodule GenServerTest do test "start/2" do {:ok, pid} = GenServer.start(Stack, [:hello]) - {:links, links} = Process.info(self, :links) + {:links, links} = Process.info(self(), :links) refute pid in links - GenServer.call(pid, :stop) + GenServer.stop(pid) end - test "abcast/3" do - {:ok, _} = GenServer.start_link(Stack, [], name: :stack) + test "abcast/3", %{test: name} do + {:ok, _} = GenServer.start_link(Stack, [], name: name) - assert GenServer.abcast(:stack, {:push, :hello}) == :abcast - assert GenServer.call({:stack, node()}, :pop) == :hello + assert GenServer.abcast(name, {:push, :hello}) == :abcast + assert GenServer.call({name, node()}, :pop) == :hello - assert GenServer.abcast([node, :foo@bar], :stack, {:push, :world}) == :abcast - assert GenServer.call(:stack, :pop) == :world + assert GenServer.abcast([node(), :foo@bar], name, {:push, :world}) == :abcast + assert GenServer.call(name, :pop) == :world + end + + test "multi_call/4", %{test: name} do + {:ok, _} = GenServer.start_link(Stack, [:hello, :world], name: name) + node = node() - GenServer.call(:stack, :stop) + assert {[{^node, :hello}], _} = GenServer.multi_call(name, :pop) + assert {[{^node, :world}], [:foo@bar]} = GenServer.multi_call([node(), :foo@bar], name, :pop) end - test "multi_call/4" do - {:ok, _} = GenServer.start_link(Stack, [:hello, :world], name: :stack) + test "whereis/1" do + name = :whereis_server + + {:ok, pid} = GenServer.start_link(Stack, [], name: name) + assert GenServer.whereis(name) == pid + assert GenServer.whereis({name, node()}) == pid + assert GenServer.whereis({name, :another_node}) == {name, :another_node} + assert GenServer.whereis(pid) == pid + assert GenServer.whereis(:whereis_bad_server) == nil + + {:ok, pid} = GenServer.start_link(Stack, [], name: {:global, name}) + assert GenServer.whereis({:global, name}) == pid + assert GenServer.whereis({:global, :whereis_bad_server}) == nil + assert GenServer.whereis({:via, :global, name}) == pid + assert GenServer.whereis({:via, :global, :whereis_bad_server}) == nil + end + + test "stop/3", %{test: name} do + {:ok, pid} = GenServer.start(Stack, []) + assert GenServer.stop(pid, :normal) == :ok + + stopped_pid = pid + + assert catch_exit(GenServer.stop(stopped_pid)) == + {:noproc, {GenServer, :stop, [stopped_pid, :normal, :infinity]}} + + assert catch_exit(GenServer.stop(nil)) == + {:noproc, {GenServer, :stop, [nil, :normal, :infinity]}} + + {:ok, pid} = GenServer.start(Stack, []) - assert GenServer.multi_call(:stack, :pop) == - {[{node(), :hello}], []} - assert GenServer.multi_call([node, :foo@bar], :stack, :pop) == - {[{node, :world}], [:foo@bar]} + assert GenServer.call(pid, :stop_self) == + {:calling_self, {GenServer, :stop, [pid, :normal, :infinity]}} - GenServer.call(:stack, :stop) + {:ok, _} = GenServer.start(Stack, [], name: name) + assert GenServer.stop(name, :normal) == :ok end end diff --git a/lib/elixir/test/elixir/hash_dict_test.exs b/lib/elixir/test/elixir/hash_dict_test.exs deleted file mode 100644 index 3a64f3e818b..00000000000 --- a/lib/elixir/test/elixir/hash_dict_test.exs +++ /dev/null @@ -1,91 +0,0 @@ -Code.require_file "test_helper.exs", __DIR__ - -defmodule HashDictTest do - use ExUnit.Case, async: true - - @dict Enum.into([foo: :bar], HashDict.new) - - test "access" do - dict = Enum.into([foo: :baz], HashDict.new) - assert Access.get(@dict, :foo) == :bar - assert Access.get_and_update(@dict, :foo, fn :bar -> {:ok, :baz} end) == {:ok, dict} - assert Access.get_and_update(HashDict.new, :foo, fn nil -> {:ok, :baz} end) == {:ok, dict} - end - - test "is serializable as attribute" do - assert @dict == Enum.into([foo: :bar], HashDict.new) - end - - test "is accessible as attribute" do - assert @dict[:foo] == :bar - end - - test "small dict smoke test" do - smoke_test(1..8) - smoke_test(8..1) - end - - test "medium dict smoke test" do - smoke_test(1..80) - smoke_test(80..1) - end - - test "large dict smoke test" do - smoke_test(1..1200) - smoke_test(1200..1) - end - - test "reduce/3 (via to_list)" do - dict = filled_dict(8) - list = dict |> HashDict.to_list - assert length(list) == 8 - assert {1, 1} in list - assert list == Enum.to_list(dict) - - dict = filled_dict(20) - list = dict |> HashDict.to_list - assert length(list) == 20 - assert {1, 1} in list - assert list == Enum.to_list(dict) - - dict = filled_dict(120) - list = dict |> HashDict.to_list - assert length(list) == 120 - assert {1, 1} in list - assert list == Enum.to_list(dict) - end - - test "comparison when subsets" do - d1 = Enum.into [a: 0], HashDict.new - d2 = Enum.into [a: 0, b: 1], HashDict.new - - refute HashDict.equal?(d1, d2) - refute HashDict.equal?(d2, d1) - end - - defp smoke_test(range) do - {dict, _} = Enum.reduce range, {HashDict.new, 1}, fn(x, {acc, i}) -> - acc = HashDict.put(acc, x, x) - assert HashDict.size(acc) == i - {acc, i + 1} - end - - Enum.each range, fn(x) -> - assert HashDict.get(dict, x) == x - end - - {dict, _} = Enum.reduce range, {dict, Enum.count(range)}, fn(x, {acc, i}) -> - assert HashDict.size(acc) == i - acc = HashDict.delete(acc, x) - assert HashDict.size(acc) == i - 1 - assert HashDict.get(acc, x) == nil - {acc, i - 1} - end - - assert dict == HashDict.new - end - - defp filled_dict(range) do - Enum.reduce 1..range, HashDict.new, &HashDict.put(&2, &1, &1) - end -end diff --git a/lib/elixir/test/elixir/hash_set_test.exs b/lib/elixir/test/elixir/hash_set_test.exs deleted file mode 100644 index cb231a1a2a6..00000000000 --- a/lib/elixir/test/elixir/hash_set_test.exs +++ /dev/null @@ -1,55 +0,0 @@ -Code.require_file "test_helper.exs", __DIR__ - -defmodule HashSetTest do - use ExUnit.Case, async: true - - test "union" do - assert HashSet.union(filled_set(21), filled_set(22)) == filled_set(22) - assert HashSet.union(filled_set(121), filled_set(120)) == filled_set(121) - end - - test "intersection" do - assert HashSet.intersection(filled_set(21), filled_set(20)) == filled_set(20) - assert HashSet.equal?(HashSet.intersection(filled_set(120), filled_set(121)), filled_set(120)) - end - - test "difference" do - assert HashSet.equal?(HashSet.difference(filled_set(20), filled_set(21)), HashSet.new) - - diff = HashSet.difference(filled_set(9000), filled_set(9000)) - assert HashSet.equal?(diff, HashSet.new) - assert HashSet.size(diff) == 0 - end - - test "subset?" do - assert HashSet.subset?(HashSet.new, HashSet.new) - assert HashSet.subset?(filled_set(6), filled_set(10)) - assert HashSet.subset?(filled_set(6), filled_set(120)) - refute HashSet.subset?(filled_set(120), filled_set(6)) - end - - test "equal?" do - assert HashSet.equal?(HashSet.new, HashSet.new) - assert HashSet.equal?(filled_set(20), HashSet.delete(filled_set(21), 21)) - assert HashSet.equal?(filled_set(120), filled_set(120)) - end - - test "to_list" do - set = filled_set(20) - list = HashSet.to_list(set) - assert length(list) == 20 - assert 1 in list - assert Enum.sort(list) == Enum.sort(1..20) - - set = filled_set(120) - list = HashSet.to_list(set) - assert length(list) == 120 - assert 1 in list - assert Enum.sort(list) == Enum.sort(1..120) - end - - defp filled_set(range) do - Enum.into 1..range, HashSet.new - end -end - diff --git a/lib/elixir/test/elixir/inspect/algebra_test.exs b/lib/elixir/test/elixir/inspect/algebra_test.exs index 3b4b46a5263..f9ca94b3dc8 100644 --- a/lib/elixir/test/elixir/inspect/algebra_test.exs +++ b/lib/elixir/test/elixir/inspect/algebra_test.exs @@ -1,125 +1,326 @@ -Code.require_file "../test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Inspect.OptsTest do + use ExUnit.Case + + test "new" do + opts = Inspect.Opts.new(limit: 13, pretty: true) + assert opts.limit == 13 + assert opts.pretty + end + + test "default_inspect_fun" do + assert Inspect.Opts.default_inspect_fun() == (&Inspect.inspect/2) + + assert Inspect.Opts.default_inspect_fun(fn + :rewrite_atom, _ -> "rewritten_atom" + value, opts -> Inspect.inspect(value, opts) + end) == :ok + + assert inspect(:rewrite_atom) == "rewritten_atom" + after + Inspect.Opts.default_inspect_fun(&Inspect.inspect/2) + end +end defmodule Inspect.AlgebraTest do use ExUnit.Case, async: true + doctest Inspect.Algebra + import Inspect.Algebra - def helloabcd do - concat( - glue( - glue( - glue("hello", "a"), - "b"), - "c"), - "d") + defp render(doc, limit) do + doc |> group() |> format(limit) |> IO.iodata_to_binary() end - def factor(doc, w), do: format(w, 0, [{0, :flat, group(doc)}]) - test "empty doc" do - # Consistence with definitions - assert empty == :doc_nil - - # Consistence of corresponding sdoc - assert factor(empty, 80) == [] + # Consistent with definitions + assert empty() == [] # Consistent formatting - assert pretty(empty, 80) == "" + assert render(empty(), 80) == "" end - test "break doc" do - # Consistence with definitions - assert break("break") == {:doc_break, "break"} - assert break("") == {:doc_break, ""} + test "strict break doc" do + # Consistent with definitions + assert break("break") == {:doc_break, "break", :strict} + assert break("") == {:doc_break, "", :strict} # Wrong argument type assert_raise FunctionClauseError, fn -> break(42) end - # Consistence of corresponding sdoc - assert factor(break("_"), 80) == ["_"] + # Consistent formatting + assert render(break("_"), 80) == "_" + assert render(glue("foo", " ", glue("bar", " ", "baz")), 10) == "foo\nbar\nbaz" + end + + test "flex break doc" do + # Consistent with definitions + assert flex_break("break") == {:doc_break, "break", :flex} + assert flex_break("") == {:doc_break, "", :flex} + + # Wrong argument type + assert_raise FunctionClauseError, fn -> flex_break(42) end # Consistent formatting - assert pretty(break("_"), 80) == "_" + assert render(flex_break("_"), 80) == "_" + assert render(flex_glue("foo", " ", flex_glue("bar", " ", "baz")), 10) == "foo bar\nbaz" end test "glue doc" do - # Consistence with definitions - assert glue("a", "->", "b") == {:doc_cons, - "a", {:doc_cons, {:doc_break, "->"}, "b"} - } + # Consistent with definitions + assert glue("a", "->", "b") == ["a", {:doc_break, "->", :strict} | "b"] assert glue("a", "b") == glue("a", " ", "b") # Wrong argument type assert_raise FunctionClauseError, fn -> glue("a", 42, "b") end end - test "text doc" do - # Consistence of corresponding sdoc - assert factor("_", 80) == ["_"] + test "flex glue doc" do + # Consistent with definitions + assert flex_glue("a", "->", "b") == + ["a", {:doc_break, "->", :flex} | "b"] - # Consistent formatting - assert pretty("_", 80) == "_" + assert flex_glue("a", "b") == flex_glue("a", " ", "b") + + # Wrong argument type + assert_raise FunctionClauseError, fn -> flex_glue("a", 42, "b") end + end + + test "binary doc" do + assert render("_", 80) == "_" + end + + test "string doc" do + # Consistent with definitions + assert string("ólá") == {:doc_string, "ólá", 3} + + # Counts graphemes + doc = glue(string("olá"), " ", string("mundo")) + assert render(doc, 9) == "olá mundo" end test "space doc" do - # Consistency with definitions - assert space("a", "b") == {:doc_cons, - "a", {:doc_cons, " ", "b"} - } + # Consistent with definitions + assert space("a", "b") == ["a", " " | "b"] + end + + test "always nest doc" do + # Consistent with definitions + assert nest(empty(), 1) == {:doc_nest, empty(), 1, :always} + assert nest(empty(), 0) == [] + + # Wrong argument type + assert_raise FunctionClauseError, fn -> nest("foo", empty()) end + + # Consistent formatting + assert render(nest("a", 1), 80) == "a" + assert render(nest(glue("a", "b"), 1), 2) == "a\n b" + assert render(nest(line("a", "b"), 1), 20) == "a\n b" end - test "nest doc" do - # Consistence with definitions - assert nest(empty, 1) == {:doc_nest, empty, 1} - assert nest(empty, 0) == :doc_nil + test "break nest doc" do + # Consistent with definitions + assert nest(empty(), 1, :break) == {:doc_nest, empty(), 1, :break} + assert nest(empty(), 0, :break) == [] # Wrong argument type - assert_raise FunctionClauseError, fn -> nest("foo", empty) end + assert_raise FunctionClauseError, fn -> nest("foo", empty(), :break) end - # Consistence of corresponding sdoc - assert factor(nest("a", 1), 80) == ["a"] - assert format(2, 0, [{0, :break, nest(glue("a", "b"), 1)}]) == ["a", "\n ", "b"] + # Consistent formatting + assert render(nest("a", 1, :break), 80) == "a" + assert render(nest(glue("a", "b"), 1, :break), 2) == "a\n b" + assert render(nest(line("a", "b"), 1, :break), 20) == "a\nb" + end + + test "cursor nest doc" do + # Consistent with definitions + assert nest(empty(), :cursor) == {:doc_nest, empty(), :cursor, :always} # Consistent formatting - assert pretty(nest("a", 1), 80) == "a" - assert render(format 2, 0, [{0, :break, nest(glue("a", "b"), 1)}]) == "a\n b" + assert render(nest("a", :cursor), 80) == "a" + assert render(concat("prefix ", nest(glue("a", "b"), :cursor)), 2) == "prefix a\n b" + assert render(concat("prefix ", nest(line("a", "b"), :cursor)), 2) == "prefix a\n b" end - test "line doc" do - # Consistency with definitions - assert line("a", "b") == - {:doc_cons, "a", {:doc_cons, :doc_line, "b"}} + test "reset nest doc" do + # Consistent with definitions + assert nest(empty(), :reset) == {:doc_nest, empty(), :reset, :always} + + # Consistent formatting + assert render(nest("a", :reset), 80) == "a" + assert render(nest(nest(glue("a", "b"), :reset), 10), 2) == "a\nb" + assert render(nest(nest(line("a", "b"), :reset), 10), 2) == "a\nb" + end + + test "color doc" do + # Consistent with definitions + opts = %Inspect.Opts{} + assert color_doc(empty(), :atom, opts) == empty() + + opts = %Inspect.Opts{syntax_colors: [regex: :red]} + assert color_doc(empty(), :atom, opts) == empty() - # Consistence of corresponding sdoc - assert factor(line("a", "b"), 1) == ["a", "\n", "b"] - assert factor(line("a", "b"), 9) == ["a", "\n", "b"] + opts = %Inspect.Opts{syntax_colors: [atom: :red]} + doc1 = {:doc_color, "Hi", IO.ANSI.red()} + doc2 = {:doc_color, empty(), IO.ANSI.reset()} + assert color_doc("Hi", :atom, opts) == concat(doc1, doc2) + + opts = %Inspect.Opts{syntax_colors: [reset: :red]} + assert color_doc(empty(), :atom, opts) == empty() + + opts = %Inspect.Opts{syntax_colors: [number: :cyan, reset: :red]} + doc1 = {:doc_color, "123", IO.ANSI.cyan()} + doc2 = {:doc_color, empty(), IO.ANSI.red()} + assert color_doc("123", :number, opts) == concat(doc1, doc2) + + # Consistent formatting + opts = %Inspect.Opts{syntax_colors: [atom: :cyan]} + assert render(glue(color_doc("AA", :atom, opts), "BB"), 5) == "\e[36mAA\e[0m BB" + assert render(glue(color_doc("AA", :atom, opts), "BB"), 3) == "\e[36mAA\e[0m\nBB" + assert render(glue("AA", color_doc("BB", :atom, opts)), 6) == "AA \e[36mBB\e[0m" + end + + test "line doc" do + # Consistent with definitions + assert line("a", "b") == ["a", :doc_line | "b"] # Consistent formatting - assert pretty(line(glue("aaa", "bbb"), glue("ccc", "ddd")), 10) == - "aaa bbb\nccc ddd" + assert render(line(glue("aaa", "bbb"), glue("ccc", "ddd")), 10) == "aaa bbb\nccc ddd" end test "group doc" do - # Consistency with definitions - assert group(glue("a", "b")) == - {:doc_group, {:doc_cons, "a", concat(break, "b")}} - assert group(empty) == {:doc_group, empty} + # Consistent with definitions + assert group("ab") == {:doc_group, "ab", :normal} + assert group(empty()) == {:doc_group, empty(), :normal} + + # Consistent formatting + doc = concat(glue(glue(glue("hello", "a"), "b"), "c"), "d") + assert render(group(doc), 5) == "hello\na\nb\ncd" + end + + test "group modes doc" do + doc = glue(glue("hello", "a"), "b") + assert render(doc, 10) == "hello a b" + + assert render(doc |> glue("c") |> group(), 10) == + "hello\na\nb\nc" + + assert render(doc |> group() |> glue("c") |> group() |> glue("d"), 10) == + "hello a b\nc\nd" + + assert render(doc |> group(:optimistic) |> glue("c") |> group() |> glue("d"), 10) == + "hello\na\nb c d" + + assert render(doc |> group(:optimistic) |> glue("c") |> group(:pessimistic) |> glue("d"), 10) == + "hello\na\nb c\nd" + end + + test "no limit doc" do + doc = no_limit(group(glue(glue("hello", "a"), "b"))) + assert render(doc, 5) == "hello a b" + assert render(doc, :infinity) == "hello a b" + end + + test "collapse lines" do + # Consistent with definitions + assert collapse_lines(3) == {:doc_collapse, 3} + + # Wrong argument type + assert_raise FunctionClauseError, fn -> collapse_lines(0) end + assert_raise FunctionClauseError, fn -> collapse_lines(empty()) end + + # Consistent formatting + doc = concat([collapse_lines(2), line(), line(), line()]) + assert render(doc, 10) == "\n\n" + assert render(nest(doc, 2), 10) == "\n\n " + + doc = concat([collapse_lines(2), line(), line()]) + assert render(doc, 10) == "\n\n" + assert render(nest(doc, 2), 10) == "\n\n " + + doc = concat([collapse_lines(2), line()]) + assert render(doc, 10) == "\n" + assert render(nest(doc, 2), 10) == "\n " - # Consistence of corresponding sdoc - assert factor(glue("a", "b"), 1) == ["a", " ", "b"] - assert factor(glue("a", "b"), 9) == ["a", " ", "b"] + doc = concat([collapse_lines(2), line(), "", line(), "", line()]) + assert render(doc, 10) == "\n\n" + assert render(nest(doc, 2), 10) == "\n\n " + + doc = concat([collapse_lines(2), line(), "foo", line(), "bar", line()]) + assert render(doc, 10) == "\nfoo\nbar\n" + assert render(nest(doc, 2), 10) == "\n foo\n bar\n " + end + + test "force doc and cancel doc" do + # Consistent with definitions + assert force_unfit("ab") == {:doc_force, "ab"} + assert force_unfit(empty()) == {:doc_force, empty()} # Consistent formatting - assert pretty(helloabcd, 5) == "hello\na b\ncd" - assert pretty(helloabcd, 80) == "hello a b cd" + doc = force_unfit(glue(glue("hello", "a"), "b")) + assert render(doc, 20) == "hello\na\nb" + + assert render(doc |> glue("c") |> group(), 20) == + "hello\na\nb\nc" + + assert render(doc |> group(:optimistic) |> glue("c") |> group() |> glue("d"), 20) == + "hello\na\nb c d" + + assert render(doc |> group(:optimistic) |> glue("c") |> group(:pessimistic) |> glue("d"), 20) == + "hello\na\nb c\nd" + end + + test "formatting groups with lines" do + doc = line(glue("a", "b"), glue("hello", "world")) + assert render(group(doc), 5) == "a\nb\nhello\nworld" + assert render(group(doc), 100) == "a b\nhello world" end test "formatting with infinity" do - s = String.duplicate "x", 50 - g = ";" - doc = group(glue(s, g, s) |> glue(g, s) |> glue(g, s) |> glue(g, s)) + str = String.duplicate("x", 50) + colon = ";" + + doc = + str + |> glue(colon, str) + |> glue(colon, str) + |> glue(colon, str) + |> glue(colon, str) + |> group() + + assert render(doc, :infinity) == + str <> colon <> str <> colon <> str <> colon <> str <> colon <> str + end + + test "formatting container_doc with empty" do + sm = &container_doc("[", &1, "]", %Inspect.Opts{}, fn d, _ -> d end, separator: ",") + + assert sm.([]) |> render(80) == "[]" + assert sm.([empty()]) |> render(80) == "[]" + assert sm.([empty(), empty()]) |> render(80) == "[]" + assert sm.(["a"]) |> render(80) == "[a]" + assert sm.(["a", empty()]) |> render(80) == "[a]" + assert sm.([empty(), "a"]) |> render(80) == "[a]" + assert sm.(["a", empty(), "b"]) |> render(80) == "[a, b]" + assert sm.([empty(), "a", "b"]) |> render(80) == "[a, b]" + assert sm.(["a", "b", empty()]) |> render(80) == "[a, b]" + assert sm.(["a", "b" | "c"]) |> render(80) == "[a, b | c]" + assert sm.(["a" | "b"]) |> render(80) == "[a | b]" + assert sm.(["a" | empty()]) |> render(80) == "[a]" + assert sm.([empty() | "b"]) |> render(80) == "[b]" + end + + test "formatting container_doc with empty and limit" do + opts = %Inspect.Opts{limit: 2} + value = ["a", empty(), "b"] - assert pretty(doc, :infinity) == s <> g <> s <> g <> s <> g <> s <> g <> s + assert container_doc("[", value, "]", opts, fn d, _ -> d end, separator: ",") |> render(80) == + "[a, b]" end end diff --git a/lib/elixir/test/elixir/inspect_test.exs b/lib/elixir/test/elixir/inspect_test.exs index 307801d8efc..67a1544c9e6 100644 --- a/lib/elixir/test/elixir/inspect_test.exs +++ b/lib/elixir/test/elixir/inspect_test.exs @@ -1,343 +1,1025 @@ -Code.require_file "test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("test_helper.exs", __DIR__) defmodule Inspect.AtomTest do use ExUnit.Case, async: true - test :basic do + doctest Inspect + + test "basic" do assert inspect(:foo) == ":foo" end - test :empty do + test "empty" do assert inspect(:"") == ":\"\"" end - test :true_false_nil do + test "true, false, nil" do assert inspect(false) == "false" assert inspect(true) == "true" assert inspect(nil) == "nil" end - test :with_uppercase do + test "with uppercase letters" do assert inspect(:fOO) == ":fOO" assert inspect(:FOO) == ":FOO" end - test :alias_atom do + test "aliases" do assert inspect(Foo) == "Foo" assert inspect(Foo.Bar) == "Foo.Bar" assert inspect(Elixir) == "Elixir" + assert inspect(Elixir.Foo) == "Foo" assert inspect(Elixir.Elixir) == "Elixir.Elixir" + assert inspect(Elixir.Elixir.Foo) == "Elixir.Elixir.Foo" end - test :with_integers do - assert inspect(User1) == "User1" + test "with integers" do + assert inspect(User1) == "User1" assert inspect(:user1) == ":user1" end - test :with_punctuation do + test "with trailing ? or !" do assert inspect(:foo?) == ":foo?" assert inspect(:bar!) == ":bar!" + assert inspect(:Foo?) == ":Foo?" end - test :op do - assert inspect(:+) == ":+" + test "operators" do + assert inspect(:+) == ":+" + assert inspect(:<~) == ":<~" + assert inspect(:~>) == ":~>" assert inspect(:&&&) == ":&&&" - assert inspect(:~~~) == ":~~~" + assert inspect(:"~~~") == ":\"~~~\"" + assert inspect(:<<~) == ":<<~" + assert inspect(:~>>) == ":~>>" + assert inspect(:<~>) == ":<~>" + assert inspect(:+++) == ":+++" + assert inspect(:---) == ":---" end - test :... do - assert inspect(:...) == ":..." + test "::" do + assert inspect(:"::") == ~s[:"::"] end - test :@ do + test "with @" do assert inspect(:@) == ":@" assert inspect(:foo@bar) == ":foo@bar" assert inspect(:foo@bar@) == ":foo@bar@" assert inspect(:foo@bar@baz) == ":foo@bar@baz" end - test :others do + test "others" do + assert inspect(:...) == ":..." assert inspect(:<<>>) == ":<<>>" - assert inspect(:{}) == ":{}" - assert inspect(:%{}) == ":%{}" - assert inspect(:%) == ":%" + assert inspect(:{}) == ":{}" + assert inspect(:%{}) == ":%{}" + assert inspect(:%) == ":%" + assert inspect(:->) == ":->" + end + + test "escaping" do + assert inspect(:"hy-phen") == ~s(:"hy-phen") + assert inspect(:"@hello") == ~s(:"@hello") + assert inspect(:"Wat!?") == ~s(:"Wat!?") + assert inspect(:"'quotes' and \"double quotes\"") == ~S(:"'quotes' and \"double quotes\"") + end + + test "colors" do + opts = [syntax_colors: [atom: :red]] + assert inspect(:hello, opts) == "\e[31m:hello\e[0m" + opts = [syntax_colors: [reset: :cyan]] + assert inspect(:hello, opts) == ":hello" + end + + test "Unicode" do + assert inspect(:olá) == ":olá" + assert inspect(:Olá) == ":Olá" + assert inspect(:Ólá) == ":Ólá" + assert inspect(:こんにちは世界) == ":こんにちは世界" + + nfd = :unicode.characters_to_nfd_binary("olá") + assert inspect(String.to_atom(nfd)) == ":\"#{nfd}\"" end end defmodule Inspect.BitStringTest do use ExUnit.Case, async: true - test :bitstring do - assert inspect(<<1 :: [size(12), integer, signed]>>) == "<<0, 1::size(4)>>" + test "bitstring" do + assert inspect(<<1::12-integer-signed>>) == "<<0, 1::size(4)>>" + + assert inspect(<<1::12-integer-signed>>, syntax_colors: [number: :blue]) == + "<<\e[34m0\e[0m, \e[34m1\e[0m::size(4)>>" + + assert inspect(<<1, 2, 3, 4, 5>>, pretty: true, width: 10) == "<<1, 2, 3,\n 4, 5>>" end - test :binary do + test "binary" do assert inspect("foo") == "\"foo\"" assert inspect(<>) == "\"abc\"" end - test :escape do + test "escaping" do assert inspect("f\no") == "\"f\\no\"" assert inspect("f\\o") == "\"f\\\\o\"" assert inspect("f\ao") == "\"f\\ao\"" + + assert inspect("\a\b\d\e\f\n\r\s\t\v") == "\"\\a\\b\\d\\e\\f\\n\\r \\t\\v\"" end - test :utf8 do + test "UTF-8" do assert inspect(" ゆんゆん") == "\" ゆんゆん\"" + # BOM + assert inspect("\uFEFFhello world") == "\"\\uFEFFhello world\"" + # Invisible characters + assert inspect("\u2063") == "\"\\u2063\"" end - test :all_escapes do - assert inspect("\a\b\d\e\f\n\r\s\t\v") == - "\"\\a\\b\\d\\e\\f\\n\\r \\t\\v\"" - end + test "infer" do + assert inspect(<<"john", 193, "doe">>, binaries: :infer) == + ~s(<<106, 111, 104, 110, 193, 100, 111, 101>>) - test :opt_infer do - assert inspect(<<"eric", 193, "mj">>, binaries: :infer) == ~s(<<101, 114, 105, 99, 193, 109, 106>>) - assert inspect(<<"eric">>, binaries: :infer) == ~s("eric") + assert inspect(<<"john">>, binaries: :infer) == ~s("john") assert inspect(<<193>>, binaries: :infer) == ~s(<<193>>) end - test :opt_as_strings do - assert inspect(<<"eric", 193, "mj">>, binaries: :as_strings) == ~s("eric\\301mj") - assert inspect(<<"eric">>, binaries: :as_strings) == ~s("eric") - assert inspect(<<193>>, binaries: :as_strings) == ~s("\\301") + test "as strings" do + assert inspect(<<"john", 193, "doe">>, binaries: :as_strings) == ~s("john\\xC1doe") + assert inspect(<<"john">>, binaries: :as_strings) == ~s("john") + assert inspect(<<193>>, binaries: :as_strings) == ~s("\\xC1") + assert inspect(<<193>>, base: :hex, binaries: :as_strings) == ~s("\\xC1") end - test :opt_as_binaries do - assert inspect(<<"eric", 193, "mj">>, binaries: :as_binaries) == "<<101, 114, 105, 99, 193, 109, 106>>" - assert inspect(<<"eric">>, binaries: :as_binaries) == "<<101, 114, 105, 99>>" + test "as binaries" do + assert inspect(<<"john", 193, "doe">>, binaries: :as_binaries) == + "<<106, 111, 104, 110, 193, 100, 111, 101>>" + + assert inspect(<<"john">>, binaries: :as_binaries) == "<<106, 111, 104, 110>>" assert inspect(<<193>>, binaries: :as_binaries) == "<<193>>" + + # Any base other than :decimal implies "binaries: :as_binaries" + assert inspect("abc", base: :hex) == "<<0x61, 0x62, 0x63>>" + assert inspect("abc", base: :octal) == "<<0o141, 0o142, 0o143>>" + + # Size is still represented as decimal + assert inspect(<<10, 11, 12::4>>, base: :hex) == "<<0xA, 0xB, 0xC::size(4)>>" end - test :unprintable_with_opts do + test "unprintable with limit" do assert inspect(<<193, 193, 193, 193>>, limit: 3) == "<<193, 193, 193, ...>>" end + + test "printable limit" do + assert inspect("hello world", printable_limit: 4) == ~s("hell" <> ...) + + # Non-printable characters after the limit don't matter + assert inspect("hello world" <> <<0>>, printable_limit: 4) == ~s("hell" <> ...) + + # Non printable strings aren't affected by printable limit + assert inspect(<<0, 1, 2, 3, 4>>, printable_limit: 3) == ~s(<<0, 1, 2, 3, 4>>) + end end defmodule Inspect.NumberTest do use ExUnit.Case, async: true - test :integer do + test "integer" do assert inspect(100) == "100" end - test :float do + test "decimal" do + assert inspect(100, base: :decimal) == "100" + end + + test "hex" do + assert inspect(100, base: :hex) == "0x64" + assert inspect(-100, base: :hex) == "-0x64" + end + + test "octal" do + assert inspect(100, base: :octal) == "0o144" + assert inspect(-100, base: :octal) == "-0o144" + end + + test "binary" do + assert inspect(86, base: :binary) == "0b1010110" + assert inspect(-86, base: :binary) == "-0b1010110" + end + + test "float" do assert inspect(1.0) == "1.0" - assert inspect(1.0E10) == "1.0e10" - assert inspect(1.0e10) == "1.0e10" + assert inspect(1.0e10) == "10000000000.0" assert inspect(1.0e-10) == "1.0e-10" end + + test "integer colors" do + opts = [syntax_colors: [number: :red]] + assert inspect(123, opts) == "\e[31m123\e[0m" + opts = [syntax_colors: [reset: :cyan]] + assert inspect(123, opts) == "123" + end + + test "float colors" do + opts = [syntax_colors: [number: :red]] + assert inspect(1.3, opts) == "\e[31m1.3\e[0m" + opts = [syntax_colors: [reset: :cyan]] + assert inspect(1.3, opts) == "1.3" + end end defmodule Inspect.TupleTest do - use ExUnit.Case + use ExUnit.Case, async: true - test :basic do + test "basic" do assert inspect({1, "b", 3}) == "{1, \"b\", 3}" - assert inspect({1, "b", 3}, [pretty: true, width: 1]) == "{1,\n \"b\",\n 3}" + assert inspect({1, "b", 3}, pretty: true, width: 1) == "{1,\n \"b\",\n 3}" + assert inspect({1, "b", 3}, pretty: true, width: 10) == "{1, \"b\",\n 3}" end - test :empty do + test "empty" do assert inspect({}) == "{}" end - test :with_limit do + test "with limit" do assert inspect({1, 2, 3, 4}, limit: 3) == "{1, 2, 3, ...}" end + + test "colors" do + opts = [syntax_colors: []] + assert inspect({}, opts) == "{}" + + opts = [syntax_colors: [reset: :cyan]] + assert inspect({}, opts) == "{}" + assert inspect({:x, :y}, opts) == "{:x, :y}" + + opts = [syntax_colors: [reset: :cyan, atom: :red]] + assert inspect({}, opts) == "{}" + assert inspect({:x, :y}, opts) == "{\e[31m:x\e[36m, \e[31m:y\e[36m}" + + opts = [syntax_colors: [tuple: :green, reset: :cyan, atom: :red]] + assert inspect({}, opts) == "\e[32m{\e[36m\e[32m}\e[36m" + + assert inspect({:x, :y}, opts) == + "\e[32m{\e[36m\e[31m:x\e[36m\e[32m,\e[36m \e[31m:y\e[36m\e[32m}\e[36m" + end end defmodule Inspect.ListTest do use ExUnit.Case, async: true - test :basic do - assert inspect([ 1, "b", 3 ]) == "[1, \"b\", 3]" - assert inspect([ 1, "b", 3 ], [pretty: true, width: 1]) == "[1,\n \"b\",\n 3]" + test "basic" do + assert inspect([1, "b", 3]) == "[1, \"b\", 3]" + assert inspect([1, "b", 3], pretty: true, width: 1) == "[1,\n \"b\",\n 3]" end - test :printable do - assert inspect('abc') == "'abc'" + test "printable" do + assert inspect(~c"abc") == ~s(~c"abc") end - test :keyword do - assert inspect([a: 1]) == "[a: 1]" - assert inspect([a: 1, b: 2]) == "[a: 1, b: 2]" - assert inspect([a: 1, a: 2, b: 2]) == "[a: 1, a: 2, b: 2]" - assert inspect(["123": 1]) == ~s(["123": 1]) + test "printable limit" do + assert inspect(~c"hello world", printable_limit: 4) == ~s(~c"hell" ++ ...) + # Non printable characters after the limit don't matter + assert inspect(~c"hello world" ++ [0], printable_limit: 4) == ~s(~c"hell" ++ ...) + # Non printable strings aren't affected by printable limit + assert inspect([0, 1, 2, 3, 4], printable_limit: 3) == ~s([0, 1, 2, 3, 4]) + end + + test "keyword" do + assert inspect(a: 1) == "[a: 1]" + assert inspect(a: 1, b: 2) == "[a: 1, b: 2]" + assert inspect(a: 1, a: 2, b: 2) == "[a: 1, a: 2, b: 2]" + assert inspect("123": 1) == ~s(["123": 1]) + + assert inspect([foo: [1, 2, 3], baz: [4, 5, 6]], pretty: true, width: 20) == + "[\n foo: [1, 2, 3],\n baz: [4, 5, 6]\n]" + end - assert inspect([foo: [1,2,3,:bar], bazzz: :bat], [pretty: true, width: 30]) == - "[foo: [1, 2, 3, :bar],\n bazzz: :bat]" + test "keyword operators" do + assert inspect("::": 1, +: 2) == ~s(["::": 1, +: 2]) end - test :opt_infer do - assert inspect('eric' ++ [0] ++ 'mj', char_lists: :infer) == "[101, 114, 105, 99, 0, 109, 106]" - assert inspect('eric', char_lists: :infer) == "'eric'" - assert inspect([0], char_lists: :infer) == "[0]" + test "opt infer" do + assert inspect(~c"john" ++ [0] ++ ~c"doe", charlists: :infer) == + "[106, 111, 104, 110, 0, 100, 111, 101]" + + assert inspect(~c"john", charlists: :infer) == ~s(~c"john") + assert inspect([0], charlists: :infer) == "[0]" end - test :opt_as_strings do - assert inspect('eric' ++ [0] ++ 'mj', char_lists: :as_char_lists) == "'eric\\000mj'" - assert inspect('eric', char_lists: :as_char_lists) == "'eric'" - assert inspect([0], char_lists: :as_char_lists) == "'\\000'" + test "opt as strings" do + assert inspect(~c"john" ++ [0] ++ ~c"doe", charlists: :as_charlists) == ~s(~c"john\\0doe") + assert inspect(~c"john", charlists: :as_charlists) == ~s(~c"john") + assert inspect([0], charlists: :as_charlists) == ~s(~c"\\0") end - test :opt_as_lists do - assert inspect('eric' ++ [0] ++ 'mj', char_lists: :as_lists) == "[101, 114, 105, 99, 0, 109, 106]" - assert inspect('eric', char_lists: :as_lists) == "[101, 114, 105, 99]" - assert inspect([0], char_lists: :as_lists) == "[0]" + test "opt as lists" do + assert inspect(~c"john" ++ [0] ++ ~c"doe", charlists: :as_lists) == + "[106, 111, 104, 110, 0, 100, 111, 101]" + + assert inspect(~c"john", charlists: :as_lists) == "[106, 111, 104, 110]" + assert inspect([0], charlists: :as_lists) == "[0]" end - test :non_printable do + test "non printable" do assert inspect([{:b, 1}, {:a, 1}]) == "[b: 1, a: 1]" end - test :unproper do + test "improper" do assert inspect([:foo | :bar]) == "[:foo | :bar]" - assert inspect([1,2,3,4,5|42], [pretty: true, width: 1]) == "[1,\n 2,\n 3,\n 4,\n 5 |\n 42]" + assert inspect([1, 2, 3, 4, 5 | 42], pretty: true, width: 1) == + "[1,\n 2,\n 3,\n 4,\n 5 |\n 42]" + end + + test "nested" do + assert inspect(Enum.reduce(1..5, [0], &[&2, &1]), limit: 5) == + "[[[[[[...], ...], ...], ...], ...], ...]" + + assert inspect(Enum.reduce(1..5, [0], &[&2, &1]), limit: 10) == + "[[[[[[0], 1], 2], 3], 4], ...]" + + assert inspect(Enum.reduce(1..6, [0], &[&2, &1]), limit: 10) == + "[[[[[[[0], 1], 2], 3], ...], ...], ...]" + + assert inspect(Enum.reduce(1..100, [0], &[&2 | &1]), limit: 5) == + "[[[[[[...] | 96] | 97] | 98] | 99] | 100]" end - test :codepoints do - assert inspect('é') == "[233]" + test "codepoints" do + assert inspect(~c"é") == "[233]" end - test :empty do + test "empty" do assert inspect([]) == "[]" end - test :with_limit do - assert inspect([ 1, 2, 3, 4 ], limit: 3) == "[1, 2, 3, ...]" + test "with limit" do + assert inspect([1, 2, 3, 4], limit: 3) == "[1, 2, 3, ...]" + end + + test "colors" do + opts = [syntax_colors: []] + assert inspect([], opts) == "[]" + + opts = [syntax_colors: [reset: :cyan]] + assert inspect([], opts) == "[]" + assert inspect([:x, :y], opts) == "[:x, :y]" + + opts = [syntax_colors: [reset: :cyan, atom: :red]] + assert inspect([], opts) == "[]" + assert inspect([:x, :y], opts) == "[\e[31m:x\e[36m, \e[31m:y\e[36m]" + + opts = [syntax_colors: [reset: :cyan, atom: :red, list: :green]] + assert inspect([], opts) == "\e[32m[]\e[36m" + + assert inspect([:x, :y], opts) == + "\e[32m[\e[36m\e[31m:x\e[36m\e[32m,\e[36m \e[31m:y\e[36m\e[32m]\e[36m" + end + + test "keyword with colors" do + opts = [syntax_colors: [reset: :cyan, list: :green, number: :blue]] + assert inspect([], opts) == "\e[32m[]\e[36m" + + assert inspect([a: 9999], opts) == "\e[32m[\e[36ma: \e[34m9999\e[36m\e[32m]\e[36m" + + opts = [syntax_colors: [reset: :cyan, atom: :red, list: :green, number: :blue]] + assert inspect([], opts) == "\e[32m[]\e[36m" + + assert inspect([a: 9999], opts) == "\e[32m[\e[36m\e[31ma:\e[36m \e[34m9999\e[36m\e[32m]\e[36m" + end + + test "limit with colors" do + opts = [limit: 1, syntax_colors: [reset: :cyan, list: :green, atom: :red]] + assert inspect([], opts) == "\e[32m[]\e[36m" + + assert inspect([:x, :y], opts) == "\e[32m[\e[36m\e[31m:x\e[36m\e[32m,\e[36m ...\e[32m]\e[36m" end end defmodule Inspect.MapTest do - use ExUnit.Case + use ExUnit.Case, async: true - test :basic do + test "basic" do assert inspect(%{1 => "b"}) == "%{1 => \"b\"}" - assert inspect(%{1 => "b", 2 => "c"}, [pretty: true, width: 1]) == "%{1 => \"b\",\n 2 => \"c\"}" + + assert inspect(%{1 => "b", 2 => "c"}, + pretty: true, + width: 1, + custom_options: [sort_maps: true] + ) == + "%{\n 1 => \"b\",\n 2 => \"c\"\n}" end - test :keyword do + test "keyword" do assert inspect(%{a: 1}) == "%{a: 1}" - assert inspect(%{a: 1, b: 2}) == "%{a: 1, b: 2}" - assert inspect(%{a: 1, b: 2, c: 3}) == "%{a: 1, b: 2, c: 3}" + assert inspect(%{a: 1, b: 2}, custom_options: [sort_maps: true]) == "%{a: 1, b: 2}" + + assert inspect(%{a: 1, b: 2, c: 3}, custom_options: [sort_maps: true]) == + "%{a: 1, b: 2, c: 3}" end - test :with_limit do - assert inspect(%{1 => 1, 2 => 2, 3 => 3, 4 => 4}, limit: 3) == "%{1 => 1, 2 => 2, 3 => 3, ...}" + test "with limit" do + assert inspect(%{1 => 1, 2 => 2, 3 => 3, 4 => 4}, limit: 3) == + "%{1 => 1, 2 => 2, 3 => 3, ...}" end defmodule Public do - def __struct__ do - %{key: 0, __struct__: Public} - end + defstruct key: 0 end defmodule Private do end - test :public_struct do + test "public struct" do assert inspect(%Public{key: 1}) == "%Inspect.MapTest.Public{key: 1}" end - test :public_modified_struct do + test "public modified struct" do public = %Public{key: 1} - assert inspect(Map.put(public, :foo, :bar)) == - "%{__struct__: Inspect.MapTest.Public, foo: :bar, key: 1}" + + assert inspect(Map.put(public, :foo, :bar), custom_options: [sort_maps: true]) == + "%{__struct__: Inspect.MapTest.Public, foo: :bar, key: 1}" end - test :private_struct do - assert inspect(%{__struct__: Private, key: 1}) == "%{__struct__: Inspect.MapTest.Private, key: 1}" + test "public modified struct with defimpl" do + map_set = MapSet.new([1, 2]) + + assert inspect(Map.put(map_set, :foo, :bar), custom_options: [sort_maps: true]) == + "%{__struct__: MapSet, foo: :bar, map: %{1 => [], 2 => []}}" + end + + test "private struct" do + assert inspect(%{__struct__: Private, key: 1}, custom_options: [sort_maps: true]) == + "%{__struct__: Inspect.MapTest.Private, key: 1}" end defmodule Failing do - def __struct__ do - %{key: 0} - end + @enforce_keys [:name] + defstruct @enforce_keys defimpl Inspect do - def inspect(_, _) do - raise "failing" + def inspect(%Failing{name: name}, _) do + Atom.to_string(name) end end end - test :bad_implementation do - msg = "Got RuntimeError with message \"failing\" " <> - "while inspecting %{__struct__: Inspect.MapTest.Failing, key: 0}" + test "safely inspect bad implementation" do + assert_raise ArgumentError, ~r/argument error/, fn -> + raise(ArgumentError) + end + + message = ~s''' + #Inspect.Error< + got ArgumentError with message: + + """ + errors were found at the given arguments: + + * 1st argument: not an atom + """ + + while inspecting: + + %{__struct__: Inspect.MapTest.Failing, name: "Foo"} + ''' + + assert inspect(%Failing{name: "Foo"}, custom_options: [sort_maps: true]) =~ message + end + + test "safely inspect bad implementation disables colors" do + message = ~s''' + #Inspect.Error< + got ArgumentError with message: + + """ + errors were found at the given arguments: + + * 1st argument: not an atom + """ + + while inspecting: + + %{__struct__: Inspect.MapTest.Failing, name: "Foo"} + ''' + + assert inspect(%Failing{name: "Foo"}, + syntax_colors: [atom: [:green]], + custom_options: [sort_maps: true] + ) =~ message + end - assert_raise ArgumentError, msg, fn -> - inspect(%Failing{}) + test "unsafely inspect bad implementation" do + exception_message = ~s''' + got ArgumentError with message: + + """ + errors were found at the given arguments: + + * 1st argument: not an atom + """ + + while inspecting: + + %{__struct__: Inspect.MapTest.Failing, name: "Foo"} + ''' + + try do + inspect(%Failing{name: "Foo"}, safe: false, custom_options: [sort_maps: true]) + rescue + exception in Inspect.Error -> + assert Exception.message(exception) =~ exception_message + assert [{:erlang, :atom_to_binary, [_], [_ | _]} | _] = __STACKTRACE__ + else + _ -> flunk("expected failure") end end - test :exception do + test "raise when trying to inspect with a bad implementation from inside another exception that is being raised" do + # Inspect.Error is raised here when we tried to print the error message + # called by another exception (Protocol.UndefinedError in this case) + exception_message = ~s''' + protocol Enumerable not implemented for Inspect.MapTest.Failing (a struct) + + Got value: + + #Inspect.Error< + got ArgumentError with message: + + """ + errors were found at the given arguments: + + * 1st argument: not an atom + """ + + while inspecting: + + ''' + + try do + Enum.to_list(%Failing{name: "Foo"}) + rescue + exception in Protocol.UndefinedError -> + message = Exception.message(exception) + assert message =~ exception_message + assert message =~ "__struct__: Inspect.MapTest.Failing" + assert message =~ "name: \"Foo\"" + + assert [ + {Enumerable, :impl_for!, 1, _} | _ + ] = __STACKTRACE__ + + # The culprit + assert Enum.any?(__STACKTRACE__, fn + {Enum, :to_list, 1, _} -> true + _ -> false + end) + + # The line calling the culprit + assert Enum.any?(__STACKTRACE__, fn + {Inspect.MapTest, _test_name, 1, file: file, line: _line_number} -> + String.ends_with?(List.to_string(file), "test/elixir/inspect_test.exs") + + _ -> + false + end) + else + _ -> flunk("expected failure") + end + end + + test "Exception.message/1 with bad implementation" do + failing = %Failing{name: "Foo"} + + message = ~s''' + #Inspect.Error< + got ArgumentError with message: + + """ + errors were found at the given arguments: + + * 1st argument: not an atom + """ + + while inspecting: + + %{__struct__: Inspect.MapTest.Failing, name: "Foo"} + ''' + + {my_argument_error, stacktrace} = + try do + atom_to_string(Process.get(:unused, failing.name)) + rescue + e -> {e, __STACKTRACE__} + end + + inspected = + inspect( + Inspect.Error.exception( + exception: my_argument_error, + stacktrace: stacktrace, + inspected_struct: "%{__struct__: Inspect.MapTest.Failing, name: \"Foo\"}" + ) + ) + + assert inspect(failing, custom_options: [sort_maps: true]) =~ message + assert inspected =~ message + end + + defp atom_to_string(atom) do + Atom.to_string(atom) + end + + test "exception" do assert inspect(%RuntimeError{message: "runtime error"}) == - "%RuntimeError{message: \"runtime error\"}" + "%RuntimeError{message: \"runtime error\"}" + end + + test "colors" do + opts = [syntax_colors: [reset: :cyan, atom: :red, number: :magenta]] + assert inspect(%{1 => 2}, opts) == "%{\e[35m1\e[36m => \e[35m2\e[36m}" + + assert inspect(%{a: 1}, opts) == "%{\e[31ma:\e[36m \e[35m1\e[36m}" + + assert inspect(%Public{key: 1}, opts) == + "%Inspect.MapTest.Public{\e[31mkey:\e[36m \e[35m1\e[36m}" + + opts = [syntax_colors: [reset: :cyan, atom: :red, map: :green, number: :blue]] + + assert inspect(%{a: 9999}, opts) == + "\e[32m%{\e[36m" <> "\e[31ma:\e[36m " <> "\e[34m9999\e[36m" <> "\e[32m}\e[36m" + end + + defmodule StructWithoutOptions do + @derive Inspect + defstruct [:a, :b, :c, :d] + end + + test "struct without options" do + struct = %StructWithoutOptions{a: 1, b: 2, c: 3, d: 4} + assert inspect(struct) == "%Inspect.MapTest.StructWithoutOptions{a: 1, b: 2, c: 3, d: 4}" + + assert inspect(struct, pretty: true, width: 1) == + "%Inspect.MapTest.StructWithoutOptions{\n a: 1,\n b: 2,\n c: 3,\n d: 4\n}" + end + + defmodule StructWithOnlyOption do + @derive {Inspect, only: [:b, :c]} + defstruct [:a, :b, :c, :d] + end + + test "struct with :only option" do + struct = %StructWithOnlyOption{a: 1, b: 2, c: 3, d: 4} + assert inspect(struct) == "#Inspect.MapTest.StructWithOnlyOption" + + assert inspect(struct, pretty: true, width: 1) == + "#Inspect.MapTest.StructWithOnlyOption<\n b: 2,\n c: 3,\n ...\n>" + + struct = %{struct | c: [1, 2, 3, 4]} + assert inspect(struct) == "#Inspect.MapTest.StructWithOnlyOption" + end + + defmodule StructWithEmptyOnlyOption do + @derive {Inspect, only: []} + defstruct [:a, :b, :c, :d] + end + + test "struct with empty :only option" do + struct = %StructWithEmptyOnlyOption{a: 1, b: 2, c: 3, d: 4} + assert inspect(struct) == "#Inspect.MapTest.StructWithEmptyOnlyOption<...>" + end + + defmodule StructWithAllFieldsInOnlyOption do + @derive {Inspect, only: [:a, :b]} + defstruct [:a, :b] + end + + test "struct with all fields in the :only option" do + struct = %StructWithAllFieldsInOnlyOption{a: 1, b: 2} + assert inspect(struct) == "%Inspect.MapTest.StructWithAllFieldsInOnlyOption{a: 1, b: 2}" + + assert inspect(struct, pretty: true, width: 1) == + "%Inspect.MapTest.StructWithAllFieldsInOnlyOption{\n a: 1,\n b: 2\n}" + end + + test "struct missing fields in the :only option" do + assert_raise ArgumentError, + "unknown fields [:c] in :only when deriving the Inspect protocol for Inspect.MapTest.StructMissingFieldsInOnlyOption", + fn -> + defmodule StructMissingFieldsInOnlyOption do + @derive {Inspect, only: [:c]} + defstruct [:a, :b] + end + end + end + + test "struct missing fields in the :except option" do + assert_raise ArgumentError, + "unknown fields [:c, :d] in :except when deriving the Inspect protocol for Inspect.MapTest.StructMissingFieldsInExceptOption", + fn -> + defmodule StructMissingFieldsInExceptOption do + @derive {Inspect, except: [:c, :d]} + defstruct [:a, :b] + end + end + end + + test "passing a non-list to the :only option" do + assert_raise ArgumentError, + "invalid value :not_a_list in :only when deriving the Inspect protocol for Inspect.MapTest.StructInvalidListInOnlyOption (expected a list)", + fn -> + defmodule StructInvalidListInOnlyOption do + @derive {Inspect, only: :not_a_list} + defstruct [:a, :b] + end + end + end + + defmodule StructWithExceptOption do + @derive {Inspect, except: [:b, :c]} + defstruct [:a, :b, :c, :d] + end + + test "struct with :except option" do + struct = %StructWithExceptOption{a: 1, b: 2, c: 3, d: 4} + assert inspect(struct) == "#Inspect.MapTest.StructWithExceptOption" + + assert inspect(struct, pretty: true, width: 1) == + "#Inspect.MapTest.StructWithExceptOption<\n a: 1,\n d: 4,\n ...\n>" + end + + defmodule StructWithBothOnlyAndExceptOptions do + @derive {Inspect, only: [:a, :b], except: [:b, :c]} + defstruct [:a, :b, :c, :d] + end + + test "struct with both :only and :except options" do + struct = %StructWithBothOnlyAndExceptOptions{a: 1, b: 2, c: 3, d: 4} + assert inspect(struct) == "#Inspect.MapTest.StructWithBothOnlyAndExceptOptions" + + assert inspect(struct, pretty: true, width: 1) == + "#Inspect.MapTest.StructWithBothOnlyAndExceptOptions<\n a: 1,\n ...\n>" + end + + defmodule StructWithOptionalAndOrder do + @derive {Inspect, optional: [:b, :c]} + defstruct [:c, :d, :a, :b] + end + + test "struct with both :order and :optional options" do + struct = %StructWithOptionalAndOrder{a: 1, b: 2, c: 3, d: 4} + + assert inspect(struct) == + "%Inspect.MapTest.StructWithOptionalAndOrder{c: 3, d: 4, a: 1, b: 2}" + + struct = %StructWithOptionalAndOrder{} + assert inspect(struct) == "%Inspect.MapTest.StructWithOptionalAndOrder{d: nil, a: nil}" + end + + defmodule StructWithExceptOptionalAndOrder do + @derive {Inspect, optional: [:b, :c], except: [:e]} + defstruct [:c, :d, :e, :a, :b] + end + + test "struct with :except, :order, and :optional options" do + struct = %StructWithExceptOptionalAndOrder{a: 1, b: 2, c: 3, d: 4} + + assert inspect(struct) == + "#Inspect.MapTest.StructWithExceptOptionalAndOrder" + + struct = %StructWithExceptOptionalAndOrder{} + + assert inspect(struct) == + "#Inspect.MapTest.StructWithExceptOptionalAndOrder" + end + + defmodule StructWithOptionalAll do + @derive {Inspect, optional: :all} + defstruct [:a, :b, :c, :d] + end + + test "struct with :optional set to :all" do + struct = %StructWithOptionalAll{a: 1, b: 2} + + assert inspect(struct) == "%Inspect.MapTest.StructWithOptionalAll{a: 1, b: 2}" + + struct = %StructWithOptionalAll{} + assert inspect(struct) == "%Inspect.MapTest.StructWithOptionalAll{}" end end defmodule Inspect.OthersTest do use ExUnit.Case, async: true - def f do - fn() -> :ok end + def fun() do + fn -> :ok end end - test :external_elixir_funs do + def unquote(:"weirdly named/fun-")() do + fn -> :ok end + end + + test "external Elixir funs" do bin = inspect(&Enum.map/2) assert bin == "&Enum.map/2" + + assert inspect(&__MODULE__."weirdly named/fun-"/0) == + ~s(&Inspect.OthersTest."weirdly named/fun-"/0) end - test :external_erlang_funs do + test "external Erlang funs" do bin = inspect(&:lists.map/2) assert bin == "&:lists.map/2" end - test :outdated_functions do + test "outdated functions" do defmodule V do def fun do fn -> 1 end end end - Application.put_env(:elixir, :anony, V.fun) + Application.put_env(:elixir, :anony, V.fun()) Application.put_env(:elixir, :named, &V.fun/0) - :code.delete(V) :code.purge(V) + :code.delete(V) anony = Application.get_env(:elixir, :anony) named = Application.get_env(:elixir, :named) - assert inspect(anony) =~ ~r"#Function<0.\d+/0 in Inspect.OthersTest.V>" + assert inspect(anony) =~ ~r"#Function<0.\d+/0" assert inspect(named) =~ ~r"&Inspect.OthersTest.V.fun/0" after Application.delete_env(:elixir, :anony) Application.delete_env(:elixir, :named) end - test :other_funs do - assert "#Function<" <> _ = inspect(fn(x) -> x + 1 end) - assert "#Function<" <> _ = inspect(f) + test "other funs" do + assert "#Function<" <> _ = inspect(fn x -> x + 1 end) + assert "#Function<" <> _ = inspect(fun()) + opts = [syntax_colors: []] + assert "#Function<" <> _ = inspect(fun(), opts) + opts = [syntax_colors: [reset: :red]] + assert "#Function<" <> rest = inspect(fun(), opts) + assert String.ends_with?(rest, ">") + + inspected = inspect(__MODULE__."weirdly named/fun-"()) + assert inspected =~ ~r(#Function<\d+\.\d+/0 in Inspect\.OthersTest\."weirdly named/fun-"/0>) + end + + test "map set" do + assert "MapSet.new(" <> _ = inspect(MapSet.new()) + end + + test "PIDs" do + assert "#PID<" <> _ = inspect(self()) + opts = [syntax_colors: []] + assert "#PID<" <> _ = inspect(self(), opts) + opts = [syntax_colors: [reset: :cyan]] + assert "#PID<" <> rest = inspect(self(), opts) + assert String.ends_with?(rest, ">") + end + + test "references" do + assert "#Reference<" <> _ = inspect(make_ref()) end - test :hash_dict_set do - assert "#HashDict<" <> _ = inspect(HashDict.new) - assert "#HashSet<" <> _ = inspect(HashSet.new) + test "regex" do + assert inspect(~r(foo)m) == "~r/foo/m" + assert inspect(~r[\\\#{2,}]iu) == ~S"~r/\\\#{2,}/iu" + + assert inspect(Regex.compile!("a\\/b")) == "~r/a\\/b/" + + assert inspect(Regex.compile!("\a\b\d\e\f\n\r\s\t\v/")) == + "~r/\\a\\x08\\x7F\\x1B\\f\\n\\r \\t\\v\\//" + + assert inspect(~r<\a\b\d\e\f\n\r\s\t\v/>) == "~r/\\a\\b\\d\\e\\f\\n\\r\\s\\t\\v\\//" + assert inspect(~r" \\/ ") == "~r/ \\\\\\/ /" + assert inspect(~r/hi/, syntax_colors: [regex: :red]) == "\e[31m~r/hi/\e[0m" + + assert inspect(Regex.compile!("foo", "i")) == "~r/foo/i" + assert inspect(Regex.compile!("foo", [:ucp])) == ~S'Regex.compile!("foo", [:ucp])' end - test :pids do - assert "#PID<" <> _ = inspect(self) + @tag :re_import + test "exported regex" do + assert inspect(~r/foo/E) == "~r/foo/E" end - test :references do - assert "#Reference<" <> _ = inspect(make_ref) + test "inspect_fun" do + fun = fn + integer, _opts when is_integer(integer) -> + "<#{integer}>" + + %URI{} = uri, _opts -> + "#URI<#{uri}>" + + term, opts -> + Inspect.inspect(term, opts) + end + + opts = [inspect_fun: fun] + + assert inspect(1000, opts) == "<1000>" + assert inspect([1000], opts) == "[<1000>]" + + uri = URI.parse("https://elixir-lang.org") + assert inspect(uri, opts) == "#URI" + assert inspect([uri], opts) == "[#URI]" end - test :regex do - "~r/foo/m" = inspect(~r(foo)m) - "~r/\\a\\010\\177\\033\\f\\n\\r \\t\\v\\//" = inspect(Regex.compile!("\a\b\d\e\f\n\r\s\t\v/")) - "~r/\\a\\b\\d\\e\\f\\n\\r\\s\\t\\v\\//" = inspect(~r<\a\b\d\e\f\n\r\s\t\v/>) + defmodule Nested do + defstruct nested: nil + + defimpl Inspect do + import Inspect.Algebra + + def inspect(%Nested{nested: nested}, opts) do + indent = Keyword.get(opts.custom_options, :indent, 2) + level = Keyword.get(opts.custom_options, :level, 1) + + nested_str = + Kernel.inspect(nested, custom_options: [level: level + 1, indent: indent + 2]) + + concat( + nest(line("#Nested[##{level}/#{indent}]<", nested_str), indent), + nest(line("", ">"), indent - 2) + ) + end + end + end + + test "custom_options" do + assert inspect(%Nested{nested: %Nested{nested: 42}}) == + "#Nested[#1/2]<\n #Nested[#2/4]<\n 42\n >\n>" + end +end + +defmodule Inspect.CustomProtocolTest do + use ExUnit.Case, async: true + + defprotocol CustomInspect do + def inspect(term, opts) + end + + defmodule MissingImplementation do + defstruct [] + end + + test "unsafely inspect missing implementation" do + msg = ~S''' + got Protocol.UndefinedError with message: + + """ + protocol Inspect.CustomProtocolTest.CustomInspect not implemented for Inspect.CustomProtocolTest.MissingImplementation (a struct) + + Got value: + + %Inspect.CustomProtocolTest.MissingImplementation{} + """ + + while inspecting: + + %{__struct__: Inspect.CustomProtocolTest.MissingImplementation} + ''' + + opts = [safe: false, inspect_fun: &CustomInspect.inspect/2] + + try do + inspect(%MissingImplementation{}, opts) + rescue + e in Inspect.Error -> + assert Exception.message(e) =~ msg + assert [{Inspect.CustomProtocolTest.CustomInspect, :impl_for!, 1, _} | _] = __STACKTRACE__ + else + _ -> flunk("expected failure") + end + end + + test "safely inspect missing implementation" do + msg = ~S''' + #Inspect.Error< + got Protocol.UndefinedError with message: + + """ + protocol Inspect.CustomProtocolTest.CustomInspect not implemented for Inspect.CustomProtocolTest.MissingImplementation (a struct) + + Got value: + + %Inspect.CustomProtocolTest.MissingImplementation{} + """ + + while inspecting: + + %{__struct__: Inspect.CustomProtocolTest.MissingImplementation} + ''' + + opts = [inspect_fun: &CustomInspect.inspect/2] + assert inspect(%MissingImplementation{}, opts) =~ msg end end diff --git a/lib/elixir/test/elixir/integer_test.exs b/lib/elixir/test/elixir/integer_test.exs index 547c71567a0..5069eb58da0 100644 --- a/lib/elixir/test/elixir/integer_test.exs +++ b/lib/elixir/test/elixir/integer_test.exs @@ -1,33 +1,157 @@ -Code.require_file "test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("test_helper.exs", __DIR__) defmodule IntegerTest do use ExUnit.Case, async: true + + doctest Integer + require Integer - test :odd? do - assert Integer.odd?(0) == false - assert Integer.odd?(1) == true - assert Integer.odd?(2) == false - assert Integer.odd?(3) == true - assert Integer.odd?(-1) == true - assert Integer.odd?(-2) == false - assert Integer.odd?(-3) == true + def test_is_odd_in_guards(number) when Integer.is_odd(number), do: number + def test_is_odd_in_guards(_number), do: false + + def test_is_even_in_guards(number) when Integer.is_even(number), do: number + def test_is_even_in_guards(_number), do: false + + test "is_odd/1" do + assert Integer.is_odd(0) == false + assert Integer.is_odd(1) == true + assert Integer.is_odd(2) == false + assert Integer.is_odd(3) == true + assert Integer.is_odd(-1) == true + assert Integer.is_odd(-2) == false + assert Integer.is_odd(-3) == true + assert test_is_odd_in_guards(10) == false + assert test_is_odd_in_guards(11) == 11 + assert test_is_odd_in_guards(:not_integer) == false + end + + test "is_even/1" do + assert Integer.is_even(0) == true + assert Integer.is_even(1) == false + assert Integer.is_even(2) == true + assert Integer.is_even(3) == false + assert Integer.is_even(-1) == false + assert Integer.is_even(-2) == true + assert Integer.is_even(-3) == false + assert test_is_even_in_guards(10) == 10 + assert test_is_even_in_guards(11) == false + assert test_is_even_in_guards(:not_integer) == false + end + + test "mod/2" do + assert Integer.mod(10, -5) == 0 + assert Integer.mod(3, 2) == 1 + assert Integer.mod(0, 10) == 0 + assert Integer.mod(0, -5) == 0 + assert Integer.mod(30000, 2001) == 1986 + assert Integer.mod(30000, -2001) == -15 + assert Integer.mod(-20, 11) == 2 + assert Integer.mod(-55, -22) == -11 + end + + test "mod/2 raises ArithmeticError when divisor is 0" do + assert_raise ArithmeticError, fn -> Integer.mod(3, 0) end + assert_raise ArithmeticError, fn -> Integer.mod(-50, 0) end + end + + test "floor_div/2" do + assert Integer.floor_div(10, -5) == -2 + assert Integer.floor_div(3, 2) == 1 + assert Integer.floor_div(0, 10) == 0 + assert Integer.floor_div(0, -5) == 0 + assert Integer.floor_div(30000, 2001) == 14 + assert Integer.floor_div(30000, -2001) == -15 + assert Integer.floor_div(-20, 11) == -2 + assert Integer.floor_div(-55, -22) == 2 + end + + test "floor_div/2 raises ArithmeticError when divisor is 0" do + assert_raise ArithmeticError, fn -> Integer.floor_div(3, 0) end + assert_raise ArithmeticError, fn -> Integer.floor_div(-50, 0) end + end + + test "floor_div/2 raises ArithmeticError when non-integers used as arguments" do + assert_raise ArithmeticError, fn -> Integer.floor_div(3.0, 2) end + assert_raise ArithmeticError, fn -> Integer.floor_div(20, 1.2) end + end + + test "ceil_div/2" do + assert Integer.ceil_div(10, -5) == -2 + assert Integer.ceil_div(3, 2) == 2 + assert Integer.ceil_div(0, 10) == 0 + assert Integer.ceil_div(0, -10) == 0 + assert Integer.ceil_div(30000, -2001) == -14 + assert Integer.ceil_div(-20, 11) == -1 + assert Integer.ceil_div(-55, -22) == 3 + end + + test "ceil_div/2 raises ArithmeticError when divisor is 0" do + assert_raise ArithmeticError, fn -> Integer.ceil_div(3, 0) end + assert_raise ArithmeticError, fn -> Integer.ceil_div(-50, 0) end + end + + test "ceil_div/2 raises ArithmeticError when non-integers used as arguments" do + assert_raise ArithmeticError, fn -> Integer.ceil_div(3.0, 2) end + assert_raise ArithmeticError, fn -> Integer.ceil_div(20, 1.2) end + end + + test "digits/2" do + assert Integer.digits(0) == [0] + assert Integer.digits(0, 2) == [0] + assert Integer.digits(1) == [1] + assert Integer.digits(-1) == [-1] + assert Integer.digits(123, 123) == [1, 0] + assert Integer.digits(-123, 123) == [-1, 0] + assert Integer.digits(456, 1000) == [456] + assert Integer.digits(-456, 1000) == [-456] + assert Integer.digits(123) == [1, 2, 3] + assert Integer.digits(-123) == [-1, -2, -3] + assert Integer.digits(58127, 2) == [1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1] + assert Integer.digits(-58127, 2) == [-1, -1, -1, 0, 0, 0, -1, -1, 0, 0, 0, 0, -1, -1, -1, -1] + + for n <- Enum.to_list(-1..1) do + assert_raise FunctionClauseError, fn -> + Integer.digits(10, n) + Integer.digits(-10, n) + end + end end - test :even? do - assert Integer.even?(0) == true - assert Integer.even?(1) == false - assert Integer.even?(2) == true - assert Integer.even?(3) == false - assert Integer.even?(-1) == false - assert Integer.even?(-2) == true - assert Integer.even?(-3) == false + test "undigits/2" do + assert Integer.undigits([]) == 0 + assert Integer.undigits([0]) == 0 + assert Integer.undigits([1]) == 1 + assert Integer.undigits([1, 0, 1]) == 101 + assert Integer.undigits([1, 4], 16) == 0x14 + assert Integer.undigits([1, 4], 8) == 0o14 + assert Integer.undigits([1, 1], 2) == 0b11 + assert Integer.undigits([1, 2, 3, 4, 5]) == 12345 + assert Integer.undigits([1, 0, -5]) == 95 + assert Integer.undigits([-1, -1, -5]) == -115 + assert Integer.undigits([0, 0, 0, -1, -1, -5]) == -115 + + for n <- Enum.to_list(-1..1) do + assert_raise FunctionClauseError, fn -> + Integer.undigits([1, 0, 1], n) + end + end + + assert_raise ArgumentError, "invalid digit 17 in base 16", fn -> + Integer.undigits([1, 2, 17], 16) + end end - test :parse do + test "parse/2" do assert Integer.parse("12") === {12, ""} + assert Integer.parse("012") === {12, ""} + assert Integer.parse("+12") === {12, ""} assert Integer.parse("-12") === {-12, ""} - assert Integer.parse("123456789") === {123456789, ""} + assert Integer.parse("123456789") === {123_456_789, ""} assert Integer.parse("12.5") === {12, ".5"} assert Integer.parse("7.5e-3") === {7, ".5e-3"} assert Integer.parse("12x") === {12, "x"} @@ -35,5 +159,128 @@ defmodule IntegerTest do assert Integer.parse("--1") === :error assert Integer.parse("+-1") === :error assert Integer.parse("three") === :error + + assert Integer.parse("12", 10) === {12, ""} + assert Integer.parse("-12", 12) === {-14, ""} + assert Integer.parse("12345678", 9) === {6_053_444, ""} + assert Integer.parse("3.14", 4) === {3, ".14"} + assert Integer.parse("64eb", 16) === {25835, ""} + assert Integer.parse("64eb", 10) === {64, "eb"} + assert Integer.parse("10", 2) === {2, ""} + assert Integer.parse("++4", 10) === :error + + # Base should be in range 2..36 + assert_raise ArgumentError, "invalid base 1", fn -> Integer.parse("2", 1) end + assert_raise ArgumentError, "invalid base 37", fn -> Integer.parse("2", 37) end + + # Base should be an integer + assert_raise ArgumentError, "invalid base 10.2", fn -> Integer.parse("2", 10.2) end + + assert_raise ArgumentError, "invalid base nil", fn -> Integer.parse("2", nil) end + end + + test "to_string/2" do + assert Integer.to_string(42) == "42" + assert Integer.to_string(+42) == "42" + assert Integer.to_string(-42) == "-42" + assert Integer.to_string(-0001) == "-1" + + for n <- [42.0, :forty_two, ~c"42", "42"] do + assert_raise ArgumentError, fn -> + Integer.to_string(n) + end + end + + assert Integer.to_string(42, 2) == "101010" + assert Integer.to_string(42, 10) == "42" + assert Integer.to_string(42, 16) == "2A" + assert Integer.to_string(+42, 16) == "2A" + assert Integer.to_string(-42, 16) == "-2A" + assert Integer.to_string(-042, 16) == "-2A" + + for n <- [42.0, :forty_two, ~c"42", "42"] do + assert_raise ArgumentError, fn -> + Integer.to_string(n, 42) + end + end + + for n <- [-1, 0, 1, 37] do + assert_raise ArgumentError, fn -> + Integer.to_string(42, n) + end + + assert_raise ArgumentError, fn -> + Integer.to_string(n, n) + end + end + end + + test "to_charlist/2" do + module = String.to_atom("Elixir.Integer") + + assert Integer.to_charlist(42) == ~c"42" + assert Integer.to_charlist(+42) == ~c"42" + assert Integer.to_charlist(-42) == ~c"-42" + assert Integer.to_charlist(-0001) == ~c"-1" + + for n <- [42.0, :forty_two, ~c"42", "42"] do + assert_raise ArgumentError, fn -> + Integer.to_charlist(n) + end + end + + assert module.to_char_list(42) == ~c"42" + assert module.to_char_list(42, 2) == ~c"101010" + + assert Integer.to_charlist(42, 2) == ~c"101010" + assert Integer.to_charlist(42, 10) == ~c"42" + assert Integer.to_charlist(42, 16) == ~c"2A" + assert Integer.to_charlist(+42, 16) == ~c"2A" + assert Integer.to_charlist(-42, 16) == ~c"-2A" + assert Integer.to_charlist(-042, 16) == ~c"-2A" + + for n <- [42.0, :forty_two, ~c"42", "42"] do + assert_raise ArgumentError, fn -> + Integer.to_charlist(n, 42) + end + end + + for n <- [-1, 0, 1, 37] do + assert_raise ArgumentError, fn -> + Integer.to_charlist(42, n) + end + + assert_raise ArgumentError, fn -> + Integer.to_charlist(n, n) + end + end + end + + test "gcd/2" do + assert Integer.gcd(1, 5) == 1 + assert Integer.gcd(2, 3) == 1 + assert Integer.gcd(8, 12) == 4 + assert Integer.gcd(-8, 12) == 4 + assert Integer.gcd(8, -12) == 4 + assert Integer.gcd(-8, -12) == 4 + assert Integer.gcd(27, 27) == 27 + assert Integer.gcd(-27, -27) == 27 + assert Integer.gcd(-27, 27) == 27 + assert Integer.gcd(0, 3) == 3 + assert Integer.gcd(0, -3) == 3 + assert Integer.gcd(3, 0) == 3 + assert Integer.gcd(-3, 0) == 3 + assert Integer.gcd(0, 0) == 0 + end + + test "extended_gcd" do + # Poor's man properby based testing + for _ <- 1..100 do + left = :rand.uniform(1000) + right = :rand.uniform(1000) + {gcd, m, n} = Integer.extended_gcd(left, right) + assert Integer.gcd(left, right) == gcd + assert m * left + n * right == gcd + end end end diff --git a/lib/elixir/test/elixir/io/ansi/docs_test.exs b/lib/elixir/test/elixir/io/ansi/docs_test.exs index 66173bc3b5c..23ace94e8c1 100644 --- a/lib/elixir/test/elixir/io/ansi/docs_test.exs +++ b/lib/elixir/test/elixir/io/ansi/docs_test.exs @@ -1,219 +1,668 @@ -Code.require_file "../../test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../../test_helper.exs", __DIR__) defmodule IO.ANSI.DocsTest do use ExUnit.Case, async: true import ExUnit.CaptureIO - def format_heading(str) do - capture_io(fn -> IO.ANSI.Docs.print_heading(str, []) end) |> String.strip - end - - def format(str) do - capture_io(fn -> IO.ANSI.Docs.print(str, []) end) |> String.strip - end - - test "heading is formatted" do - result = format_heading("wibble") - assert String.starts_with?(result, "\e[0m\n\e[7m\e[33m\e[1m") - assert String.ends_with?(result, "\e[0m\n\e[0m") - assert String.contains?(result, " wibble ") - end - - test "first level heading is converted" do - result = format("# wibble\n\ntext\n") - assert result == "\e[33m\e[1mWIBBLE\e[0m\n\e[0m\ntext\n\e[0m" - end - - test "second level heading is converted" do - result = format("## wibble\n\ntext\n") - assert result == "\e[33m\e[1mwibble\e[0m\n\e[0m\ntext\n\e[0m" - end - - test "third level heading is converted" do - result = format("## wibble\n\ntext\n") - assert result == "\e[33m\e[1mwibble\e[0m\n\e[0m\ntext\n\e[0m" - end - - test "code block is converted" do - result = format("line\n\n code\n code2\n\nline2\n") - assert result == "line\n\e[0m\n\e[36m\e[1m┃ code\n┃ code2\e[0m\n\e[0m\nline2\n\e[0m" - end - - test "* list is converted" do - result = format("* one\n* two\n* three\n") - assert result == "• one\n• two\n• three\n\e[0m" - end - - test "* list surrounded by text is converted" do - result = format("Count:\n\n* one\n* two\n* three\n\nDone") - assert result == "Count:\n\e[0m\n• one\n• two\n• three\n\e[0m\nDone\n\e[0m" - end - - test "* list with continuation is converted" do - result = format("* one\n two\n three\n* four") - assert result == "• one two three\n• four" - end - - test "* nested lists are converted" do - result = format("* one\n * one.one\n * one.two\n* two") - assert result == "• one\n • one.one\n • one.two\n• two" - end - - test "* lists with spaces are converted" do - result = format(" * one\n * two\n * three") - assert result == "• one\n• two\n• three" - end + def format_headings(list) do + capture_io(fn -> IO.ANSI.Docs.print_headings(list, []) end) |> String.trim_trailing() + end + + def format_metadata(map) do + capture_io(fn -> IO.ANSI.Docs.print_metadata(map, []) end) + end + + def format_markdown(str, opts \\ []) do + capture_io(fn -> IO.ANSI.Docs.print(str, "text/markdown", opts) end) + |> String.trim_trailing() + end + + describe "heading" do + test "is formatted" do + result = format_headings(["foo"]) + assert String.starts_with?(result, "\e[0m\n\e[7m\e[33m") + assert String.ends_with?(result, "\e[0m\n\e[0m") + assert String.contains?(result, " foo ") + end + + test "multiple entries formatted" do + result = format_headings(["foo", "bar"]) + assert :binary.matches(result, "\e[0m\n\e[7m\e[33m") |> length() == 2 + assert String.starts_with?(result, "\e[0m\n\e[7m\e[33m") + assert String.ends_with?(result, "\e[0m\n\e[0m") + assert String.contains?(result, " foo ") + assert String.contains?(result, " bar ") + end + + test "is correctly formatted when newline character is present" do + result = format_headings(["foo\nbar"]) + assert :binary.matches(result, "\e[0m\n\e[7m\e[33m") |> length() == 2 + assert ["\e[0m", foo_line, bar_line, "\e[0m"] = String.split(result, "\n") + assert Regex.match?(~r/\e\[7m\e\[33m +foo +\e\[0m/, foo_line) + assert Regex.match?(~r/\e\[7m\e\[33m +bar +\e\[0m/, bar_line) + end + end + + describe "metadata" do + test "is formatted" do + result = + format_metadata(%{ + since: "1.2.3", + deprecated: "Use that other one", + author: "Alice", + delegate_to: {Foo, :bar, 3} + }) + + assert result == """ + \e[33mdelegate_to:\e[0m Foo.bar/3 + \e[33mdeprecated:\e[0m Use that other one + \e[33msince:\e[0m 1.2.3 + + """ + + assert format_metadata(%{author: "Alice"}) == "" + end + end + + describe "markdown" do + test "first level heading is converted" do + result = format_markdown("# wibble\n\ntext\n") + assert result == "\e[33m# wibble\e[0m\n\e[0m\ntext\n\e[0m" + end + + test "second level heading is converted" do + result = format_markdown("## wibble\n\ntext\n") + assert result == "\e[33m## wibble\e[0m\n\e[0m\ntext\n\e[0m" + end + + test "third level heading is converted" do + result = format_markdown("### wibble\n\ntext\n") + assert result == "\e[33m### wibble\e[0m\n\e[0m\ntext\n\e[0m" + end + + test "short single-line quote block is converted into single-line quote" do + result = + format_markdown(""" + line + + > normal *italics* `code` + + line2 + """) + + assert result == + """ + line + \e[0m + \e[90m> \e[0mnormal \e[4mitalics\e[0m \e[36mcode\e[0m + \e[0m + line2 + \e[0m\ + """ + end + + test "short multi-line quote block is converted into single-line quote" do + result = + format_markdown(""" + line + + > normal + > *italics* + > `code` + + line2 + """) + + assert result == + """ + line + \e[0m + \e[90m> \e[0mnormal \e[4mitalics\e[0m \e[36mcode\e[0m + \e[0m + line2 + \e[0m\ + """ + end + + test "long multi-line quote block is converted into wrapped multi-line quote" do + result = + format_markdown(""" + line + + > normal + > *italics* + > `code` + > some-extremely-long-word-which-can-not-possibly-fit-into-the-previous-line + + line2 + """) + + assert result == + """ + line + \e[0m + \e[90m> \e[0mnormal \e[4mitalics\e[0m \e[36mcode\e[0m + \e[90m> \e[0msome-extremely-long-word-which-can-not-possibly-fit-into-the-previous-line + \e[0m + line2 + \e[0m\ + """ + end + + test "multi-line quote block containing empty lines is converted into wrapped multi-line quote" do + result = + format_markdown(""" + line + + > normal + > *italics* + > + > `code` + > some-extremely-long-word-which-can-not-possibly-fit-into-the-previous-line + + line2 + """) + + assert result == + """ + line + \e[0m + \e[90m> \e[0mnormal \e[4mitalics\e[0m + \e[90m> \e[0m + \e[90m> \e[0m\e[36mcode\e[0m + \e[90m> \e[0msome-extremely-long-word-which-can-not-possibly-fit-into-the-previous-line + \e[0m + line2 + \e[0m\ + """ + end + + test "code block is converted" do + result = format_markdown("line\n\n code\n code2\n\nline2\n") + assert result == "line\n\e[0m\n\e[36m code\n code2\e[0m\n\e[0m\nline2\n\e[0m" + end + + test "fenced code block is converted" do + result = format_markdown("line\n```\ncode\ncode2\n```\nline2\n") + assert result == "line\n\e[0m\n\e[36m code\n code2\e[0m\n\e[0m\nline2\n\e[0m" + result = format_markdown("line\n```elixir\ncode\ncode2\n```\nline2\n") + assert result == "line\n\e[0m\n\e[36m code\n code2\e[0m\n\e[0m\nline2\n\e[0m" + end + + test "mermaid fenced code block is discarded" do + result = format_markdown("line\n```mermaid\ncode\ncode2\n```\nline2\n") + assert result == "line\n\e[0m\nline2\n\e[0m" + end + + test "* list is converted" do + result = format_markdown("* one\n* two\n* three\n") + assert result == " • one\n • two\n • three\n\e[0m" + end + + test "* list is converted without ansi" do + result = format_markdown("* one\n* two\n* three\n", enabled: false) + assert result == " * one\n * two\n * three" + end + + test "* list surrounded by text is converted" do + result = format_markdown("Count:\n\n* one\n* two\n* three\n\nDone") + assert result == "Count:\n\e[0m\n • one\n • two\n • three\n\e[0m\nDone\n\e[0m" + end + + test "* list with continuation is converted" do + result = format_markdown("* one\ntwo\n\n three\nfour\n* five") + assert result == " • one two\n three four\n\e[0m\n • five\n\e[0m" + end + + test "* nested lists are converted" do + result = format_markdown("* one\n * one.one\n * one.two\n* two") + assert result == " • one\n • one.one\n • one.two\n\e[0m\n • two\n\e[0m" + end + + test "* deep nested lists are converted" do + result = + format_markdown(""" + * level 1 + * level 2a + * level 2b + * level 3 + * level 4a + * level 4b + * level 5 + * level 6 + """) + + assert result == + " • level 1\n • level 2a\n • level 2b\n • level 3\n • level 4a\n • level 4b\n • level 5\n • level 6\n\e[0m\n\e[0m\n\e[0m\n\e[0m\n\e[0m\n\e[0m" + end + + test "* lists with spaces are converted" do + result = format_markdown(" * one\n * two\n * three") + assert result == " • one\n • two\n • three\n\e[0m" + end + + test "* lists with code" do + result = format_markdown(" * one\n two three") + assert result == " • one\n\e[36m two three\e[0m\n\e[0m\n\e[0m" + end + + test "- list is converted" do + result = format_markdown("- one\n- two\n- three\n") + assert result == " • one\n • two\n • three\n\e[0m" + end + + test "+ list is converted" do + result = format_markdown("+ one\n+ two\n+ three\n") + assert result == " • one\n • two\n • three\n\e[0m" + end + + test "+ and - nested lists are converted" do + result = format_markdown("- one\n + one.one\n + one.two\n- two") + assert result == " • one\n • one.one\n • one.two\n\e[0m\n • two\n\e[0m" + end + + test "paragraphs are split" do + result = format_markdown("para1\n\npara2") + assert result == "para1\n\e[0m\npara2\n\e[0m" + end + + test "extra whitespace is ignored between paras" do + result = format_markdown("para1\n \npara2") + assert result == "para1\n\e[0m\npara2\n\e[0m" + end + + test "extra whitespace doesn't mess up a following list" do + result = format_markdown("para1\n \n* one\n* two") + assert result == "para1\n\e[0m\n • one\n • two\n\e[0m" + end + + test "star/underscore/backtick works" do + result = format_markdown("*world*") + assert result == "\e[4mworld\e[0m\n\e[0m" + + result = format_markdown("*world*.") + assert result == "\e[4mworld\e[0m.\n\e[0m" + + result = format_markdown("**world**") + assert result == "\e[1mworld\e[0m\n\e[0m" + + result = format_markdown("_world_") + assert result == "\e[4mworld\e[0m\n\e[0m" + + result = format_markdown("__world__") + assert result == "\e[1mworld\e[0m\n\e[0m" + + result = format_markdown("`world`") + assert result == "\e[36mworld\e[0m\n\e[0m" + end + + test "star/underscore/backtick works across words" do + result = format_markdown("*hello world*") + assert result == "\e[4mhello world\e[0m\n\e[0m" + + result = format_markdown("**hello world**") + assert result == "\e[1mhello world\e[0m\n\e[0m" + + result = format_markdown("_hello world_") + assert result == "\e[4mhello world\e[0m\n\e[0m" + + result = format_markdown("__hello world__") + assert result == "\e[1mhello world\e[0m\n\e[0m" + + result = format_markdown("`hello world`") + assert result == "\e[36mhello world\e[0m\n\e[0m" + end + + test "star/underscore/backtick works across words with ansi disabled" do + result = format_markdown("*hello world*", enabled: false) + assert result == "*hello world*" + + result = format_markdown("**hello world**", enabled: false) + assert result == "**hello world**" + + result = format_markdown("_hello world_", enabled: false) + assert result == "_hello world_" + + result = format_markdown("__hello world__", enabled: false) + assert result == "__hello world__" - test "- list is converted" do - result = format("- one\n- two\n- three\n") - assert result == "• one\n• two\n• three\n\e[0m" - end + result = format_markdown("`hello world`", enabled: false) + assert result == "`hello world`" + end - test "- list surrounded by text is converted" do - result = format("Count:\n\n- one\n- two\n- three\n\nDone") - assert result == "Count:\n\e[0m\n• one\n• two\n• three\n\e[0m\nDone\n\e[0m" - end + test "multiple stars/underscores/backticks work" do + result = format_markdown("*hello world* *hello world*") + assert result == "\e[4mhello world\e[0m \e[4mhello world\e[0m\n\e[0m" - test "- list with continuation is converted" do - result = format("- one\n two\n three\n- four") - assert result == "• one two three\n• four" - end + result = format_markdown("_hello world_ _hello world_") + assert result == "\e[4mhello world\e[0m \e[4mhello world\e[0m\n\e[0m" - test "+ list is converted" do - result = format("+ one\n+ two\n+ three\n") - assert result == "• one\n• two\n• three\n\e[0m" - end + result = format_markdown("`hello world` `hello world`") + assert result == "\e[36mhello world\e[0m \e[36mhello world\e[0m\n\e[0m" + end - test "+ and - nested lists are converted" do - result = format("- one\n + one.one\n + one.two\n- two") - assert result == "• one\n • one.one\n • one.two\n• two" - end + test "multiple stars/underscores/backticks work when separated by other words" do + result = format_markdown("*hello world* unit test *hello world*") + assert result == "\e[4mhello world\e[0m unit test \e[4mhello world\e[0m\n\e[0m" - test "paragraphs are split" do - result = format("para1\n\npara2") - assert result == "para1\n\e[0m\npara2\n\e[0m" - end + result = format_markdown("_hello world_ unit test _hello world_") + assert result == "\e[4mhello world\e[0m unit test \e[4mhello world\e[0m\n\e[0m" - test "extra whitespace is ignored between paras" do - result = format("para1\n \npara2") - assert result == "para1\n\e[0m\npara2\n\e[0m" - end + result = format_markdown("`hello world` unit test `hello world`") + assert result == "\e[36mhello world\e[0m unit test \e[36mhello world\e[0m\n\e[0m" + end - test "extra whitespace doesn't mess up a following list" do - result = format("para1\n \n* one\n* two") - assert result == "para1\n\e[0m\n• one\n• two" - end + test "star/underscore preceded by space doesn't get interpreted" do + result = format_markdown("_unit _size") + assert result == "_unit _size\n\e[0m" - test "star/underscore/backtick works" do - result = format("*world*") - assert result == "\e[1mworld\e[0m\n\e[0m" + result = format_markdown("**unit **size") + assert result == "**unit **size\n\e[0m" - result = format("**world**") - assert result == "\e[1mworld\e[0m\n\e[0m" + result = format_markdown("*unit *size") + assert result == "*unit *size\n\e[0m" + end - result = format("_world_") - assert result == "\e[4mworld\e[0m\n\e[0m" + test "star/underscore/backtick preceded by non-space delimiters gets interpreted" do + result = format_markdown("(`hello world`)") + assert result == "(\e[36mhello world\e[0m)\n\e[0m" + result = format_markdown("<`hello world`>") + assert result == "<\e[36mhello world\e[0m>\n\e[0m" - result = format("`world`") - assert result == "\e[36mworld\e[0m\n\e[0m" - end + result = format_markdown("(*hello world*)") + assert result == "(\e[4mhello world\e[0m)\n\e[0m" + result = format_markdown("@*hello world*@") + assert result == "@\e[4mhello world\e[0m@\n\e[0m" - test "star/underscore/backtick works accross words" do - result = format("*hello world*") - assert result == "\e[1mhello world\e[0m\n\e[0m" + result = format_markdown("(_hello world_)") + assert result == "(\e[4mhello world\e[0m)\n\e[0m" + result = format_markdown("'_hello world_'") + assert result == "'\e[4mhello world\e[0m'\n\e[0m" + end - result = format("**hello world**") - assert result == "\e[1mhello world\e[0m\n\e[0m" + test "star/underscore/backtick starts/ends within a word doesn't get interpreted" do + result = format_markdown("foo_bar, foo_bar_baz!") + assert result == "foo_bar, foo_bar_baz!\n\e[0m" - result = format("_hello world_") - assert result == "\e[4mhello world\e[0m\n\e[0m" + result = format_markdown("_foo_bar") + assert result == "_foo_bar\n\e[0m" - result = format("`hello world`") - assert result == "\e[36mhello world\e[0m\n\e[0m" - end + result = format_markdown("foo_bar_") + assert result == "foo_bar_\n\e[0m" - test "star/underscore preceeded by space doesn't get interpreted" do - result = format("_unit _size") - assert result == "_unit _size\n\e[0m" + result = format_markdown("foo*bar, foo*bar*baz!") + assert result == "foo*bar, foo*bar*baz!\n\e[0m" - result = format("**unit **size") - assert result == "**unit **size\n\e[0m" + result = format_markdown("*foo*bar") + assert result == "*foo*bar\n\e[0m" - result = format("*unit *size") - assert result == "*unit *size\n\e[0m" - end + result = format_markdown("foo*bar*") + assert result == "foo*bar*\n\e[0m" + end - test "backtick preceeded by space gets interpreted" do - result = format("`unit `size") - assert result == "\e[36munit \e[0msize\n\e[0m" - end - - test "star/underscore/backtick with leading escape" do - result = format("\\_unit_") - assert result == "_unit_\n\e[0m" - - result = format("\\*unit*") - assert result == "*unit*\n\e[0m" + test "backtick preceded by space gets interpreted" do + result = format_markdown("`unit `size") + assert result == "\e[36munit \e[0msize\n\e[0m" + end - result = format("\\`unit`") - assert result == "`unit`\n\e[0m" - end + test "backtick does not escape characters" do + result = format_markdown("`Ctrl+\\ `") + assert result == "\e[36mCtrl+\\ \e[0m\n\e[0m" + end - test "star/underscore/backtick with closing escape" do - result = format("_unit\\_") - assert result == "_unit_\n\e[0m" + test "star/underscore/backtick with leading escape" do + result = format_markdown("\\_unit_") + assert result == "_unit_\n\e[0m" - result = format("*unit\\*") - assert result == "*unit*\n\e[0m" + result = format_markdown("\\*unit*") + assert result == "*unit*\n\e[0m" - result = format("`unit\\`") - assert result == "\e[36munit\\\e[0m\n\e[0m" - end + result = format_markdown("\\`unit`") + assert result == "`unit`\n\e[0m" + end - test "star/underscore/backtick with double escape" do - result = format("\\\\*world*") - assert result == "\\\e[1mworld\e[0m\n\e[0m" + test "star/underscore/backtick with closing escape" do + result = format_markdown("_unit\\_") + assert result == "_unit_\n\e[0m" - result = format("\\\\_world_") - assert result == "\\\e[4mworld\e[0m\n\e[0m" + result = format_markdown("*unit\\*") + assert result == "*unit*\n\e[0m" - result = format("\\\\`world`") - assert result == "\\\e[36mworld\e[0m\n\e[0m" - end + result = format_markdown("`unit\\`") + assert result == "\e[36munit\\\e[0m\n\e[0m" + end - test "star/underscore/backtick when incomplete" do - result = format("unit_size") - assert result == "unit_size\n\e[0m" + test "star/underscore/backtick with double escape" do + result = format_markdown("\\\\*world*") + assert result == "\\\e[4mworld\e[0m\n\e[0m" + + result = format_markdown("\\\\_world_") + assert result == "\\\e[4mworld\e[0m\n\e[0m" + + result = format_markdown("\\\\`world`") + assert result == "\\\e[36mworld\e[0m\n\e[0m" + end + + test "star/underscore/backtick when incomplete" do + result = format_markdown("unit_size") + assert result == "unit_size\n\e[0m" + + result = format_markdown("unit`size") + assert result == "unit`size\n\e[0m" + + result = format_markdown("unit*size") + assert result == "unit*size\n\e[0m" + + result = format_markdown("unit**size") + assert result == "unit**size\n\e[0m" + end + + test "backtick with escape" do + result = format_markdown("`\\`") + assert result == "\e[36m\\\e[0m\n\e[0m" + end + + test "backtick close to underscores gets interpreted as code" do + result = format_markdown("`__world__`") + assert result == "\e[36m__world__\e[0m\n\e[0m" + end + + test "escaping of underlines within links" do + result = format_markdown("(https://en.wikipedia.org/wiki/ANSI_escape_code)") + assert result == "(https://en.wikipedia.org/wiki/ANSI_escape_code)\n\e[0m" + + result = + format_markdown("[ANSI escape code](https://en.wikipedia.org/wiki/ANSI_escape_code)") + + assert result == "ANSI escape code (https://en.wikipedia.org/wiki/ANSI_escape_code)\n\e[0m" + + result = format_markdown("(ftp://example.com/ANSI_escape_code.zip)") + assert result == "(ftp://example.com/ANSI_escape_code.zip)\n\e[0m" + end + + test "escaping of underlines within links does not escape surrounding text" do + result = + format_markdown( + "_emphasis_ (https://en.wikipedia.org/wiki/ANSI_escape_code) more _emphasis_" + ) + + assert result == + "\e[4memphasis\e[0m (https://en.wikipedia.org/wiki/ANSI_escape_code) more \e[4memphasis\e[0m\n\e[0m" + end + + test "escaping of underlines within links avoids false positives" do + assert format_markdown("`https_proxy`") == "\e[36mhttps_proxy\e[0m\n\e[0m" + end + + test "escaping of several Markdown links in one line" do + assert format_markdown("[List](`List`) (`[1, 2, 3]`), [Map](`Map`)") == + "List (\e[36mList\e[0m) (\e[36m[1, 2, 3]\e[0m), Map (\e[36mMap\e[0m)\n\e[0m" + end + + test "one reference link label per line" do + assert format_markdown(" [id]: //example.com\n [Elixir]: https://elixir-lang.org") == + " [id]: //example.com\n [Elixir]: https://elixir-lang.org" + end + end + + describe "markdown tables" do + test "lone thing that looks like a table line isn't" do + assert format_markdown("one\n2 | 3\ntwo\n") == "one 2 | 3 two\n\e[0m" + end + + test "lone table line at end of input isn't" do + assert format_markdown("one\n2 | 3") == "one 2 | 3\n\e[0m" + end + + test "two successive table lines are a table" do + # note spacing + assert format_markdown("a | b\none | two\n") == "a | b \none | two\n\e[0m" + end + + test "table with heading" do + assert format_markdown("column 1 | and 2\n-- | --\na | b\none | two\n") == + "\e[7mcolumn 1 | and 2\e[0m\na | b \none | two \n\e[0m" + end + + test "table with heading alignment" do + table = """ + column 1 | 2 | and three + -------: | :------: | :----- + a | even | c\none | odd | three + """ + + expected = + """ + \e[7mcolumn 1 | 2 | and three\e[0m + a | even | c\s\s\s\s\s\s\s\s + one | odd | three\s\s\s\s + \e[0m + """ + |> String.trim_trailing() + + assert format_markdown(table) == expected + end + + test "table with heading alignment and no space around \"|\"" do + table = """ + | Value | Encoding | Value | Encoding | + |------:|:---------|------:|:---------| + | 0 | A | 17 | R | + | 1 | B | 18 | S | + """ + + expected = + "\e[7m" <> + "Value | Encoding | Value | Encoding\e[0m\n" <> + " 0 | A | 17 | R \n" <> + " 1 | B | 18 | S \n\e[0m" + + assert format_markdown(table) == expected + end + + test "table with formatting in cells" do + assert format_markdown("`a` | _b_\nc | d") == "\e[36ma\e[0m | \e[4mb\e[0m\nc | d\n\e[0m" + + assert format_markdown("`abc` | d \n`e` | f") == + "\e[36mabc\e[0m | d\n\e[36me\e[0m | f\n\e[0m" + end + + test "table with variable number of columns" do + assert format_markdown("a | b | c\nd | e") == "a | b | c\nd | e | \n\e[0m" + end + + test "table with escaped \"|\" inside cell" do + table = "a | smth\\|smth_else | c\nd | e | f" + + expected = + """ + a | smth|smth_else | c + d | e | f + \e[0m + """ + |> String.trim_trailing() + + assert format_markdown(table) == expected + end + + test "table with last two columns empty" do + table = """ + AAA | | | + BBB | CCC | | + GGG | HHH | III | + JJJ | KKK | LLL | MMM + """ + + expected = + """ + AAA | | |\s\s\s\s + BBB | CCC | |\s\s\s\s + GGG | HHH | III |\s\s\s\s + JJJ | KKK | LLL | MMM + \e[0m + """ + |> String.trim_trailing() + + assert format_markdown(table) == expected + end + + test "HTML comments are ignored" do + markdown = """ + + hello + """ + + assert format_markdown(markdown) == "hello\n\e[0m" + + markdown = """ + + + hello + """ + + assert format_markdown(markdown) == "hello\n\e[0m" + + markdown = """ + hello + + world + """ + + assert format_markdown(markdown) == "hello\n\e[0m\nworld\n\e[0m" + + markdown = """ + hello + + + + world + """ - result = format("unit`size") - assert result == "unit`size\n\e[0m" + assert format_markdown(markdown) == "hello\n\e[0m\nworld\n\e[0m" - result = format("unit*size") - assert result == "unit*size\n\e[0m" + markdown = """ + hello + world + """ + + assert format_markdown(markdown) == "hello world\n\e[0m" - result = format("unit**size") - assert result == "unit**size\n\e[0m" - end - - test "backtick with escape" do - result = format("`\\`") - assert result == "\e[36m\\\e[0m\n\e[0m" - end - - test "backtick close to underscores gets interpreted as code" do - result = format("`__world__`") - assert result == "\e[36m__world__\e[0m\n\e[0m" - end + markdown = """ + hello world + """ - test "backtick works inside parenthesis" do - result = format("(`hello world`)") - assert result == "(\e[36mhello world\e[0m)\n\e[0m" + assert format_markdown(markdown) == "hello world\n\e[0m" + end end - test "escaping of underlines within links" do - result = format("(http://en.wikipedia.org/wiki/ANSI_escape_code)") - assert result == "(http://en.wikipedia.org/wiki/ANSI_escape_code)\n\e[0m" - result = format("[ANSI escape code](http://en.wikipedia.org/wiki/ANSI_escape_code)") - assert result == "ANSI escape code (http://en.wikipedia.org/wiki/ANSI_escape_code)\n\e[0m" + describe "invalid format" do + test "prints message" do + assert capture_io(fn -> IO.ANSI.Docs.print("hello", "text/unknown", []) end) == + "\nUnknown documentation format \"text/unknown\"\n\n" + end end end diff --git a/lib/elixir/test/elixir/io/ansi_test.exs b/lib/elixir/test/elixir/io/ansi_test.exs index f05a0679985..ecafe2196a1 100644 --- a/lib/elixir/test/elixir/io/ansi_test.exs +++ b/lib/elixir/test/elixir/io/ansi_test.exs @@ -1,54 +1,239 @@ -Code.require_file "../test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) defmodule IO.ANSITest do use ExUnit.Case, async: true - test :escape_single do - assert IO.ANSI.escape("Hello, %{red}world!", true) == - "Hello, #{IO.ANSI.red}world!#{IO.ANSI.reset}" - assert IO.ANSI.escape("Hello, %{red}world!", true) == - "Hello, #{IO.ANSI.red}world!#{IO.ANSI.reset}" + doctest IO.ANSI + + test "format ansicode" do + assert IO.chardata_to_string(IO.ANSI.format(:green, true)) == + "#{IO.ANSI.green()}#{IO.ANSI.reset()}" + + assert IO.chardata_to_string(IO.ANSI.format(:green, false)) == "" + end + + test "format binary" do + assert IO.chardata_to_string(IO.ANSI.format("Hello, world!", true)) == "Hello, world!" + + assert IO.chardata_to_string(IO.ANSI.format("A map: %{foo: :bar}", false)) == + "A map: %{foo: :bar}" + end + + test "format empty list" do + assert IO.chardata_to_string(IO.ANSI.format([], true)) == "" + assert IO.chardata_to_string(IO.ANSI.format([], false)) == "" + end + + test "format ansicode list" do + assert IO.chardata_to_string(IO.ANSI.format([:red, :bright], true)) == + "#{IO.ANSI.red()}#{IO.ANSI.bright()}#{IO.ANSI.reset()}" + + assert IO.chardata_to_string(IO.ANSI.format([:red, :bright], false)) == "" + end + + test "format binary list" do + assert IO.chardata_to_string(IO.ANSI.format(["Hello, ", "world!"], true)) == "Hello, world!" + assert IO.chardata_to_string(IO.ANSI.format(["Hello, ", "world!"], false)) == "Hello, world!" + end + + test "format charlist" do + assert IO.chardata_to_string(IO.ANSI.format(~c"Hello, world!", true)) == "Hello, world!" + assert IO.chardata_to_string(IO.ANSI.format(~c"Hello, world!", false)) == "Hello, world!" + end + + test "format mixed list" do + data = ["Hello", ?,, 32, :red, "world!"] + + assert IO.chardata_to_string(IO.ANSI.format(data, true)) == + "Hello, #{IO.ANSI.red()}world!#{IO.ANSI.reset()}" + + assert IO.chardata_to_string(IO.ANSI.format(data, false)) == "Hello, world!" + end + + test "format nested list" do + data = ["Hello, ", ["nested", 32, :red, "world!"]] + + assert IO.chardata_to_string(IO.ANSI.format(data, true)) == + "Hello, nested #{IO.ANSI.red()}world!#{IO.ANSI.reset()}" + + assert IO.chardata_to_string(IO.ANSI.format(data, false)) == "Hello, nested world!" + end + + test "format improper list" do + data = ["Hello, ", :red, "world" | "!"] + + assert IO.chardata_to_string(IO.ANSI.format(data, true)) == + "Hello, #{IO.ANSI.red()}world!#{IO.ANSI.reset()}" + + assert IO.chardata_to_string(IO.ANSI.format(data, false)) == "Hello, world!" + end + + test "format nested improper list" do + data = [["Hello, " | :red], "world!" | :green] + + assert IO.chardata_to_string(IO.ANSI.format(data, true)) == + "Hello, #{IO.ANSI.red()}world!#{IO.ANSI.green()}#{IO.ANSI.reset()}" + + assert IO.chardata_to_string(IO.ANSI.format(data, false)) == "Hello, world!" + end + + test "format fragment" do + assert IO.chardata_to_string(IO.ANSI.format_fragment([:red, "Hello!"], true)) == + "#{IO.ANSI.red()}Hello!" + end + + test "format invalid sequence" do + assert_raise ArgumentError, "invalid ANSI sequence specification: :brigh", fn -> + IO.ANSI.format([:brigh, "Hello!"], true) + end + + assert_raise ArgumentError, "invalid ANSI sequence specification: nil", fn -> + IO.ANSI.format(["Hello!", nil], true) + end + + assert_raise ArgumentError, "invalid ANSI sequence specification: :brigh", fn -> + IO.ANSI.format([:brigh, "Hello!"], false) + end + + assert_raise ArgumentError, "invalid ANSI sequence specification: nil", fn -> + IO.ANSI.format(["Hello!", nil], false) + end + + assert_raise ArgumentError, "invalid ANSI sequence specification: :invalid", fn -> + IO.ANSI.format(:invalid, false) + end end - test :escape_non_attribute do - assert IO.ANSI.escape("Hello %{clear}world!", true) == - "Hello #{IO.ANSI.clear}world!#{IO.ANSI.reset}" - assert IO.ANSI.escape("Hello %{home}world!", true) == - "Hello #{IO.ANSI.home}world!#{IO.ANSI.reset}" + test "colors" do + assert IO.ANSI.red() == "\e[31m" + assert IO.ANSI.light_red() == "\e[91m" + + assert IO.ANSI.red_background() == "\e[41m" + assert IO.ANSI.light_red_background() == "\e[101m" end - test :escape_multiple do - assert IO.ANSI.escape("Hello, %{red,bright}world!", true) == - "Hello, #{IO.ANSI.red}#{IO.ANSI.bright}world!#{IO.ANSI.reset}" - assert IO.ANSI.escape("Hello, %{red, bright}world!", true) == - "Hello, #{IO.ANSI.red}#{IO.ANSI.bright}world!#{IO.ANSI.reset}" - assert IO.ANSI.escape("Hello, %{red , bright}world!", true) == - "Hello, #{IO.ANSI.red}#{IO.ANSI.bright}world!#{IO.ANSI.reset}" + test "color/1" do + assert IO.ANSI.color(0) == "\e[38;5;0m" + assert IO.ANSI.color(42) == "\e[38;5;42m" + assert IO.ANSI.color(255) == "\e[38;5;255m" + + assert_raise FunctionClauseError, fn -> + IO.ANSI.color(-1) + end + + assert_raise FunctionClauseError, fn -> + IO.ANSI.color(256) + end end - test :no_emit do - assert IO.ANSI.escape("Hello, %{}world!", false) == - "Hello, world!" + test "color/3" do + assert IO.ANSI.color(0, 4, 2) == "\e[38;5;42m" + assert IO.ANSI.color(1, 1, 1) == "\e[38;5;59m" + assert IO.ANSI.color(5, 5, 5) == "\e[38;5;231m" - assert IO.ANSI.escape("Hello, %{red,bright}world!", false) == - "Hello, world!" + assert_raise FunctionClauseError, fn -> + IO.ANSI.color(0, 6, 1) + end + + assert_raise FunctionClauseError, fn -> + IO.ANSI.color(5, -1, 1) + end end - test :fragment do - assert IO.ANSI.escape("%{red}", true) == "#{IO.ANSI.red}#{IO.ANSI.reset}" - assert IO.ANSI.escape_fragment("", true) == "" + test "color_background/1" do + assert IO.ANSI.color_background(0) == "\e[48;5;0m" + assert IO.ANSI.color_background(42) == "\e[48;5;42m" + assert IO.ANSI.color_background(255) == "\e[48;5;255m" + + assert_raise FunctionClauseError, fn -> + IO.ANSI.color_background(-1) + end + + assert_raise FunctionClauseError, fn -> + IO.ANSI.color_background(256) + end end - test :noop do - assert IO.ANSI.escape("") == "" + test "color_background/3" do + assert IO.ANSI.color_background(0, 4, 2) == "\e[48;5;42m" + assert IO.ANSI.color_background(1, 1, 1) == "\e[48;5;59m" + assert IO.ANSI.color_background(5, 5, 5) == "\e[48;5;231m" + + assert_raise FunctionClauseError, fn -> + IO.ANSI.color_background(0, 6, 1) + end + + assert_raise FunctionClauseError, fn -> + IO.ANSI.color_background(5, -1, 1) + end end - test :invalid do - assert_raise ArgumentError, "invalid ANSI sequence specification: brigh", fn -> - IO.ANSI.escape("%{brigh}, yes") + test "cursor/2" do + assert IO.ANSI.cursor(0, 0) == "\e[0;0H" + assert IO.ANSI.cursor(11, 12) == "\e[11;12H" + + assert_raise FunctionClauseError, fn -> + IO.ANSI.cursor(-1, 5) + end + + assert_raise FunctionClauseError, fn -> + IO.ANSI.cursor(5, -1) end - assert_raise ArgumentError, "invalid ANSI sequence specification: brigh", fn -> - IO.ANSI.escape("%{brigh,red}, yes") + end + + test "cursor_up/1" do + assert IO.ANSI.cursor_up() == "\e[1A" + assert IO.ANSI.cursor_up(12) == "\e[12A" + + assert_raise FunctionClauseError, fn -> + IO.ANSI.cursor_up(0) + end + + assert_raise FunctionClauseError, fn -> + IO.ANSI.cursor_up(-1) + end + end + + test "cursor_down/1" do + assert IO.ANSI.cursor_down() == "\e[1B" + assert IO.ANSI.cursor_down(2) == "\e[2B" + + assert_raise FunctionClauseError, fn -> + IO.ANSI.cursor_right(0) + end + + assert_raise FunctionClauseError, fn -> + IO.ANSI.cursor_down(-1) + end + end + + test "cursor_left/1" do + assert IO.ANSI.cursor_left() == "\e[1D" + assert IO.ANSI.cursor_left(3) == "\e[3D" + + assert_raise FunctionClauseError, fn -> + IO.ANSI.cursor_left(0) + end + + assert_raise FunctionClauseError, fn -> + IO.ANSI.cursor_left(-1) + end + end + + test "cursor_right/1" do + assert IO.ANSI.cursor_right() == "\e[1C" + assert IO.ANSI.cursor_right(4) == "\e[4C" + + assert_raise FunctionClauseError, fn -> + IO.ANSI.cursor_right(0) + end + + assert_raise FunctionClauseError, fn -> + IO.ANSI.cursor_right(-1) end end end diff --git a/lib/elixir/test/elixir/io_test.exs b/lib/elixir/test/elixir/io_test.exs index 7d46dbbf563..594d35feb80 100644 --- a/lib/elixir/test/elixir/io_test.exs +++ b/lib/elixir/test/elixir/io_test.exs @@ -1,29 +1,58 @@ -Code.require_file "test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("test_helper.exs", __DIR__) defmodule IOTest do use ExUnit.Case, async: true + + doctest IO + import ExUnit.CaptureIO - test :read_with_count do - {:ok, file} = File.open(Path.expand('fixtures/file.txt', __DIR__), [:char_list]) - assert 'FOO' == IO.read(file, 3) + test "read with count" do + {:ok, file} = File.open(Path.expand(~c"fixtures/file.txt", __DIR__), [:charlist]) + assert ~c"FOO" == IO.read(file, 3) assert File.close(file) == :ok end - test :read_with_utf8_and_binary do - {:ok, file} = File.open(Path.expand('fixtures/utf8.txt', __DIR__), [:utf8]) + test "read with UTF-8 and binary" do + {:ok, file} = File.open(Path.expand(~c"fixtures/utf8.txt", __DIR__), [:utf8]) assert "Русский" == IO.read(file, 7) assert File.close(file) == :ok end - test :binread do - {:ok, file} = File.open(Path.expand('fixtures/utf8.txt', __DIR__)) + test "read all charlist" do + {:ok, file} = File.open(Path.expand(~c"fixtures/multiline_file.txt", __DIR__), [:charlist]) + assert ~c"this is the first line\nthis is the second line\n" == IO.read(file, :eof) + assert File.close(file) == :ok + end + + test "read empty file" do + {:ok, file} = File.open(Path.expand(~c"fixtures/cp_mode", __DIR__), []) + assert IO.read(file, :eof) == :eof + assert File.close(file) == :ok + + {:ok, file} = File.open(Path.expand(~c"fixtures/cp_mode", __DIR__), [:charlist]) + assert IO.read(file, :eof) == :eof + assert File.close(file) == :ok + end + + test "binread" do + {:ok, file} = File.open(Path.expand(~c"fixtures/utf8.txt", __DIR__)) assert "Русский" == IO.binread(file, 14) assert File.close(file) == :ok end - test :getn do - {:ok, file} = File.open(Path.expand('fixtures/file.txt', __DIR__)) + test "binread eof" do + {:ok, file} = File.open(Path.expand(~c"fixtures/file.bin", __DIR__)) + assert "LF\nCR\rCRLF\r\nLFCR\n\r" == IO.binread(file, :eof) + assert File.close(file) == :ok + end + + test "getn" do + {:ok, file} = File.open(Path.expand(~c"fixtures/file.txt", __DIR__)) assert "F" == IO.getn(file, "") assert "O" == IO.getn(file, "") assert "O" == IO.getn(file, "") @@ -32,78 +61,221 @@ defmodule IOTest do assert File.close(file) == :ok end - test :getn_with_count do - {:ok, file} = File.open(Path.expand('fixtures/file.txt', __DIR__), [:char_list]) - assert 'FOO' == IO.getn(file, "", 3) + test "getn with count" do + {:ok, file} = File.open(Path.expand(~c"fixtures/file.txt", __DIR__), [:charlist]) + assert ~c"F" == IO.getn(file, "λ") + assert ~c"OO" == IO.getn(file, "", 2) + assert ~c"\n" == IO.getn(file, "λ", 99) + assert :eof == IO.getn(file, "λ", 1) assert File.close(file) == :ok end - test :getn_with_utf8_and_binary do - {:ok, file} = File.open(Path.expand('fixtures/utf8.txt', __DIR__), [:utf8]) + test "getn with eof" do + {:ok, file} = File.open(Path.expand(~c"fixtures/file.txt", __DIR__), [:charlist]) + assert ~c"F" == IO.getn(file, "λ") + assert ~c"OO\n" == IO.getn(file, "", :eof) + assert :eof == IO.getn(file, "", :eof) + assert File.close(file) == :ok + end + + test "getn with UTF-8 and binary" do + {:ok, file} = File.open(Path.expand(~c"fixtures/utf8.txt", __DIR__), [:utf8]) assert "Русский" == IO.getn(file, "", 7) + assert "\n日\n" == IO.getn(file, "", :eof) + assert :eof == IO.getn(file, "", :eof) assert File.close(file) == :ok end - test :gets do - {:ok, file} = File.open(Path.expand('fixtures/file.txt', __DIR__), [:char_list]) - assert 'FOO\n' == IO.gets(file, "") + test "gets" do + {:ok, file} = File.open(Path.expand(~c"fixtures/file.txt", __DIR__), [:charlist]) + assert ~c"FOO\n" == IO.gets(file, "") assert :eof == IO.gets(file, "") assert File.close(file) == :ok end - test :gets_with_utf8_and_binary do - {:ok, file} = File.open(Path.expand('fixtures/utf8.txt', __DIR__), [:utf8]) + test "gets with UTF-8 and binary" do + {:ok, file} = File.open(Path.expand(~c"fixtures/utf8.txt", __DIR__), [:utf8]) assert "Русский\n" == IO.gets(file, "") assert "日\n" == IO.gets(file, "") assert File.close(file) == :ok end - test :readline do - {:ok, file} = File.open(Path.expand('fixtures/file.txt', __DIR__)) + test "read with eof" do + {:ok, file} = File.open(Path.expand(~c"fixtures/file.txt", __DIR__)) + assert "FOO\n" == IO.read(file, :eof) + assert :eof == IO.read(file, :eof) + assert File.close(file) == :ok + end + + test "read with eof and UTF-8 and binary" do + {:ok, file} = File.open(Path.expand(~c"fixtures/utf8.txt", __DIR__), [:utf8]) + assert "Русский\n日\n" == IO.read(file, :eof) + assert :eof == IO.read(file, :eof) + assert File.close(file) == :ok + end + + test "readline" do + {:ok, file} = File.open(Path.expand(~c"fixtures/file.txt", __DIR__)) assert "FOO\n" == IO.read(file, :line) assert :eof == IO.read(file, :line) assert File.close(file) == :ok end - test :readline_with_utf8_and_binary do - {:ok, file} = File.open(Path.expand('fixtures/utf8.txt', __DIR__), [:utf8]) + test "readline with UTF-8 and binary" do + {:ok, file} = File.open(Path.expand(~c"fixtures/utf8.txt", __DIR__), [:utf8]) assert "Русский\n" == IO.read(file, :line) assert "日\n" == IO.read(file, :line) assert File.close(file) == :ok end - test :binreadline do - {:ok, file} = File.open(Path.expand('fixtures/utf8.txt', __DIR__)) + test "binread with eof" do + {:ok, file} = File.open(Path.expand(~c"fixtures/utf8.txt", __DIR__)) + assert "Русский\n日\n" == IO.binread(file, :eof) + assert :eof == IO.binread(file, :eof) + assert File.close(file) == :ok + end + + test "binread with line" do + {:ok, file} = File.open(Path.expand(~c"fixtures/utf8.txt", __DIR__)) assert "Русский\n" == IO.binread(file, :line) assert "日\n" == IO.binread(file, :line) assert File.close(file) == :ok end - test :puts_with_chardata do + test "puts with chardata" do assert capture_io(fn -> IO.puts("hello") end) == "hello\n" - assert capture_io(fn -> IO.puts('hello') end) == "hello\n" + assert capture_io(fn -> IO.puts(~c"hello") end) == "hello\n" assert capture_io(fn -> IO.puts(:hello) end) == "hello\n" assert capture_io(fn -> IO.puts(13) end) == "13\n" end - test :write_with_chardata do + describe "warn" do + test "with chardata" do + capture_io(:stderr, fn -> IO.warn("hello") end) + |> assert_emits(["hello", "(ex_unit #{System.version()}) lib/ex_unit"]) + + capture_io(:stderr, fn -> IO.warn(~c"hello") end) + |> assert_emits(["hello", "(ex_unit #{System.version()}) lib/ex_unit"]) + + capture_io(:stderr, fn -> IO.warn(:hello) end) + |> assert_emits(["hello", "(ex_unit #{System.version()}) lib/ex_unit"]) + + capture_io(:stderr, fn -> IO.warn(13) end) + |> assert_emits(["13", "(ex_unit #{System.version()}) lib/ex_unit"]) + end + + test "no stacktrace" do + assert capture_io(:stderr, fn -> IO.warn("hello", []) end) =~ "hello\n" + end + + test "with stacktrace" do + stacktrace = [{IEx.Evaluator, :eval, 4, [file: ~c"lib/iex/evaluator.ex", line: 108]}] + + output = capture_io(:stderr, fn -> IO.warn("hello", stacktrace) end) + + assert output =~ "hello" + assert output =~ "lib/iex/evaluator.ex:108: IEx.Evaluator.eval/4" + end + + test "with env" do + output = capture_io(:stderr, fn -> IO.warn("hello", __ENV__) end) + + assert output =~ "hello" + + assert output =~ + ~r"(lib/elixir/)?test/elixir/io_test.exs:#{__ENV__.line - 5}: IOTest.\"test warn with env\"/1" + end + + test "with options" do + capture_io(:stderr, fn -> + IO.warn("hello", line: 13, file: "lib/foo.ex", module: Foo, function: {:bar, 1}) + end) + |> assert_emits(["hello", "lib/foo.ex:13: Foo.bar/1"]) + + capture_io(:stderr, fn -> + IO.warn("hello", file: "lib/foo.ex", module: Foo, function: {:bar, 1}) + end) + |> assert_emits(["hello", "lib/foo.ex: Foo.bar/1"]) + + capture_io(:stderr, fn -> IO.warn("hello", file: "lib/foo.ex", module: Foo) end) + |> assert_emits(["hello", "lib/foo.ex: Foo (module)"]) + + capture_io(:stderr, fn -> IO.warn("hello", file: "lib/foo.ex") end) + |> assert_emits(["hello", "lib/foo.ex: (file)"]) + + capture_io(:stderr, fn -> + IO.warn("hello", file: "lib/foo.ex", function: {:bar, 1}) + end) + |> assert_emits(["hello", "lib/foo.ex: (file)"]) + + assert capture_io(:stderr, fn -> + IO.warn("hello", line: 13, module: Foo, function: {:bar, 1}) + end) =~ "hello" + end + end + + test "write with chardata" do assert capture_io(fn -> IO.write("hello") end) == "hello" - assert capture_io(fn -> IO.write('hello') end) == "hello" + assert capture_io(fn -> IO.write(~c"hello") end) == "hello" assert capture_io(fn -> IO.write(:hello) end) == "hello" assert capture_io(fn -> IO.write(13) end) == "13" end - test :gets_with_chardata do + test "gets with chardata" do assert capture_io("foo\n", fn -> IO.gets("hello") end) == "hello" - assert capture_io("foo\n", fn -> IO.gets('hello') end) == "hello" + assert capture_io("foo\n", fn -> IO.gets(~c"hello") end) == "hello" assert capture_io("foo\n", fn -> IO.gets(:hello) end) == "hello" assert capture_io("foo\n", fn -> IO.gets(13) end) == "13" end - test :getn_with_chardata do + test "getn with chardata" do assert capture_io("foo\n", fn -> IO.getn("hello", 3) end) == "hello" - assert capture_io("foo\n", fn -> IO.getn('hello', 3) end) == "hello" + assert capture_io("foo\n", fn -> IO.getn(~c"hello", 3) end) == "hello" assert capture_io("foo\n", fn -> IO.getn(:hello, 3) end) == "hello" assert capture_io("foo\n", fn -> IO.getn(13, 3) end) == "13" end + + test "getn with different arities" do + assert capture_io("hello", fn -> + input = IO.getn(">") + IO.write(input) + end) == ">h" + + assert capture_io("hello", fn -> + input = IO.getn(">", 3) + IO.write(input) + end) == ">hel" + + assert capture_io("hello", fn -> + input = IO.getn(Process.group_leader(), ">") + IO.write(input) + end) == ">h" + + assert capture_io("hello", fn -> + input = IO.getn(Process.group_leader(), ">") + IO.write(input) + end) == ">h" + + assert capture_io("hello", fn -> + input = IO.getn(Process.group_leader(), ">", 99) + IO.write(input) + end) == ">hello" + end + + test "inspect" do + assert capture_io(fn -> IO.inspect(1) end) == "1\n" + assert capture_io(fn -> IO.inspect(1, label: "foo") end) == "foo: 1\n" + assert capture_io(fn -> IO.inspect(1, label: :foo) end) == "foo: 1\n" + end + + test "stream" do + assert IO.stream() == IO.stream(:stdio, :line) + assert IO.binstream() == IO.binstream(:stdio, :line) + end + + defp assert_emits(output, messages) do + for m <- messages do + assert output =~ m + end + end end diff --git a/lib/elixir/test/elixir/json_test.exs b/lib/elixir/test/elixir/json_test.exs new file mode 100644 index 00000000000..79bff828702 --- /dev/null +++ b/lib/elixir/test/elixir/json_test.exs @@ -0,0 +1,199 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team + +Code.require_file("test_helper.exs", __DIR__) + +defmodule JSONTest do + use ExUnit.Case, async: true + + defmodule Token do + defstruct [:value] + + defimpl JSON.Encoder do + def encode(token, encoder) do + [?[, encoder.(token.value, encoder), ?]] + end + end + end + + doctest JSON + + describe "encode" do + test "atoms" do + assert JSON.encode!([nil, false, true, :another]) == "[null,false,true,\"another\"]" + end + + test "binaries" do + assert JSON.encode!("hello\0world\t✂️") == "\"hello\\u0000world\\t✂️\"" + end + + test "integers" do + assert JSON.encode!(123_456) == "123456" + end + + test "floats" do + assert JSON.encode!(123.456) == "123.456" + end + + test "maps" do + assert JSON.encode!(%{1 => 2, 3.0 => 4.0, ~c"list" => ~c"list", key: :bar}) == + "{\"1\":2,\"3.0\":4.0,\"key\":\"bar\",\"list\":[108,105,115,116]}" + end + + test "lists" do + assert JSON.encode!([1, 1.0, "one", %{1 => 2, 3.0 => 4.0, key: :bar}]) == + "[1,1.0,\"one\",{\"1\":2,\"3.0\":4.0,\"key\":\"bar\"}]" + end + + test "structs" do + assert JSON.encode!(%Token{value: :example}) == "[\"example\"]" + assert JSON.encode!(%Token{value: "hello\0world"}) == "[\"hello\\u0000world\"]" + end + + test "calendar" do + assert JSON.encode!(~D[2010-04-17]) == "\"2010-04-17\"" + assert JSON.encode!(~T[14:00:00.123]) == "\"14:00:00.123\"" + assert JSON.encode!(~N[2010-04-17 14:00:00.123]) == "\"2010-04-17T14:00:00.123\"" + assert JSON.encode!(~U[2010-04-17 14:00:00.123Z]) == "\"2010-04-17T14:00:00.123Z\"" + assert JSON.encode!(Duration.new!(month: 2, hour: 3)) == "\"P2MT3H\"" + end + end + + describe "JSON.Encoder" do + defp protocol_encode(term) do + term + |> JSON.Encoder.encode(&JSON.protocol_encode/2) + |> IO.iodata_to_binary() + end + + test "atoms" do + assert protocol_encode(:another) == "\"another\"" + assert protocol_encode([nil, false, true, :another]) == "[null,false,true,\"another\"]" + end + + test "binaries" do + assert protocol_encode("hello\0world\t✂️") == "\"hello\\u0000world\\t✂️\"" + end + + test "integers" do + assert protocol_encode(123_456) == "123456" + end + + test "floats" do + assert protocol_encode(123.456) == "123.456" + end + + test "maps" do + assert protocol_encode(%{1 => 2, 3.0 => 4.0, ~c"list" => ~c"list", key: :bar}) == + "{\"1\":2,\"3.0\":4.0,\"key\":\"bar\",\"list\":[108,105,115,116]}" + end + + test "lists" do + assert protocol_encode([1, 1.0, "one", %{1 => 2, 3.0 => 4.0, key: :bar}]) == + "[1,1.0,\"one\",{\"1\":2,\"3.0\":4.0,\"key\":\"bar\"}]" + end + + test "structs" do + assert protocol_encode(%Token{value: :example}) == "[\"example\"]" + assert protocol_encode(%Token{value: "hello\0world"}) == "[\"hello\\u0000world\"]" + end + end + + test "encode_to_iodata" do + list = JSON.encode_to_iodata!([1, 1.0, "one", %{1 => 2, 3.0 => 4.0, key: :bar}]) + assert is_list(list) + assert IO.iodata_to_binary(list) == "[1,1.0,\"one\",{\"1\":2,\"3.0\":4.0,\"key\":\"bar\"}]" + + list = + JSON.encode_to_iodata!([ + ~T[12:34:56.78], + ~D[2024-12-31], + ~N[2010-04-17 14:00:00.123], + ~U[2010-04-17 14:00:00.123Z], + Duration.new!(month: 2, hour: 3) + ]) + + assert IO.iodata_to_binary(list) == + ~s'["12:34:56.78","2024-12-31","2010-04-17T14:00:00.123","2010-04-17T14:00:00.123Z","P2MT3H"]' + end + + test "deprecated" do + assert JSON.encode!([:hello, "world"]) == "[\"hello\",\"world\"]" + + list = JSON.encode_to_iodata!([:hello, "world"]) + assert is_list(list) + assert IO.iodata_to_binary(list) == "[\"hello\",\"world\"]" + end + + describe "deriving" do + defmodule WithOnly do + @derive {JSON.Encoder, only: [:a, :b, :d]} + # The encoded order depends on only + defstruct Enum.shuffle([:a, :b, :c, :d]) + end + + test "with only" do + assert ["{\"a\":", _, ",\"b\":", _, ",\"d\":", _, 125] = + json = JSON.encode_to_iodata!(%WithOnly{a: :a, b: "b", c: make_ref(), d: [?d]}) + + assert IO.iodata_to_binary(json) == "{\"a\":\"a\",\"b\":\"b\",\"d\":[100]}" + end + + defmodule WithExcept do + @derive {JSON.Encoder, except: [:c]} + defstruct [:a, :b, :c, :d] + end + + test "with except" do + assert ["{\"a\":", _, ",\"b\":", _, ",\"d\":", _, 125] = + json = JSON.encode_to_iodata!(%WithExcept{a: :a, b: "b", c: make_ref(), d: [?d]}) + + assert IO.iodata_to_binary(json) == "{\"a\":\"a\",\"b\":\"b\",\"d\":[100]}" + end + + defmodule WithEmpty do + @derive {JSON.Encoder, only: []} + defstruct [:a, :b] + end + + test "with empty" do + assert JSON.encode_to_iodata!(%WithEmpty{}) == "{}" + end + end + + describe "decode" do + test "succeeds" do + assert JSON.decode("[null,123,456.7,\"string\",{\"key\":\"value\"}]") == + {:ok, [nil, 123, 456.7, "string", %{"key" => "value"}]} + + assert JSON.decode!("[null,123,456.7,\"string\",{\"key\":\"value\"}]") == + [nil, 123, 456.7, "string", %{"key" => "value"}] + end + + test "unexpected end" do + assert JSON.decode("{") == {:error, {:unexpected_end, 1}} + + assert_raise JSON.DecodeError, + "unexpected end of JSON binary at position (byte offset) 1", + fn -> JSON.decode!("{") end + end + + test "invalid byte" do + assert JSON.decode(",") == {:error, {:invalid_byte, 0, ?,}} + assert JSON.decode("123o") == {:error, {:invalid_byte, 3, ?o}} + + assert_raise JSON.DecodeError, + "invalid byte 111 at position (byte offset) 3", + fn -> JSON.decode!("123o") end + end + + test "unexpected sequence" do + assert JSON.decode("\"\\ud8aa\\udcxx\"") == + {:error, {:unexpected_sequence, 1, "\\ud8aa\\udcxx"}} + + assert_raise JSON.DecodeError, + "unexpected sequence \"\\\\ud8aa\\\\udcxx\" at position (byte offset) 1", + fn -> JSON.decode!("\"\\ud8aa\\udcxx\"") end + end + end +end diff --git a/lib/elixir/test/elixir/kernel/alias_test.exs b/lib/elixir/test/elixir/kernel/alias_test.exs index be552061d10..d897fedb5ed 100644 --- a/lib/elixir/test/elixir/kernel/alias_test.exs +++ b/lib/elixir/test/elixir/kernel/alias_test.exs @@ -1,4 +1,8 @@ -Code.require_file "../test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) alias Kernel.AliasTest.Nested, as: Nested @@ -9,26 +13,33 @@ end defmodule Kernel.AliasTest do use ExUnit.Case, async: true - test :alias_erlang do + test "alias Erlang" do alias :lists, as: MyList assert MyList.flatten([1, [2], 3]) == [1, 2, 3] assert Elixir.MyList.Bar == :"Elixir.MyList.Bar" assert MyList.Bar == :"Elixir.lists.Bar" end - test :double_alias do + test "double alias" do alias Kernel.AliasTest.Nested, as: Nested2 - assert Nested.value == 1 - assert Nested2.value == 1 + assert Nested.value() == 1 + assert Nested2.value() == 1 end - test :overwriten_alias do - alias List, as: Nested + test "overwritten alias" do + assert alias(List, as: Nested) == List assert Nested.flatten([[13]]) == [13] end - test :lexical do - if true do + test "non-recursive alias" do + alias Billing, as: BillingLib + alias MyApp.Billing + assert BillingLib == :"Elixir.Billing" + assert Billing == :"Elixir.MyApp.Billing" + end + + test "lexical" do + if true_fun() do alias OMG, as: List, warn: false else alias ABC, as: List, warn: false @@ -37,12 +48,29 @@ defmodule Kernel.AliasTest do assert List.flatten([1, [2], 3]) == [1, 2, 3] end + defp true_fun(), do: true + defmodule Elixir do def sample, do: 1 end - test :nested_elixir_alias do - assert Kernel.AliasTest.Elixir.sample == 1 + test "nested Elixir alias" do + assert Kernel.AliasTest.Elixir.sample() == 1 + end + + test "multi-call" do + result = alias unquote(Inspect).{Opts, Algebra} + assert result == [Inspect.Opts, Inspect.Algebra] + assert %Opts{} == %Inspect.Opts{} + assert Algebra.empty() == [] + end + + test "alias removal" do + alias __MODULE__.Foo + assert Foo == __MODULE__.Foo + alias Elixir.Foo + assert Foo == Elixir.Foo + alias Elixir.Bar end end @@ -54,7 +82,7 @@ defmodule Kernel.AliasNestingGenerator do end defmodule Parent.Child do - def b, do: Parent.a + def b, do: Parent.a() end end end @@ -63,20 +91,46 @@ end defmodule Kernel.AliasNestingTest do use ExUnit.Case, async: true - require Kernel.AliasNestingGenerator - Kernel.AliasNestingGenerator.create + test "aliases nesting" do + require Kernel.AliasNestingGenerator + Kernel.AliasNestingGenerator.create() - test :aliases_nesting do - assert Parent.a == :a - assert Parent.Child.b == :a + assert Parent.a() == :a + assert Parent.Child.b() == :a end defmodule Nested do def value, do: 2 end - test :aliases_nesting_with_previous_alias do - assert Nested.value == 2 + test "aliases nesting with previous alias" do + assert Nested.value() == 2 + end + + alias Another.AliasEnv, warn: false + def aliases_before, do: __ENV__.aliases + + defmodule Elixir.AliasEnv do + def aliases_nested, do: __ENV__.aliases + end + + def aliases_after, do: __ENV__.aliases + + test "keeps env after overriding nested Elixir module of the same name" do + assert aliases_before() == [ + {Elixir.Nested, Kernel.AliasNestingTest.Nested}, + {Elixir.AliasEnv, Another.AliasEnv} + ] + + assert Elixir.AliasEnv.aliases_nested() == [ + {Elixir.Nested, Kernel.AliasNestingTest.Nested}, + {Elixir.AliasEnv, Another.AliasEnv} + ] + + assert aliases_after() == [ + {Elixir.Nested, Kernel.AliasNestingTest.Nested}, + {Elixir.AliasEnv, Another.AliasEnv} + ] end end @@ -96,6 +150,7 @@ defmodule Macro.AliasTest.Definer do defmodule First do defstruct foo: :bar end + defmodule Second do defstruct baz: %First{} end @@ -118,7 +173,7 @@ defmodule Macro.AliasTest.User do use Macro.AliasTest.Aliaser test "has a struct defined from after compile" do - assert is_map struct(Macro.AliasTest.User.First, []) - assert is_map struct(Macro.AliasTest.User.Second, []).baz + assert is_map(struct(Macro.AliasTest.User.First, [])) + assert is_map(struct(Macro.AliasTest.User.Second, []).baz) end end diff --git a/lib/elixir/test/elixir/kernel/binary_test.exs b/lib/elixir/test/elixir/kernel/binary_test.exs index e6e7934dd16..c48c2b9613c 100644 --- a/lib/elixir/test/elixir/kernel/binary_test.exs +++ b/lib/elixir/test/elixir/kernel/binary_test.exs @@ -1,56 +1,79 @@ -Code.require_file "../test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) defmodule Kernel.BinaryTest do use ExUnit.Case, async: true - test :heredoc do - assert 7 == __ENV__.line + test "heredoc" do + assert 11 == __ENV__.line + assert "foo\nbar\n" == """ -foo -bar -""" + foo + bar + """ + + assert 18 == __ENV__.line - assert 13 == __ENV__.line assert "foo\nbar \"\"\"\n" == """ -foo -bar """ -""" + foo + bar \""" + """ end - test :aligned_heredoc do + test "aligned heredoc" do assert "foo\nbar\n" == """ - foo - bar - """ + foo + bar + """ end - test :heredoc_with_interpolation do - assert "29\n" == """ - #{__ENV__.line} - """ + test "heredoc with interpolation" do + assert "35\n" == """ + #{__ENV__.line} + """ + + assert "\n40\n" == """ - assert "\n34\n" == """ + #{__ENV__.line} + """ + end - #{__ENV__.line} - """ + test "heredoc in call" do + assert "foo\nbar" == + Kernel.<>( + """ + foo + """, + "bar" + ) end - test :heredoc_in_call do - assert "foo\nbar" == Kernel.<>(""" - foo - """, "bar") + test "heredoc with heredoc inside interpolation" do + assert """ + 1 + #{""" + 2 + """} + """ == "1\n2\n\n" end - test :utf8 do + test "UTF-8" do assert byte_size(" ゆんゆん") == 13 end - test :utf8_char do + test "UTF-8 char" do assert ?ゆ == 12422 - assert ?\ゆ == 12422 end - test :string_concatenation_as_match do + test "size outside match" do + x = 16 + assert <<0::size(x)>> == <<0, 0>> + end + + test "string concatenation as match" do "foo" <> x = "foobar" assert x == "bar" @@ -60,142 +83,255 @@ bar """ <<"f", "oo">> <> x = "foobar" assert x == "bar" - <> <> _ = "foobar" + <> <> _ = "foobar" assert x == "foo" size = 3 - <> <> _ = "foobar" + <> <> _ = "foobar" assert x == "foo" - <> <> _ = "foobar" + size = 3 + <> <> _ = <<10, "foobar">> assert x == "foo" + assert size == 10 + + <> <> _ = "foobar" + assert x == "foo" + + <> <> _ = "foobar" + assert x == "foo" + + <> <> _ = "foobar" + assert x == "foo" + + <> <> _ = "foobar" + assert x == ?f + end + + test "string concatenation outside match" do + x = "bar" + assert "foobar" = "foo" <> x + assert "barfoo" = x <> "foo" + end - assert_raise ErlangError, fn -> - Code.eval_string(~s{<> <> _ = "foobar"}) + test "invalid string concatenation arguments" do + assert_raise ArgumentError, ~r"expected binary argument in <> operator but got: :bar", fn -> + Code.eval_string(~s["foo" <> :bar]) end - assert_raise ErlangError, fn -> - Code.eval_string(~s{<> <> _ = "foobar"}) + assert_raise ArgumentError, ~r"expected binary argument in <> operator but got: 1", fn -> + Code.eval_string(~s["foo" <> 1]) end - end - test :octals do - assert "\1" == <<1>> - assert "\12" == "\n" - assert "\123" == "S" - assert "\123" == "S" - assert "\377" == "ÿ" - assert "\128" == "\n8" - assert "\18" == <<1, ?8>> + message = ~r"cannot perform prefix match because the left operand of <> has unknown size." + + assert_raise ArgumentError, message, fn -> + Code.eval_string(~s[a <> "b" = "ab"]) + end + + assert_raise ArgumentError, message, fn -> + Code.eval_string(~s["a" <> b <> "c" = "abc"]) + end end - test :hex do - assert "\xa" == "\n" - assert "\xE9" == "é" - assert "\xFF" == "ÿ" - assert "\x{A}"== "\n" - assert "\x{E9}"== "é" - assert "\x{10F}" == <<196, 143>> - assert "\x{10FF}" == <<225, 131, 191>> - assert "\x{10FFF}" == <<240, 144, 191, 191>> - assert "\x{10FFFF}" == <<244, 143, 191, 191>> + test "hex" do + assert "\x76" == "v" + assert "\u00FF" == "ÿ" + assert "\u{A}" == "\n" + assert "\u{E9}" == "é" + assert "\u{10F}" == <<196, 143>> + assert "\u{10FF}" == <<225, 131, 191>> + assert "\u{10FFF}" == <<240, 144, 191, 191>> + assert "\u{10FFFF}" == <<244, 143, 191, 191>> end - test :match do - assert match?(<< ?a, _ :: binary >>, "ab") - refute match?(<< ?a, _ :: binary >>, "cd") - assert match?(<< _ :: utf8 >> <> _, "éf") + test "match" do + assert match?(<>, "ab") + refute match?(<>, "cd") + assert match?(<<_::utf8>> <> _, "éf") end - test :interpolation do + test "interpolation" do res = "hello \\abc" assert "hello #{"\\abc"}" == res assert "hello #{"\\abc" <> ""}" == res end - test :pattern_match do + test "pattern match" do s = 16 - assert <<_a, _b :: size(s)>> = "foo" + assert <<_a, _b::size(^s)>> = "foo" end - test :pattern_match_with_splice do - assert << 1, <<2, 3, 4>>, 5 >> = <<1, 2, 3, 4, 5>> + test "pattern match with splice" do + assert <<1, <<2, 3, 4>>, 5>> = <<1, 2, 3, 4, 5>> end - test :partial_application do - assert (&<< &1, 2 >>).(1) == << 1, 2 >> - assert (&<< &1, &2 >>).(1, 2) == << 1, 2 >> - assert (&<< &2, &1 >>).(2, 1) == << 1, 2 >> + test "partial application" do + assert (&<<&1, 2>>).(1) == <<1, 2>> + assert (&<<&1, &2>>).(1, 2) == <<1, 2>> + assert (&<<&2, &1>>).(2, 1) == <<1, 2>> end - test :literal do - assert <<106,111,115,195,169>> == << "josé" :: binary >> - assert <<106,111,115,195,169>> == << "josé" :: bits >> - assert <<106,111,115,195,169>> == << "josé" :: bitstring >> - assert <<106,111,115,195,169>> == << "josé" :: bytes >> - - assert <<106,111,115,195,169>> == << "josé" :: utf8 >> - assert <<0,106,0,111,0,115,0,233>> == << "josé" :: utf16 >> - assert <<106,0,111,0,115,0,233,0>> == << "josé" :: [utf16, little] >> - assert <<0,0,0,106,0,0,0,111,0,0,0,115,0,0,0,233>> == << "josé" :: utf32 >> + test "literal" do + assert <<106, 111, 115, 195, 169>> == <<"josé">> + assert <<106, 111, 115, 195, 169>> == <<"#{:josé}">> + assert <<106, 111, 115, 195, 169>> == <<"josé"::binary>> + assert <<106, 111, 115, 195, 169>> == <<"josé"::bits>> + assert <<106, 111, 115, 195, 169>> == <<"josé"::bitstring>> + assert <<106, 111, 115, 195, 169>> == <<"josé"::bytes>> + + assert <<106, 111, 115, 195, 169>> == <<"josé"::utf8>> + assert <<0, 106, 0, 111, 0, 115, 0, 233>> == <<"josé"::utf16>> + assert <<106, 0, 111, 0, 115, 0, 233, 0>> == <<"josé"::little-utf16>> + assert <<0, 0, 0, 106, 0, 0, 0, 111, 0, 0, 0, 115, 0, 0, 0, 233>> == <<"josé"::utf32>> end - test :literal_errors do - assert_raise CompileError, fn -> - Code.eval_string(~s[<< "foo" :: integer >>]) - end + test "literal errors" do + message = "conflicting type specification for bit field" - assert_raise CompileError, fn -> - Code.eval_string(~s[<< "foo" :: float >>]) - end + assert_compile_error(message, fn -> + Code.eval_string(~s[<<"foo"::integer>>]) + end) - assert_raise CompileError, fn -> - Code.eval_string(~s[<< 'foo' :: binary >>]) - end + assert_compile_error(message, fn -> + Code.eval_string(~s[<<"foo"::float>>]) + end) + end - assert_raise ArgumentError, fn -> - Code.eval_string(~s[<<1::size(4)>> <> "foo"]) - end + @bitstring <<"foo", 16::4>> + + test "bitstring attribute" do + assert @bitstring == <<"foo", 16::4>> end @binary "new " - test :bitsyntax_with_expansion do + test "bitsyntax expansion" do assert <<@binary, "world">> == "new world" end - test :bitsyntax_translation do + test "bitsyntax translation" do refb = "sample" sec_data = "another" - << byte_size(refb) :: [size(1), big, signed, integer, unit(8)], - refb :: binary, - byte_size(sec_data) :: [size(1), big, signed, integer, unit(16)], - sec_data :: binary >> + + << + byte_size(refb)::size(1)-big-signed-integer-unit(8), + refb::binary, + byte_size(sec_data)::1*16-big-signed-integer, + sec_data::binary + >> + end + + test "bitsyntax size shortcut" do + assert <<1::3>> == <<1::size(3)>> + assert <<1::3*8>> == <<1::size(3)-unit(8)>> + end + + test "bitsyntax variable size" do + x = 8 + assert <<_, _::size(^x)>> = <> + assert (fn <<_, _::size(^x)>> -> true end).(<>) end - test :bitsyntax_size_shorcut do - assert << 1 :: 3 >> == << 1 :: size(3) >> - assert << 1 :: [unit(8), 3] >> == << 1 :: [unit(8), size(3)] >> + test "bitsyntax size using expressions" do + x = 8 + assert <<1::size(x - 5)>> + + foo = %{bar: 5} + assert <<1::size(foo.bar)>> + assert <<1::size(length(~c"abcd"))>> + assert <<255::size(hd(List.flatten([3])))>> end - defmacrop signed_16 do + test "bitsyntax size using guard expressions in match context" do + x = 8 + assert <<1::size(^x - 5)>> = <<1::3>> + assert <<1::size(^x - 5)-unit(8)>> = <<1::3*8>> + assert <<1::size(length(~c"abcd"))>> = <<1::4>> + + foo = %{bar: 5} + assert <<1::size((^foo).bar)>> = <<1::5>> + end + + test "bitsyntax size with pinned integer" do + a = 1 + b = <<2, 3>> + assert <<^a, ^b::binary>> = <<1, 2, 3>> + end + + test "automatic size computation of matched bitsyntax variable" do + var = "foo" + <<^var::binary, rest::binary>> = "foobar" + assert rest == "bar" + + <<^var::bytes, rest::bytes>> = "foobar" + assert rest == "bar" + + ^var <> rest = "foobar" + assert rest == "bar" + + var = <<0, 1>> + <<^var::bitstring, rest::bitstring>> = <<0, 1, 2, 3>> + assert rest == <<2, 3>> + + <<^var::bits, rest::bits>> = <<0, 1, 2, 3>> + assert rest == <<2, 3>> + + ^var <> rest = <<0, 1, 2, 3>> + assert rest == <<2, 3>> + end + + defmacro signed_16 do quote do - [big, signed, integer, unit(16)] + big - signed - integer - unit(16) end end - defmacrop refb_spec do + defmacro refb_spec do quote do - [size(1), big, signed, integer, unit(8)] + 1 * 8 - big - signed - integer end end - test :bitsyntax_macro do + test "bitsyntax macro" do refb = "sample" sec_data = "another" - << byte_size(refb) :: refb_spec, - refb :: binary, - byte_size(sec_data) :: [size(1), signed_16], - sec_data :: binary >> + + << + byte_size(refb)::refb_spec(), + refb::binary, + byte_size(sec_data)::size(1)-signed_16(), + sec_data::binary + >> + end + + test "bitsyntax macro is expanded with a warning" do + assert capture_err(fn -> + Code.eval_string("<<1::refb_spec>>", [], __ENV__) + end) =~ + "bitstring specifier \"refb_spec\" does not exist and is being expanded to \"refb_spec()\"" + + assert capture_err(fn -> + Code.eval_string("<<1::size(1)-signed_16>>", [], __ENV__) + end) =~ + "bitstring specifier \"signed_16\" does not exist and is being expanded to \"signed_16()\"" + end + + test "bitsyntax with extra parentheses warns" do + assert capture_err(fn -> + Code.eval_string("<<1::big()>>") + end) =~ "extra parentheses on a bitstring specifier \"big()\" have been deprecated" + + assert capture_err(fn -> + Code.eval_string("<<1::size(8)-integer()>>") + end) =~ "extra parentheses on a bitstring specifier \"integer()\" have been deprecated" + end + + defp capture_err(fun) do + ExUnit.CaptureIO.capture_io(:stderr, fun) + end + + defp assert_compile_error(message, fun) do + assert capture_err(fn -> assert_raise CompileError, fun end) =~ message end end diff --git a/lib/elixir/test/elixir/kernel/case_test.exs b/lib/elixir/test/elixir/kernel/case_test.exs deleted file mode 100644 index 9c9755c5564..00000000000 --- a/lib/elixir/test/elixir/kernel/case_test.exs +++ /dev/null @@ -1,66 +0,0 @@ -Code.require_file "../test_helper.exs", __DIR__ - -defmodule Kernel.CaseTest do - use ExUnit.Case, async: true - - test :inline_case do - assert (case 1, do: (1 -> :ok; 2 -> :wrong)) == :ok - end - - test :nested_variables do - assert vars_case(400, 1) == {400, 1} - assert vars_case(401, 1) == {400, -1} - assert vars_case(0, -1) == {0, -1} - assert vars_case(-1, -1) == {0, 1} - end - - test :nested_vars_match do - x = {:error, {:ok, :done}} - assert (case x do - {:ok, right} -> - right - {_left, right} -> - case right do - {:ok, right} -> right - end - end) == :done - end - - test :in_operator_outside_case do - x = 1 - y = 4 - assert x in [1, 2, 3], "in assertion" - assert not y in [1, 2, 3], "not in assertion" - end - - test :in_with_match do - refute 1.0 in [1, 2, 3], "not in assertion" - end - - test :in_cond_clause do - assert (cond do - format() && (f = format()) -> - f - true -> - :text - end) == :html - end - - defp format, do: :html - - defp vars_case(x, vx) do - case x > 400 do - true -> - x = 400 - vx = -vx - _ -> - case x < 0 do - true -> - x = 0 - vx = -vx - _ -> nil - end - end - {x, vx} - end -end diff --git a/lib/elixir/test/elixir/kernel/char_list_test.exs b/lib/elixir/test/elixir/kernel/char_list_test.exs deleted file mode 100644 index 7bbc381f2b2..00000000000 --- a/lib/elixir/test/elixir/kernel/char_list_test.exs +++ /dev/null @@ -1,45 +0,0 @@ -Code.require_file "../test_helper.exs", __DIR__ - -defmodule CharListTest do - use ExUnit.Case, async: true - - test :heredoc do - assert __ENV__.line == 7 - assert 'foo\nbar\n' == ''' -foo -bar -''' - - assert __ENV__.line == 13 - assert 'foo\nbar \'\'\'\n' == ''' -foo -bar ''' -''' - end - - test :utf8 do - assert length(' ゆんゆん') == 5 - end - - test :octals do - assert '\1' == [1] - assert '\12' == '\n' - assert '\123' == 'S' - assert '\123' == 'S' - assert '\377' == 'ÿ' - assert '\128' == '\n8' - assert '\18' == [1, ?8] - end - - test :hex do - assert '\xa' == '\n' - assert '\xE9' == 'é' - assert '\xfF' == 'ÿ' - assert '\x{A}' == '\n' - assert '\x{e9}' == 'é' - assert '\x{10F}' == [271] - assert '\x{10FF}' == [4351] - assert '\x{10FFF}' == [69631] - assert '\x{10FFFF}' == [1114111] - end -end diff --git a/lib/elixir/test/elixir/kernel/charlist_test.exs b/lib/elixir/test/elixir/kernel/charlist_test.exs new file mode 100644 index 00000000000..6266a67bf66 --- /dev/null +++ b/lib/elixir/test/elixir/kernel/charlist_test.exs @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) + +defmodule CharlistTest do + use ExUnit.Case, async: true + + test "heredoc" do + assert __ENV__.line == 11 + + assert ~c"foo\nbar\n" == ~c""" + foo + bar + """ + + assert __ENV__.line == 18 + + assert ~c"foo\nbar '''\n" == ~c""" + foo + bar \'\'\' + """ + end + + test "UTF-8" do + assert length(~c" ゆんゆん") == 5 + end + + test "hex" do + assert ~c"\x76" == ~c"v" + assert ~c"\u00fF" == ~c"ÿ" + assert ~c"\u{A}" == ~c"\n" + assert ~c"\u{e9}" == ~c"é" + assert ~c"\u{10F}" == [271] + assert ~c"\u{10FF}" == [4351] + assert ~c"\u{10FFF}" == [69631] + assert ~c"\u{10FFFF}" == [1_114_111] + end +end diff --git a/lib/elixir/test/elixir/kernel/cli_test.exs b/lib/elixir/test/elixir/kernel/cli_test.exs index 65369f4d78e..5f2e9f3edff 100644 --- a/lib/elixir/test/elixir/kernel/cli_test.exs +++ b/lib/elixir/test/elixir/kernel/cli_test.exs @@ -1,166 +1,349 @@ -Code.require_file "../test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) import PathHelpers -defmodule Kernel.CLI.ARGVTest do +defmodule Retry do + # Tests that write to stderr fail on Windows due to late writes, + # so we do a simple retry already them. + defmacro stderr_test(msg, context \\ quote(do: _), do: block) do + if windows?() do + quote do + test unquote(msg), unquote(context) do + unquote(__MODULE__).retry(fn -> unquote(block) end, 3) + end + end + else + quote do + test(unquote(msg), unquote(context), do: unquote(block)) + end + end + end + + def retry(fun, 1) do + fun.() + end + + def retry(fun, n) do + try do + fun.() + rescue + _ -> retry(fun, n - 1) + end + end +end + +defmodule Kernel.CLITest do use ExUnit.Case, async: true import ExUnit.CaptureIO defp run(argv) do - {config, argv} = Kernel.CLI.parse_argv(argv) + {config, argv} = Kernel.CLI.parse_argv(Enum.map(argv, &String.to_charlist/1)) assert Kernel.CLI.process_commands(config) == [] - argv + Enum.map(argv, &IO.chardata_to_string/1) end test "argv handling" do assert capture_io(fn -> - assert run(["-e", "IO.puts :ok", "sample.exs", "-o", "1", "2"]) == - ["sample.exs", "-o", "1", "2"] - end) == "ok\n" - - assert capture_io(fn -> - assert run(["-e", "IO.puts :ok", "--", "sample.exs", "-o", "1", "2"]) == - ["sample.exs", "-o", "1", "2"] - end) == "ok\n" + assert run(["-e", "IO.puts :ok", "sample.exs", "-o", "1", "2"]) == + ["sample.exs", "-o", "1", "2"] + end) == "ok\n" assert capture_io(fn -> - assert run(["-e", "IO.puts :ok", "--hidden", "sample.exs", "-o", "1", "2"]) == - ["sample.exs", "-o", "1", "2"] - end) == "ok\n" + assert run(["-e", "IO.puts :ok", "--", "sample.exs", "-o", "1", "2"]) == + ["sample.exs", "-o", "1", "2"] + end) == "ok\n" assert capture_io(fn -> - assert run(["-e", "IO.puts :ok", "--", "--hidden", "sample.exs", "-o", "1", "2"]) == - ["--hidden", "sample.exs", "-o", "1", "2"] - end) == "ok\n" + assert run(["-e", "", "--", "sample.exs", "-o", "1", "2"]) == + ["sample.exs", "-o", "1", "2"] + end) end end -defmodule Kernel.CLI.OptionParsingTest do - use ExUnit.Case, async: true +test_parameters = + if(PathHelpers.windows?(), + do: [%{cli_extension: ".bat"}], + else: [%{cli_extension: ""}] + ) + +defmodule Kernel.CLI.ExecutableTest do + use ExUnit.Case, + async: true, + parameterize: test_parameters + + import Retry + + @tag :tmp_dir + test "file smoke test", context do + file = Path.join(context.tmp_dir, "hello_world!.exs") + File.write!(file, "IO.puts :hello_world123") + {output, 0} = System.cmd(elixir_executable(context.cli_extension), [file]) + assert output =~ "hello_world123" + end + + test "--eval smoke test", context do + {output, 0} = + System.cmd(elixir_executable(context.cli_extension), ["--eval", "IO.puts :hello_world123"]) + + assert output =~ "hello_world123" + + # Check for -e and exclamation mark handling on Windows + assert {_output, 0} = + System.cmd(elixir_executable(context.cli_extension), ["-e", "Time.new!(0, 0, 0)"]) + + # TODO: remove this once we bump CI to Erlang/OTP 27 + if not (windows?() and System.otp_release() == "26") do + {output, 0} = + System.cmd(iex_executable(context.cli_extension), [ + "--eval", + "IO.puts :hello_world123; System.halt()" + ]) + + assert output =~ "hello_world123" + + {output, 0} = + System.cmd(iex_executable(context.cli_extension), [ + "-e", + "IO.puts :hello_world123; System.halt()" + ]) + + assert output =~ "hello_world123" + end + end + + @tag :unix + # This test hangs on Windows but "iex --version" works on the command line + test "iex smoke test", %{cli_extension: cli_extension} do + output = iex(~c"--version", cli_extension) + assert output =~ "Erlang/OTP #{System.otp_release()}" + assert output =~ "IEx #{System.version()}" + end + + test "--version smoke test", %{cli_extension: cli_extension} do + output = elixir(~c"--version", cli_extension) + assert output =~ "Erlang/OTP #{System.otp_release()}" + assert output =~ "Elixir #{System.version()}" + + output = elixir(~c"--version -e \"IO.puts(:test_output)\"", cli_extension) + assert output =~ "Erlang/OTP #{System.otp_release()}" + assert output =~ "Elixir #{System.version()}" + assert output =~ "Standalone options can't be combined with other options" + end + + test "--short-version smoke test", %{cli_extension: cli_extension} do + output = elixir(~c"--short-version", cli_extension) + assert output =~ System.version() + refute output =~ "Erlang" + end + + stderr_test "--help smoke test", %{cli_extension: cli_extension} do + output = elixir(~c"--help", cli_extension) + assert output =~ "Usage: elixir" + end + + stderr_test "combining --help results in error", %{cli_extension: cli_extension} do + output = elixir(~c"-e 1 --help", cli_extension) + assert output =~ "--help : Standalone options can't be combined with other options" + + output = elixir(~c"--help -e 1", cli_extension) + assert output =~ "--help : Standalone options can't be combined with other options" + end + + stderr_test "combining --short-version results in error", %{cli_extension: cli_extension} do + output = elixir(~c"--short-version -e 1", cli_extension) + assert output =~ "--short-version : Standalone options can't be combined with other options" + + output = elixir(~c"-e 1 --short-version", cli_extension) + assert output =~ "--short-version : Standalone options can't be combined with other options" + end - test "properly parses paths" do - root = fixture_path("../../..") |> to_char_list - list = elixir('-pa "#{root}/*" -pz "#{root}/lib/*" -e "IO.inspect(:code.get_path, limit: :infinity)"') - {path, _} = Code.eval_string list, [] + test "parses paths", %{cli_extension: cli_extension} do + root = fixture_path("../../..") |> to_charlist() + + args = + ~c"-pa \"#{root}/*\" -pz \"#{root}/lib/*\" -e \"IO.inspect(:code.get_path(), limit: :infinity)\"" + + list = elixir(args, cli_extension) + {path, _} = Code.eval_string(list, []) # pa - assert to_char_list(Path.expand('ebin', root)) in path - assert to_char_list(Path.expand('lib', root)) in path - assert to_char_list(Path.expand('src', root)) in path + assert to_charlist(Path.expand(~c"ebin", root)) in path + assert to_charlist(Path.expand(~c"lib", root)) in path + assert to_charlist(Path.expand(~c"src", root)) in path # pz - assert to_char_list(Path.expand('lib/list', root)) in path + assert to_charlist(Path.expand(~c"lib/list", root)) in path end -end -defmodule Kernel.CLI.AtExitTest do - use ExUnit.Case, async: true + stderr_test "formats errors", %{cli_extension: cli_extension} do + assert String.starts_with?(elixir(~c"-e \":erlang.throw 1\"", cli_extension), "** (throw) 1") + + assert String.starts_with?( + elixir(~c"-e \":erlang.error 1\"", cli_extension), + "** (ErlangError) Erlang error: 1" + ) + + assert String.starts_with?(elixir(~c"-e \"1 +\"", cli_extension), "** (TokenMissingError)") + + assert elixir( + ~c"-e \"Task.async(fn -> raise ArgumentError end) |> Task.await\"", + cli_extension + ) =~ + "an exception was raised:\n ** (ArgumentError) argument error" + + assert elixir( + ~c"-e \"IO.puts(Process.flag(:trap_exit, false)); exit({:shutdown, 1})\"", + cli_extension + ) == + "false\n" + end + + stderr_test "blames exceptions", %{cli_extension: cli_extension} do + error = elixir(~c"-e \"Access.fetch :foo, :bar\"", cli_extension) + assert error =~ "** (FunctionClauseError) no function clause matching in Access.fetch/2" + assert error =~ "The following arguments were given to Access.fetch/2" + assert error =~ ":foo" + assert error =~ "def fetch(-%module{} = container-, +key+)" + assert error =~ ~r"\(elixir #{System.version()}\) lib/access\.ex:\d+: Access\.fetch/2" + end test "invokes at_exit callbacks" do - assert elixir(fixture_path("at_exit.exs") |> to_char_list) == - 'goodbye cruel world with status 0\n' + assert elixir(fixture_path("at_exit.exs") |> to_charlist()) == + "goodbye cruel world with status 1\n" end end -defmodule Kernel.CLI.ErrorTest do +defmodule Kernel.CLI.RPCTest do use ExUnit.Case, async: true - test "properly format errors" do - assert :string.str('** (throw) 1', elixir('-e "throw 1"')) == 0 - assert :string.str('** (ErlangError) erlang error: 1', elixir('-e "error 1"')) == 0 + import Retry - # It does not catch exits with integers nor strings... - assert elixir('-e "exit 1"') == '' + defp rpc_eval(command) do + node = "cli-rpc#{System.unique_integer()}@127.0.0.1" + elixir(~c"--name #{node} --rpc-eval #{node} \"#{command}\"") end -end -defmodule Kernel.CLI.CompileTest do - use ExUnit.Case, async: true + test "invokes command on remote node" do + assert rpc_eval("IO.puts :ok") == "ok\n" + end - test "compiles code" do - fixture = fixture_path "compile_sample.ex" - assert elixirc('#{fixture} -o #{tmp_path}') == '' - assert File.regular?(tmp_path "Elixir.CompileSample.beam") - after - File.rm(tmp_path("Elixir.CompileSample.beam")) + test "invokes command on remote node without host and --name after --rpc-eval" do + node = "cli-rpc#{System.unique_integer()}" + assert elixir(~c"--rpc-eval #{node} \"IO.puts :ok\" --name #{node}@127.0.0.1 ") == "ok\n" end - test "compiles code with verbose mode" do - fixture = fixture_path "compile_sample.ex" - assert elixirc('#{fixture} -o #{tmp_path} --verbose') == - 'Compiled #{fixture}\n' - assert File.regular?(tmp_path "Elixir.CompileSample.beam") - after - File.rm(tmp_path("Elixir.CompileSample.beam")) + test "can be invoked multiple times" do + node = "cli-rpc#{System.unique_integer()}" + + assert elixir( + ~c"--name #{node}@127.0.0.1 --rpc-eval #{node} \"IO.puts :foo\" --rpc-eval #{node} \"IO.puts :bar\"" + ) == "foo\nbar\n" end - test "fails on missing patterns" do - fixture = fixture_path "compile_sample.ex" - output = elixirc('#{fixture} non_existing.ex -o #{tmp_path}') - assert :string.str(output, 'non_existing.ex') > 0, "expected non_existing.ex to be mentionned" - assert :string.str(output, 'compile_sample.ex') == 0, "expected compile_sample.ex to not be mentionned" - refute File.exists?(tmp_path("Elixir.CompileSample.beam")) , "expected the sample to not be compiled" + # Windows does not provide an easy to check for missing args + @tag :unix + test "fails on wrong arguments" do + node = "cli-rpc#{System.unique_integer()}" + + assert elixir(~c"--name #{node}@127.0.0.1 --rpc-eval") == + "--rpc-eval : wrong number of arguments\n" + + assert elixir(~c"--name #{node}@127.0.0.1 --rpc-eval #{node}") == + "--rpc-eval : wrong number of arguments\n" + end + + stderr_test "properly formats errors" do + assert String.starts_with?(rpc_eval(":erlang.throw 1"), "** (throw) 1") + assert String.starts_with?(rpc_eval(":erlang.error 1"), "** (ErlangError) Erlang error: 1") + assert String.starts_with?(rpc_eval("1 +"), "** (TokenMissingError)") + + assert rpc_eval("Task.async(fn -> raise ArgumentError end) |> Task.await") =~ + "an exception was raised:\n ** (ArgumentError) argument error" + + assert rpc_eval("IO.puts(Process.flag(:trap_exit, false)); exit({:shutdown, 1})") == + "false\n" end end -defmodule Kernel.CLI.ParallelCompilerTest do - use ExUnit.Case - import ExUnit.CaptureIO +defmodule Kernel.CLI.CompileTest do + use ExUnit.Case, + async: true, + parameterize: test_parameters - test "compiles files solving dependencies" do - fixtures = [fixture_path("parallel_compiler/bar.ex"), fixture_path("parallel_compiler/foo.ex")] - assert capture_io(fn -> - assert [Bar, Foo] = Kernel.ParallelCompiler.files fixtures - end) =~ "message_from_foo" - after - Enum.map [Foo, Bar], fn mod -> - :code.purge(mod) - :code.delete(mod) - end + import Retry + @moduletag :tmp_dir + + setup context do + beam_file_path = Path.join([context.tmp_dir, "Elixir.CompileSample.beam"]) + fixture = fixture_path("compile_sample.ex") + {:ok, [beam_file_path: beam_file_path, fixture: fixture]} end - test "compiles files with structs solving dependencies" do - fixtures = [fixture_path("parallel_struct/bar.ex"), fixture_path("parallel_struct/foo.ex")] - assert [Bar, Foo] = Kernel.ParallelCompiler.files(fixtures) |> Enum.sort + test "compiles code", context do + assert elixirc(~c"#{context.fixture} -o #{context.tmp_dir}", context.cli_extension) == "" + assert File.regular?(context.beam_file_path) + + # Assert that the module is loaded into memory with the proper destination for the BEAM file. + Code.append_path(context.tmp_dir) + assert :code.which(CompileSample) |> List.to_string() == Path.expand(context.beam_file_path) after - Enum.map [Foo, Bar], fn mod -> - :code.purge(mod) - :code.delete(mod) + :code.purge(CompileSample) + :code.delete(CompileSample) + Code.delete_path(context.tmp_dir) + end + + @tag :windows + stderr_test "compiles code with Windows paths", context do + try do + fixture = String.replace(context.fixture, "/", "\\") + tmp_dir_path = String.replace(context.tmp_dir, "/", "\\") + assert elixirc(~c"#{fixture} -o #{tmp_dir_path}", context.cli_extension) == "" + assert File.regular?(context[:beam_file_path]) + + # Assert that the module is loaded into memory with the proper destination for the BEAM file. + Code.append_path(context.tmp_dir) + + assert :code.which(CompileSample) |> List.to_string() == + Path.expand(context[:beam_file_path]) + after + :code.purge(CompileSample) + :code.delete(CompileSample) + Code.delete_path(context.tmp_dir) end end - test "does not hang on missing dependencies" do - fixtures = [fixture_path("parallel_compiler/bat.ex")] - assert capture_io(fn -> - assert catch_exit(Kernel.ParallelCompiler.files(fixtures)) == 1 - end) =~ "Compilation error" + stderr_test "fails on missing patterns", context do + output = + elixirc(~c"#{context.fixture} non_existing.ex -o #{context.tmp_dir}", context.cli_extension) + + assert output =~ "non_existing.ex" + refute output =~ "compile_sample.ex" + refute File.exists?(context.beam_file_path) end - test "handles possible deadlocks" do - fixtures = [fixture_path("parallel_deadlock/foo.ex"), - fixture_path("parallel_deadlock/bar.ex")] + stderr_test "fails on missing write access to .beam file", context do + compilation_args = ~c"#{context.fixture} -o #{context.tmp_dir}" - msg = capture_io(fn -> - assert catch_exit(Kernel.ParallelCompiler.files fixtures) == 1 - end) + assert elixirc(compilation_args, context.cli_extension) == "" + assert File.regular?(context.beam_file_path) - assert msg =~ ~r"== Compilation error on file .+parallel_deadlock/foo\.ex ==" - assert msg =~ ~r"== Compilation error on file .+parallel_deadlock/bar\.ex ==" - end + # Set the .beam file to read-only + File.chmod!(context.beam_file_path, 4) + {:ok, %{access: access}} = File.stat(context.beam_file_path) - test "warnings as errors" do - warnings_as_errors = Code.compiler_options[:warnings_as_errors] - fixtures = [fixture_path("warnings_sample.ex")] + # Can only assert when read-only applies to the user + if access != :read_write do + output = elixirc(compilation_args, context.cli_extension) - try do - Code.compiler_options(warnings_as_errors: true) + expected = + "(File.Error) could not write to file #{inspect(context.beam_file_path)}: permission denied" - capture_io :stderr, fn -> - assert catch_exit(Kernel.ParallelCompiler.files fixtures) == 1 - end - after - Code.compiler_options(warnings_as_errors: warnings_as_errors) + assert output =~ expected end end end diff --git a/lib/elixir/test/elixir/kernel/comprehension_test.exs b/lib/elixir/test/elixir/kernel/comprehension_test.exs index 1a731460407..97a0a6b733e 100644 --- a/lib/elixir/test/elixir/kernel/comprehension_test.exs +++ b/lib/elixir/test/elixir/kernel/comprehension_test.exs @@ -1,4 +1,8 @@ -Code.require_file "../test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) defmodule Kernel.ComprehensionTest do use ExUnit.Case, async: true @@ -6,8 +10,24 @@ defmodule Kernel.ComprehensionTest do import ExUnit.CaptureIO require Integer + defmodule Pdict do + defstruct [] + + defimpl Collectable do + def into(struct) do + fun = fn + _, {:cont, x} -> Process.put(:into_cont, [x | Process.get(:into_cont)]) + _, :done -> Process.put(:into_done, true) + _, :halt -> Process.put(:into_halt, true) + end + + {struct, fun} + end + end + end + defp to_bin(x) do - << x >> + <> end defp nilly, do: nil @@ -20,46 +40,161 @@ defmodule Kernel.ComprehensionTest do end test "for comprehensions with matching" do - assert for({_,x} <- 1..3, do: x * 2) == [] + assert for({_, x} <- 1..3, do: x * 2) == [] + end + + test "for comprehensions with pin matching" do + maps = [x: 1, y: 2, x: 3] + assert for({:x, v} <- maps, do: v * 2) == [2, 6] + x = :x + assert for({^x, v} <- maps, do: v * 2) == [2, 6] + end + + test "for comprehensions with guards" do + assert for(x when x < 4 <- 1..10, do: x) == [1, 2, 3] + assert for(x when x == 3 when x == 7 <- 1..10, do: x) == [3, 7] + end + + test "for comprehensions with guards and filters" do + assert for( + {var, _} + when is_atom(var) <- [{:foo, 1}, {2, :bar}], + var = Atom.to_string(var), + do: var + ) == ["foo"] + end + + test "for comprehensions with map key matching" do + maps = [%{x: 1}, %{y: 2}, %{x: 3}] + assert for(%{x: v} <- maps, do: v * 2) == [2, 6] + x = :x + assert for(%{^x => v} <- maps, do: v * 2) == [2, 6] end test "for comprehensions with filters" do assert for(x <- 1..3, x > 1, x < 3, do: x * 2) == [4] end + test "for comprehensions with unique values" do + list = [1, 1, 2, 3] + assert for(x <- list, uniq: true, do: x * 2) == [2, 4, 6] + assert for(x <- list, uniq: true, into: [], do: x * 2) == [2, 4, 6] + assert for(x <- list, uniq: true, into: %{}, do: {x, 1}) == %{1 => 1, 2 => 1, 3 => 1} + assert for(x <- list, uniq: true, into: "", do: to_bin(x * 2)) == <<2, 4, 6>> + assert for(<>, uniq: true, into: "", do: to_bin(x)) == "abc" + + Process.put(:into_cont, []) + Process.put(:into_done, false) + Process.put(:into_halt, false) + + for x <- list, uniq: true, into: %Pdict{} do + x * 2 + end + + assert Process.get(:into_cont) == [6, 4, 2] + assert Process.get(:into_done) + refute Process.get(:into_halt) + + assert_raise RuntimeError, "oops", fn -> + for _ <- [1, 2, 3], uniq: true, into: %Pdict{}, do: raise("oops") + end + + assert Process.get(:into_halt) + end + + test "nested for comprehensions with unique values" do + assert for(x <- [1, 1, 2], uniq: true, do: for(y <- [3, 3], uniq: true, do: x * y)) == [ + [3], + [6] + ] + + assert for(<>, + uniq: true, + into: "", + do: for(<>, uniq: true, into: "", do: to_bin(x) <> to_bin(y)) + ) == "azbzcz" + end + test "for comprehensions with nilly filters" do - assert for(x <- 1..3, nilly, do: x * 2) == [] + assert for(x <- 1..3, nilly(), do: x * 2) == [] + end + + test "for comprehensions with unique option where value is not used" do + assert capture_io(:stderr, fn -> + assert capture_io(fn -> + Code.eval_quoted( + quote do + for x <- [1, 2, 1, 2], uniq: true, do: IO.puts(x) + nil + end + ) + end) == + "1\n2\n1\n2\n" + end) =~ + "the :uniq option has no effect since the result of the for comprehension is not used" + end + + test "for comprehensions with unique option where value is assigned to _" do + assert capture_io(:stderr, fn -> + assert capture_io(fn -> + Code.eval_quoted( + quote do + _ = for x <- [1, 2, 1, 2], uniq: true, do: IO.puts(x) + nil + end + ) + end) == + "1\n2\n1\n2\n" + end) =~ + "the :uniq option has no effect since the result of the for comprehension is not used" end test "for comprehensions with errors on filters" do assert_raise ArgumentError, fn -> - for(x <- 1..3, hd(x), do: x * 2) + for x <- 1..3, hd(x), do: :ok end end test "for comprehensions with variables in filters" do - assert for(x <- 1..3, y = x + 1, y > 2, z = y, do: x * z) == - [6, 12] + assert for(x <- 1..3, y = x + 1, y > 2, z = y, do: x * z) == [6, 12] end test "for comprehensions with two enum generators" do - assert (for x <- [1, 2, 3], y <- [4, 5, 6], do: x * y) == - [4, 5, 6, 8, 10, 12, 12, 15, 18] + assert for( + x <- [1, 2, 3], + y <- [4, 5, 6], + do: x * y + ) == [4, 5, 6, 8, 10, 12, 12, 15, 18] end test "for comprehensions with two enum generators and filters" do - assert (for x <- [1, 2, 3], y <- [4, 5, 6], y / 2 == x, do: x * y) == - [8, 18] + assert for( + x <- [1, 2, 3], + y <- [4, 5, 6], + y / 2 == x, + do: x * y + ) == [8, 18] end test "for comprehensions generators precedence" do - assert (for {_, _} = x <- [foo: :bar], do: x) == - [foo: :bar] + assert for({_, _} = x <- [foo: :bar], do: x) == [foo: :bar] + end + + test "for comprehensions with shadowing" do + assert for( + a <- + ( + b = 1 + _ = b + [1] + ), + b <- [2], + do: a + b + ) == [3] end test "for comprehensions with binary, enum generators and filters" do - assert (for x <- [1, 2, 3], << y <- <<4, 5, 6>> >>, y / 2 == x, do: x * y) == - [8, 18] + assert for(x <- [1, 2, 3], <<(y <- <<4, 5, 6>>)>>, y / 2 == x, do: x * y) == [8, 18] end test "for comprehensions into list" do @@ -68,17 +203,47 @@ defmodule Kernel.ComprehensionTest do end test "for comprehensions into binary" do - enum = 1..3 - assert for(x <- enum, into: "", do: to_bin(x * 2)) == <<2, 4, 6>> + enum = 0..3 + + assert (for x <- enum, into: "" do + to_bin(x * 2) + end) == <<0, 2, 4, 6>> + + assert (for x <- enum, into: "" do + if Integer.is_even(x), do: <>, else: <> + end) == <<0::size(2), 1::size(1), 2::size(2), 3::size(1)>> + end + + test "for comprehensions into dynamic binary" do + enum = 0..3 + into = "" + + assert (for x <- enum, into: into do + to_bin(x * 2) + end) == <<0, 2, 4, 6>> + + assert (for x <- enum, into: into do + if Integer.is_even(x), do: <>, else: <> + end) == <<0::size(2), 1::size(1), 2::size(2), 3::size(1)>> + + into = <<7::size(1)>> + + assert (for x <- enum, into: into do + to_bin(x * 2) + end) == <<7::size(1), 0, 2, 4, 6>> + + assert (for x <- enum, into: into do + if Integer.is_even(x), do: <>, else: <> + end) == <<7::size(1), 0::size(2), 1::size(1), 2::size(2), 3::size(1)>> end test "for comprehensions where value is not used" do enum = 1..3 assert capture_io(fn -> - for(x <- enum, do: IO.puts x) - nil - end) == "1\n2\n3\n" + for x <- enum, do: IO.puts(x) + nil + end) == "1\n2\n3\n" end test "for comprehensions with into" do @@ -86,7 +251,7 @@ defmodule Kernel.ComprehensionTest do Process.put(:into_done, false) Process.put(:into_halt, false) - for x <- 1..3, into: collectable_pdict do + for x <- 1..3, into: %Pdict{} do x * 2 end @@ -101,7 +266,7 @@ defmodule Kernel.ComprehensionTest do Process.put(:into_halt, false) catch_error( - for x <- 1..3, into: collectable_pdict do + for x <- 1..3, into: %Pdict{} do if x > 2, do: raise("oops"), else: x end ) @@ -114,19 +279,38 @@ defmodule Kernel.ComprehensionTest do test "for comprehension with into, generators and filters" do Process.put(:into_cont, []) - for x <- 1..3, Integer.odd?(x), << y <- "hello" >>, into: collectable_pdict do + for x <- 1..3, Integer.is_odd(x), <>, into: %Pdict{} do x + y end assert IO.iodata_to_binary(Process.get(:into_cont)) == "roohkpmmfi" end - defp collectable_pdict do - fn - _, {:cont, x} -> Process.put(:into_cont, [x|Process.get(:into_cont)]) - _, :done -> Process.put(:into_done, true) - _, :halt -> Process.put(:into_halt, true) - end + test "for comprehensions of map into map" do + enum = %{a: 2, b: 3} + assert for({k, v} <- enum, into: %{}, do: {k, v * v}) == %{a: 4, b: 9} + end + + test "for comprehensions with reduce, generators and filters" do + acc = + for x <- 1..3, Integer.is_odd(x), <>, reduce: %{} do + acc -> Map.update(acc, x, [y], &[y | &1]) + end + + assert acc == %{1 => ~c"olleh", 3 => ~c"olleh"} + end + + test "for comprehensions with matched reduce" do + acc = + for entry <- [1, 2, 3], reduce: {:ok, nil} do + {:ok, _} -> + {:ok, entry} + + {:error, _} = error -> + error + end + + assert acc == {:ok, 3} end ## List generators (inlined by the compiler) @@ -137,26 +321,33 @@ defmodule Kernel.ComprehensionTest do end test "list for comprehensions with matching" do - assert for({_,x} <- [1, 2, a: 3, b: 4, c: 5], do: x * 2) == [6, 8, 10] + assert for({_, x} <- [1, 2, a: 3, b: 4, c: 5], do: x * 2) == [6, 8, 10] + end + + test "list for comprehension matched to '_' on last line of block" do + assert (if true_fun() do + _ = for x <- [1, 2, 3], do: x * 2 + end) == [2, 4, 6] end + defp true_fun(), do: true + test "list for comprehensions with filters" do assert for(x <- [1, 2, 3], x > 1, x < 3, do: x * 2) == [4] end test "list for comprehensions with nilly filters" do - assert for(x <- [1, 2, 3], nilly, do: x * 2) == [] + assert for(x <- [1, 2, 3], nilly(), do: x * 2) == [] end test "list for comprehensions with errors on filters" do assert_raise ArgumentError, fn -> - for(x <- [1, 2, 3], hd(x), do: x * 2) + for x <- [1, 2, 3], hd(Process.get(:unused, x)), do: x * 2 end end test "list for comprehensions with variables in filters" do - assert for(x <- [1, 2, 3], y = x + 1, y > 2, z = y, do: x * z) == - [6, 12] + assert for(x <- [1, 2, 3], y = x + 1, y > 2, z = y, do: x * z) == [6, 12] end test "list for comprehensions into list" do @@ -164,53 +355,185 @@ defmodule Kernel.ComprehensionTest do assert for(x <- enum, into: [], do: x * 2) == [2, 4, 6] end - test "list for comprehensions into binaries" do - enum = [1, 2, 3] - assert for(x <- enum, into: "", do: to_bin(x * 2)) == <<2, 4, 6>> + test "list for comprehensions into binary" do + enum = [0, 1, 2, 3] + + assert (for x <- enum, into: "" do + to_bin(x * 2) + end) == <<0, 2, 4, 6>> + + assert (for x <- enum, into: "" do + if Integer.is_even(x), do: <>, else: <> + end) == <<0::size(2), 1::size(1), 2::size(2), 3::size(1)>> + end + + test "list for comprehensions into dynamic binary" do + enum = [0, 1, 2, 3] + into = "" + + assert (for x <- enum, into: into do + to_bin(x * 2) + end) == <<0, 2, 4, 6>> + + assert (for x <- enum, into: into do + if Integer.is_even(x), do: <>, else: <> + end) == <<0::size(2), 1::size(1), 2::size(2), 3::size(1)>> + + into = <<7::size(1)>> + + assert (for x <- enum, into: into do + to_bin(x * 2) + end) == <<7::size(1), 0, 2, 4, 6>> + + assert (for x <- enum, into: into do + if Integer.is_even(x), do: <>, else: <> + end) == <<7::size(1), 0::size(2), 1::size(1), 2::size(2), 3::size(1)>> end test "list for comprehensions where value is not used" do - enum = [1,2,3] + enum = [1, 2, 3] assert capture_io(fn -> - for(x <- enum, do: IO.puts x) - nil - end) == "1\n2\n3\n" + for x <- enum, do: IO.puts(x) + nil + end) == "1\n2\n3\n" + end + + test "list for comprehensions with reduce, generators and filters" do + acc = + for x <- [1, 2, 3], Integer.is_odd(x), <>, reduce: %{} do + acc -> Map.update(acc, x, [y], &[y | &1]) + end + + assert acc == %{1 => ~c"olleh", 3 => ~c"olleh"} end ## Binary generators (inlined by the compiler) test "binary for comprehensions" do bin = <<1, 2, 3>> - assert for(<< x <- bin >>, do: x * 2) == [2, 4, 6] + assert for(<>, do: x * 2) == [2, 4, 6] end test "binary for comprehensions with inner binary" do bin = <<1, 2, 3>> - assert for(<< <> <- bin >>, do: x * 2) == [2, 4, 6] + assert for(<<(<> <- bin)>>, do: x * 2) == [2, 4, 6] end test "binary for comprehensions with two generators" do - assert (for << x <- <<1, 2, 3>> >>, << y <- <<4, 5, 6>> >>, y / 2 == x, do: x * y) == - [8, 18] + assert for(<<(x <- <<1, 2, 3>>)>>, <<(y <- <<4, 5, 6>>)>>, y / 2 == x, do: x * y) == [8, 18] end test "binary for comprehensions into list" do bin = <<1, 2, 3>> - assert for(<< x <- bin >>, into: [], do: x * 2) == [2, 4, 6] + assert for(<>, into: [], do: x * 2) == [2, 4, 6] end - test "binary for comprehensions into binaries" do - bin = <<1, 2, 3>> - assert for(<< x <- bin >>, into: "", do: to_bin(x * 2)) == <<2, 4, 6>> + test "binary for comprehensions into binary" do + bin = <<0, 1, 2, 3>> + + assert (for <>, into: "" do + to_bin(x * 2) + end) == <<0, 2, 4, 6>> + + assert (for <>, into: "" do + if Integer.is_even(x), do: <>, else: <> + end) == <<0::size(2), 1::size(1), 2::size(2), 3::size(1)>> + end + + test "binary for comprehensions into dynamic binary" do + bin = <<0, 1, 2, 3>> + into = "" + + assert (for <>, into: into do + to_bin(x * 2) + end) == <<0, 2, 4, 6>> + + assert (for <>, into: into do + if Integer.is_even(x), do: <>, else: <> + end) == <<0::size(2), 1::size(1), 2::size(2), 3::size(1)>> + + into = <<7::size(1)>> + + assert (for <>, into: into do + to_bin(x * 2) + end) == <<7::size(1), 0, 2, 4, 6>> + + assert (for <>, into: into do + if Integer.is_even(x), do: <>, else: <> + end) == <<7::size(1), 0::size(2), 1::size(1), 2::size(2), 3::size(1)>> + end + + test "binary for comprehensions with literal matches" do + # Integers + bin = <<1, 2, 1, 3, 1, 4>> + assert for(<<1, x <- bin>>, into: "", do: to_bin(x)) == <<2, 3, 4>> + assert for(<<1, x <- bin>>, into: %{}, do: {x, x}) == %{2 => 2, 3 => 3, 4 => 4} + + bin = <<1, 2, 3, 1, 4>> + assert for(<<1, x <- bin>>, into: "", do: to_bin(x)) == <<2>> + assert for(<<1, x <- bin>>, into: %{}, do: {x, x}) == %{2 => 2} + + # Floats + bin = <<1.0, 2, 1.0, 3, 1.0, 4>> + assert for(<<1.0, x <- bin>>, into: "", do: to_bin(x)) == <<2, 3, 4>> + assert for(<<1.0, x <- bin>>, into: %{}, do: {x, x}) == %{2 => 2, 3 => 3, 4 => 4} + + bin = <<1.0, 2, 3, 1.0, 4>> + assert for(<<1.0, x <- bin>>, into: "", do: to_bin(x)) == <<2>> + assert for(<<1.0, x <- bin>>, into: %{}, do: {x, x}) == %{2 => 2} + + # Binaries + bin = <<"foo", 2, "foo", 3, "foo", 4>> + assert for(<<"foo", x <- bin>>, into: "", do: to_bin(x)) == <<2, 3, 4>> + assert for(<<"foo", x <- bin>>, into: %{}, do: {x, x}) == %{2 => 2, 3 => 3, 4 => 4} + + bin = <<"foo", 2, 3, "foo", 4>> + assert for(<<"foo", x <- bin>>, into: "", do: to_bin(x)) == <<2>> + assert for(<<"foo", x <- bin>>, into: %{}, do: {x, x}) == %{2 => 2} + + bin = <<"foo", 2, 3, 4, "foo", 5>> + assert for(<<"foo", x <- bin>>, into: "", do: to_bin(x)) == <<2>> + assert for(<<"foo", x <- bin>>, into: %{}, do: {x, x}) == %{2 => 2} + end + + test "binary for comprehensions with variable size" do + s = 16 + bin = <<1, 2, 3, 4, 5, 6>> + assert for(<>, into: "", do: to_bin(div(x, 2))) == <<129, 130, 131>> + + s = 8 + bin = <<1, 2, 3, 4, 5, 6>> + assert for(<>, into: "", do: <>) == <<2, 12, 30>> + + # Aligned + bin = <<8, 1, 16, 2, 3>> + assert for(<>, into: "", do: <>) == <<1, 2, 3>> + assert for(<>, into: %{}, do: {s, x}) == %{8 => 1, 16 => 515} + + # Unaligned + bin = <<8, 1, 32, 2, 3>> + assert for(<>, into: "", do: <>) == <<1>> + assert for(<>, into: %{}, do: {s, x}) == %{8 => 1} end test "binary for comprehensions where value is not used" do bin = <<1, 2, 3>> assert capture_io(fn -> - for(<>, do: IO.puts x) - nil - end) == "1\n2\n3\n" + for <>, do: IO.puts(x) + nil + end) == "1\n2\n3\n" + end + + test "binary for comprehensions with reduce, generators and filters" do + bin = <<1, 2, 3>> + + acc = + for <>, Integer.is_odd(x), <>, reduce: %{} do + acc -> Map.update(acc, x, [y], &[y | &1]) + end + + assert acc == %{1 => ~c"olleh", 3 => ~c"olleh"} end end diff --git a/lib/elixir/test/elixir/kernel/defaults_test.exs b/lib/elixir/test/elixir/kernel/defaults_test.exs new file mode 100644 index 00000000000..4c5df72118d --- /dev/null +++ b/lib/elixir/test/elixir/kernel/defaults_test.exs @@ -0,0 +1,115 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team + +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Kernel.DefaultsTest do + use ExUnit.Case, async: true + + import ExUnit.CaptureIO + + def fun_with_fn_defaults( + x, + fun1 \\ & &1, + fun2 \\ & &1, + y + ) do + {fun1.(x), fun2.(y)} + end + + def fun_with_block_defaults( + x, + y \\ ( + default = "y" + default + ), + z \\ ( + default = "z" + default + ) + ) do + {x, y, z} + end + + test "with anonymous function defaults" do + assert {1, 2} = fun_with_fn_defaults(1, 2) + assert {100, 2} = fun_with_fn_defaults(1, &(&1 * 100), 2) + assert {100, 12} = fun_with_fn_defaults(1, &(&1 * 100), &(&1 + 10), 2) + end + + test "with block defaults" do + assert {1, "y", "z"} = fun_with_block_defaults(1) + assert {1, 2, "z"} = fun_with_block_defaults(1, 2) + assert {1, 2, 3} = fun_with_block_defaults(1, 2, 3) + end + + test "errors on accessing variable from default block" do + assert_compile_error(~r/undefined variable \"default\"/, fn -> + defmodule VarDefaultScope do + def test(_ \\ default = 1), + do: default + end + end) + end + + test "errors on multiple defaults" do + message = ~r"def hello/1 defines defaults multiple times" + + assert_compile_error(message, fn -> + defmodule Kernel.ErrorsTest.ClauseWithDefaults do + def hello(_arg \\ 0) + def hello(_arg \\ 1) + end + end) + + assert_compile_error(message, fn -> + defmodule Kernel.ErrorsTest.ClauseWithDefaults do + def hello(_arg \\ 0), do: nil + def hello(_arg \\ 1), do: nil + end + end) + + assert_compile_error(message, fn -> + defmodule Kernel.ErrorsTest.ClauseWithDefaults do + def hello(_arg \\ 0) + def hello(_arg \\ 1), do: nil + end + end) + + assert_compile_error(message, fn -> + defmodule Kernel.ErrorsTest.ClauseWithDefaults do + def hello(_arg \\ 0), do: nil + def hello(_arg \\ 1) + end + end) + + assert_compile_error("undefined variable \"foo\"", fn -> + defmodule Kernel.ErrorsTest.ClauseWithDefaults5 do + def hello(foo, bar \\ foo) + def hello(foo, bar), do: foo + bar + end + end) + end + + test "errors on conflicting defaults" do + assert_compile_error(~r"def hello/3 defaults conflicts with hello/2", fn -> + defmodule Kernel.ErrorsTest.DifferentDefsWithDefaults1 do + def hello(a, b \\ nil), do: a + b + def hello(a, b \\ nil, c \\ nil), do: a + b + c + end + end) + + assert_compile_error(~r"def hello/2 conflicts with defaults from hello/3", fn -> + defmodule Kernel.ErrorsTest.DifferentDefsWithDefaults2 do + def hello(a, b \\ nil, c \\ nil), do: a + b + c + def hello(a, b \\ nil), do: a + b + end + end) + end + + defp assert_compile_error(message, fun) do + assert capture_io(:stderr, fn -> + assert_raise CompileError, fun + end) =~ message + end +end diff --git a/lib/elixir/test/elixir/kernel/deprecated_test.exs b/lib/elixir/test/elixir/kernel/deprecated_test.exs new file mode 100644 index 00000000000..be2145b19e8 --- /dev/null +++ b/lib/elixir/test/elixir/kernel/deprecated_test.exs @@ -0,0 +1,49 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Kernel.DeprecatedTest do + use ExUnit.Case + + import PathHelpers + + test "raises on invalid @deprecated" do + assert_raise ArgumentError, ~r"should be a string with the reason", fn -> + defmodule InvalidDeprecated do + @deprecated 1.2 + def foo, do: :bar + end + end + end + + test "takes into account deprecated from defaults" do + defmodule DefaultDeprecated do + @deprecated "reason" + def foo(x \\ true), do: x + end + + assert DefaultDeprecated.__info__(:deprecated) == [ + {{:foo, 0}, "reason"}, + {{:foo, 1}, "reason"} + ] + end + + test "add deprecated to __info__" do + write_beam( + defmodule SampleDeprecated do + @deprecated "Use SampleDeprecated.bar/0 instead" + def foo, do: true + + def bar, do: false + end + ) + + deprecated = [ + {{:foo, 0}, "Use SampleDeprecated.bar/0 instead"} + ] + + assert SampleDeprecated.__info__(:deprecated) == deprecated + end +end diff --git a/lib/elixir/test/elixir/kernel/diagnostics_test.exs b/lib/elixir/test/elixir/kernel/diagnostics_test.exs new file mode 100644 index 00000000000..cf34c5ce741 --- /dev/null +++ b/lib/elixir/test/elixir/kernel/diagnostics_test.exs @@ -0,0 +1,1512 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team + +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Kernel.DiagnosticsTest do + use ExUnit.Case, async: false + + import ExUnit.CaptureIO + + setup_all do + previous = Application.get_env(:elixir, :ansi_enabled, false) + Application.put_env(:elixir, :ansi_enabled, false) + on_exit(fn -> Application.put_env(:elixir, :ansi_enabled, previous) end) + end + + describe "mismatched delimiter" do + test "same line - handles unicode input" do + output = + capture_raise( + """ + [1, 2, 3, 4, 5, 6) <- 😎 + """, + MismatchedDelimiterError + ) + + assert output == """ + ** (MismatchedDelimiterError) mismatched delimiter found on nofile:1:18: + error: unexpected token: ) + │ + 1 │ [1, 2, 3, 4, 5, 6) <- 😎 + │ │ └ mismatched closing delimiter (expected "]") + │ └ unclosed delimiter + │ + └─ nofile:1:18\ + """ + end + + test "same line" do + output = + capture_raise( + """ + [1, 2, 3, 4, 5, 6) + """, + MismatchedDelimiterError + ) + + assert output == """ + ** (MismatchedDelimiterError) mismatched delimiter found on nofile:1:18: + error: unexpected token: ) + │ + 1 │ [1, 2, 3, 4, 5, 6) + │ │ └ mismatched closing delimiter (expected "]") + │ └ unclosed delimiter + │ + └─ nofile:1:18\ + """ + end + + test "same line with offset" do + output = + capture_raise( + """ + [1, 2, 3, 4, 5, 6) + """, + MismatchedDelimiterError, + line: 3 + ) + + assert output == """ + ** (MismatchedDelimiterError) mismatched delimiter found on nofile:3:18: + error: unexpected token: ) + │ + 3 │ [1, 2, 3, 4, 5, 6) + │ │ └ mismatched closing delimiter (expected "]") + │ └ unclosed delimiter + │ + └─ nofile:3:18\ + """ + end + + test "two-line span" do + output = + capture_raise( + """ + [a, b, c + d, f, g} + """, + MismatchedDelimiterError + ) + + assert output == """ + ** (MismatchedDelimiterError) mismatched delimiter found on nofile:2:9: + error: unexpected token: } + │ + 1 │ [a, b, c + │ └ unclosed delimiter + 2 │ d, f, g} + │ └ mismatched closing delimiter (expected "]") + │ + └─ nofile:2:9\ + """ + end + + test "two-line span with offset" do + output = + capture_raise( + """ + [a, b, c + d, f, g} + """, + MismatchedDelimiterError, + line: 3 + ) + + assert output == """ + ** (MismatchedDelimiterError) mismatched delimiter found on nofile:4:9: + error: unexpected token: } + │ + 3 │ [a, b, c + │ └ unclosed delimiter + 4 │ d, f, g} + │ └ mismatched closing delimiter (expected "]") + │ + └─ nofile:4:9\ + """ + end + + test "many-line span" do + output = + capture_raise( + """ + [ a, + b, + c, + d + e ) + """, + MismatchedDelimiterError + ) + + assert output == """ + ** (MismatchedDelimiterError) mismatched delimiter found on nofile:5:5: + error: unexpected token: ) + │ + 1 │ [ a, + │ └ unclosed delimiter + 2 │ b, + 3 │ c, + 4 │ d + 5 │ e ) + │ └ mismatched closing delimiter (expected "]") + │ + └─ nofile:5:5\ + """ + end + + test "many-line span with offset" do + output = + capture_raise( + """ + fn always_forget_end -> + IO.inspect(2 + 2) + 2 + ) + """, + MismatchedDelimiterError, + line: 3 + ) + + assert output == """ + ** (MismatchedDelimiterError) mismatched delimiter found on nofile:5:1: + error: unexpected token: ) + │ + 3 │ fn always_forget_end -> + │ └ unclosed delimiter + 4 │ IO.inspect(2 + 2) + 2 + 5 │ ) + │ └ mismatched closing delimiter (expected "end") + │ + └─ nofile:5:1\ + """ + end + + test "line range - handles unicode input" do + output = + capture_raise( + """ + defmodule A do + IO.inspect(2 + 2) + ) <- 😎 + """, + MismatchedDelimiterError + ) + + assert output == """ + ** (MismatchedDelimiterError) mismatched delimiter found on nofile:3:1: + error: unexpected token: ) + │ + 1 │ defmodule A do + │ └ unclosed delimiter + 2 │ IO.inspect(2 + 2) + 3 │ ) <- 😎 + │ └ mismatched closing delimiter (expected "end") + │ + └─ nofile:3:1\ + """ + end + + test "trim in between lines if too many" do + output = + capture_raise( + """ + [ :a, + :b, + :c, + :d, + :e, + :f, + :g, + :h + ) + """, + MismatchedDelimiterError + ) + + assert output == """ + ** (MismatchedDelimiterError) mismatched delimiter found on nofile:9:1: + error: unexpected token: ) + │ + 1 │ [ :a, + │ └ unclosed delimiter + ... + 9 │ ) + │ └ mismatched closing delimiter (expected "]") + │ + └─ nofile:9:1\ + """ + end + + test "trimmed line range - handles unicode input" do + output = + capture_raise( + """ + [ :a, + :b, + :c, + :d, + :e, + :f, + :g, + :h + ) <- 😎 + """, + MismatchedDelimiterError + ) + + assert output == """ + ** (MismatchedDelimiterError) mismatched delimiter found on nofile:9:1: + error: unexpected token: ) + │ + 1 │ [ :a, + │ └ unclosed delimiter + ... + 9 │ ) <- 😎 + │ └ mismatched closing delimiter (expected "]") + │ + └─ nofile:9:1\ + """ + end + + test "pads according to line number digits" do + output = + capture_raise( + """ + [ a, + #{String.duplicate("\n", 10)} + b ) + """, + MismatchedDelimiterError + ) + + assert output == """ + ** (MismatchedDelimiterError) mismatched delimiter found on nofile:13:5: + error: unexpected token: ) + │ + 1 │ [ a, + │ └ unclosed delimiter + ... + 13 │ b ) + │ └ mismatched closing delimiter (expected "]") + │ + └─ nofile:13:5\ + """ + + output = + capture_raise( + """ + [ a, + #{String.duplicate("\n", 400)} + b ) + """, + MismatchedDelimiterError + ) + + assert output == """ + ** (MismatchedDelimiterError) mismatched delimiter found on nofile:403:5: + error: unexpected token: ) + │ + 1 │ [ a, + │ └ unclosed delimiter + ... + 403 │ b ) + │ └ mismatched closing delimiter (expected "]") + │ + └─ nofile:403:5\ + """ + + output = + capture_raise( + """ + #{String.duplicate("\n", 97)} + [ a, + #{String.duplicate("\n", 6)} + b ) + """, + MismatchedDelimiterError + ) + + assert output == """ + ** (MismatchedDelimiterError) mismatched delimiter found on nofile:107:5: + error: unexpected token: ) + │ + 99 │ [ a, + │ └ unclosed delimiter + ... + 107 │ b ) + │ └ mismatched closing delimiter (expected "]") + │ + └─ nofile:107:5\ + """ + end + end + + describe "token missing error" do + test "missing parens terminator" do + output = + capture_raise( + """ + my_numbers = [1, 2, 3, 4, 5, 6 + IO.inspect(my_numbers) + """, + TokenMissingError + ) + + assert output == """ + ** (TokenMissingError) token missing on nofile:2:23: + error: missing terminator: ] + │ + 1 │ my_numbers = [1, 2, 3, 4, 5, 6 + │ └ unclosed delimiter + 2 │ IO.inspect(my_numbers) + │ └ missing closing delimiter (expected "]") + │ + └─ nofile:2:23\ + """ + end + + test "missing heredoc terminator" do + output = + capture_raise( + """ + a = \""" + test string + + IO.inspect(10 + 20) + """, + TokenMissingError + ) + + assert output == """ + ** (TokenMissingError) token missing on nofile:4:20: + error: missing terminator: \""" (for heredoc starting at line 1) + │ + 1 │ a = \""" + │ └ unclosed delimiter + 2 │ test string + 3 │\s + 4 │ IO.inspect(10 + 20) + │ └ missing closing delimiter (expected \""") + │ + └─ nofile:4:20\ + """ + end + + test "missing sigil terminator" do + output = + capture_raise("~s (for sigil ~s< starting at line 1) + │ + 1 │ ~s") + │ └ unclosed delimiter + │ + └─ nofile:1:10\ + """ + + output = + capture_raise("~s|foobar", TokenMissingError) + + assert output == """ + ** (TokenMissingError) token missing on nofile:1:10: + error: missing terminator: | (for sigil ~s| starting at line 1) + │ + 1 │ ~s|foobar + │ │ └ missing closing delimiter (expected "|") + │ └ unclosed delimiter + │ + └─ nofile:1:10\ + """ + end + + test "missing string terminator" do + output = + capture_raise("\"foobar", TokenMissingError) + + assert output == """ + ** (TokenMissingError) token missing on nofile:1:8: + error: missing terminator: " (for string starting at line 1) + │ + 1 │ "foobar + │ │ └ missing closing delimiter (expected ") + │ └ unclosed delimiter + │ + └─ nofile:1:8\ + """ + end + + test "missing atom terminator" do + output = + capture_raise(":\"foobar", TokenMissingError) + + assert output == """ + ** (TokenMissingError) token missing on nofile:1:9: + error: missing terminator: " (for atom starting at line 1) + │ + 1 │ :"foobar + │ │ └ missing closing delimiter (expected ") + │ └ unclosed delimiter + │ + └─ nofile:1:9\ + """ + end + + test "missing function terminator" do + output = + capture_raise("K.\"foobar", TokenMissingError) + + assert output == """ + ** (TokenMissingError) token missing on nofile:1:10: + error: missing terminator: " (for function name starting at line 1) + │ + 1 │ K."foobar + │ │ └ missing closing delimiter (expected ") + │ └ unclosed delimiter + │ + └─ nofile:1:10\ + """ + end + + test "shows in between lines if EOL is not far below" do + output = + capture_raise( + """ + my_numbers = [1, 2, 3, 4, 5, 6 + my_numbers + |> Enum.map(&(&1 + 1)) + |> Enum.map(&(&1 * &1)) + |> IO.inspect() + """, + TokenMissingError + ) + + assert output == """ + ** (TokenMissingError) token missing on nofile:5:16: + error: missing terminator: ] + │ + 1 │ my_numbers = [1, 2, 3, 4, 5, 6 + │ └ unclosed delimiter + 2 │ my_numbers + 3 │ |> Enum.map(&(&1 + 1)) + 4 │ |> Enum.map(&(&1 * &1)) + 5 │ |> IO.inspect() + │ └ missing closing delimiter (expected "]") + │ + └─ nofile:5:16\ + """ + end + + test "trims lines" do + output = + capture_raise( + """ + my_numbers = (1, 2, 3, 4, 5, 6 + + + + + + + IO.inspect(my_numbers) + """, + TokenMissingError + ) + + assert output == """ + ** (TokenMissingError) token missing on nofile:8:23: + error: missing terminator: ) + │ + 1 │ my_numbers = (1, 2, 3, 4, 5, 6 + │ └ unclosed delimiter + ... + 8 │ IO.inspect(my_numbers) + │ └ missing closing delimiter (expected ")") + │ + └─ nofile:8:23\ + """ + end + + test "shows the last non-empty line of a file" do + output = + capture_raise( + """ + my_numbers = {1, 2, 3, 4, 5, 6 + IO.inspect(my_numbers) + + + + + """, + TokenMissingError + ) + + assert output == """ + ** (TokenMissingError) token missing on nofile:2:23: + error: missing terminator: } + │ + 1 │ my_numbers = {1, 2, 3, 4, 5, 6 + │ └ unclosed delimiter + 2 │ IO.inspect(my_numbers) + │ └ missing closing delimiter (expected "}") + │ + └─ nofile:2:23\ + """ + end + + test "supports unicode" do + output = + capture_raise( + """ + my_emojis = [1, 2, 3, 4 # ⚗️ + IO.inspect(my_numbers) + """, + TokenMissingError + ) + + assert output == """ + ** (TokenMissingError) token missing on nofile:2:23: + error: missing terminator: ] + │ + 1 │ my_emojis = [1, 2, 3, 4 # ⚗️ + │ └ unclosed delimiter + 2 │ IO.inspect(my_numbers) + │ └ missing closing delimiter (expected "]") + │ + └─ nofile:2:23\ + """ + end + end + + describe "compile-time exceptions" do + test "SyntaxError (snippet)" do + output = + capture_raise( + """ + [1, 2, 3, 4, 5, *] + """, + SyntaxError + ) + + assert output == """ + ** (SyntaxError) invalid syntax found on nofile:1:17: + error: syntax error before: '*' + │ + 1 │ [1, 2, 3, 4, 5, *] + │ ^ + │ + └─ nofile:1:17\ + """ + end + + test "SyntaxError (snippet) with offset" do + output = + capture_raise( + """ + [1, 2, 3, 4, 5, *] + """, + SyntaxError, + line: 3 + ) + + assert output == """ + ** (SyntaxError) invalid syntax found on nofile:3:17: + error: syntax error before: '*' + │ + 3 │ [1, 2, 3, 4, 5, *] + │ ^ + │ + └─ nofile:3:17\ + """ + end + + test "TokenMissingError (snippet)" do + output = + capture_raise( + """ + 1 + + """, + TokenMissingError + ) + + assert output == """ + ** (TokenMissingError) token missing on nofile:1:4: + error: syntax error: expression is incomplete + │ + 1 │ 1 + + │ ^ + │ + └─ nofile:1:4\ + """ + end + + test "TokenMissingError (snippet) with offset and column" do + output = + capture_raise( + """ + 1 + + """, + TokenMissingError, + line: 3, + column: 3 + ) + + assert output == """ + ** (TokenMissingError) token missing on nofile:3:6: + error: syntax error: expression is incomplete + │ + 3 │ 1 + + │ ^ + │ + └─ nofile:3:6\ + """ + end + + test "TokenMissingError (unclosed delimiter)" do + expected = """ + ** (TokenMissingError) token missing on nofile:1:5: + error: missing terminator: end + │ + 1 │ fn a + │ │ └ missing closing delimiter (expected "end") + │ └ unclosed delimiter + │ + └─ nofile:1:5\ + """ + + output = + capture_raise( + """ + fn a + """, + TokenMissingError + ) + + assert output == expected + end + + test "keeps trailing whitespace if under threshold" do + expected = """ + ** (SyntaxError) invalid syntax found on nofile:1:23: + error: unexpected token: "😎" (column 23, code point U+****) + │ + 1 │ a + 😎 + │ ^ + │ + └─ nofile:1:23\ + """ + + output = + capture_raise( + """ + a + 😎 + """, + SyntaxError + ) + + assert output == expected + end + + test "limits trailing whitespace if too many" do + expected = """ + ** (SyntaxError) invalid syntax found on nofile:1:43: + error: unexpected token: "😎" (column 43, code point U+****) + │ + 1 │ ... a + 😎 + │ ^ + │ + └─ nofile:1:43\ + """ + + output = + capture_raise( + """ + a + 😎 + """, + SyntaxError + ) + + assert output == expected + end + + test "shows stacktrace if present" do + fake_stacktrace = [ + {:fake, :fun, 3, [file: "nofile", line: 10]}, + {:real, :fun, 2, [file: "nofile", line: 10]} + ] + + expected = """ + ** (TokenMissingError) token missing on nofile:1:4: + error: syntax error: expression is incomplete + │ + 1 │ 1 - + │ ^ + │ + └─ nofile:1:4 + nofile:10: :fake.fun/3 + nofile:10: :real.fun/2 + """ + + output = + capture_raise( + """ + 1 - + """, + TokenMissingError, + stacktrace: fake_stacktrace + ) + + assert output == expected + end + + test "2-digit line errors stay aligned 1-digit line errors" do + fake_stacktrace = [ + {:fake, :fun, 3, [file: "nofile", line: 10]} + ] + + expected = """ + ** (TokenMissingError) token missing on nofile:12:4: + error: syntax error: expression is incomplete + │ + 12 │ 1 - + │ ^ + │ + └─ nofile:12:4 + nofile:10: :fake.fun/3 + """ + + output = + capture_raise( + """ + #{String.duplicate("\n", 10)} + 1 - + """, + TokenMissingError, + stacktrace: fake_stacktrace + ) + + assert output == expected + end + + test "handles unicode" do + source = """ + defmodule Sample do + def a do + 10 + 😎 + end + end + """ + + output = capture_raise(source, SyntaxError) + + assert output =~ "😎" + after + purge(Sample) + end + end + + describe "compiler warnings" do + @tag :tmp_dir + test "simple warning (line + column + file)", %{tmp_dir: tmp_dir} do + path = make_relative_tmp(tmp_dir, "long-warning.ex") + + source = """ + defmodule Sample do + @file "#{path}" + defp a, do: Unknown.b() + end + """ + + File.write!(path, source) + + expected = """ + warning: Unknown.b/0 is undefined (module Unknown is not available or is yet to be defined). Make sure the module name is correct and has been specified in full (or that an alias has been defined) + │ + 3 │ defp a, do: Unknown.b() + │ ~ + │ + └─ #{path}:3:23: Sample.a/0 + """ + + assert capture_eval(source) =~ expected + after + purge(Sample) + end + + @tag :tmp_dir + test "simple warning (line + file)", %{tmp_dir: tmp_dir} do + path = make_relative_tmp(tmp_dir, "long-warning.ex") + + source = """ + defmodule Sample do + @file "#{path}" + defp a, do: Unknown.b() + end + """ + + File.write!(path, source) + + expected = """ + warning: Unknown.b/0 is undefined (module Unknown is not available or is yet to be defined). Make sure the module name is correct and has been specified in full (or that an alias has been defined) + │ + 3 │ defp a, do: Unknown.b() + │ ~~~~~~~~~~~~~~~~~~~~~~~ + │ + └─ #{path}:3: Sample.a/0 + """ + + assert capture_eval(source, columns: false) =~ expected + after + purge(Sample) + end + + @tag :tmp_dir + test "simple warning with tabs (line + file)", %{tmp_dir: tmp_dir} do + path = make_relative_tmp(tmp_dir, "long-warning.ex") + + source = """ + defmodule Sample do + \t@file "#{path}" + \tdefp a do + \t\tUnknown.b() + \tend + end + """ + + File.write!(path, source) + + expected = """ + warning: Unknown.b/0 is undefined (module Unknown is not available or is yet to be defined). Make sure the module name is correct and has been specified in full (or that an alias has been defined) + │ + 4 │ \t\tUnknown.b() + │ ~~~~~~~~~~~ + │ + └─ #{path}:4: Sample.a/0 + """ + + assert capture_eval(source, columns: false) =~ expected + after + purge(Sample) + end + + test "simple warning (no file)" do + source = """ + defmodule Sample do + defp a, do: Unknown.b() + end + """ + + expected = """ + warning: Unknown.b/0 is undefined (module Unknown is not available or is yet to be defined). Make sure the module name is correct and has been specified in full (or that an alias has been defined) + └─ nofile:2:23: Sample.a/0 + """ + + assert capture_eval(source) =~ expected + after + purge(Sample) + end + + @tag :tmp_dir + test "IO.warn file+line", %{tmp_dir: tmp_dir} do + path = make_relative_tmp(tmp_dir, "io-warn-file-line.ex") + + source = """ + IO.warn("oops\\nmulti\\nline", file: __ENV__.file, line: __ENV__.line) + """ + + File.write!(path, source) + + expected = """ + warning: oops + multi + line + │ + 1 │ IO.warn("oops\\nmulti\\nline", file: __ENV__.file, line: __ENV__.line) + │ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + │ + └─ tmp\ + """ + + assert capture_io(:stderr, fn -> Code.eval_file(path) end) =~ expected + end + + @tag :tmp_dir + test "IO.warn file+line+column", %{tmp_dir: tmp_dir} do + path = make_relative_tmp(tmp_dir, "io-warn-file-line-column.ex") + + source = """ + IO.warn("oops\\nmulti\\nline", file: __ENV__.file, line: __ENV__.line, column: 4) + """ + + File.write!(path, source) + + expected = """ + warning: oops + multi + line + │ + 1 │ IO.warn("oops\\nmulti\\nline", file: __ENV__.file, line: __ENV__.line, column: 4) + │ ~ + │ + └─ tmp\ + """ + + assert capture_io(:stderr, fn -> Code.eval_file(path) end) =~ expected + end + + test "IO.warn with missing data" do + assert capture_eval(""" + IO.warn("oops-bad", file: #{inspect(__ENV__.file)}, line: 3, column: nil) + """) =~ "warning: oops-bad" + + assert capture_eval(""" + IO.warn("oops-bad", file: #{inspect(__ENV__.file)}, line: nil) + """) =~ "oops-bad" + + assert capture_eval(""" + IO.warn("oops-bad", file: nil) + """) =~ "oops-bad" + end + + @tag :tmp_dir + test "trims lines if too many whitespaces", %{tmp_dir: tmp_dir} do + path = make_relative_tmp(tmp_dir, "trim_warning_line.ex") + + source = """ + defmodule Sample do + @file "#{path}" + + def a do + Unknown.bar(:test) + end + end + """ + + File.write!(path, source) + + expected = """ + warning: Unknown.bar/1 is undefined (module Unknown is not available or is yet to be defined). Make sure the module name is correct and has been specified in full (or that an alias has been defined) + │ + 5 │ ... Unknown.bar(:test) + │ ~ + │ + └─ #{path}:5:53: Sample.a/0 + + """ + + assert capture_eval(source) == expected + after + purge(Sample) + end + + @tag :tmp_dir + test "handles unicode", %{tmp_dir: tmp_dir} do + path = make_relative_tmp(tmp_dir, "warning_group_unicode.ex") + + source = """ + defmodule Sample do + @file "#{path}" + + def a do + Unknown.bar("😎") + Unknown.bar("😎") + end + end + """ + + File.write!(path, source) + + assert capture_eval(source) =~ "😎" + after + purge(Sample) + end + end + + describe "warning groups" do + test "no file" do + source = """ + defmodule Sample do + def a do + Unknown.bar() + Unknown.bar() + Unknown.bar() + Unknown.bar() + end + end + """ + + expected = """ + warning: Unknown.bar/0 is undefined (module Unknown is not available or is yet to be defined). Make sure the module name is correct and has been specified in full (or that an alias has been defined) + └─ nofile:3:13: Sample.a/0 + └─ nofile:4:13: Sample.a/0 + └─ nofile:5:13: Sample.a/0 + └─ nofile:6:13: Sample.a/0 + + """ + + assert capture_eval(source) =~ expected + after + purge(Sample) + end + + @tag :tmp_dir + test "file + line + column", %{tmp_dir: tmp_dir} do + path = make_relative_tmp(tmp_dir, "warning_group_nofile.ex") + + source = """ + defmodule Sample do + @file "#{path}" + + def a do + Unknown.bar() + Unknown.bar() + Unknown.bar() + Unknown.bar() + end + end + """ + + File.write!(path, source) + + expected = """ + warning: Unknown.bar/0 is undefined (module Unknown is not available or is yet to be defined). Make sure the module name is correct and has been specified in full (or that an alias has been defined) + │ + 5 │ Unknown.bar() + │ ~ + │ + └─ #{path}:5:13: Sample.a/0 + └─ #{path}:6:13: Sample.a/0 + └─ #{path}:7:13: Sample.a/0 + └─ #{path}:8:13: Sample.a/0 + + """ + + assert capture_eval(source) == expected + after + purge(Sample) + end + + @tag :tmp_dir + test "file + line", %{tmp_dir: tmp_dir} do + path = make_relative_tmp(tmp_dir, "warning_group_nofile.ex") + + source = """ + defmodule Sample do + @file "#{path}" + + def a do + Unknown.bar() + Unknown.bar() + Unknown.bar() + Unknown.bar() + end + end + """ + + File.write!(path, source) + + expected = """ + warning: Unknown.bar/0 is undefined (module Unknown is not available or is yet to be defined). Make sure the module name is correct and has been specified in full (or that an alias has been defined) + │ + 5 │ Unknown.bar() + │ ~~~~~~~~~~~~~ + │ + └─ #{path}:5: Sample.a/0 + └─ #{path}:6: Sample.a/0 + └─ #{path}:7: Sample.a/0 + └─ #{path}:8: Sample.a/0 + + """ + + assert capture_eval(source, columns: false) == expected + after + purge(Sample) + end + end + + describe "error diagnostics" do + @tag :tmp_dir + test "line only", %{tmp_dir: tmp_dir} do + path = make_relative_tmp(tmp_dir, "error_line_only.ex") + + source = """ + defmodule Sample do + @file "#{path}" + def CamelCase do + end + end + """ + + File.write!(path, source) + + expected = """ + error: function names should start with lowercase characters or underscore, invalid name CamelCase + │ + 3 │ def CamelCase do + │ ^^^^^^^^^^^^^^^^ + │ + └─ #{path}:3 + + """ + + assert capture_compile(source, columns: false) == expected + after + purge(Sample) + end + + @tag :tmp_dir + test "shows span for unused variables", %{tmp_dir: tmp_dir} do + path = make_relative_tmp(tmp_dir, "error_line_column.ex") + + source = """ + defmodule Sample do + @file "#{path}" + + def foo(unused_param) do + :constant + end + end + """ + + File.write!(path, source) + + expected = """ + warning: variable "unused_param" is unused (if the variable is not meant to be used, prefix it with an underscore) + │ + 4 │ def foo(unused_param) do + │ ~~~~~~~~~~~~ + │ + └─ #{path}:4:11: Sample.foo/1 + + """ + + assert capture_eval(source) == expected + after + purge(Sample) + end + + @tag :tmp_dir + test "shows span for undefined variables", %{tmp_dir: tmp_dir} do + path = make_relative_tmp(tmp_dir, "undefined_variable_span.ex") + + source = """ + defmodule Sample do + @file "#{path}" + + def foo(a) do + a - unknown_var + end + end + """ + + File.write!(path, source) + + expected = """ + error: undefined variable "unknown_var" + │ + 5 │ a - unknown_var + │ ^^^^^^^^^^^ + │ + └─ #{path}:5:9: Sample.foo/1 + + """ + + assert capture_compile(source) == expected + after + purge(Sample) + end + + @tag :tmp_dir + test "shows span for unknown local function calls", %{tmp_dir: tmp_dir} do + path = make_relative_tmp(tmp_dir, "unknown_local_function_call.ex") + + source = """ + defmodule Sample do + @file "#{path}" + + def foo do + _result = unknown_func_call!(:hello!) + end + end + """ + + File.write!(path, source) + + expected = """ + error: undefined function unknown_func_call!/1 (expected Sample to define such a function or for it to be imported, but none are available) + │ + 5 │ _result = unknown_func_call!(:hello!) + │ ^^^^^^^^^^^^^^^^^^ + │ + └─ #{path}:5:15: Sample.foo/0 + + """ + + assert capture_compile(source) == expected + after + purge(Sample) + end + + @tag :tmp_dir + test "line + column", %{tmp_dir: tmp_dir} do + path = make_relative_tmp(tmp_dir, "error_line_column.ex") + + source = """ + defmodule Sample do + @file "#{path}" + + def foo do + IO.puts(bar) + end + end + """ + + File.write!(path, source) + + expected = """ + error: undefined variable "bar" + │ + 5 │ IO.puts(bar) + │ ^^^ + │ + └─ #{path}:5:13: Sample.foo/0 + + """ + + assert capture_compile(source) == expected + after + purge(Sample) + end + + test "no file" do + expected = """ + error: undefined function module_info/0 (this function is auto-generated by the compiler and must always be called as a remote, as in __MODULE__.module_info/0) + └─ nofile:2:16: Sample.foo/0 + + """ + + output = + capture_compile(""" + defmodule Sample do + def foo, do: module_info() + end + """) + + assert expected == output + after + purge(Sample) + end + end + + describe "warning diagnostics" do + @tag :tmp_dir + test "line only", %{tmp_dir: tmp_dir} do + path = make_relative_tmp(tmp_dir, "warn_line.ex") + + source = """ + defmodule Sample do + @file "#{path}" + def a(unused), do: 1 + end + """ + + File.write!(path, source) + + expected = """ + warning: variable "unused" is unused (if the variable is not meant to be used, prefix it with an underscore) + │ + 3 │ def a(unused), do: 1 + │ ~~~~~~~~~~~~~~~~~~~~ + │ + └─ #{path}:3: Sample.a/1 + + """ + + assert capture_eval(source, columns: false) == expected + after + purge(Sample) + end + + @tag :tmp_dir + test "line + column", %{tmp_dir: tmp_dir} do + path = make_relative_tmp(tmp_dir, "warn_line_column.ex") + + source = """ + defmodule Sample do + @file "#{path}" + @foo 1 + + def bar do + @foo + :ok + end + end + """ + + File.write!(path, source) + + expected = """ + warning: module attribute @foo in code block has no effect as it is never returned (remove the attribute or assign it to _ to avoid warnings) + │ + 6 │ @foo + │ ~ + │ + └─ #{path}:6:5: Sample.bar/0 + + """ + + assert capture_eval(source) == expected + after + purge(Sample) + end + + test "no file" do + expected = """ + warning: unused alias List + └─ nofile:2:3 + + """ + + output = + capture_eval(""" + defmodule Sample do + alias :lists, as: List + import MapSet + new() + end + """) + + assert output == expected + after + purge(Sample) + end + end + + describe "Code.print_diagnostic" do + @tag :tmp_dir + test "handles diagnostic with span", %{tmp_dir: tmp_dir} do + path = make_relative_tmp(tmp_dir, "diagnostic_length.ex") + + diagnostic = %{ + file: path, + severity: :error, + message: "Diagnostic span test", + stacktrace: [], + position: {4, 7}, + span: {4, 10}, + compiler_name: "Elixir" + } + + source = """ + defmodule Sample do + @file "#{path}" + + def bar do + nil + end + end + """ + + File.write!(path, source) + + result = + ExUnit.CaptureIO.capture_io(:stderr, fn -> + Code.print_diagnostic(diagnostic) + end) + + assert result == """ + error: Diagnostic span test + │ + 4 │ def bar do + │ ^^^ + │ + └─ #{path}:4:7 + + """ + end + + @tag :tmp_dir + test "prints single marker for multiline diagnostic", %{tmp_dir: tmp_dir} do + path = make_relative_tmp(tmp_dir, "diagnostic_length.ex") + + diagnostic = %{ + file: path, + severity: :error, + message: "Diagnostic span test", + stacktrace: [], + position: {4, 7}, + span: {5, 2}, + compiler_name: "Elixir" + } + + source = """ + defmodule Sample do + @file "#{path}" + + def bar do + nil + end + end + """ + + File.write!(path, source) + + result = + ExUnit.CaptureIO.capture_io(:stderr, fn -> + Code.print_diagnostic(diagnostic) + end) + + assert result == """ + error: Diagnostic span test + │ + 4 │ def bar do + │ ^ + │ + └─ #{path}:4:7 + + """ + end + end + + defp make_relative_tmp(tmp_dir, filename) do + # Compiler outputs relative, so we just grab the tmp dir + tmp_dir + |> Path.join(filename) + |> Path.relative_to_cwd() + end + + defp capture_eval(source, opts \\ [columns: true]) do + capture_io(:stderr, fn -> + quoted = Code.string_to_quoted!(source, opts) + Code.eval_quoted(quoted) + end) + end + + defp capture_compile(source, opts \\ [columns: true]) do + capture_io(:stderr, fn -> + assert_raise CompileError, fn -> + ast = Code.string_to_quoted!(source, opts) + Code.eval_quoted(ast) + end + end) + end + + defp capture_raise(source, exception, opts \\ []) do + {stacktrace, opts} = Keyword.pop(opts, :stacktrace, []) + + e = + assert_raise exception, fn -> + ast = Code.string_to_quoted!(source, [columns: true] ++ opts) + Code.eval_quoted(ast) + end + + Exception.format(:error, e, stacktrace) + end + + defp purge(module) when is_atom(module) do + :code.purge(module) + :code.delete(module) + end +end diff --git a/lib/elixir/test/elixir/kernel/dialyzer_test.exs b/lib/elixir/test/elixir/kernel/dialyzer_test.exs new file mode 100644 index 00000000000..84182774343 --- /dev/null +++ b/lib/elixir/test/elixir/kernel/dialyzer_test.exs @@ -0,0 +1,218 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Kernel.DialyzerTest do + use ExUnit.Case, async: true + + @moduletag :dialyzer + @moduletag :require_ast + import PathHelpers + + setup_all do + dir = tmp_path("dialyzer") + File.rm_rf!(dir) + File.mkdir_p!(dir) + + plt = + dir + |> Path.join("base_plt") + |> String.to_charlist() + + # Some OSs (like Windows) do not provide the HOME environment variable. + if !System.get_env("HOME") do + System.put_env("HOME", System.user_home()) + end + + # Add a few key Elixir modules for types and macro functions + mods = [ + :elixir, + :elixir_env, + :elixir_erl_pass, + :maps, + :sets, + ArgumentError, + Atom, + Code, + EEx, + Enum, + Exception, + ExUnit.AssertionError, + ExUnit.Assertions, + IO, + Kernel, + Kernel.Utils, + List, + Macro, + Macro.Env, + MapSet, + Module, + Protocol, + String, + String.Chars, + Task, + Task.Supervisor + ] + + files = Enum.map(mods, &:code.which/1) + dialyzer_run(analysis_type: :plt_build, output_plt: plt, apps: [:erts], files: files) + + # Compile Dialyzer fixtures + source_files = Path.wildcard(Path.join(fixture_path("dialyzer"), "*")) + + {:ok, _, _} = + Kernel.ParallelCompiler.compile_to_path(source_files, dir, return_diagnostics: true) + + {:ok, [base_dir: dir, base_plt: plt]} + end + + setup context do + dir = String.to_charlist(context.tmp_dir) + + plt = + dir + |> Path.join("plt") + |> String.to_charlist() + + File.cp!(context.base_plt, plt) + warnings = Map.get(context, :warnings, []) + + dialyzer = [ + analysis_type: :succ_typings, + check_plt: false, + files_rec: [dir], + plts: [plt], + warnings: warnings + ] + + {:ok, [outdir: dir, dialyzer: dialyzer]} + end + + @moduletag :tmp_dir + + @tag warnings: [:specdiffs] + test "no warnings on specdiffs", context do + copy_beam!(context, Dialyzer.RemoteCall) + assert_dialyze_no_warnings!(context) + end + + test "no warnings on valid remote calls", context do + copy_beam!(context, Dialyzer.RemoteCall) + assert_dialyze_no_warnings!(context) + end + + test "no warnings on rewrites", context do + copy_beam!(context, Dialyzer.Rewrite) + assert_dialyze_no_warnings!(context) + end + + test "no warnings on raise", context do + copy_beam!(context, Dialyzer.Raise) + assert_dialyze_no_warnings!(context) + end + + test "no warnings on macrocallback", context do + copy_beam!(context, Dialyzer.Macrocallback) + copy_beam!(context, Dialyzer.Macrocallback.Impl) + assert_dialyze_no_warnings!(context) + end + + test "no warnings on callback", context do + copy_beam!(context, Dialyzer.Callback) + copy_beam!(context, Dialyzer.Callback.ImplAtom) + copy_beam!(context, Dialyzer.Callback.ImplList) + assert_dialyze_no_warnings!(context) + end + + test "no warnings on and/2 and or/2", context do + copy_beam!(context, Dialyzer.BooleanCheck) + assert_dialyze_no_warnings!(context) + end + + test "no warnings on cond", context do + copy_beam!(context, Dialyzer.Cond) + assert_dialyze_no_warnings!(context) + end + + test "no warnings on for comprehensions with bitstrings", context do + copy_beam!(context, Dialyzer.ForBitstring) + assert_dialyze_no_warnings!(context) + end + + test "no warnings on for falsy check that always boolean", context do + copy_beam!(context, Dialyzer.ForBooleanCheck) + assert_dialyze_no_warnings!(context) + end + + test "no warnings on with/else", context do + copy_beam!(context, Dialyzer.With) + assert_dialyze_no_warnings!(context) + end + + test "no warnings on with when else has a no_return type", context do + copy_beam!(context, Dialyzer.WithNoReturn) + assert_dialyze_no_warnings!(context) + end + + test "no warnings on with when multiple else clauses and one is a no_return", context do + copy_beam!(context, Dialyzer.WithThrowingElse) + assert_dialyze_no_warnings!(context) + end + + test "no warnings on defmacrop", context do + copy_beam!(context, Dialyzer.Defmacrop) + assert_dialyze_no_warnings!(context) + end + + test "no warnings on try", context do + copy_beam!(context, Dialyzer.Try) + assert_dialyze_no_warnings!(context) + end + + test "no warning on is_struct/2", context do + copy_beam!(context, Dialyzer.IsStruct) + assert_dialyze_no_warnings!(context) + end + + test "no warning on ExUnit assertions", context do + copy_beam!(context, Dialyzer.Assertions) + assert_dialyze_no_warnings!(context) + end + + test "no warning due to opaqueness edge cases", context do + copy_beam!(context, Dialyzer.Opaqueness) + assert_dialyze_no_warnings!(context) + end + + test "no warning in various non-regression cases", context do + copy_beam!(context, Dialyzer.Regressions) + assert_dialyze_no_warnings!(context) + end + + defp copy_beam!(context, module) do + name = "#{module}.beam" + File.cp!(Path.join(context.base_dir, name), Path.join(context.outdir, name)) + end + + defp assert_dialyze_no_warnings!(context) do + case dialyzer_run(context.dialyzer) do + [] -> + :ok + + warnings -> + formatted = for warn <- warnings, do: [:dialyzer.format_warning(warn), ?\n] + formatted |> IO.chardata_to_string() |> flunk() + end + end + + defp dialyzer_run(opts) do + try do + :dialyzer.run(opts) + catch + :throw, {:dialyzer_error, chardata} -> + raise "dialyzer error: " <> IO.chardata_to_string(chardata) + end + end +end diff --git a/lib/elixir/test/elixir/kernel/docs_test.exs b/lib/elixir/test/elixir/kernel/docs_test.exs index 1b0d55a4986..fde228722a4 100644 --- a/lib/elixir/test/elixir/kernel/docs_test.exs +++ b/lib/elixir/test/elixir/kernel/docs_test.exs @@ -1,60 +1,457 @@ -Code.require_file "../test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) defmodule Kernel.DocsTest do use ExUnit.Case - test "compiled with docs" do - deftestmodule(SampleDocs) - docs = Code.get_docs(SampleDocs, :all) + import PathHelpers - assert [{{:fun, 2}, _, :def, [{:x, [], nil}, {:y, [], nil}], "This is fun!\n"}, - {{:nofun, 0}, _, :def, [], nil}, - {{:sneaky, 1}, _, :def, [{:bool1, [], Elixir}], false}] = docs[:docs] - assert {_, "Hello, I am a module"} = docs[:moduledoc] + defmacro wrong_doc_baz do + quote do + @doc "Wrong doc" + @doc since: "1.2" + def baz(_arg) + def baz(arg), do: arg + 1 + end + end + + test "attributes format" do + defmodule DocAttributes do + @moduledoc "Module doc" + assert @moduledoc == "Module doc" + assert Module.get_attribute(__MODULE__, :moduledoc) == {__ENV__.line - 2, "Module doc"} + + @typedoc "Type doc" + assert @typedoc == "Type doc" + assert Module.get_attribute(__MODULE__, :typedoc) == {__ENV__.line - 2, "Type doc"} + @type foobar :: any + + @doc "Function doc" + assert @doc == "Function doc" + assert Module.get_attribute(__MODULE__, :doc) == {__ENV__.line - 2, "Function doc"} + + def foobar() do + :ok + end + end end test "compiled without docs" do Code.compiler_options(docs: false) - deftestmodule(SampleNoDocs) + write_beam( + defmodule WithoutDocs do + @moduledoc "Module doc" + + @doc "Some doc" + def foobar(arg), do: arg + end + ) - assert Code.get_docs(SampleNoDocs, :docs) == nil - assert Code.get_docs(SampleNoDocs, :moduledoc) == nil + assert Code.fetch_docs(WithoutDocs) == {:error, :chunk_not_found} after Code.compiler_options(docs: true) end test "compiled in memory does not have accessible docs" do - defmodule NoDocs do - @moduledoc "moduledoc" + defmodule InMemoryDocs do + @moduledoc "Module doc" - @doc "Some example" - def example(var), do: var + @doc "Some doc" + def foobar(arg), do: arg end - assert Code.get_docs(NoDocs, :docs) == nil - assert Code.get_docs(NoDocs, :moduledoc) == nil + assert Code.fetch_docs(InMemoryDocs) == {:error, :module_not_found} end - defp deftestmodule(name) do - import PathHelpers + test "non-existent beam file" do + assert {:error, :module_not_found} = Code.fetch_docs("bad.beam") + end - write_beam(defmodule name do - @moduledoc "Hello, I am a module" + test "raises on invalid @doc since: ..." do + assert_raise ArgumentError, ~r"should be a string representing the version", fn -> + defmodule InvalidSince do + @doc since: 1.2 + def foo, do: :bar + end + end + end - @doc """ - This is fun! - """ - def fun(x, y) do - {x, y} + test "raises on invalid @doc" do + assert_raise ArgumentError, ~r/When set dynamically, it should be {line, doc}/, fn -> + defmodule DocAttributesFormat do + Module.put_attribute(__MODULE__, :moduledoc, "Other") end + end + + message = ~r/should be either false, nil, a string, or a keyword list/ - @doc false - def sneaky(true), do: false + assert_raise ArgumentError, message, fn -> + defmodule AtSyntaxDocAttributesFormat do + @moduledoc :not_a_binary + end + end - def nofun() do - 'not fun at all' + assert_raise ArgumentError, message, fn -> + defmodule AtSyntaxDocAttributesFormat do + @moduledoc true end - end) + end + end + + describe "compiled with docs" do + test "infers signatures" do + write_beam( + defmodule SignatureDocs do + def arg_names([], [], %{}, [], %{}), do: false + + @year 2015 + def with_defaults(@year, arg \\ 0, year \\ @year, fun \\ &>=/2) do + {fun, arg + year} + end + + def with_map_and_default(%{key: value} \\ %{key: :default}), do: value + def with_struct(%URI{}), do: :ok + + def with_underscore({_, _} = _two_tuple), do: :ok + def with_underscore(_), do: :error + + def only_underscore(_), do: :ok + + def two_good_names(first, :ok), do: first + def two_good_names(second, :error), do: second + + def really_long_signature( + really_long_var_named_one, + really_long_var_named_two, + really_long_var_named_three + ) do + {really_long_var_named_one, really_long_var_named_two, really_long_var_named_three} + end + end + ) + + assert {:docs_v1, _, :elixir, _, _, _, docs} = Code.fetch_docs(SignatureDocs) + signatures = for {{:function, n, a}, _, signature, _, %{}} <- docs, do: {{n, a}, signature} + + assert [ + arg_names, + only_underscore, + really_long_signature, + two_good_names, + with_defaults, + with_map_and_default, + with_struct, + with_underscore + ] = Enum.sort(signatures) + + # arg_names/5 + assert {{:arg_names, 5}, ["arg_names(list1, list2, map1, list3, map2)"]} = arg_names + + # only_underscore/1 + assert {{:only_underscore, 1}, ["only_underscore(_)"]} = only_underscore + + # really_long_signature/3 + assert {{:really_long_signature, 3}, + [ + "really_long_signature(really_long_var_named_one, really_long_var_named_two, really_long_var_named_three)" + ]} = really_long_signature + + # two_good_names/2 + assert {{:two_good_names, 2}, ["two_good_names(first, atom)"]} = two_good_names + + # with_defaults/4 + assert {{:with_defaults, 4}, + ["with_defaults(int, arg \\\\ 0, year \\\\ 2015, fun \\\\ &>=/2)"]} = with_defaults + + # with_map_and_default/1 + assert {{:with_map_and_default, 1}, ["with_map_and_default(map \\\\ %{key: :default})"]} = + with_map_and_default + + # with_struct/1 + assert {{:with_struct, 1}, ["with_struct(uri)"]} = with_struct + + # with_underscore/1 + assert {{:with_underscore, 1}, ["with_underscore(two_tuple)"]} = with_underscore + end + + test "includes docs for functions, modules, types and callbacks" do + line = __ENV__.line + + write_beam( + defmodule SampleDocs do + @moduledoc "Module doc" + @moduledoc authors: "Elixir Contributors", purpose: :test + + @doc "My struct" + defstruct [:sample] + + @typedoc "Type doc" + @typedoc since: "1.2.3", color: :red + @type foo(any) :: any + + @typedoc "Opaque type doc" + @opaque bar(any) :: any + + @doc "Callback doc" + @doc since: "1.2.3", color: :red, deprecated: "use baz/2 instead" + @doc color: :blue, stable: true + @callback foo(any) :: any + + @doc false + @doc since: "1.2.3" + @callback bar() :: term + @callback baz(any, term) :: any + + @doc "Callback with multiple clauses" + @callback callback_multi(integer) :: integer + @callback callback_multi(atom) :: atom + + @doc "Macrocallback doc" + @macrocallback qux(any) :: any + + @doc "Macrocallback with multiple clauses" + @macrocallback macrocallback_multi(integer) :: integer + @macrocallback macrocallback_multi(atom) :: atom + + @doc "Function doc" + @doc since: "1.2.3", color: :red + @doc color: :blue, stable: true + @deprecated "use baz/2 instead" + def foo(arg \\ 0), do: arg + 1 + + @doc "Multiple function head doc" + @deprecated "something else" + def bar(_arg) + def bar(arg), do: arg + 1 + + require Kernel.DocsTest + Kernel.DocsTest.wrong_doc_baz() + + @doc "Multiple function head and docs" + @doc since: "1.2.3" + def baz(_arg) + + @doc false + def qux(true), do: false + + @doc "A guard" + defguard is_zero(v) when v == 0 + + # We do this to avoid the deprecation warning. + module = Module + module.add_doc(__MODULE__, __ENV__.line, :def, {:nullary, 0}, [], "add_doc") + def nullary, do: 0 + + defmodule SampleBehaviour do + @callback foo(any()) :: any() + end + + @behaviour SampleBehaviour + end + ) + + assert {:docs_v1, _, :elixir, "text/markdown", %{"en" => module_doc}, module_doc_meta, docs} = + Code.fetch_docs(SampleDocs) + + assert module_doc == "Module doc" + + file = String.to_charlist(__ENV__.file) + + source_annos = [:erl_anno.new({line + 3, 19})] + + assert %{ + # Generated meta + source_path: ^file, + source_annos: ^source_annos, + behaviours: [SampleDocs.SampleBehaviour], + # User meta + authors: "Elixir Contributors", + purpose: :test + } = module_doc_meta + + [ + callback_bar, + callback_baz, + callback_multi, + callback_foo, + function_struct_0, + function_struct_1, + function_bar, + function_baz, + function_foo, + function_nullary, + function_qux, + guard_is_zero, + macrocallback_multi, + macrocallback_qux, + type_bar, + type_foo + ] = Enum.sort(docs) + + assert {{:callback, :bar, 0}, _, [], :hidden, %{}} = callback_bar + assert {{:callback, :baz, 2}, _, [], :none, %{}} = callback_baz + + source_annos = [:erl_anno.new({line + 20, 21})] + + assert {{:callback, :foo, 1}, _, [], %{"en" => "Callback doc"}, + %{ + source_annos: ^source_annos, + since: "1.2.3", + deprecated: "use baz/2 instead", + color: :blue, + stable: true + }} = callback_foo + + assert {{:callback, :callback_multi, 1}, _, [], %{"en" => "Callback with multiple clauses"}, + %{}} = callback_multi + + assert {{:function, :__struct__, 0}, _, ["%Kernel.DocsTest.SampleDocs{}"], + %{"en" => "My struct"}, %{}} = function_struct_0 + + assert {{:function, :__struct__, 1}, _, ["__struct__(kv)"], :hidden, %{}} = + function_struct_1 + + assert {{:function, :bar, 1}, _, ["bar(arg)"], %{"en" => "Multiple function head doc"}, + %{deprecated: "something else"}} = function_bar + + assert {{:function, :baz, 1}, _, ["baz(arg)"], %{"en" => "Multiple function head and docs"}, + %{since: "1.2.3"}} = function_baz + + source_annos = [:erl_anno.new({line + 42, 15})] + + assert {{:function, :foo, 1}, _, ["foo(arg \\\\ 0)"], %{"en" => "Function doc"}, + %{ + source_annos: ^source_annos, + since: "1.2.3", + deprecated: "use baz/2 instead", + color: :blue, + stable: true, + defaults: 1 + }} = function_foo + + assert {{:function, :nullary, 0}, _, ["nullary()"], %{"en" => "add_doc"}, %{}} = + function_nullary + + assert {{:function, :qux, 1}, _, ["qux(bool)"], :hidden, %{}} = function_qux + + source_annos = [:erl_anno.new({line + 60, 20})] + + assert {{:macro, :is_zero, 1}, _, ["is_zero(v)"], %{"en" => "A guard"}, + %{source_annos: ^source_annos, guard: true}} = guard_is_zero + + assert {{:macrocallback, :macrocallback_multi, 1}, _, [], + %{"en" => "Macrocallback with multiple clauses"}, %{}} = macrocallback_multi + + assert {{:macrocallback, :qux, 1}, _, [], %{"en" => "Macrocallback doc"}, %{}} = + macrocallback_qux + + assert {{:type, :bar, 1}, _, [], %{"en" => "Opaque type doc"}, %{opaque: true}} = type_bar + + source_annos = [:erl_anno.new({line + 12, 17})] + + assert {{:type, :foo, 1}, _, [], %{"en" => "Type doc"}, + %{ + source_annos: ^source_annos, + since: "1.2.3", + color: :red + }} = type_foo + end + end + + test "fetch docs chunk from doc/chunks" do + Code.compiler_options(docs: false) + + doc_chunks_path = Path.join([tmp_path(), "doc", "chunks"]) + File.rm_rf!(doc_chunks_path) + File.mkdir_p!(doc_chunks_path) + + write_beam( + defmodule ExternalDocs do + end + ) + + assert Code.fetch_docs(ExternalDocs) == {:error, :chunk_not_found} + + path = Path.join([doc_chunks_path, "#{ExternalDocs}.chunk"]) + chunk = {:docs_v1, 1, :elixir, "text/markdown", %{"en" => "Some docs"}, %{}} + File.write!(path, :erlang.term_to_binary(chunk)) + + assert Code.fetch_docs(ExternalDocs) == chunk + after + Code.compiler_options(docs: true) + end + + test "@impl true doesn't set @doc false if previous implementation has docs" do + write_beam( + defmodule Docs do + defmodule SampleBehaviour do + @callback foo(any()) :: any() + @callback bar() :: any() + @callback baz() :: any() + end + + @behaviour SampleBehaviour + + @doc "Foo docs" + def foo(nil), do: nil + + @impl true + def foo(_), do: false + + @impl true + def bar(), do: true + + @doc "Baz docs" + @impl true + def baz(), do: true + + def fuz(), do: true + end + ) + + {:docs_v1, _, _, _, _, _, docs} = Code.fetch_docs(Docs) + function_docs = for {{:function, name, arity}, _, _, doc, _} <- docs, do: {{name, arity}, doc} + + assert [ + {{:bar, 0}, :hidden}, + {{:baz, 0}, %{"en" => "Baz docs"}}, + {{:foo, 1}, %{"en" => "Foo docs"}}, + {{:fuz, 0}, :none} + ] = Enum.sort(function_docs) + end + + test "generated functions are annotated as such" do + line = __ENV__.line + + write_beam( + defmodule ToBeUsed do + defmacro __using__(_) do + quote generated: true do + @doc "Hello" + def foo, do: :bar + end + end + end + ) + + write_beam( + defmodule WillBeUsing do + use ToBeUsed + end + ) + + {:docs_v1, _, _, _, _, _, docs} = Code.fetch_docs(WillBeUsing) + + doc_anno = :erl_anno.new(line + 15) + source_anno = :erl_anno.set_generated(true, :erl_anno.new(line + 15)) + + assert [ + {{:function, :foo, 0}, ^doc_anno, ["foo()"], %{"en" => "Hello"}, + %{source_annos: [^source_anno]}} + ] = docs end end diff --git a/lib/elixir/test/elixir/kernel/errors_test.exs b/lib/elixir/test/elixir/kernel/errors_test.exs index 5e5fa194a7a..f6dd6c4190f 100644 --- a/lib/elixir/test/elixir/kernel/errors_test.exs +++ b/lib/elixir/test/elixir/kernel/errors_test.exs @@ -1,717 +1,1042 @@ -Code.require_file "../test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) defmodule Kernel.ErrorsTest do use ExUnit.Case, async: true - import CompileAssertion - - defmodule UnproperMacro do - defmacro unproper(args), do: args - defmacro exit(args), do: args - end - test :invalid_token do - assert_compile_fail SyntaxError, - "nofile:1: invalid token: \end", - '\end\nlol\nbarbecue' + defmacro hello do + quote location: :keep do + def hello, do: :world + end end - test :invalid_quoted_token do - assert_compile_fail SyntaxError, - "nofile:1: syntax error before: \"world\"", - '"hello" "world"' - - assert_compile_fail SyntaxError, - "nofile:1: syntax error before: foo", - 'Foo.:foo' - - assert_compile_fail SyntaxError, - "nofile:1: syntax error before: \"foo\"", - 'Foo.:"foo\#{:bar}"' + test "no default arguments in fn" do + assert_compile_error( + ["nofile:1:1", "anonymous functions cannot have optional arguments"], + ~c"fn x \\\\ 1 -> x end" + ) + + assert_compile_error( + ["nofile:1:1", "anonymous functions cannot have optional arguments"], + ~c"fn x, y \\\\ 1 -> x + y end" + ) + end + + test "invalid __CALLER__" do + assert_compile_error( + ["nofile:1:34", "__CALLER__ is available only inside defmacro and defmacrop"], + ~c"defmodule Sample do def hello do __CALLER__ end end" + ) + end + + test "invalid __STACKTRACE__" do + assert_compile_error( + [ + "nofile:1:34", + "__STACKTRACE__ is available only inside catch and rescue clauses of try expressions" + ], + ~c"defmodule Sample do def hello do __STACKTRACE__ end end" + ) + + assert_compile_error( + [ + "nofile:1:66", + "__STACKTRACE__ is available only inside catch and rescue clauses of try expressions" + ], + ~c"defmodule Sample do try do raise \"oops\" rescue _ -> def hello do __STACKTRACE__ end end end" + ) + end + + test "undefined function" do + assert_compile_error( + [ + "hello.ex:4:5: ", + "undefined function bar/0 (expected Kernel.ErrorsTest.BadForm to define such a function or for it to be imported, but none are available)" + ], + ~c""" + defmodule Kernel.ErrorsTest.BadForm do + @file "hello.ex" + def foo do + bar() + end + end + """ + ) + + assert_compile_error( + [ + "nofile:2:16", + "undefined function module_info/0 (this function is auto-generated by the compiler and must always be called as a remote, as in __MODULE__.module_info/0)" + ], + ~c""" + defmodule Kernel.ErrorsTest.Info do + def foo, do: module_info() + end + """ + ) + + assert_compile_error( + [ + "nofile:3:16", + "undefined function behaviour_info/1 (this function is auto-generated by the compiler and must always be called as a remote, as in __MODULE__.behaviour_info/1)" + ], + ~c""" + defmodule Kernel.ErrorsTest.BehaviourInfo do + @callback dummy() :: :ok + def foo, do: behaviour_info(:callbacks) + end + """ + ) + + assert_compile_error( + [ + "nofile:3:5", + "undefined function bar/1 (expected Kernel.ErrorsTest.BadForm to define such a function or for it to be imported, but none are available)" + ], + ~c""" + defmodule Kernel.ErrorsTest.BadForm do + def foo do + bar( + baz(1, 2) + ) + end + end + """ + ) + + assert_compile_error( + [ + "nofile:8:5", + "undefined function baz/0 (expected Sample to define such a function or for it to be imported, but none are available)" + ], + ~c""" + defmodule Sample do + def foo do + bar() + end - assert_compile_fail SyntaxError, - "nofile:1: syntax error before: \"", - 'Foo.:"\#{:bar}"' + defoverridable [foo: 0] + def foo do + baz() + end + end + """ + ) end - test :invalid_or_reserved_codepoint do - assert_compile_fail ArgumentError, - "invalid or reserved unicode codepoint 55296", - '?\\x{D800}' + test "undefined function within unused function" do + assert_compile_error( + ["nofile:2:8", "undefined function bar/0"], + ~c""" + defmodule Sample do + defp foo, do: bar() + end + """ + ) end - test :sigil_terminator do - assert_compile_fail TokenMissingError, - "nofile:3: missing terminator: \" (for sigil ~r\" starting at line 1)", - '~r"foo\n\n' - - assert_compile_fail TokenMissingError, - "nofile:3: missing terminator: } (for sigil ~r{ starting at line 1)", - '~r{foo\n\n' + test "undefined non-local function" do + assert_compile_error( + ["nofile:1:1", "undefined function call/2 (there is no such import)"], + ~c"call foo, do: :foo" + ) end - test :dot_terminator do - assert_compile_fail TokenMissingError, - "nofile:1: missing terminator: \" (for function name starting at line 1)", - 'foo."bar' + test "undefined variables" do + assert_compile_error( + ["nofile:3:13", "undefined variable \"bar\"", "nofile:4:13", "undefined variable \"baz\""], + ~c""" + defmodule Sample do + def foo do + IO.puts bar + IO.puts baz + end + end + """ + ) + end + + test "recursive variables on definition" do + assert_compile_error( + [ + "nofile:2:7: ", + "recursive variable definition in patterns:", + "foo(x = y, y = z, z = x)", + "the following variables form a cycle: \"x\", \"y\", \"z\"" + ], + ~c""" + defmodule Kernel.ErrorsTest.RecursiveVars do + def foo(x = y, y = z, z = x), do: {x, y, z} + end + """ + ) end - test :string_terminator do - assert_compile_fail TokenMissingError, - "nofile:1: missing terminator: \" (for string starting at line 1)", - '"bar' - end + test "function without definition" do + assert_compile_error( + ["nofile:2:7: ", "implementation not provided for predefined def foo/0"], + ~c""" + defmodule Kernel.ErrorsTest.FunctionWithoutDefinition do + def foo + end + """ + ) + + assert_compile_error( + ["nofile:10: ", "implementation not provided for predefined def example/2"], + ~c""" + defmodule Kernel.ErrorsTest.FunctionTemplate do + defmacro __using__(_) do + quote do + def example(foo, bar \\\\ []) + end + end + end - test :heredoc_start do - assert_compile_fail SyntaxError, - "nofile:1: heredoc start must be followed by a new line after \"\"\"", - '"""bar\n"""' + defmodule Kernel.ErrorsTest.UseFunctionTemplate do + use Kernel.ErrorsTest.FunctionTemplate + end + """ + ) end - test :heredoc_terminator do - assert_compile_fail TokenMissingError, - "nofile:2: missing terminator: \"\"\" (for heredoc starting at line 1)", - '"""\nbar' + test "guard without definition" do + assert_compile_error( + ["nofile:2:12: ", "implementation not provided for predefined defmacro foo/1"], + ~c""" + defmodule Kernel.ErrorsTest.GuardWithoutDefinition do + defguard foo(bar) + end + """ + ) end - test :unexpected_end do - assert_compile_fail SyntaxError, - "nofile:1: unexpected token: end", - '1 end' + test "literal on map and struct" do + assert_compile_error( + ["nofile:1:10", "expected key-value pairs in a map, got: put_in(foo.bar.baz, nil)"], + ~c"foo = 1; %{put_in(foo.bar.baz, nil), foo}" + ) end - test :syntax_error do - assert_compile_fail SyntaxError, - "nofile:1: syntax error before: '.'", - '+.foo' + test "struct fields on defstruct" do + assert_eval_raise ArgumentError, ["struct field names must be atoms, got: 1"], ~c""" + defmodule Kernel.ErrorsTest.StructFieldsOnDefstruct do + defstruct [1, 2, 3] + end + """ + end + + test "struct access on body" do + assert_compile_error( + [ + "nofile:3:3", + "cannot access struct Kernel.ErrorsTest.StructAccessOnBody, " <> + "the struct was not yet defined or the struct " <> + "is being accessed in the same context that defines it" + ], + ~c""" + defmodule Kernel.ErrorsTest.StructAccessOnBody do + defstruct %{name: "Brasilia"} + %Kernel.ErrorsTest.StructAccessOnBody{} + end + """ + ) end - test :compile_error_on_op_ambiguity do - msg = "nofile:1: \"a -1\" looks like a function call but there is a variable named \"a\", " <> - "please use explicit parenthesis or even spaces" - assert_compile_fail CompileError, msg, 'a = 1; a -1' + describe "struct errors" do + test "bad errors" do + assert_compile_error( + ["nofile:1:1", "BadStruct.__struct__/1 is undefined, cannot expand struct BadStruct"], + ~c"%BadStruct{}" + ) - max = 1 - assert max == 1 - assert (max 1, 2) == 2 - end + assert_compile_error( + ["nofile:1:1", "BadStruct.__struct__/1 is undefined, cannot expand struct BadStruct"], + ~c"%BadStruct{} = %{}" + ) - test :syntax_error_on_parens_call do - msg = "nofile:1: unexpected parenthesis. If you are making a function call, do not " <> - "insert spaces in between the function name and the opening parentheses. " <> - "Syntax error before: '('" + bad_struct_type_error = + ~r"expected Kernel.ErrorsTest.BadStructType.__struct__/(0|1) to return a map.*, got: :invalid" - assert_compile_fail SyntaxError, msg, 'foo (hello, world)' - assert_compile_fail SyntaxError, msg, 'foo ()' - assert_compile_fail SyntaxError, msg, 'foo (), 1' - end + defmodule BadStructType do + def __struct__, do: :invalid + def __struct__(_), do: :invalid + end - test :syntax_error_on_nested_no_parens_call do - msg = "nofile:1: unexpected comma. Parentheses are required to solve ambiguity in " <> - "nested calls. Syntax error before: ','" + assert_compile_error(bad_struct_type_error, ~c"%#{BadStructType}{}") - assert_compile_fail SyntaxError, msg, '[foo 1, 2]' - assert_compile_fail SyntaxError, msg, '[do: foo 1, 2]' - assert_compile_fail SyntaxError, msg, 'foo(do: bar 1, 2)' - assert_compile_fail SyntaxError, msg, '{foo 1, 2}' - assert_compile_fail SyntaxError, msg, 'foo 1, foo 2, 3' - assert_compile_fail SyntaxError, msg, 'foo(1, foo 2, 3)' + assert_raise ArgumentError, bad_struct_type_error, fn -> + struct(BadStructType) + end - assert is_list List.flatten [1] - assert is_list Enum.reverse [3, 2, 1], [4, 5, 6] - assert is_list(Enum.reverse [3, 2, 1], [4, 5, 6]) - end + assert_raise ArgumentError, bad_struct_type_error, fn -> + struct(BadStructType, foo: 1) + end + end - test :syntax_error_with_no_token do - assert_compile_fail TokenMissingError, - "nofile:1: missing terminator: ) (for \"(\" starting at line 1)", - 'case 1 (' - end + test "bad struct on module conflict" do + Code.put_compiler_option(:ignore_module_conflict, true) - test :clause_with_defaults do - assert_compile_fail CompileError, - "nofile:3: def hello/1 has default values and multiple clauses, " <> - "define a function head with the defaults", - ~C''' - defmodule ErrorsTest do - def hello(arg \\ 0), do: nil - def hello(arg \\ 1), do: nil + assert_compile_error(~r'MissingStructOnReload\.__struct__/1 is undefined', ~c''' + defmodule MissingStructOnReload do + defstruct [:title] + def d(), do: %MissingStructOnReload{} end - ''' - end - test :invalid_match_pattern do - assert_compile_fail CompileError, - "nofile:2: invalid expression in match", - ''' - case true do - true && true -> true + defmodule MissingStructOnReload do + def d(), do: %MissingStructOnReload{} + end + ''') + after + Code.put_compiler_option(:ignore_module_conflict, false) end - ''' - end - test :different_defs_with_defaults do - assert_compile_fail CompileError, - "nofile:3: def hello/3 defaults conflicts with def hello/2", - ~C''' - defmodule ErrorsTest do - def hello(a, b \\ nil), do: a + b - def hello(a, b \\ nil, c \\ nil), do: a + b + c + test "missing struct key" do + missing_struct_key_error = + ~r"expected Kernel.ErrorsTest.MissingStructKey.__struct__/(0|1) to return a map.*, got: %\{\}" + + defmodule MissingStructKey do + def __struct__, do: %{} + def __struct__(_), do: %{} end - ''' - assert_compile_fail CompileError, - "nofile:3: def hello/2 conflicts with defaults from def hello/3", - ~C''' - defmodule ErrorsTest do - def hello(a, b \\ nil, c \\ nil), do: a + b + c - def hello(a, b \\ nil), do: a + b + assert_compile_error(missing_struct_key_error, ~c"%#{MissingStructKey}{}") + + assert_raise ArgumentError, missing_struct_key_error, fn -> + struct(MissingStructKey) end - ''' - end - test :bad_form do - assert_compile_fail CompileError, - "nofile:2: function bar/0 undefined", - ''' - defmodule ErrorsTest do - def foo, do: bar + assert_raise ArgumentError, missing_struct_key_error, fn -> + struct(MissingStructKey, foo: 1) end - ''' - end - test :unbound_var do - assert_compile_fail CompileError, - "nofile:1: unbound variable ^x", - '^x = 1' - end + invalid_struct_key_error = + ~r"expected Kernel.ErrorsTest.InvalidStructKey.__struct__/(0|1) to return a map.*, got: %\{__struct__: 1\}" - test :unbound_not_match do - assert_compile_fail CompileError, - "nofile:1: cannot use ^x outside of match clauses", - '^x' - end + defmodule InvalidStructKey do + def __struct__, do: %{__struct__: 1} + def __struct__(_), do: %{__struct__: 1} + end - test :unbound_expr do - assert_compile_fail CompileError, - "nofile:1: invalid argument for unary operator ^, expected an existing variable, got: ^x(1)", - '^x(1) = 1' - end + assert_compile_error(invalid_struct_key_error, ~c"%#{InvalidStructKey}{}") - test :literal_on_map_and_struct do - assert_compile_fail SyntaxError, - "nofile:1: syntax error before: '}'", - '%{{:a, :b}}' + assert_raise ArgumentError, invalid_struct_key_error, fn -> + struct(InvalidStructKey) + end - assert_compile_fail SyntaxError, - "nofile:1: syntax error before: '{'", - '%{:a, :b}{a: :b}' - end + assert_raise ArgumentError, invalid_struct_key_error, fn -> + struct(InvalidStructKey, foo: 1) + end + end + + test "invalid struct" do + invalid_struct_name_error = + ~r"expected struct name returned by Kernel.ErrorsTest.InvalidStructName.__struct__/(0|1) to be Kernel.ErrorsTest.InvalidStructName, got: InvalidName" - test :struct_fields_on_defstruct do - assert_compile_fail ArgumentError, - "struct field names must be atoms, got: 1", - ''' - defmodule TZ do - defstruct [1, 2, 3] + defmodule InvalidStructName do + def __struct__, do: %{__struct__: InvalidName} + def __struct__(_), do: %{__struct__: InvalidName} end - ''' - end - test :struct_access_on_body do - assert_compile_fail CompileError, - "nofile:3: cannot access struct TZ in body of the module that defines it " <> - "as the struct fields are not yet accessible", - ''' - defmodule TZ do - defstruct %{name: "Brasilia"} - %TZ{} + assert_compile_error(invalid_struct_name_error, ~c"%#{InvalidStructName}{}") + + assert_raise ArgumentError, invalid_struct_name_error, fn -> + struct(InvalidStructName) end - ''' - end - test :unbound_map_key_var do - assert_compile_fail CompileError, - "nofile:1: illegal use of variable x in map key", - '%{x => 1} = %{}' + assert_raise ArgumentError, invalid_struct_name_error, fn -> + struct(InvalidStructName, foo: 1) + end + end - assert_compile_fail CompileError, - "nofile:1: illegal use of variable x in map key", - '%{x = 1 => 1}' - end + test "good struct" do + defmodule GoodStruct do + defstruct name: "john" + end + + assert_eval_raise KeyError, + ["key :age not found"], + ~c"%#{GoodStruct}{age: 27}" - test :struct_errors do - assert_compile_fail CompileError, - "nofile:1: BadStruct.__struct__/0 is undefined, cannot expand struct BadStruct", - '%BadStruct{}' + assert_compile_error( + ["nofile:1:1", "unknown key :age for struct Kernel.ErrorsTest.GoodStruct"], + ~c"%#{GoodStruct}{age: 27} = %{}" + ) + end - defmodule BadStruct do - def __struct__ do - [] + test "enforce @enforce_keys" do + defmodule EnforceKeys do + @enforce_keys [:foo] + defstruct(foo: nil) end + + assert_raise ArgumentError, + "@enforce_keys required keys ([:fo, :bar]) that are not defined in defstruct: [foo: nil]", + fn -> + defmodule EnforceKeysError do + @enforce_keys [:foo, :fo, :bar] + defstruct(foo: nil) + end + end end + end + + test "invalid unquote" do + assert_compile_error(["nofile:1:1", "unquote called outside quote"], ~c"unquote 1") + end - assert_compile_fail CompileError, - "nofile:1: expected Kernel.ErrorsTest.BadStruct.__struct__/0 to return a map, got: []", - '%#{BadStruct}{}' + test "invalid unquote splicing in one-liners" do + assert_eval_raise ArgumentError, + [ + "unquote_splicing only works inside arguments and block contexts, " <> + "wrap it in parens if you want it to work with one-liners" + ], + ~c""" + defmodule Kernel.ErrorsTest.InvalidUnquoteSplicingInOneliners do + defmacro oneliner2 do + quote do: unquote_splicing 1 + end - defmodule GoodStruct do - def __struct__ do - %{name: "josé"} + def callme do + oneliner2 + end + end + """ + end + + test "invalid attribute" do + msg = ~r"cannot inject attribute @foo into function/macro because cannot escape " + + assert_raise ArgumentError, msg, fn -> + defmodule InvalidAttribute do + @foo fn -> nil end + def bar, do: @foo end end - - assert_compile_fail CompileError, - "nofile:1: unknown key :age for struct Kernel.ErrorsTest.GoodStruct", - '%#{GoodStruct}{age: 27}' end - test :name_for_defmodule do - assert_compile_fail CompileError, - "nofile:1: invalid module name: 3", - 'defmodule 1 + 2, do: 3' + test "typespec attributes set via Module.put_attribute/4" do + message = + "attributes type, typep, opaque, spec, callback, and macrocallback " <> + "must be set directly via the @ notation" + + for kind <- [:type, :typep, :opaque, :spec, :callback, :macrocallback] do + assert_eval_raise ArgumentError, + [message], + """ + defmodule PutTypespecAttribute do + Module.put_attribute(__MODULE__, #{inspect(kind)}, {}) + end + """ + end end - test :invalid_unquote do - assert_compile_fail CompileError, - "nofile:1: unquote called outside quote", - 'unquote 1' + test "invalid struct field value" do + msg = ~r"invalid default value for struct field baz, cannot escape " + + assert_raise ArgumentError, msg, fn -> + defmodule InvalidStructFieldValue do + defstruct baz: fn -> nil end + end + end end - test :invalid_quote_args do - assert_compile_fail CompileError, - "nofile:1: invalid arguments for quote", - 'quote 1' + test "invalid case clauses" do + assert_compile_error( + ["nofile:1:37", "expected one argument for \"do\" clauses (->) in \"case\""], + ~c"case nil do 0, z when not is_nil(z) -> z end" + ) end - test :invalid_calls do - assert_compile_fail CompileError, - "nofile:1: invalid call foo(1)(2)", - 'foo(1)(2)' + test "invalid fn args" do + exception = + assert_eval_raise TokenMissingError, + [ + "nofile:1:5:", + ~r/missing terminator: end/ + ], + ~c"fn 1" - assert_compile_fail CompileError, - "nofile:1: invalid call 1.foo()", - '1.foo' + assert exception.opening_delimiter == :fn end - test :unhandled_stab do - assert_compile_fail CompileError, - "nofile:3: unhandled operator ->", - ''' - defmodule Mod do - def fun do - casea foo, do: (bar -> baz) - end - end - ''' + test "invalid escape" do + assert_eval_raise TokenMissingError, + ["nofile:1:3:", "invalid escape \\ at end of file"], + ~c"1 \\" end - test :undefined_non_local_function do - assert_compile_fail CompileError, - "nofile:1: undefined function casea/2", - 'casea foo, do: 1' + test "show snippet on missing tokens" do + assert_eval_raise TokenMissingError, + [ + "nofile:1:25:", + "missing terminator: end", + "defmodule ShowSnippet do\n", + "└ unclosed delimiter" + ], + ~c"defmodule ShowSnippet do" end - test :invalid_fn_args do - assert_compile_fail TokenMissingError, - "nofile:1: missing terminator: end (for \"fn\" starting at line 1)", - 'fn 1' + test "don't show snippet when error line is empty" do + assert_eval_raise TokenMissingError, + ["nofile:1:25:", "missing terminator: end"], + ~c"defmodule ShowSnippet do\n\n" end - test :function_local_conflict do - assert_compile_fail CompileError, - "nofile:1: imported Kernel.&&/2 conflicts with local function", - ''' - defmodule ErrorsTest do + test "function local conflict" do + assert_compile_error( + ["nofile:3:9: ", "imported Kernel.&&/2 conflicts with local function"], + ~c""" + defmodule Kernel.ErrorsTest.FunctionLocalConflict do def other, do: 1 && 2 def _ && _, do: :error end - ''' - end - - test :macro_local_conflict do - assert_compile_fail CompileError, - "nofile:6: call to local macro &&/2 conflicts with imported Kernel.&&/2, " <> - "please rename the local macro or remove the conflicting import", - ''' - defmodule ErrorsTest do + """ + ) + end + + test "macro local conflict" do + assert_compile_error( + [ + "nofile:6:20", + "call to local macro &&/2 conflicts with imported Kernel.&&/2, " <> + "please rename the local macro or remove the conflicting import" + ], + ~c""" + defmodule Kernel.ErrorsTest.MacroLocalConflict do def hello, do: 1 || 2 defmacro _ || _, do: :ok defmacro _ && _, do: :error def world, do: 1 && 2 end - ''' - end - - test :macro_with_undefined_local do - assert_compile_fail UndefinedFunctionError, - "undefined function: ErrorsTest.unknown/1", - ''' - defmodule ErrorsTest do - defmacrop bar, do: unknown(1) - def baz, do: bar() + """ + ) + end + + test "macro with undefined local" do + assert_eval_raise UndefinedFunctionError, + [ + "function Kernel.ErrorsTest.MacroWithUndefinedLocal.unknown/1 is undefined (function not available)" + ], + ~c""" + defmodule Kernel.ErrorsTest.MacroWithUndefinedLocal do + defmacrop bar, do: unknown(1) + def baz, do: bar() + end + """ + end + + test "private macro" do + assert_eval_raise UndefinedFunctionError, + [ + "function Kernel.ErrorsTest.PrivateMacro.foo/0 is undefined (function not available)" + ], + ~c""" + defmodule Kernel.ErrorsTest.PrivateMacro do + defmacrop foo, do: 1 + defmacro bar, do: __MODULE__.foo() + defmacro baz, do: bar() + end + """ + end + + test "macro invoked before its definition" do + assert_compile_error( + ["nofile:2:16", "cannot invoke macro bar/0 before its definition"], + ~c""" + defmodule Kernel.ErrorsTest.IncorrectMacroDispatch do + def foo, do: bar() + defmacro bar, do: :bar end - ''' - end - - test :private_macro do - assert_compile_fail UndefinedFunctionError, - "undefined function: ErrorsTest.foo/0", - ''' - defmodule ErrorsTest do - defmacrop foo, do: 1 - defmacro bar, do: __MODULE__.foo - defmacro baz, do: bar + """ + ) + + assert_compile_error( + ["nofile:2:16", "cannot invoke macro bar/0 before its definition"], + ~c""" + defmodule Kernel.ErrorsTest.IncorrectMacropDispatch do + def foo, do: bar() + defmacrop bar, do: :ok end - ''' - end + """ + ) + + assert_compile_error( + ["nofile:2:40", "cannot invoke macro bar/1 before its definition"], + ~c""" + defmodule Kernel.ErrorsTest.IncorrectMacroDispatch do + defmacro bar(a) when is_atom(a), do: bar([a]) + end + """ + ) + end + + test "macro captured before its definition" do + assert_compile_error( + ["nofile:3:18", "cannot invoke macro is_ok/1 before its definition"], + ~c""" + defmodule Kernel.ErrorsTest.IncorrectMacroDispatch.Capture do + def foo do + predicate = &is_ok/1 + Enum.any?([:ok, :error, :foo], predicate) + end - test :function_definition_with_alias do - assert_compile_fail CompileError, - "nofile:2: function names should start with lowercase characters or underscore, invalid name Bar", - ''' - defmodule ErrorsTest do + defmacro is_ok(atom), do: atom == :ok + end + """ + ) + end + + test "function definition with alias" do + assert_compile_error( + [ + "nofile:2:7\n", + "function names should start with lowercase characters or underscore, invalid name Bar" + ], + ~c""" + defmodule Kernel.ErrorsTest.FunctionDefinitionWithAlias do def Bar do :baz end end - ''' + """ + ) end - test :function_import_conflict do - assert_compile_fail CompileError, - "nofile:3: function exit/1 imported from both :erlang and Kernel, call is ambiguous", - ''' - defmodule ErrorsTest do - import :erlang, warn: false + test "function import conflict" do + assert_compile_error( + ["nofile:3:16", "function exit/1 imported from both :erlang and Kernel, call is ambiguous"], + ~c""" + defmodule Kernel.ErrorsTest.FunctionImportConflict do + import :erlang, only: [exit: 1], warn: false def foo, do: exit(:test) end - ''' - end - - test :import_invalid_macro do - assert_compile_fail CompileError, - "nofile:1: cannot import Kernel.invalid/1 because it doesn't exist", - 'import Kernel, only: [invalid: 1]' - end - - test :unrequired_macro do - assert_compile_fail SyntaxError, - "nofile:2: you must require Kernel.ErrorsTest.UnproperMacro before invoking " <> - "the macro Kernel.ErrorsTest.UnproperMacro.unproper/1 " - ''' - defmodule ErrorsTest do - Kernel.ErrorsTest.UnproperMacro.unproper([]) + """ + ) + + assert_compile_error( + ["nofile:3:17", "function exit/1 imported from both :erlang and Kernel, call is ambiguous"], + ~c""" + defmodule Kernel.ErrorsTest.FunctionImportConflict do + import :erlang, only: [exit: 1], warn: false + def foo, do: &exit/1 + end + """ + ) + end + + test "ensure valid import :only option" do + assert_compile_error( + [ + "nofile:3:3", + "invalid :only option for import, expected value to be an atom :functions, :macros, or a literal keyword list of function names with arity as values, got: x" + ], + ~c""" + defmodule Kernel.ErrorsTest.Only do + x = [flatten: 1] + import List, only: x end - ''' + """ + ) + end + + test "ensure valid import :except option" do + assert_compile_error( + [ + "nofile:3:3", + "invalid :except option for import, expected value to be a literal keyword list of function names " <> + "with arity as values, got: Module.__get_attribute__(Kernel.ErrorsTest.Only, :x, 3, true)" + ], + ~c""" + defmodule Kernel.ErrorsTest.Only do + @x [flatten: 1] + import List, except: @x + end + """ + ) end - test :def_defmacro_clause_change do - assert_compile_fail CompileError, - "nofile:3: defmacro foo/1 already defined as def", - ''' - defmodule ErrorsTest do + test "def defmacro clause change" do + assert_compile_error( + ["nofile:3:12\n", "defmacro foo/1 already defined as def in nofile:2"], + ~c""" + defmodule Kernel.ErrorsTest.DefDefmacroClauseChange do def foo(1), do: 1 defmacro foo(x), do: x end - ''' + """ + ) end - test :internal_function_overridden do - assert_compile_fail CompileError, - "nofile:1: function __info__/1 is internal and should not be overridden", - ''' - defmodule ErrorsTest do - def __info__(_), do: [] - end - ''' + test "def defp clause change from another file" do + assert_compile_error(["nofile:4\n", "def hello/0 already defined as defp"], ~c""" + defmodule Kernel.ErrorsTest.DefDefmacroClauseChange do + require Kernel.ErrorsTest + defp hello, do: :world + Kernel.ErrorsTest.hello() + end + """) end - test :no_macros do - assert_compile_fail CompileError, - "nofile:2: could not load macros from module :lists", - ''' - defmodule ErrorsTest do - import :lists, only: :macros + test "internal function overridden" do + assert_compile_error( + ["nofile:2:7\n", "cannot define def __info__/1 as it is automatically defined by Elixir"], + ~c""" + defmodule Kernel.ErrorsTest.InternalFunctionOverridden do + def __info__(_), do: [] end - ''' + """ + ) end - test :invalid_macro do - assert_compile_fail CompileError, - "nofile: invalid quoted expression: {:foo, :bar, :baz, :bat}", - ''' - defmodule ErrorsTest do + test "invalid macro" do + assert_compile_error( + "invalid quoted expression: {:foo, :bar, :baz, :bat}", + ~c""" + defmodule Kernel.ErrorsTest.InvalidMacro do defmacrop oops do {:foo, :bar, :baz, :bat} end - def test, do: oops + def test, do: oops() end - ''' + """ + ) end - test :unloaded_module do - assert_compile_fail CompileError, - "nofile:1: module Certainly.Doesnt.Exist is not loaded and could not be found", - 'import Certainly.Doesnt.Exist' + test "unloaded module" do + assert_compile_error( + ["nofile:1:1", "module Certainly.Doesnt.Exist is not loaded and could not be found"], + ~c"import Certainly.Doesnt.Exist" + ) end - test :scheduled_module do - assert_compile_fail CompileError, - "nofile:4: module ErrorsTest.Hygiene is not loaded but was defined. " <> - "This happens because you are trying to use a module in the same context it is defined. " <> - "Try defining the module outside the context that requires it.", - ''' - defmodule ErrorsTest do + test "module imported from the context it was defined in" do + assert_compile_error( + [ + "nofile:4:3", + "module Kernel.ErrorsTest.ScheduledModule.Hygiene is not loaded but was defined." + ], + ~c""" + defmodule Kernel.ErrorsTest.ScheduledModule do + defmodule Hygiene do + end + import Kernel.ErrorsTest.ScheduledModule.Hygiene + end + """ + ) + end + + test "module imported from the same module" do + assert_compile_error( + [ + "nofile:3:5", + "you are trying to use/import/require the module Kernel.ErrorsTest.ScheduledModule.Hygiene which is currently being defined" + ], + ~c""" + defmodule Kernel.ErrorsTest.ScheduledModule do defmodule Hygiene do + import Kernel.ErrorsTest.ScheduledModule.Hygiene end - import ErrorsTest.Hygiene end - ''' + """ + ) end - test :already_compiled_module do - assert_compile_fail ArgumentError, - "could not call eval_quoted on module Record " <> - "because it was already compiled", - 'Module.eval_quoted Record, quote(do: 1), [], file: __ENV__.file' - end + test "invalid @compile inline" do + assert_compile_error( + ["nofile:1: ", "undefined function foo/1 given to @compile :inline"], + ~c"defmodule Test do @compile {:inline, foo: 1} end" + ) - test :interpolation_error do - assert_compile_fail SyntaxError, - "nofile:1: \"do\" starting at line 1 is missing terminator \"end\". Unexpected token: )", - '"foo\#{case 1 do )}bar"' + assert_compile_error( + ["nofile:1: ", "macro foo/1 given to @compile :inline"], + ~c"defmodule Test do @compile {:inline, foo: 1}; defmacro foo(_), do: :ok end" + ) end - test :in_definition_module do - assert_compile_fail CompileError, - "nofile:1: cannot define module ErrorsTest because it is currently being defined in nofile:1", - 'defmodule ErrorsTest, do: (defmodule Elixir.ErrorsTest, do: true)' - end + test "invalid @nifs attribute" do + assert_compile_error( + ["nofile:1: ", "undefined function foo/1 given to @nifs"], + ~c"defmodule Test do @nifs [foo: 1] end" + ) - test :invalid_definition do - assert_compile_fail CompileError, - "nofile:1: invalid syntax in def 1.(hello)", - 'defmodule ErrorsTest, do: (def 1.(hello), do: true)' - end + assert_compile_error( + ["nofile:1: ", "undefined function foo/1 given to @nifs"], + ~c"defmodule Test do @nifs [foo: 1]; defmacro foo(_) end" + ) - test :duplicated_bitstring_size do - assert_compile_fail CompileError, - "nofile:1: duplicated size definition in bitstring", - '<<1 :: [size(12), size(13)]>>' + assert_eval_raise ArgumentError, + ["@nifs is a built-in module attribute"], + ~c"defmodule Test do @nifs :not_an_option end" end - test :invalid_bitstring_specified do - assert_compile_fail CompileError, - "nofile:1: unknown bitstring specifier :atom", - '<<1 :: :atom>>' - - assert_compile_fail CompileError, - "nofile:1: unknown bitstring specifier unknown()", - '<<1 :: unknown>>' + test "invalid @dialyzer options" do + assert_compile_error( + ["nofile:1: ", "undefined function foo/1 given to @dialyzer :nowarn_function"], + ~c"defmodule Test do @dialyzer {:nowarn_function, {:foo, 1}} end" + ) - assert_compile_fail CompileError, - "nofile:1: unknown bitstring specifier another(12)", - '<<1 :: another(12)>>' + assert_compile_error( + ["nofile:1: ", "macro foo/1 given to @dialyzer :nowarn_function"], + ~c"defmodule Test do @dialyzer {:nowarn_function, {:foo, 1}}; defmacro foo(_), do: :ok end" + ) - assert_compile_fail CompileError, - "nofile:1: size in bitstring expects an integer or a variable as argument, got: :a", - '<<1 :: size(:a)>>' + assert_compile_error( + ["nofile:1: ", "undefined function foo/1 given to @dialyzer :no_opaque"], + ~c"defmodule Test do @dialyzer {:no_opaque, {:foo, 1}} end" + ) - assert_compile_fail CompileError, - "nofile:1: unit in bitstring expects an integer as argument, got: :x", - '<<1 :: unit(:x)>>' + assert_eval_raise ArgumentError, + ["invalid value for @dialyzer attribute: :not_an_option"], + ~c"defmodule Test do @dialyzer :not_an_option end" end - test :invalid_var! do - assert_compile_fail CompileError, - "nofile:1: expected var x to expand to an existing variable or be a part of a match", - 'var!(x)' - end - - test :invalid_alias do - assert_compile_fail CompileError, - "nofile:1: invalid value for keyword :as, expected an alias, got nested alias: Sample.Lists", - 'alias :lists, as: Sample.Lists' - end - - test :invalid_import_option do - assert_compile_fail CompileError, - "nofile:1: unsupported option :ops given to import", - 'import :lists, [ops: 1]' - end - - test :invalid_rescue_clause do - assert_compile_fail CompileError, - "nofile:4: invalid rescue clause. The clause should match on an alias, a variable or be in the `var in [alias]` format", - 'try do\n1\nrescue\n%UndefinedFunctionError{arity: 1} -> false\nend' + test "@on_load attribute format" do + assert_raise ArgumentError, ~r/should be an atom or an {atom, 0} tuple/, fn -> + defmodule BadOnLoadAttribute do + Module.put_attribute(__MODULE__, :on_load, "not an atom") + end + end end - test :invalid_for_without_generators do - assert_compile_fail CompileError, - "nofile:1: for comprehensions must start with a generator", - 'for x, do: x' + test "duplicated @on_load attribute" do + assert_raise ArgumentError, "the @on_load attribute can only be set once per module", fn -> + defmodule DuplicatedOnLoadAttribute do + @on_load :foo + @on_load :bar + end + end end - test :invalid_for_bit_generator do - assert_compile_fail CompileError, - "nofile:1: bitstring fields without size are not allowed in bitstring generators", - 'for << x :: binary <- "123" >>, do: x' + test "@on_load attribute with undefined function" do + assert_compile_error( + ["nofile:1: ", "undefined function foo/0 given to @on_load"], + ~c"defmodule UndefinedOnLoadFunction do @on_load :foo end" + ) end - test :unbound_cond do - assert_compile_fail CompileError, - "nofile:1: unbound variable _ inside cond. If you want the last clause to always match, " <> - "you probably meant to use: true ->", - 'cond do _ -> true end' + test "wrong kind for @on_load attribute" do + assert_compile_error( + ["nofile:1: ", "macro foo/0 given to @on_load"], + ~c""" + defmodule PrivateOnLoadFunction do + @on_load :foo + defmacro foo, do: :ok + end + """ + ) + end + + test "in definition module" do + assert_compile_error( + [ + "nofile:2: ", + "cannot define module Kernel.ErrorsTest.InDefinitionModule " <> + "because it is currently being defined in nofile:1" + ], + ~c""" + defmodule Kernel.ErrorsTest.InDefinitionModule do + defmodule Elixir.Kernel.ErrorsTest.InDefinitionModule, do: true + end + """ + ) end - test :fun_different_arities do - assert_compile_fail CompileError, - "nofile:1: cannot mix clauses with different arities in function definition", - 'fn x -> x; x, y -> x + y end' + test "invalid definition" do + assert_compile_error( + ["nofile:1: ", "invalid syntax in def 1.(hello)"], + ~c"defmodule Kernel.ErrorsTest.InvalidDefinition, do: (def 1.(hello), do: true)" + ) end - test :new_line_error do - assert_compile_fail SyntaxError, - "nofile:3: syntax error before: newline", - 'if true do\n foo = [],\n baz\nend' - end + test "function head with guard" do + assert_compile_error(["nofile:2:7: ", "missing :do option in \"def\""], ~c""" + defmodule Kernel.ErrorsTest.BodyessFunctionWithGuard do + def foo(n) when is_number(n) + end + """) - test :invalid_var_or_function_on_guard do - assert_compile_fail CompileError, - "nofile:2: unknown variable something_that_does_not_exist or " <> - "cannot invoke function something_that_does_not_exist/0 inside guard", - ''' - case [] do - [] when something_that_does_not_exist == [] -> :ok + assert_compile_error(["nofile:2:7: ", "missing :do option in \"def\""], ~c""" + defmodule Kernel.ErrorsTest.BodyessFunctionWithGuard do + def foo(n) when is_number(n), true + end + """) + end + + test "invalid args for function head" do + assert_compile_error( + [ + "nofile:2:7: ", + "patterns are not allowed in function head, only variables and default arguments (using \\\\)" + ], + ~c""" + defmodule Kernel.ErrorsTest.InvalidArgsForBodylessClause do + def foo(nil) + def foo(_), do: :ok end - ''' + """ + ) + end + + test "bad multi-call" do + assert_compile_error( + [ + "nofile:1:1", + "invalid argument for alias, expected a compile time atom or alias, got: 42" + ], + ~c"alias IO.{ANSI, 42}" + ) + + assert_compile_error( + ["nofile:1:1", ":as option is not supported by multi-alias call"], + ~c"alias Elixir.{Map}, as: Dict" + ) + + assert_eval_raise UndefinedFunctionError, + ["function List.\"{}\"/1 is undefined or private"], + ~c"[List.{Chars}, \"one\"]" + end + + test "macros error stacktrace" do + assert [ + {:erlang, :+, [1, :foo], _}, + {Kernel.ErrorsTest.MacrosErrorStacktrace, :sample, 1, _} | _ + ] = + rescue_stacktrace(""" + defmodule Kernel.ErrorsTest.MacrosErrorStacktrace do + defmacro sample(num), do: num + :foo + def other, do: sample(1) + end + """) + end + + test "macros function clause stacktrace" do + assert [{__MODULE__, :sample, 1, _} | _] = + rescue_stacktrace(""" + defmodule Kernel.ErrorsTest.MacrosFunctionClauseStacktrace do + import Kernel.ErrorsTest + sample(1) + end + """) + end + + test "macros interpreted function clause stacktrace" do + assert [{Kernel.ErrorsTest.MacrosInterpretedFunctionClauseStacktrace, :sample, 1, _} | _] = + rescue_stacktrace(""" + defmodule Kernel.ErrorsTest.MacrosInterpretedFunctionClauseStacktrace do + defmacro sample(0), do: 0 + def other, do: sample(1) + end + """) + end + + test "macros compiled callback" do + assert [{Kernel.ErrorsTest, :__before_compile__, [env], _} | _] = + rescue_stacktrace(""" + defmodule Kernel.ErrorsTest.MacrosCompiledCallback do + Module.put_attribute(__MODULE__, :before_compile, Kernel.ErrorsTest) + end + """) + + assert %Macro.Env{module: Kernel.ErrorsTest.MacrosCompiledCallback} = env + end + + test "failed remote call stacktrace includes file/line info" do + try do + bad_remote_call(Process.get(:unused, 1)) + rescue + ArgumentError -> + assert [ + {:erlang, :apply, [1, :foo, []], _}, + {__MODULE__, :bad_remote_call, 1, [file: _, line: _]} | _ + ] = __STACKTRACE__ + end end - test :bodyless_function_with_guard do - assert_compile_fail CompileError, - "nofile:2: missing do keyword in def", - ''' - defmodule ErrorsTest do - def foo(n) when is_number(n) + test "def fails when rescue, else or catch don't have clauses" do + assert_compile_error( + ~r"invalid \"rescue\" block in \"def\", it expects \"pattern -> expr\" clauses", + """ + defmodule Example do + def foo do + bar() + rescue + baz() + end end - ''' + """ + ) end - test :invalid_args_for_bodyless_clause do - assert_compile_fail CompileError, - "nofile:2: can use only variables and \\\\ as arguments of bodyless clause", - ''' - defmodule ErrorsTest do - def foo(arg // nil) - def foo(_), do: :ok - end - ''' - end + test "duplicate map keys" do + assert_compile_error(["nofile:1:3", "key :a will be overridden in map"], """ + %{a: :b, a: :c} = %{a: :c} + """) - test :invalid_function_on_match do - assert_compile_fail CompileError, - "nofile:1: cannot invoke function something_that_does_not_exist/0 inside match", - 'case [] do; something_that_does_not_exist() -> :ok; end' + assert_compile_error(["nofile:1:3", "key :a will be overridden in map"], """ + %{a: :b, a: :c, a: :d} = %{a: :c} + """) end - test :invalid_remote_on_match do - assert_compile_fail CompileError, - "nofile:1: cannot invoke remote function Hello.something_that_does_not_exist/0 inside match", - 'case [] do; Hello.something_that_does_not_exist() -> :ok; end' - end + test "| outside of cons" do + assert_compile_error(["nofile:1:3", "misplaced operator |/2"], "1 | 2") - test :invalid_remote_on_guard do - assert_compile_fail CompileError, - "nofile:1: cannot invoke remote function Hello.something_that_does_not_exist/0 inside guard", - 'case [] do; [] when Hello.something_that_does_not_exist == [] -> :ok; end' + assert_compile_error( + ["nofile:1:45", "misplaced operator |/2"], + "defmodule MisplacedOperator, do: (def bar(1 | 2), do: :ok)" + ) end - test :typespec_errors do - assert_compile_fail CompileError, - "nofile:2: type foo() undefined", - ''' - defmodule ErrorsTest do - @type omg :: foo - end - ''' + defp bad_remote_call(x), do: x.foo() - assert_compile_fail CompileError, - "nofile:2: spec for undefined function ErrorsTest.omg/0", - ''' - defmodule ErrorsTest do - @spec omg :: atom - end - ''' - end + defmacro sample(0), do: 0 - test :bad_unquoting do - assert_compile_fail CompileError, - "nofile: invalid quoted expression: {:foo, 0, 1}", - ''' - defmodule ErrorsTest do - def range(unquote({:foo, 0, 1})), do: :ok - end - ''' + defmacro before_compile(_) do + quote(do: _) end - test :macros_error_stacktrace do - assert [{:erlang, :+, [1, :foo], _}, {ErrorsTest, :sample, 1, _}|_] = - rescue_stacktrace(""" - defmodule ErrorsTest do - defmacro sample(num), do: num + :foo - def other, do: sample(1) - end - """) - end + ## Helpers - test :macros_function_clause_stacktrace do - assert [{__MODULE__, :sample, 1, _}|_] = - rescue_stacktrace(""" - defmodule ErrorsTest do - import Kernel.ErrorsTest - sample(1) + defp assert_eval_raise(given_exception, messages, source) do + exception = + assert_raise given_exception, fn -> + Code.eval_string(source) end - """) - end - test :macros_interpreted_function_clause_stacktrace do - assert [{ErrorsTest, :sample, 1, _}|_] = - rescue_stacktrace(""" - defmodule ErrorsTest do - defmacro sample(0), do: 0 - def other, do: sample(1) - end - """) - end + error_msg = Exception.format(:error, exception, []) - test :macros_compiled_callback do - assert [{Kernel.ErrorsTest, :__before_compile__, [%Macro.Env{module: ErrorsTest}], _}|_] = - rescue_stacktrace(""" - defmodule ErrorsTest do - Module.put_attribute(__MODULE__, :before_compile, Kernel.ErrorsTest) - end - """) + for msg <- messages do + assert error_msg =~ msg + end + + exception end - defmacro sample(0), do: 0 + defp assert_compile_error(messages, string) do + captured = + ExUnit.CaptureIO.capture_io(:stderr, fn -> + ast = Code.string_to_quoted!(string, columns: true) + assert_raise CompileError, fn -> Code.eval_quoted(ast) end + end) - defmacro before_compile(_) do - quote(do: _) + for message <- List.wrap(messages) do + assert captured =~ message + end end - ## Helpers - - defp rescue_stacktrace(expr) do - result = try do - :elixir.eval(to_char_list(expr), []) + defp rescue_stacktrace(string) do + try do + Code.eval_string(string) nil rescue - _ -> System.stacktrace + _ -> __STACKTRACE__ + else + _ -> flunk("Expected expression to fail") end - - result || raise(ExUnit.AssertionError, message: "Expected function given to rescue_stacktrace to fail") end end diff --git a/lib/elixir/test/elixir/kernel/expansion_test.exs b/lib/elixir/test/elixir/kernel/expansion_test.exs index ca6048bd386..d1b2af82280 100644 --- a/lib/elixir/test/elixir/kernel/expansion_test.exs +++ b/lib/elixir/test/elixir/kernel/expansion_test.exs @@ -1,474 +1,3091 @@ -Code.require_file "../test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) defmodule Kernel.ExpansionTarget do defmacro seventeen, do: 17 + defmacro bar, do: "bar" + + defmacro message_hello(arg) do + send(self(), :hello) + arg + end end defmodule Kernel.ExpansionTest do use ExUnit.Case, async: true - ## __block__ + import ExUnit.CaptureIO - test "__block__: expands to nil when empty" do - assert expand(quote do: __block__()) == nil - end + describe "__block__" do + test "expands to nil when empty" do + assert expand(quote(do: unquote(:__block__)())) == nil + end - test "__block__: expands to argument when arity is 1" do - assert expand(quote do: __block__(1)) == 1 - end + test "expands to argument when arity is 1" do + assert expand(quote(do: unquote(:__block__)(1))) == 1 + end - test "__block__: is recursive to argument when arity is 1" do - assert expand(quote do: __block__(1, __block__(2))) == quote do: __block__(1, 2) - end + test "is recursive to argument when arity is 1" do + expanded = + quote do + _ = 1 + 2 + end - test "__block__: accumulates vars" do - assert expand(quote(do: (a = 1; a))) == quote do: (a = 1; a) + assert expand(quote(do: unquote(:__block__)(_ = 1, unquote(:__block__)(2)))) == expanded + end + + test "accumulates vars" do + before_expansion = + quote do + a = 1 + a + end + + after_expansion = + quote do + a = 1 + a + end + + assert expand(before_expansion) == after_expansion + end end - ## alias + describe "alias" do + test "expand args, defines alias and returns itself" do + alias true, as: True - test "alias: expand args, defines alias and returns itself" do - alias true, as: True + input = quote(do: alias(:hello, as: World, warn: True)) + {output, env} = expand_env(input, __ENV__) - input = quote do: (alias :hello, as: World, warn: True) - {output, env} = expand_env(input, __ENV__) + assert output == :hello + assert env.aliases == [{:"Elixir.True", true}, {:"Elixir.World", :hello}] + end - assert output == quote do: (alias :hello, as: :"Elixir.World", warn: true) - assert env.aliases == [{:"Elixir.True", true}, {:"Elixir.World", :hello}] - end + test "invalid alias" do + message = + ~r"invalid value for option :as, expected a simple alias, got nested alias: Sample.Lists" - ## __aliases__ + assert_compile_error(message, fn -> + expand(quote(do: alias(:lists, as: Sample.Lists))) + end) - test "__aliases__: expands even if no alias" do - assert expand(quote do: World) == :"Elixir.World" - assert expand(quote do: Elixir.World) == :"Elixir.World" - end + message = ~r"invalid argument for alias, expected a compile time atom or alias, got: 1 \+ 2" - test "__aliases__: expands with alias" do - alias Hello, as: World - assert expand_env(quote(do: World), __ENV__) |> elem(0) == :"Elixir.Hello" - end + assert_compile_error(message, fn -> + expand(quote(do: alias(1 + 2))) + end) - test "__aliases__: expands with alias is recursive" do - alias Source, as: Hello - alias Hello, as: World - assert expand_env(quote(do: World), __ENV__) |> elem(0) == :"Elixir.Source" - end + message = ~r"invalid value for option :as, expected an alias, got: :foobar" - test "__aliases__: expands to elixir_aliases on runtime" do - assert expand(quote do: hello.World) == - quote do: :elixir_aliases.concat([hello(), :World]) - end + assert_compile_error(message, fn -> + expand(quote(do: alias(:lists, as: :foobar))) + end) - ## = + message = ~r"invalid value for option :as, expected an alias, got: :\"Elixir.foobar\"" - test "=: sets context to match" do - assert expand(quote do: __ENV__.context = :match) == quote do: :match = :match - end + assert_compile_error(message, fn -> + expand(quote(do: alias(:lists, as: :"Elixir.foobar"))) + end) - test "=: defines vars" do - {output, env} = expand_env(quote(do: a = 1), __ENV__) - assert output == quote(do: a = 1) - assert {:a, __MODULE__} in env.vars - end + message = + ~r"alias cannot be inferred automatically for module: :lists, please use the :as option" - test "=: does not carry rhs imports" do - assert expand(quote(do: flatten([1,2,3]) = import List)) == - quote(do: flatten([1,2,3]) = import :"Elixir.List", []) - end + assert_compile_error(message, fn -> + expand(quote(do: alias(:lists))) + end) + end - test "=: does not define _" do - {output, env} = expand_env(quote(do: _ = 1), __ENV__) - assert output == quote(do: _ = 1) - assert env.vars == [] - end + test "invalid expansion" do + assert_compile_error(~r"invalid alias: \"foo\.Foo\"", fn -> + code = + quote do + foo = :foo + foo.Foo + end - ## Pseudo vars + expand(code) + end) + end - test "__MODULE__" do - assert expand(quote do: __MODULE__) == __MODULE__ - end + test "raises if :as is passed to multi-alias aliases" do + assert_compile_error(~r":as option is not supported by multi-alias call", fn -> + expand(quote(do: alias(Foo.{Bar, Baz}, as: BarBaz))) + end) + end - test "__DIR__" do - assert expand(quote do: __DIR__) == __DIR__ - end + test "raises on multi-alias with non-atom base" do + assert_compile_error(~r"invalid alias: \"foo\"", fn -> + expand(quote(do: alias(foo.{Bar, Baz}))) + end) + end - test "__CALLER__" do - assert expand(quote do: __CALLER__) == quote do: __CALLER__ + test "invalid options" do + assert_compile_error(~r"unsupported option :ops given to alias", fn -> + expand(quote(do: alias(Foo, ops: 1))) + end) + end end - test "__ENV__" do - env = %{__ENV__ | line: 0} - assert expand_env(quote(do: __ENV__), env) == - {{:%{}, [], Map.to_list(env)}, env} - end + describe "__aliases__" do + test "expands even if no alias" do + assert expand(quote(do: World)) == :"Elixir.World" + assert expand(quote(do: Elixir.World)) == :"Elixir.World" + end - test "__ENV__.accessor" do - env = %{__ENV__ | line: 0} - assert expand_env(quote(do: __ENV__.file), env) == {__ENV__.file, env} - assert expand_env(quote(do: __ENV__.unknown), env) == - {quote(do: unquote({:%{}, [], Map.to_list(env)}).unknown), env} + test "expands with alias" do + alias Hello, as: World + assert expand_env(quote(do: World), __ENV__) |> elem(0) == :"Elixir.Hello" + end + + test "expands with alias is recursive" do + alias Source, as: Hello + alias Hello, as: World + assert expand_env(quote(do: World), __ENV__) |> elem(0) == :"Elixir.Source" + end end - ## Super + describe "import" do + test "raises on conflicting options" do + message = + ~r":only and :except can only be given together to import when :only is :functions, :macros, or :sigils" + + assert_compile_error(message, fn -> + expand(quote(do: import(Kernel, only: [], except: []))) + end) + end + + test "invalid import option" do + assert_compile_error(~r"unsupported option :ops given to import", fn -> + expand(quote(do: import(:lists, ops: 1))) + end) + end - test "super: expand args" do - assert expand(quote do: super(a, b)) == quote do: super(a(), b()) + test "raises for non-compile-time module" do + assert_compile_error(~r"invalid argument for import, .*, got: {:a, :tuple}", fn -> + expand(quote(do: import({:a, :tuple}))) + end) + end end - ## Vars + describe "require" do + test "raises for non-compile-time module" do + assert_compile_error(~r"invalid argument for require, .*, got: {:a, :tuple}", fn -> + expand(quote(do: require({:a, :tuple}))) + end) + end - test "vars: expand to local call" do - {output, env} = expand_env(quote(do: a), __ENV__) - assert output == quote(do: a()) - assert env.vars == [] + test "invalid options" do + assert_compile_error(~r"unsupported option :ops given to require", fn -> + expand(quote(do: require(Foo, ops: 1))) + end) + end end - test "vars: forces variable to exist" do - assert expand(quote do: (var!(a) = 1; var!(a))) + describe "=" do + test "defines vars" do + {output, env} = expand_env(quote(do: a = 1), __ENV__) + assert output == quote(do: a = 1) + assert Macro.Env.has_var?(env, {:a, __MODULE__}) + end + + test "does not define _" do + {output, env} = expand_env(quote(do: _ = 1), __ENV__) + assert output == quote(do: _ = 1) + assert Macro.Env.vars(env) == [] + end - message = ~r"expected var a to expand to an existing variable or be a part of a match" - assert_raise CompileError, message, fn -> expand(quote do: var!(a)) end + test "errors on directly recursive definitions" do + assert_compile_error( + ~r""" + recursive variable definition in patterns: - message = ~r"expected var a \(context Unknown\) to expand to an existing variable or be a part of a match" - assert_raise CompileError, message, fn -> expand(quote do: var!(a, Unknown)) end - end + \{x = \{:ok, x\}\} - test "^: expands args" do - assert expand(quote do: ^a = 1) == quote do: ^a = 1 - end + the variable "x" \(context Kernel.ExpansionTest\) is defined in function of itself + """, + fn -> expand(quote(do: {x = {:ok, x}} = :ok)) end + ) + + assert_compile_error( + ~r""" + recursive variable definition in patterns: - test "^: raises outside match" do - assert_raise CompileError, ~r"cannot use \^a outside of match clauses", fn -> - expand(quote do: ^a) + \{\{x, y\} = \{y, x\}\} + + the variable "x" \(context Kernel.ExpansionTest\) is defined in function of itself + """, + fn -> expand(quote(do: {{x, y} = {y, x}} = :ok)) end + ) + + assert_compile_error( + ~r""" + recursive variable definition in patterns: + + \{\{:x, y\} = \{x, :y\}, x = y\} + + the variable "x" \(context Kernel.ExpansionTest\) is defined recursively in function of "y" \(context Kernel.ExpansionTest\) + """, + fn -> expand(quote(do: {{:x, y} = {x, :y}, x = y} = :ok)) end + ) + + assert_compile_error( + ~r""" + recursive variable definition in patterns: + + \{x = y, y = z, z = x\} + + the following variables form a cycle: "x" \(context Kernel.ExpansionTest\), "y" \(context Kernel.ExpansionTest\), "z" \(context Kernel.ExpansionTest\) + """, + fn -> expand(quote(do: {x = y, y = z, z = x} = :ok)) end + ) end - end - test "^: raises without var" do - assert_raise CompileError, ~r"invalid argument for unary operator \^, expected an existing variable, got: \^1", fn -> - expand(quote do: ^1 = 1) + test "complex recursive variable definitions" do + assert expand( + quote do: + {%{type: type, client_id: client_id} = message, + %{type: type, client_id: client_id} = state} = :ok + ) + + assert_compile_error( + ~r"recursive variable definition in patterns", + fn -> + expand( + quote do: + {%{type: type, client_id: client_id} = message, + %{type: type, client_id: client_id} = state, client_id = type} = :ok + ) + end + ) end end - ## Locals + describe "environment macros" do + test "__MODULE__" do + assert expand(quote(do: __MODULE__)) == __MODULE__ + end - test "locals: expands to remote calls" do - assert {{:., _, [Kernel, :=~]}, _, [{:a, _, []}, {:b, _, []}]} = - expand(quote do: a =~ b) - end + test "__DIR__" do + assert expand(quote(do: __DIR__)) == __DIR__ + end - test "locals: expands to configured local" do - assert expand_env(quote(do: a), %{__ENV__ | local: Hello}) |> elem(0) == - quote(do: :"Elixir.Hello".a()) - end + test "__ENV__" do + env = %{__ENV__ | line: 0} + assert expand_env(quote(do: __ENV__), env) == {Macro.escape(env), env} + assert %{lexical_tracker: nil, tracers: []} = __ENV__ + end - test "locals: in guards" do - assert expand(quote(do: fn pid when :erlang.==(pid, self) -> pid end)) == - quote(do: fn pid when :erlang.==(pid, :erlang.self()) -> pid end) - end + test "__ENV__.accessor" do + env = %{__ENV__ | line: 0} + assert expand_env(quote(do: __ENV__.file), env) == {__ENV__.file, env} - test "locals: custom imports" do - assert expand(quote do: (import Kernel.ExpansionTarget; seventeen)) == - quote do: (import :"Elixir.Kernel.ExpansionTarget", []; 17) - end + assert expand_env(quote(do: __ENV__.unknown), env) == + {quote(do: unquote(Macro.escape(env)).unknown), env} - ## Tuples + assert __ENV__.lexical_tracker == nil + assert __ENV__.tracers == [] + end - test "tuples: expanded as arguments" do - assert expand(quote(do: {a = 1, a})) == quote do: {a = 1, a()} - assert expand(quote(do: {b, a = 1, a})) == quote do: {b(), a = 1, a()} + test "on match" do + assert_compile_error( + ~r"invalid pattern in match, __ENV__ is not allowed in matches", + fn -> expand(quote(do: __ENV__ = :ok)) end + ) + + assert_compile_error( + ~r"invalid pattern in match, __CALLER__ is not allowed in matches", + fn -> expand(quote(do: __CALLER__ = :ok)) end + ) + + assert_compile_error( + ~r"invalid pattern in match, __STACKTRACE__ is not allowed in matches", + fn -> expand(quote(do: __STACKTRACE__ = :ok)) end + ) + end end - ## Maps & structs + describe "vars" do + test "raises on undefined var by default" do + assert_compile_error(~r"undefined variable \"a\"", fn -> + expand_env({:a, [], nil}, __ENV__, []) + end) + end + + test "expands vars to local call when :on_undefined_variable is :warn" do + Code.put_compiler_option(:on_undefined_variable, :warn) - test "maps: expanded as arguments" do - assert expand(quote(do: %{a: a = 1, b: a})) == quote do: %{a: a = 1, b: a()} - end + {output, env} = expand_env({:a, [], nil}, __ENV__, []) + assert output == {:a, [if_undefined: :warn], []} + assert Macro.Env.vars(env) == [] + after + Code.put_compiler_option(:on_undefined_variable, :raise) + end - test "structs: expanded as arguments" do - assert expand(quote(do: %:elixir{a: a = 1, b: a})) == - quote do: %:elixir{a: a = 1, b: a()} + test "expands vars to local call without warning" do + env = __ENV__ - assert expand(quote(do: %:"Elixir.Kernel"{a: a = 1, b: a})) == - quote do: %:"Elixir.Kernel"{a: a = 1, b: a()} - end + {output, _, env} = + :elixir_expand.expand({:a, [if_undefined: :apply], nil}, :elixir_env.env_to_ex(env), env) - test "structs: expects atoms" do - assert_raise CompileError, ~r"expected struct name to be a compile time atom or alias", fn -> - expand(quote do: %unknown{a: 1}) + assert output == {:a, [if_undefined: :apply], []} + assert Macro.Env.vars(env) == [] end - end - ## quote + test "raises when expanding var to local call" do + env = __ENV__ - test "quote: expanded to raw forms" do - assert expand(quote do: (quote do: hello)) == {:{}, [], [:hello, [], __MODULE__]} - end + assert_compile_error(~r"undefined variable \"a\"", fn -> + :elixir_expand.expand({:a, [if_undefined: :raise], nil}, :elixir_env.env_to_ex(env), env) + end) + end - ## Anonymous calls + test "forces variable to exist" do + code = + quote do + var!(a) = 1 + var!(a) + end - test "anonymous calls: expands base and args" do - assert expand(quote do: a.(b)) == quote do: a().(b()) - end + assert expand(code) + + message = ~r"undefined variable \"a\"" + + assert_compile_error(message, fn -> + expand(quote(do: var!(a))) + end) - test "anonymous calls: raises on atom base" do - assert_raise CompileError, ~r"invalid function call :foo.()", fn -> - expand(quote do: :foo.(a)) + message = ~r"undefined variable \"a\" \(context Unknown\)" + + assert_compile_error(message, fn -> + expand(quote(do: var!(a, Unknown))) + end) end - end - ## Remote calls + test "raises for _ used outside of a match" do + assert_compile_error(~r"invalid use of _", fn -> + expand(quote(do: {1, 2, _})) + end) + end - test "remote calls: expands to erlang" do - assert expand(quote do: Kernel.is_atom(a)) == quote do: :erlang.is_atom(a()) - end + defmacrop var_ver(var, version) do + quote do + {unquote(var), [version: unquote(version)], __MODULE__} + end + end - test "remote calls: expands macros" do - assert expand(quote do: Kernel.ExpansionTest.thirteen) == 13 - end + defp expand_with_version(expr) do + env = :elixir_env.reset_vars(__ENV__) + {expr, _, _} = :elixir_expand.expand(expr, :elixir_env.env_to_ex(env), env) + expr + end - test "remote calls: expands receiver and args" do - assert expand(quote do: a.is_atom(b)) == quote do: a().is_atom(b()) - assert expand(quote do: (a = :foo).is_atom(a)) == quote do: (a = :foo).is_atom(a()) + test "tracks variable version" do + assert {:__block__, _, [{:=, _, [var_ver(:x, 0), 0]}, {:=, _, [_, var_ver(:x, 0)]}]} = + expand_with_version( + quote do + x = 0 + _ = x + end + ) + + assert {:__block__, _, + [ + {:=, _, [var_ver(:x, 0), 0]}, + {:=, _, [_, var_ver(:x, 0)]}, + {:=, _, [var_ver(:x, 1), 1]}, + {:=, _, [_, var_ver(:x, 1)]} + ]} = + expand_with_version( + quote do + x = 0 + _ = x + x = 1 + _ = x + end + ) + + assert {:__block__, _, + [ + {:=, _, [var_ver(:x, 0), 0]}, + {:fn, _, [{:->, _, [[var_ver(:x, 1)], {:=, _, [var_ver(:x, 2), 2]}]}]}, + {:=, _, [_, var_ver(:x, 0)]}, + {:=, _, [var_ver(:x, 3), 3]} + ]} = + expand_with_version( + quote do + x = 0 + fn x -> x = 2 end + _ = x + x = 3 + end + ) + + assert {:__block__, _, + [ + {:=, _, [var_ver(:x, 0), 0]}, + {:case, _, [:foo, [do: [{:->, _, [[var_ver(:x, 1)], var_ver(:x, 1)]}]]]}, + {:=, _, [_, var_ver(:x, 0)]}, + {:=, _, [var_ver(:x, 2), 2]} + ]} = + expand_with_version( + quote do + x = 0 + case(:foo, do: (x -> x)) + _ = x + x = 2 + end + ) + end end - test "remote calls: modules must be required for macros" do - assert expand(quote do: (require Kernel.ExpansionTarget; Kernel.ExpansionTarget.seventeen)) == - quote do: (require :"Elixir.Kernel.ExpansionTarget", []; 17) - end + describe "^" do + test "expands args" do + before_expansion = + quote do + after_expansion = 1 + ^after_expansion = 1 + end + + after_expansion = + quote do + after_expansion = 1 + ^after_expansion = 1 + end - test "remote calls: raises when not required" do - msg = ~r"you must require Kernel\.ExpansionTarget before invoking the macro Kernel\.ExpansionTarget\.seventeen/0" - assert_raise CompileError, msg, fn -> - expand(quote do: Kernel.ExpansionTarget.seventeen) + assert expand(before_expansion) == after_expansion end - end - ## Comprehensions + test "raises outside match" do + assert_compile_error(~r"misplaced operator \^a", fn -> + expand(quote(do: ^a)) + end) + end - test "variables inside comprehensions do not leak with enums" do - assert expand(quote do: (for(a <- b, do: c = 1); c)) == - quote do: (for(a <- b(), do: c = 1); c()) - end + test "raises without var" do + message = + ~r"invalid argument for unary operator \^, expected an existing variable, got: \^1" - test "variables inside comprehensions do not leak with binaries" do - assert expand(quote do: (for(<>, do: c = 1); c)) == - quote do: (for(<< <> <- b() >>, do: c = 1); c()) - end + assert_compile_error(message, fn -> + expand(quote(do: ^1 = 1)) + end) + end - test "variables inside filters are available in blocks" do - assert expand(quote do: for(a <- b, c = a, do: c)) == - quote do: (for(a <- b(), c = a, do: c)) + test "raises when the var is undefined" do + assert_compile_error(~r"undefined variable \^foo", fn -> + expand(quote(do: ^foo = :foo), []) + end) + end end - test "variables inside comprehensions options do not leak" do - assert expand(quote do: (for(a <- c = b, into: [], do: 1); c)) == - quote do: (for(a <- c = b(), do: 1, into: []); c()) + describe "locals" do + test "expands to remote calls" do + assert {{:., _, [Kernel, :=~]}, _, [{:a, _, []}, {:b, _, []}]} = expand(quote(do: a =~ b)) + end - assert expand(quote do: (for(a <- b, into: c = [], do: 1); c)) == - quote do: (for(a <- b(), do: 1, into: c = []); c()) - end + test "in matches" do + assert_compile_error( + ~r"cannot find or invoke local foo/1 inside a match. .+ Called as: foo\(:bar\)", + fn -> + expand(quote(do: foo(:bar) = :bar)) + end + ) + end - ## Capture + test "in guards" do + code = quote(do: fn pid when :erlang.==(pid, self) -> pid end) + expanded_code = quote(do: fn pid when :erlang.==(pid, :erlang.self()) -> pid end) + assert clean_meta(expand(code), [:imports, :context]) == expanded_code - test "&: keeps locals" do - assert expand(quote do: &unknown/2) == - {:&, [], [{:/, [], [{:unknown,[],nil}, 2]}]} - assert expand(quote do: &unknown(&1, &2)) == - {:&, [], [{:/, [], [{:unknown,[],nil}, 2]}]} - end + assert_compile_error(~r"cannot find or invoke local foo/1", fn -> + expand(quote(do: fn arg when foo(arg) -> arg end)) + end) + end - test "&: expands remotes" do - assert expand(quote do: &List.flatten/2) == - quote do: :erlang.make_fun(:"Elixir.List", :flatten, 2) + test "custom imports" do + before_expansion = + quote do + import Kernel.ExpansionTarget + seventeen() + end - assert expand(quote do: &Kernel.is_atom/1) == - quote do: :erlang.make_fun(:erlang, :is_atom, 1) - end + after_expansion = + quote do + :"Elixir.Kernel.ExpansionTarget" + 17 + end - test "&: expands macros" do + assert expand(before_expansion) == after_expansion + end - assert expand(quote do: (require Kernel.ExpansionTarget; &Kernel.ExpansionTarget.seventeen/0)) == - quote do: (require :"Elixir.Kernel.ExpansionTarget", []; fn -> 17 end) + test "invalid metadata" do + assert expand({:foo, [imports: 2, context: :unknown], [1, 2]}) == + {:foo, [imports: 2, context: :unknown], [1, 2]} + end end - ## fn + describe "floats" do + test "cannot be 0.0 inside match" do + assert capture_io(:stderr, fn -> expand(quote(do: 0.0 = 0.0)) end) =~ + "pattern matching on 0.0 is equivalent to matching only on +0.0 from Erlang/OTP 27+" - test "fn: expands each clause" do - assert expand(quote do: fn x -> x; _ -> x end) == - quote do: fn x -> x; _ -> x() end + assert {:=, [], [+0.0, +0.0]} = expand(quote(do: +0.0 = 0.0)) + assert {:=, [], [-0.0, +0.0]} = expand(quote(do: -0.0 = 0.0)) + end end - test "fn: does not share lexical in between clauses" do - assert expand(quote do: fn 1 -> import List; 2 -> flatten([1,2,3]) end) == - quote do: fn 1 -> import :"Elixir.List", []; 2 -> flatten([1,2,3]) end - end + describe "tuples" do + test "expanded as arguments" do + assert expand(quote(do: {after_expansion = 1, a})) == quote(do: {after_expansion = 1, a()}) - test "fn: expands guards" do - assert expand(quote do: fn x when x when __ENV__.context -> true end) == - quote do: fn x when x when :guard -> true end + assert expand(quote(do: {b, after_expansion = 1, a})) == + quote(do: {b(), after_expansion = 1, a()}) + end end - test "fn: does not leak vars" do - assert expand(quote do: (fn x -> x end; x)) == - quote do: (fn x -> x end; x()) - end + describe "maps" do + test "expanded as arguments" do + assert expand(quote(do: %{a: after_expansion = 1, b: a})) == + quote(do: %{a: after_expansion = 1, b: a()}) + end - ## Cond + test "with variables on keys inside patterns" do + ast = + quote do + %{(x = 1) => 1} + end - test "cond: expands each clause" do - assert expand_and_clean(quote do: (cond do x = 1 -> x; _ -> x end)) == - quote do: (cond do x = 1 -> x; _ -> x() end) - end + assert expand(ast) == ast - test "cond: does not share lexical in between clauses" do - assert expand_and_clean(quote do: (cond do 1 -> import List; 2 -> flatten([1,2,3]) end)) == - quote do: (cond do 1 -> import :"Elixir.List", []; 2 -> flatten([1,2,3]) end) - end + ast = + quote do + x = 1 + %{%{^x => 1} => 2} = y() + end - test "cond: does not leaks vars on head" do - assert expand_and_clean(quote do: (cond do x = 1 -> x; y = 2 -> y end; :erlang.+(x, y))) == - quote do: (cond do x = 1 -> x; y = 2 -> y end; :erlang.+(x(), y())) - end + assert expand(ast) == ast - test "cond: leaks vars" do - assert expand_and_clean(quote do: (cond do 1 -> x = 1; 2 -> y = 2 end; :erlang.+(x, y))) == - quote do: (cond do 1 -> x = 1; 2 -> y = 2 end; :erlang.+(x, y)) - end + ast = + quote do + x = 1 + %{{^x} => 1} = %{{1} => 1} + end - ## Case + assert expand(ast) == ast - test "case: expands each clause" do - assert expand_and_clean(quote do: (case w do x -> x; _ -> x end)) == - quote do: (case w() do x -> x; _ -> x() end) - end + assert_compile_error(~r"cannot use variable x as map key inside a pattern", fn -> + expand(quote(do: %{x => 1} = %{})) + end) - test "case: does not share lexical in between clauses" do - assert expand_and_clean(quote do: (case w do 1 -> import List; 2 -> flatten([1,2,3]) end)) == - quote do: (case w() do 1 -> import :"Elixir.List", []; 2 -> flatten([1,2,3]) end) - end + assert_compile_error(~r"undefined variable \^x", fn -> + expand(quote(do: {x, %{^x => 1}} = %{}), []) + end) + end - test "case: expands guards" do - assert expand_and_clean(quote do: (case w do x when x when __ENV__.context -> true end)) == - quote do: (case w() do x when x when :guard -> true end) - end + test "with binaries in keys inside patterns" do + before_ast = + quote do + %{<<0>> => nil} = %{<<0>> => nil} + end - test "case: does not leaks vars on head" do - assert expand_and_clean(quote do: (case w do x -> x; y -> y end; :erlang.+(x, y))) == - quote do: (case w() do x -> x; y -> y end; :erlang.+(x(), y())) - end + after_ast = + quote do + %{<<0::integer>> => nil} = %{<<0::integer>> => nil} + end - test "case: leaks vars" do - assert expand_and_clean(quote do: (case w do x -> x = x; y -> y = y end; :erlang.+(x, y))) == - quote do: (case w() do x -> x = x; y -> y = y end; :erlang.+(x, y)) - end + assert expand(before_ast) |> clean_meta([:alignment]) == clean_bit_modifiers(after_ast) + assert expand(after_ast) |> clean_meta([:alignment]) == clean_bit_modifiers(after_ast) - ## Receive + ast = + quote do + x = 8 + %{<<0::integer-size(x)>> => nil} = %{<<0::integer>> => nil} + end - test "receive: expands each clause" do - assert expand_and_clean(quote do: (receive do x -> x; _ -> x end)) == - quote do: (receive do x -> x; _ -> x() end) - end + assert expand(ast) |> clean_meta([:alignment]) == + clean_bit_modifiers(ast) |> clean_meta([:context, :imports]) - test "receive: does not share lexical in between clauses" do - assert expand_and_clean(quote do: (receive do 1 -> import List; 2 -> flatten([1,2,3]) end)) == - quote do: (receive do 1 -> import :"Elixir.List", []; 2 -> flatten([1,2,3]) end) - end + assert_compile_error(~r"cannot use variable x as map key inside a pattern", fn -> + expand(quote(do: %{<> => 1} = %{}), []) + end) + end - test "receive: expands guards" do - assert expand_and_clean(quote do: (receive do x when x when __ENV__.context -> true end)) == - quote do: (receive do x when x when :guard -> true end) + test "expects key-value pairs" do + assert_compile_error(~r"expected key-value pairs in a map, got: :foo", fn -> + expand(quote(do: unquote({:%{}, [], [:foo]}))) + end) + end end - test "receive: does not leaks clause vars" do - assert expand_and_clean(quote do: (receive do x -> x; y -> y end; :erlang.+(x, y))) == - quote do: (receive do x -> x; y -> y end; :erlang.+(x(), y())) + defmodule User do + defstruct name: "", age: 0 end - test "receive: leaks vars" do - assert expand_and_clean(quote do: (receive do x -> x = x; y -> y = y end; :erlang.+(x, y))) == - quote do: (receive do x -> x = x; y -> y = y end; :erlang.+(x, y)) - end + describe "structs" do + test "expanded as arguments" do + assert expand(quote(do: %User{})) == + quote(do: %:"Elixir.Kernel.ExpansionTest.User"{age: 0, name: ""}) - test "receive: leaks vars on after" do - assert expand_and_clean(quote do: (receive do x -> x = x after y -> y; w = y end; :erlang.+(x, w))) == - quote do: (receive do x -> x = x after y() -> y(); w = y() end; :erlang.+(x, w)) - end + assert expand(quote(do: %User{name: "john doe"})) == + quote(do: %:"Elixir.Kernel.ExpansionTest.User"{age: 0, name: "john doe"}) + end - ## Try + test "expects atoms" do + expand(quote(do: %unknown{a: 1} = x)) - test "try: expands do" do - assert expand(quote do: (try do x = y end; x)) == - quote do: (try do x = y() end; x()) - end + message = ~r"expected struct name to be a compile time atom or alias" - test "try: expands catch" do - assert expand(quote do: (try do x catch x, y -> z = :erlang.+(x, y) end; z)) == - quote do: (try do x() catch x, y -> z = :erlang.+(x, y) end; z()) - end + assert_compile_error(message, fn -> + expand(quote(do: %unknown{a: 1})) + end) - test "try: expands after" do - assert expand(quote do: (try do x after z = y end; z)) == - quote do: (try do x() after z = y() end; z()) - end + message = ~r"expected struct name to be a compile time atom or alias" - test "try: expands else" do - assert expand(quote do: (try do x else z -> z end; z)) == - quote do: (try do x() else z -> z end; z()) - end + assert_compile_error(message, fn -> + expand(quote(do: %unquote(1){a: 1})) + end) + + message = ~r"expected struct name in a match to be a compile time atom, alias or a variable" + + assert_compile_error(message, fn -> + expand(quote(do: %unquote(1){a: 1} = x)) + end) + end + + test "update syntax" do + expand(quote(do: %{%{a: 0} | a: 1})) + + assert_compile_error(~r"cannot use map/struct update syntax in match", fn -> + expand(quote(do: %{%{a: 0} | a: 1} = %{})) + end) + end + + test "dynamic syntax expands to itself" do + assert expand(quote(do: %x{} = 1)) == quote(do: %x{} = 1) + end + + test "invalid keys in structs" do + assert_compile_error(~r"invalid key :erlang\.\+\(1, 2\) for struct", fn -> + expand( + quote do + %User{(1 + 2) => :my_value} + end + ) + end) + end - test "try: expands rescue" do - assert expand(quote do: (try do x rescue x -> x; Error -> x end; x)) == - quote do: (try do x() rescue unquote(:in)(x, _) -> x; unquote(:in)(_, [:"Elixir.Error"]) -> x() end; x()) + test "unknown key in structs" do + message = ~r"unknown key :foo for struct Kernel\.ExpansionTest\.User" + + assert_compile_error(message, fn -> + expand( + quote do + %User{foo: :my_value} = %{} + end + ) + end) + end end - ## Binaries + describe "quote" do + test "expanded to raw forms" do + assert expand(quote(do: quote(do: hello)), []) == {:{}, [], [:hello, [], __MODULE__]} + end + + test "raises if the :bind_quoted option is invalid" do + assert_compile_error(~r"invalid :bind_quoted for quote", fn -> + expand(quote(do: quote(bind_quoted: self(), do: :ok))) + end) + + assert_compile_error(~r"invalid :bind_quoted for quote", fn -> + expand(quote(do: quote(bind_quoted: [{1, 2}], do: :ok))) + end) + end + + test "raises for missing do" do + assert_compile_error(~r"missing :do option in \"quote\"", fn -> + expand(quote(do: quote(context: Foo))) + end) + end - test "bitstrings: expands modifiers" do - assert expand(quote do: (import Kernel.ExpansionTarget; << x :: seventeen >>)) == - quote do: (import :"Elixir.Kernel.ExpansionTarget", []; << x() :: [unquote(:size)(17)] >>) + test "raises for invalid arguments" do + assert_compile_error(~r"invalid arguments for \"quote\"", fn -> + expand(quote(do: quote(1 + 1))) + end) + end - assert expand(quote do: (import Kernel.ExpansionTarget; << seventeen :: seventeen, x :: size(seventeen) >> = 1)) == - quote do: (import :"Elixir.Kernel.ExpansionTarget", []; - << seventeen :: [unquote(:size)(17)], x :: [unquote(:size)(seventeen)] >> = 1) + test "raises unless its options are a keyword list" do + assert_compile_error(~r"invalid options for quote, expected a keyword list", fn -> + expand(quote(do: quote(:foo, do: :foo))) + end) + end end - test "bitstrings: expands modifiers args" do - assert expand(quote do: (require Kernel.ExpansionTarget; << x :: size(Kernel.ExpansionTarget.seventeen) >>)) == - quote do: (require :"Elixir.Kernel.ExpansionTarget", []; << x() :: [unquote(:size)(17)] >>) + describe "anonymous calls" do + test "expands base and args" do + assert expand(quote(do: a.(b))) == quote(do: a().(b())) + end end - ## Invalid + describe "remotes" do + test "expands to Erlang" do + assert expand(quote(do: Kernel.is_atom(a))) == quote(do: :erlang.is_atom(a())) + end + + test "expands macros" do + assert expand(quote(do: Kernel.ExpansionTest.thirteen())) == 13 + end + + test "expands receiver and args" do + assert expand(quote(do: a.is_atom(b))) == quote(do: a().is_atom(b())) - test "handles invalid expressions" do - assert_raise CompileError, ~r"invalid quoted expression: {1, 2, 3}", fn -> - expand(quote do: unquote({1, 2, 3})) + assert expand(quote(do: (after_expansion = :foo).is_atom(a))) == + quote(do: (after_expansion = :foo).is_atom(a())) + end + + test "modules must be required for macros" do + before_expansion = + quote do + require Kernel.ExpansionTarget + Kernel.ExpansionTarget.seventeen() + end + + after_expansion = + quote do + :"Elixir.Kernel.ExpansionTarget" + 17 + end + + assert expand(before_expansion) == after_expansion + end + + test "in matches" do + message = ~r"cannot invoke remote function Hello.fun_that_does_not_exist/0 inside a match" + + assert_compile_error(message, fn -> + expand(quote(do: Hello.fun_that_does_not_exist() = :foo)) + end) + + message = ~r"cannot invoke remote function :erlang.make_ref/0 inside a match" + assert_compile_error(message, fn -> expand(quote(do: make_ref() = :foo)) end) + + message = ~r"invalid argument for \+\+ operator inside a match" + + assert_compile_error(message, fn -> + expand(quote(do: "a" ++ "b" = "ab")) + end) + + assert_compile_error(message, fn -> + expand(quote(do: [1 | 2] ++ [3] = [1, 2, 3])) + end) + + assert_compile_error(message, fn -> + expand(quote(do: [1] ++ 2 ++ [3] = [1, 2, 3])) + end) + + assert {:=, _, [-1, -1]} = + expand(quote(do: -1 = -1)) + + assert {:=, _, [1, 1]} = + expand(quote(do: +1 = +1)) + + assert {:=, _, [[{:|, _, [1, [{:|, _, [2, 3]}]]}], [1, 2, 3]]} = + expand(quote(do: [1] ++ [2] ++ 3 = [1, 2, 3])) + end + + test "in guards" do + message = + ~r"cannot invoke remote function Hello.something_that_does_not_exist/1 inside a guard" + + assert_compile_error(message, fn -> + expand(quote(do: fn arg when Hello.something_that_does_not_exist(arg) -> arg end)) + end) + + message = ~r"cannot invoke remote function :erlang.make_ref/0 inside a guard" + + assert_compile_error(message, fn -> + expand(quote(do: fn arg when make_ref() -> arg end)) + end) end - assert_raise CompileError, ~r"invalid quoted expression: #Function<", fn -> - expand(quote do: unquote({:sample, fn -> end})) + test "in guards with macros" do + message = + ~r"you must require the moduleInteger before invoking macro Integer.is_even/1 inside a guard" + + assert_compile_error(message, fn -> + expand(quote(do: fn arg when Integer.is_even(arg) -> arg end)) + end) + end + + test "in guards with bitstrings" do + message = ~r"cannot invoke remote function String.Chars.to_string/1 inside a guard" + + assert_compile_error(message, fn -> + expand(quote(do: fn arg when "#{arg}foo" == "argfoo" -> arg end)) + end) + + assert_compile_error(message, fn -> + expand( + quote do + fn arg when <<:"Elixir.Kernel".to_string(arg)::binary, "foo">> == "argfoo" -> + arg + end + end + ) + end) end end - ## Helpers + describe "comprehensions" do + test "variables do not leak with enums" do + before_expansion = + quote do + for(a <- b, do: c = 1) + c + end - defmacro thirteen do - 13 + after_expansion = + quote do + for(a <- b(), do: c = 1) + c() + end + + assert expand(before_expansion) == after_expansion + end + + test "variables do not leak with binaries" do + before_expansion = + quote do + for(<>, do: c = 1) + c + end + + after_expansion = + quote do + for(<<(<> <- b())>>, do: c = 1) + c() + end + + assert expand(before_expansion) |> clean_meta([:alignment]) == + clean_bit_modifiers(after_expansion) + end + + test "variables inside generator args do not leak" do + before_expansion = + quote do + for( + b <- + ( + a = 1 + [2] + ), + do: {a, b} + ) + + a + end + + after_expansion = + quote do + for( + b <- + ( + a = 1 + [2] + ), + do: {a(), b} + ) + + a() + end + + assert expand(before_expansion) == after_expansion + + before_expansion = + quote do + for( + b <- + ( + a = 1 + [2] + ), + d <- + ( + c = 3 + [4] + ), + do: {a, b, c, d} + ) + end + + after_expansion = + quote do + for( + b <- + ( + a = 1 + [2] + ), + d <- + ( + c = 3 + [4] + ), + do: {a(), b, c(), d}, + into: [] + ) + end + + assert expand(before_expansion) == after_expansion + end + + test "variables inside filters are available in blocks" do + assert expand(quote(do: for(a <- b, c = a, do: c))) == + quote(do: for(a <- b(), c = a, do: c, into: [])) + end + + test "variables inside options do not leak" do + before_expansion = + quote do + for(a <- c = b, into: [], do: 1) + c + end + + after_expansion = + quote do + for(a <- c = b(), do: 1, into: []) + c() + end + + assert expand(before_expansion) == after_expansion + + before_expansion = + quote do + for(a <- b, into: c = [], do: 1) + c + end + + after_expansion = + quote do + for(a <- b(), do: 1, into: c = []) + c() + end + + assert expand(before_expansion) == after_expansion + end + + test "must start with generators" do + assert_compile_error(~r"for comprehensions must start with a generator", fn -> + expand(quote(do: for(is_atom(:foo), do: :foo))) + end) + + assert_compile_error(~r"for comprehensions must start with a generator", fn -> + expand(quote(do: for(do: :foo))) + end) + end + + test "requires size on binary generators" do + message = ~r"a binary field without size is only allowed at the end of a binary pattern" + + assert_compile_error(message, fn -> + expand(quote(do: for(<>, do: x))) + end) + end + + test "require do option" do + assert_compile_error(~r"missing :do option in \"for\"", fn -> + expand(quote(do: for(_ <- 1..2))) + end) + end + + test "uniq option is boolean" do + message = ~r":uniq option for comprehensions only accepts a boolean, got: x" + + assert_compile_error(message, fn -> + expand(quote(do: for(x <- 1..2, uniq: x, do: x))) + end) + end + + test "raise error on invalid reduce" do + assert_compile_error( + ~r"cannot use :reduce alongside :into/:uniq in comprehension", + fn -> + expand(quote(do: for(x <- 1..3, reduce: %{}, into: %{}, do: (acc -> acc)))) + end + ) + + assert_compile_error( + ~r"the do block was written using acc -> expr clauses but the :reduce option was not given", + fn -> expand(quote(do: for(x <- 1..3, do: (acc -> acc)))) end + ) + + assert_compile_error( + ~r"when using :reduce with comprehensions, the do block must be written using acc -> expr clauses", + fn -> expand(quote(do: for(x <- 1..3, reduce: %{}, do: x))) end + ) + + assert_compile_error( + ~r"when using :reduce with comprehensions, the do block must be written using acc -> expr clauses", + fn -> expand(quote(do: for(x <- 1..3, reduce: %{}, do: (acc, x -> x)))) end + ) + + assert_compile_error( + ~r"when using :reduce with comprehensions, the do block must be written using acc -> expr clauses", + fn -> expand(quote(do: for(x <- 1..3, reduce: %{}, do: (acc, x when 1 == 1 -> x)))) end + ) + end + + test "raise error for unknown options" do + assert_compile_error(~r"unsupported option :else given to for", fn -> + expand(quote(do: for(_ <- 1..2, do: 1, else: 1))) + end) + + assert_compile_error(~r"unsupported option :other given to for", fn -> + expand(quote(do: for(_ <- 1..2, do: 1, other: 1))) + end) + end end - defp expand_and_clean(expr) do - cleaner = &Keyword.drop(&1, [:export]) - expr - |> expand_env(__ENV__) - |> elem(0) - |> Macro.prewalk(&Macro.update_meta(&1, cleaner)) + describe "with" do + test "variables do not leak" do + before_expansion = + quote do + with({foo} <- {bar}, do: baz = :ok) + baz + end + + after_expansion = + quote do + with({foo} <- {bar()}, do: baz = :ok) + baz() + end + + assert expand(before_expansion) == after_expansion + end + + test "variables inside args expression do not leak" do + before_expansion = + quote do + with( + b <- + ( + a = 1 + 2 + ), + do: {a, b} + ) + + a + end + + after_expansion = + quote do + with( + b <- + ( + a = 1 + 2 + ), + do: {a(), b} + ) + + a() + end + + assert expand(before_expansion) == after_expansion + + before_expansion = + quote do + with( + b <- + ( + a = 1 + 2 + ), + d <- + ( + c = 3 + 4 + ), + do: {a, b, c, d} + ) + end + + after_expansion = + quote do + with( + b <- + ( + a = 1 + 2 + ), + d <- + ( + c = 3 + 4 + ), + do: {a(), b, c(), d} + ) + end + + assert expand(before_expansion) == after_expansion + end + + test "variables are available in do option" do + before_expansion = + quote do + with({foo} <- {bar}, do: baz = foo) + baz + end + + after_expansion = + quote do + with({foo} <- {bar()}, do: baz = foo) + baz() + end + + assert expand(before_expansion) == after_expansion + end + + test "variables inside else do not leak" do + before_expansion = + quote do + with({foo} <- {bar}, do: :ok, else: (baz -> baz)) + baz + end + + after_expansion = + quote do + with({foo} <- {bar()}, do: :ok, else: (baz -> baz)) + baz() + end + + assert expand(before_expansion) == after_expansion + end + + test "fails if \"do\" is missing" do + assert_compile_error(~r"missing :do option in \"with\"", fn -> + expand(quote(do: with(_ <- true, []))) + end) + end + + test "fails on invalid else option" do + assert_compile_error( + ~r"invalid \"else\" block in \"with\", it expects \"pattern -> expr\" clauses", + fn -> + expand(quote(do: with(_ <- true, do: :ok, else: [:error]))) + end + ) + + assert_compile_error( + ~r"invalid \"else\" block in \"with\", it expects \"pattern -> expr\" clauses", + fn -> + expand(quote(do: with(_ <- true, do: :ok, else: :error))) + end + ) + + assert_compile_error( + ~r"invalid \"else\" block in \"with\", it expects \"pattern -> expr\" clauses", + fn -> + expand(quote(do: with(_ <- true, do: :ok, else: []))) + end + ) + end + + test "fails for invalid options" do + # Only the required "do" is present alongside the unexpected option. + assert_compile_error(~r"unexpected option :foo in \"with\"", fn -> + expand(quote(do: with(_ <- true, foo: :bar, do: :ok))) + end) + + # More options are present alongside the unexpected option. + assert_compile_error(~r"unexpected option :foo in \"with\"", fn -> + expand(quote(do: with(_ <- true, do: :ok, else: (_ -> :ok), foo: :bar))) + end) + + assert_compile_error(~r"unexpected option :foo in \"with\"", fn -> + expand( + quote do + with _ <- true, foo: :bar do + :ok + end + end + ) + end) + end end - defp expand(expr) do - expand_env(expr, __ENV__) |> elem(0) + describe "&" do + test "keeps locals" do + assert expand(quote(do: &unknown/2)) == {:&, [], [{:/, [], [{:unknown, [], nil}, 2]}]} + assert expand(quote(do: &unknown(&1, &2))) == {:&, [], [{:/, [], [{:unknown, [], nil}, 2]}]} + end + + test "keeps position meta on & variables" do + assert expand(Code.string_to_quoted!("& &1")) |> clean_meta([:counter]) == + {:fn, [capture: true, line: 1], + [ + {:->, [line: 1], + [ + [{:capture, [capture: 1, line: 1], nil}], + {:capture, [capture: 1, line: 1], nil} + ]} + ]} + end + + test "removes no_parens when expanding 0-arity capture to fn" do + assert expand(quote(do: &foo().bar/0)) == + quote(do: fn -> foo().bar() end) + end + + test "expands remotes" do + assert expand(quote(do: &List.flatten/2)) == + quote(do: &:"Elixir.List".flatten/2) + |> clean_meta([:imports, :context]) + + assert expand(quote(do: &Kernel.is_atom/1)) == + quote(do: &:erlang.is_atom/1) |> clean_meta([:imports, :context]) + end + + test "expands macros" do + before_expansion = + quote do + require Kernel.ExpansionTarget + &Kernel.ExpansionTarget.seventeen/0 + end + + after_expansion = + quote do + :"Elixir.Kernel.ExpansionTarget" + fn -> 17 end + end + + assert clean_meta(expand(before_expansion), [:imports, :context, :no_parens]) == + after_expansion + end + + test "fails on non-continuous" do + assert_compile_error(~r"capture argument &0 must be numbered between 1 and 255", fn -> + expand(quote(do: &foo(&0))) + end) + + assert_compile_error(~r"capture argument &2 cannot be defined without &1", fn -> + expand(quote(do: & &2)) + end) + + assert_compile_error(~r"capture argument &255 cannot be defined without &1", fn -> + expand(quote(do: & &255)) + end) + end + + test "fails on block" do + message = ~r"block expressions are not allowed inside the capture operator &, got: 1\n2" + + assert_compile_error(message, fn -> + code = + quote do + &( + 1 + 2 + ) + end + + expand(code) + end) + end + + test "fails on other types" do + assert_compile_error(~r"invalid args for &, expected one of:", fn -> + expand(quote(do: &:foo)) + end) + end + + test "fails on invalid arity" do + message = ~r"capture argument &256 must be numbered between 1 and 255" + + assert_compile_error(message, fn -> + expand(quote(do: &Mod.fun/256)) + end) + end + + test "fails when no captures" do + assert_compile_error(~r"invalid args for &, expected one of:", fn -> + expand(quote(do: &foo())) + end) + end + + test "fails on nested capture" do + assert_compile_error(~r"nested captures are not allowed", fn -> + expand(quote(do: &(& &1))) + end) + end + + test "fails on integers" do + assert_compile_error( + ~r"capture argument &1 must be used within the capture operator &", + fn -> expand(quote(do: &1)) end + ) + end end - defp expand_env(expr, env) do - :elixir_exp.expand(expr, env) + describe "fn" do + test "expands each clause" do + before_expansion = + quote do + fn + x -> x + _ -> x + end + end + + after_expansion = + quote do + fn + x -> x + _ -> x() + end + end + + assert expand(before_expansion) == after_expansion + end + + test "does not share lexical scope between clauses" do + before_expansion = + quote do + fn + 1 -> import List + 2 -> flatten([1, 2, 3]) + end + end + + after_expansion = + quote do + fn + 1 -> :"Elixir.List" + 2 -> flatten([1, 2, 3]) + end + end + + assert expand(before_expansion) == after_expansion + end + + test "expands guards" do + assert expand(quote(do: fn x when x when __ENV__.context -> true end)) == + quote(do: fn x when x when :guard -> true end) + end + + test "does not leak vars" do + before_expansion = + quote do + fn x -> x end + x + end + + after_expansion = + quote do + fn x -> x end + x() + end + + assert expand(before_expansion) == after_expansion + end + + test "raises on mixed arities" do + message = ~r"cannot mix clauses with different arities in anonymous functions" + + assert_compile_error(message, fn -> + code = + quote do + fn + x -> x + x, y -> x + y + end + end + + expand(code) + end) + end + end + + describe "cond" do + test "expands each clause" do + before_expansion = + quote do + cond do + x = 1 -> x + true -> x + end + end + + after_expansion = + quote do + cond do + x = 1 -> x + true -> x() + end + end + + assert expand(before_expansion) == after_expansion + end + + test "does not share lexical scope between clauses" do + before_expansion = + quote do + cond do + 1 -> import List + 2 -> flatten([1, 2, 3]) + end + end + + after_expansion = + quote do + cond do + 1 -> :"Elixir.List" + 2 -> flatten([1, 2, 3]) + end + end + + assert expand(before_expansion) == after_expansion + end + + test "does not leaks vars on head" do + before_expansion = + quote do + cond do + x = 1 -> x + y = 2 -> y + end + + :erlang.+(x, y) + end + + after_expansion = + quote do + cond do + x = 1 -> x + y = 2 -> y + end + + :erlang.+(x(), y()) + end + + assert expand(before_expansion) == after_expansion + end + + test "does not leak vars" do + before_expansion = + quote do + cond do + 1 -> x = 1 + 2 -> y = 2 + end + + :erlang.+(x, y) + end + + after_expansion = + quote do + cond do + 1 -> x = 1 + 2 -> y = 2 + end + + :erlang.+(x(), y()) + end + + assert expand(before_expansion) == after_expansion + end + + test "expects exactly one do" do + assert_compile_error(~r"missing :do option in \"cond\"", fn -> + expand(quote(do: cond([]))) + end) + + assert_compile_error(~r"duplicate \"do\" clauses given for \"cond\"", fn -> + expand(quote(do: cond(do: (x -> x), do: (y -> y)))) + end) + end + + test "expects clauses" do + assert_compile_error( + ~r"invalid \"do\" block in \"cond\", it expects \"pattern -> expr\" clauses", + fn -> + expand(quote(do: cond(do: :ok))) + end + ) + + assert_compile_error( + ~r"invalid \"do\" block in \"cond\", it expects \"pattern -> expr\" clauses", + fn -> + expand(quote(do: cond(do: [:not, :clauses]))) + end + ) + end + + test "expects one argument in clauses" do + assert_compile_error( + ~r"expected one argument for \"do\" clauses \(->\) in \"cond\"", + fn -> + code = + quote do + cond do + _, _ -> :ok + end + end + + expand(code) + end + ) + end + + test "raises for invalid arguments" do + assert_compile_error(~r"invalid arguments for \"cond\"", fn -> + expand(quote(do: cond(:foo))) + end) + end + + test "raises with invalid options" do + assert_compile_error(~r"unexpected option :foo in \"cond\"", fn -> + expand(quote(do: cond(do: (1 -> 1), foo: :bar))) + end) + end + + test "raises for _ in clauses" do + message = ~r"invalid use of _ inside \"cond\"\. If you want the last clause" + + assert_compile_error(message, fn -> + code = + quote do + cond do + x -> x + _ -> :raise + end + end + + expand(code) + end) + end + end + + describe "case" do + test "expands each clause" do + before_expansion = + quote do + case w do + x -> x + _ -> x + end + end + + after_expansion = + quote do + case w() do + x -> x + _ -> x() + end + end + + assert expand(before_expansion) == after_expansion + end + + test "does not share lexical scope between clauses" do + before_expansion = + quote do + case w do + 1 -> import List + 2 -> flatten([1, 2, 3]) + end + end + + after_expansion = + quote do + case w() do + 1 -> :"Elixir.List" + 2 -> flatten([1, 2, 3]) + end + end + + assert expand(before_expansion) == after_expansion + end + + test "expands guards" do + before_expansion = + quote do + case w do + x when x when __ENV__.context -> true + end + end + + after_expansion = + quote do + case w() do + x when x when :guard -> true + end + end + + assert expand(before_expansion) == after_expansion + end + + test "does not leaks vars on head" do + before_expansion = + quote do + case w do + x -> x + y -> y + end + + :erlang.+(x, y) + end + + after_expansion = + quote do + case w() do + x -> x + y -> y + end + + :erlang.+(x(), y()) + end + + assert expand(before_expansion) == after_expansion + end + + test "does not leak vars" do + before_expansion = + quote do + case w do + x -> x = x + y -> y = y + end + + :erlang.+(x, y) + end + + after_expansion = + quote do + case w() do + x -> x = x + y -> y = y + end + + :erlang.+(x(), y()) + end + + assert expand(before_expansion) == after_expansion + end + + test "expects exactly one do" do + assert_compile_error(~r"missing :do option in \"case\"", fn -> + expand(quote(do: case(e, []))) + end) + + assert_compile_error(~r"duplicate \"do\" clauses given for \"case\"", fn -> + expand(quote(do: case(e, do: (x -> x), do: (y -> y)))) + end) + end + + test "expects clauses" do + assert_compile_error( + ~r"invalid \"do\" block in \"case\", it expects \"pattern -> expr\" clauses", + fn -> + code = + quote do + case e do + x + end + end + + expand(code) + end + ) + + assert_compile_error( + ~r"invalid \"do\" block in \"case\", it expects \"pattern -> expr\" clauses", + fn -> + code = + quote do + case e do + [:not, :clauses] + end + end + + expand(code) + end + ) + end + + test "expects exactly one argument in clauses" do + assert_compile_error( + ~r"expected one argument for \"do\" clauses \(->\) in \"case\"", + fn -> + code = + quote do + case e do + _, _ -> :ok + end + end + + expand(code) + end + ) + end + + test "fails with invalid arguments" do + assert_compile_error(~r"invalid arguments for \"case\"", fn -> + expand(quote(do: case(:foo, :bar))) + end) + end + + test "fails for invalid options" do + assert_compile_error(~r"unexpected option :foo in \"case\"", fn -> + expand(quote(do: case(e, do: (x -> x), foo: :bar))) + end) + end + end + + describe "receive" do + test "expands each clause" do + before_expansion = + quote do + receive do + x -> x + _ -> x + end + end + + after_expansion = + quote do + receive do + x -> x + _ -> x() + end + end + + assert expand(before_expansion) == after_expansion + end + + test "does not share lexical scope between clauses" do + before_expansion = + quote do + receive do + 1 -> import List + 2 -> flatten([1, 2, 3]) + end + end + + after_expansion = + quote do + receive do + 1 -> :"Elixir.List" + 2 -> flatten([1, 2, 3]) + end + end + + assert expand(before_expansion) == after_expansion + end + + test "expands guards" do + before_expansion = + quote do + receive do + x when x when __ENV__.context -> true + end + end + + after_expansion = + quote do + receive do + x when x when :guard -> true + end + end + + assert expand(before_expansion) == after_expansion + end + + test "does not leaks clause vars" do + before_expansion = + quote do + receive do + x -> x + y -> y + end + + :erlang.+(x, y) + end + + after_expansion = + quote do + receive do + x -> x + y -> y + end + + :erlang.+(x(), y()) + end + + assert expand(before_expansion) == after_expansion + end + + test "does not leak vars" do + before_expansion = + quote do + receive do + x -> x = x + y -> y = y + end + + :erlang.+(x, y) + end + + after_expansion = + quote do + receive do + x -> x = x + y -> y = y + end + + :erlang.+(x(), y()) + end + + assert expand(before_expansion) == after_expansion + end + + test "does not leak vars on after" do + before_expansion = + quote do + receive do + x -> x = x + after + y -> + y + w = y + end + + :erlang.+(x, w) + end + + after_expansion = + quote do + receive do + x -> x = x + after + y() -> + y() + w = y() + end + + :erlang.+(x(), w()) + end + + assert expand(before_expansion) == after_expansion + end + + test "expects exactly one do or after" do + assert_compile_error(~r"missing :do/:after option in \"receive\"", fn -> + expand(quote(do: receive([]))) + end) + + assert_compile_error(~r"duplicate \"do\" clauses given for \"receive\"", fn -> + expand(quote(do: receive(do: (x -> x), do: (y -> y)))) + end) + + assert_compile_error(~r"duplicate \"after\" clauses given for \"receive\"", fn -> + code = + quote do + receive do + x -> x + after + y -> y + after + z -> z + end + end + + expand(code) + end) + end + + test "expects clauses" do + assert_compile_error( + ~r"invalid \"do\" block in \"receive\", it expects \"pattern -> expr\" clauses", + fn -> + code = + quote do + receive do + x + end + end + + expand(code) + end + ) + + assert_compile_error( + ~r"invalid \"do\" block in \"receive\", it expects \"pattern -> expr\" clauses", + fn -> + code = + quote do + receive do + [:not, :clauses] + end + end + + expand(code) + end + ) + end + + test "expects on argument for do/after clauses" do + assert_compile_error( + ~r"expected one argument for \"do\" clauses \(->\) in \"receive\"", + fn -> + code = + quote do + receive do + _, _ -> :ok + end + end + + expand(code) + end + ) + + message = ~r"expected one argument for \"after\" clauses \(->\) in \"receive\"" + + assert_compile_error(message, fn -> + code = + quote do + receive do + x -> x + after + _, _ -> :ok + end + end + + expand(code) + end) + end + + test "expects a single clause for \"after\"" do + assert_compile_error(~r"expected a single -> clause for :after in \"receive\"", fn -> + code = + quote do + receive do + x -> x + after + 1 -> y + 2 -> z + end + end + + expand(code) + end) + end + + test "raises for invalid arguments" do + assert_compile_error(~r"invalid arguments for \"receive\"", fn -> + expand(quote(do: receive(:foo))) + end) + end + + test "raises with invalid options" do + assert_compile_error(~r"unexpected option :foo in \"receive\"", fn -> + expand(quote(do: receive(do: (x -> x), foo: :bar))) + end) + end + end + + describe "try" do + test "expands catch" do + before_expansion = + quote do + try do + x + catch + x, y -> z = :erlang.+(x, y) + end + + z + end + + after_expansion = + quote do + try do + x() + catch + x, y -> z = :erlang.+(x, y) + end + + z() + end + + assert expand(before_expansion) == after_expansion + end + + test "expands catch with when" do + before_expansion = + quote do + try do + x + catch + x when x -> z = :erlang.-(x) + end + + z + end + + after_expansion = + quote do + try do + x() + catch + :throw, x when x -> z = :erlang.-(x) + end + + z() + end + + assert expand(before_expansion) == after_expansion + end + + test "expands after" do + before_expansion = + quote do + try do + x + after + z = y + end + + z + end + + after_expansion = + quote do + try do + x() + after + z = y() + end + + z() + end + + assert expand(before_expansion) == after_expansion + end + + test "expands else" do + before_expansion = + quote do + try do + x + catch + _, _ -> :ok + else + z -> z + end + + z + end + + after_expansion = + quote do + try do + x() + catch + _, _ -> :ok + else + z -> z + end + + z() + end + + assert expand(before_expansion) == after_expansion + end + + test "expands rescue" do + before_expansion = + quote do + try do + x + rescue + x -> x + Error -> x + end + + x + end + + after_expansion = + quote do + try do + x() + rescue + x -> x + unquote(:in)(_, [:"Elixir.Error"]) -> x() + end + + x() + end + + assert expand(before_expansion) == after_expansion + end + + test "expects more than do" do + assert_compile_error(~r"missing :catch/:rescue/:after option in \"try\"", fn -> + code = + quote do + try do + x = y + end + + x + end + + expand(code) + end) + end + + test "raises if do is missing" do + assert_compile_error(~r"missing :do option in \"try\"", fn -> + expand(quote(do: try([]))) + end) + end + + test "expects at most one clause" do + assert_compile_error(~r"duplicate \"do\" clauses given for \"try\"", fn -> + expand(quote(do: try(do: e, do: f))) + end) + + assert_compile_error(~r"duplicate \"rescue\" clauses given for \"try\"", fn -> + code = + quote do + try do + e + rescue + x -> x + rescue + y -> y + end + end + + expand(code) + end) + + assert_compile_error(~r"duplicate \"after\" clauses given for \"try\"", fn -> + code = + quote do + try do + e + after + x = y + after + x = y + end + end + + expand(code) + end) + + assert_compile_error(~r"duplicate \"else\" clauses given for \"try\"", fn -> + code = + quote do + try do + e + else + x -> x + else + y -> y + end + end + + expand(code) + end) + + assert_compile_error(~r"duplicate \"catch\" clauses given for \"try\"", fn -> + code = + quote do + try do + e + catch + x -> x + catch + y -> y + end + end + + expand(code) + end) + end + + test "raises with invalid arguments" do + assert_compile_error(~r"invalid arguments for \"try\"", fn -> + expand(quote(do: try(:foo))) + end) + end + + test "raises with invalid options" do + assert_compile_error(~r"unexpected option :foo in \"try\"", fn -> + expand(quote(do: try(do: x, foo: :bar))) + end) + end + + test "expects exactly one argument in rescue clauses" do + assert_compile_error( + ~r"expected one argument for \"rescue\" clauses \(->\) in \"try\"", + fn -> + code = + quote do + try do + x + rescue + _, _ -> :ok + end + end + + expand(code) + end + ) + end + + test "expects an alias, a variable, or \"var in [alias]\" as the argument of rescue clauses" do + assert_compile_error(~r"invalid \"rescue\" clause\. The clause should match", fn -> + code = + quote do + try do + x + rescue + function(:call) -> :ok + end + end + + expand(code) + end) + end + + test "expects one or two args for catch clauses" do + message = ~r"expected one or two args for \"catch\" clauses \(->\) in \"try\"" + + assert_compile_error(message, fn -> + code = + quote do + try do + x + catch + _, _, _ -> :ok + end + end + + expand(code) + end) + + assert_compile_error(message, fn -> + code = + quote do + try do + x + catch + _, _, _ when 1 == 1 -> :ok + end + end + + expand(code) + end) + end + + test "expects clauses for rescue, else, catch" do + assert_compile_error( + ~r"invalid \"rescue\" block in \"try\", it expects \"pattern -> expr\" clauses", + fn -> + code = + quote do + try do + e + rescue + x + end + end + + expand(code) + end + ) + + assert_compile_error( + ~r"invalid \"rescue\" block in \"try\", it expects \"pattern -> expr\" clauses", + fn -> + code = + quote do + try do + e + rescue + [:not, :clauses] + end + end + + expand(code) + end + ) + + assert_compile_error( + ~r"invalid \"rescue\" block in \"try\", it expects \"pattern -> expr\" clauses", + fn -> + code = + quote do + try do + e + rescue + [] + end + end + + expand(code) + end + ) + + assert_compile_error( + ~r"invalid \"catch\" block in \"try\", it expects \"pattern -> expr\" clauses", + fn -> + code = + quote do + try do + e + catch + x + end + end + + expand(code) + end + ) + + assert_compile_error( + ~r"invalid \"catch\" block in \"try\", it expects \"pattern -> expr\" clauses", + fn -> + code = + quote do + try do + e + catch + [:not, :clauses] + end + end + + expand(code) + end + ) + + assert_compile_error( + ~r"invalid \"catch\" block in \"try\", it expects \"pattern -> expr\" clauses", + fn -> + code = + quote do + try do + e + catch + [] + end + end + + expand(code) + end + ) + + assert_compile_error( + ~r"invalid \"else\" block in \"try\", it expects \"pattern -> expr\" clauses", + fn -> + code = + quote do + try do + e + catch + _ -> :ok + else + x + end + end + + expand(code) + end + ) + + assert_compile_error( + ~r"invalid \"else\" block in \"try\", it expects \"pattern -> expr\" clauses", + fn -> + code = + quote do + try do + e + catch + _ -> :ok + else + [:not, :clauses] + end + end + + expand(code) + end + ) + + assert_compile_error( + ~r"invalid \"else\" block in \"try\", it expects \"pattern -> expr\" clauses", + fn -> + code = + quote do + try do + e + catch + _ -> :ok + else + [] + end + end + + expand(code) + end + ) + end + end + + describe "bitstrings" do + test "parallel match" do + assert expand(quote(do: <> = <>)) |> clean_meta([:alignment]) == + quote(do: <> = <>) + |> clean_bit_modifiers() + + assert expand(quote(do: <> = baz = <>)) |> clean_meta([:alignment]) == + quote(do: <> = baz = <>) + |> clean_bit_modifiers() + + assert expand(quote(do: <> = <> = baz)) |> clean_meta([:alignment]) == + quote(do: <> = <> = baz()) + |> clean_bit_modifiers() + end + + test "invalid match" do + assert_compile_error( + "a bitstring only accepts binaries, numbers, and variables inside a match", + fn -> + expand(quote(do: <<%{}>> = foo())) + end + ) + end + + test "nested match" do + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) |> clean_bit_modifiers() + + assert expand(quote(do: <> = rest()::binary>>)) + |> clean_meta([:alignment]) == + quote(do: <<45::integer, <<_::integer, _::binary>> = rest()::binary>>) + |> clean_bit_modifiers() + end + + test "inlines binaries inside interpolation" do + import Kernel.ExpansionTarget + + # Check expansion happens only once + assert expand(quote(do: "foo#{message_hello("bar")}")) |> clean_meta([:alignment]) == + quote(do: <<"foo"::binary, "bar"::binary>>) |> clean_bit_modifiers() + + assert_received :hello + refute_received :hello + + # And it also works in match + assert expand(quote(do: "foo#{bar()}" = "foobar")) |> clean_meta([:alignment]) == + quote(do: <<"foo"::binary, "bar"::binary>> = "foobar") + |> clean_bit_modifiers() + end + + test "inlines binaries inside interpolation is isomorphic after manual expansion" do + import Kernel.ExpansionTarget + + quoted = Macro.prewalk(quote(do: "foo#{bar()}" = "foobar"), &Macro.expand(&1, __ENV__)) + + assert expand(quoted) |> clean_meta([:alignment]) == + quote(do: <<"foo"::binary, "bar"::binary>> = "foobar") + |> clean_bit_modifiers() + end + + test "expands size * unit" do + import Kernel, except: [-: 1, -: 2] + import Kernel.ExpansionTarget + + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) |> clean_bit_modifiers() + + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) |> clean_bit_modifiers() + + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) |> clean_bit_modifiers() + + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) |> clean_bit_modifiers() + + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) |> clean_bit_modifiers() + + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) |> clean_bit_modifiers() + + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) |> clean_bit_modifiers() + + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) |> clean_bit_modifiers() + + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) |> clean_bit_modifiers() + end + + test "expands binary/bitstring specifiers" do + import Kernel, except: [-: 1, -: 2] + + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) |> clean_bit_modifiers() + + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) |> clean_bit_modifiers() + + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) |> clean_bit_modifiers() + + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) |> clean_bit_modifiers() + + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) |> clean_bit_modifiers() + + message = ~r"signed and unsigned specifiers are supported only on integer and float type" + + assert_compile_error(message, fn -> + expand(quote(do: <>)) + end) + end + + test "expands utf* specifiers" do + import Kernel, except: [-: 1, -: 2] + + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) |> clean_bit_modifiers() + + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) |> clean_bit_modifiers() + + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) |> clean_bit_modifiers() + + message = ~r"signed and unsigned specifiers are supported only on integer and float type" + + assert_compile_error(message, fn -> + expand(quote(do: <>)) + end) + + assert_compile_error(~r"size and unit are not supported on utf types", fn -> + expand(quote(do: <>)) + end) + end + + test "expands numbers specifiers" do + import Kernel, except: [-: 1, -: 2] + + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) |> clean_bit_modifiers() + + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) |> clean_bit_modifiers() + + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) |> clean_bit_modifiers() + + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) |> clean_bit_modifiers() + + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) |> clean_bit_modifiers() + + message = + ~r"integer and float types require a size specifier if the unit specifier is given" + + assert_compile_error(message, fn -> + expand(quote(do: <>)) + end) + end + + test "expands macro specifiers" do + import Kernel, except: [-: 1, -: 2] + import Kernel.ExpansionTarget + + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) |> clean_bit_modifiers() + + assert expand(quote(do: <> = 1)) + |> clean_meta([:alignment]) == + quote(do: <> = 1) + |> clean_bit_modifiers() + end + + test "expands macro in args" do + import Kernel, except: [-: 1, -: 2] + + before_expansion = + quote do + require Kernel.ExpansionTarget + <> + end + + after_expansion = + quote do + :"Elixir.Kernel.ExpansionTarget" + <> + end + + assert expand(before_expansion) |> clean_meta([:alignment]) == + clean_bit_modifiers(after_expansion) + end + + test "supports dynamic size" do + import Kernel, except: [-: 1, -: 2] + + before_expansion = + quote do + var = 1 + <> + end + + after_expansion = + quote do + var = 1 + <> + end + + assert expand(before_expansion) |> clean_meta([:alignment]) == + clean_bit_modifiers(after_expansion) + end + + defmacro offset(size, binary) do + quote do + offset = unquote(size) + <<_::size(^offset)>> = unquote(binary) + end + end + + test "supports size from counters" do + assert offset(8, <<0>>) + end + + test "merges bitstrings" do + import Kernel, except: [-: 1, -: 2] + + assert expand(quote(do: <>, z>>)) |> clean_meta([:alignment]) == + quote(do: <>) + |> clean_bit_modifiers() + + assert expand(quote(do: <>::bitstring, z>>)) + |> clean_meta([:alignment]) == + quote(do: <>) + |> clean_bit_modifiers() + end + + test "merges binaries" do + import Kernel, except: [-: 1, -: 2] + + assert expand(quote(do: "foo" <> x)) |> clean_meta([:alignment]) == + quote(do: <<"foo"::binary, x()::binary>>) |> clean_bit_modifiers() + + assert expand(quote(do: "foo" <> <>)) |> clean_meta([:alignment]) == + quote(do: <<"foo"::binary, x()::integer-size(4), y()::integer-size(4)>>) + |> clean_bit_modifiers() + + assert expand(quote(do: <<"foo", <>::binary>>)) + |> clean_meta([:alignment]) == + quote(do: <<"foo"::binary, x()::integer-size(4), y()::integer-size(4)>>) + |> clean_bit_modifiers() + end + + test "guard expressions on size" do + import Kernel, except: [-: 1, -: 2, +: 1, +: 2, length: 1] + + # Arithmetic operations with literals and variables are valid expressions + # for bitstring size in OTP 23+ + + before_expansion = + quote do + var = 1 + <> + end + + after_expansion = + quote do + var = 1 + <> + end + + assert expand(before_expansion) |> clean_meta([:alignment]) == + clean_bit_modifiers(after_expansion) + + # Other valid guard expressions are also legal for bitstring size in OTP 23+ + + before_expansion = quote(do: <>) + after_expansion = quote(do: <>) + + assert expand(before_expansion) |> clean_meta([:alignment]) == + clean_bit_modifiers(after_expansion) + end + + test "map lookup on size" do + import Kernel, except: [-: 1, -: 2] + + before_expansion = + quote do + var = %{foo: 3} + <> + end + + after_expansion = + quote do + var = %{foo: 3} + <> + end + + assert expand(before_expansion) |> clean_meta([:alignment]) == + clean_bit_modifiers(after_expansion) + end + + test "raises on unaligned binaries in match" do + message = ~r"its number of bits is not divisible by 8" + + assert_compile_error(message, fn -> + expand(quote(do: <> <> _ = "foo")) + end) + + assert_compile_error(message, fn -> + expand(quote(do: <<1::4>> <> "foo")) + end) + end + + test "raises on size or unit for literal bitstrings" do + message = ~r"literal <<>> in bitstring supports only type specifiers" + + assert_compile_error(message, fn -> + expand(quote(do: <<(<<"foo">>)::32>>)) + end) + end + + test "raises on size or unit for literal strings" do + message = ~r"literal string in bitstring supports only endianness and type specifiers" + + assert_compile_error(message, fn -> + expand(quote(do: <<"foo"::32>>)) + end) + end + + test "16-bit floats" do + import Kernel, except: [-: 1, -: 2] + + assert expand(quote(do: <<12.3::float-16>>)) |> clean_meta([:alignment]) == + quote(do: <<12.3::float-size(16)>>) |> clean_bit_modifiers() + end + + test "raises for invalid size * unit for floats" do + message = ~r"float requires size\*unit to be 16, 32, or 64 \(default\), got: 128" + + assert_compile_error(message, fn -> + expand(quote(do: <<12.3::32*4>>)) + end) + + message = ~r"float requires size\*unit to be 16, 32, or 64 \(default\), got: 256" + + assert_compile_error(message, fn -> + expand(quote(do: <<12.3::256>>)) + end) + end + + test "raises for invalid size" do + assert_compile_error(~r/undefined variable "foo"/, fn -> + code = + quote do + fn <<_::size(foo)>> -> :ok end + end + + expand(code, []) + end) + + assert_compile_error(~r/undefined variable "foo"/, fn -> + code = + quote do + fn <<_::size(foo), foo::size(8)>> -> :ok end + end + + expand(code, []) + end) + + assert_compile_error(~r/undefined variable "foo"/, fn -> + code = + quote do + fn foo, <<_::size(foo)>> -> :ok end + end + + expand(code, []) + end) + + assert_compile_error(~r/undefined variable "foo"/, fn -> + code = + quote do + fn foo, <<_::size(foo + 1)>> -> :ok end + end + + expand(code, []) + end) + + assert_compile_error( + ~r"cannot find or invoke local foo/0 inside a bitstring size specifier", + fn -> + code = + quote do + fn <<_::size(foo())>> -> :ok end + end + + expand(code, []) + end + ) + + message = ~r"anonymous call is not allowed inside a bitstring size specifier" + + assert_compile_error(message, fn -> + code = + quote do + fn <<_::size(foo.())>> -> :ok end + end + + expand(code, []) + end) + + message = ~r"cannot invoke remote function inside a bitstring size specifier" + + assert_compile_error(message, fn -> + code = + quote do + foo = %{bar: true} + fn <<_::size(foo.bar())>> -> :ok end + end + + expand(code, []) + end) + + message = ~r"cannot invoke remote function Foo.bar/0 inside a bitstring size specifier" + + assert_compile_error(message, fn -> + code = + quote do + fn <<_::size(Foo.bar())>> -> :ok end + end + + expand(code, []) + end) + end + + test "raises for variable used both in pattern and size" do + assert_compile_error(~r/undefined variable "foo"/, fn -> + code = + quote do + fn <> -> :ok end + end + + expand(code, []) + end) + end + + test "raises for invalid unit" do + message = ~r"unit in bitstring expects an integer as argument, got: :oops" + + assert_compile_error(message, fn -> + expand(quote(do: <<"foo"::size(8)-unit(:oops)>>)) + end) + end + + test "raises for unknown specifier" do + assert_compile_error(~r"unknown bitstring specifier: unknown()", fn -> + expand(quote(do: <<1::unknown()>>)) + end) + end + + test "raises for conflicting specifiers" do + assert_compile_error(~r"conflicting endianness specification for bit field", fn -> + expand(quote(do: <<1::little-big>>)) + end) + + assert_compile_error(~r"conflicting unit specification for bit field", fn -> + expand(quote(do: <>)) + end) + end + + test "raises on binary fields with size in matches" do + assert expand(quote(do: <> = "foobar")) + + message = ~r"a binary field without size is only allowed at the end of a binary pattern" + + assert_compile_error(message, fn -> + expand(quote(do: <> = "foobar")) + end) + + assert_compile_error(message, fn -> + expand(quote(do: <<(<>), y::binary>> = "foobar")) + end) + + assert_compile_error(message, fn -> + expand(quote(do: <<(<>), y::bitstring>> = "foobar")) + end) + + assert_compile_error(message, fn -> + expand(quote(do: <<(<>)::bitstring, y::bitstring>> = "foobar")) + end) + end + end + + describe "op ambiguity" do + test "raises when a call is ambiguous" do + # We use string_to_quoted! here to avoid the formatter adding parentheses + message = ~r["a -1" looks like a function call but there is a variable named "a"] + + assert_compile_error(message, fn -> + code = + Code.string_to_quoted!(""" + a = 1 + a -1 + """) + + expand(code) + end) + + message = + ~r["a -1\.\.\(a \+ 1\)" looks like a function call but there is a variable named "a"] + + assert_compile_error(message, fn -> + code = + Code.string_to_quoted!(""" + a = 1 + a -1 .. a + 1 + """) + + expand(code) + end) + end + end + + test "handles invalid expressions" do + assert_compile_error(~r"invalid quoted expression: {1, 2, 3}", fn -> + expand_env({1, 2, 3}, __ENV__) + end) + + assert_compile_error(~r"invalid quoted expression: #Function\<", fn -> + expand({:sample, fn -> nil end}) + end) + + assert_compile_error(~r"invalid pattern in match", fn -> + code = + quote do + x = & &1 + + case true do + x.(false) -> true + end + end + + expand(code) + end) + + assert_compile_error(~r"anonymous call is not allowed in guards", fn -> + code = + quote do + x = & &1 + + case true do + true when x.(true) -> true + end + end + + expand(code) + end) + + assert_compile_error(~r"invalid call foo\(1\)\(2\)", fn -> + expand(quote(do: foo(1)(2))) + end) + + assert_compile_error(~r"invalid call 1\.foo", fn -> + expand(quote(do: 1.foo)) + end) + + assert_compile_error(~r"invalid call 0\.foo", fn -> + expand(quote(do: __ENV__.line.foo)) + end) + + assert_compile_error(~r"misplaced operator ->", fn -> + expand(quote(do: (foo -> bar))) + end) + end + + ## Helpers + + defmacro thirteen do + 13 + end + + defp assert_compile_error(message, fun) do + assert capture_io(:stderr, fn -> + assert_raise CompileError, fun + end) =~ message + end + + defp clean_meta(expr, vars) do + cleaner = &Keyword.drop(&1, vars) + Macro.prewalk(expr, &Macro.update_meta(&1, cleaner)) + end + + @bitstring_modifiers [ + :integer, + :float, + :binary, + :utf8, + :utf16, + :utf32, + :native, + :signed, + :bitstring, + :little + ] + + defp clean_bit_modifiers(expr) do + Macro.prewalk(expr, fn + {expr, meta, atom} when expr in @bitstring_modifiers and is_atom(atom) -> + {expr, meta, nil} + + other -> + other + end) + end + + defp expand(expr, extra_meta \\ [if_undefined: :apply]) do + add_meta = &Keyword.merge(&1, extra_meta) + + expr + |> Macro.postwalk(&Macro.update_meta(&1, add_meta)) + |> expand_env(__ENV__) + |> elem(0) + end + + defp expand_env(expr, env, to_clean \\ [:version, :inferred_bitstring_spec, :if_undefined]) do + {{expr, scope, env}, _capture} = + with_io(:stderr, fn -> + :elixir_expand.expand(expr, :elixir_env.env_to_ex(env), env) + end) + + env = :elixir_env.to_caller({env.line, scope, env}) + {clean_meta(expr, to_clean), env} end end diff --git a/lib/elixir/test/elixir/kernel/fn_test.exs b/lib/elixir/test/elixir/kernel/fn_test.exs index 5e934d4ca9d..eac16496319 100644 --- a/lib/elixir/test/elixir/kernel/fn_test.exs +++ b/lib/elixir/test/elixir/kernel/fn_test.exs @@ -1,47 +1,81 @@ -Code.require_file "../test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) defmodule Kernel.FnTest do use ExUnit.Case, async: true - import CompileAssertion test "arithmetic constants on match" do - assert (fn 1 + 2 -> :ok end).(3) == :ok - assert (fn 1 - 2 -> :ok end).(-1) == :ok - assert (fn -1 -> :ok end).(-1) == :ok - assert (fn +1 -> :ok end).(1) == :ok + assert (fn -1 -> true end).(-1) + assert (fn +1 -> true end).(1) + end + + defp fun_match(x) do + fn + ^x -> true + _ -> false + end + end + + test "pin operator on match" do + refute fun_match(1).(0) + assert fun_match(1).(1) + refute fun_match(1).(1.0) + end + + test "guards with no args" do + fun = fn () when node() == :nonode@nohost -> true end + assert is_function(fun, 0) + end + + test "case function hoisting does not affect anonymous fns" do + result = + if atom?(0) do + user = :defined + user + else + (fn -> + user = :undefined + user + end).() + end + + assert result == :undefined end test "capture with access" do - assert (&(&1[:hello])).([hello: :world]) == :world + assert (& &1[:hello]).(hello: :world) == :world end test "capture remote" do - assert (&:erlang.atom_to_list/1).(:a) == 'a' - assert (&Atom.to_char_list/1).(:a) == 'a' + assert (&:erlang.atom_to_list/1).(:a) == ~c"a" + assert (&Atom.to_charlist/1).(:a) == ~c"a" assert (&List.flatten/1).([[0]]) == [0] - assert (&(List.flatten/1)).([[0]]) == [0] + assert (&List.flatten/1).([[0]]) == [0] assert (&List.flatten(&1)).([[0]]) == [0] assert (&List.flatten(&1)) == (&List.flatten/1) end test "capture local" do - assert (&atl/1).(:a) == 'a' - assert (&(atl/1)).(:a) == 'a' - assert (&atl(&1)).(:a) == 'a' + assert (&atl/1).(:a) == ~c"a" + assert (&atl/1).(:a) == ~c"a" + assert (&atl(&1)).(:a) == ~c"a" end test "capture local with question mark" do - assert (&is_a?/2).(:atom, :a) - assert (&(is_a?/2)).(:atom, :a) - assert (&is_a?(&1, &2)).(:atom, :a) + assert (&atom?/1).(:a) + assert (&atom?/1).(:a) + assert (&atom?(&1)).(:a) end test "capture imported" do assert (&is_atom/1).(:a) - assert (&(is_atom/1)).(:a) + assert (&is_atom/1).(:a) assert (&is_atom(&1)).(:a) - assert (&is_atom(&1)) == &is_atom/1 + assert (&is_atom(&1)) == (&is_atom/1) end test "capture macro" do @@ -52,26 +86,27 @@ defmodule Kernel.FnTest do end test "capture operator" do - assert is_function &+/2 - assert is_function &(&&/2) - assert is_function & &1 + &2, 2 + assert is_function(&+/2) + assert is_function(& &&/2) + assert is_function(&(&1 + &2), 2) + assert is_function(&and/2) end test "capture with variable module" do mod = List assert (&mod.flatten(&1)).([1, [2], 3]) == [1, 2, 3] assert (&mod.flatten/1).([1, [2], 3]) == [1, 2, 3] - assert (&mod.flatten/1) == &List.flatten/1 + assert (&mod.flatten/1) == (&List.flatten/1) end test "local partial application" do assert (&atb(&1, :utf8)).(:a) == "a" - assert (&atb(List.to_atom(&1), :utf8)).('a') == "a" + assert (&atb(List.to_atom(&1), :utf8)).(~c"a") == "a" end test "imported partial application" do import Record - assert (&record?(&1, :sample)).({:sample, 1}) + assert (&is_record(&1, :sample)).({:sample, 1}) end test "remote partial application" do @@ -88,62 +123,42 @@ defmodule Kernel.FnTest do end test "capture and partially apply lists" do - assert (&[ &1, &2 ]).(1, 2) == [ 1, 2 ] - assert (&[ &1, &2, &3 ]).(1, 2, 3) == [ 1, 2, 3 ] + assert (&[&1, &2]).(1, 2) == [1, 2] + assert (&[&1, &2, &3]).(1, 2, 3) == [1, 2, 3] - assert (&[ 1, &1 ]).(2) == [ 1, 2 ] - assert (&[ 1, &1, &2 ]).(2, 3) == [ 1, 2, 3 ] + assert (&[1, &1]).(2) == [1, 2] + assert (&[1, &1, &2]).(2, 3) == [1, 2, 3] - assert (&[&1|&2]).(1, 2) == [1|2] + assert (&[&1 | &2]).(1, 2) == [1 | 2] end test "capture and partially apply on call" do - assert (&(&1.module)).(__ENV__) == __MODULE__ + assert (& &1.module).(__ENV__) == __MODULE__ end test "capture block like" do assert (&(!is_atom(&1))).(:foo) == false end - test "capture other" do + test "capture with function call" do assert (& &1).(:ok) == :ok fun = fn a, b -> a + b end assert (&fun.(&1, 2)).(1) == 3 end - test "failure on non-continuous" do - assert_compile_fail CompileError, "nofile:1: capture &2 cannot be defined without &1", "&(&2)" - end - - test "failure on integers" do - assert_compile_fail CompileError, "nofile:1: unhandled &1 outside of a capture", "&1" - assert_compile_fail CompileError, "nofile:1: capture &0 is not allowed", "&foo(&0)" - end - - test "failure on block" do - assert_compile_fail CompileError, - "nofile:1: invalid args for &, block expressions " <> - "are not allowed, got: (\n 1\n 2\n)", - "&(1;2)" - end - - test "failure on other types" do - assert_compile_fail CompileError, - "nofile:1: invalid args for &, expected an expression in the format of &Mod.fun/arity, " <> - "&local/arity or a capture containing at least one argument as &1, got: :foo", - "&:foo" + defmacro c(x) do + quote do + &(unquote(x) <> &1) + end end - test "failure when no captures" do - assert_compile_fail CompileError, - "nofile:1: invalid args for &, expected an expression in the format of &Mod.fun/arity, " <> - "&local/arity or a capture containing at least one argument as &1, got: foo()", - "&foo()" + test "capture within capture through macro" do + assert (&c(&1).("b")).("a") == "ab" end - defp is_a?(:atom, atom) when is_atom(atom), do: true - defp is_a?(_, _), do: false + defp atom?(atom) when is_atom(atom), do: true + defp atom?(_), do: false defp atl(arg) do :erlang.atom_to_list(arg) diff --git a/lib/elixir/test/elixir/kernel/guard_test.exs b/lib/elixir/test/elixir/kernel/guard_test.exs new file mode 100644 index 00000000000..342d81638fa --- /dev/null +++ b/lib/elixir/test/elixir/kernel/guard_test.exs @@ -0,0 +1,499 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Kernel.GuardTest do + use ExUnit.Case, async: true + + import ExUnit.CaptureIO + + describe "defguard(p) usage" do + defmodule GuardsInMacros do + defguard is_foo(atom) when atom == :foo + + defmacro is_compile_time_foo(atom) when is_foo(atom) do + quote do: unquote(__MODULE__).is_foo(unquote(atom)) + end + end + + test "guards can be used in other macros in the same module" do + require GuardsInMacros + assert GuardsInMacros.is_foo(:foo) + refute GuardsInMacros.is_foo(:baz) + assert GuardsInMacros.is_compile_time_foo(:foo) + end + + defmodule GuardsInFuns do + defguard is_foo(atom) when atom == :foo + defguard is_equal(foo, bar) when foo == bar + + def foo?(atom) when is_foo(atom) do + is_foo(atom) + end + end + + test "guards can be used in other funs in the same module" do + require GuardsInFuns + assert GuardsInFuns.is_foo(:foo) + refute GuardsInFuns.is_foo(:bar) + end + + test "guards do not change code evaluation semantics" do + require GuardsInFuns + x = 1 + assert GuardsInFuns.is_equal(x = 2, x) == false + assert x == 2 + end + + defmodule MacrosInGuards do + defmacro is_foo(atom) do + quote do + unquote(atom) == :foo + end + end + + defguard is_foobar(atom) when is_foo(atom) or atom == :bar + end + + test "macros can be used in other guards in the same module" do + require MacrosInGuards + assert MacrosInGuards.is_foobar(:foo) + assert MacrosInGuards.is_foobar(:bar) + refute MacrosInGuards.is_foobar(:baz) + end + + defmodule UnquotedInGuardCall do + @value :foo + + defguard unquote(String.to_atom("is_#{@value}"))(x) when x == unquote(@value) + end + + test "guards names can be defined dynamically using unquote" do + require UnquotedInGuardCall + assert UnquotedInGuardCall.is_foo(:foo) + refute UnquotedInGuardCall.is_foo(:bar) + end + + defmodule GuardsInGuards do + defguard is_foo(atom) when atom == :foo + defguard is_foobar(atom) when is_foo(atom) or atom == :bar + end + + test "guards can be used in other guards in the same module" do + require GuardsInGuards + assert GuardsInGuards.is_foobar(:foo) + assert GuardsInGuards.is_foobar(:bar) + refute GuardsInGuards.is_foobar(:baz) + end + + defmodule DefaultArgs do + defguard is_divisible(value, remainder \\ 2) + when is_integer(value) and rem(value, remainder) == 0 + end + + test "permits default values in args" do + require DefaultArgs + assert DefaultArgs.is_divisible(2) + refute DefaultArgs.is_divisible(1) + assert DefaultArgs.is_divisible(3, 3) + refute DefaultArgs.is_divisible(3, 4) + end + + test "doesn't allow matching in args" do + assert_raise ArgumentError, ~r"invalid syntax in defguard", fn -> + defmodule Integer.Args do + defguard foo(value, 1) when is_integer(value) + end + end + + assert_raise ArgumentError, ~r"invalid syntax in defguard", fn -> + defmodule String.Args do + defguard foo(value, "string") when is_integer(value) + end + end + + assert_raise ArgumentError, ~r"invalid syntax in defguard", fn -> + defmodule Atom.Args do + defguard foo(value, :atom) when is_integer(value) + end + end + + assert_raise ArgumentError, ~r"invalid syntax in defguard", fn -> + defmodule Tuple.Args do + defguard foo(value, {foo, bar}) when is_integer(value) + end + end + end + + defmodule GuardFromMacro do + defmacro __using__(_) do + quote do + defguard is_even(value) when is_integer(value) and rem(value, 2) == 0 + end + end + end + + test "defguard defines a guard from inside another macro" do + defmodule UseGuardFromMacro do + use GuardFromMacro + + def assert! do + assert is_even(0) + refute is_even(1) + end + end + + UseGuardFromMacro.assert!() + end + + defmodule IntegerPrivateGuards do + defguardp is_even(value) when is_integer(value) and rem(value, 2) == 0 + + def even_and_large?(value) when is_even(value) and value > 100, do: true + def even_and_large?(_), do: false + + def even_and_small?(value) do + if is_even(value) and value <= 100, do: true, else: false + end + end + + test "defguardp defines private guards that work inside and outside guard clauses" do + assert IntegerPrivateGuards.even_and_large?(102) + refute IntegerPrivateGuards.even_and_large?(98) + refute IntegerPrivateGuards.even_and_large?(99) + refute IntegerPrivateGuards.even_and_large?(103) + + assert IntegerPrivateGuards.even_and_small?(98) + refute IntegerPrivateGuards.even_and_small?(99) + refute IntegerPrivateGuards.even_and_small?(102) + refute IntegerPrivateGuards.even_and_small?(103) + + assert_compile_error(~r"cannot find or invoke local is_even/1", fn -> + defmodule IntegerPrivateGuardUtils do + import IntegerPrivateGuards + + def even_and_large?(value) when is_even(value) and value > 100, do: true + def even_and_large?(_), do: false + end + end) + + assert_compile_error(~r"undefined function is_even/1", fn -> + defmodule IntegerPrivateFunctionUtils do + import IntegerPrivateGuards + + def even_and_small?(value) do + if is_even(value) and value <= 100, do: true, else: false + end + end + end) + end + + test "requires a proper macro name" do + assert_raise ArgumentError, ~r"invalid syntax in defguard", fn -> + defmodule(LiteralUsage, do: defguard("literal is bad")) + end + + assert_raise ArgumentError, ~r"invalid syntax in defguard", fn -> + defmodule(RemoteUsage, do: defguard(Remote.call(is_bad))) + end + end + + test "handles overriding appropriately" do + assert_compile_error(~r"defmacro (.*?) already defined as def", fn -> + defmodule OverriddenFunUsage do + def foo(bar), do: bar + defguard foo(bar) when bar + end + end) + + assert_compile_error(~r"defmacro (.*?) already defined as defp", fn -> + defmodule OverriddenPrivateFunUsage do + defp foo(bar), do: bar + defguard foo(bar) when bar + end + end) + + assert_compile_error(~r"defmacro (.*?) already defined as defmacrop", fn -> + defmodule OverriddenPrivateFunUsage do + defmacrop foo(bar), do: bar + defguard foo(bar) when bar + end + end) + + assert_compile_error(~r"defmacrop (.*?) already defined as def", fn -> + defmodule OverriddenFunUsage do + def foo(bar), do: bar + defguardp foo(bar) when bar + end + end) + + assert_compile_error(~r"defmacrop (.*?) already defined as defp", fn -> + defmodule OverriddenPrivateFunUsage do + defp foo(bar), do: bar + defguardp foo(bar) when bar + end + end) + + assert_compile_error(~r"defmacrop (.*?) already defined as defmacro", fn -> + defmodule OverriddenPrivateFunUsage do + defmacro foo(bar), do: bar + defguardp foo(bar) when bar + end + end) + end + + test "does not allow multiple guard clauses" do + assert_raise ArgumentError, ~r"invalid syntax in defguard", fn -> + defmodule MultiGuardUsage do + defguardp foo(bar, baz) when bar == 1 when baz == 2 + end + end + end + + test "does not accept a block" do + assert_compile_error(~r"undefined function defguard/2", fn -> + defmodule OnelinerBlockUsage do + defguard(foo(bar), do: one_liner) + end + end) + + assert_compile_error(~r"undefined function defguard/2", fn -> + defmodule MultilineBlockUsage do + defguard foo(bar) do + multi + liner + end + end + end) + + assert_compile_error(~r"undefined function defguard/2", fn -> + defmodule ImplAndBlockUsage do + defguard(foo(bar) when both_given, do: error) + end + end) + end + end + + describe "defguard(p) compilation" do + test "refuses to compile nonsensical code" do + assert_compile_error("cannot find or invoke local undefined/1", fn -> + defmodule UndefinedUsage do + defguard foo(function) when undefined(function) + end + end) + end + + test "fails on expressions not allowed in guards" do + # Slightly unique errors + + assert_raise ArgumentError, ~r{invalid right argument for operator "in"}, fn -> + defmodule RuntimeListUsage do + defguard foo(bar, baz) when bar in baz + end + end + + assert_compile_error("cannot invoke remote function", fn -> + defmodule BadErlangFunctionUsage do + defguard foo(bar) when :erlang.binary_to_atom("foo") + end + end) + + assert_compile_error("cannot invoke remote function", fn -> + defmodule SendUsage do + defguard foo(bar) when send(self(), :baz) + end + end) + + # Consistent errors + + assert_raise ArgumentError, ~r"invalid expression in guard, ! is not allowed", fn -> + defmodule SoftNegationLogicUsage do + defguard foo(logic) when !logic + end + end + + assert_raise ArgumentError, ~r"invalid expression in guard, && is not allowed", fn -> + defmodule SoftAndLogicUsage do + defguard foo(soft, logic) when soft && logic + end + end + + assert_raise ArgumentError, ~r"invalid expression in guard, || is not allowed", fn -> + defmodule SoftOrLogicUsage do + defguard foo(soft, logic) when soft || logic + end + end + + assert_compile_error( + "cannot invoke remote function :erlang\.is_record/2 inside a guard", + fn -> + defmodule IsRecord2Usage do + defguard foo(rec) when :erlang.is_record(rec, :tag) + end + end + ) + + assert_compile_error( + "cannot invoke remote function :erlang\.is_record/3 inside a guard", + fn -> + defmodule IsRecord3Usage do + defguard foo(rec) when :erlang.is_record(rec, :tag, 7) + end + end + ) + + assert_compile_error( + ~r"cannot invoke remote function :erlang\.\+\+/2 inside a guard", + fn -> + defmodule ListSubtractionUsage do + defguard foo(list) when list ++ [] + end + end + ) + + assert_compile_error( + "cannot invoke remote function :erlang\.\-\-/2 inside a guard", + fn -> + defmodule ListSubtractionUsage do + defguard foo(list) when list -- [] + end + end + ) + + assert_compile_error("invalid expression in guard", fn -> + defmodule LocalCallUsage do + defguard foo(local, call) when local.(call) + end + end) + + assert_compile_error("invalid expression in guard", fn -> + defmodule ComprehensionUsage do + defguard foo(bar) when for(x <- [1, 2, 3], do: x * bar) + end + end) + + assert_compile_error("invalid expression in guard", fn -> + defmodule AliasUsage do + defguard foo(bar) when alias(bar) + end + end) + + assert_compile_error("invalid expression in guard", fn -> + defmodule ImportUsage do + defguard foo(bar) when import(bar) + end + end) + + assert_compile_error("invalid expression in guard", fn -> + defmodule RequireUsage do + defguard foo(bar) when require(bar) + end + end) + + assert_compile_error("invalid expression in guard", fn -> + defmodule SuperUsage do + defguard foo(bar) when super(bar) + end + end) + + assert_compile_error("invalid expression in guard", fn -> + defmodule SpawnUsage do + defguard foo(bar) when spawn(& &1) + end + end) + + assert_compile_error("invalid expression in guard", fn -> + defmodule ReceiveUsage do + defguard foo(bar) when receive(do: (baz -> baz)) + end + end) + + assert_compile_error("invalid expression in guard", fn -> + defmodule CaseUsage do + defguard foo(bar) when case(bar, do: (baz -> :baz)) + end + end) + + assert_compile_error("invalid expression in guard", fn -> + defmodule CondUsage do + defguard foo(bar) when cond(do: (bar -> :baz)) + end + end) + + assert_compile_error("invalid expression in guard", fn -> + defmodule TryUsage do + defguard foo(bar) when try(do: (baz -> baz)) + end + end) + + assert_compile_error("invalid expression in guard", fn -> + defmodule WithUsage do + defguard foo(bar) when with(do: (baz -> baz)) + end + end) + + assert_compile_error( + "cannot invoke remote function inside a guard. " <> + "If you want to do a map lookup instead, please remove parens from map.field()", + fn -> + defmodule MapDot do + def map_dot(map) when map.field(), do: true + end + end + ) + + assert_compile_error("cannot invoke remote function Module.fun/0 inside a guard", fn -> + defmodule MapDot do + def map_dot(map) when Module.fun(), do: true + end + end) + end + end + + describe "defguard(p) expansion" do + defguard with_unused_vars(foo, bar, _baz) when foo + bar + + test "doesn't obscure unused variables" do + args = quote(do: [1 + 1, 2 + 2, 3 + 3]) + + assert expand_defguard_to_string(:with_unused_vars, args, :guard) == """ + :erlang.+(1 + 1, 2 + 2) + """ + + assert expand_defguard_to_string(:with_unused_vars, args, nil) == """ + {arg1, arg2} = {1 + 1, 2 + 2} + :erlang.+(arg1, arg2) + """ + end + + defguard with_reused_vars(foo, bar, baz) when foo + foo + bar + baz + + test "handles re-used variables" do + args = quote(do: [1 + 1, 2 + 2, 3 + 3]) + + assert expand_defguard_to_string(:with_reused_vars, args, :guard) == """ + :erlang.+(:erlang.+(:erlang.+(1 + 1, 1 + 1), 2 + 2), 3 + 3) + """ + + assert expand_defguard_to_string(:with_reused_vars, args, nil) == """ + {arg1, arg2, arg3} = {1 + 1, 2 + 2, 3 + 3} + :erlang.+(:erlang.+(:erlang.+(arg1, arg1), arg2), arg3) + """ + end + + defp expand_defguard_to_string(fun, args, context) do + {{:., [], [__MODULE__, fun]}, [], args} + |> Macro.expand(%{__ENV__ | context: context}) + |> Macro.to_string() + |> Kernel.<>("\n") + end + end + + defp assert_compile_error(message, fun) do + assert capture_io(:stderr, fn -> + assert_raise CompileError, fun + end) =~ message + end +end diff --git a/lib/elixir/test/elixir/kernel/impl_test.exs b/lib/elixir/test/elixir/kernel/impl_test.exs new file mode 100644 index 00000000000..95a0b2e837d --- /dev/null +++ b/lib/elixir/test/elixir/kernel/impl_test.exs @@ -0,0 +1,628 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Kernel.ImplTest do + use ExUnit.Case, async: true + + defp capture_err(fun) do + ExUnit.CaptureIO.capture_io(:stderr, fun) + end + + defp purge(module) do + :code.purge(module) + :code.delete(module) + end + + setup do + on_exit(fn -> purge(Kernel.ImplTest.ImplAttributes) end) + end + + defprotocol AProtocol do + def foo(term) + def bar(term) + end + + defmodule Behaviour do + @callback foo() :: any + end + + defmodule BehaviourWithArgument do + @callback foo(any) :: any + end + + defmodule BehaviourWithThreeArguments do + @callback foo(any, any, any) :: any + end + + defmodule UseBehaviourWithoutImpl do + @callback foo_without_impl() :: any + @callback bar_without_impl() :: any + @callback baz_without_impl() :: any + + defmacro __using__(_opts) do + quote do + @behaviour Kernel.ImplTest.UseBehaviourWithoutImpl + def foo_without_impl(), do: :auto_generated + end + end + end + + defmodule UseBehaviourWithImpl do + @callback foo_with_impl() :: any + @callback bar_with_impl() :: any + @callback baz_with_impl() :: any + + defmacro __using__(_opts) do + quote do + @behaviour Kernel.ImplTest.UseBehaviourWithImpl + @impl true + def foo_with_impl(), do: :auto_generated + def bar_with_impl(), do: :auto_generated + end + end + end + + defmodule MacroBehaviour do + @macrocallback bar :: any + end + + defmodule ManualBehaviour do + def behaviour_info(:callbacks), do: [foo: 0] + def behaviour_info(:optional_callbacks), do: :undefined + end + + test "sets @impl to boolean" do + defmodule ImplAttributes do + @behaviour Behaviour + + @impl true + def foo(), do: :ok + + @impl false + def foo(term) do + term + end + end + end + + test "sets @impl to nil" do + assert_raise ArgumentError, ~r/should be a module or a boolean/, fn -> + defmodule ImplAttributes do + @behaviour Behaviour + @impl nil + def foo(), do: :ok + end + end + end + + test "sets @impl to behaviour" do + defmodule ImplAttributes do + @behaviour Behaviour + @impl Behaviour + def foo(), do: :ok + end + end + + test "does not set @impl" do + defmodule ImplAttributes do + @behaviour Behaviour + def foo(), do: :ok + end + end + + test "sets @impl to boolean on manual behaviour" do + defmodule ImplAttributes do + @behaviour ManualBehaviour + + @impl true + def foo(), do: :ok + end + end + + test "warns of undefined module, but does not warn at @impl line" do + capture_err = + capture_err(fn -> + Code.eval_string(""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Abc + + @impl Abc + def foo(), do: :ok + end + """) + end) + + assert capture_err =~ + "@behaviour Abc does not exist (in module Kernel.ImplTest.ImplAttributes)" + + refute capture_err =~ + "got \"@impl Abc\"" + end + + test "warns of undefined behaviour, but does not warn at @impl line" do + capture_err = + capture_err(fn -> + Code.eval_string(""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Enum + + @impl Enum + def foo(), do: :ok + end + """) + end) + + assert capture_err =~ + "module Enum is not a behaviour (in module Kernel.ImplTest.ImplAttributes)" + + refute capture_err =~ + "got \"@impl Abc\"" + end + + test "warns of callbacks without impl and @impl has been set before" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.Behaviour + @behaviour Kernel.ImplTest.MacroBehaviour + + @impl true + def foo(), do: :ok + + defmacro bar(), do: :ok + end + """) + end) =~ + "module attribute @impl was not set for macro bar/0 callback (specified in Kernel.ImplTest.MacroBehaviour)" + end + + test "warns of callbacks without impl and @impl has been set after" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.Behaviour + @behaviour Kernel.ImplTest.MacroBehaviour + + defmacro bar(), do: :ok + + @impl true + def foo(), do: :ok + end + """) + end) =~ + "module attribute @impl was not set for macro bar/0 callback (specified in Kernel.ImplTest.MacroBehaviour)" + end + + test "warns when @impl is set on private function" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.Behaviour + @impl true + defp foo(), do: :ok + end + """) + end) =~ + "function foo/0 is private, @impl attribute is always discarded for private functions/macros" + end + + test "warns when @impl is set and no function" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.Behaviour + @impl true + end + """) + end) =~ "module attribute @impl was set but no definition follows it" + end + + test "warns of @impl true and no behaviour" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Kernel.ImplTest.ImplAttributes do + @impl true + def foo(), do: :ok + end + """) + end) =~ "got \"@impl true\" for function foo/0 but no behaviour was declared" + end + + test "warns of @impl true with callback name not in behaviour" do + message = + capture_err(fn -> + Code.eval_string(""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.Behaviour + @impl true + def bar(), do: :ok + end + """) + end) + + assert message =~ + "got \"@impl true\" for function bar/0 but no behaviour specifies such callback" + + assert message =~ "The known callbacks are" + assert message =~ "* Kernel.ImplTest.Behaviour.foo/0 (function)" + end + + test "warns of @impl true with macro callback name not in behaviour" do + message = + capture_err(fn -> + Code.eval_string(""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.MacroBehaviour + @impl true + defmacro foo(), do: :ok + end + """) + end) + + assert message =~ + "got \"@impl true\" for macro foo/0 but no behaviour specifies such callback" + + assert message =~ "The known callbacks are" + assert message =~ "* Kernel.ImplTest.MacroBehaviour.bar/0 (macro)" + end + + test "warns of @impl true with callback kind not in behaviour" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.MacroBehaviour + @impl true + def foo(), do: :ok + end + """) + end) =~ + "got \"@impl true\" for function foo/0 but no behaviour specifies such callback" + end + + test "warns of @impl true with wrong arity" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.Behaviour + @impl true + def foo(arg), do: arg + end + """) + end) =~ + "got \"@impl true\" for function foo/1 but no behaviour specifies such callback" + end + + test "warns of @impl false and there are no callbacks" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Kernel.ImplTest.ImplAttributes do + @impl false + def baz(term), do: term + end + """) + end) =~ "got \"@impl false\" for function baz/1 but no behaviour was declared" + end + + test "warns of @impl false and it is a callback" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.Behaviour + @impl false + def foo(), do: :ok + end + """) + end) =~ + "got \"@impl false\" for function foo/0 but it is a callback specified in Kernel.ImplTest.Behaviour" + end + + test "warns of @impl module and no behaviour" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Kernel.ImplTest.ImplAttributes do + @impl Kernel.ImplTest.Behaviour + def foo(), do: :ok + end + """) + end) =~ + "got \"@impl Kernel.ImplTest.Behaviour\" for function foo/0 but no behaviour was declared" + end + + test "warns of @impl module with callback name not in behaviour" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.Behaviour + @impl Kernel.ImplTest.Behaviour + def bar(), do: :ok + end + """) + end) =~ + "got \"@impl Kernel.ImplTest.Behaviour\" for function bar/0 but Kernel.ImplTest.Behaviour does not specify such callback" + end + + test "warns of @impl module with macro callback name not in behaviour" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.MacroBehaviour + @impl Kernel.ImplTest.MacroBehaviour + defmacro foo(), do: :ok + end + """) + end) =~ + "got \"@impl Kernel.ImplTest.MacroBehaviour\" for macro foo/0 but Kernel.ImplTest.MacroBehaviour does not specify such callback" + end + + test "warns of @impl module with macro callback kind not in behaviour" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.MacroBehaviour + @impl Kernel.ImplTest.MacroBehaviour + def foo(), do: :ok + end + """) + end) =~ + "got \"@impl Kernel.ImplTest.MacroBehaviour\" for function foo/0 but Kernel.ImplTest.MacroBehaviour does not specify such callback" + end + + test "warns of @impl module and callback belongs to another known module" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.Behaviour + @behaviour Kernel.ImplTest.MacroBehaviour + @impl Kernel.ImplTest.Behaviour + def bar(), do: :ok + end + """) + end) =~ + "got \"@impl Kernel.ImplTest.Behaviour\" for function bar/0 but Kernel.ImplTest.Behaviour does not specify such callback" + end + + test "warns of @impl module and callback belongs to another unknown module" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.Behaviour + @impl Kernel.ImplTest.MacroBehaviour + def bar(), do: :ok + end + """) + end) =~ + "got \"@impl Kernel.ImplTest.MacroBehaviour\" for function bar/0 but this behaviour was not declared with @behaviour" + end + + test "does not warn of @impl when the function with default conforms with several typespecs" do + Code.eval_string(~S""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.Behaviour + @behaviour Kernel.ImplTest.BehaviourWithArgument + + @impl true + def foo(args \\ []), do: args + end + """) + end + + test "does not warn of @impl when the function conforms to behaviour but has default value for arg" do + Code.eval_string(~S""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.BehaviourWithArgument + + @impl true + def foo(args \\ []), do: args + end + """) + end + + test "does not warn of @impl when the function conforms to behaviour but has additional trailing default args" do + Code.eval_string(~S""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.BehaviourWithArgument + + @impl true + def foo(arg_1, _args \\ []), do: arg_1 + end + """) + end + + test "does not warn of @impl when the function conforms to behaviour but has additional leading default args" do + Code.eval_string(~S""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.BehaviourWithArgument + + @impl true + def foo(_defaulted_arg \\ [], args), do: args + end + """) + end + + test "does not warn of @impl when the function has more args than callback, but they're all defaulted" do + Code.eval_string(~S""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.BehaviourWithArgument + + @impl true + def foo(args \\ [], _bar \\ []), do: args + end + """) + end + + test "does not warn of @impl with defaults when the same function is defined multiple times" do + Code.eval_string(~S""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.BehaviourWithArgument + @behaviour Kernel.ImplTest.BehaviourWithThreeArguments + + @impl Kernel.ImplTest.BehaviourWithArgument + def foo(_foo \\ [], _bar \\ []), do: :ok + + @impl Kernel.ImplTest.BehaviourWithThreeArguments + def foo(_foo, _bar, _baz, _qux \\ []), do: :ok + end + """) + end + + test "does not warn of no @impl when overriding callback" do + Code.eval_string(~S""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.Behaviour + + def foo(), do: :overridable + + defoverridable Kernel.ImplTest.Behaviour + + def foo(), do: :overridden + end + """) + end + + test "does not warn of overridable function missing @impl" do + Code.eval_string(~S""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.Behaviour + + def foo(), do: :overridable + + defoverridable Kernel.ImplTest.Behaviour + + @impl Kernel.ImplTest.Behaviour + def foo(), do: :overridden + end + """) + end + + test "warns correctly of missing @impl only for end-user implemented function" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.Behaviour + @behaviour Kernel.ImplTest.MacroBehaviour + + def foo(), do: :overridable + + defoverridable Kernel.ImplTest.Behaviour + + def foo(), do: :overridden + + @impl true + defmacro bar(), do: :overridden + end + """) + end) =~ + "module attribute @impl was not set for function foo/0 callback (specified in Kernel.ImplTest.Behaviour)" + end + + test "warns correctly of incorrect @impl in overridable callback" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.Behaviour + @behaviour Kernel.ImplTest.MacroBehaviour + + @impl Kernel.ImplTest.MacroBehaviour + def foo(), do: :overridable + + defoverridable Kernel.ImplTest.Behaviour + + @impl Kernel.ImplTest.Behaviour + def foo(), do: :overridden + end + """) + end) =~ + "got \"@impl Kernel.ImplTest.MacroBehaviour\" for function foo/0 but Kernel.ImplTest.MacroBehaviour does not specify such callback" + end + + test "warns only of non-generated functions in non-generated @impl" do + message = + capture_err(fn -> + Code.eval_string(""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.Behaviour + use Kernel.ImplTest.UseBehaviourWithoutImpl + + @impl true + def bar_without_impl(), do: :overridden + def baz_without_impl(), do: :overridden + + defdelegate foo(), to: __MODULE__, as: :baz + def baz(), do: :ok + end + """) + end) + + assert message =~ + "module attribute @impl was not set for function baz_without_impl/0 callback" + + assert message =~ + "module attribute @impl was not set for function foo/0 callback" + + refute message =~ "foo_without_impl/0" + end + + test "warns only of non-generated functions in non-generated @impl in protocols" do + message = + capture_err(fn -> + Code.eval_string(""" + defimpl Kernel.ImplTest.AProtocol, for: List do + @impl true + def foo(_list), do: :ok + + defdelegate bar(list), to: __MODULE__, as: :baz + def baz(_list), do: :ok + end + """) + end) + + assert message =~ + "module attribute @impl was not set for function bar/1 callback" + end + + test "warns only of generated functions in generated @impl" do + message = + capture_err(fn -> + Code.eval_string(""" + defmodule Kernel.ImplTest.ImplAttributes do + use Kernel.ImplTest.UseBehaviourWithImpl + def baz_with_impl(), do: :overridden + end + """) + end) + + assert message =~ "module attribute @impl was not set for function bar_with_impl/0 callback" + refute message =~ "foo_with_impl/0" + end + + test "does not warn of overridable callback when using __before_compile__/1 hook" do + Code.eval_string(~S""" + defmodule BeforeCompile do + defmacro __before_compile__(_) do + quote do + @behaviour Kernel.ImplTest.Behaviour + + def foo(), do: :overridable + + defoverridable Kernel.ImplTest.Behaviour + end + end + end + + defmodule Kernel.ImplTest.ImplAttributes do + @before_compile BeforeCompile + @behaviour Kernel.ImplTest.MacroBehaviour + + defmacro bar(), do: :overridable + + defoverridable Kernel.ImplTest.MacroBehaviour + + @impl Kernel.ImplTest.MacroBehaviour + defmacro bar(), do: :overridden + end + """) + end +end diff --git a/lib/elixir/test/elixir/kernel/import_test.exs b/lib/elixir/test/elixir/kernel/import_test.exs index 2b107cdd83f..900d069f193 100644 --- a/lib/elixir/test/elixir/kernel/import_test.exs +++ b/lib/elixir/test/elixir/kernel/import_test.exs @@ -1,16 +1,41 @@ -Code.require_file "../test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) defmodule Kernel.ImportTest do use ExUnit.Case, async: true + # This should not warn due to the empty only + import URI, only: [] + defmodule ImportAvailable do defmacro flatten do [flatten: 1] end end + test "multi-call" do + assert [List, String] = import(Elixir.{List, unquote(:String)}) + assert keymember?([a: 1], :a, 0) + assert valid?("ø") + end + + test "blank multi-call" do + assert [] = import(List.{}) + # Buggy local duplicate is untouched + assert duplicate([1], 2) == [1] + end + + test "multi-call with options" do + assert [List] = import(Elixir.{List}, only: []) + # Buggy local duplicate is untouched + assert duplicate([1], 2) == [1] + end + test "import all" do - import :lists + assert :lists = import(:lists) assert flatten([1, [2], 3]) == [1, 2, 3] end @@ -20,13 +45,15 @@ defmodule Kernel.ImportTest do end test "import except one" do - import :lists, except: [each: 2] + import :lists, except: [duplicate: 2] assert flatten([1, [2], 3]) == [1, 2, 3] + # Buggy local duplicate is untouched + assert duplicate([1], 2) == [1] end test "import only via macro" do require ImportAvailable - import :lists, only: ImportAvailable.flatten + import :lists, only: ImportAvailable.flatten() assert flatten([1, [2], 3]) == [1, 2, 3] end @@ -35,7 +62,7 @@ defmodule Kernel.ImportTest do end test "import with options via macro" do - import :lists, dynamic_opts + import :lists, dynamic_opts() assert flatten([1, [2], 3]) == [1, 2, 3] end @@ -47,8 +74,24 @@ defmodule Kernel.ImportTest do assert duplicate([1], 2) == [1] end + test "import except none respects previous import with except" do + import :lists, except: [duplicate: 2] + import :lists, except: [] + assert append([1], [2, 3]) == [1, 2, 3] + # Buggy local duplicate is untouched + assert duplicate([1], 2) == [1] + end + + test "import except none respects previous import with only" do + import :lists, only: [append: 2] + import :lists, except: [] + assert append([1], [2, 3]) == [1, 2, 3] + # Buggy local duplicate is untouched + assert duplicate([1], 2) == [1] + end + defmodule Underscored do - def hello(x), do: x + def hello(x), do: x def __underscore__(x), do: x end @@ -61,7 +104,7 @@ defmodule Kernel.ImportTest do assert __underscore__(3) == 3 end - test "import non underscored" do + test "import non-underscored" do import ExplicitUnderscored, only: [__underscore__: 1] import Underscored assert hello(2) == 2 @@ -69,14 +112,14 @@ defmodule Kernel.ImportTest do end defmodule MessedBitwise do - defmacro bnot(x), do: x + defmacro bnot(x), do: x defmacro bor(x, _), do: x end - import Bitwise, only: :macros + import Bitwise, only: :functions - test "conflicing imports with only and except" do - import Bitwise, only: :macros, except: [bnot: 1] + test "conflicting imports with only and except" do + import Bitwise, only: :functions, except: [bnot: 1] import MessedBitwise, only: [bnot: 1] assert bnot(0) == 0 assert bor(0, 1) == 1 @@ -100,15 +143,79 @@ defmodule Kernel.ImportTest do test "import many" do [import(List), import(String)] - assert capitalize("foo") == "Foo" + assert capitalize("foo") == "Foo" assert flatten([1, [2], 3]) == [1, 2, 3] end + test "does not import *_info in Erlang" do + import :gen_server, warn: false + assert Macro.Env.lookup_import(__ENV__, {:module_info, 1}) == [] + assert Macro.Env.lookup_import(__ENV__, {:behaviour_info, 1}) == [] + end + + test "does not import *_info in Elixir" do + import GenServer, warn: false + assert Macro.Env.lookup_import(__ENV__, {:module_info, 1}) == [] + assert Macro.Env.lookup_import(__ENV__, {:behaviour_info, 1}) == [] + end + + defmodule ModuleWithSigils do + def sigil_i(string, []), do: String.to_integer(string) + + defmacro sigil_I(string, []) do + quote do + String.to_integer(unquote(string)) + end + end + + defmacro sigil_III(string, []) do + quote do + 3 * String.to_integer(unquote(string)) + end + end + + def sigil_w(_string, []), do: [] + + def bnot(x), do: x + defmacro bor(x, _), do: x + end + + test "import only sigils" do + import Kernel, except: [sigil_w: 2] + import ModuleWithSigils, only: :sigils + + # Ensure that both function and macro sigils are imported + assert ~i'10' == 10 + assert ~I'10' == 10 + assert ~III'10' == 30 + assert ~w(abc def) == [] + + # Ensure that non-sigil functions and macros from ModuleWithSigils were not loaded + assert bnot(0) == -1 + assert bor(0, 1) == 1 + end + + test "import only sigils with except" do + import ModuleWithSigils, only: :sigils, except: [sigil_w: 2] + + assert ~i'10' == 10 + assert ~I'10' == 10 + assert ~III'10' == 30 + assert ~w(abc def) == ["abc", "def"] + end + + test "import only removes the non-import part" do + import List + import List, only: :macros + # Buggy local duplicate is used because we asked only for macros + assert duplicate([1], 2) == [1] + end + test "import lexical on if" do if false do - import :lists + import List flatten([1, [2], 3]) - flunk + flunk() else # Buggy local duplicate is untouched assert duplicate([1], 2) == [1] @@ -116,22 +223,45 @@ defmodule Kernel.ImportTest do end test "import lexical on case" do - case true do + case Process.get(:unused, true) do false -> - import :lists + import List flatten([1, [2], 3]) - flunk + flunk() + true -> # Buggy local duplicate is untouched assert duplicate([1], 2) == [1] end end + test "import lexical on for" do + for x <- [1, 2, 3], x > 10 do + import List + flatten([1, [2], 3]) + flunk() + end + + # Buggy local duplicate is untouched + assert duplicate([1], 2) == [1] + end + + test "import lexical on with" do + with [_ | _] <- List.flatten([]) do + import List + flatten([1, [2], 3]) + flunk() + end + + # Buggy local duplicate is untouched + assert duplicate([1], 2) == [1] + end + test "import lexical on try" do try do - import :lists + import List flatten([1, [2], 3]) - flunk + flunk() catch _, _ -> # Buggy local duplicate is untouched diff --git a/lib/elixir/test/elixir/kernel/lexical_tracker_test.exs b/lib/elixir/test/elixir/kernel/lexical_tracker_test.exs index da2a9cf4931..979abdc2bb2 100644 --- a/lib/elixir/test/elixir/kernel/lexical_tracker_test.exs +++ b/lib/elixir/test/elixir/kernel/lexical_tracker_test.exs @@ -1,58 +1,606 @@ -Code.require_file "../test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) defmodule Kernel.LexicalTrackerTest do use ExUnit.Case, async: true alias Kernel.LexicalTracker, as: D + defstruct used_by_tests: :ok setup do - {:ok, [pid: D.start_link]} + {:ok, pid} = D.start_link() + {:ok, [pid: pid]} end - test "can add remote dispatches", config do - D.remote_dispatch(config[:pid], String) - assert D.remotes(config[:pid]) == [String] + test "can add remote dispatch", config do + D.remote_dispatch(config[:pid], String, :runtime) + assert D.references(config[:pid]) == {[], [], [String], []} + + D.remote_dispatch(config[:pid], String, :compile) + assert D.references(config[:pid]) == {[String], [], [], []} + + D.remote_dispatch(config[:pid], String, :runtime) + assert D.references(config[:pid]) == {[String], [], [], []} end - test "can add imports", config do - D.add_import(config[:pid], String, 1, true) - assert D.remotes(config[:pid]) == [String] + test "can add requires", config do + D.add_export(config[:pid], URI) + assert D.references(config[:pid]) == {[], [URI], [], []} + + D.remote_dispatch(config[:pid], URI, :runtime) + assert D.references(config[:pid]) == {[], [URI], [URI], []} + + D.remote_dispatch(config[:pid], URI, :compile) + assert D.references(config[:pid]) == {[URI], [URI], [], []} + end + + test "can add module imports", config do + D.add_export(config[:pid], String) + D.add_import(config[:pid], String, [], 1, true) + + D.import_dispatch(config[:pid], String, {:upcase, 1}, :runtime) + assert D.references(config[:pid]) == {[], [String], [String], []} + + D.import_dispatch(config[:pid], String, {:upcase, 1}, :compile) + assert D.references(config[:pid]) == {[String], [String], [], []} + end + + test "can add module with {function, arity} imports", config do + D.add_export(config[:pid], String) + D.add_import(config[:pid], String, [upcase: 1], 1, true) + + D.import_dispatch(config[:pid], String, {:upcase, 1}, :compile) + assert D.references(config[:pid]) == {[String], [String], [], []} end test "can add aliases", config do - D.add_alias(config[:pid], String, 1, true) - assert D.remotes(config[:pid]) == [String] + D.alias_dispatch(config[:pid], String) + assert D.references(config[:pid]) == {[], [], [], []} end - test "unused imports", config do - D.add_import(config[:pid], String, 1, true) - assert D.collect_unused_imports(config[:pid]) == [{String,1}] + test "unused module imports", config do + D.add_import(config[:pid], String, [], 1, true) + assert D.collect_unused_imports(config[:pid]) == [{String, %{String => 1}}] end - test "used imports are not unused", config do - D.add_import(config[:pid], String, 1, true) - D.import_dispatch(config[:pid], String) - assert D.collect_unused_imports(config[:pid]) == [] + test "used module imports are not unused", config do + D.add_import(config[:pid], String, [], 1, true) + D.import_dispatch(config[:pid], String, {:upcase, 1}, :compile) + assert D.collect_unused_imports(config[:pid]) == [{String, %{}}] + end + + test "unused {module, function, arity} imports", config do + D.add_import(config[:pid], String, [upcase: 1], 1, true) + assert D.collect_unused_imports(config[:pid]) == [{String, %{String => 1, {:upcase, 1} => 1}}] + end + + test "used {module, function, arity} imports are not unused", config do + D.add_import(config[:pid], String, [upcase: 1], 1, true) + D.add_import(config[:pid], String, [downcase: 1], 1, true) + D.import_dispatch(config[:pid], String, {:upcase, 1}, :compile) + assert D.collect_unused_imports(config[:pid]) == [{String, %{{:downcase, 1} => 1}}] + end + + test "overwriting {module, function, arity} import with module import", config do + D.add_import(config[:pid], String, [upcase: 1], 1, true) + D.add_import(config[:pid], String, [], 1, true) + D.import_dispatch(config[:pid], String, {:downcase, 1}, :compile) + assert D.collect_unused_imports(config[:pid]) == [{String, %{}}] end test "imports with no warn are not unused", config do - D.add_import(config[:pid], String, 1, false) + D.add_import(config[:pid], String, [], 1, false) assert D.collect_unused_imports(config[:pid]) == [] end + test "unused requires", config do + D.warn_require(config[:pid], [], String) + D.warn_require(config[:pid], [], List) + D.remote_dispatch(config[:pid], String, :compile) + assert D.collect_unused_requires(config[:pid]) == [{List, []}] + end + + test "function calls do not count as macro usage", config do + D.warn_require(config[:pid], [], String) + D.remote_dispatch(config[:pid], String, :runtime) + assert D.collect_unused_requires(config[:pid]) == [{String, []}] + end + test "unused aliases", config do - D.add_alias(config[:pid], String, 1, true) - assert D.collect_unused_aliases(config[:pid]) == [{String,1}] + D.warn_alias(config[:pid], [], String, String) + assert D.collect_unused_aliases(config[:pid]) == [{String, []}] end test "used aliases are not unused", config do - D.add_alias(config[:pid], String, 1, true) + D.warn_alias(config[:pid], [], String, String) D.alias_dispatch(config[:pid], String) assert D.collect_unused_aliases(config[:pid]) == [] end - test "aliases with no warn are not unused", config do - D.add_alias(config[:pid], String, 1, false) + test "used aliases are not unused in reverse order", config do + D.alias_dispatch(config[:pid], String) + D.warn_alias(config[:pid], [], String, String) assert D.collect_unused_aliases(config[:pid]) == [] end + + describe "references" do + test "typespecs do not tag aliases nor types" do + Code.eval_string(""" + defmodule Kernel.LexicalTrackerTest.AliasTypespecs do + alias Foo.Bar, as: Bar, warn: false + @type bar :: Foo.Bar | Foo.Bar.t + @opaque bar2 :: Foo.Bar.t + @typep bar3 :: Foo.Bar.t + @callback foo :: Foo.Bar.t + @macrocallback foo2(Foo.Bar.t) :: Foo.Bar.t + @spec foo(bar3) :: Foo.Bar.t + def foo(_), do: :ok + + # References from specs are processed only late + @after_compile __MODULE__ + def __after_compile__(env, _) do + send(self(), {:references, Kernel.LexicalTracker.references(env.lexical_tracker)}) + end + end + """) + + assert_received {:references, {compile, _exports, runtime, _}} + + refute Elixir.Bar in runtime + refute Elixir.Bar in compile + + refute Foo.Bar in runtime + refute Foo.Bar in compile + end + + test "typespecs track structs as exports" do + Code.eval_string(""" + defmodule Kernel.LexicalTrackerTest.StructTypespecs do + @type uri :: %URI{} + + # References from specs are processed only late + @after_compile __MODULE__ + def __after_compile__(env, _) do + send(self(), {:references, Kernel.LexicalTracker.references(env.lexical_tracker)}) + end + end + """) + + assert_received {:references, {compile, exports, runtime, _}} + + assert URI in runtime + assert URI in exports + refute URI in compile + end + + test "attributes adds dependency based on expansion" do + {{compile, _, _, _}, _binding} = + Code.eval_string(""" + defmodule Kernel.LexicalTrackerTest.Attribute1 do + @example [String, Enum, 3 + 10] + def foo(atom) when atom in @example, do: atom + Kernel.LexicalTracker.references(__ENV__.lexical_tracker) + end |> elem(3) + """) + + refute String in compile + refute Enum in compile + + {{compile, _, _, _}, _binding} = + Code.eval_string(""" + defmodule Kernel.LexicalTrackerTest.Attribute2 do + @example [String, Enum] + def foo(atom) when atom in @example, do: atom + _ = @example + Kernel.LexicalTracker.references(__ENV__.lexical_tracker) + end |> elem(3) + """) + + assert String in compile + assert Enum in compile + + {{compile, _, _, _}, _binding} = + Code.eval_string(""" + defmodule Kernel.LexicalTrackerTest.Attribute3 do + @example [String, Enum] + _ = Module.get_attribute(__MODULE__, :example) + Kernel.LexicalTracker.references(__ENV__.lexical_tracker) + end |> elem(3) + """) + + assert String in compile + assert Enum in compile + + {{compile, _, _, _}, _binding} = + Code.eval_string(""" + defmodule Kernel.LexicalTrackerTest.Attribute4 do + Module.register_attribute(__MODULE__, :example, accumulate: true) + @example String + def foo(atom) when atom in @example, do: atom + @example Enum + def bar(atom) when atom in @example, do: atom + Kernel.LexicalTracker.references(__ENV__.lexical_tracker) + end |> elem(3) + """) + + refute String in compile + refute Enum in compile + + {{compile, _, _, _}, _binding} = + Code.eval_string(""" + defmodule Kernel.LexicalTrackerTest.Attribute5 do + Module.register_attribute(__MODULE__, :example, accumulate: true) + @example String + def foo(atom) when atom in @example, do: atom + @example Enum + def bar(atom) when atom in @example, do: atom + _ = Module.get_attribute(__MODULE__, :example) + Kernel.LexicalTracker.references(__ENV__.lexical_tracker) + end |> elem(3) + """) + + assert String in compile + assert Enum in compile + + {{compile, _, _, _}, _binding} = + Code.eval_string(""" + defmodule Kernel.LexicalTrackerTest.Attribute6 do + @example %{foo: Application.compile_env(:elixir, Enum, String)} + def foo(atom) when atom == @example.foo, do: atom + Kernel.LexicalTracker.references(__ENV__.lexical_tracker) + end |> elem(3) + """) + + refute String in compile + refute Enum in compile + + {{compile, _, _, _}, _binding} = + Code.eval_string(""" + defmodule Kernel.LexicalTrackerTest.Attribute7 do + Module.register_attribute(__MODULE__, :example, accumulate: true) + @example String + @example Enum + _ = Module.get_last_attribute(__MODULE__, :example) + + Kernel.LexicalTracker.references(__ENV__.lexical_tracker) + end |> elem(3) + """) + + assert String in compile + assert Enum in compile + end + + test "@compile adds a runtime dependency" do + {{compile, exports, runtime, _}, _binding} = + Code.eval_string(""" + defmodule Kernel.LexicalTrackerTest.Compile do + @compile {:no_warn_undefined, String} + @compile {:no_warn_undefined, {Enum, :concat, 1}} + Kernel.LexicalTracker.references(__ENV__.lexical_tracker) + end |> elem(3) + """) + + refute String in compile + refute String in exports + assert String in runtime + + refute Enum in compile + refute Enum in exports + assert Enum in runtime + end + + def __before_compile__(_), do: :ok + def __after_compile__(_, _), do: :ok + def __on_definition__(_, _, _, _, _, _), do: :ok + + test "module callbacks add a compile dependency" do + {{compile, exports, runtime, _}, _binding} = + Code.eval_string(""" + defmodule Kernel.LexicalTrackerTest.BeforeCompile do + @before_compile Kernel.LexicalTrackerTest + Kernel.LexicalTracker.references(__ENV__.lexical_tracker) + end |> elem(3) + """) + + assert Kernel.LexicalTrackerTest in compile + refute Kernel.LexicalTrackerTest in exports + refute Kernel.LexicalTrackerTest in runtime + + {{compile, exports, runtime, _}, _binding} = + Code.eval_string(""" + defmodule Kernel.LexicalTrackerTest.AfterCompile do + @after_compile Kernel.LexicalTrackerTest + Kernel.LexicalTracker.references(__ENV__.lexical_tracker) + end |> elem(3) + """) + + assert Kernel.LexicalTrackerTest in compile + refute Kernel.LexicalTrackerTest in exports + refute Kernel.LexicalTrackerTest in runtime + + {{compile, exports, runtime, _}, _binding} = + Code.eval_string(""" + defmodule Kernel.LexicalTrackerTest.OnDefinition do + @on_definition Kernel.LexicalTrackerTest + Kernel.LexicalTracker.references(__ENV__.lexical_tracker) + end |> elem(3) + """) + + assert Kernel.LexicalTrackerTest in compile + refute Kernel.LexicalTrackerTest in exports + refute Kernel.LexicalTrackerTest in runtime + end + + test "defdelegate with literal adds runtime dependency" do + {{compile, _exports, runtime, _}, _binding} = + Code.eval_string(""" + defmodule Kernel.LexicalTrackerTest.Defdelegate do + defdelegate decode_query(query), to: URI + + opts = [to: Enum] + defdelegate concat(enum), opts + + Kernel.LexicalTracker.references(__ENV__.lexical_tracker) + end |> elem(3) + """) + + refute URI in compile + assert Enum in compile + assert URI in runtime + end + + test "dbg adds a compile dependency" do + {{compile, exports, runtime, _}, _binding} = + Code.eval_string(""" + defmodule Kernel.LexicalTrackerTest.Dbg do + def foo, do: dbg(:ok) + Kernel.LexicalTracker.references(__ENV__.lexical_tracker) + end |> elem(3) + """) + + assert Macro in compile + refute Macro in exports + refute Macro in runtime + + {{compile, exports, runtime, _}, _binding} = + Code.eval_string(""" + defmodule Kernel.LexicalTrackerTest.NoDbg do + Kernel.LexicalTracker.references(__ENV__.lexical_tracker) + end |> elem(3) + """) + + refute Macro in compile + refute Macro in exports + refute Macro in runtime + end + + test "imports adds an export dependency" do + {{compile, exports, runtime, _}, _binding} = + Code.eval_string(""" + defmodule Kernel.LexicalTrackerTest.Imports do + import String, warn: false + Kernel.LexicalTracker.references(__ENV__.lexical_tracker) + end |> elem(3) + """) + + refute String in compile + assert String in exports + refute String in runtime + end + + test "structs are exports or compile time" do + {{compile, exports, runtime, _}, _binding} = + Code.eval_string(""" + defmodule Kernel.LexicalTrackerTest.StructRuntime do + def expand, do: %URI{} + Kernel.LexicalTracker.references(__ENV__.lexical_tracker) + end |> elem(3) + """) + + refute URI in compile + assert URI in exports + assert URI in runtime + + {{compile, exports, runtime, _}, _binding} = + Code.eval_string(""" + defmodule Kernel.LexicalTrackerTest.StructCompile do + _ = %URI{} + Kernel.LexicalTracker.references(__ENV__.lexical_tracker) + end |> elem(3) + """) + + assert URI in compile + assert URI in exports + refute URI in runtime + + {{compile, exports, runtime, _}, _binding} = + Code.eval_string(""" + defmodule Kernel.LexicalTrackerTest.StructPattern do + def uri?(%URI{}), do: true + Kernel.LexicalTracker.references(__ENV__.lexical_tracker) + end |> elem(3) + """) + + refute URI in compile + assert URI in exports + assert URI in runtime + end + + test "Macro.struct_info! adds an export dependency" do + {{compile, exports, runtime, _}, _binding} = + Code.eval_string(""" + defmodule Kernel.LexicalTrackerTest.MacroStruct do + # We do not use the alias because it would be a compile time + # dependency. The alias may happen in practice, which is the + # mechanism to make this expansion become a compile-time one. + # However, in some cases, such as typespecs, we don't necessarily + # want the compile-time dependency to happen. + Macro.struct_info!(:"Elixir.URI", __ENV__) + Kernel.LexicalTracker.references(__ENV__.lexical_tracker) + end |> elem(3) + """) + + refute URI in compile + assert URI in exports + refute URI in runtime + end + + test "aliases in patterns and guards inside functions do not add runtime dependency" do + {{compile, exports, runtime, _}, _binding} = + Code.eval_string(""" + defmodule Kernel.LexicalTrackerTest.PatternGuardsRuntime do + def uri_atom?(URI), do: true + def range_struct?(range) when is_struct(range, Range), do: true + Kernel.LexicalTracker.references(__ENV__.lexical_tracker) + end |> elem(3) + """) + + refute URI in compile + refute URI in exports + refute URI in runtime + refute Range in compile + refute Range in exports + refute Range in runtime + end + + test "aliases in patterns and guards outside functions do add compile dependency" do + {{compile, exports, runtime, _}, _binding} = + Code.eval_string(""" + defmodule Kernel.LexicalTrackerTest.PatternGuardsCompile do + %URI{} = URI.parse("/") + case Range.new(1, 3) do + range when is_struct(range, Range) -> :ok + end + Kernel.LexicalTracker.references(__ENV__.lexical_tracker) + end |> elem(3) + """) + + assert URI in compile + assert URI in exports + refute URI in runtime + assert Range in compile + refute Range in exports + refute Range in runtime + end + + test "compile_env! does not add a compile dependency" do + {{compile, exports, runtime, _}, _binding} = + Code.eval_string(""" + defmodule Kernel.LexicalTrackerTest.CompileEnvStruct do + require Application + Application.compile_env(:elixir, URI) + Application.compile_env(:elixir, [:foo, URI, :bar]) + Kernel.LexicalTracker.references(__ENV__.lexical_tracker) + end |> elem(3) + """) + + refute URI in compile + refute URI in exports + assert URI in runtime + end + + test "defmodule does not add a compile dependency" do + {{compile, exports, runtime, _}, _binding} = + Code.eval_string(""" + defmodule Kernel.LexicalTrackerTest.Defmodule do + Kernel.LexicalTracker.references(__ENV__.lexical_tracker) + end |> elem(3) + """) + + refute Kernel.LexicalTrackerTest.Defmodule in compile + refute Kernel.LexicalTrackerTest.Defmodule in exports + refute Kernel.LexicalTrackerTest.Defmodule in runtime + end + + test "defmacro adds a compile-time dependency for local calls" do + {{compile, exports, runtime, _}, _binding} = + Code.eval_string(""" + defmodule Kernel.LexicalTrackerTest.Defmacro do + defmacro uri(path) do + Macro.escape(URI.parse(path)) + end + + def fun() do + uri("/hello") + end + + Kernel.LexicalTracker.references(__ENV__.lexical_tracker) + end |> elem(3) + """) + + assert URI in compile + refute URI in exports + refute URI in runtime + end + + test "imported functions from quote adds dependencies" do + {{compile, exports, runtime, _}, _binding} = + Code.eval_string(""" + defmodule Kernel.LexicalTrackerTest.QuotedFun do + import URI + + defmacro parse_root() do + quote do + parse("/") + end + end + end + + defmodule Kernel.LexicalTrackerTest.UsingQuotedFun do + require Kernel.LexicalTrackerTest.QuotedFun, as: QF + QF.parse_root() + Kernel.LexicalTracker.references(__ENV__.lexical_tracker) + end |> elem(3) + """) + + assert URI in compile + refute URI in exports + refute URI in runtime + end + + test "imported macro from quote adds dependencies" do + {{compile, exports, runtime, _}, _binding} = + Code.eval_string(""" + defmodule Kernel.LexicalTrackerTest.QuotedMacro do + import Config + + defmacro config_env() do + quote do + config_env() + end + end + end + + defmodule Kernel.LexicalTrackerTest.UsingQuotedMacro do + require Kernel.LexicalTrackerTest.QuotedMacro, as: QM + def fun(), do: QM.config_env() + Kernel.LexicalTracker.references(__ENV__.lexical_tracker) + end |> elem(3) + """) + + assert Config in compile + refute Config in exports + refute Config in runtime + end + + test "defimpl does not add dependencies on for only on impl" do + {{compile, exports, runtime, _}, _binding} = + Code.eval_string(""" + defimpl String.Chars, for: Kernel.LexicalTrackerTest do + def to_string(val), do: val.used_by_tests + Kernel.LexicalTracker.references(__ENV__.lexical_tracker) + end |> elem(3) + """) + + refute String.Chars in compile + assert String.Chars in exports + + refute Kernel.LexicalTrackerTest in compile + refute Kernel.LexicalTrackerTest in exports + refute Kernel.LexicalTrackerTest in runtime + end + end end diff --git a/lib/elixir/test/elixir/kernel/macros_test.exs b/lib/elixir/test/elixir/kernel/macros_test.exs index fd2f7511e7c..c5f2975280f 100644 --- a/lib/elixir/test/elixir/kernel/macros_test.exs +++ b/lib/elixir/test/elixir/kernel/macros_test.exs @@ -1,4 +1,8 @@ -Code.require_file "../test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) defmodule Kernel.MacrosTest.Nested do defmacro value, do: 1 @@ -6,49 +10,85 @@ defmodule Kernel.MacrosTest.Nested do defmacro do_identity!(do: x) do x end + + defmacro unused_require do + quote do + require Integer + end + end end defmodule Kernel.MacrosTest do - require Kernel.MacrosTest.Nested, as: Nested - use ExUnit.Case, async: true + Kernel.MacrosTest.Nested = require Kernel.MacrosTest.Nested, as: Nested + + # Unused require from macro should not warn + Kernel.MacrosTest.Nested.unused_require() + + @spec my_macro :: Macro.t() defmacro my_macro do - quote do: 1 + 1 + quote(do: 1 + 1) end + @spec my_private_macro :: Macro.t() defmacrop my_private_macro do - quote do: 1 + 3 + quote(do: 1 + 3) end defmacro my_macro_with_default(value \\ 5) do - quote do: 1 + unquote(value) + quote(do: 1 + unquote(value)) + end + + defp by_two(x), do: x * 2 + + defmacro my_macro_with_local(value) do + value = by_two(by_two(value)) + quote(do: 1 + unquote(value)) end - test :require do - assert Kernel.MacrosTest.Nested.value == 1 + defmacro my_macro_with_capture(value) do + Enum.map(value, &by_two/1) end - test :require_with_alias do - assert Nested.value == 1 + test "require" do + assert Kernel.MacrosTest.Nested.value() == 1 end - test :local_but_private_macro do - assert my_private_macro == 4 + test "require with alias" do + assert Nested.value() == 1 end - test :local_with_defaults_macro do - assert my_macro_with_default == 6 + test "local with private macro" do + assert my_private_macro() == 4 end - test :macros_cannot_be_called_dynamically do - x = Nested - assert_raise UndefinedFunctionError, fn -> x.value end + test "local with defaults macro" do + assert my_macro_with_default() == 6 end - test :bang_do_block do + test "local with local call" do + assert my_macro_with_local(4) == 17 + end + + test "local with capture" do + assert my_macro_with_capture([1, 2, 3]) == [2, 4, 6] + end + + test "macros cannot be called dynamically" do + x = String.to_atom("Elixir.Nested") + assert_raise UndefinedFunctionError, fn -> x.func() end + end + + test "macros with bang and do block have proper precedence" do import Kernel.MacrosTest.Nested - assert (do_identity! do 1 end) == 1 - assert (Kernel.MacrosTest.Nested.do_identity! do 1 end) == 1 + + assert (do_identity! do + 1 + end) == 1 + + assert (Kernel.MacrosTest.Nested.do_identity! do + 1 + end) == 1 end -end \ No newline at end of file +end diff --git a/lib/elixir/test/elixir/kernel/overridable_test.exs b/lib/elixir/test/elixir/kernel/overridable_test.exs index 1397e53d793..51992f0b4ba 100644 --- a/lib/elixir/test/elixir/kernel/overridable_test.exs +++ b/lib/elixir/test/elixir/kernel/overridable_test.exs @@ -1,10 +1,10 @@ -Code.require_file "../test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec -defmodule Kernel.Overridable do - defmacrop super? do - Module.overridable?(__CALLER__.module, __CALLER__.function) - end +Code.require_file("../test_helper.exs", __DIR__) +defmodule Kernel.Overridable do def sample do 1 end @@ -17,50 +17,54 @@ defmodule Kernel.Overridable do 1 end - def explicit_nested_super do - {super?, 2} + def super_with_multiple_args(x, y) do + x + y end - false = Module.overridable? __MODULE__, {:explicit_nested_super, 0} - - defoverridable [sample: 0, with_super: 0, without_super: 0, explicit_nested_super: 0] - - true = Module.overridable? __MODULE__, {:explicit_nested_super, 0} - - def explicit_nested_super do - {super, super?, 1} + def capture_super(x) do + x end - true = Module.overridable? __MODULE__, {:explicit_nested_super, 0} - - defoverridable [explicit_nested_super: 0] - - true = Module.overridable? __MODULE__, {:explicit_nested_super, 0} - - def implicit_nested_super do - {super?, 1} + defmacro capture_super_macro(x) do + x end - defoverridable [implicit_nested_super: 0] + def many_clauses(0) do + 11 + end - def implicit_nested_super do - {super, super?, 0} + def many_clauses(1) do + 13 end - def super_with_explicit_args(x, y) do - x + y + def locals do + undefined_function() end - def many_clauses(0) do - 11 + def multiple_overrides do + [1] end - def many_clauses(1) do - 13 + def public_to_private do + :public end - defoverridable [implicit_nested_super: 0, - super_with_explicit_args: 2, many_clauses: 1] + defoverridable sample: 0, + with_super: 0, + without_super: 0, + super_with_multiple_args: 2, + capture_super: 1, + capture_super_macro: 1, + many_clauses: 1, + locals: 0, + multiple_overrides: 0, + public_to_private: 0 + + true = Module.overridable?(__MODULE__, {:without_super, 0}) + true = Module.overridable?(__MODULE__, {:with_super, 0}) + + true = {:with_super, 0} in Module.overridables_in(__MODULE__) + true = {:without_super, 0} in Module.overridables_in(__MODULE__) def without_super do :without_super @@ -70,16 +74,22 @@ defmodule Kernel.Overridable do super() + 2 end - def no_overridable do - {:no_overridable, super?} + true = Module.overridable?(__MODULE__, {:without_super, 0}) + true = Module.overridable?(__MODULE__, {:with_super, 0}) + + true = {:with_super, 0} in Module.overridables_in(__MODULE__) + true = {:without_super, 0} in Module.overridables_in(__MODULE__) + + def super_with_multiple_args(x, y) do + super(x, y * 2) end - def explicit_nested_super do - {super, super?, 0} + def capture_super(x) do + Enum.map(1..x, &super(&1)) ++ Enum.map(1..x, &super/1) end - def super_with_explicit_args(x, y) do - super x, y * 2 + defmacro capture_super_macro(x) do + Enum.map(1..x, &super(&1)) ++ Enum.map(1..x, &super/1) end def many_clauses(2) do @@ -93,38 +103,171 @@ defmodule Kernel.Overridable do def many_clauses(x) do super(x) end + + def locals do + :ok + end + + def multiple_overrides do + [2 | super()] + end + + defp public_to_private do + :private + end + + def test_public_to_private do + public_to_private() + end + + defoverridable multiple_overrides: 0 + + def multiple_overrides do + [3 | super()] + end + + ## Macros + + defmacro overridable_macro(x) do + quote do + unquote(x) + 100 + end + end + + defoverridable overridable_macro: 1 + + defmacro overridable_macro(x) do + quote do + unquote(super(x)) + 1000 + end + end + + defmacrop private_macro(x) do + quote do + unquote(x) + 100 + end + end + + defoverridable private_macro: 1 + + defmacrop private_macro(x) do + quote do + unquote(super(x)) + 1000 + end + end + + def private_macro_call(val \\ 11) do + private_macro(val) + end +end + +defmodule Kernel.OverridableExampleBehaviour do + @callback required_callback :: any + @callback optional_callback :: any + @macrocallback required_macro_callback(arg :: any) :: Macro.t() + @macrocallback optional_macro_callback(arg :: any, arg2 :: any) :: Macro.t() + @optional_callbacks optional_callback: 0, optional_macro_callback: 2 end defmodule Kernel.OverridableTest do require Kernel.Overridable, as: Overridable - use ExUnit.Case, async: true + use ExUnit.Case + + defp purge(module) do + :code.purge(module) + :code.delete(module) + end + + test "overridable keeps function ordering" do + defmodule OverridableOrder do + def not_private(str) do + process_url(str) + end + + def process_url(_str) do + :first + end + + # There was a bug where the order in which we removed + # overridable expressions lead to errors. This module + # aims to guarantee removing process_url/1 before we + # remove the function that depends on it does not cause + # errors. If it compiles, it works! + defoverridable process_url: 1, not_private: 1 + + def process_url(_str) do + :second + end + end + end + + test "overridable works with defaults" do + defmodule OverridableDefault do + def fun(value, opt \\ :from_parent) do + {value, opt} + end + + defmacro macro(value, opt \\ :from_parent) do + {{value, opt}, Macro.escape(__CALLER__)} + end + + # There was a bug where the default function would + # attempt to call its overridable name instead of + # func/1. If it compiles, it works! + defoverridable fun: 1, fun: 2, macro: 1, macro: 2 + + def fun(value) do + {value, super(value)} + end + + defmacro macro(value) do + {{value, super(value)}, Macro.escape(__CALLER__)} + end + end + + defmodule OverridableCall do + require OverridableDefault + OverridableDefault.fun(:foo) + OverridableDefault.macro(:bar) + end + end test "overridable is made concrete if no other is defined" do - assert Overridable.sample == 1 + assert Overridable.sample() == 1 end test "overridable overridden with super" do - assert Overridable.with_super == 3 + assert Overridable.with_super() == 3 end test "overridable overridden without super" do - assert Overridable.without_super == :without_super + assert Overridable.without_super() == :without_super end - test "overridable overridden with nested super" do - assert Overridable.explicit_nested_super == {{{false, 2}, true, 1}, true, 0} + test "public overridable overridden as private function" do + assert Overridable.test_public_to_private() == :private + refute {:public_to_private, 0} in Overridable.module_info(:exports) end - test "overridable node overridden with nested super" do - assert Overridable.implicit_nested_super == {{false, 1}, true, 0} + test "overridable locals are ignored without super" do + assert Overridable.locals() == :ok end - test "calling super with explicit args" do - assert Overridable.super_with_explicit_args(1, 2) == 5 + test "calling super with multiple args" do + assert Overridable.super_with_multiple_args(1, 2) == 5 end - test "function without overridable returns false for super?" do - assert Overridable.no_overridable == {:no_overridable, false} + test "calling super using function captures" do + assert Overridable.capture_super(5) == [1, 2, 3, 4, 5, 1, 2, 3, 4, 5] + end + + test "calling super of an overridable macro using function captures" do + assert Overridable.capture_super_macro(5) == [1, 2, 3, 4, 5, 1, 2, 3, 4, 5] + end + + test "super as a variable" do + super = :ok + assert super == :ok end test "overridable with many clauses" do @@ -135,18 +278,283 @@ defmodule Kernel.OverridableTest do end test "overridable definitions are private" do - refute {:"with_super (overridable 0)", 0} in Overridable.__info__(:exports) + refute {:"with_super (overridable 0)", 0} in Overridable.module_info(:exports) + refute {:"with_super (overridable 1)", 0} in Overridable.module_info(:exports) + end + + test "multiple overrides" do + assert Overridable.multiple_overrides() == [3, 2, 1] + end + + test "overridable macros" do + a = 11 + assert Overridable.overridable_macro(a) == 1111 + assert Overridable.private_macro_call() == 1111 end test "invalid super call" do - try do - :elixir.eval 'defmodule Foo.Forwarding do\ndef bar, do: 1\ndefoverridable [bar: 0]\ndef foo, do: super\nend', [] - flunk "expected eval to fail" - rescue - error -> - assert Exception.message(error) == - "nofile:4: no super defined for foo/0 in module Foo.Forwarding. " <> - "Overridable functions available are: bar/0" + messages = [ + "nofile:4", + "no super defined for foo/0 in module Kernel.OverridableOrder.Forwarding. " <> + "Overridable functions available are: bar/0" + ] + + assert_compile_error(messages, fn -> + Code.eval_string(""" + defmodule Kernel.OverridableOrder.Forwarding do + def bar(), do: 1 + defoverridable bar: 0 + def foo(), do: super() + end + """) + end) + + purge(Kernel.OverridableOrder.Forwarding) + end + + test "invalid super call with different arity" do + messages = [ + "nofile:4", + "super must be called with the same number of arguments as the current definition" + ] + + assert_compile_error(messages, fn -> + Code.eval_string(""" + defmodule Kernel.OverridableSuper.DifferentArities do + def bar(a), do: a + defoverridable bar: 1 + def bar(_), do: super() + end + """) + end) + end + + test "invalid super capture with different arity" do + messages = [ + "nofile:4", + "super must be called with the same number of arguments as the current definition" + ] + + assert_compile_error(messages, fn -> + Code.eval_string(""" + defmodule Kernel.OverridableSuperCapture.DifferentArities do + def bar(a), do: a + defoverridable bar: 1 + def bar(_), do: (&super/0).() + end + """) + end) + end + + test "does not allow to override a macro as a function" do + messages = [ + "nofile:4", + "cannot override macro (defmacro, defmacrop) foo/0 in module " <> + "Kernel.OverridableMacro.FunctionOverride as a function (def, defp)" + ] + + assert_compile_error(messages, fn -> + Code.eval_string(""" + defmodule Kernel.OverridableMacro.FunctionOverride do + defmacro foo(), do: :ok + defoverridable foo: 0 + def foo(), do: :invalid + end + """) + end) + + purge(Kernel.OverridableMacro.FunctionOverride) + + assert_compile_error(messages, fn -> + Code.eval_string(""" + defmodule Kernel.OverridableMacro.FunctionOverride do + defmacro foo(), do: :ok + defoverridable foo: 0 + def foo(), do: :invalid + defoverridable foo: 0 + def foo(), do: :invalid + end + """) + end) + + purge(Kernel.OverridableMacro.FunctionOverride) + + assert_compile_error(messages, fn -> + Code.eval_string(""" + defmodule Kernel.OverridableMacro.FunctionOverride do + defmacro foo(), do: :ok + defoverridable foo: 0 + def foo(), do: super() + end + """) + end) + + purge(Kernel.OverridableMacro.FunctionOverride) + end + + test "does not allow to override a function as a macro" do + messages = [ + "nofile:4", + "cannot override function (def, defp) foo/0 in module " <> + "Kernel.OverridableFunction.MacroOverride as a macro (defmacro, defmacrop)" + ] + + assert_compile_error(messages, fn -> + Code.eval_string(""" + defmodule Kernel.OverridableFunction.MacroOverride do + def foo(), do: :ok + defoverridable foo: 0 + defmacro foo(), do: :invalid + end + """) + end) + + purge(Kernel.OverridableFunction.MacroOverride) + + assert_compile_error(messages, fn -> + Code.eval_string(""" + defmodule Kernel.OverridableFunction.MacroOverride do + def foo(), do: :ok + defoverridable foo: 0 + defmacro foo(), do: :invalid + defoverridable foo: 0 + defmacro foo(), do: :invalid + end + """) + end) + + purge(Kernel.OverridableFunction.MacroOverride) + + assert_compile_error(messages, fn -> + Code.eval_string(""" + defmodule Kernel.OverridableFunction.MacroOverride do + def foo(), do: :ok + defoverridable foo: 0 + defmacro foo(), do: super() + end + """) + end) + + purge(Kernel.OverridableFunction.MacroOverride) + end + + test "undefined functions can't be marked as overridable" do + message = "cannot make function foo/2 overridable because it was not defined" + + assert_raise ArgumentError, message, fn -> + Code.eval_string(""" + defmodule Kernel.OverridableOrder.Foo do + defoverridable foo: 2 + end + """) + end + + purge(Kernel.OverridableOrder.Foo) + end + + test "overrides with behaviour" do + defmodule OverridableWithBehaviour do + @behaviour Elixir.Kernel.OverridableExampleBehaviour + + def required_callback(), do: "original" + + def optional_callback(), do: "original" + + def not_a_behaviour_callback(), do: "original" + + defmacro required_macro_callback(boolean) do + quote do + if unquote(boolean) do + "original" + end + end + end + + defoverridable Elixir.Kernel.OverridableExampleBehaviour + + defmacro optional_macro_callback(arg1, arg2), do: {arg1, arg2} + + assert Module.overridable?(__MODULE__, {:required_callback, 0}) + assert Module.overridable?(__MODULE__, {:optional_callback, 0}) + assert Module.overridable?(__MODULE__, {:required_macro_callback, 1}) + refute Module.overridable?(__MODULE__, {:optional_macro_callback, 1}) + refute Module.overridable?(__MODULE__, {:not_a_behaviour_callback, 1}) + end + end + + test "undefined module can't be passed as argument to defoverridable" do + message = + "cannot pass module Kernel.OverridableTest.Bar as argument to defoverridable/1 because it was not defined" + + assert_raise ArgumentError, message, fn -> + Code.eval_string(""" + defmodule Kernel.OverridableTest.Foo do + defoverridable Kernel.OverridableTest.Bar + end + """) + end + + purge(Kernel.OverridableTest.Foo) + end + + test "module without @behaviour can't be passed as argument to defoverridable" do + message = + "cannot pass module Kernel.OverridableExampleBehaviour as argument to defoverridable/1" <> + " because its corresponding behaviour is missing. Did you forget to add " <> + "@behaviour Kernel.OverridableExampleBehaviour?" + + assert_raise ArgumentError, message, fn -> + Code.eval_string(""" + defmodule Kernel.OverridableTest.Foo do + defoverridable Kernel.OverridableExampleBehaviour + end + """) + end + + purge(Kernel.OverridableTest.Foo) + end + + test "module with no callbacks can't be passed as argument to defoverridable" do + message = + "cannot pass module Kernel.OverridableTest.Bar as argument to defoverridable/1 because it does not define any callbacks" + + assert_raise ArgumentError, message, fn -> + Code.eval_string(""" + defmodule Kernel.OverridableTest.Bar do + end + defmodule Kernel.OverridableTest.Foo do + @behaviour Kernel.OverridableTest.Bar + defoverridable Kernel.OverridableTest.Bar + end + """) + end + + purge(Kernel.OverridableTest.Bar) + purge(Kernel.OverridableTest.Foo) + end + + test "atom which is not a module can't be passed as argument to defoverridable" do + message = "cannot pass module :abc as argument to defoverridable/1 because it was not defined" + + assert_raise ArgumentError, message, fn -> + Code.eval_string(""" + defmodule Kernel.OverridableTest.Foo do + defoverridable :abc + end + """) + end + + purge(Kernel.OverridableTest.Foo) + end + + defp assert_compile_error(messages, fun) do + captured = + ExUnit.CaptureIO.capture_io(:stderr, fn -> + assert_raise CompileError, fun + end) + + for message <- List.wrap(messages) do + assert captured =~ message end end end diff --git a/lib/elixir/test/elixir/kernel/parallel_compiler_test.exs b/lib/elixir/test/elixir/kernel/parallel_compiler_test.exs new file mode 100644 index 00000000000..1bd2c86d8eb --- /dev/null +++ b/lib/elixir/test/elixir/kernel/parallel_compiler_test.exs @@ -0,0 +1,662 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) + +import PathHelpers + +defmodule Kernel.ParallelCompilerTest do + use ExUnit.Case + import ExUnit.CaptureIO + + @no_warnings %{compile_warnings: [], runtime_warnings: []} + + defp compile(files, opts \\ []) do + Kernel.ParallelCompiler.compile(files, [return_diagnostics: true] ++ opts) + end + + defp purge(modules) do + Enum.map(modules, fn mod -> + :code.purge(mod) + :code.delete(mod) + end) + end + + defp write_tmp(context, kv) do + dir = tmp_path(context) + File.rm_rf!(dir) + File.mkdir_p!(dir) + + for {key, contents} <- kv do + path = Path.join(dir, "#{key}.ex") + File.write!(path, contents) + path + end + end + + describe "compile" do + test "with profiling" do + fixtures = + write_tmp( + "profile_time", + bar: """ + defmodule HelloWorld do + end + """ + ) + + profile = + capture_io(:stderr, fn -> + assert {:ok, modules, @no_warnings} = compile(fixtures, profile: :time) + + assert HelloWorld in modules + end) + + assert profile =~ + ~r"\[profile\] [\s\d]{6}ms compiling \+ 0ms waiting while compiling .*tmp/profile_time/bar.ex" + + assert profile =~ ~r"\[profile\] Finished compilation cycle of 1 modules in \d+ms" + assert profile =~ ~r"\[profile\] Finished group pass check of 1 modules in \d+ms" + after + purge([HelloWorld]) + end + + test "immediately loads modules when not writing them to disk" do + fixtures = + write_tmp( + "compile_loads", + will_be_loaded: """ + defmodule WillBeLoaded do + end + true = Code.loaded?(WillBeLoaded) + """ + ) + + assert {:ok, _modules, @no_warnings} = compile(fixtures) + end + + test "lazily loads modules when writing them to disk" do + fixtures = + write_tmp( + "compile_lazy_loads", + will_be_lazy_loaded: """ + defmodule WillBeLazyLoaded do + end + false = Code.loaded?(WillBeLazyLoaded) + {:error, _} = Code.ensure_loaded(WillBeLazyLoaded) + {:module, _} = Code.ensure_compiled(WillBeLazyLoaded) + """, + is_autoloaded: """ + defmodule WillBeAutoLoaded do + @compile {:autoload, true} + end + true = Code.loaded?(WillBeAutoLoaded) + """ + ) + + assert {:ok, _modules, @no_warnings} = + Kernel.ParallelCompiler.compile_to_path(fixtures, tmp_path("pcload"), + return_diagnostics: true + ) + end + + test "solves dependencies between modules" do + fixtures = + write_tmp( + "parallel_compiler", + bar: """ + defmodule BarParallel do + end + + require FooParallel + IO.puts(FooParallel.message()) + """, + foo: """ + defmodule FooParallel do + # We use this ensure_compiled clause so both Foo and + # Bar block. Foo depends on Unknown and Bar depends on + # Foo. The compiler will see this dependency and first + # release Foo and then Bar, compiling with success. + {:error, _} = Code.ensure_compiled(Unknown) + def message, do: "message_from_foo" + end + """ + ) + + assert capture_io(fn -> + assert {:ok, modules, @no_warnings} = compile(fixtures) + assert BarParallel in modules + assert FooParallel in modules + end) =~ "message_from_foo" + after + purge([FooParallel, BarParallel]) + end + + test "solves dependencies between structs" do + fixtures = + write_tmp( + "parallel_struct", + bar: """ + defmodule BarStruct do + defstruct name: "", foo: %FooStruct{} + end + """, + foo: """ + defmodule FooStruct do + defstruct name: "" + def bar?(%BarStruct{}), do: true + end + """ + ) + + assert {:ok, modules, @no_warnings} = compile(fixtures) + assert [BarStruct, FooStruct] = Enum.sort(modules) + after + purge([FooStruct, BarStruct]) + end + + test "solves dependencies between structs in typespecs" do + fixtures = + write_tmp( + "parallel_typespec_struct", + bar: """ + defmodule BarStruct do + defstruct name: "" + @type t :: %FooStruct{} + end + """, + foo: """ + defmodule FooStruct do + defstruct name: "" + @type t :: %BarStruct{} + end + """ + ) + + assert {:ok, modules, @no_warnings} = compile(fixtures) + assert [BarStruct, FooStruct] = Enum.sort(modules) + after + purge([FooStruct, BarStruct]) + end + + test "returns struct undefined error when local struct is undefined" do + [fixture] = + write_tmp( + "compile_struct", + undef: """ + defmodule Undef do + def undef() do + %__MODULE__{} + end + end + """ + ) + + expected_msg = "Undef.__struct__/1 is undefined, cannot expand struct Undef" + + assert capture_io(:stderr, fn -> + assert {:error, + [ + %{file: ^fixture, position: {3, 5}, message: msg}, + %{file: ^fixture, position: 0, message: compile_msg} + ], @no_warnings} = + compile([fixture]) + + assert msg =~ expected_msg + assert compile_msg =~ "cannot compile module Undef (errors have been logged)" + end) =~ expected_msg + end + + test "returns error when fails to expand struct" do + [fixture] = + write_tmp( + "compile_struct_invalid_key", + undef: """ + defmodule InvalidStructKey do + def invalid_struct_key() do + %Date{invalid_key: 2020} + end + end + """ + ) + + expected_msg = "** (KeyError) key :invalid_key not found" + + assert capture_io(:stderr, fn -> + assert {:error, [%{file: ^fixture, position: 3, message: msg}], @no_warnings} = + compile([fixture]) + + assert msg =~ expected_msg + end) =~ expected_msg + end + + test "does not crash with pending monitor message" do + {pid, ref} = spawn_monitor(fn -> :ok end) + + [fixture] = + write_tmp( + "quick_example", + quick_example: """ + defmodule QuickExample do + end + """ + ) + + assert {:ok, [QuickExample], @no_warnings} = compile([fixture]) + assert_received {:DOWN, ^ref, _, ^pid, :normal} + after + purge([QuickExample]) + end + + test "does not crash on external reports" do + [fixture] = + write_tmp( + "compile_quoted", + quick_example: """ + defmodule CompileQuoted do + try do + Code.compile_quoted({:fn, [], [{:->, [], [[], quote(do: unknown_var)]}]}) + rescue + _ -> :ok + end + end + """ + ) + + assert capture_io(:stderr, fn -> + assert {:ok, [CompileQuoted], @no_warnings} = compile([fixture]) + end) =~ "undefined variable \"unknown_var\"" + after + purge([CompileQuoted]) + end + + test "does not hang on missing dependencies" do + [fixture] = + write_tmp( + "compile_does_not_hang", + with_behaviour_and_struct: """ + # We need to ensure it won't block even after multiple calls. + # So we use both behaviour and struct expansion below. + defmodule WithBehaviourAndStruct do + # @behaviour will call ensure_compiled(). + @behaviour :unknown + # Struct expansion calls it as well. + %ThisModuleWillNeverBeAvailable{} + end + """ + ) + + expected_msg = + "ThisModuleWillNeverBeAvailable.__struct__/1 is undefined, cannot expand struct ThisModuleWillNeverBeAvailable" + + assert capture_io(:stderr, fn -> + assert {:error, + [ + %{file: ^fixture, position: {7, 3}, message: msg}, + %{file: ^fixture, position: 0, message: compile_msg} + ], @no_warnings} = + compile([fixture]) + + assert msg =~ expected_msg + + assert compile_msg =~ + "cannot compile module WithBehaviourAndStruct (errors have been logged)" + end) =~ expected_msg + end + + test "does not deadlock on missing dependencies" do + [missing_struct, depends_on] = + write_tmp( + "does_not_deadlock", + missing_struct: """ + defmodule MissingStruct do + %ThisModuleWillNeverBeAvailable{} + def hello, do: :ok + end + """, + depends_on_missing_struct: """ + MissingStruct.hello() + """ + ) + + expected_msg = + "ThisModuleWillNeverBeAvailable.__struct__/1 is undefined, cannot expand struct ThisModuleWillNeverBeAvailable" + + assert capture_io(:stderr, fn -> + assert {:error, + [ + %{file: ^missing_struct, position: {2, 3}, message: msg}, + %{file: ^missing_struct, position: 0, message: compile_msg} + ], @no_warnings} = + compile([missing_struct, depends_on]) + + assert msg =~ expected_msg + + assert compile_msg =~ + "cannot compile module MissingStruct (errors have been logged)" + end) =~ expected_msg + end + + test "does not deadlock on missing import/struct dependencies" do + [missing_import, depends_on] = + write_tmp( + "import_and_structs", + missing_import: """ + defmodule MissingStruct do + import Unknown.Module + end + """, + depends_on_missing_struct: """ + %MissingStruct{} + """ + ) + + expected_msg = "module Unknown.Module is not loaded and could not be found" + + assert capture_io(:stderr, fn -> + assert {:error, + [ + %{file: ^missing_import, position: {2, 3}, message: msg}, + %{file: ^missing_import, position: 0, message: compile_msg} + ], @no_warnings} = + compile([missing_import, depends_on]) + + assert msg =~ expected_msg + + assert compile_msg =~ + "cannot compile module MissingStruct (errors have been logged)" + end) =~ expected_msg + end + + test "handles deadlocks" do + [foo, bar] = + write_tmp( + "parallel_deadlock", + foo: """ + defmodule FooDeadlock do + BarDeadlock.__info__(:module) + end + """, + bar: """ + defmodule BarDeadlock do + %FooDeadlock{} + end + """ + ) + + msg = + capture_io(:stderr, fn -> + fixtures = [foo, bar] + assert {:error, [bar_error, foo_error], @no_warnings} = compile(fixtures) + + assert %{file: ^bar, position: 2, message: "deadlocked waiting on struct FooDeadlock"} = + bar_error + + assert %{file: ^foo, position: nil, message: "deadlocked waiting on module BarDeadlock"} = + foo_error + end) + + assert msg =~ "Compilation failed because of a deadlock between files." + assert msg =~ "parallel_deadlock/foo.ex => BarDeadlock" + assert msg =~ "parallel_deadlock/bar.ex => FooDeadlock" + assert msg =~ ~r"== Compilation error in file .+parallel_deadlock/foo\.ex ==" + assert msg =~ "** (CompileError) deadlocked waiting on module BarDeadlock" + assert msg =~ ~r"== Compilation error in file .+parallel_deadlock/bar\.ex:2 ==" + assert msg =~ "** (CompileError) deadlocked waiting on struct FooDeadlock" + end + + test "does not deadlock from Code.ensure_compiled" do + [foo, bar] = + write_tmp( + "parallel_ensure_nodeadlock", + foo: """ + defmodule FooCircular do + {:error, :unavailable} = Code.ensure_compiled(BarCircular) + end + """, + bar: """ + defmodule BarCircular do + {:error, :unavailable} = Code.ensure_compiled(FooCircular) + end + """ + ) + + assert {:ok, _modules, @no_warnings} = compile([foo, bar]) + assert Enum.sort([FooCircular, BarCircular]) == [BarCircular, FooCircular] + after + purge([FooCircular, BarCircular]) + end + + test "handles pmap compilation" do + [foo, bar] = + write_tmp( + "async_compile", + foo: """ + defmodule FooAsync do + true = Code.can_await_module_compilation?() + + [BarAsync] = + Kernel.ParallelCompiler.pmap([:ok], fn :ok -> + true = Code.can_await_module_compilation?() + BarAsync.__info__(:module) + end) + end + """, + bar: """ + defmodule BarAsync do + true = Code.can_await_module_compilation?() + end + """ + ) + + capture_io(:stderr, fn -> + fixtures = [foo, bar] + assert {:ok, modules, @no_warnings} = compile(fixtures) + assert FooAsync in modules + assert BarAsync in modules + end) + after + purge([FooAsync, BarAsync]) + end + + test "handles pmap deadlocks" do + [foo, bar] = + write_tmp( + "async_deadlock", + foo: """ + defmodule FooAsyncDeadlock do + Kernel.ParallelCompiler.pmap([:ok], fn :ok -> + BarAsyncDeadlock.__info__(:module) + end) + end + """, + bar: """ + defmodule BarAsyncDeadlock do + FooAsyncDeadlock.__info__(:module) + end + """ + ) + + capture_io(:stderr, fn -> + fixtures = [foo, bar] + assert {:error, [bar_error, foo_error], @no_warnings} = compile(fixtures) + + assert %{ + file: ^bar, + position: nil, + message: "deadlocked waiting on module FooAsyncDeadlock" + } = bar_error + + assert %{file: ^foo, position: nil, message: "deadlocked waiting on pmap [#PID<" <> _} = + foo_error + end) + end + + test "does not use incorrect line number when error originates in another file" do + File.mkdir_p!(tmp_path()) + + [a, b] = + write_tmp( + "error_line", + a: """ + defmodule PCA do + def fun(arg), do: arg / 2 + end + """, + b: """ + defmodule PCB do + def fun(arg) do + PCA.fun(arg) + :ok + end + end + PCB.fun(:not_a_number) + """ + ) + + capture_io(:stderr, fn -> + assert {:error, [%{file: ^b, position: 0, message: _}], _} = compile([a, b]) + end) + end + + test "gets both source and file on @file annotations" do + File.mkdir_p!(tmp_path()) + + [a] = + write_tmp( + "file_source", + a: """ + defmodule FileAttr do + @file "unknown.foo.bar" + def fun, do: (unused = :ok) + end + """ + ) + + capture_io(:stderr, fn -> + assert {:ok, [FileAttr], %{compile_warnings: [%{source: ^a, file: file, message: _}]}} = + compile([a]) + + assert String.ends_with?(file, "unknown.foo.bar") + assert Path.type(file) == :absolute + end) + after + purge([FileAttr]) + end + + test "gets correct line number for UndefinedFunctionError" do + File.mkdir_p!(tmp_path()) + + [fixture] = + write_tmp("undef", + undef: """ + defmodule UndefErrorLine do + Bogus.fun() + end + """ + ) + + capture_io(:stderr, fn -> + assert {:error, [%{file: ^fixture, position: 2, message: _}], _} = compile([fixture]) + end) + end + + test "gets correct file+line+column number for SyntaxError" do + File.mkdir_p!(tmp_path()) + + [fixture] = + write_tmp("error", + error: """ + raise SyntaxError, file: "foo/bar.ex", line: 3, column: 10 + """ + ) + + file = Path.absname("foo/bar.ex") + + capture_io(:stderr, fn -> + assert {:error, [%{file: ^file, source: ^fixture, position: {3, 10}}], _} = + compile([fixture]) + end) + end + + test "gets proper beam destinations from dynamic modules" do + fixtures = + write_tmp( + "dynamic", + dynamic: """ + Module.create(Dynamic, quote(do: :ok), file: "dynamic.ex") + [_ | _] = :code.which(Dynamic) + """ + ) + + assert {:ok, [Dynamic], @no_warnings} = compile(fixtures, dest: "sample") + after + purge([Dynamic]) + end + end + + describe "require" do + test "returns struct undefined error when local struct is undefined" do + [fixture] = + write_tmp( + "require_struct", + undef: """ + defmodule Undef do + def undef() do + %__MODULE__{} + end + end + """ + ) + + expected_msg = "Undef.__struct__/1 is undefined, cannot expand struct Undef" + + assert capture_io(:stderr, fn -> + assert {:error, + [ + %{file: ^fixture, position: {3, 5}, message: msg}, + %{file: ^fixture, position: 0, message: compile_msg} + ], @no_warnings} = + Kernel.ParallelCompiler.require([fixture], return_diagnostics: true) + + assert msg =~ expected_msg + assert compile_msg =~ "cannot compile module Undef (errors have been logged)" + end) =~ expected_msg + end + + test "does not hang on missing dependencies" do + [fixture] = + write_tmp( + "require_does_not_hang", + with_behaviour_and_struct: """ + # We need to ensure it won't block even after multiple calls. + # So we use both behaviour and struct expansion below. + defmodule WithBehaviourAndStruct do + # @behaviour will call ensure_compiled(). + @behaviour :unknown + # Struct expansion calls it as well. + %ThisModuleWillNeverBeAvailable{} + end + """ + ) + + expected_msg = + "ThisModuleWillNeverBeAvailable.__struct__/1 is undefined, cannot expand struct ThisModuleWillNeverBeAvailable" + + assert capture_io(:stderr, fn -> + assert {:error, + [ + %{file: ^fixture, position: {7, 3}, message: msg}, + %{file: ^fixture, position: 0, message: compile_msg} + ], @no_warnings} = + Kernel.ParallelCompiler.require([fixture], return_diagnostics: true) + + assert msg =~ expected_msg + + assert compile_msg =~ + "cannot compile module WithBehaviourAndStruct (errors have been logged)" + end) =~ expected_msg + end + end +end diff --git a/lib/elixir/test/elixir/kernel/parser_test.exs b/lib/elixir/test/elixir/kernel/parser_test.exs new file mode 100644 index 00000000000..8502a5a0b4d --- /dev/null +++ b/lib/elixir/test/elixir/kernel/parser_test.exs @@ -0,0 +1,1677 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team + +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Kernel.ParserTest do + use ExUnit.Case, async: true + + test "empty" do + assert Code.string_to_quoted!("") == {:__block__, [line: 1], []} + assert Code.string_to_quoted!("", columns: true) == {:__block__, [line: 1, column: 1], []} + + assert Code.string_to_quoted!(" \n") == {:__block__, [line: 1], []} + assert Code.string_to_quoted!(" \n", columns: true) == {:__block__, [line: 1, column: 1], []} + end + + describe "nullary ops" do + test "in expressions" do + assert parse!("..") == {:.., [line: 1], []} + assert parse!("...") == {:..., [line: 1], []} + end + + test "in capture" do + assert parse!("&../0") == {:&, [line: 1], [{:/, [line: 1], [{:.., [line: 1], nil}, 0]}]} + assert parse!("&.../0") == {:&, [line: 1], [{:/, [line: 1], [{:..., [line: 1], nil}, 0]}]} + end + + test "raises on ambiguous uses when also binary" do + assert_raise SyntaxError, ~r/syntax error before: do/, fn -> + parse!("if .. do end") + end + end + end + + describe "unary ops" do + test "in keywords" do + assert parse!("f(!: :ok)") == {:f, [line: 1], [[!: :ok]]} + assert parse!("f @: :ok") == {:f, [line: 1], [[@: :ok]]} + end + + test "in maps" do + assert parse!("%{+foo, bar => bat, ...baz}") == + {:%{}, [line: 1], + [ + {:+, [line: 1], [{:foo, [line: 1], nil}]}, + {{:bar, [line: 1], nil}, {:bat, [line: 1], nil}}, + {:..., [line: 1], [{:baz, [line: 1], nil}]} + ]} + end + + test "ambiguous ops" do + assert parse!("f -var") == + {:f, [ambiguous_op: nil, line: 1], [{:-, [line: 1], [{:var, [line: 1], nil}]}]} + + assert parse!("f -(var)") == + {:f, [ambiguous_op: nil, line: 1], [{:-, [line: 1], [{:var, [line: 1], nil}]}]} + + assert parse!("f +-var") == + {:f, [{:ambiguous_op, nil}, {:line, 1}], + [{:+, [line: 1], [{:-, [line: 1], [{:var, [line: 1], nil}]}]}]} + + assert parse!("f - var") == + {:-, [line: 1], [{:f, [line: 1], nil}, {:var, [line: 1], nil}]} + + assert parse!("f --var") == + {:--, [line: 1], [{:f, [line: 1], nil}, {:var, [line: 1], nil}]} + + assert parse!("(f ->var)") == + [{:->, [line: 1], [[{:f, [line: 1], nil}], {:var, [line: 1], nil}]}] + end + + test "ambiguous ops in keywords" do + assert parse!("f(+: :ok)") == {:f, [line: 1], [[+: :ok]]} + assert parse!("f +: :ok") == {:f, [line: 1], [[+: :ok]]} + assert parse!("f +:\n:ok") == {:f, [line: 1], [[+: :ok]]} + end + end + + describe "ternary ops" do + test "root" do + assert parse!("1..2//3") == {:..//, [line: 1], [1, 2, 3]} + assert parse!("(1..2)//3") == {:..//, [line: 1], [1, 2, 3]} + end + + test "with do-blocks" do + assert parse!("foo do end..bar do end//baz do end") == { + :..//, + [line: 1], + [ + {:foo, [line: 1], [[do: {:__block__, [line: 1], []}]]}, + {:bar, [line: 1], [[do: {:__block__, [line: 1], []}]]}, + {:baz, [line: 1], [[do: {:__block__, [line: 1], []}]]} + ] + } + end + + test "with no parens" do + assert parse!("1..foo do end//bar bat") == { + :..//, + [line: 1], + [ + 1, + {:foo, [line: 1], [[do: {:__block__, [line: 1], []}]]}, + {:bar, [line: 1], [{:bat, [line: 1], nil}]} + ] + } + end + + test "errors" do + msg = + "the range step operator (//) must immediately follow the range definition operator (..)" + + assert_syntax_error([msg], "foo..bar baz//bat") + assert_syntax_error([msg], "foo++bar//bat") + assert_syntax_error([msg], "foo..(bar//bat)") + end + end + + describe "\\\\ + newline" do + test "with ambiguous ops" do + assert parse!("f \\\n-var") == + {:f, [ambiguous_op: nil, line: 1], [{:-, [line: 2], [{:var, [line: 2], nil}]}]} + + assert parse!("f \\\n- var") == + {:-, [line: 2], [{:f, [line: 1], nil}, {:var, [line: 2], nil}]} + + assert parse!("f -\\\nvar") == + {:-, [line: 1], [{:f, [line: 1], nil}, {:var, [line: 2], nil}]} + + assert parse!("f -\\\n var") == + {:-, [line: 1], [{:f, [line: 1], nil}, {:var, [line: 2], nil}]} + end + + test "with capture" do + assert parse!("&..//\\\n/3") == + {:&, [line: 1], [{:/, [line: 2], [{:..//, [line: 1], nil}, 3]}]} + + assert parse!("&\\\n+/2") == {:&, [line: 1], [{:/, [line: 2], [{:+, [line: 2], nil}, 2]}]} + assert parse!("&\\\n//2") == {:&, [line: 1], [{:/, [line: 2], [{:/, [line: 2], nil}, 2]}]} + assert parse!("&\\\nor/2") == {:&, [line: 1], [{:/, [line: 2], [{:or, [line: 2], nil}, 2]}]} + + assert parse!("&+\\\n/2") == {:&, [line: 1], [{:/, [line: 2], [{:+, [line: 1], nil}, 2]}]} + assert parse!("&/\\\n/2") == {:&, [line: 1], [{:/, [line: 2], [{:/, [line: 1], nil}, 2]}]} + assert parse!("&or\\\n/2") == {:&, [line: 1], [{:/, [line: 2], [{:or, [line: 1], nil}, 2]}]} + + assert parse!("&+/\\\n2") == {:&, [line: 1], [{:/, [line: 1], [{:+, [line: 1], nil}, 2]}]} + assert parse!("&//\\\n2") == {:&, [line: 1], [{:/, [line: 1], [{:/, [line: 1], nil}, 2]}]} + assert parse!("&or/\\\n2") == {:&, [line: 1], [{:/, [line: 1], [{:or, [line: 1], nil}, 2]}]} + end + end + + describe "identifier unicode normalization" do + test "stops at ascii codepoints" do + assert {:ok, {:ç, _, nil}} = Code.string_to_quoted("ç\n") + assert {:ok, {:\\, _, [{:ç, _, nil}, 1]}} = Code.string_to_quoted(~S"ç\\1") + end + + test "nfc normalization is performed" do + # before elixir 1.14, non-nfc would error + # non-nfc: "ç" (code points 0x0063 0x0327) + # nfc-normalized: "ç" (code points 0x00E7) + assert Code.eval_string("ç = 1; ç") == {1, [ç: 1]} + end + + test "elixir's additional normalization is performed" do + # Common micro => Greek mu. See code formatter test too. + assert Code.eval_string("µs = 1; μs") == {1, [{:μs, 1}]} + + # commented out: math symbols capability in elixir + # normalizations, to ensure that we *can* handle codepoints + # that are Common-script and non-ASCII + # assert Code.eval_string("_ℕ𝕩 = 1") == {1, [{:"_ℕ𝕩", 1}]} + end + + test "handles graphemes inside quoted identifiers" do + string_to_quoted = + fn code -> + Code.string_to_quoted!(code, + token_metadata: true, + literal_encoder: &{:ok, {:__block__, &2, [&1]}}, + emit_warnings: false + ) + end + + assert { + {:., _, [{:foo, _, nil}, :"➡️"]}, + [no_parens: true, delimiter: ~S["], line: 1], + [] + } = string_to_quoted.(~S|foo."➡️"|) + + assert { + {:., _, [{:foo, _, nil}, :"➡️"]}, + [no_parens: true, delimiter: ~S['], line: 1], + [] + } = string_to_quoted.(~S|foo.'➡️'|) + + assert {:__block__, [delimiter: ~S["], line: 1], [:"➡️"]} = string_to_quoted.(~S|:"➡️"|) + + assert {:__block__, [delimiter: ~S['], line: 1], [:"➡️"]} = string_to_quoted.(~S|:'➡️'|) + + assert {:__block__, [closing: [line: 1], line: 1], + [ + [ + {{:__block__, [delimiter: ~S["], format: :keyword, line: 1], [:"➡️"]}, + {:x, [line: 1], nil}} + ] + ]} = string_to_quoted.(~S|["➡️": x]|) + + assert {:__block__, [closing: [line: 1], line: 1], + [ + [ + {{:__block__, [delimiter: ~S['], format: :keyword, line: 1], [:"➡️"]}, + {:x, [line: 1], nil}} + ] + ]} = string_to_quoted.(~S|['➡️': x]|) + end + end + + describe "strings/sigils" do + test "delimiter information for sigils is included" do + string_to_quoted = &Code.string_to_quoted!(&1, token_metadata: false) + + assert parse!("~r/foo/") == + {:sigil_r, [delimiter: "/", line: 1], [{:<<>>, [line: 1], ["foo"]}, []]} + + assert string_to_quoted.("~r[foo]") == + {:sigil_r, [delimiter: "[", line: 1], [{:<<>>, [line: 1], ["foo"]}, []]} + + assert string_to_quoted.("~r\"foo\"") == + {:sigil_r, [delimiter: "\"", line: 1], [{:<<>>, [line: 1], ["foo"]}, []]} + + meta = [delimiter: "\"\"\"", line: 1] + args = {:sigil_S, meta, [{:<<>>, [indentation: 0, line: 1], ["sigil heredoc\n"]}, []]} + assert string_to_quoted.("~S\"\"\"\nsigil heredoc\n\"\"\"") == args + + meta = [delimiter: "'''", line: 1] + args = {:sigil_S, meta, [{:<<>>, [indentation: 0, line: 1], ["sigil heredoc\n"]}, []]} + assert string_to_quoted.("~S'''\nsigil heredoc\n'''") == args + end + + test "valid multi-letter sigils" do + string_to_quoted = &Code.string_to_quoted!(&1, token_metadata: false) + + assert string_to_quoted.("~REGEX/foo/") == + {:sigil_REGEX, [delimiter: "/", line: 1], [{:<<>>, [line: 1], ["foo"]}, []]} + + assert string_to_quoted.("~REGEX/foo/mods") == + {:sigil_REGEX, [delimiter: "/", line: 1], [{:<<>>, [line: 1], ["foo"]}, ~c"mods"]} + + assert string_to_quoted.("~REGEX[foo]") == + {:sigil_REGEX, [delimiter: "[", line: 1], [{:<<>>, [line: 1], ["foo"]}, []]} + + meta = [delimiter: "\"\"\"", line: 1] + args = {:sigil_MAT, meta, [{:<<>>, [indentation: 0, line: 1], ["1,2,3\n"]}, []]} + assert string_to_quoted.("~MAT\"\"\"\n1,2,3\n\"\"\"") == args + + args = {:sigil_FOO1, meta, [{:<<>>, [indentation: 0, line: 1], ["1,2,3\n"]}, []]} + assert string_to_quoted.("~FOO1\"\"\"\n1,2,3\n\"\"\"") == args + + args = {:sigil_BAR321, meta, [{:<<>>, [indentation: 0, line: 1], ["1,2,3\n"]}, []]} + assert string_to_quoted.("~BAR321\"\"\"\n1,2,3\n\"\"\"") == args + + args = {:sigil_I18N, meta, [{:<<>>, [indentation: 0, line: 1], ["1,2,3\n"]}, []]} + assert string_to_quoted.("~I18N\"\"\"\n1,2,3\n\"\"\"") == args + end + + test "invalid multi-letter sigils" do + msg = + ~r/invalid sigil name, it should be either a one-letter lowercase letter or an uppercase letter optionally followed by uppercase letters and digits/ + + assert_syntax_error(["nofile:1:1:", msg], "~Regex/foo/") + + assert_syntax_error(["nofile:1:1:", msg], "~FOo1{bar]") + + assert_syntax_error(["nofile:1:1:", msg], "~foo1{bar]") + end + + test "sigil newlines" do + assert {:sigil_s, _, [{:<<>>, _, ["here\ndoc"]}, []]} = + Code.string_to_quoted!(~s|~s"here\ndoc"|) + + assert {:sigil_s, _, [{:<<>>, _, ["here\r\ndoc"]}, []]} = + Code.string_to_quoted!(~s|~s"here\r\ndoc"|) + end + + test "string newlines" do + assert Code.string_to_quoted!(~s|"here\ndoc"|) == "here\ndoc" + assert Code.string_to_quoted!(~s|"here\r\ndoc"|) == "here\r\ndoc" + assert Code.string_to_quoted!(~s|"here\\\ndoc"|) == "heredoc" + assert Code.string_to_quoted!(~s|"here\\\r\ndoc"|) == "heredoc" + end + + test "heredoc newlines" do + assert Code.string_to_quoted!(~s|"""\nhere\ndoc\n"""|) == "here\ndoc\n" + assert Code.string_to_quoted!(~s|"""\r\nhere\r\ndoc\r\n"""|) == "here\r\ndoc\r\n" + assert Code.string_to_quoted!(~s| """\n here\n doc\n """|) == "here\ndoc\n" + assert Code.string_to_quoted!(~s| """\r\n here\r\n doc\r\n """|) == "here\r\ndoc\r\n" + assert Code.string_to_quoted!(~s|"""\nhere\\\ndoc\\\n"""|) == "heredoc" + assert Code.string_to_quoted!(~s|"""\r\nhere\\\r\ndoc\\\r\n"""|) == "heredoc" + end + + test "heredoc indentation" do + meta = [delimiter: "'''", line: 1] + args = {:sigil_S, meta, [{:<<>>, [indentation: 2, line: 1], [" sigil heredoc\n"]}, []]} + assert Code.string_to_quoted!("~S'''\n sigil heredoc\n '''") == args + end + end + + describe "string_to_quoted/2" do + test "converts strings to quoted expressions" do + assert Code.string_to_quoted("1 + 2") == {:ok, {:+, [line: 1], [1, 2]}} + + assert Code.string_to_quoted("a.1") == + {:error, {[line: 1, column: 3], "syntax error before: ", "\"1\""}} + end + end + + describe "string_to_quoted/2 and atom handling" do + test "ensures :existing_atoms_only" do + assert Code.string_to_quoted(":there_is_no_such_atom", existing_atoms_only: true) == + {:error, + {[line: 1, column: 1], "unsafe atom does not exist: ", "there_is_no_such_atom"}} + + assert Code.string_to_quoted("~UNKNOWN'foo bar'", existing_atoms_only: true) == + {:error, {[line: 1, column: 1], "unsafe atom does not exist: ", "sigil_UNKNOWN"}} + end + + test "encodes atoms" do + ref = make_ref() + + encoder = fn atom, meta -> + assert atom == "there_is_no_such_atom" + assert meta[:line] == 1 + assert meta[:column] == 1 + {:ok, {:my, "atom", ref}} + end + + assert {:ok, {:my, "atom", ^ref}} = + Code.string_to_quoted(":there_is_no_such_atom", static_atoms_encoder: encoder) + end + + test "encodes vars" do + ref = make_ref() + + encoder = fn atom, meta -> + assert atom == "there_is_no_such_var" + assert meta[:line] == 1 + assert meta[:column] == 1 + {:ok, {:my, "atom", ref}} + end + + assert {:ok, {{:my, "atom", ^ref}, [line: 1], nil}} = + Code.string_to_quoted("there_is_no_such_var", static_atoms_encoder: encoder) + end + + test "encodes quoted keyword keys" do + ref = make_ref() + + encoder = fn atom, meta -> + assert atom == "there is no such key" + assert meta[:line] == 1 + assert meta[:column] == 2 + {:ok, {:my, "atom", ref}} + end + + assert {:ok, [{{:my, "atom", ^ref}, true}]} = + Code.string_to_quoted(~S(["there is no such key": true]), + static_atoms_encoder: encoder + ) + end + + test "encodes multi-letter sigils" do + ref = make_ref() + + encoder = fn atom, meta -> + assert atom == "sigil_UNKNOWN" + assert meta[:line] == 1 + assert meta[:column] == 1 + {:ok, ref} + end + + assert {:ok, {^ref, [delimiter: "'", line: 1], [{:<<>>, [line: 1], ["abc"]}, []]}} = + Code.string_to_quoted("~UNKNOWN'abc'", static_atoms_encoder: encoder) + end + + test "addresses ambiguities" do + encoder = fn string, _meta -> {:ok, {:atom, string}} end + + # We check a=1 for precedence issues with a!=1, make sure it works + assert Code.string_to_quoted!("a = 1", static_atoms_encoder: encoder) + assert Code.string_to_quoted!("a=1", static_atoms_encoder: encoder) + end + + test "does not encode keywords" do + encoder = fn atom, _meta -> raise "shouldn't be invoked for #{atom}" end + + assert {:ok, {:fn, [line: 1], [{:->, [line: 1], [[1], 2]}]}} = + Code.string_to_quoted("fn 1 -> 2 end", static_atoms_encoder: encoder) + + assert {:ok, {:or, [line: 1], [true, false]}} = + Code.string_to_quoted("true or false", static_atoms_encoder: encoder) + + encoder = fn atom, _meta -> {:ok, {:encoded, atom}} end + + assert {:ok, [encoded: "true", encoded: "do", encoded: "and"]} = + Code.string_to_quoted("[:true, :do, :and]", static_atoms_encoder: encoder) + + assert {:ok, [{{:encoded, "do"}, 1}, {{:encoded, "true"}, 2}, {{:encoded, "end"}, 3}]} = + Code.string_to_quoted("[do: 1, true: 2, end: 3]", static_atoms_encoder: encoder) + end + + test "does not encode one-letter sigils" do + encoder = fn atom, _meta -> raise "shouldn't be invoked for #{atom}" end + + assert {:ok, {:sigil_z, [{:delimiter, "'"}, {:line, 1}], [{:<<>>, [line: 1], ["foo"]}, []]}} = + Code.string_to_quoted("~z'foo'", static_atoms_encoder: encoder) + + assert {:ok, {:sigil_Z, [{:delimiter, "'"}, {:line, 1}], [{:<<>>, [line: 1], ["foo"]}, []]}} = + Code.string_to_quoted("~Z'foo'", static_atoms_encoder: encoder) + end + + test "returns errors on long atoms even when using static_atoms_encoder" do + atom = String.duplicate("a", 256) + + encoder = fn atom, _meta -> {:ok, atom} end + + assert Code.string_to_quoted(atom, static_atoms_encoder: encoder) == + {:error, + {[line: 1, column: 1], "atom length must be less than system limit: ", atom}} + end + + test "avoids crashes on invalid AST" do + encoder = fn atom, _meta -> {:ok, {:atom, [], [atom]}} end + + assert {:error, {_, "missing terminator: )", ""}} = + Code.string_to_quoted("Module(", static_atoms_encoder: encoder) + + assert {:error, {_, "syntax error before: ", "'('"}} = + Code.string_to_quoted("Module()", static_atoms_encoder: encoder) + end + + test "may return errors" do + encoder = fn _atom, _meta -> + {:error, "Invalid atom name"} + end + + assert {:error, {[line: 1, column: 1], "Invalid atom name: ", "there_is_no_such_atom"}} = + Code.string_to_quoted(":there_is_no_such_atom", static_atoms_encoder: encoder) + + assert {:error, {[line: 1, column: 1], "Invalid atom name: ", "sigil_UNKNOWN"}} = + Code.string_to_quoted("~UNKNOWN'foo bar'", static_atoms_encoder: encoder) + end + + test "may return tuples" do + encoder = fn string, _metadata -> + try do + {:ok, String.to_existing_atom(string)} + rescue + ArgumentError -> + {:ok, {:user_atom, string}} + end + end + + assert {:ok, {:try, _, [[do: {:test, _, [{{:user_atom, "atom_does_not_exist"}, _, []}]}]]}} = + Code.string_to_quoted("try do: test(atom_does_not_exist())", + static_atoms_encoder: encoder + ) + end + end + + describe "string_to_quoted/2 with :columns" do + test "includes column information" do + string_to_quoted = &Code.string_to_quoted(&1, columns: true) + assert string_to_quoted.("1 + 2") == {:ok, {:+, [line: 1, column: 3], [1, 2]}} + + foo = {:foo, [line: 1, column: 1], nil} + bar = {:bar, [line: 1, column: 7], nil} + assert string_to_quoted.("foo + bar") == {:ok, {:+, [line: 1, column: 5], [foo, bar]}} + + nfc_abba = [225, 98, 98, 224] + nfd_abba = [97, 769, 98, 98, 97, 768] + context = [line: 1, column: 8] + expr = "\"ábbà\" = 1" + + assert string_to_quoted.(String.normalize(expr, :nfc)) == + {:ok, {:=, context, [List.to_string(nfc_abba), 1]}} + + assert string_to_quoted.(String.normalize(expr, :nfd)) == + {:ok, {:=, context, [List.to_string(nfd_abba), 1]}} + end + + test "not in" do + assert Code.string_to_quoted!("a not in b", columns: true) == + {:not, [line: 1, column: 3], + [ + {:in, [line: 1, column: 7], + [{:a, [line: 1, column: 1], nil}, {:b, [line: 1, column: 10], nil}]} + ]} + + assert Code.string_to_quoted!("a not in b", columns: true, token_metadata: true) == + {:not, [line: 1, column: 3], + [ + {:in, [line: 1, column: 8], + [{:a, [line: 1, column: 1], nil}, {:b, [line: 1, column: 11], nil}]} + ]} + + assert Code.string_to_quoted!("a\nnot in b", columns: true, token_metadata: true) == + {:not, [newlines: 1, line: 2, column: 1], + [ + {:in, [line: 2, column: 5], + [{:a, [line: 1, column: 1], nil}, {:b, [line: 2, column: 8], nil}]} + ]} + + assert Code.string_to_quoted!("a not in\nb", columns: true, token_metadata: true) == + {:not, [newlines: 1, line: 1, column: 3], + [ + {:in, [line: 1, column: 7], + [{:a, [line: 1, column: 1], nil}, {:b, [line: 2, column: 1], nil}]} + ]} + + assert Code.string_to_quoted!("a\nnot in\nb", columns: true, token_metadata: true) == + {:not, [newlines: 1, line: 2, column: 1], + [ + {:in, [line: 2, column: 5], + [{:a, [line: 1, column: 1], nil}, {:b, [line: 3, column: 1], nil}]} + ]} + end + + test "deprecated not/in" do + assert ExUnit.CaptureIO.capture_io(:stderr, fn -> + assert Code.string_to_quoted!("not a in b", columns: true) == + {:not, [line: 1, column: 1], + [ + {:in, [line: 1, column: 7], + [ + {:a, [line: 1, column: 5], nil}, + {:b, [line: 1, column: 10], nil} + ]} + ]} + end) =~ "not expr1 in expr2" + + assert ExUnit.CaptureIO.capture_io(:stderr, fn -> + assert Code.string_to_quoted!("!a in b", columns: true) == + {:!, [line: 1, column: 1], + [ + {:in, [line: 1, column: 4], + [ + {:a, [line: 1, column: 2], nil}, + {:b, [line: 1, column: 7], nil} + ]} + ]} + end) =~ "!expr1 in expr2" + end + + test "handles maps and structs" do + assert Code.string_to_quoted("%{}", columns: true) == + {:ok, {:%{}, [line: 1, column: 1], []}} + + assert Code.string_to_quoted("%:atom{}", columns: true) == + {:ok, {:%, [line: 1, column: 1], [:atom, {:%{}, [line: 1, column: 7], []}]}} + end + end + + describe "string_to_quoted/2 with :token_metadata" do + test "adds end_of_expression information to blocks" do + file = """ + one();two() + three() + + four() + + + five() + """ + + args = [ + {:one, + [ + end_of_expression: [newlines: 0, line: 1, column: 6], + closing: [line: 1, column: 5], + line: 1, + column: 1 + ], []}, + {:two, + [ + end_of_expression: [newlines: 1, line: 1, column: 12], + closing: [line: 1, column: 11], + line: 1, + column: 7 + ], []}, + {:three, + [ + end_of_expression: [newlines: 2, line: 2, column: 8], + closing: [line: 2, column: 7], + line: 2, + column: 1 + ], []}, + {:four, + [ + end_of_expression: [newlines: 3, line: 4, column: 7], + closing: [line: 4, column: 6], + line: 4, + column: 1 + ], []}, + {:five, + [ + end_of_expression: [newlines: 1, line: 7, column: 7], + closing: [line: 7, column: 6], + line: 7, + column: 1 + ], []} + ] + + assert Code.string_to_quoted!(file, token_metadata: true, columns: true) == + {:__block__, [], args} + end + + test "adds end_of_expression to the right hand side of ->" do + file = """ + case true do + :foo -> bar(); two() + :baz -> bat() + end + """ + + assert Code.string_to_quoted!(file, token_metadata: true) == + {:case, + [ + end_of_expression: [newlines: 1, line: 4], + do: [line: 1], + end: [line: 4], + line: 1 + ], + [ + true, + [ + do: [ + {:->, [line: 2], + [ + [:foo], + {:__block__, [], + [ + {:bar, + [ + end_of_expression: [newlines: 0, line: 2], + closing: [line: 2], + line: 2 + ], []}, + {:two, + [ + end_of_expression: [newlines: 1, line: 2], + closing: [line: 2], + line: 2 + ], []} + ]} + ]}, + {:->, [line: 3], + [ + [:baz], + {:bat, + [ + end_of_expression: [newlines: 1, line: 3], + closing: [line: 3], + line: 3 + ], []} + ]} + ] + ] + ]} + end + + test "end of expression with literal" do + file = """ + a do + d -> + ( + b -> c + ) + end + """ + + assert Code.string_to_quoted!(file, + token_metadata: true, + literal_encoder: &{:ok, {:__block__, &2, [&1]}} + ) == + {:a, + [ + end_of_expression: [newlines: 1, line: 6], + do: [line: 1], + end: [line: 6], + line: 1 + ], + [ + [ + {{:__block__, [line: 1], [:do]}, + [ + {:->, [newlines: 1, line: 2], + [ + [{:d, [line: 2], nil}], + {:__block__, + [ + end_of_expression: [newlines: 1, line: 5], + newlines: 1, + closing: [line: 5], + line: 3 + ], + [ + [ + {:->, [line: 4], + [ + [{:b, [line: 4], nil}], + {:c, [end_of_expression: [newlines: 1, line: 4], line: 4], nil} + ]} + ] + ]} + ]} + ]} + ] + ]} + end + + test "does not add end of expression to ->" do + file = """ + case true do + :foo -> :bar + :baz -> :bat + end\ + """ + + assert Code.string_to_quoted!(file, token_metadata: true) == + {:case, [do: [line: 1], end: [line: 4], line: 1], + [ + true, + [ + do: [ + {:->, [line: 2], [[:foo], :bar]}, + {:->, [line: 3], [[:baz], :bat]} + ] + ] + ]} + end + + test "adds pairing information" do + string_to_quoted = &Code.string_to_quoted!(&1, token_metadata: true) + + assert string_to_quoted.("foo") == {:foo, [line: 1], nil} + assert string_to_quoted.("foo()") == {:foo, [closing: [line: 1], line: 1], []} + + assert string_to_quoted.("foo(\n)") == + {:foo, [newlines: 1, closing: [line: 2], line: 1], []} + + assert string_to_quoted.("%{\n}") == {:%{}, [newlines: 1, closing: [line: 2], line: 1], []} + + assert string_to_quoted.("foo(\n) do\nend") == + {:foo, [do: [line: 2], end: [line: 3], newlines: 1, closing: [line: 2], line: 1], + [[do: {:__block__, [line: 2], []}]]} + + assert string_to_quoted.("foo(\n)(\n)") == + {{:foo, [newlines: 1, closing: [line: 2], line: 1], []}, + [newlines: 1, closing: [line: 3], line: 1], []} + end + + test "adds opening and closing information for single-expression block" do + file = "1 + (2 + 3)" + + assert Code.string_to_quoted!(file, token_metadata: true, columns: true) == + {:+, [line: 1, column: 3], + [ + 1, + {:+, + [ + parens: [closing: [line: 1, column: 11], line: 1, column: 5], + line: 1, + column: 8 + ], [2, 3]} + ]} + + file = "1 + ((2 + 3))" + + assert Code.string_to_quoted!(file, token_metadata: true, columns: true) == + {:+, [line: 1, column: 3], + [ + 1, + {:+, + [ + parens: [closing: [line: 1, column: 13], line: 1, column: 5], + parens: [closing: [line: 1, column: 12], line: 1, column: 6], + line: 1, + column: 9 + ], [2, 3]} + ]} + end + + test "adds opening and closing information for tuples" do + string_to_quoted = &Code.string_to_quoted!(&1, token_metadata: true, columns: true) + + assert string_to_quoted.("{}") == + {:{}, [closing: [line: 1, column: 2], line: 1, column: 1], []} + + assert string_to_quoted.("{123}") == + {:{}, [closing: [line: 1, column: 5], line: 1, column: 1], [123]} + + assert string_to_quoted.("x.{}") == + {{:., [line: 1, column: 2], [{:x, [line: 1, column: 1], nil}, :{}]}, + [closing: [line: 1, column: 4], line: 1, column: 2], []} + + assert string_to_quoted.("x.{123}") == + {{:., [line: 1, column: 2], [{:x, [line: 1, column: 1], nil}, :{}]}, + [closing: [line: 1, column: 7], line: 1, column: 2], [123]} + end + + test "adds opening and closing information for empty block" do + string_to_quoted = + &Code.string_to_quoted!(&1, token_metadata: true, columns: true, emit_warnings: false) + + file = "()" + + assert string_to_quoted.(file) == + {:__block__, [parens: [closing: [line: 1, column: 2], line: 1, column: 1]], []} + + file = "(())" + + assert string_to_quoted.(file) == + {:__block__, + [ + parens: [closing: [line: 1, column: 4], line: 1, column: 1], + parens: [closing: [line: 1, column: 3], line: 1, column: 2] + ], []} + + file = """ + ( + # Foo + ( + # Bar + ) + ) + """ + + assert string_to_quoted.(file) == + {:__block__, + [ + end_of_expression: [newlines: 1, line: 6, column: 2], + parens: [closing: [line: 6, column: 1], line: 1, column: 1], + end_of_expression: [newlines: 1, line: 5, column: 4], + parens: [closing: [line: 5, column: 3], line: 3, column: 3] + ], []} + end + + test "adds opening and closing information for stab arguments" do + file = "fn () -> x end " + + assert Code.string_to_quoted!(file, token_metadata: true, columns: true) == + {:fn, [closing: [line: 1, column: 12], line: 1, column: 1], + [ + {:->, + [ + parens: [closing: [line: 1, column: 5], line: 1, column: 4], + line: 1, + column: 7 + ], [[], {:x, [line: 1, column: 10], nil}]} + ]} + + file = "fn (x, y) -> x end " + + assert Code.string_to_quoted!(file, token_metadata: true, columns: true) == + { + :fn, + [{:closing, [line: 1, column: 16]}, {:line, 1}, {:column, 1}], + [ + {:->, + [ + parens: [closing: [line: 1, column: 9], line: 1, column: 4], + line: 1, + column: 11 + ], + [ + [{:x, [line: 1, column: 5], nil}, {:y, [line: 1, column: 8], nil}], + {:x, [line: 1, column: 14], nil} + ]} + ] + } + + file = "if true do (x, y) -> x end" + + assert Code.string_to_quoted!(file, token_metadata: true, columns: true) == + { + :if, + [ + {:do, [line: 1, column: 9]}, + {:end, [line: 1, column: 24]}, + {:line, 1}, + {:column, 1} + ], + [ + true, + [ + do: [ + {:->, + [ + parens: [closing: [line: 1, column: 17], line: 1, column: 12], + line: 1, + column: 19 + ], + [ + [{:x, [line: 1, column: 13], nil}, {:y, [line: 1, column: 16], nil}], + {:x, [line: 1, column: 22], nil} + ]} + ] + ] + ] + } + end + + test "with :literal_encoder" do + opts = [literal_encoder: &{:ok, {:__block__, &2, [&1]}}, token_metadata: true] + string_to_quoted = &Code.string_to_quoted!(&1, opts) + + assert string_to_quoted.(~s("one")) == {:__block__, [delimiter: "\"", line: 1], ["one"]} + assert string_to_quoted.("?é") == {:__block__, [token: "?é", line: 1], [233]} + assert string_to_quoted.("0b10") == {:__block__, [token: "0b10", line: 1], [2]} + assert string_to_quoted.("12") == {:__block__, [token: "12", line: 1], [12]} + assert string_to_quoted.("0o123") == {:__block__, [token: "0o123", line: 1], [83]} + assert string_to_quoted.("0xEF") == {:__block__, [token: "0xEF", line: 1], [239]} + assert string_to_quoted.("12.3") == {:__block__, [token: "12.3", line: 1], [12.3]} + assert string_to_quoted.("nil") == {:__block__, [line: 1], [nil]} + assert string_to_quoted.(":one") == {:__block__, [line: 1], [:one]} + + assert string_to_quoted.("true") == {:__block__, [line: 1], [true]} + assert string_to_quoted.(":true") == {:__block__, [format: :atom, line: 1], [true]} + + assert string_to_quoted.("[one: :two]") == { + :__block__, + [{:closing, [line: 1]}, {:line, 1}], + [ + [ + {{:__block__, [format: :keyword, line: 1], [:one]}, + {:__block__, [line: 1], [:two]}} + ] + ] + } + + assert string_to_quoted.("[1]") == + {:__block__, [closing: [line: 1], line: 1], + [[{:__block__, [token: "1", line: 1], [1]}]]} + + assert string_to_quoted.(~s("""\nhello\n""")) == + {:__block__, [delimiter: ~s["""], indentation: 0, line: 1], ["hello\n"]} + + assert string_to_quoted.(~s[fn (1) -> "hello" end]) == + {:fn, [closing: [line: 1], line: 1], + [ + {:->, [line: 1], + [ + [ + {:__block__, + [ + parens: [closing: [line: 1], line: 1], + token: "1", + line: 1 + ], [1]} + ], + {:__block__, [delimiter: "\"", line: 1], ["hello"]} + ]} + ]} + + assert string_to_quoted.("(1)") == + {:__block__, [parens: [closing: [line: 1], line: 1], token: "1", line: 1], [1]} + end + + test "adds identifier_location for qualified identifiers" do + string_to_quoted = &Code.string_to_quoted!(&1, token_metadata: true, columns: true) + + assert string_to_quoted.("foo.\nbar") == + {{:., [line: 1, column: 4], + [ + {:foo, [line: 1, column: 1], nil}, + :bar + ]}, [no_parens: true, line: 2, column: 1], []} + + assert string_to_quoted.("foo\n.\nbar") == + {{:., [line: 2, column: 1], + [ + {:foo, [line: 1, column: 1], nil}, + :bar + ]}, [no_parens: true, line: 3, column: 1], []} + + assert string_to_quoted.(~s[Foo.\nbar(1)]) == + {{:., [line: 1, column: 4], + [ + {:__aliases__, [last: [line: 1, column: 1], line: 1, column: 1], [:Foo]}, + :bar + ]}, [closing: [line: 2, column: 6], line: 2, column: 1], [1]} + end + + test "adds metadata for the last alias segment" do + string_to_quoted = &Code.string_to_quoted!(&1, token_metadata: true) + + assert string_to_quoted.("Foo") == {:__aliases__, [last: [line: 1], line: 1], [:Foo]} + + assert string_to_quoted.("Foo.\nBar\n.\nBaz") == + {:__aliases__, [last: [line: 4], line: 1], [:Foo, :Bar, :Baz]} + + assert string_to_quoted.("foo.\nBar\n.\nBaz") == + {:__aliases__, [last: [line: 4], line: 1], [{:foo, [line: 1], nil}, :Bar, :Baz]} + end + + test "adds metadata about assoc operator position in maps" do + opts = [ + literal_encoder: &{:ok, {:__block__, &2, [&1]}}, + token_metadata: true, + columns: true + ] + + string_to_quoted = &Code.string_to_quoted!(&1, opts) + + file = "%{:key => 1, {} => {}}" + + assert string_to_quoted.(file) == + { + :%{}, + [closing: [line: 1, column: 22], line: 1, column: 1], + [ + {{:__block__, [assoc: [line: 1, column: 8], line: 1, column: 3], [:key]}, + {:__block__, [token: "1", line: 1, column: 11], [1]}}, + { + {:{}, + [ + assoc: [line: 1, column: 17], + closing: [line: 1, column: 15], + line: 1, + column: 14 + ], []}, + {:{}, [closing: [line: 1, column: 21], line: 1, column: 20], []} + } + ] + } + end + end + + describe "syntax errors" do + test "invalid heredoc start" do + assert_syntax_error( + [ + "nofile:1:4:", + ~r/heredoc allows only whitespace characters followed by a new line after opening \"\"\"/ + ], + ~c"\"\"\"bar\n\"\"\"" + ) + end + + test "invalid fn" do + assert_syntax_error( + ["nofile:1:1:", "expected anonymous functions to be defined with -> inside: 'fn'"], + ~c"fn 1 end" + ) + + assert_syntax_error( + ["nofile:2:", "unexpected operator ->. If you want to define multiple clauses,"], + ~c"fn 1\n2 -> 3 end" + ) + end + + test "invalid token" do + assert_syntax_error( + ["nofile:1:1:", ~s/unexpected token: "#{"\u3164"}" (column 1, code point U+3164)/], + ~c"ㅤ = 1" + ) + + assert_syntax_error( + ["nofile:1:7:", ~s/unexpected token: "#{"\u200B"}" (column 7, code point U+200B)/], + ~c"[foo: \u200B]\noops" + ) + + assert_syntax_error( + ["nofile:1:1:", ~s/unexpected token: carriage return (column 1, code point U+000D)/], + ~c"\r" + ) + end + + test "invalid bidi in source" do + assert_syntax_error( + ["nofile:1:1:", ~s/invalid bidirectional formatting character in comment: \\u202A/], + ~c"# This is a \u202A" + ) + + assert_syntax_error( + ["nofile:1:6:", "invalid bidirectional formatting character in comment: \\u202A"], + ~c"foo. # This is a \u202A" + ) + + assert_syntax_error( + [ + "nofile:1:12:", + "invalid bidirectional formatting character in string: \\u202A. If you want to use such character, use it in its escaped \\u202A form instead" + ], + ~c"\"this is a \u202A\"" + ) + + assert_syntax_error( + [ + "nofile:1:13:", + "invalid bidirectional formatting character in string: \\u202A. If you want to use such character, use it in its escaped \\u202A form instead" + ], + ~c"\"this is a \\\u202A\"" + ) + end + + test "invalid newline in source" do + assert_syntax_error( + ["nofile:1:1:", ~s/invalid line break character in comment: \\u2028/], + ~c"# This is a \u2028" + ) + + assert_syntax_error( + ["nofile:1:6:", "invalid line break character in comment: \\u2028"], + ~c"foo. # This is a \u2028" + ) + + assert_syntax_error( + [ + "nofile:1:12:", + "invalid line break character in string: \\u2028. If you want to use such character, use it in its escaped \\u2028 form instead" + ], + ~c"\"this is a \u2028\"" + ) + + assert_syntax_error( + [ + "nofile:1:13:", + "invalid line break character in string: \\u2028. If you want to use such character, use it in its escaped \\u2028 form instead" + ], + ~c"\"this is a \\\u2028\"" + ) + end + + test "reserved tokens" do + assert_syntax_error(["nofile:1:1:", "reserved token: __aliases__"], ~c"__aliases__") + assert_syntax_error(["nofile:1:1:", "reserved token: __block__"], ~c"__block__") + end + + test "invalid alias terminator" do + assert_syntax_error(["nofile:1:4:", "unexpected ( after alias Foo"], ~c"Foo()") + end + + test "invalid quoted token" do + assert_syntax_error( + ["nofile:1:9:", "syntax error before: \"world\""], + ~c"\"hello\" \"world\"" + ) + + assert_syntax_error( + ["nofile:1:3:", "syntax error before: 'Foobar'"], + ~c"1 Foobar" + ) + + assert_syntax_error( + ["nofile:1:5:", "syntax error before: foo"], + ~c"Foo.:foo" + ) + + assert_syntax_error( + ["nofile:1:5:", "syntax error before: \"foo\""], + ~c"Foo.:\"foo\#{:bar}\"" + ) + + assert_syntax_error( + ["nofile:1:5:", "syntax error before: \""], + ~c"Foo.:\"\#{:bar}\"" + ) + end + + test "invalid identifier" do + message = + &["nofile:1:1:", ~s/invalid character "@" (code point U+0040) in identifier: #{&1}/] + + assert_syntax_error(message.("foo@"), ~c"foo@") + assert_syntax_error(message.("foo@"), ~c"foo@ ") + assert_syntax_error(message.("foo@bar"), ~c"foo@bar") + + message = + &["nofile:1:1:", "invalid character \"@\" (code point U+0040) in alias: #{&1}"] + + assert_syntax_error(message.("Foo@"), ~c"Foo@") + assert_syntax_error(message.("Foo@bar"), ~c"Foo@bar") + + message = + [ + "nofile:1:1:", + ~s/invalid character "!" (code point U+0021) in alias (only ASCII characters, without punctuation, are allowed): Foo!/ + ] + + assert_syntax_error(message, ~c"Foo!") + + message = + [ + "nofile:1:1:", + ~s/invalid character "?" (code point U+003F) in alias (only ASCII characters, without punctuation, are allowed): Foo?/ + ] + + assert_syntax_error(message, ~c"Foo?") + + message = + [ + "nofile:1:1:", + ~s/invalid character "ó" (code point U+00F3) in alias (only ASCII characters, without punctuation, are allowed): Foó/ + ] + + assert_syntax_error(message, ~c"Foó") + + # token suggestion heuristic: + # "for foO𝚳, NFKC isn't enough because 𝚳 nfkc's to Greek Μ, would be mixed script. + # however the 'confusability skeleton' for that token produces an all-Latin foOM + # and would tokenize -- so suggest that, in case that's what they want" + message = [ + "Codepoint failed identifier tokenization, but a simpler form was found.", + "Got:", + ~s/"foO𝚳" (code points 0x00066 0x0006F 0x0004F 0x1D6B3)/, + "Hint: You could write the above in a similar way that is accepted by Elixir:", + ~s/"foOM" (code points 0x00066 0x0006F 0x0004F 0x0004D)/, + "See https://hexdocs.pm/elixir/unicode-syntax.html for more information." + ] + + assert_syntax_error(message, ~c"foO𝚳") + + # token suggestion heuristic: + # "for fooی𝚳, both NKFC and confusability would result in mixed scripts, + # because the Farsi letter is confusable with a different Arabic letter. + # Well, can't fix it all at once -- let's check for a suggestion just on + # the one codepoint that triggered this, the 𝚳 -- that would at least + # nudge them forwards." + message = [ + "Elixir expects unquoted Unicode atoms, variables, and calls to use allowed codepoints and to be in NFC form.", + "Got:", + ~s/"𝚳" (code points 0x1D6B3)/, + "Hint: You could write the above in a compatible format that is accepted by Elixir:", + ~s/"Μ" (code points 0x0039C)/, + "See https://hexdocs.pm/elixir/unicode-syntax.html for more information." + ] + + assert_syntax_error(message, ~c"fooی𝚳") + end + + test "keyword missing space" do + msg = ["nofile:1:1:", "keyword argument must be followed by space after: foo:"] + + assert_syntax_error(msg, "foo:bar") + assert_syntax_error(msg, "foo:+") + assert_syntax_error(msg, "foo:+1") + end + + test "invalid keyword list in tuple/binary" do + assert_syntax_error( + ["unexpected keyword list inside tuple"], + ~c"{foo: :bar}" + ) + + assert_syntax_error( + ["unexpected keyword list inside tuple"], + ~c"{foo: :bar, baz: :bar}" + ) + + assert_syntax_error( + ["unexpected keyword list inside bitstring"], + ~c"<>" + ) + end + + test "expression after keyword lists" do + assert_syntax_error( + ["unexpected expression after keyword list"], + ~c"call foo: 1, :bar" + ) + + assert_syntax_error( + ["unexpected expression after keyword list"], + ~c"call(foo: 1, :bar)" + ) + + assert_syntax_error( + ["unexpected expression after keyword list"], + ~c"[foo: 1, :bar]" + ) + + assert_syntax_error( + ["unexpected expression after keyword list"], + ~c"%{foo: 1, :bar => :bar}" + ) + end + + test "syntax errors include formatted snippet" do + message = ["nofile:1:5:", "syntax error before:", "1 + * 3", "^"] + assert_syntax_error(message, "1 + * 3") + end + + test "invalid map start" do + assert_syntax_error( + ["nofile:1:7:", "expected %{ to define a map, got: %["], + "{:ok, %[], %{}}" + ) + + assert_syntax_error( + ["nofile:1:3:", "unexpected space between % and {"], + "% {1, 2, 3}" + ) + end + + test "invalid access" do + msg = ["nofile:1:6:", "too many arguments when accessing a value"] + assert_syntax_error(msg, "foo[1, 2]") + assert_syntax_error(msg, "foo[1, 2, 3]") + assert_syntax_error(msg, "foo[1, 2, 3,]") + end + + test "unexpected end" do + assert_syntax_error(["nofile:1:3:", "unexpected reserved word: end"], ~c"1 end") + + assert_syntax_error( + [ + "hint:", + "the \"end\" on line 2 may not have a matching \"do\" defined before it (based on indentation)" + ], + ~c""" + defmodule MyApp do + def one end + def two do end + end + """ + ) + + assert_syntax_error( + [ + "hint:", + "the \"end\" on line 3 may not have a matching \"do\" defined before it (based on indentation)" + ], + ~c""" + defmodule MyApp do + def one + end + + def two do + end + end + """ + ) + + assert_syntax_error( + [ + "hint:", + "the \"end\" on line 6 may not have a matching \"do\" defined before it (based on indentation)" + ], + ~c""" + defmodule MyApp do + def one do + end + + def two + end + end + """ + ) + end + + test "invalid keywords" do + assert_syntax_error( + ["nofile:1:2:", "syntax error before: '.'"], + ~c"+.foo" + ) + + assert_syntax_error( + ["nofile:1:1:", "syntax error before: after. \"after\" is a reserved word"], + ~c"after = 1" + ) + end + + test "before sigil" do + msg = &["nofile:1:9:", "syntax error before: sigil ~s starting with content '#{&1}'"] + + assert_syntax_error(msg.("bar baz"), ~c"~s(foo) ~s(bar baz)") + assert_syntax_error(msg.(""), ~c"~s(foo) ~s()") + assert_syntax_error(msg.("bar "), ~c"~s(foo) ~s(bar \#{:baz})") + assert_syntax_error(msg.(""), ~c"~s(foo) ~s(\#{:bar} baz)") + end + + test "invalid do" do + assert_syntax_error( + ["nofile:1:10:", "unexpected reserved word: do."], + ~c"if true, do\n" + ) + + assert_syntax_error(["nofile:1:9:", "unexpected keyword: do:."], ~c"if true do:\n") + end + + test "invalid parens call" do + msg = + [ + "nofile:1:5:", + "unexpected parentheses", + "If you are making a function call, do not insert spaces between the function name and the opening parentheses.", + "Syntax error before: '\('" + ] + + assert_syntax_error(msg, ~c"foo (hello, world)") + end + + test "invalid nested no parens call" do + msg = ["nofile:1:", "unexpected comma. Parentheses are required to solve ambiguity"] + + assert_syntax_error(msg, ~c"[foo 1, 2]") + assert_syntax_error(msg, ~c"[foo bar 1, 2]") + assert_syntax_error(msg, ~c"[do: foo 1, 2]") + assert_syntax_error(msg, ~c"foo(do: bar 1, 2)") + assert_syntax_error(msg, ~c"{foo 1, 2}") + assert_syntax_error(msg, ~c"{foo bar 1, 2}") + assert_syntax_error(msg, ~c"foo 1, foo 2, 3") + assert_syntax_error(msg, ~c"foo 1, @bar 3, 4") + assert_syntax_error(msg, ~c"foo 1, 2 + bar 3, 4") + assert_syntax_error(msg, ~c"foo(1, foo 2, 3)") + + interpret = fn x -> Macro.to_string(Code.string_to_quoted!(x)) end + assert interpret.("f 1 + g h 2, 3") == "f(1 + g(h(2, 3)))" + + assert interpret.("assert [] = TestRepo.all from p in Post, where: p.title in ^[]") == + "assert [] = TestRepo.all(from(p in Post, where: p.title in ^[]))" + end + + test "invalid atom dot alias" do + msg = + [ + "nofile:1:6:", + "atom cannot be followed by an alias. If the '.' was meant to be " <> + "part of the atom's name, the atom name must be quoted. Syntax error before: '.'" + ] + + assert_syntax_error(msg, ~c":foo.Bar") + assert_syntax_error(msg, ~c":\"+\".Bar") + end + + test "invalid map/struct" do + assert_syntax_error(["nofile:1:15:", "syntax error before: '}'"], ~c"%{foo bar, baz}") + assert_syntax_error(["nofile:1:8:", "syntax error before: '{'"], ~c"%{a, b}{a: :b}") + end + + test "mismatching delimiters" do + assert_mismatched_delimiter_error( + [ + "nofile:1:9:", + "unexpected token:", + "└ unclosed delimiter", + "└ mismatched closing delimiter" + ], + ~c"fn a -> )" + ) + + assert_mismatched_delimiter_error( + [ + "nofile:1:16:", + "unexpected token:", + "└ unclosed delimiter", + "└ mismatched closing delimiter" + ], + ~c"defmodule A do ]" + ) + + assert_mismatched_delimiter_error( + [ + "nofile:1:9:", + "unexpected token:", + "└ unclosed delimiter", + "└ mismatched closing delimiter" + ], + ~c"(1, 2, 3}" + ) + + assert_mismatched_delimiter_error( + [ + "nofile:1:14:", + "unexpected reserved word:", + "└ unclosed delimiter", + "└ mismatched closing delimiter" + ], + ~c"<<1, 2, 3, 4 end" + ) + end + + test "invalid interpolation" do + assert_mismatched_delimiter_error( + [ + "nofile:1:17:", + "unexpected token:", + "└ unclosed delimiter", + "└ mismatched closing delimiter" + ], + ~c"\"foo\#{case 1 do )}bar\"" + ) + + assert_mismatched_delimiter_error( + [ + "nofile:8:3:", + "unexpected token: )", + "└ unclosed delimiter", + "└ mismatched closing delimiter" + ], + ~c""" + defmodule MyApp do + ( + def one do + # end + + def two do + end + ) + end + """ + ) + end + + test "invalid end of expression" do + # All valid examples + Code.eval_quoted(~c""" + 1; + 2; + 3 + + (;) + (;1) + (1;) + (1; 2) + + fn -> 1; 2 end + fn -> ; end + + if true do + ; + end + + try do + ; + catch + _, _ -> ; + after + ; + end + """) + + # All invalid examples + assert_syntax_error(["nofile:1:3:", "syntax error before: ';'"], ~c"1+;\n2") + + assert_syntax_error(["nofile:1:8:", "syntax error before: ';'"], ~c"max(1, ;2)") + end + + test "invalid new line" do + assert_syntax_error( + [ + "nofile:3:6:", + "unexpectedly reached end of line. The current expression is invalid or incomplete", + "baz", + "^" + ], + ~c"if true do\n foo = [],\n baz\nend" + ) + end + + test "invalid \"fn do expr end\"" do + assert_syntax_error( + [ + "nofile:1:4:", + "unexpected reserved word: do. Anonymous functions are written as:", + "fn pattern -> expression end", + "Please remove the \"do\" keyword", + "fn do :ok end", + "^" + ], + ~c"fn do :ok end" + ) + end + + test "characters literal are printed correctly in syntax errors" do + assert_syntax_error(["nofile:1:5:", "syntax error before: ?a"], ~c":ok ?a") + assert_syntax_error(["nofile:1:5:", "syntax error before: ?\\s"], ~c":ok ?\\s") + assert_syntax_error(["nofile:1:5:", "syntax error before: ?す"], ~c":ok ?す") + end + + test "character literals take newlines into account" do + ExUnit.CaptureIO.capture_io(:stderr, fn -> + assert parse!("{?\n}\n{123}") == + {:__block__, [], [{:{}, [line: 1], ~c"\n"}, {:{}, [line: 3], ~c"{"}]} + + assert parse!("{?\\n}\n{123}") == + {:__block__, [], [{:{}, [line: 1], ~c"\n"}, {:{}, [line: 2], ~c"{"}]} + + assert parse!("{?\\\n}\n{123}") == + {:__block__, [], [{:{}, [line: 1], ~c"\n"}, {:{}, [line: 3], ~c"{"}]} + end) + end + + test "numbers are printed correctly in syntax errors" do + assert_syntax_error(["nofile:1:5:", ~s/syntax error before: "12"/], ~c":ok 12") + assert_syntax_error(["nofile:1:5:", ~s/syntax error before: "0b1"/], ~c":ok 0b1") + assert_syntax_error(["nofile:1:5:", ~s/syntax error before: "12.3"/], ~c":ok 12.3") + + assert_syntax_error( + ["nofile:1:1:", ~s/invalid character "_" after number 123_456/], + ~c"123_456_foo" + ) + end + + test "on hex errors" do + msg = + "invalid hex escape character, expected \\xHH where H is a hexadecimal digit. Syntax error after: \\x" + + assert_syntax_error(["nofile:1:2:", msg], ~S["\x"]) + assert_syntax_error(["nofile:1:1:", msg], ~S[:"\x"]) + assert_syntax_error(["nofile:1:2:", msg], ~S["\x": 123]) + assert_syntax_error(["nofile:1:1:", msg], ~s["""\n\\x\n"""]) + end + + test "on unicode errors" do + msg = "invalid Unicode escape character" + + assert_syntax_error(["nofile:1:2:", msg], ~S["\u"]) + assert_syntax_error(["nofile:1:1:", msg], ~S[:"\u"]) + assert_syntax_error(["nofile:1:2:", msg], ~S["\u": 123]) + assert_syntax_error(["nofile:1:1:", msg], ~s["""\n\\u\n"""]) + + assert_syntax_error( + [ + "nofile:1:2:", + "invalid or reserved Unicode code point \\u{FFFFFF}. Syntax error after: \\u" + ], + ~S["\u{FFFFFF}"] + ) + end + + test "on interpolation in calls" do + msg = + "interpolation is not allowed when calling function/macro. Found interpolation in a call starting with: \"" + + assert_syntax_error([msg], ".\"\#{}\"") + assert_syntax_error([msg], ".\"a\#{:b}\"c") + end + + test "on long atoms" do + atom = + "@GR{+z]`_XrNla!d0ptDp(amr.oS&,UbT}v$L|rHHXGV{;W!>avHbD[T-G5xrzR6m?rQPot-37B@" + + assert_syntax_error( + ["atom length must be less than system limit: "], + ~s{:"#{atom}"} + ) + + assert_syntax_error( + ["atom length must be less than system limit: "], + ~s{["#{atom}": 123]} + ) + end + end + + defp parse!(string), do: Code.string_to_quoted!(string) + + defp assert_syntax_error(given_messages, source) do + e = assert_raise SyntaxError, fn -> parse!(source) end + assert_exception_msg(e, given_messages) + end + + defp assert_mismatched_delimiter_error(given_messages, source) do + e = assert_raise MismatchedDelimiterError, fn -> parse!(source) end + assert_exception_msg(e, given_messages) + end + + defp assert_exception_msg(exception, messages) do + error_msg = Exception.format(:error, exception, []) + + for msg <- messages do + assert error_msg =~ msg + end + end +end diff --git a/lib/elixir/test/elixir/kernel/quote_test.exs b/lib/elixir/test/elixir/kernel/quote_test.exs index 2d8de03c0ae..8cfa3be07de 100644 --- a/lib/elixir/test/elixir/kernel/quote_test.exs +++ b/lib/elixir/test/elixir/kernel/quote_test.exs @@ -1,109 +1,222 @@ -Code.require_file "../test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) defmodule Kernel.QuoteTest do use ExUnit.Case, async: true - test :list do + @some_fun &List.flatten/1 + + test "fun" do + assert is_function(@some_fun) + end + + test "list" do assert quote(do: [1, 2, 3]) == [1, 2, 3] end - test :tuple do + test "tuple" do assert quote(do: {:a, 1}) == {:a, 1} end - test :keep_line do - ## DO NOT MOVE THIS LINE - assert quote(location: :keep, do: bar(1, 2, 3)) == {:bar, [keep: 16], [1, 2, 3]} + test "keep line" do + line = __ENV__.line + 2 + + assert quote(location: :keep, do: bar(1, 2, 3)) == + {:bar, [keep: {__ENV__.file, line}], [1, 2, 3]} end - test :fixed_line do + test "fixed line" do assert quote(line: 3, do: bar(1, 2, 3)) == {:bar, [line: 3], [1, 2, 3]} + assert quote(line: false, do: bar(1, 2, 3)) == {:bar, [], [1, 2, 3]} + assert quote(line: true, do: bar(1, 2, 3)) == {:bar, [line: __ENV__.line], [1, 2, 3]} + end + + test "file line" do + assert quote(file: "foo", line: 3, do: bar(1, 2, 3)) == + {:bar, [keep: {"foo", 3}], [1, 2, 3]} + + assert quote(file: "foo", line: false, do: bar(1, 2, 3)) == + {:bar, [keep: {"foo", 0}], [1, 2, 3]} + + assert quote(file: "foo", line: true, do: bar(1, 2, 3)) == + {:bar, [keep: {"foo", __ENV__.line - 1}], [1, 2, 3]} end - test :quote_line_var do - ## DO NOT MOVE THIS LINE + test "quote line var" do line = __ENV__.line - assert quote(line: line, do: bar(1, 2, 3)) == {:bar, [line: 25], [1, 2, 3]} + assert quote(line: line, do: bar(1, 2, 3)) == {:bar, [line: line], [1, 2, 3]} + + assert_raise ArgumentError, fn -> + line = "oops" + quote(line: line, do: bar(1, 2, 3)) + end + + assert_raise ArgumentError, fn -> + line = true + quote(line: line, do: bar(1, 2, 3)) + end end - test :unquote_call do + test "quote context var" do + context = :dynamic + assert quote(context: context, do: bar) == {:bar, [], :dynamic} + + assert_raise ArgumentError, fn -> + context = "oops" + quote(context: context, do: bar) + end + + assert_raise ArgumentError, fn -> + context = nil + quote(context: context, do: bar) + end + end + + test "quote context bind_quoted" do + assert {:__block__, _, + [{:=, [], [{:some_var, _, :fallback}, 321]}, {:some_var, _, :fallback}]} = + (quote bind_quoted: [some_var: 321], context: __ENV__.context || :fallback do + some_var + end) + end + + test "operator precedence" do + assert {:+, _, [{:+, _, [1, _]}, 1]} = quote(do: 1 + Foo.l() + 1) + assert {:+, _, [1, {_, _, [{:+, _, [1]}]}]} = quote(do: 1 + Foo.l(+1)) + end + + test "generated" do + assert quote(generated: true, do: bar(1)) == {:bar, [generated: true], [1]} + end + + test "unquote call" do assert quote(do: foo(bar)[unquote(:baz)]) == quote(do: foo(bar)[:baz]) assert quote(do: unquote(:bar)()) == quote(do: bar()) - assert quote(do: unquote(:bar)(1) do 2 + 3 end) == quote(do: bar(1) do 2 + 3 end) + + assert (quote do + unquote(:bar)(1) do + 2 + 3 + end + end) == + (quote do + bar 1 do + 2 + 3 + end + end) + assert quote(do: foo.unquote(:bar)) == quote(do: foo.bar) + assert quote(do: foo.unquote(:bar)()) == quote(do: foo.bar()) assert quote(do: foo.unquote(:bar)(1)) == quote(do: foo.bar(1)) - assert quote(do: foo.unquote(:bar)(1) do 2 + 3 end) == quote(do: foo.bar(1) do 2 + 3 end) + + assert (quote do + foo.unquote(:bar)(1) do + 2 + 3 + end + end) == + (quote do + foo.bar 1 do + 2 + 3 + end + end) + assert quote(do: foo.unquote({:bar, [], nil})) == quote(do: foo.bar) - assert quote(do: foo.unquote({:bar, [], [1,2]})) == quote(do: foo.bar(1,2)) + assert quote(do: foo.unquote({:bar, [], nil})()) == quote(do: foo.bar()) + assert quote(do: foo.unquote({:bar, [], [1, 2]})) == quote(do: foo.bar(1, 2)) - assert Code.eval_quoted(quote(do: Foo.unquote(Bar))) == {Elixir.Foo.Bar, []} - assert Code.eval_quoted(quote(do: Foo.unquote(quote do: Bar))) == {Elixir.Foo.Bar, []} + assert Code.eval_quoted(quote(do: Foo.unquote(Bar))) == {Elixir.Foo.Bar, []} + assert Code.eval_quoted(quote(do: Foo.unquote(quote(do: Bar)))) == {Elixir.Foo.Bar, []} assert_raise ArgumentError, fn -> quote(do: foo.unquote(1)) end end - test :nested_quote do + test "unquote call with dynamic line" do + assert quote(line: String.to_integer("123"), do: Foo.unquote(:bar)()) == + quote(line: 123, do: Foo.bar()) + end + + test "nested quote" do assert {:quote, _, [[do: {:unquote, _, _}]]} = quote(do: quote(do: unquote(x))) end + test "import inside nested quote" do + # Check that we can evaluate imports from quote inside quote + assert {{:to_string, meta, [123]}, _} = Code.eval_quoted(quote(do: quote(do: to_string(123)))) + assert meta[:imports] == [{1, Kernel}] + end + defmacrop nested_quote_in_macro do x = 1 + quote do x = unquote(x) + quote do unquote(x) end end end - test :nested_quote_in_macro do - assert nested_quote_in_macro == 1 + test "nested quote in macro" do + assert nested_quote_in_macro() == 1 end - Enum.each [foo: 1, bar: 2, baz: 3], fn {k, v} -> - def unquote(k)(arg) do - unquote(v) + arg + defmodule Dyn do + for {k, v} <- [foo: 1, bar: 2, baz: 3] do + # Local call unquote + def unquote(k)(), do: unquote(v) + + # Remote call unquote + def unquote(k)(arg), do: __MODULE__.unquote(k)() + arg end end - test :dynamic_definition_with_unquote do - assert foo(1) == 2 - assert bar(2) == 4 - assert baz(3) == 6 + test "dynamic definition with unquote" do + assert Dyn.foo() == 1 + assert Dyn.bar() == 2 + assert Dyn.baz() == 3 + + assert Dyn.foo(1) == 2 + assert Dyn.bar(2) == 4 + assert Dyn.baz(3) == 6 end - test :splice_on_root do + test "splice on root" do contents = [1, 2, 3] - assert quote(do: (unquote_splicing(contents))) == quote do: (1; 2; 3) + + assert quote(do: (unquote_splicing(contents))) == + (quote do + 1 + 2 + 3 + end) end - test :splice_with_tail do + test "splice with tail" do contents = [1, 2, 3] - assert quote(do: [unquote_splicing(contents)|[1, 2, 3]]) == - [1, 2, 3, 1, 2, 3] - assert quote(do: [unquote_splicing(contents)|val]) == - quote(do: [1, 2, 3 | val]) + assert quote(do: [unquote_splicing(contents) | [1, 2, 3]]) == [1, 2, 3, 1, 2, 3] - assert quote(do: [unquote_splicing(contents)|unquote([4])]) == - quote(do: [1, 2, 3, 4]) + assert quote(do: [unquote_splicing(contents) | val]) == quote(do: [1, 2, 3 | val]) + + assert quote(do: [unquote_splicing(contents) | unquote([4])]) == quote(do: [1, 2, 3, 4]) end - test :splice_on_stab do - {fun, []} = - Code.eval_quoted(quote(do: fn(unquote_splicing([1, 2, 3])) -> :ok end), []) + test "splice on stab" do + {fun, []} = Code.eval_quoted(quote(do: fn unquote_splicing([1, 2, 3]) -> :ok end), []) assert fun.(1, 2, 3) == :ok - {fun, []} = - Code.eval_quoted(quote(do: fn(1, unquote_splicing([2, 3])) -> :ok end), []) + {fun, []} = Code.eval_quoted(quote(do: fn 1, unquote_splicing([2, 3]) -> :ok end), []) assert fun.(1, 2, 3) == :ok end - test :splice_on_definition do + test "splice on definition" do defmodule Hello do - def world([unquote_splicing(["foo", "bar"])|rest]) do + def world([unquote_splicing(["foo", "bar"]) | rest]) do rest end end @@ -111,79 +224,150 @@ defmodule Kernel.QuoteTest do assert Hello.world(["foo", "bar", "baz"]) == ["baz"] end - test :splice_on_map do - assert %{unquote_splicing([foo: :bar])} == %{foo: :bar} - assert %{unquote_splicing([foo: :bar]), baz: :bat} == %{foo: :bar, baz: :bat} - assert %{unquote_splicing([foo: :bar]), :baz => :bat} == %{foo: :bar, baz: :bat} - assert %{:baz => :bat, unquote_splicing([foo: :bar])} == %{foo: :bar, baz: :bat} + test "splice on map" do + assert %{unquote_splicing(foo: :bar)} == %{foo: :bar} + assert %{unquote_splicing(foo: :bar), baz: :bat} == %{foo: :bar, baz: :bat} + assert %{unquote_splicing(foo: :bar), :baz => :bat} == %{foo: :bar, baz: :bat} + assert %{:baz => :bat, unquote_splicing(foo: :bar)} == %{foo: :bar, baz: :bat} map = %{foo: :default} - assert %{map | unquote_splicing([foo: :bar])} == %{foo: :bar} + assert %{map | unquote_splicing(foo: :bar)} == %{foo: :bar} end - test :when do - assert [{:->,_,[[{:when,_,[1,2,3,4]}],5]}] = quote(do: (1, 2, 3 when 4 -> 5)) - assert [{:->,_,[[{:when,_,[1,2,3,4]}],5]}] = quote(do: ((1, 2, 3) when 4 -> 5)) + test "when" do + assert [{:->, _, [[{:when, _, [1, 2, 3, 4]}], 5]}] = quote(do: (1, 2, 3 when 4 -> 5)) - assert [{:->,_,[[{:when,_,[1,2,3,{:when,_,[4,5]}]}],6]}] = - quote(do: ((1, 2, 3) when 4 when 5 -> 6)) + assert [{:->, _, [[{:when, _, [1, 2, 3, {:when, _, [4, 5]}]}], 6]}] = + quote(do: (1, 2, 3 when 4 when 5 -> 6)) end - test :stab do - assert [{:->, _, [[], nil]}] = (quote do -> end) - assert [{:->, _, [[], nil]}] = (quote do: (->)) + test "stab" do + assert [{:->, _, [[], 1]}] = + (quote do + () -> 1 + end) - assert [{:->, _, [[1], nil]}] = (quote do 1 -> end) - assert [{:->, _, [[1], nil]}] = (quote do: (1 ->)) + assert [{:->, _, [[], 1]}] = quote(do: (-> 1)) + end - assert [{:->, _, [[], 1]}] = (quote do -> 1 end) - assert [{:->, _, [[], 1]}] = (quote do: (-> 1)) + test "empty block" do + # Since ; is allowed by itself, it must also be allowed inside () + # The exception to this rule is an empty (). While empty expressions + # are allowed, an empty () is ambiguous. We also can't use quote here, + # since the formatter will rewrite (;) to something else. + assert {:ok, {:__block__, [line: 1], []}} = Code.string_to_quoted("(;)") end - test :bind_quoted do - assert quote(bind_quoted: [foo: 1 + 2], do: foo) == {:__block__, [], [ - {:=, [], [{:foo, [], Kernel.QuoteTest}, 3]}, + test "bind quoted" do + args = [ + {:=, [], [{:foo, [line: __ENV__.line + 4], Kernel.QuoteTest}, 3]}, {:foo, [], Kernel.QuoteTest} - ]} - end + ] - test :literals do - assert (quote do: []) == [] - assert (quote do: nil) == nil - assert (quote do [] end) == [] - assert (quote do nil end) == nil + quoted = quote(bind_quoted: [foo: 1 + 2], do: foo) + assert quoted == {:__block__, [], args} end - defmacrop dynamic_opts do - [line: 3] - end + test "literals" do + assert quote(do: []) == [] + assert quote(do: nil) == nil + + assert (quote do + [] + end) == [] - test :with_dynamic_opts do - assert quote(dynamic_opts, do: bar(1, 2, 3)) == {:bar, [line: 3], [1, 2, 3]} + assert (quote do + nil + end) == nil end - test :unary_with_integer_precedence do - assert quote(do: +1.foo) == quote(do: (+1).foo) - assert quote(do: @1.foo) == quote(do: (@1).foo) - assert quote(do: &1.foo) == quote(do: (&1).foo) + defmacrop dynamic_opts do + [line: 3] end - test :operators_slash_arity do - assert {:/, _, [{:+, _, _}, 2]} = quote do: +/2 - assert {:/, _, [{:&&, _, _}, 3]} = quote do: &&/3 + test "with dynamic opts" do + assert quote(dynamic_opts(), do: bar(1, 2, 3)) == {:bar, [line: 3], [1, 2, 3]} + end + + test "unary with integer precedence" do + assert quote(do: +1.foo) == quote(do: +1.foo) + assert quote(do: (@1).foo) == quote(do: (@1).foo) + assert quote(do: &1.foo) == quote(do: &1.foo) + end + + test "pipe precedence" do + assert {:|>, _, [{:|>, _, [{:foo, _, _}, {:bar, _, _}]}, {:baz, _, _}]} = + quote(do: foo |> bar |> baz) + + assert {:|>, _, [{:|>, _, [{:foo, _, _}, {:bar, _, _}]}, {:baz, _, _}]} = + (quote do + foo do + end + |> bar + |> baz + end) + + assert {:|>, _, [{:|>, _, [{:foo, _, _}, {:bar, _, _}]}, {:baz, _, _}]} = + (quote do + foo + |> bar do + end + |> baz + end) + + assert {:|>, _, [{:|>, _, [{:foo, _, _}, {:bar, _, _}]}, {:baz, _, _}]} = + (quote do + foo + |> bar + |> baz do + end + end) + + assert {:|>, _, [{:|>, _, [{:foo, _, _}, {:bar, _, _}]}, {:baz, _, _}]} = + (quote do + foo do + end + |> bar + |> baz do + end + end) + + assert {:|>, _, [{:|>, _, [{:foo, _, _}, {:bar, _, _}]}, {:baz, _, _}]} = + (quote do + foo do + end + |> bar do + end + |> baz do + end + end) + end + + test "capture" do + assert Code.string_to_quoted!("&1[:foo]") == Code.string_to_quoted!("(&1)[:foo]") + assert Code.string_to_quoted!("&1 [:foo]") == Code.string_to_quoted!("(&1)[:foo]") + assert Code.string_to_quoted!("& 1[:foo]") == Code.string_to_quoted!("&(1[:foo])") + end + + test "not and ! as rearrange ops" do + assert {:__block__, _, [{:not, [line: 1], [true]}]} = Code.string_to_quoted!("(not true)") + + assert {:fn, _, [{:->, _, [[], {:not, _, [true]}]}]} = + Code.string_to_quoted!("fn -> not true end") end end -## DO NOT MOVE THIS LINE defmodule Kernel.QuoteTest.Errors do - defmacro defadd do + def line, do: __ENV__.line + 4 + + defmacro defraise do quote location: :keep do - def add(a, b), do: a + b + def will_raise(_a, _b), do: raise("oops") end end defmacro will_raise do - quote location: :keep, do: raise "omg" + quote(location: :keep, do: raise("oops")) end end @@ -192,44 +376,49 @@ defmodule Kernel.QuoteTest.ErrorsTest do import Kernel.QuoteTest.Errors # Defines the add function - defadd - - test :inside_function_error do - assert_raise ArithmeticError, fn -> - add(:a, :b) + defraise() + + @line line() + test "inside function error" do + try do + will_raise(:a, :b) + rescue + RuntimeError -> + mod = Kernel.QuoteTest.ErrorsTest + file = __ENV__.file |> Path.relative_to_cwd() |> String.to_charlist() + assert [{^mod, :will_raise, 2, [file: ^file, line: @line] ++ _} | _] = __STACKTRACE__ end - - mod = Kernel.QuoteTest.ErrorsTest - file = __ENV__.file |> Path.relative_to_cwd |> String.to_char_list - assert [{^mod, :add, 2, [file: ^file, line: 181]}|_] = System.stacktrace end - test :outside_function_error do - assert_raise RuntimeError, fn -> - will_raise + @line __ENV__.line + 3 + test "outside function error" do + try do + will_raise() + flunk("expected failure") + rescue + RuntimeError -> + mod = Kernel.QuoteTest.ErrorsTest + file = __ENV__.file |> Path.relative_to_cwd() |> String.to_charlist() + assert [{^mod, _, _, [file: ^file, line: @line] ++ _} | _] = __STACKTRACE__ end - - mod = Kernel.QuoteTest.ErrorsTest - file = __ENV__.file |> Path.relative_to_cwd |> String.to_char_list - assert [{^mod, _, _, [file: ^file, line: 209]}|_] = System.stacktrace end end defmodule Kernel.QuoteTest.VarHygiene do defmacro no_interference do - quote do: a = 1 + quote(do: a = 1) end defmacro write_interference do - quote do: var!(a) = 1 + quote(do: var!(a) = 1) end defmacro read_interference do - quote do: 10 = var!(a) + quote(do: 10 = var!(a)) end defmacro cross_module_interference do - quote do: var!(a, Kernel.QuoteTest.VarHygieneTest) = 1 + quote(do: var!(a, Kernel.QuoteTest.VarHygieneTest) = 1) end end @@ -238,11 +427,11 @@ defmodule Kernel.QuoteTest.VarHygieneTest do import Kernel.QuoteTest.VarHygiene defmacrop cross_module_no_interference do - quote do: a = 10 + quote(do: a = 10) end defmacrop read_cross_module do - quote do: var!(a, __MODULE__) + quote(do: var!(a, __MODULE__)) end defmacrop nested(var, do: block) do @@ -255,44 +444,88 @@ defmodule Kernel.QuoteTest.VarHygieneTest do defmacrop hat do quote do - var = 1 + var = 1 ^var = 1 var end end - test :no_interference do + test "no interference" do a = 10 - no_interference + no_interference() assert a == 10 end - test :cross_module_interference do - cross_module_no_interference - cross_module_interference - assert read_cross_module == 1 + test "cross module interference" do + cross_module_no_interference() + cross_module_interference() + assert read_cross_module() == 1 end - test :write_interference do - write_interference + test "write interference" do + write_interference() assert a == 1 end - test :read_interference do + test "read interference" do a = 10 - read_interference + read_interference() end - test :nested do + test "hat" do + assert hat() == 1 + end + + test "nested macro" do assert (nested 1 do - nested 2 do - :ok + nested 2 do + _ = :ok + end + end) == 1 + end + + test "nested quoted" do + defmodule NestedQuote do + defmacro __using__(_) do + quote unquote: false do + arg = quote(do: arg) + + def test(arg) do + unquote(arg) + end + end end - end) == 1 + end + + defmodule UseNestedQuote do + use NestedQuote + end + + assert UseNestedQuote.test("foo") == "foo" end - test :hat do - assert hat == 1 + test "nested bind quoted" do + defmodule NestedBindQuoted do + defmacrop macro(arg) do + quote bind_quoted: [arg: arg] do + quote bind_quoted: [arg: arg], do: String.duplicate(arg, 2) + end + end + + defmacro __using__(_) do + quote do + def test do + unquote(macro("foo")) + end + end + end + end + + defmodule UseNestedBindQuoted do + use NestedBindQuoted + end + + assert UseNestedBindQuoted.test() == "foofoo" end end @@ -300,11 +533,11 @@ defmodule Kernel.QuoteTest.AliasHygiene do alias Dict, as: SuperDict defmacro dict do - quote do: Dict.Bar + quote(do: Dict.Bar) end defmacro super_dict do - quote do: SuperDict.Bar + quote(do: SuperDict.Bar) end end @@ -313,64 +546,79 @@ defmodule Kernel.QuoteTest.AliasHygieneTest do alias Dict, as: SuperDict - test :annotate_aliases do - assert {:__aliases__, [alias: false], [:Foo, :Bar]} = - quote(do: Foo.Bar) - assert {:__aliases__, [alias: false], [:Dict, :Bar]} = - quote(do: Dict.Bar) - assert {:__aliases__, [alias: Dict.Bar], [:SuperDict, :Bar]} = - quote(do: SuperDict.Bar) + test "annotate aliases" do + assert {:__aliases__, [alias: false], [:Foo, :Bar]} = quote(do: Foo.Bar) + assert {:__aliases__, [alias: false], [:Dict, :Bar]} = quote(do: Dict.Bar) + assert {:__aliases__, [alias: Dict.Bar], [:SuperDict, :Bar]} = quote(do: SuperDict.Bar) + + # Edge-case + assert {:__aliases__, _, [Elixir]} = quote(do: Elixir) end - test :expand_aliases do - assert Code.eval_quoted(quote do: SuperDict.Bar) == {Elixir.Dict.Bar, []} - assert Code.eval_quoted(quote do: alias!(SuperDict.Bar)) == {Elixir.SuperDict.Bar, []} + test "expand aliases" do + assert Code.eval_quoted(quote(do: SuperDict.Bar)) == {Elixir.Dict.Bar, []} + assert Code.eval_quoted(quote(do: alias!(SuperDict.Bar))) == {Elixir.SuperDict.Bar, []} end - test :expand_aliases_without_macro do + test "expand aliases without macro" do alias HashDict, as: SuperDict assert SuperDict.Bar == Elixir.HashDict.Bar end - test :expand_aliases_with_macro_does_not_expand_source_alias do + test "expand aliases with macro does not expand source alias" do alias HashDict, as: Dict, warn: false require Kernel.QuoteTest.AliasHygiene - assert Kernel.QuoteTest.AliasHygiene.dict == Elixir.Dict.Bar + assert Kernel.QuoteTest.AliasHygiene.dict() == Elixir.Dict.Bar end - test :expand_aliases_with_macro_has_higher_preference do + test "expand aliases with macro has higher preference" do alias HashDict, as: SuperDict, warn: false require Kernel.QuoteTest.AliasHygiene - assert Kernel.QuoteTest.AliasHygiene.super_dict == Elixir.Dict.Bar + assert Kernel.QuoteTest.AliasHygiene.super_dict() == Elixir.Dict.Bar end end defmodule Kernel.QuoteTest.ImportsHygieneTest do use ExUnit.Case, async: true + # We are redefining |> and using it inside the quote + # and only inside the quote. This code should still compile. + defmacro x |> f do + quote do + unquote(x) |> unquote(f) + end + end + defmacrop get_list_length do quote do - length('hello') + length(~c"hello") + end + end + + defmacrop get_list_length_with_pipe do + quote do + ~c"hello" |> length() end end defmacrop get_list_length_with_partial do quote do - (&length(&1)).('hello') + (&length(&1)).(~c"hello") end end defmacrop get_list_length_with_function do quote do - (&length/1).('hello') + (&length/1).(~c"hello") end end - test :expand_imports do + test "expand imports" do import Kernel, except: [length: 1] - assert get_list_length == 5 - assert get_list_length_with_partial == 5 - assert get_list_length_with_function == 5 + assert get_list_length() == 5 + assert get_list_length_with_pipe() == 5 + assert get_list_length_with_partial() == 5 + assert get_list_length_with_function() == 5 end defmacrop get_string_length do @@ -381,30 +629,209 @@ defmodule Kernel.QuoteTest.ImportsHygieneTest do end end - test :lazy_expand_imports do + test "lazy expand imports" do import Kernel, except: [length: 1] import String, only: [length: 1] - assert get_string_length == 5 + assert get_string_length() == 5 end - test :lazy_expand_imports_no_conflicts do + test "lazy expand imports no conflicts" do import Kernel, except: [length: 1] - import String, only: [length: 1] + import String, only: [length: 1], warn: false - assert get_list_length == 5 - assert get_list_length_with_partial == 5 - assert get_list_length_with_function == 5 + assert get_list_length() == 5 + assert get_list_length_with_partial() == 5 + assert get_list_length_with_function() == 5 end defmacrop with_length do quote do import Kernel, except: [length: 1] import String, only: [length: 1] - length('hello') + length(~c"hello") + end + end + + test "explicitly overridden imports" do + assert with_length() == 5 + end + + defmodule BinaryUtils do + defmacro int32 do + quote do + integer - size(32) + end end end - test :explicitly_overridden_imports do - assert with_length == 5 + test "checks the context also for variables to zero-arity functions" do + import BinaryUtils + {:int32, meta, __MODULE__} = quote(do: int32) + assert meta[:imports] == [{0, BinaryUtils}] + end +end + +defmodule Kernel.QuoteTest.HasUnquoteTest do + use ExUnit.Case, async: true + + test "expression without unquote returns false" do + ast = + quote unquote: false do + opts = [x: 5] + x = Keyword.fetch!(opts, :x) + x + 1 + end + + refute :elixir_quote.has_unquotes(ast) + end + + test "expression with unquote returns true" do + ast = + quote unquote: false do + [x: unquote(x)] + end + + assert :elixir_quote.has_unquotes(ast) + + ast = + quote unquote: false do + unquote(module).fun(x) + end + + assert :elixir_quote.has_unquotes(ast) + + ast = + quote unquote: false do + module.unquote(fun)(x) + end + + assert :elixir_quote.has_unquotes(ast) + + ast = + quote unquote: false do + module.fun(unquote(x)) + end + + assert :elixir_quote.has_unquotes(ast) + + ast = + quote unquote: false do + module.fun(unquote_splicing(args)) + end + + assert :elixir_quote.has_unquotes(ast) + end + + test "expression with unquote within quote returns false" do + ast = + quote unquote: false do + quote do + x + unquote(y) + end + end + + refute :elixir_quote.has_unquotes(ast) + + ast = + quote unquote: false do + quote do + foo = bar(unquote_splicing(args)) + end + end + + refute :elixir_quote.has_unquotes(ast) + end + + test "expression with unquote within unquote within quote returns true" do + ast = + quote unquote: false do + quote do + x + unquote(unquote(y)) + end + end + + assert :elixir_quote.has_unquotes(ast) + + ast = + quote unquote: false do + quote do + foo = bar(unquote_splicing(unquote(args))) + end + end + + assert :elixir_quote.has_unquotes(ast) + + ast = + quote unquote: false do + quote do + foo = bar(unquote(unquote_splicing(args))) + end + end + + assert :elixir_quote.has_unquotes(ast) + end + + test "expression within quote disabling unquotes always returns false" do + ast = + quote unquote: false do + quote unquote: false do + x + unquote(unquote(y)) + end + end + + refute :elixir_quote.has_unquotes(ast) + + ast = + quote unquote: false do + quote bind_quoted: [x: x] do + x + unquote(unquote(y)) + end + end + + refute :elixir_quote.has_unquotes(ast) + end + + test "unquote with invalid AST (shallow check)" do + for term <- [ + %{unescaped: :map}, + 1..10, + {:bad_meta, nil, []}, + {:bad_arg, nil, 1}, + {:bad_tuple}, + make_ref(), + [:improper | :list], + [nested: {}] + ] do + message = """ + tried to unquote invalid AST: #{inspect(term)} + Did you forget to escape term using Macro.escape/1?\ + """ + + assert_raise ArgumentError, message, fn -> quote do: unquote(term) end + end + end + + test "unquote with invalid AST is not checked deeply" do + assert quote do: unquote(foo: [1 | 2]) == [foo: [1 | 2]] + assert quote do: unquote(foo: [bar: %{}]) == [foo: [bar: %{}]] + end + + test "unquote_splicing with invalid AST" do + for args <- [ + "not_a_list", + [:improper | :list], + [%{unescaped: :map}], + [1..10], + [{:bad_meta, nil, []}], + [{:bad_arg, nil, 1}], + [{:bad_tuple}], + [make_ref()], + [nested: {}] + ] do + message = + "expected a list with quoted expressions in unquote_splicing/1, got: #{inspect(args)}" + + assert_raise ArgumentError, message, fn -> quote do: [unquote_splicing(args)] end + end end end diff --git a/lib/elixir/test/elixir/kernel/raise_test.exs b/lib/elixir/test/elixir/kernel/raise_test.exs index 2b6567e949a..6a4e2d95046 100644 --- a/lib/elixir/test/elixir/kernel/raise_test.exs +++ b/lib/elixir/test/elixir/kernel/raise_test.exs @@ -1,4 +1,8 @@ -Code.require_file "../test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) defmodule Kernel.RaiseTest do use ExUnit.Case, async: true @@ -9,8 +13,23 @@ defmodule Kernel.RaiseTest do defp opts, do: [message: "message"] defp struct, do: %RuntimeError{message: "message"} + @compile {:no_warn_undefined, DoNotExist} @trace [{:foo, :bar, 0, []}] + test "raise preserves the stacktrace" do + stacktrace = + try do + raise "a" + rescue + _ -> Enum.fetch!(__STACKTRACE__, 0) + end + + file = __ENV__.file |> Path.relative_to_cwd() |> String.to_charlist() + + assert {__MODULE__, :"test raise preserves the stacktrace", _, [file: ^file, line: 22] ++ _} = + stacktrace + end + test "raise message" do assert_raise RuntimeError, "message", fn -> raise "message" @@ -56,318 +75,511 @@ defmodule Kernel.RaiseTest do end end + test "raise with error_info" do + {exception, stacktrace} = + try do + raise "a" + rescue + e -> {e, __STACKTRACE__} + end + + assert [{__MODULE__, _, _, meta} | _] = stacktrace + assert meta[:error_info] == %{module: Exception} + + assert Exception.format_error(exception, stacktrace) == + %{general: "a", reason: "#Elixir.RuntimeError"} + end + test "reraise message" do try do reraise "message", @trace - flunk "should not reach" + flunk("should not reach") rescue RuntimeError -> - assert @trace == :erlang.get_stacktrace() + assert @trace == __STACKTRACE__ end try do var = binary() reraise var, @trace - flunk "should not reach" + flunk("should not reach") rescue RuntimeError -> - assert @trace == :erlang.get_stacktrace() + assert @trace == __STACKTRACE__ end end test "reraise with no arguments" do try do reraise RuntimeError, @trace - flunk "should not reach" + flunk("should not reach") rescue RuntimeError -> - assert @trace == :erlang.get_stacktrace() + assert @trace == __STACKTRACE__ end try do var = atom() reraise var, @trace - flunk "should not reach" + flunk("should not reach") rescue RuntimeError -> - assert @trace == :erlang.get_stacktrace() + assert @trace == __STACKTRACE__ end end test "reraise with arguments" do try do reraise RuntimeError, [message: "message"], @trace - flunk "should not reach" + flunk("should not reach") rescue RuntimeError -> - assert @trace == :erlang.get_stacktrace() + assert @trace == __STACKTRACE__ end try do atom = atom() opts = opts() reraise atom, opts, @trace - flunk "should not reach" + flunk("should not reach") rescue RuntimeError -> - assert @trace == :erlang.get_stacktrace() + assert @trace == __STACKTRACE__ end end test "reraise existing exception" do try do reraise %RuntimeError{message: "message"}, @trace - flunk "should not reach" + flunk("should not reach") rescue RuntimeError -> - assert @trace == :erlang.get_stacktrace() + assert @trace == __STACKTRACE__ end try do var = struct() reraise var, @trace - flunk "should not reach" + flunk("should not reach") rescue RuntimeError -> - assert @trace == :erlang.get_stacktrace() + assert @trace == __STACKTRACE__ end end - test :rescue_with_underscore_no_exception do - result = try do - RescueUndefinedModule.go - rescue - _ -> true - end + describe "rescue" do + test "runtime error" do + result = + try do + raise "an exception" + rescue + RuntimeError -> true + catch + :error, _ -> false + end + + assert result + + result = + try do + raise "an exception" + rescue + ArgumentError -> true + catch + :error, _ -> false + end - assert result - end + refute result + end + + test "named runtime error" do + result = + try do + raise "an exception" + rescue + x in [RuntimeError] -> Exception.message(x) + catch + :error, _ -> false + end + + assert result == "an exception" + end + + test "named runtime or argument error" do + result = + try do + raise "an exception" + rescue + x in [ArgumentError, RuntimeError] -> Exception.message(x) + catch + :error, _ -> false + end - test :rescue_with_higher_precedence_than_catch do - result = try do - RescueUndefinedModule.go - catch - _, _ -> false - rescue - _ -> true + assert result == "an exception" end - assert result - end + test "named function clause (stacktrace) or runtime (no stacktrace) error" do + result = + try do + Access.get("foo", 0) + rescue + x in [FunctionClauseError, CaseClauseError] -> Exception.message(x) + end - test :rescue_runtime_error do - result = try do - raise "an exception" - rescue - RuntimeError -> true - catch - :error, _ -> false + assert result == "no function clause matching in Access.get/3" end - assert result + test "with higher precedence than catch" do + result = + try do + raise "an exception" + rescue + _ -> true + catch + _, _ -> false + end - result = try do - raise "an exception" - rescue - AnotherError -> true - catch - :error, _ -> false + assert result end - refute result - end + test "argument error from Erlang" do + result = + try do + :erlang.error(:badarg) + rescue + ArgumentError -> true + end - test :rescue_named_runtime_error do - result = try do - raise "an exception" - rescue - x in [RuntimeError] -> Exception.message(x) - catch - :error, _ -> false + assert result end - assert result == "an exception" - end + test "argument error from Elixir" do + result = + try do + raise ArgumentError, "" + rescue + ArgumentError -> true + end - test :rescue_argument_error_from_elixir do - result = try do - raise ArgumentError, "" - rescue - ArgumentError -> true + assert result end - assert result - end + test "catch-all variable" do + result = + try do + raise "an exception" + rescue + x -> Exception.message(x) + end - test :rescue_named_with_underscore do - result = try do - raise "an exception" - rescue - x in _ -> Exception.message(x) + assert result == "an exception" end - assert result == "an exception" - end + test "catch-all underscore" do + result = + try do + raise "an exception" + rescue + _ -> true + end - test :wrap_custom_erlang_error do - result = try do - :erlang.error(:sample) - rescue - x in [RuntimeError, ErlangError] -> Exception.message(x) + assert result end - assert result == "erlang error: :sample" - end + test "catch-all unused variable" do + result = + try do + raise "an exception" + rescue + _any -> true + end - test :undefined_function_error do - result = try do - DoNotExist.for_sure() - rescue - x in [UndefinedFunctionError] -> Exception.message(x) + assert result end - assert result == "undefined function: DoNotExist.for_sure/0" - end + test "catch-all with \"x in _\" syntax" do + result = + try do + raise "an exception" + rescue + exception in _ -> + Exception.message(exception) + end - test :function_clause_error do - result = try do - zero(1) - rescue - x in [FunctionClauseError] -> Exception.message(x) + assert result == "an exception" + end + + defmacrop argerr(e) do + quote(do: unquote(e) in ArgumentError) end - assert result == "no function clause matching in Kernel.RaiseTest.zero/1" + test "with rescue macro" do + result = + try do + raise ArgumentError, "oops, badarg" + rescue + argerr(e) -> Exception.message(e) + end + + assert result == "oops, badarg" + end end - test :badarg_error do - result = try do - :erlang.error(:badarg) - rescue - x in [ArgumentError] -> Exception.message(x) + describe "normalize" do + test "wrap custom Erlang error" do + result = + try do + :erlang.error(:sample) + rescue + x in [ErlangError] -> Exception.message(x) + end + + assert result == "Erlang error: :sample" end - assert result == "argument error" - end + test "undefined function error" do + result = + try do + DoNotExist.for_sure() + rescue + x in [UndefinedFunctionError] -> Exception.message(x) + end - test :tuple_badarg_error do - result = try do - :erlang.error({:badarg, [1, 2, 3]}) - rescue - x in [ArgumentError] -> Exception.message(x) + assert result == + "function DoNotExist.for_sure/0 is undefined (module DoNotExist is not available). " <> + "Make sure the module name is correct and has been specified in full (or that an alias has been defined)" end - assert result == "argument error: [1, 2, 3]" - end + test "function clause error" do + result = + try do + Access.get(:ok, :error) + rescue + x in [FunctionClauseError] -> Exception.message(x) + end - test :badarith_error do - result = try do - :erlang.error(:badarith) - rescue - x in [ArithmeticError] -> Exception.message(x) + assert result == "no function clause matching in Access.get/3" end - assert result == "bad argument in arithmetic expression" - end + test "badarg error" do + result = + try do + :erlang.error(:badarg) + rescue + x in [ArgumentError] -> Exception.message(x) + end + + assert result == "argument error" + end - test :badarity_error do - fun = fn(x) -> x end - string = "#{inspect(fun)} with arity 1 called with 2 arguments (1, 2)" + test "tuple badarg error" do + result = + try do + :erlang.error({:badarg, [1, 2, 3]}) + rescue + x in [ArgumentError] -> Exception.message(x) + end - result = try do - fun.(1, 2) - rescue - x in [BadArityError] -> Exception.message(x) + assert result == "argument error: [1, 2, 3]" end - assert result == string - end + test "badarith error" do + result = + try do + :erlang.error(:badarith) + rescue + x in [ArithmeticError] -> Exception.message(x) + end - test :badfun_error do - x = :example - result = try do - x.(2) - rescue - x in [BadFunctionError] -> Exception.message(x) + assert result == "bad argument in arithmetic expression" end - assert result == "expected a function, got: :example" - end + test "badarity error" do + fun = fn x -> x end + string = "#{inspect(fun)} with arity 1 called with 2 arguments (1, 2)" - test :badmatch_error do - x = :example - result = try do - ^x = zero(0) - rescue - x in [MatchError] -> Exception.message(x) + result = + try do + Process.get(:unused, fun).(1, 2) + rescue + x in [BadArityError] -> Exception.message(x) + end + + assert result == string end - assert result == "no match of right hand side value: 0" - end + test "badfun error" do + result = + try do + Process.get(:unused, :example).(2) + rescue + x in [BadFunctionError] -> Exception.message(x) + end - test :case_clause_error do - x = :example - result = try do - case zero(0) do - ^x -> nil - end - rescue - x in [CaseClauseError] -> Exception.message(x) + assert result == "expected a function, got: :example" end - assert result == "no case clause matching: 0" - end + test "badfun error when the function is gone" do + defmodule BadFunction.Missing do + def fun, do: fn -> :ok end + end + + fun = BadFunction.Missing.fun() + + :code.purge(BadFunction.Missing) + :code.delete(BadFunction.Missing) - test :cond_clause_error do - result = try do - cond do - !zero(0) -> :ok + defmodule BadFunction.Missing do + def fun, do: fn -> :another end end - rescue - x in [CondClauseError] -> Exception.message(x) - end - assert result == "no cond clause evaluated to a true value" - end + :code.purge(BadFunction.Missing) - test :try_clause_error do - f = fn() -> :example end - result = try do try do - f.() + fun.() + rescue + x in [BadFunctionError] -> + assert Exception.message(x) =~ + ~r/function #Function<[0-9]\.[0-9]*\/0[^>]*> is invalid, likely because it points to an old version of the code/ else - :other -> - :ok + _ -> flunk("this should not be invoked") end - rescue - x in [TryClauseError] -> Exception.message(x) end - assert result == "no try clause matching: :example" - end + test "badmatch error" do + result = + try do + [] = Range.to_list(1000_000..1_000_009) + rescue + x in [MatchError] -> Exception.message(x) + end - test :undefined_function_error_as_erlang_error do - result = try do - DoNotExist.for_sure() - rescue - x in [ErlangError] -> Exception.message(x) + assert result == + """ + no match of right hand side value: + + [1000000, 1000001, 1000002, 1000003, 1000004, 1000005, 1000006, 1000007, + 1000008, 1000009] + """ + end + + test "bad key error" do + result = + try do + %{Process.get(:unused, %{}) | foo: :bar} + rescue + x in [KeyError] -> Exception.message(x) + end + + assert result == "key :foo not found" + + result = + try do + Process.get(:unused, %{}).foo + rescue + x in [KeyError] -> Exception.message(x) + end + + assert result == "key :foo not found in:\n\n %{}\n" + end + + test "bad map error" do + result = + try do + %{Process.get(:unused, 0) | foo: :bar} + rescue + x in [BadMapError] -> Exception.message(x) + end + + assert result == "expected a map, got:\n\n 0\n" + end + + test "bad boolean error" do + result = + try do + Process.get(:unused, 1) and true + rescue + x in [BadBooleanError] -> Exception.message(x) + end + + assert result == "expected a boolean on left-side of \"and\", got:\n\n 1\n" + end + + test "case clause error" do + x = :example + + result = + try do + case Process.get(:unused, 0) do + ^x -> nil + end + rescue + x in [CaseClauseError] -> Exception.message(x) + end + + assert result == "no case clause matching:\n\n 0\n" + end + + test "cond clause error" do + result = + try do + cond do + !Process.get(:unused, 0) -> :ok + end + rescue + x in [CondClauseError] -> Exception.message(x) + end + + assert result == "no cond clause evaluated to a truthy value" + end + + test "try clause error" do + result = + try do + try do + Process.get(:unused, :example) + rescue + _exception -> + :ok + else + :other -> + :ok + end + rescue + x in [TryClauseError] -> Exception.message(x) + end + + assert result == "no try clause matching:\n\n :example\n" + end + + test "undefined function error as Erlang error" do + result = + try do + DoNotExist.for_sure() + rescue + x in [ErlangError] -> Exception.message(x) + end + + assert result == + "function DoNotExist.for_sure/0 is undefined (module DoNotExist is not available). " <> + "Make sure the module name is correct and has been specified in full (or that an alias has been defined)" end - - assert result == "undefined function: DoNotExist.for_sure/0" end defmacrop exceptions do [ErlangError] end - test :with_macros do - result = try do - DoNotExist.for_sure() - rescue - x in exceptions -> Exception.message(x) - end + test "with macros" do + result = + try do + DoNotExist.for_sure() + rescue + x in exceptions() -> Exception.message(x) + end - assert result == "undefined function: DoNotExist.for_sure/0" + assert result == + "function DoNotExist.for_sure/0 is undefined (module DoNotExist is not available). " <> + "Make sure the module name is correct and has been specified in full (or that an alias has been defined)" end - - defp zero(0), do: 0 end diff --git a/lib/elixir/test/elixir/kernel/sigils_test.exs b/lib/elixir/test/elixir/kernel/sigils_test.exs index 28e9f232771..d1e3b905974 100644 --- a/lib/elixir/test/elixir/kernel/sigils_test.exs +++ b/lib/elixir/test/elixir/kernel/sigils_test.exs @@ -1,21 +1,25 @@ -Code.require_file "../test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) defmodule Kernel.SigilsTest do use ExUnit.Case, async: true - test :sigil_s do + test "sigil s" do assert ~s(foo) == "foo" assert ~s(f#{:o}o) == "foo" assert ~s(f\no) == "f\no" end - test :sigil_s_with_heredoc do + test "sigil s with heredoc" do assert " foo\n\n" == ~s""" - f#{:o}o\n - """ + f#{:o}o\n + """ end - test :sigil_S do + test "sigil S" do assert ~S(foo) == "foo" assert ~S[foo] == "foo" assert ~S{foo} == "foo" @@ -25,37 +29,56 @@ defmodule Kernel.SigilsTest do assert ~S/foo/ == "foo" assert ~S|foo| == "foo" assert ~S(f#{o}o) == "f\#{o}o" + assert ~S(f\#{o}o) == "f\\\#{o}o" assert ~S(f\no) == "f\\no" end - test :sigil_S_with_heredoc do + test "sigil S newline" do + assert ~S(foo\ +bar) in ["foo\\\nbar", "foo\\\r\nbar"] + end + + test "sigil S with heredoc" do assert " f\#{o}o\\n\n" == ~S""" - f#{o}o\n - """ + f#{o}o\n + """ end - test :sigil_c do - assert ~c(foo) == 'foo' - assert ~c(f#{:o}o) == 'foo' - assert ~c(f\no) == 'f\no' + test "sigil s/S expand to binary when possible" do + assert Macro.expand(quote(do: ~s(foo)), __ENV__) == "foo" + assert Macro.expand(quote(do: ~S(foo)), __ENV__) == "foo" end - test :sigil_C do - assert ~C(foo) == 'foo' - assert ~C[foo] == 'foo' - assert ~C{foo} == 'foo' - assert ~C'foo' == 'foo' - assert ~C"foo" == 'foo' - assert ~C|foo| == 'foo' - assert ~C(f#{o}o) == 'f\#{o}o' - assert ~C(f\no) == 'f\\no' + test "sigil c" do + assert ~c(foo) == ~c"foo" + assert ~c(f#{:o}o) == ~c"foo" + assert ~c(f\no) == ~c"f\no" end - test :sigil_w do + test "sigil C" do + assert ~C(foo) == ~c"foo" + assert ~C[foo] == ~c"foo" + assert ~C{foo} == ~c"foo" + assert ~C'foo' == ~c"foo" + assert ~C"foo" == ~c"foo" + assert ~C|foo| == ~c"foo" + assert ~C(f#{o}o) == ~c"f\#{o}o" + assert ~C(f\no) == ~c"f\\no" + end + + test "sigil w" do assert ~w() == [] + assert ~w([ , ]) == ["[", ",", "]"] assert ~w(foo bar baz) == ["foo", "bar", "baz"] assert ~w(foo #{:bar} baz) == ["foo", "bar", "baz"] + assert ~w(#{""}) == [] + assert ~w(foo #{""}) == ["foo"] + assert ~w(#{" foo bar "}) == ["foo", "bar"] + + assert ~w(foo\ #{:bar}) == ["foo", "bar"] + assert ~w(foo\ bar) == ["foo", "bar"] + assert ~w( foo bar @@ -64,23 +87,27 @@ defmodule Kernel.SigilsTest do assert ~w(foo bar baz)s == ["foo", "bar", "baz"] assert ~w(foo bar baz)a == [:foo, :bar, :baz] - assert ~w(foo bar baz)c == ['foo', 'bar', 'baz'] + assert ~w(foo bar baz)c == [~c"foo", ~c"bar", ~c"baz"] - bad_modifier = quote do: ~w(foo bar baz)x + bad_modifier = quote(do: ~w(foo bar baz)x) assert %ArgumentError{} = catch_error(Code.eval_quoted(bad_modifier)) - assert ~w(Foo Bar)a == [:"Foo", :"Bar"] + assert ~w(Foo Bar)a == [:Foo, :Bar] assert ~w(Foo.#{Bar}.Baz)a == [:"Foo.Elixir.Bar.Baz"] assert ~w(Foo.Bar)s == ["Foo.Bar"] - assert ~w(Foo.#{Bar})c == ['Foo.Elixir.Bar'] + assert ~w(Foo.#{Bar})c == [~c"Foo.Elixir.Bar"] # Ensure it is fully expanded at compile time assert Macro.expand(quote(do: ~w(a b c)a), __ENV__) == [:a, :b, :c] end - test :sigil_W do + test "sigil W" do + assert ~W() == [] + assert ~W([ , ]) == ["[", ",", "]"] assert ~W(foo #{bar} baz) == ["foo", "\#{bar}", "baz"] + assert ~W(foo\ bar) == ["foo\\", "bar"] + assert ~W( foo bar @@ -89,20 +116,32 @@ defmodule Kernel.SigilsTest do assert ~W(foo bar baz)s == ["foo", "bar", "baz"] assert ~W(foo bar baz)a == [:foo, :bar, :baz] - assert ~W(foo bar baz)c == ['foo', 'bar', 'baz'] + assert ~W(foo bar baz)c == [~c"foo", ~c"bar", ~c"baz"] - bad_modifier = quote do: ~W(foo bar baz)x + bad_modifier = quote(do: ~W(foo bar baz)x) assert %ArgumentError{} = catch_error(Code.eval_quoted(bad_modifier)) - assert ~W(Foo #{Bar})a == [:"Foo", :"\#{Bar}"] + assert ~W(Foo #{Bar})a == [:Foo, :"\#{Bar}"] assert ~W(Foo.Bar.Baz)a == [:"Foo.Bar.Baz"] end - test :sigils_matching do + test "sigils matching" do assert ~s(f\(oo) == "f(oo" assert ~s(fo\)o) == "fo)o" assert ~s(f\(o\)o) == "f(o)o" assert ~s(f[oo) == "f[oo" assert ~s(fo]o) == "fo]o" end + + describe "multi-letter sigils" do + def sigil_MAT(string, modifiers) do + %{matrix: string, modifiers: modifiers} + end + + test "sigil MAT" do + assert ~MAT"foo" == %{matrix: "foo", modifiers: []} + assert ~MAT[foo]i == %{matrix: "foo", modifiers: ~c"i"} + assert ~MAT("1")mod == %{matrix: "\"1\"", modifiers: ~c"mod"} + end + end end diff --git a/lib/elixir/test/elixir/kernel/special_forms_test.exs b/lib/elixir/test/elixir/kernel/special_forms_test.exs new file mode 100644 index 00000000000..c05a6aac1cf --- /dev/null +++ b/lib/elixir/test/elixir/kernel/special_forms_test.exs @@ -0,0 +1,88 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Kernel.SpecialFormsTest do + use ExUnit.Case, async: true + + doctest Kernel.SpecialForms + + describe "cond" do + test "does not leak variables for one clause" do + x = 0 + + cond do + true -> + x = 1 + x + end + + assert x == 0 + end + + test "does not leak variables for one clause with non-boolean as catch-all" do + x = 0 + + cond do + :otherwise -> + x = 1 + x + end + + assert x == 0 + end + + test "does not leak variables for multiple clauses" do + x = 0 + + cond do + List.flatten([]) == [] -> + x = 1 + x + + true -> + x = 1 + x + end + + assert x == 0 + end + + test "does not leak variables from conditions" do + x = :not_nil + + result = + cond do + x = List.first([]) -> + x + + true -> + x + end + + assert result == :not_nil + end + + test "does not warn on non-boolean as catch-all" do + cond do + List.flatten([]) == [] -> :good + :otherwise -> :also_good + end + end + + test "cond_clause error keeps line number in stacktrace" do + try do + cond do + Process.get(:unused, false) -> :ok + end + rescue + _ -> + assert [{Kernel.SpecialFormsTest, _, _, meta} | _] = __STACKTRACE__ + assert meta[:file] + assert meta[:line] + end + end + end +end diff --git a/lib/elixir/test/elixir/kernel/string_tokenizer_test.exs b/lib/elixir/test/elixir/kernel/string_tokenizer_test.exs new file mode 100644 index 00000000000..316b5d1de41 --- /dev/null +++ b/lib/elixir/test/elixir/kernel/string_tokenizer_test.exs @@ -0,0 +1,197 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Kernel.StringTokenizerTest do + use ExUnit.Case, async: true + + defp var({var, _, nil}), do: var + defp aliases({:__aliases__, _, [alias]}), do: alias + + test "tokenizes vars" do + assert Code.string_to_quoted!("_12") |> var() == :_12 + assert Code.string_to_quoted!("ola") |> var() == :ola + assert Code.string_to_quoted!("ólá") |> var() == :ólá + assert Code.string_to_quoted!("óLÁ") |> var() == :óLÁ + assert Code.string_to_quoted!("ólá?") |> var() == :ólá? + assert Code.string_to_quoted!("ólá!") |> var() == :ólá! + assert Code.string_to_quoted!("こんにちは世界") |> var() == :こんにちは世界 + assert {:error, _} = Code.string_to_quoted("v@r") + assert {:error, _} = Code.string_to_quoted("1var") + end + + test "tokenizes atoms" do + assert Code.string_to_quoted!(":_12") == :_12 + assert Code.string_to_quoted!(":ola") == :ola + assert Code.string_to_quoted!(":ólá") == :ólá + assert Code.string_to_quoted!(":ólá?") == :ólá? + assert Code.string_to_quoted!(":ólá!") == :ólá! + assert Code.string_to_quoted!(":ól@") == :ól@ + assert Code.string_to_quoted!(":ól@!") == :ól@! + assert Code.string_to_quoted!(":ó@@!") == :ó@@! + assert Code.string_to_quoted!(":Ola") == :Ola + assert Code.string_to_quoted!(":Ólá") == :Ólá + assert Code.string_to_quoted!(":ÓLÁ") == :ÓLÁ + assert Code.string_to_quoted!(":ÓLÁ?") == :ÓLÁ? + assert Code.string_to_quoted!(":ÓLÁ!") == :ÓLÁ! + assert Code.string_to_quoted!(":ÓL@!") == :ÓL@! + assert Code.string_to_quoted!(":Ó@@!") == :Ó@@! + assert Code.string_to_quoted!(":こんにちは世界") == :こんにちは世界 + assert {:error, _} = Code.string_to_quoted(":123") + assert {:error, _} = Code.string_to_quoted(":@123") + end + + test "tokenizes keywords" do + assert Code.string_to_quoted!("[_12: 0]") == [_12: 0] + assert Code.string_to_quoted!("[ola: 0]") == [ola: 0] + assert Code.string_to_quoted!("[ólá: 0]") == [ólá: 0] + assert Code.string_to_quoted!("[ólá?: 0]") == [ólá?: 0] + assert Code.string_to_quoted!("[ólá!: 0]") == [ólá!: 0] + assert Code.string_to_quoted!("[ól@: 0]") == [ól@: 0] + assert Code.string_to_quoted!("[ól@!: 0]") == [ól@!: 0] + assert Code.string_to_quoted!("[ó@@!: 0]") == [ó@@!: 0] + assert Code.string_to_quoted!("[Ola: 0]") == [Ola: 0] + assert Code.string_to_quoted!("[Ólá: 0]") == [Ólá: 0] + assert Code.string_to_quoted!("[ÓLÁ: 0]") == [ÓLÁ: 0] + assert Code.string_to_quoted!("[ÓLÁ?: 0]") == [ÓLÁ?: 0] + assert Code.string_to_quoted!("[ÓLÁ!: 0]") == [ÓLÁ!: 0] + assert Code.string_to_quoted!("[ÓL@!: 0]") == [ÓL@!: 0] + assert Code.string_to_quoted!("[Ó@@!: 0]") == [Ó@@!: 0] + assert Code.string_to_quoted!("[こんにちは世界: 0]") == [こんにちは世界: 0] + assert {:error, _} = Code.string_to_quoted("[123: 0]") + assert {:error, _} = Code.string_to_quoted("[@123: 0]") + end + + test "tokenizes aliases" do + assert Code.string_to_quoted!("Ola") |> aliases() == String.to_atom("Ola") + assert Code.string_to_quoted!("M_123") |> aliases() == String.to_atom("M_123") + assert {:error, _} = Code.string_to_quoted("Óla") + assert {:error, _} = Code.string_to_quoted("Olá") + assert {:error, _} = Code.string_to_quoted("Ol@") + assert {:error, _} = Code.string_to_quoted("Ola?") + assert {:error, _} = Code.string_to_quoted("Ola!") + end + + test "tokenizes remote calls" do + # We chose the atom below because Erlang represents it using nested lists + assert {{:., _, [:foo, :บูมเมอแรง]}, _, []} = + Code.string_to_quoted!(":foo.บูมเมอแรง()") + + assert {{:., _, [:foo, :บูมเมอแรง]}, _, []} = + Code.string_to_quoted!(":foo.\"บูมเมอแรง\"()") + end + + describe "script mixing" do + test "prevents Restricted codepoints in identifiers" do + exception = assert_raise SyntaxError, fn -> Code.string_to_quoted!("_shibㅤ = 1") end + + assert Exception.message(exception) =~ + "unexpected token: \"ㅤ\" (column 6, code point U+3164)" + end + + test "prevents unsafe mixing in identifiers" do + exception = + assert_raise SyntaxError, fn -> + Code.string_to_quoted!("if аdmin_, do: :ok, else: :err") + end + + assert Exception.message(exception) =~ "nofile:1:9:" + assert Exception.message(exception) =~ "invalid mixed-script identifier found: аdmin" + + for s <- [ + "\\u0430 а {Cyrillic}", + "\\u0064 d {Latin}", + "\\u006D m {Latin}", + "\\u0069 i {Latin}", + "\\u006E n {Latin}", + "\\u005F _" + ] do + assert Exception.message(exception) =~ s + end + + # includes suggestion about what to change + assert Exception.message(exception) =~ """ + Hint: You could write the above in a similar way that is accepted by Elixir: + """ + + assert Exception.message(exception) =~ """ + "admin_" (code points 0x00061 0x00064 0x0006D 0x00069 0x0006E 0x0005F) + """ + + # a is in cyrillic + assert_raise SyntaxError, ~r/mixed/, fn -> Code.string_to_quoted!("[аdmin: 1]") end + assert_raise SyntaxError, ~r/mixed/, fn -> Code.string_to_quoted!("[{:аdmin, 1}]") end + assert_raise SyntaxError, ~r/mixed/, fn -> Code.string_to_quoted!("quote do: аdmin(1)") end + + # c is Latin + assert_raise SyntaxError, ~r/mixed/, fn -> Code.string_to_quoted!("http_cервер = 1") end + + # T is in cyrillic + assert_raise SyntaxError, ~r/mixed/, fn -> Code.string_to_quoted!("[Тシャツ: 1]") end + end + + test "allows legitimate script mixing" do + # Mixed script with supersets, numbers, and underscores + assert Code.eval_string("幻한 = 1") == {1, [幻한: 1]} + assert Code.eval_string("幻한1 = 1") == {1, [幻한1: 1]} + assert Code.eval_string("__सवव_1? = 1") == {1, [__सवव_1?: 1]} + + # Elixir's normalizations combine scriptsets of the 'from' and 'to' characters, + # ex: {Common} MICRO => {Greek} MU == {Common, Greek}; Common intersects w/all + assert Code.eval_string("μs = 1") == {1, [μs: 1]} + + # Mixed scripts in variables + assert Code.eval_string("http_сервер = 1") == {1, [http_сервер: 1]} + assert Code.eval_string("сервер_http = 1") == {1, [сервер_http: 1]} + + # Mixed scripts in atoms + assert Code.eval_string(":T_シャツ") == {:T_シャツ, []} + end + + test "bidi" do + # test that the implementation of String.Tokenizer.Security.unbidify/1 agrees + # w/Unicode Bidi Algo (UAX9) for these (identifier-specific, no-bracket) examples + # + # you can create new examples with: https://util.unicode.org/UnicodeJsps/bidic.jsp?s=foo_%D9%84%D8%A7%D9%85%D8%AF%D8%A7_baz&b=0&u=140&d=2 + # inspired by (none of these are directly usable for our idents): https://www.unicode.org/Public/UCD/latest/ucd/BidiCharacterTest.txt + # + # there's a spurious ;A; after the identifier, because the semicolon is dir-neutral, and + # deleting it makes these examples hard to read in many/most editors! + """ + foo;A;0066 006F 006F;0 1 2 + _foo_ ;A;005F 0066 006F 006F 005F;0 1 2 3 4 + __foo__ ;A;005F 005F 0066 006F 006F 005F 005F;0 1 2 3 4 5 6 + لامدا_foo ;A;0644 0627 0645 062F 0627 005F 0066 006F 006F;4 3 2 1 0 5 6 7 8 + foo_لامدا_baz ;A;0066 006F 006F 005F 0644 0627 0645 062F 0627 005F 0062 0061 007A;0 1 2 3 8 7 6 5 4 9 10 11 12 + foo_لامدا ;A;0066 006F 006F 005F 0644 0627 0645 062F 0627;0 1 2 3 8 7 6 5 4 + foo_لامدا1 ;A;0066 006F 006F 005F 0644 0627 0645 062F 0627 0031;0 1 2 3 9 8 7 6 5 4 + foo_لامدا_حدد ;A;0066 006F 006F 005F 0644 0627 0645 062F 0627 005F 062D 062F 062F;0 1 2 3 12 11 10 9 8 7 6 5 4 + foo_لامدا_حدد1 ;A;0066 006F 006F 005F 0644 0627 0645 062F 0627 005F 062D 062F 062F 0031;0 1 2 3 13 12 11 10 9 8 7 6 5 4 + foo_لامدا_حدد1_bar ;A; 0066 006F 006F 005F 0644 0627 0645 062F 0627 005F 062D 062F 062F 0031 005F 0062 0061 0072;0 1 2 3 13 12 11 10 9 8 7 6 5 4 14 15 16 17 + foo_لامدا_حدد1_bar1 ;A;0066 006F 006F 005F 0644 0627 0645 062F 0627 005F 062D 062F 062F 0031 005F 0062 0061 0072 0031;0 1 2 3 13 12 11 10 9 8 7 6 5 4 14 15 16 17 18 + """ + |> String.split("\n", trim: true) + |> Enum.map(&String.split(&1, ";", trim: true)) + |> Enum.each(fn + [ident, _, bytes, indices | _rest] -> + bytes = String.split(bytes, " ", trim: true) |> Enum.map(&String.to_integer(&1, 16)) + indices = String.split(indices, " ", trim: true) |> Enum.map(&String.to_integer/1) + display_ordered = for i <- indices, do: Enum.at(bytes, i) + unbidified = String.Tokenizer.Security.unbidify(bytes) + + if display_ordered != unbidified do + raise """ + Failing String.Tokenizer.Security.unbidify/1 case for: '#{ident}' + bytes : #{bytes |> Enum.map(&Integer.to_string(&1, 16)) |> Enum.join(" ")} + byte order : #{bytes |> Enum.intersperse(32)} + uax9 order : #{display_ordered |> Enum.intersperse(32)} + uax9 indices : #{indices |> Enum.join(" ")} + unbidify/1 : #{unbidified |> Enum.intersperse(32)} + """ + end + end) + end + end +end diff --git a/lib/elixir/test/elixir/kernel/tracers_test.exs b/lib/elixir/test/elixir/kernel/tracers_test.exs new file mode 100644 index 00000000000..240cb24a9ed --- /dev/null +++ b/lib/elixir/test/elixir/kernel/tracers_test.exs @@ -0,0 +1,401 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Kernel.TracersTest do + use ExUnit.Case + + defp compile_string(string) do + string + |> Code.string_to_quoted!(columns: true) + |> Code.compile_quoted() + end + + def trace(event, %Macro.Env{} = env) do + for {pid, _} <- Registry.lookup(__MODULE__, :tracers) do + send(pid, {event, env}) + end + + :ok + end + + setup_all do + start_supervised!({Registry, keys: :duplicate, name: __MODULE__}) + Code.put_compiler_option(:tracers, [__MODULE__]) + + on_exit(fn -> + Code.put_compiler_option(:tracers, []) + end) + end + + setup do + Registry.register(__MODULE__, :tracers, :unused) + :ok + end + + test "traces start and stop" do + compile_string(""" + Foo + """) + + assert_received {:start, %{lexical_tracker: pid}} when is_pid(pid) + assert_received {:stop, %{lexical_tracker: pid}} when is_pid(pid) + end + + test "traces alias references" do + compile_string(""" + Foo + """) + + assert_received {{:alias_reference, meta, Foo}, _} + assert meta[:line] == 1 + assert meta[:column] == 1 + end + + test "traces aliases" do + compile_string(""" + alias Hello.World + World + + alias Foo, as: Bar, warn: true + Bar + """) + + assert_received {{:alias, meta, Hello.World, World, []}, _} + assert meta[:line] == 1 + assert meta[:column] == 1 + assert_received {{:alias_expansion, meta, World, Hello.World}, _} + assert meta[:line] == 2 + assert meta[:column] == 1 + + assert_received {{:alias, meta, Foo, Bar, [as: Bar, warn: true]}, _} + assert meta[:line] == 4 + assert meta[:column] == 1 + assert_received {{:alias_expansion, meta, Bar, Foo}, _} + assert meta[:line] == 5 + assert meta[:column] == 1 + end + + test "traces imports" do + compile_string(""" + import Integer, only: [is_odd: 1, parse: 1] + true = is_odd(1) + {1, ""} = parse("1") + """) + + assert_received {{:import, meta, Integer, only: [is_odd: 1, parse: 1]}, _} + assert meta[:line] == 1 + assert meta[:column] == 1 + + assert_received {{:imported_macro, meta, Integer, :is_odd, 1}, _} + assert meta[:line] == 2 + assert meta[:column] == 8 + + assert_received {{:imported_function, meta, Integer, :parse, 1}, _} + assert meta[:line] == 3 + assert meta[:column] == 11 + + refute_received {{:remote_function, _, Integer, :parse, 1}, _} + end + + test "traces imports via capture" do + compile_string(""" + import Integer, only: [is_odd: 1, parse: 1] + &is_odd/1 + &parse/1 + """) + + assert_received {{:import, meta, Integer, only: [is_odd: 1, parse: 1]}, _} + assert meta[:line] == 1 + assert meta[:column] == 1 + + assert_received {{:imported_macro, meta, Integer, :is_odd, 1}, _} + assert meta[:line] == 2 + assert meta[:column] == 2 + + assert_received {{:imported_function, meta, Integer, :parse, 1}, _} + assert meta[:line] == 3 + assert meta[:column] == 2 + + refute_received {{:remote_function, _meta, Integer, :parse, 1}, _} + end + + test "traces structs" do + compile_string(""" + %URI{path: "/"} + """) + + assert_received {{:struct_expansion, meta, URI, [:path]}, _} + assert meta[:line] == 1 + assert meta[:column] == 1 + end + + test "traces remote" do + compile_string(""" + require Integer + true = Integer.is_odd(1) + {1, ""} = Integer.parse("1") + "foo" = Atom.to_string(:foo) + """) + + assert_received {{:remote_macro, meta, Integer, :is_odd, 1}, _} + assert meta[:line] == 2 + assert meta[:column] == 16 + + assert_received {{:remote_function, meta, Integer, :parse, 1}, _} + assert meta[:line] == 3 + assert meta[:column] == 19 + + assert_received {{:remote_function, meta, Atom, :to_string, 1}, _} + assert meta[:line] == 4 + assert meta[:column] == 14 + end + + test "traces remote via captures" do + compile_string(""" + require Integer + &Integer.is_odd/1 + &Integer.parse/1 + """) + + assert_received {{:remote_macro, meta, Integer, :is_odd, 1}, _} + assert meta[:line] == 2 + assert meta[:column] == 10 + + assert_received {{:remote_function, meta, Integer, :parse, 1}, _} + assert meta[:line] == 3 + assert meta[:column] == 10 + end + + test "traces locals" do + compile_string(""" + defmodule Sample do + defmacro foo(arg), do: arg + def bar(arg), do: arg + def baz(arg), do: foo(arg) + bar(arg) + end + """) + + assert_received {{:local_macro, meta, :foo, 1}, _} + assert meta[:line] == 4 + assert meta[:column] == 21 + + assert_received {{:local_function, meta, :bar, 1}, _} + assert meta[:line] == 4 + assert meta[:column] == 32 + after + :code.purge(Sample) + :code.delete(Sample) + end + + test "traces locals with capture" do + compile_string(""" + defmodule Sample do + defmacro foo(arg), do: arg + def bar(arg), do: arg + def baz(_), do: {&foo/1, &bar/1} + end + """) + + assert_received {{:local_macro, meta, :foo, 1}, _} + assert meta[:line] == 4 + assert meta[:column] == 21 + + assert_received {{:local_function, meta, :bar, 1}, _} + assert meta[:line] == 4 + assert meta[:column] == 29 + after + :code.purge(Sample) + :code.delete(Sample) + end + + test "traces modules" do + compile_string(""" + defmodule Sample do + :ok + end + """) + + assert_received {:defmodule, %{module: Sample, function: nil}} + assert_received {{:on_module, <<_::binary>>, :none}, %{module: Sample, function: nil}} + after + :code.purge(Sample) + :code.delete(Sample) + end + + test "traces dynamic modules" do + compile_string(""" + Module.create(Sample, :ok, __ENV__) + """) + + assert_received {:defmodule, %{module: Sample, function: nil}} + assert_received {{:on_module, <<_::binary>>, :none}, %{module: Sample, function: nil}} + after + :code.purge(Sample) + :code.delete(Sample) + end + + test "traces module attribute expansion" do + compile_string(""" + defmodule TracersModuleAttribute do + @module URI + @module + end + """) + + assert_received {{:alias_reference, [line: 3], URI}, %{file: "@module"}} + end + + test "traces string interpolation" do + compile_string(""" + arg = 1 + 2 + "foo\#{arg}" + """) + + assert_received {{:remote_macro, meta, Kernel, :to_string, 1}, _env} + assert meta[:from_interpolation] + end + + test "traces bracket access" do + compile_string(""" + foo = %{bar: 3} + foo[:bar] + """) + + assert_received {{:remote_function, meta, Access, :get, 2}, _env} + assert meta[:from_brackets] + + compile_string(""" + defmodule TracerBracketAccess do + @foo %{bar: 3} + def a() do + @foo[:bar] + end + end + """) + + assert_received {{:remote_function, meta, Access, :get, 2}, _env} + assert meta[:from_brackets] + + compile_string(""" + %{bar: 3}[:bar] + """) + + assert_received {{:remote_function, meta, Access, :get, 2}, _env} + assert meta[:from_brackets] + end + + test "traces on_load" do + compile_string(""" + defmodule TracerOnLoad do + @on_load :init + def init, do: :ok + end + """) + + assert_received {{:local_function, meta, :init, 0}, _} + assert meta[:line] == 1 + end + + def __before_compile__(_), do: :ok + def __after_compile__(_, _), do: :ok + def __after_verify__(_), do: :ok + def __on_definition__(_, _, _, _, _, _), do: :ok + + test "traces compile time attributes" do + compile_string(""" + defmodule TracerCompileAttributes do + @before_compile Kernel.TracersTest + @after_compile Kernel.TracersTest + @on_definition Kernel.TracersTest + @after_verify Kernel.TracersTest + def hello, do: :world + end + """) + + assert_received {{:remote_function, meta, __MODULE__, :__before_compile__, 1}, _} + assert meta[:line] == 1 + + assert_received {{:remote_function, meta, __MODULE__, :__after_compile__, 2}, _} + assert meta[:line] == 1 + + assert_received {{:remote_function, meta, __MODULE__, :__after_verify__, 1}, _} + assert meta[:line] == 1 + + assert_received {{:remote_function, meta, __MODULE__, :__on_definition__, 6}, _} + assert meta[:line] == 6 + end + + test "traces super" do + compile_string(""" + defmodule TracerOverridable do + def local(x), do: x + defoverridable [local: 1] + def local(x), do: super(x) + + defmacro macro(x), do: x + defoverridable [macro: 1] + defmacro macro(x), do: super(x) + + def capture(x), do: x + defoverridable [capture: 1] + def capture(x), do: tap(x, &super/1) + + def capture_arg(x), do: x + defoverridable [capture_arg: 1] + def capture_arg(x), do: tap(x, &super(&1)) + end + """) + + assert_received {{:local_function, _, :"local (overridable 1)", 1}, _} + assert_received {{:local_function, _, :"macro (overridable 1)", 1}, _} + assert_received {{:local_function, _, :"capture (overridable 1)", 1}, _} + assert_received {{:local_function, _, :"capture_arg (overridable 1)", 1}, _} + refute_received {{:local_function, _, _, _}, _} + end + + test "does not trace bind quoted twice" do + compile_string(""" + quote bind_quoted: [foo: List.flatten([])] do + foo + end + """) + + assert_received {{:remote_function, _, List, :flatten, 1}, _} + refute_received {{:remote_function, _, List, :flatten, 1}, _} + end + + test "does not trace captures twice" do + compile_string(""" + &List.flatten/1 + """) + + assert_received {{:remote_function, _, List, :flatten, 1}, _} + refute_received {{:remote_function, _, List, :flatten, 1}, _} + end + + """ + # Make sure this module is compiled with column information + defmodule MacroWithColumn do + defmacro some_macro(list) do + quote do + Enum.map(unquote(list), fn str -> String.upcase(str) end) + end + end + end + """ + |> Code.string_to_quoted!(columns: true) + |> Code.compile_quoted() + + test "traces quoted from macro expansion without column information" do + compile_string(""" + require MacroWithColumn + MacroWithColumn.some_macro(["hello", "world", "!"]) + """) + + assert_received {{:alias_reference, meta, Enum}, _env} + refute meta[:column] + end +end diff --git a/lib/elixir/test/elixir/kernel/typespec_test.exs b/lib/elixir/test/elixir/kernel/typespec_test.exs deleted file mode 100644 index 80908987812..00000000000 --- a/lib/elixir/test/elixir/kernel/typespec_test.exs +++ /dev/null @@ -1,549 +0,0 @@ -Code.require_file "../test_helper.exs", __DIR__ - -defmodule Kernel.TypespecTest do - use ExUnit.Case, async: true - - # This macro allows us to focus on the result of the - # definition and not on the hassles of handling test - # module - defmacrop test_module([{:do, block}]) do - quote do - {:module, _, binary, _} = defmodule TestTypespec do - unquote(block) - end - :code.delete(TestTypespec) - :code.purge(TestTypespec) - binary - end - end - - defp types(module) do - Kernel.Typespec.beam_types(module) - |> Enum.sort - end - - @skip_specs [__info__: 1] - - defp specs(module) do - Kernel.Typespec.beam_specs(module) - |> Enum.reject(fn {sign, _} -> sign in @skip_specs end) - |> Enum.sort() - end - - defp callbacks(module) do - Kernel.Typespec.beam_callbacks(module) - |> Enum.sort - end - - test "invalid type specification" do - assert_raise CompileError, ~r"invalid type specification: mytype = 1", fn -> - test_module do - @type mytype = 1 - end - end - end - - test "invalid function specification" do - assert_raise CompileError, ~r"invalid function type specification: myfun = 1", fn -> - test_module do - @spec myfun = 1 - end - end - end - - test "@type with a single type" do - module = test_module do - @type mytype :: term - end - - assert [type: {:mytype, {:type, _, :term, []}, []}] = - types(module) - end - - test "@type with an atom" do - module = test_module do - @type mytype :: :atom - end - - assert [type: {:mytype, {:atom, _, :atom}, []}] = - types(module) - end - - test "@type with an atom alias" do - module = test_module do - @type mytype :: Atom - end - - assert [type: {:mytype, {:atom, _, Atom}, []}] = - types(module) - end - - test "@type with an integer" do - module = test_module do - @type mytype :: 10 - end - assert [type: {:mytype, {:integer, _, 10}, []}] = - types(module) - end - - test "@type with a negative integer" do - module = test_module do - @type mytype :: -10 - end - - assert [type: {:mytype, {:op, _, :-, {:integer, _, 10}}, []}] = - types(module) - end - - test "@type with a remote type" do - module = test_module do - @type mytype :: Remote.Some.type - @type mytype_arg :: Remote.type(integer) - end - - assert [type: {:mytype, {:remote_type, _, [{:atom, _, Remote.Some}, {:atom, _, :type}, []]}, []}, - type: {:mytype_arg, {:remote_type, _, [{:atom, _, Remote}, {:atom, _, :type}, [{:type, _, :integer, []}]]}, []}] = - types(module) - end - - test "@type with a binary" do - module = test_module do - @type mytype :: binary - end - - assert [type: {:mytype, {:type, _, :binary, []}, []}] = - types(module) - end - - test "@type with an empty binary" do - module = test_module do - @type mytype :: <<>> - end - - assert [type: {:mytype, {:type, _, :binary, [{:integer, _, 0}, {:integer, _, 0}]}, []}] = - types(module) - end - - test "@type with a binary with a base size" do - module = test_module do - @type mytype :: <<_ :: 3>> - end - - assert [type: {:mytype, {:type, _, :binary, [{:integer, _, 3}, {:integer, _, 0}]}, []}] = - types(module) - end - - test "@type with a binary with a unit size" do - module = test_module do - @type mytype :: <<_ :: _ * 8>> - end - - assert [type: {:mytype, {:type, _, :binary, [{:integer, _, 0}, {:integer, _, 8}]}, []}] = - types(module) - end - - test "@type with a range" do - module = test_module do - @type mytype :: range(1, 10) - end - - assert [type: {:mytype, {:type, _, :range, [{:integer, _, 1}, {:integer, _, 10}]}, []}] = - types(module) - end - - test "@type with a range op" do - module = test_module do - @type mytype :: 1..10 - end - - assert [type: {:mytype, {:type, _, :range, [{:integer, _, 1}, {:integer, _, 10}]}, []}] = - types(module) - end - - test "@type with a map" do - module = test_module do - @type mytype :: %{hello: :world} - end - - assert [type: {:mytype, - {:type, _, :map, [ - {:type, _, :map_field_assoc, {:atom, _, :hello}, {:atom, _, :world}} - ]}, - []}] = types(module) - end - - test "@type with a struct" do - module = test_module do - @type mytype :: %User{hello: :world} - end - - assert [type: {:mytype, - {:type, _, :map, [ - {:type, _, :map_field_assoc, {:atom, _, :__struct__}, {:atom, _, User}}, - {:type, _, :map_field_assoc, {:atom, _, :hello}, {:atom, _, :world}} - ]}, - []}] = types(module) - end - - test "@type with a tuple" do - module = test_module do - @type mytype :: tuple - @type mytype1 :: {} - @type mytype2 :: {1, 2} - end - - assert [type: {:mytype, {:type, _, :tuple, :any}, []}, - type: {:mytype1, {:type, _, :tuple, []}, []}, - type: {:mytype2, {:type, _, :tuple, [{:integer, _, 1}, {:integer, _, 2}]}, []}] = - types(module) - end - - test "@type with list shortcuts" do - module = test_module do - @type mytype :: [] - @type mytype1 :: [integer] - @type mytype2 :: [integer, ...] - end - - assert [type: {:mytype, {:type, _, :nil, []}, []}, - type: {:mytype1, {:type, _, :list, [{:type, _, :integer, []}]}, []}, - type: {:mytype2, {:type, _, :nonempty_list, [{:type, _, :integer, []}]}, []}] = - types(module) - end - - test "@type with a fun" do - module = test_module do - @type mytype :: (... -> any) - end - - assert [type: {:mytype, {:type, _, :fun, []}, []}] = - types(module) - end - - test "@type with a fun with multiple arguments and return type" do - module = test_module do - @type mytype :: (integer, integer -> integer) - end - - assert [type: {:mytype, {:type, _, :fun, [{:type, _, :product, - [{:type, _, :integer, []}, {:type, _, :integer, []}]}, - {:type, _, :integer, []}]}, []}] = - types(module) - end - - test "@type with a fun with no arguments and return type" do - module = test_module do - @type mytype :: (() -> integer) - end - - assert [type: {:mytype, {:type, _, :fun, [{:type, _, :product, []}, - {:type, _, :integer, []}]}, []}] = - types(module) - end - - test "@type with a fun with any arity and return type" do - module = test_module do - @type mytype :: (... -> integer) - end - - assert [type: {:mytype, {:type, _, :fun, [{:type, _, :any}, - {:type, _, :integer, []}]}, []}] = - types(module) - end - - test "@type with a union" do - module = test_module do - @type mytype :: integer | char_list | atom - end - - assert [type: {:mytype, {:type, _, :union, [{:type, _, :integer, []}, - {:remote_type, _, [{:atom, _, :elixir}, {:atom, _, :char_list}, []]}, - {:type, _, :atom, []}]}, []}] = - types(module) - end - - test "@type with keywords" do - module = test_module do - @type mytype :: [first: integer, step: integer, last: integer] - end - - assert [type: {:mytype, {:type, _, :list, [ - {:type, _, :union, [ - {:type, _, :tuple, [{:atom, _, :first}, {:type, _, :integer, []}]}, - {:type, _, :tuple, [{:atom, _, :step}, {:type, _, :integer, []}]}, - {:type, _, :tuple, [{:atom, _, :last}, {:type, _, :integer, []}]} - ]} - ]}, []}] = types(module) - end - - test "@type with parameters" do - module = test_module do - @type mytype(x) :: x - @type mytype1(x) :: list(x) - @type mytype2(x, y) :: {x, y} - end - - assert [type: {:mytype, {:var, _, :x}, [{:var, _, :x}]}, - type: {:mytype1, {:type, _, :list, [{:var, _, :x}]}, [{:var, _, :x}]}, - type: {:mytype2, {:type, _, :tuple, [{:var, _, :x}, {:var, _, :y}]}, [{:var, _, :x}, {:var, _, :y}]}] = - types(module) - end - - test "@type with annotations" do - module = test_module do - @type mytype :: (named :: integer) - @type mytype1 :: (a :: integer -> integer) - end - - assert [type: {:mytype, {:ann_type, _, [{:var, _, :named}, {:type, _, :integer, []}]}, []}, - type: {:mytype1, {:type, _, :fun, [{:type, _, :product, [{:ann_type, _, [{:var, _, :a}, {:type, _, :integer, []}]}]}, {:type, _, :integer, []}]}, []}] = - types(module) - end - - test "@opaque(type)" do - module = test_module do - @opaque mytype(x) :: x - end - - assert [opaque: {:mytype, {:var, _, :x}, [{:var, _, :x}]}] = - types(module) - end - - test "@type + opaque" do - module = test_module do - @type mytype :: tuple - @opaque mytype1 :: {} - end - - assert [opaque: {:mytype1, _, []}, - type: {:mytype, _, []},] = - types(module) - end - - test "@type from structs" do - module = test_module do - defstruct name: nil, age: 0 :: non_neg_integer - end - - assert [type: {:t, {:type, _, :map, [ - {:type, _, :map_field_assoc, {:atom, _, :name}, {:type, _, :term, []}}, - {:type, _, :map_field_assoc, {:atom, _, :age}, {:type, _, :non_neg_integer, []}}, - {:type, _, :map_field_assoc, {:atom, _, :__struct__}, {:atom, _, TestTypespec}} - ]}, []}] = types(module) - end - - test "@type from dynamic structs" do - module = test_module do - fields = [name: nil, age: 0] - defstruct fields - end - - assert [type: {:t, {:type, _, :map, [ - {:type, _, :map_field_assoc, {:atom, _, :name}, {:type, _, :term, []}}, - {:type, _, :map_field_assoc, {:atom, _, :age}, {:type, _, :term, []}}, - {:type, _, :map_field_assoc, {:atom, _, :__struct__}, {:atom, _, TestTypespec}} - ]}, []}] = types(module) - end - - test "@type unquote fragment" do - module = test_module do - quoted = quote unquote: false do - name = :mytype - type = :atom - @type unquote(name)() :: unquote(type) - end - Module.eval_quoted(__MODULE__, quoted) |> elem(0) - end - - assert [type: {:mytype, {:atom, _, :atom}, []}] = - types(module) - end - - test "defines_type?" do - test_module do - @type mytype :: tuple - @type mytype(a) :: [a] - assert Kernel.Typespec.defines_type?(__MODULE__, :mytype, 0) - assert Kernel.Typespec.defines_type?(__MODULE__, :mytype, 1) - refute Kernel.Typespec.defines_type?(__MODULE__, :mytype, 2) - end - end - - test "@spec(spec)" do - module = test_module do - def myfun1(x), do: x - def myfun2(), do: :ok - def myfun3(x, y), do: {x, y} - def myfun4(x), do: x - @spec myfun1(integer) :: integer - @spec myfun2() :: integer - @spec myfun3(integer, integer) :: {integer, integer} - @spec myfun4(x :: integer) :: integer - end - - assert [{{:myfun1, 1}, [{:type, _, :fun, [{:type, _, :product, [{:type, _, :integer, []}]}, {:type, _, :integer, []}]}]}, - {{:myfun2, 0}, [{:type, _, :fun, [{:type, _, :product, []}, {:type, _, :integer, []}]}]}, - {{:myfun3, 2}, [{:type, _, :fun, [{:type, _, :product, [{:type, _, :integer, []}, {:type, _, :integer, []}]}, {:type, _, :tuple, [{:type, _, :integer, []}, {:type, _, :integer, []}]}]}]}, - {{:myfun4, 1}, [{:type, _, :fun, [{:type, _, :product, [{:ann_type, _, [{:var, _, :x}, {:type, _, :integer, []}]}]}, {:type, _, :integer, []}]}]}] = - specs(module) - end - - test "@spec(spec) with guards" do - module = test_module do - def myfun1(x), do: x - @spec myfun1(x) :: boolean when [x: integer] - - def myfun2(x), do: x - @spec myfun2(x) :: x when [x: var] - - def myfun3(_x, y), do: y - @spec myfun3(x, y) :: y when [y: x, x: var] - end - - assert [{{:myfun1, 1}, [{:type, _, :bounded_fun, [{:type, _, :fun, [{:type, _, :product, [{:var, _, :x}]}, {:type, _, :boolean, []}]}, [{:type, _, :constraint, [{:atom, _, :is_subtype}, [{:var, _, :x}, {:type, _, :integer, []}]]}]]}]}, - {{:myfun2, 1}, [{:type, _, :fun, [{:type, _, :product, [{:var, _, :x}]}, {:var, _, :x}]}]}, - {{:myfun3, 2}, [{:type, _, :bounded_fun, [{:type, _, :fun, [{:type, _, :product, [{:var, _, :x}, {:var, _, :y}]}, {:var, _, :y}]}, [{:type, _, :constraint, [{:atom, _, :is_subtype}, [{:var, _, :y}, {:var, _, :x}]]}]]}]}] = - specs(module) - end - - test "@callback(callback)" do - module = test_module do - @callback myfun(integer) :: integer - @callback myfun() :: integer - @callback myfun(integer, integer) :: {integer, integer} - end - - assert [{{:myfun, 0}, [{:type, _, :fun, [{:type, _, :product, []}, {:type, _, :integer, []}]}]}, - {{:myfun, 1}, [{:type, _, :fun, [{:type, _, :product, [{:type, _, :integer, []}]}, {:type, _, :integer, []}]}]}, - {{:myfun, 2}, [{:type, _, :fun, [{:type, _, :product, [{:type, _, :integer, []}, {:type, _, :integer, []}]}, {:type, _, :tuple, [{:type, _, :integer, []}, {:type, _, :integer, []}]}]}]}] = - callbacks(module) - end - - test "@spec + @callback" do - module = test_module do - def myfun(x), do: x - @spec myfun(integer) :: integer - @spec myfun(char_list) :: char_list - @callback cb(integer) :: integer - end - - assert [{{:cb, 1}, [{:type, _, :fun, [{:type, _, :product, [{:type, _, :integer, []}]}, {:type, _, :integer, []}]}]}] = - callbacks(module) - - assert [{{:myfun, 1}, [ - {:type, _, :fun, [{:type, _, :product, [ - {:remote_type, _, [{:atom, _, :elixir}, {:atom, _, :char_list}, []]}]}, - {:remote_type, _, [{:atom, _, :elixir}, {:atom, _, :char_list}, []]}]}, - {:type, _, :fun, [{:type, _, :product, [{:type, _, :integer, []}]}, {:type, _, :integer, []}]}]}] = - specs(module) - end - - test "block handling" do - module = test_module do - @spec foo((() -> [ integer ])) :: integer - def foo(_), do: 1 - end - assert [{{:foo, 1}, - [{:type, _, :fun, [{:type, _, :product, [ - {:type, _, :fun, [{:type, _, :product, []}, {:type, _, :list, [{:type, _, :integer, []}]}]}]}, - {:type, _, :integer, []}]}]}] = - specs(module) - end - - # Conversion to AST - - test "type_to_ast" do - quoted = [ - (quote do: @type with_ann() :: (t :: atom())), - (quote do: @type empty_tuple_type() :: {}), - (quote do: @type imm_type_1() :: 1), - (quote do: @type imm_type_2() :: :atom), - (quote do: @type simple_type() :: integer()), - (quote do: @type param_type(p) :: [p]), - (quote do: @type union_type() :: integer() | binary() | boolean()), - (quote do: @type binary_type1() :: <<_ :: _ * 8>>), - (quote do: @type binary_type2() :: <<_ :: 3 * 8>>), - (quote do: @type binary_type3() :: <<_ :: 3>>), - (quote do: @type tuple_type() :: {integer()}), - (quote do: @type ftype() :: (() -> any()) | (() -> integer()) | ((integer() -> integer()))), - (quote do: @type cl() :: char_list()), - (quote do: @type ab() :: as_boolean(term())), - (quote do: @type vaf() :: (... -> any())), - (quote do: @type rng() :: 1 .. 10), - (quote do: @type opts() :: [first: integer(), step: integer(), last: integer()]), - (quote do: @type ops() :: {+1,-1}), - (quote do: @type my_map() :: %{hello: :world}), - (quote do: @type my_struct() :: %User{hello: :world}), - ] |> Enum.sort - - module = test_module do - Module.eval_quoted __MODULE__, quote do: (unquote_splicing(quoted)) - end - - types = types(module) - - Enum.each(Enum.zip(types, quoted), fn {{:type, type}, definition} -> - ast = Kernel.Typespec.type_to_ast(type) - assert Macro.to_string(quote do: @type unquote(ast)) == Macro.to_string(definition) - end) - end - - test "type_to_ast for paren_type" do - type = {:my_type, {:paren_type, 0, [{:type, 0, :integer, []}]}, []} - assert Kernel.Typespec.type_to_ast(type) == - {:::, [], [{:my_type, [], []}, {:integer, [line: 0], []}]} - end - - test "spec_to_ast" do - quoted = [ - (quote do: @spec a() :: integer()), - (quote do: @spec a(atom()) :: integer() | [{}]), - (quote do: @spec a(b) :: integer() when [b: integer()]), - (quote do: @spec a(b) :: b when [b: var]), - (quote do: @spec a(c :: atom()) :: atom()), - ] |> Enum.sort - - module = test_module do - def a, do: 1 - def a(a), do: a - Module.eval_quoted __MODULE__, quote do: (unquote_splicing(quoted)) - end - - specs = Enum.flat_map(specs(module), fn {{_, _}, specs} -> - Enum.map(specs, fn spec -> - quote do: @spec unquote(Kernel.Typespec.spec_to_ast(:a, spec)) - end) - end) |> Enum.sort - - Enum.each(Enum.zip(specs, quoted), fn {spec, definition} -> - assert Macro.to_string(spec) == Macro.to_string(definition) - end) - end - - test "typedoc retrieval" do - {:module, _, binary, _} = defmodule T do - @typedoc "A" - @type a :: any - @typep b :: any - @typedoc "C" - @opaque c(x, y) :: {x, y} - @type d :: any - @spec uses_b() :: b - def uses_b(), do: nil - end - - :code.delete(T) - :code.purge(T) - - assert [ - {{:c, 2}, "C"}, - {{:a, 0}, "A"} - ] = Kernel.Typespec.beam_typedocs(binary) - end - - test "retrieval invalid data" do - assert Kernel.Typespec.beam_typedocs(Unknown) == nil - assert Kernel.Typespec.beam_types(Unknown) == nil - assert Kernel.Typespec.beam_specs(Unknown) == nil - end -end diff --git a/lib/elixir/test/elixir/kernel/warning_test.exs b/lib/elixir/test/elixir/kernel/warning_test.exs index fbfa7dad5f6..58aa6a047ba 100644 --- a/lib/elixir/test/elixir/kernel/warning_test.exs +++ b/lib/elixir/test/elixir/kernel/warning_test.exs @@ -1,493 +1,2314 @@ -Code.require_file "../test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) defmodule Kernel.WarningTest do use ExUnit.Case import ExUnit.CaptureIO - defp capture_err(fun) do - capture_io(:stderr, fun) + defp capture_err(fun) when is_function(fun), do: capture_io(:stderr, fun) + + defp assert_messages_match(messages, output) when is_list(messages) do + for message <- messages do + assert output =~ message + end end - test :unused_variable do - assert capture_err(fn -> - Code.eval_string """ + defp assert_warn_eval(messages, source) do + captured = + capture_err(fn -> + quoted = Code.string_to_quoted!(source, columns: true) + Code.eval_quoted(quoted) + end) + + assert_messages_match(messages, captured) + end + + defp assert_warn_quoted(messages, source) do + captured = + capture_err(fn -> + Code.string_to_quoted!(source, columns: true) + end) + + assert_messages_match(messages, captured) + end + + defp assert_warn_compile(messages, source) do + captured = + capture_err(fn -> + quoted = Code.string_to_quoted!(source, columns: true) + Code.compile_quoted(quoted) + end) + + assert_messages_match(messages, captured) + end + + defp capture_eval(source) do + capture_err(fn -> + quoted = Code.string_to_quoted!(source, columns: true) + Code.eval_quoted(quoted) + end) + end + + defp capture_compile(source) do + capture_err(fn -> + quoted = Code.string_to_quoted!(source, columns: true) + Code.compile_quoted(quoted) + end) + end + + defmacro will_warn do + quote file: "demo", line: true do + %{dup: 1, dup: 2} + end + end + + test "warnings from macro" do + assert_warn_eval( + ["demo:64\n", "key :dup will be overridden in map\n"], + """ + import Kernel.WarningTest + will_warn() + """ + ) + end + + test "outdented heredoc" do + assert_warn_eval( + ["nofile:2:3", "outdented heredoc line"], + """ + ''' + outdented + ''' + """ + ) + end + + test "does not warn on incomplete tokenization" do + assert {:error, _} = Code.string_to_quoted(~s[:"foobar" do]) + end + + describe "unicode identifier security" do + test "warns on confusables" do + assert_warn_quoted( + ["nofile:1:6", "confusable identifier: 'a' looks like 'а' on line 1"], + "а=1; a=1" + ) + + assert_warn_quoted( + ["nofile:1:12", "confusable identifier: 'a' looks like 'а' on line 1"], + "[{:а, 1}, {:a, 1}]" + ) + + assert_warn_quoted( + ["nofile:1:8", "confusable identifier: 'a' looks like 'а' on line 1"], + "[а: 1, a: 1]" + ) + + assert_warn_quoted( + ["nofile:1:18", "confusable identifier: 'a' looks like 'а' on line 1"], + "quote do: [а(1), a(1)]" + ) + + assert_warn_quoted( + ["nofile:1:6", "confusable identifier: 'カ' looks like '力' on line 1"], + "力=1; カ=1" + ) + + # by convention, doesn't warn on ascii-only confusables + assert capture_eval("x0 = xO = 1") == "" + assert capture_eval("l1 = ll = 1") == "" + + # works with a custom atom encoder + assert capture_err(fn -> + Code.string_to_quoted("[{:а, 1}, {:a, 1}]", + static_atoms_encoder: fn token, _ -> {:ok, {:wrapped, token}} end + ) + end) =~ + "confusable identifier: 'a' looks like 'а' on line 1" + end + + test "warns on LTR-confusables" do + # warning outputs in byte order (vs bidi algo display order, uax9), mentions presence of rtl + assert_warn_quoted( + ["nofile:1:9", "'_1א' looks like '_א1'", "right-to-left characters"], + "_א1 and _1א" + ) + + assert_warn_quoted( + [ + "'a_1א' includes right-to-left characters", + "\\u0061 a ltr", + "\\u005F _ neutral", + "\\u0031 1 weak_number", + "\\u05D0 א rtl", + "'a_א1' includes right-to-left characters:", + "\\u0061 a ltr", + "\\u005F _ neutral", + "\\u05D0 א rtl", + "\\u0031 1 weak_number" + ], + "a_א1 or a_1א" + ) + end + end + + test "operators formed by many of the same character followed by that character" do + assert_warn_eval( + [ + "nofile:1:12", + "found \"+++\" followed by \"+\", please use a space between \"+++\" and the next \"+\"" + ], + "quote do: 1++++1" + ) + end + + test "identifier that ends in ! followed by the = operator without a space in between" do + assert_warn_eval( + ["nofile:1:1", "found identifier \"foo!\", ending with \"!\""], + "foo!= 1" + ) + + assert_warn_eval( + ["nofile:1:1", "found atom \":foo!\", ending with \"!\""], + ":foo!= :foo!" + ) + end + + describe "unnecessary quotes" do + test "does not warn of unnecessary quotes in uppercase atoms/keywords" do + assert capture_eval(~s/:"Foo"/) == "" + assert capture_eval(~s/["Foo": :bar]/) == "" + assert capture_eval(~s/:"Foo"/) == "" + assert capture_eval(~s/:"foo@bar"/) == "" + assert capture_eval(~s/:"héllò"/) == "" + assert capture_eval(~s/:"3L1X1R"/) == "" + end + + test "warns of unnecessary quotes" do + assert_warn_eval( + ["nofile:1:1", "found quoted atom \"foo\" but the quotes are not required"], + ~s/:"foo"/ + ) + + assert_warn_eval( + ["nofile:1:2", "found quoted keyword \"foo\" but the quotes are not required"], + ~s/["foo": :bar]/ + ) + + assert_warn_eval( + ["nofile:1:9", "found quoted call \"length\" but the quotes are not required"], + ~s/[Kernel."length"([])]/ + ) + end + end + + describe "deprecated single quotes in atoms" do + test "warns of single quotes in atoms" do + assert_warn_eval( + [ + "nofile:1:1", + "single quotes around atoms are deprecated. Use double quotes instead" + ], + ~s/:'a+b'/ + ) + end + + test "warns twice of single and unnecessary atom quotes" do + assert_warn_eval( + [ + "nofile:1:1", + "single quotes around atoms are deprecated. Use double quotes instead", + "nofile:1:1", + "found quoted atom \"ab\" but the quotes are not required" + ], + ~s/:'ab'/ + ) + end + + test "warns twice of single and unnecessary call quotes" do + assert_warn_eval( + [ + "nofile:1:9", + "single quotes around calls are deprecated. Use double quotes instead", + "nofile:1:9", + "found quoted call \"length\" but the quotes are not required" + ], + ~s/[Kernel.'length'([])]/ + ) + end + end + + test "warns on :: as atom" do + assert_warn_eval( + [ + "nofile:1:1", + "atom ::: must be written between quotes, as in :\"::\", to avoid ambiguity" + ], + ~s/:::/ + ) + end + + test "unused variable" do + # Note we use compile_string because eval_string does not emit unused vars warning + assert_warn_compile( + [ + "nofile:2:3", + "variable \"module\" is unused", + "nofile:3:13", + "variable \"arg\" is unused", + "nofile:5:1", + "variable \"file\" is unused" + ], + """ defmodule Sample do + module = 1 def hello(arg), do: nil end + file = 2 + file = 3 + file """ - end) =~ "warning: variable arg is unused" + ) after - purge Sample + purge(Sample) end - test :unused_function do - assert capture_err(fn -> - Code.eval_string """ + test "unused variable that could be pinned" do + # Note we use compile_string because eval_string does not emit unused vars warning + assert_warn_compile( + [ + "nofile:4:12", + "variable \"compare_local\" is unused (there is a variable with the same name in the context, use the pin operator (^) to match on it or prefix this variable with underscore if it is not meant to be used)", + "nofile:8:7", + "variable \"compare_nested\" is unused (there is a variable with the same name in the context, use the pin operator (^) to match on it or prefix this variable with underscore if it is not meant to be used)" + ], + """ + defmodule Sample do + def test do + compare_local = "hello" + match?(compare_local, "hello") + + compare_nested = "hello" + case "hello" do + compare_nested -> true + _other -> false + end + end + end + """ + ) + after + purge(Sample) + end + + test "duplicate pattern" do + output = + capture_eval(""" + defmodule Sample do + var = quote(do: x) + def hello(unquote(var) = unquote(var)), do: unquote(var) + end + """) + + assert output =~ "this pattern is matched against itself inside a match: x = x" + after + purge(Sample) + end + + test "unused compiler variable" do + output = + capture_eval(""" + defmodule Sample do + def hello(__MODULE___), do: :ok + def world(_R), do: :ok + end + """) + + assert output =~ "unknown compiler variable \"__MODULE___\"" + assert output =~ "nofile:2:13" + refute output =~ "unknown compiler variable \"_R\"" + after + purge(Sample) + end + + test "nested unused variable" do + messages = ["undefined variable \"x\"", "variable \"x\" is unused"] + + assert_compile_error( + ["nofile:5:1", "nofile:2:11" | messages], + """ + case false do + true -> x = 1 + _ -> 1 + end + x + """ + ) + + assert_compile_error( + ["nofile:1:12", "nofile:2:1" | messages], + """ + false and (x = 1) + x + """ + ) + + assert_compile_error( + ["nofile:1:10", "nofile:2:1" | messages], + """ + true or (x = 1) + x + """ + ) + + assert_compile_error( + ["nofile:2:3", "nofile:4:1" | messages], + """ + if false do + x = 1 + end + x + """ + ) + + assert_compile_error( + ["nofile:2:12", "nofile:5:1" | messages], + """ + cond do + false -> x = 1 + true -> 1 + end + x + """ + ) + + assert_compile_error( + ["nofile:2:11", "nofile:6:1" | messages], + """ + receive do + :foo -> x = 1 + after + 0 -> 1 + end + x + """ + ) + + assert_compile_error( + ["nofile:1:11", "nofile:2:1" | messages], + """ + false && (x = 1) + x + """ + ) + + assert_compile_error( + ["nofile:1:10", "nofile:2:1" | messages], + """ + true || (x = 1) + x + """ + ) + + assert_compile_error( + ["nofile:2:3", "nofile:4:1" | messages], + """ + with true <- true do + x = false + end + x + """ + ) + + assert_compile_error( + ["nofile:2:3", "nofile:4:1" | messages], + """ + fn -> + x = true + end + x + """ + ) + end + + test "unused variable in redefined function in different file" do + output = + capture_err(fn -> + Code.eval_string(""" + defmodule Sample do + defmacro __using__(_) do + quote location: :keep do + def function(arg) + end + end + end + """) + + code = """ + defmodule RedefineSample do + use Sample + def function(var123), do: nil + end + """ + + Code.eval_string(code, [], file: "redefine_sample.ex") + end) + + assert output =~ "redefine_sample.ex:3: " + assert output =~ "variable \"var123\" is unused" + after + purge(Sample) + purge(RedefineSample) + end + + test "unused variable because re-declared in a match? pattern" do + assert_warn_eval( + [ + "nofile:1:16", + "variable \"x\" is unused (there is a variable with the same name in the context,", + "variable \"x\" is unused (if the variable is not meant to be used," + ], + """ + fn x -> match?(x, :value) end + """ + ) + end + + test "useless literal" do + message = "code block contains unused literal \"oops\"" + + assert_warn_eval( + ["nofile:1\n", message], + """ + "oops" + :ok + """ + ) + + assert_warn_eval( + ["nofile:1\n", message], + """ + fn -> + "oops" + :ok + end + """ + ) + + assert_warn_eval( + ["nofile:1:5\n", message], + """ + try do + "oops" + :ok + after + :ok + end + """ + ) + end + + test "useless attr" do + assert_warn_eval( + [ + "nofile:4:3", + "module attribute @foo in code block has no effect as it is never returned ", + "nofile:7:5", + "module attribute @bar in code block has no effect as it is never returned " + ], + """ + defmodule Sample do + @foo 1 + @bar 1 + @foo + + def bar do + @bar + :ok + end + end + """ + ) + after + purge(Sample) + end + + test "useless var" do + message = "variable foo in code block has no effect as it is never returned " + + assert_warn_eval( + ["nofile:2:1", message], + """ + foo = 1 + foo + :ok + """ + ) + + assert_warn_eval( + ["nofile:3:3", message], + """ + fn -> + foo = 1 + foo + :ok + end + """ + ) + + assert_warn_eval( + ["nofile:3:3", message], + """ + try do + foo = 1 + foo + :ok + after + :ok + end + """ + ) + + assert capture_eval(""" + node() + :ok + """) == "" + end + + test "underscored variable on match" do + assert_warn_eval( + ["nofile:1:8", "the underscored variable \"_arg\" appears more than once in a match"], + """ + {_arg, _arg} = {1, 1} + """ + ) + end + + test "underscored variable on use" do + assert_warn_eval( + ["nofile:1:12", "the underscored variable \"_var\" is used after being set"], + """ + fn _var -> _var + 1 end + """ + ) + + assert capture_eval(""" + fn var!(_var, Foo) -> var!(_var, Foo) + 1 end + """) == "" + end + + test "unused function" do + assert_warn_eval( + ["nofile:2:8: ", "function hello/0 is unused\n"], + """ defmodule Sample1 do defp hello, do: nil end """ - end) =~ "warning: function hello/0 is unused" + ) - assert capture_err(fn -> - Code.eval_string """ + assert_warn_eval( + ["nofile:2:8: ", "function hello/1 is unused\n"], + """ defmodule Sample2 do defp hello(0), do: hello(1) defp hello(1), do: :ok end - """ - end) =~ "function hello/1 is unused" + """ + ) + + assert_warn_eval( + ["nofile:4:8: ", "function c/2 is unused\n"], + ~S""" + defmodule Sample3 do + def a, do: nil + def b, do: d(10) + defp c(x, y \\ 1), do: [x, y] + defp d(x), do: x + end + """ + ) + + assert_warn_eval( + ["nofile:3:8: ", "function b/2 is unused\n"], + ~S""" + defmodule Sample4 do + def a, do: nil + defp b(x \\ 1, y \\ 1) + defp b(x, y), do: [x, y] + end + """ + ) + + assert_warn_eval( + ["nofile:3:8: ", "function b/0 is unused\n"], + ~S""" + defmodule Sample5 do + def a, do: nil + defp b(), do: unquote(1) + end + """ + ) + after + purge([Sample1, Sample2, Sample3, Sample4, Sample5]) + end + + test "unused cyclic functions" do + assert_warn_eval( + [ + "nofile:2:8: ", + "function a/0 is unused\n", + "nofile:3:8: ", + "function b/0 is unused\n" + ], + """ + defmodule Sample do + defp a, do: b() + defp b, do: a() + end + """ + ) + after + purge(Sample) + end + + test "unused macro" do + assert_warn_eval( + ["nofile:2:13: ", "macro hello/0 is unused"], + """ + defmodule Sample do + defmacrop hello, do: nil + end + """ + ) + + assert_warn_eval( + ["nofile:2:13: ", "macro hello/0 is unused\n"], + ~S""" + defmodule Sample2 do + defmacrop hello do + quote do: unquote(1) + end + end + """ + ) + after + purge([Sample, Sample2]) + end + + test "shadowing" do + assert capture_eval(""" + defmodule Sample do + def test(x) do + case x do + {:file, fid} -> fid + {:path, _} -> fn(fid) -> fid end + end + end + end + """) == "" + after + purge(Sample) + end + + test "unused default args" do + assert_warn_eval( + [ + "nofile:3:8: ", + "default values for the optional arguments in the private function b/3 are never used" + ], + ~S""" + defmodule Sample1 do + def a, do: b(1, 2, 3) + defp b(arg1 \\ 1, arg2 \\ 2, arg3 \\ 3), do: [arg1, arg2, arg3] + end + """ + ) + + assert_warn_eval( + [ + "nofile:3:8: ", + "the default value for the last optional argument in the private function b/3 is never used" + ], + ~S""" + defmodule Sample2 do + def a, do: b(1, 2) + defp b(arg1 \\ 1, arg2 \\ 2, arg3 \\ 3), do: [arg1, arg2, arg3] + end + """ + ) + + assert_warn_eval( + [ + "nofile:3:8: ", + "the default values for the last 2 optional arguments in the private function b/4 are never used" + ], + ~S""" + defmodule Sample3 do + def a, do: b(1, 2) + defp b(arg1, arg2 \\ 2, arg3 \\ 3, arg4 \\ 4), do: [arg1, arg2, arg3, arg4] + end + """ + ) + + assert capture_eval(~S""" + defmodule Sample4 do + def a, do: b(1) + defp b(arg1 \\ 1, arg2, arg3 \\ 3), do: [arg1, arg2, arg3] + end + """) == "" + + assert_warn_eval( + [ + "nofile:3:8: ", + "the default value for the last optional argument in the private function b/3 is never used" + ], + ~S""" + defmodule Sample5 do + def a, do: b(1, 2) + defp b(arg1 \\ 1, arg2 \\ 2, arg3 \\ 3) + + defp b(arg1, arg2, arg3), do: [arg1, arg2, arg3] + end + """ + ) + after + purge([Sample1, Sample2, Sample3, Sample4, Sample5]) + end + + test "unused import" do + assert_warn_compile( + ["nofile:2:3", "unused import :lists"], + """ + defmodule Sample do + import :lists + def a, do: nil + end + """ + ) + + assert_warn_compile( + ["nofile:1:1", "unused import :lists"], + """ + import :lists + """ + ) + after + purge(Sample) + end + + test "unknown import" do + assert_warn_compile( + ["nofile:1:1", "cannot import Kernel.invalid/1 because it is undefined or private"], + """ + import(Kernel, only: [invalid: 1]) + """ + ) + end + + test "conditional import" do + assert capture_err(fn -> + defmodule KernelTest.ConditionalImport do + if false do + import Map, only: [new: 0] + def fun, do: new() + end + end + end) == "" + after + purge(Sample) + end + + test "unused import of one of the functions in :only" do + assert_warn_compile( + [ + "nofile:2:3", + "unused import String.downcase/1", + "nofile:2:3", + "unused import String.trim/1" + ], + """ + defmodule Sample do + import String, only: [upcase: 1, downcase: 1, trim: 1] + def a, do: upcase("hello") + end + """ + ) + after + purge(Sample) + end + + test "unused import of any of the functions in :only" do + assert_warn_compile( + ["nofile:1:1", "unused import String"], + """ + import String, only: [upcase: 1, downcase: 1] + """ + ) + end + + test "unused import inside dynamic module" do + import List, only: [flatten: 1], warn: false + + assert capture_err(fn -> + defmodule Sample do + import String, only: [downcase: 1] + + def world do + flatten([1, 2, 3]) + end + end + end) =~ "unused import String" + after + purge(Sample) + end + + def with(a, b, c), do: [a, b, c] + + test "import matches special form" do + assert_warn_compile( + [ + "nofile:1:1", + "cannot import Kernel.WarningTest.with/3 because it conflicts with Elixir special forms" + ], + """ + import Kernel.WarningTest, only: [with: 3] + :ok = with true <- true, true <- true, do: :ok + """ + ) + end + + test "duplicated function on import options" do + assert_warn_compile( + ["nofile:2:3", "invalid :only option for import, wrap/1 is duplicated"], + """ + defmodule Kernel.WarningsTest.DuplicatedFunctionOnImportOnly do + import List, only: [wrap: 1, keyfind: 3, wrap: 1] + end + """ + ) + + assert_warn_compile( + ["nofile:2:3", "invalid :except option for import, wrap/1 is duplicated"], + """ + defmodule Kernel.WarningsTest.DuplicatedFunctionOnImportExcept do + import List, except: [wrap: 1, keyfind: 3, wrap: 1] + end + """ + ) + end + + test "conditional alias" do + assert capture_err(fn -> + defmodule KernelTest.ConditionaAlias do + if false do + alias Map, as: M + def fun, do: M.new() + end + end + end) == "" + end + + test "unused alias" do + assert_warn_compile( + ["nofile:2:3", "unused alias List"], + """ + defmodule Sample do + alias :lists, as: List + def a, do: nil + end + """ + ) + after + purge(Sample) + end + + test "unused alias when also import" do + assert_warn_compile( + ["nofile:2:3", "unused alias List"], + """ + defmodule Sample do + alias :lists, as: List + import MapSet + new() + end + """ + ) + after + purge(Sample) + end + + test "duplicate map keys" do + assert_warn_eval( + [ + "key :a will be overridden in map", + "nofile:4:10\n", + "key :m will be overridden in map", + "nofile:5:10\n", + "key 1 will be overridden in map", + "nofile:6:10\n" + ], + """ + defmodule DuplicateMapKeys do + import ExUnit.Assertions + + assert %{a: :b, a: :c} == %{a: :c} + assert %{m: :n, m: :o, m: :p} == %{m: :p} + assert %{1 => 2, 1 => 3} == %{1 => 3} + end + """ + ) + + assert map_size(%{System.unique_integer() => 1, System.unique_integer() => 2}) == 2 + end + + test "length(list) == 0 in guard" do + assert_warn_eval( + [ + "nofile:5:24", + "do not use \"length(v) == 0\" to check if a list is empty", + "Prefer to pattern match on an empty list or use \"v == []\" as a guard" + ], + """ + defmodule Sample do + def list_case do + v = [] + case v do + _ when length(v) == 0 -> :ok + _ -> :fail + end + end + end + """ + ) + after + purge(Sample) + end + + test "length(list) > 0 in guard" do + assert_warn_eval( + [ + "nofile:5:24", + "do not use \"length(v) > 0\" to check if a list is not empty", + "Prefer to pattern match on a non-empty list, such as [_ | _], or use \"v != []\" as a guard" + ], + """ + defmodule Sample do + def list_case do + v = [] + case v do + _ when length(v) > 0 -> :ok + _ -> :fail + end + end + end + """ + ) + after + purge(Sample) + end + + test "late function heads" do + assert_warn_eval( + [ + "nofile:4:7\n", + "function head for def add/2 must come at the top of its direct implementation" + ], + """ + defmodule Sample do + def add(a, b), do: a + b + @doc "hello" + def add(a, b) + end + """ + ) + after + purge(Sample) + end + + test "late function heads do not warn of meta programming" do + assert capture_eval(""" + defmodule Sample1 do + defmacro __using__(_) do + quote do + def add(a, b), do: a + b + end + end + end + + defmodule Sample2 do + use Sample1 + @doc "hello" + def add(a, b) + end + """) == "" + + assert capture_eval(""" + defmodule Sample3 do + for fun <- [:foo, :bar] do + def unquote(fun)(), do: unquote(fun) + end + + def foo() + def bar() + end + """) == "" + after + purge([Sample1, Sample2, Sample3]) + end + + test "used import via alias" do + assert capture_eval(""" + defmodule Sample1 do + import List, only: [flatten: 1] + + defmacro generate do + List.duplicate(quote(do: flatten([1, 2, 3])), 100) + end + end + + defmodule Sample2 do + import Sample1 + generate() + end + """) == "" + after + purge([Sample1, Sample2]) + end + + test "clause not match" do + assert_warn_eval( + [ + "nofile:3:7\n", + ~r"this clause( for hello/0)? cannot match because a previous clause at line 2 always matches" + ], + """ + defmodule Sample do + def hello, do: nil + def hello, do: nil + end + """ + ) + after + purge(Sample) + end + + test "generated clause not match" do + assert_warn_eval( + [ + "nofile:10\n", + ~r"this clause( for hello/0)? cannot match because a previous clause at line 10 always matches" + ], + """ + defmodule Sample do + defmacro __using__(_) do + quote do + def hello, do: nil + def hello, do: nil + end + end + end + defmodule UseSample do + use Sample + end + """ + ) + after + purge(Sample) + purge(UseSample) + end + + test "deprecated closing sigil delimiter" do + assert_warn_eval(["nofile:1:7", "deprecated"], "~S(foo\\))") + end + + test "deprecated not left in right" do + assert_warn_eval(["nofile:1:7", "deprecated"], "not 1 in [1, 2, 3]") + end + + test "clause with defaults should be first" do + message = "def hello/1 has multiple clauses and also declares default values" + + assert_warn_eval( + ["nofile:3:7\n", message, "the previous clause is defined on line 2"], + ~S""" + defmodule Sample1 do + def hello(arg), do: arg + def hello(arg \\ 0), do: arg + end + """ + ) + + assert_warn_eval( + ["nofile:3:7\n", message, "the previous clause is defined on line 2"], + ~S""" + defmodule Sample2 do + def hello(_arg) + def hello(arg \\ 0), do: arg + end + """ + ) + after + purge([Sample1, Sample2]) + end + + test "clauses with default should use header" do + assert_warn_eval( + [ + "nofile:3:7\n", + "def hello/1 has multiple clauses and also declares default values", + "the previous clause is defined on line 2" + ], + ~S""" + defmodule Sample do + def hello(arg \\ 0), do: arg + def hello(arg), do: arg + end + """ + ) + after + purge(Sample) + end + + test "unused with local with overridable" do + assert_warn_eval( + ["nofile:3:8: ", "function world/0 is unused"], + """ + defmodule Sample do + def hello, do: world() + defp world, do: :ok + defoverridable [hello: 0] + def hello, do: :ok + end + """ + ) + after + purge(Sample) + end + + test "undefined module attribute" do + assert_warn_eval( + [ + "nofile:2: ", + "undefined module attribute @foo, please remove access to @foo or explicitly set it before access" + ], + """ + defmodule Sample do + @foo + end + """ + ) + after + purge(Sample) + end + + test "parens with module attribute" do + assert_warn_eval( + [ + "nofile:3: ", + "the @foo() notation (with parentheses) is deprecated, please use @foo (without parentheses) instead" + ], + """ + defmodule Sample do + @foo 13 + @foo() + end + """ + ) + after + purge(Sample) + end + + test "undefined module attribute in function" do + assert_warn_eval( + [ + "nofile:3: ", + "undefined module attribute @foo, please remove access to @foo or explicitly set it before access" + ], + """ + defmodule Sample do + def hello do + @foo + end + end + """ + ) + after + purge(Sample) + end + + test "undefined module attribute with file" do + assert_warn_eval( + [ + "nofile:2: ", + "undefined module attribute @foo, please remove access to @foo or explicitly set it before access" + ], + """ + defmodule Sample do + @foo + end + """ + ) + after + purge(Sample) + end + + test "parse transform" do + assert_warn_eval( + ["nofile:1: ", "@compile {:parse_transform, :ms_transform} is deprecated"], + """ + defmodule Sample do + @compile {:parse_transform, :ms_transform} + end + """ + ) + after + purge(Sample) + end + + test "@compile inline no warning for unreachable function" do + refute capture_eval(""" + defmodule Sample do + @compile {:inline, foo: 1} + + defp foo(_), do: :ok + end + """) =~ "inlined function foo/1 undefined" + after + purge(Sample) + end + + test "no effect operator" do + assert_warn_eval( + ["nofile:3:7\n", "use of operator != has no effect"], + """ + defmodule Sample do + def a(x) do + x != :foo + :ok + end + end + """ + ) + after + purge(Sample) + end + + test "undefined function for behaviour" do + assert_warn_eval( + [ + "nofile:5: ", + "function foo/0 required by behaviour Sample1 is not implemented (in module Sample2)" + ], + """ + defmodule Sample1 do + @callback foo :: term + end + + defmodule Sample2 do + @behaviour Sample1 + end + """ + ) + after + purge([Sample1, Sample2]) + end + + test "undefined macro for behaviour" do + assert_warn_eval( + [ + "nofile:5: ", + "macro foo/0 required by behaviour Sample1 is not implemented (in module Sample2)" + ], + """ + defmodule Sample1 do + @macrocallback foo :: Macro.t + end + + defmodule Sample2 do + @behaviour Sample1 + end + """ + ) + after + purge([Sample1, Sample2]) + end + + test "wrong kind for behaviour" do + assert_warn_eval( + [ + "nofile:5: ", + "function foo/0 required by behaviour Sample1 was implemented as \"defmacro\" but should have been \"def\"" + ], + """ + defmodule Sample1 do + @callback foo :: term + end + + defmodule Sample2 do + @behaviour Sample1 + defmacro foo, do: :ok + end + """ + ) + after + purge([Sample1, Sample2]) + end + + test "conflicting behaviour" do + assert_warn_eval( + [ + "nofile:9: ", + "conflicting behaviours found. Callback function foo/0 is defined by both Sample1 and Sample2 (in module Sample3)" + ], + """ + defmodule Sample1 do + @callback foo :: term + end + + defmodule Sample2 do + @callback foo :: term + end + + defmodule Sample3 do + @behaviour Sample1 + @behaviour Sample2 + end + """ + ) + after + purge([Sample1, Sample2, Sample3]) + end + + test "conflicting behaviour (but one optional callback)" do + message = + capture_compile(""" + defmodule Sample1 do + @callback foo :: term + end + + defmodule Sample2 do + @callback foo :: term + @callback bar :: term + @optional_callbacks foo: 0 + end + + defmodule Sample3 do + @behaviour Sample1 + @behaviour Sample2 + + @impl Sample1 + def foo, do: 1 + @impl Sample2 + def bar, do: 2 + end + """) + + assert message =~ + "conflicting behaviours found. Callback function foo/0 is defined by both Sample1 and Sample2 (in module Sample3)" + + refute message =~ "module attribute @impl was not set" + refute message =~ "this behaviour does not specify such callback" + after + purge([Sample1, Sample2, Sample3]) + end + + test "duplicate behaviour" do + assert_warn_eval( + [ + "nofile:5: ", + "the behaviour Sample1 has been declared twice (conflict in function foo/0 in module Sample2)" + ], + """ + defmodule Sample1 do + @callback foo :: term + end + + defmodule Sample2 do + @behaviour Sample1 + @behaviour Sample1 + end + """ + ) + after + purge([Sample1, Sample2]) + end + + test "unknown remote call" do + assert capture_compile(""" + defmodule Sample do + def perform(), do: Unknown.call() + end + """) =~ + "Unknown.call/0 is undefined (module Unknown is not available or is yet to be defined)" + after + purge(Sample) + end + + test "undefined behaviour" do + assert_warn_eval( + ["nofile:1: ", "@behaviour UndefinedBehaviour does not exist (in module Sample)"], + """ + defmodule Sample do + @behaviour UndefinedBehaviour + end + """ + ) + after + purge(Sample) + end + + test "empty behaviours" do + assert_warn_eval( + ["nofile:3: ", "module EmptyBehaviour is not a behaviour (in module Sample)"], + """ + defmodule EmptyBehaviour do + end + defmodule Sample do + @behaviour EmptyBehaviour + end + """ + ) + after + purge(Sample) + purge(EmptyBehaviour) + end + + test "undefined function for protocol" do + assert_warn_eval( + [ + "nofile:5: ", + "function foo/1 required by protocol Sample1 is not implemented (in module Sample1.Atom)" + ], + """ + defprotocol Sample1 do + def foo(subject) + end - assert capture_err(fn -> - Code.eval_string ~S""" - defmodule Sample3 do - def a, do: nil - def b, do: d(10) - defp c(x, y \\ 1), do: [x, y] - defp d(x), do: x + defimpl Sample1, for: Atom do end """ - end) =~ "warning: function c/2 is unused" + ) after - purge [Sample1, Sample2, Sample3] + purge([Sample1, Sample1.Atom]) end - test :unused_cyclic_functions do - assert capture_err(fn -> - Code.eval_string """ + test "ungrouped def name" do + assert_warn_eval( + [ + "nofile:4:7\n", + "clauses with the same name should be grouped together, \"def foo/2\" was previously defined (nofile:2)" + ], + """ defmodule Sample do - defp a, do: b - defp b, do: a + def foo(x, 1), do: x + 1 + def foo(), do: nil + def foo(x, 2), do: x * 2 end """ - end) =~ "warning: function a/0 is unused" + ) after - purge Sample + purge(Sample) end - test :unused_macro do - assert capture_err(fn -> - Code.eval_string """ + test "ungrouped def name and arity" do + assert_warn_eval( + [ + "nofile:4:7\n", + "clauses with the same name and arity (number of arguments) should be grouped together, \"def foo/2\" was previously defined (nofile:2)" + ], + """ defmodule Sample do - defmacrop hello, do: nil + def foo(x, 1), do: x + 1 + def bar(), do: nil + def foo(x, 2), do: x * 2 end """ - end) =~ "warning: macro hello/0 is unused" + ) after - purge Sample + purge(Sample) end - test :shadowing do - assert capture_err(fn -> - Code.eval_string """ + test "ungrouped defs do not warn of meta programming" do + assert capture_eval(""" + defmodule Sample do + for atom <- [:foo, :bar] do + def from_string(unquote(to_string(atom))), do: unquote(atom) + def to_string(unquote(atom)), do: unquote(to_string(atom)) + end + end + """) == "" + after + purge(Sample) + end + + test "warning with overridden file" do + assert_warn_eval( + ["sample:3:11:", "variable \"x\" is unused"], + """ defmodule Sample do - def test(x) do - case x do - {:file, fid} -> fid - {:path, _} -> fn(fid) -> fid end - end - end + @file "sample" + def foo(x), do: :ok end """ - end) == "" + ) after - purge Sample + purge(Sample) end - test :unused_default_args do - assert capture_err(fn -> - Code.eval_string ~S""" - defmodule Sample1 do - def a, do: b(1, 2, 3) - defp b(arg1 \\ 1, arg2 \\ 2, arg3 \\ 3), do: [arg1, arg2, arg3] - end - """ - end) =~ "warning: default arguments in b/3 are never used" + test "warning on unnecessary code point escape" do + assert capture_eval("?\\n + ?\\\\") == "" - assert capture_err(fn -> - Code.eval_string ~S""" - defmodule Sample2 do - def a, do: b(1, 2) - defp b(arg1 \\ 1, arg2 \\ 2, arg3 \\ 3), do: [arg1, arg2, arg3] - end - """ - end) =~ "warning: the first 2 default arguments in b/3 are never used" + assert_warn_eval( + ["nofile:1:1", "unknown escape sequence ?\\w, use ?w instead"], + "?\\w" + ) + end - assert capture_err(fn -> - Code.eval_string ~S""" - defmodule Sample3 do - def a, do: b(1) - defp b(arg1 \\ 1, arg2 \\ 2, arg3 \\ 3), do: [arg1, arg2, arg3] - end - """ - end) =~ "warning: the first default argument in b/3 is never used" + test "warning on code point escape" do + assert_warn_eval( + ["nofile:1:1", "found ? followed by code point 0x20 (space), please use ?\\s instead"], + "? " + ) - assert capture_err(fn -> - Code.eval_string ~S""" - defmodule Sample4 do - def a, do: b(1) - defp b(arg1 \\ 1, arg2, arg3 \\ 3), do: [arg1, arg2, arg3] - end - """ - end) == "" - after - purge [Sample1, Sample2, Sample3, Sample4] + assert_warn_eval( + ["nofile:1:1", "found ?\\ followed by code point 0x20 (space), please use ?\\s instead"], + "?\\ " + ) end - test :unused_import do - assert capture_err(fn -> - Code.compile_string """ + test "duplicated docs in the same clause" do + output = + capture_eval(""" defmodule Sample do - import :lists, only: [flatten: 1] - def a, do: nil + @doc "Something" + @doc "Another" + def foo, do: :ok end - """ - end) =~ "warning: unused import :lists" + """) - assert capture_err(fn -> - Code.compile_string """ - import :lists, only: [flatten: 1] - """ - end) =~ "warning: unused import :lists" + assert output =~ "redefining @doc attribute previously set at line 2" + assert output =~ "nofile:3: Sample (module)" after - purge [Sample] + purge(Sample) end - test :unused_alias do - assert capture_err(fn -> - Code.compile_string """ - defmodule Sample do - alias :lists, as: List - def a, do: nil + test "duplicate docs across clauses" do + assert capture_eval(""" + defmodule Sample1 do + defmacro __using__(_) do + quote do + @doc "hello" + def add(a, 1), do: a + 1 + end + end + end + + defmodule Sample2 do + use Sample1 + @doc "world" + def add(a, 2), do: a + 2 + end + """) == "" + + assert_warn_eval( + ["nofile:4: ", "redefining @doc attribute previously set at line"], + """ + defmodule Sample3 do + @doc "hello" + def add(a, 1), do: a + 1 + @doc "world" + def add(a, 2), do: a + 2 end """ - end) =~ "warning: unused alias List" + ) after - purge [Sample] + purge([Sample1, Sample2, Sample3]) end - test :unused_inside_dynamic_module do - import List, only: [flatten: 1], warn: false + test "reserved doc metadata keys" do + {output, diagnostics} = + Code.with_diagnostics([log: true], fn -> + capture_eval(""" + defmodule Sample do + @typedoc opaque: false + @type t :: binary - assert capture_err(fn -> - defmodule Sample do - import String, only: [downcase: 1] + @doc defaults: 3, since: "1.2.3" + def foo(a), do: a + end + """) + end) + + assert output =~ "ignoring reserved documentation metadata key: :opaque" + assert output =~ "nofile:2: " + assert output =~ "ignoring reserved documentation metadata key: :defaults" + assert output =~ "nofile:5: " + refute output =~ ":since" + + assert [ + %{ + message: "ignoring reserved documentation metadata key: :opaque", + position: 2, + file: "nofile", + severity: :warning + }, + %{ + message: "ignoring reserved documentation metadata key: :defaults", + position: 5, + file: "nofile", + severity: :warning + } + ] = diagnostics + after + purge(Sample) + end + + describe "typespecs" do + test "unused types" do + output = + capture_eval(""" + defmodule Sample do + @type pub :: any + @opaque op :: any + @typep priv :: any + @typep priv_args(var1, var2) :: {var1, var2} + @typep priv2 :: any + @typep priv3 :: priv2 | atom + + @spec my_fun(priv3) :: pub + def my_fun(var), do: var + end + """) + + assert output =~ "nofile:4: " + assert output =~ "type priv/0 is unused" + assert output =~ "nofile:5: " + assert output =~ "type priv_args/2 is unused" + refute output =~ "type pub/0 is unused" + refute output =~ "type op/0 is unused" + refute output =~ "type priv2/0 is unused" + refute output =~ "type priv3/0 is unused" + after + purge(Sample) + end + + test "underspecified opaque types" do + output = + capture_eval(""" + defmodule Sample do + @opaque op1 :: term + @opaque op2 :: any + @opaque op3 :: atom + end + """) + + assert output =~ "nofile:2: " + assert output =~ "@opaque type op1/0 is underspecified and therefore meaningless" + assert output =~ "nofile:3: " + assert output =~ "@opaque type op2/0 is underspecified and therefore meaningless" + refute output =~ "nofile:4: " + refute output =~ "op3" + after + purge(Sample) + end + + test "underscored types variables" do + output = + capture_eval(""" + defmodule Sample do + @type in_typespec_vars(_var1, _var1) :: atom + @type in_typespec(_var2) :: {atom, _var2} + + @spec in_spec(_var3) :: {atom, _var3} when _var3: var + def in_spec(a), do: {:ok, a} + end + """) + + assert output =~ "nofile:2: " + assert output =~ ~r/the underscored type variable "_var1" is used more than once/ + assert output =~ "nofile:3: " + assert output =~ ~r/the underscored type variable "_var2" is used more than once/ + assert output =~ "nofile:5: " + assert output =~ ~r/the underscored type variable "_var3" is used more than once/ + after + purge(Sample) + end + + test "typedoc on typep" do + assert_warn_eval( + [ + "nofile:2: ", + "type priv/0 is private, @typedoc's are always discarded for private types" + ], + """ + defmodule Sample do + @typedoc "Something" + @typep priv :: any + @spec foo() :: priv + def foo(), do: nil + end + """ + ) + after + purge(Sample) + end + + test "discouraged types" do + string_discouraged = + "string() type use is discouraged. " <> + "For character lists, use charlist() type, for strings, String.t()\n" + + nonempty_string_discouraged = + "nonempty_string() type use is discouraged. " <> + "For non-empty character lists, use nonempty_charlist() type, for strings, String.t()\n" + + assert_warn_eval( + [ + "nofile:2: ", + string_discouraged, + "nofile:3: ", + nonempty_string_discouraged + ], + """ + defmodule Sample do + @type foo :: string() + @type bar :: nonempty_string() + end + """ + ) + after + purge(Sample) + end + + test "nested type annotations" do + message = "invalid type annotation. Type annotations cannot be nested" + + assert_warn_eval( + ["nofile:2: ", message], + """ + defmodule Sample1 do + @type my_type :: ann_type :: nested_ann_type :: atom + end + """ + ) + + purge(Sample1) + + assert_warn_eval( + ["nofile:2: ", message], + """ + defmodule Sample2 do + @type my_type :: ann_type :: nested_ann_type :: atom | port + end + """ + ) + + purge(Sample2) - def world do - flatten([1,2,3]) + assert_warn_eval( + ["nofile:2: ", message], + """ + defmodule Sample3 do + @spec foo :: {pid, ann_type :: nested_ann_type :: atom} + def foo, do: nil end + """ + ) + after + purge([Sample1, Sample2, Sample3]) + end + + test "invalid fun" do + assert_warn_eval( + [ + "nofile:2: ", + "fun/1 is not valid in typespecs. Either specify fun() or use (... -> return) instead" + ], + """ + defmodule InvalidFunType do + @type my_type :: fun(integer()) + end + """ + ) + end + + test "invalid type annotations" do + assert_warn_eval( + [ + "nofile:2: ", + "invalid type annotation. The left side of :: must be a variable, got: pid()" + ], + """ + defmodule Sample1 do + @type my_type :: (pid() :: atom) + end + """ + ) + + message = + "invalid type annotation. The left side of :: must be a variable, got: pid | ann_type. " <> + "Note \"left | right :: ann\" is the same as \"(left | right) :: ann\"" + + assert_warn_eval( + ["nofile:2: ", message], + """ + defmodule Sample2 do + @type my_type :: pid | ann_type :: atom + end + """ + ) + after + purge([Sample1, Sample2]) + end + end + + test "attribute with no use" do + assert_warn_eval( + ["nofile:2: ", "module attribute @at was set but never used"], + """ + defmodule Sample do + @at "Something" end - end) =~ "warning: unused import String" + """ + ) after - purge [Sample] + purge(Sample) end - test :unused_guard do - assert capture_err(fn -> - Code.eval_string """ - defmodule Sample1 do - def is_atom_case do - v = "bc" - case v do - _ when is_atom(v) -> :ok - _ -> :fail - end - end + test "registered attribute with no use" do + assert_warn_eval( + ["nofile:3: ", "module attribute @at was set but never used"], + """ + defmodule Sample do + Module.register_attribute(__MODULE__, :at, []) + @at "Something" end """ - end) =~ "nofile:5: warning: the guard for this clause evaluates to 'false'" + ) + after + purge(Sample) + end - assert capture_err(fn -> - Code.eval_string """ - defmodule Sample2 do - def is_binary_cond do - v = "bc" - cond do - is_binary(v) -> :bin - true -> :ok - end - end + test "typedoc with no type" do + assert_warn_eval( + ["nofile:2: ", "module attribute @typedoc was set but no type follows it"], + """ + defmodule Sample do + @typedoc "Something" end """ - end) =~ "nofile:6: warning: this clause cannot match because a previous clause at line 5 always matches" + ) after - purge [Sample1, Sample2] + purge(Sample) end - test :empty_clause do - assert capture_err(fn -> - Code.eval_string """ - defmodule Sample1 do - def hello + test "doc with no function" do + assert_warn_eval( + ["nofile:2: ", "module attribute @doc was set but no definition follows it"], + """ + defmodule Sample do + @doc "Something" end """ - end) =~ "warning: empty clause provided for nonexistent function or macro hello/0" + ) after - purge [Sample1] + purge(Sample) end - test :used_import_via_alias do - assert capture_err(fn -> - Code.eval_string """ - defmodule Sample1 do - import List, only: [flatten: 1] + test "pipe without explicit parentheses" do + assert_warn_eval( + ["nofile:2:1", "parentheses are required when piping into a function call"], + """ + [5, 6, 7, 3] + |> Enum.map_join "", &(Integer.to_string(&1)) + |> String.to_integer + """ + ) + end - defmacro generate do - List.duplicate(quote(do: flatten([1,2,3])), 100) - end + test "keywords without explicit parentheses" do + assert_warn_eval( + ["nofile:2\n", "missing parentheses for expression following \"label:\" keyword. "], + """ + quote do + IO.inspect arg, label: if true, do: "foo", else: "baz" end + """ + ) + end - defmodule Sample2 do - import Sample1 - generate + test "do+end with operator without explicit parentheses" do + assert_warn_eval( + ["nofile:3\n", "missing parentheses on expression following operator \"||\""], + """ + quote do + case do + end || raise 1, 2 end """ - end) == "" - after - purge [Sample1, Sample2] + ) end - test :clause_not_match do - assert capture_err(fn -> - Code.eval_string """ + test "variable is being expanded to function call (on_undefined_variable: warn)" do + capture_io(:stderr, fn -> + Code.put_compiler_option(:on_undefined_variable, :warn) + end) + + output = + capture_eval(""" + self defmodule Sample do - def hello, do: nil - def hello, do: nil + def my_node(), do: node end - """ - end) =~ "warning: this clause cannot match because a previous clause at line 2 always matches" + """) + + assert output =~ "variable \"self\" does not exist and is being expanded to \"self()\"" + assert output =~ "nofile:1:1" + assert output =~ "variable \"node\" does not exist and is being expanded to \"node()\"" + assert output =~ "nofile:3:22" after - purge Sample + Code.put_compiler_option(:on_undefined_variable, :raise) + purge(Sample) + end + + defmodule User do + defstruct [:name] end - test :clause_with_defaults_should_be_first do + test ":__struct__ is ignored when using structs" do assert capture_err(fn -> - Code.eval_string ~S""" - defmodule Sample do - def hello(arg), do: nil - def hello(arg \\ 0), do: nil + code = """ + assert %Kernel.WarningTest.User{__struct__: Ignored, name: "joe"} == + %Kernel.WarningTest.User{name: "joe"} + """ + + Code.eval_string(code, [], __ENV__) + end) =~ "key :__struct__ is ignored when using structs" + + assert capture_err(fn -> + code = """ + user = %Kernel.WarningTest.User{name: "meg"} + assert %Kernel.WarningTest.User{user | __struct__: Ignored, name: "joe"} == + %Kernel.WarningTest.User{__struct__: Kernel.WarningTest.User, name: "joe"} + """ + + Code.eval_string(code, [], __ENV__) + end) =~ "key :__struct__ is ignored when using structs" + end + + test "catch comes before rescue in try block" do + assert_warn_eval( + ["nofile:1:1\n", ~s("catch" should always come after "rescue" in try)], + """ + try do + :trying + catch + _ -> :caught + rescue + _ -> :error end """ - end) =~ "warning: clause with defaults should be the first clause in def hello/1" - after - purge Sample + ) end - test :unused_with_local_with_overridable do - assert capture_err(fn -> - Code.eval_string """ + test "catch comes before rescue in def" do + assert_warn_eval( + ["nofile:2:7\n", ~s("catch" should always come after "rescue" in def)], + """ defmodule Sample do - def hello, do: world - defp world, do: :ok - defoverridable [hello: 0] - def hello, do: :ok + def foo do + :trying + catch + _, _ -> :caught + rescue + _ -> :error + end end """ - end) =~ "warning: function world/0 is unused" + ) after - purge Sample + purge(Sample) end - test :used_with_local_with_reattached_overridable do - assert capture_err(fn -> - Code.eval_string """ + test "unused variable in defguard" do + assert_warn_eval( + ["nofile:2:21", "variable \"baz\" is unused"], + """ defmodule Sample do - def hello, do: world - defp world, do: :ok - defoverridable [hello: 0, world: 0] + defguard foo(bar, baz) when bar end """ - end) == "" + ) after - purge Sample + purge(Sample) end - test :undefined_module_attribute do - assert capture_err(fn -> - Code.eval_string """ + test "unused import in defguard" do + assert_warn_eval( + ["nofile:2:3", "unused import Record\n"], + """ defmodule Sample do - @foo + import Record + defguard is_record(baz) when baz end """ - end) =~ "warning: undefined module attribute @foo, please remove access to @foo or explicitly set it to nil before access" + ) after - purge Sample + purge(Sample) end - test :undefined_module_attribute_in_function do - assert capture_err(fn -> - Code.eval_string """ + test "unused private guard" do + assert_warn_eval( + ["nofile:2:13: ", "macro foo/2 is unused\n"], + """ defmodule Sample do - def hello do - @foo - end + defguardp foo(bar, baz) when bar + baz end """ - end) =~ "warning: undefined module attribute @foo, please remove access to @foo or explicitly set it to nil before access" + ) after - purge Sample + purge(Sample) end - test :undefined_module_attribute_with_file do - assert capture_err(fn -> - Code.eval_string """ + test "defguard overriding defmacro" do + assert_warn_eval( + [ + "nofile:3:12\n", + ~r"this clause( for foo/1)? cannot match because a previous clause at line 2 always matches" + ], + """ defmodule Sample do - @foo + defmacro foo(bar), do: bar == :bar + defguard foo(baz) when baz == :baz end """ - end) =~ "warning: undefined module attribute @foo, please remove access to @foo or explicitly set it to nil before access" + ) after - purge Sample + purge(Sample) end - test :in_guard_empty_list do - assert capture_err(fn -> - Code.eval_string """ + test "defmacro overriding defguard" do + assert_warn_eval( + [ + "nofile:3:12\n", + ~r"this clause( for foo/1)? cannot match because a previous clause at line 2 always matches" + ], + """ defmodule Sample do - def a(x) when x in [], do: x + defguard foo(baz) when baz == :baz + defmacro foo(bar), do: bar == :bar end """ - end) =~ "warning: the guard for this clause evaluates to 'false'" + ) after - purge Sample + purge(Sample) end - test :no_effect_operator do - assert capture_err(fn -> - Code.eval_string """ + test "deprecated GenServer super on callbacks" do + assert_warn_eval( + ["nofile:1: ", "calling super for GenServer callback handle_call/3 is deprecated"], + """ defmodule Sample do - def a(x) do - x != :foo - :ok + use GenServer + + def handle_call(a, b, c) do + super(a, b, c) end end """ - end) =~ "warning: use of operator != has no effect" + ) after - purge Sample + purge(Sample) end - test :undefined_function_for_behaviour do - assert capture_err(fn -> - Code.eval_string """ - defmodule Sample1 do - use Behaviour - defcallback foo - end + test "super is allowed on GenServer.child_spec/1" do + refute capture_eval(""" + defmodule Sample do + use GenServer - defmodule Sample2 do - @behaviour Sample1 - end - """ - end) =~ "warning: undefined behaviour function foo/0 (for behaviour Sample1)" + def child_spec(opts) do + super(opts) + end + end + """) =~ "calling super for GenServer callback child_spec/1 is deprecated" after - purge [Sample1, Sample2, Sample3] + purge(Sample) end - test :undefined_macro_for_behaviour do - assert capture_err(fn -> - Code.eval_string """ - defmodule Sample1 do - use Behaviour - defmacrocallback foo - end - - defmodule Sample2 do - @behaviour Sample1 + test "def warns if only clause is else" do + assert_warn_compile( + ["nofile:2:7\n", "\"else\" shouldn't be used as the only clause in \"def\""], + """ + defmodule Sample do + def foo do + :bar + else + _other -> :ok + end end """ - end) =~ "warning: undefined behaviour macro foo/0 (for behaviour Sample1)" + ) after - purge [Sample1, Sample2, Sample3] + purge(Sample) end - test :undefined_behavior do - assert capture_err(fn -> - Code.eval_string """ - defmodule Sample do - @behavior Hello + test "try warns if only clause is else" do + assert_warn_compile( + ["nofile:1:1\n", "\"else\" shouldn't be used as the only clause in \"try\""], + """ + try do + :ok + else + other -> other end """ - end) =~ "warning: @behavior attribute is not supported, please use @behaviour instead" - after - purge [Sample] + ) end - test :undefined_macro_for_protocol do - assert capture_err(fn -> - Code.eval_string """ - defprotocol Sample1 do - def foo(subject) - end + test "sigil w/W warns on trailing comma at macro expansion time" do + for sigil <- ~w(w W), + modifier <- ~w(a s c) do + output = + capture_err(fn -> + {:ok, ast} = + "~#{sigil}(foo, bar baz)#{modifier}" + |> Code.string_to_quoted() - defimpl Sample1, for: Atom do - end - """ - end) =~ "warning: undefined protocol function foo/1 (for protocol Sample1)" - after - purge [Sample1, Sample1.Atom] + Macro.expand(ast, __ENV__) + end) + + assert output =~ "the sigils ~w/~W do not allow trailing commas" + end end - test :overidden_def do - assert capture_err(fn -> - Code.eval_string """ + test "warnings on trailing comma on call" do + assert_warn_eval( + ["nofile:1:25\n", "trailing commas are not allowed inside function/macro call arguments"], + "Keyword.merge([], foo: 1,)" + ) + end + + test "defstruct warns with duplicate keys" do + assert_warn_eval( + ["nofile:2: Sample", "duplicate key :foo found in struct"], + """ defmodule Sample do - def foo(x, 1), do: x + 1 - def bar(), do: nil - def foo(x, 2), do: x * 2 + defstruct [:foo, :bar, foo: 1] end """ - end) =~ "nofile:4: warning: clauses for the same def should be grouped together, def foo/2 was previously defined (nofile:2)" + ) after - purge [Sample] + purge(Sample) end - test :warning_with_overridden_file do - assert capture_err(fn -> - Code.eval_string """ - defmodule Sample do - @file "sample" - def foo(x), do: :ok - end + test "deprecate nullary remote zero-arity capture with parens" do + assert capture_eval(""" + import System, only: [pid: 0] + &pid/0 + """) == "" + + assert_warn_eval( + [ + "nofile:1:1\n", + "extra parentheses on a remote function capture &System.pid()/0 have been deprecated. Please remove the parentheses: &System.pid/0" + ], """ - end) =~ "sample:3: warning: variable x is unused" - after - purge [Sample] + &System.pid()/0 + """ + ) end - test :typedoc_on_typep do - assert capture_err(fn -> - Code.eval_string """ + test "deprecate &module.fun/arity captures with complex expressions as modules" do + assert_warn_eval( + [ + "nofile:2:", + """ + expected the module in &module.fun/arity to expand to a variable or an atom, got: hd(modules) + You can either compute the module name outside of & or convert it to a regular anonymous function. + """ + ], + """ defmodule Sample do - @typedoc "Something" - @typep priv :: any - @spec foo() :: priv - def foo(), do: nil + def foo(modules), do: &hd(modules).module_info/0 end """ - end) =~ "nofile:3: warning: type priv/0 is private, @typedoc's are always discarded for private types" + ) after - purge [Sample] + purge(Sample) end - test :typedoc_with_no_type do - assert capture_err(fn -> - Code.eval_string """ + test "deprecate non-quoted variables in bitstring size modifiers" do + assert_warn_eval( + [ + "the variable \"a\" is accessed inside size(...) of a bitstring but it was defined outside of the match", + "You must precede it with the pin operator" + ], + """ + a = "foo" + <> <> _ = a + "fo" = a + """ + ) + end + + defp assert_compile_error(messages, string) do + captured = + capture_err(fn -> + assert_raise CompileError, fn -> + ast = Code.string_to_quoted!(string, columns: true) + Code.eval_quoted(ast) + end + end) + + for message <- List.wrap(messages) do + assert captured =~ message + end + end + + defp purge(list) when is_list(list) do + Enum.each(list, &purge/1) + end + + test "unused require" do + assert_warn_compile( + ["nofile:2:3", "unused require Application"], + """ defmodule Sample do - @typedoc "Something" + require Application + def a, do: nil end """ - end) =~ "nofile:1: warning: @typedoc provided but no type follows it" + ) + + assert_warn_compile( + ["nofile:1:1", "unused require Logger"], + """ + require Logger + """ + ) after - purge [Sample] + purge(Sample) end - defp purge(list) when is_list(list) do - Enum.each list, &purge/1 + test "conditional require" do + assert capture_err(fn -> + defmodule KernelTest.ConditionalRequire do + if false do + require Integer + def fun(x), do: Integer.is_odd(x) + end + end + end) == "" + after + purge(Sample) end defp purge(module) when is_atom(module) do - :code.delete module - :code.purge module + :code.purge(module) + :code.delete(module) end end diff --git a/lib/elixir/test/elixir/kernel/with_test.exs b/lib/elixir/test/elixir/kernel/with_test.exs new file mode 100644 index 00000000000..7da7b49bb9f --- /dev/null +++ b/lib/elixir/test/elixir/kernel/with_test.exs @@ -0,0 +1,176 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Kernel.WithTest do + use ExUnit.Case, async: true + + test "basic with" do + assert with({:ok, res} <- ok(41), do: res) == 41 + assert with(res <- four(), do: res + 10) == 14 + end + + test "matching with" do + assert with(_..42//_ <- 1..42, do: :ok) == :ok + assert with({:ok, res} <- error(), do: res) == :error + assert with({:ok, _} = res <- ok(42), do: elem(res, 1)) == 42 + end + + test "with guards" do + assert with(x when x < 2 <- four(), do: :ok) == 4 + assert with(x when x > 2 <- four(), do: :ok) == :ok + assert with(x when x < 2 when x == 4 <- four(), do: :ok) == :ok + end + + test "pin matching with" do + key = :ok + assert with({^key, res} <- ok(42), do: res) == 42 + end + + test "pin matching with multiple else" do + key = :error + + first_else = + with nil <- error() do + :ok + else + ^key -> :pinned + _other -> :other + end + + assert first_else == :pinned + + second_else = + with nil <- ok(42) do + :ok + else + ^key -> :pinned + _other -> :other + end + + assert second_else == :other + end + + test "two levels with" do + result = + with {:ok, n1} <- ok(11), + n2 <- 22, + do: n1 + n2 + + assert result == 33 + + result = + with n1 <- 11, + {:ok, n2} <- error(), + do: n1 + n2 + + assert result == :error + end + + test "binding inside with" do + result = + with {:ok, n1} <- ok(11), + n2 = n1 + 10, + {:ok, n3} <- ok(22), + do: n2 + n3 + + assert result == 43 + + result = + with {:ok, n1} <- ok(11), + n2 = n1 + 10, + {:ok, n3} <- error(), + do: n2 + n3 + + assert result == :error + end + + test "does not leak variables to else" do + state = 1 + + result = + with 1 <- state, + state = 2, + :ok <- error(), + do: state, + else: (_ -> state) + + assert result == 1 + assert state == 1 + end + + test "with shadowing" do + assert with( + a <- + ( + b = 1 + _ = b + 1 + ), + b <- 2, + do: a + b + ) == 3 + end + + test "with extra guards" do + var = + with %_{} = a <- struct(URI), + %_{} <- a do + :ok + end + + assert var == :ok + end + + test "errors in with" do + assert_raise RuntimeError, fn -> + with({:ok, res} <- oops(), do: res) + end + + assert_raise RuntimeError, fn -> + with({:ok, res} <- ok(42), oops(), do: res) + end + end + + test "else conditions" do + assert (with {:ok, res} <- four() do + res + else + {:error, error} -> error + res -> res + 1 + end) == 5 + + assert (with {:ok, res} <- four() do + res + else + res when res == 4 -> res + 1 + res -> res + end) == 5 + + assert with({:ok, res} <- four(), do: res, else: (_ -> :error)) == :error + end + + test "else conditions with match error" do + assert_raise WithClauseError, "no with clause matching:\n\n :error\n", fn -> + with({:ok, res} <- error(), do: res, else: ({:error, error} -> error)) + end + end + + defp four() do + 4 + end + + defp error() do + :error + end + + defp ok(num) do + {:ok, num} + end + + defp oops() do + raise("oops") + end +end diff --git a/lib/elixir/test/elixir/kernel_test.exs b/lib/elixir/test/elixir/kernel_test.exs index eee1a160d58..b4f1033fca5 100644 --- a/lib/elixir/test/elixir/kernel_test.exs +++ b/lib/elixir/test/elixir/kernel_test.exs @@ -1,15 +1,111 @@ -Code.require_file "test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("test_helper.exs", __DIR__) defmodule KernelTest do use ExUnit.Case, async: true + # Skip these doctests are they emit warnings + doctest Kernel, + except: + [===: 2, !==: 2, and: 2, or: 2] ++ + [is_exception: 1, is_exception: 2, is_nil: 1, is_struct: 1, is_non_struct_map: 1] + + def id(arg), do: arg + def id(arg1, arg2), do: {arg1, arg2} + def empty_map, do: %{} + + defp purge(module) do + :code.purge(module) + :code.delete(module) + end + + defp assert_eval_raise(error, msg, string) do + assert_raise error, msg, fn -> + Code.eval_string(string) + end + end + + test "op ambiguity" do + max = 1 + assert max == 1 + assert max(1, 2) == 2 + end + + describe "=/2" do + test "can be reassigned" do + var = 1 + id(var) + var = 2 + assert var == 2 + end + + test "can be reassigned inside a list" do + _ = [var = 1, 2, 3] + id(var) + _ = [var = 2, 3, 4] + assert var == 2 + end + + test "can be reassigned inside a keyword list" do + _ = [a: var = 1, b: 2] + id(var) + _ = [b: var = 2, c: 3] + assert var == 2 + end + + test "can be reassigned inside a call" do + id(var = 1) + id(var) + id(var = 2) + assert var == 2 + end + + test "can be reassigned inside a multi-argument call" do + id(:arg, var = 1) + id(:arg, var) + id(:arg, var = 2) + assert var == 2 + + id(:arg, a: 1, b: var = 2) + id(:arg, var) + id(:arg, b: 2, c: var = 3) + assert var == 3 + end + + test "++/2 works in matches" do + [1, 2] ++ var = [1, 2] + assert var == [] + + [1, 2] ++ var = [1, 2, 3] + assert var == [3] + + ~c"ab" ++ var = ~c"abc" + assert var == ~c"c" + + [:a, :b] ++ var = [:a, :b, :c] + assert var == [:c] + end + end + test "=~/2" do - assert ("abcd" =~ ~r/c(d)/) == true - assert ("abcd" =~ ~r/e/) == false + assert "abcd" =~ ~r/c(d)/ == true + assert "abcd" =~ ~r/e/ == false string = "^ab+cd*$" - assert (string =~ "ab+") == true - assert (string =~ "bb") == false + assert string =~ "ab+" == true + assert string =~ "bb" == false + + assert "abcd" =~ ~r// == true + assert "abcd" =~ "" == true + + assert "" =~ ~r// == true + assert "" =~ "" == true + + assert "" =~ "abcd" == false + assert "" =~ ~r/abcd/ == false assert_raise FunctionClauseError, "no function clause matching in Kernel.=~/2", fn -> 1234 =~ "hello" @@ -18,6 +114,34 @@ defmodule KernelTest do assert_raise FunctionClauseError, "no function clause matching in Kernel.=~/2", fn -> 1234 =~ ~r"hello" end + + assert_raise FunctionClauseError, "no function clause matching in Kernel.=~/2", fn -> + 1234 =~ ~r"hello" + end + + assert_raise FunctionClauseError, "no function clause matching in Kernel.=~/2", fn -> + ~r"hello" =~ "hello" + end + + assert_raise FunctionClauseError, "no function clause matching in Kernel.=~/2", fn -> + ~r"hello" =~ ~r"hello" + end + + assert_raise FunctionClauseError, "no function clause matching in Kernel.=~/2", fn -> + :abcd =~ ~r// + end + + assert_raise FunctionClauseError, "no function clause matching in Kernel.=~/2", fn -> + :abcd =~ "" + end + + assert_raise FunctionClauseError, "no function clause matching in Regex.match?/2", fn -> + "abcd" =~ nil + end + + assert_raise FunctionClauseError, "no function clause matching in Regex.match?/2", fn -> + "abcd" =~ :abcd + end end test "^" do @@ -29,392 +153,1101 @@ defmodule KernelTest do end end + # Note we use `==` in assertions so `assert` does not rewrite `match?/2`. test "match?/2" do - assert match?(_, List.first(1)) == true - assert binding([:x]) == [] - a = List.first([0]) assert match?(b when b > a, 1) == true - assert binding([:b]) == [] + assert binding() == [a: 0] assert match?(b when b > a, -1) == false - assert binding([:b]) == [] + assert binding() == [a: 0] + + # Does not warn on underscored variables + assert match?(_unused, a) == true end - test "nil?/1" do - assert nil?(nil) == true - assert nil?(0) == false - assert nil?(false) == false + def exported?, do: not_exported?() + defp not_exported?, do: true + + test "function_exported?/3" do + assert function_exported?(__MODULE__, :exported?, 0) + refute function_exported?(__MODULE__, :not_exported?, 0) end - test "in/2" do - assert 2 in [1, 2, 3] - assert 2 in 1..3 - refute 4 in [1, 2, 3] - refute 4 in 1..3 + test "macro_exported?/3" do + assert macro_exported?(Kernel, :in, 2) == true + assert macro_exported?(Kernel, :def, 1) == true + assert macro_exported?(Kernel, :def, 2) == true + assert macro_exported?(Kernel, :def, 3) == false + assert macro_exported?(Kernel, :no_such_macro, 2) == false + assert macro_exported?(:erlang, :abs, 1) == false + end - list = [1, 2, 3] - assert 2 in list - refute 4 in list + test "apply/3 and apply/2" do + assert apply(Enum, :reverse, [[1 | [2, 3]]]) == [3, 2, 1] + assert apply(fn x -> x * 2 end, [2]) == 4 end - @at_list [4,5] - @at_range 6..8 - def fun_in(x) when x in [0], do: :list - def fun_in(x) when x in 1..3, do: :range - def fun_in(x) when x in @at_list, do: :at_list - def fun_in(x) when x in @at_range, do: :at_range + test "binding/0 and binding/1" do + x = 1 + assert binding() == [x: 1] + + x = 2 + assert binding() == [x: 2] - test "in/2 in function guard" do - assert fun_in(0) == :list - assert fun_in(2) == :range - assert fun_in(5) == :at_list - assert fun_in(8) == :at_range + y = 3 + assert binding() == [x: 2, y: 3] + + var!(x, :foo) = 4 + assert binding() == [x: 2, y: 3] + assert binding(:foo) == [x: 4] + + # No warnings + _x = 1 + assert binding() == [_x: 1, x: 2, y: 3] end - defmacrop case_in(x, y) do - quote do - case 0 do - _ when unquote(x) in unquote(y) -> true - _ -> false - end - end + defmodule User do + assert is_map(defstruct name: "john") + # Ensure we keep the line information around. + # It is important for debugging tools, ExDoc, etc. + {:v1, :def, anno, _clauses} = Module.get_definition(__MODULE__, {:__struct__, 1}) + anno[:line] == __ENV__.line - 4 end - test "in/2 in case guard" do - assert case_in(1, [1,2,3]) == true - assert case_in(1, 1..3) == true - assert case_in(2, 1..3) == true - assert case_in(3, 1..3) == true - assert case_in(-3, -1..-3) == true + test "struct/1 and struct/2" do + assert struct(User) == %User{name: "john"} + + user = struct(User, name: "meg") + assert user == %User{name: "meg"} + assert struct(user, %{name: "meg"}) == user + + assert struct(user, unknown: "key") == user + assert struct(user, %{name: "john"}) == %User{name: "john"} + assert struct(user, name: "other", __struct__: Post) == %User{name: "other"} end - test "paren as nil" do - assert nil?(()) == true - assert ((); ();) == nil - assert [ 1, (), 3 ] == [1, nil, 3 ] - assert [do: ()] == [do: nil] - assert {1, (), 3} == {1, nil, 3} - assert (Kernel.&& nil, ()) == nil - assert (Kernel.&& nil, ()) == nil - assert (() && ()) == nil - assert (if(() && ()) do - :ok - else - :error - end) == :error + test "struct!/1 and struct!/2" do + assert struct!(User) == %User{name: "john"} + + user = struct!(User, name: "meg") + assert user == %User{name: "meg"} + + assert_raise KeyError, fn -> + struct!(user, unknown: "key") + end + + assert struct!(user, %{name: "john"}) == %User{name: "john"} + assert struct!(user, name: "other", __struct__: Post) == %User{name: "other"} end - test "__info__(:macros)" do - assert {:in, 2} in Kernel.__info__(:macros) + test "if/2 with invalid keys" do + error_message = + "invalid or duplicate keys for if, only \"do\" and an optional \"else\" are permitted" + + assert_raise ArgumentError, error_message, fn -> + Code.eval_string("if true, foo: 7") + end + + assert_raise ArgumentError, error_message, fn -> + Code.eval_string("if true, do: 6, boo: 7") + end + + assert_raise ArgumentError, error_message, fn -> + Code.eval_string("if true, do: 7, do: 6") + end + + assert_raise ArgumentError, error_message, fn -> + Code.eval_string("if true, do: 8, else: 7, else: 6") + end + + assert_raise ArgumentError, error_message, fn -> + Code.eval_string("if true, else: 6") + end + + assert_raise ArgumentError, error_message, fn -> + Code.eval_string("if true, []") + end end - test "__info__(:functions)" do - assert not ({:__info__, 1} in Kernel.__info__(:functions)) + test "unless/2 with invalid keys" do + error_message = + "invalid or duplicate keys for unless, only \"do\" " <> + "and an optional \"else\" are permitted" + + assert_raise ArgumentError, error_message, fn -> + Code.eval_string("unless true, foo: 7") + end + + assert_raise ArgumentError, error_message, fn -> + Code.eval_string("unless true, do: 6, boo: 7") + end + + assert_raise ArgumentError, error_message, fn -> + Code.eval_string("unless true, do: 7, do: 6") + end + + assert_raise ArgumentError, error_message, fn -> + Code.eval_string("unless true, do: 8, else: 7, else: 6") + end + + assert_raise ArgumentError, error_message, fn -> + Code.eval_string("unless true, else: 6") + end + + assert_raise ArgumentError, error_message, fn -> + Code.eval_string("unless true, []") + end end - test "macro_exported?/3" do - assert macro_exported?(Kernel, :in, 2) == true - assert macro_exported?(Kernel, :def, 1) == true - assert macro_exported?(Kernel, :def, 2) == true - assert macro_exported?(Kernel, :def, 3) == false - assert macro_exported?(Kernel, :no_such_macro, 2) == false + test "and/2" do + assert (true and false) == false + assert (true and true) == true + assert (true and 0) == 0 + assert (false and false) == false + assert (false and true) == false + assert (false and 0) == false + assert (false and raise("oops")) == false + assert ((x = Process.get(:unused, true)) and not x) == false + assert_raise BadBooleanError, fn -> Process.get(:unused, 0) and 1 end end - test "apply/3 and apply/2" do - assert apply(Enum, :reverse, [[1|[2, 3]]]) == [3, 2, 1] - assert apply(fn x -> x * 2 end, [2]) == 4 + test "or/2" do + assert (true or false) == true + assert (true or true) == true + assert (true or 0) == true + assert (true or raise("foo")) == true + assert (false or false) == false + assert (false or true) == true + assert (false or 0) == 0 + assert ((x = Process.get(:unused, false)) or not x) == true + assert_raise BadBooleanError, fn -> Process.get(:unused, 0) or 1 end end - test "binding/0, binding/1 and binding/2" do - x = 1 - assert binding == [x: 1] - assert binding([:x, :y]) == [x: 1] - assert binding([:x, :y], nil) == [x: 1] + defp delegate_is_struct(arg), do: is_struct(arg) - x = 2 - assert binding == [x: 2] + defp guarded_is_struct(arg) when is_struct(arg), do: true + defp guarded_is_struct(_arg), do: false - y = 3 - assert binding == [x: 2, y: 3] + defp struct_or_map?(arg) when is_struct(arg) or is_map(arg), do: true + defp struct_or_map?(_arg), do: false - var!(x, :foo) = 2 - assert binding(:foo) == [x: 2] - assert binding([:x, :y], :foo) == [x: 2] + test "is_struct/1" do + assert delegate_is_struct(%{}) == false + assert delegate_is_struct([]) == false + assert delegate_is_struct(%Macro.Env{}) == true + assert delegate_is_struct(%{__struct__: "foo"}) == false + assert guarded_is_struct(%Macro.Env{}) == true + assert guarded_is_struct(%{__struct__: "foo"}) == false + assert guarded_is_struct([]) == false + assert guarded_is_struct(%{}) == false end - defmodule User do - defstruct name: "jose" + test "is_struct/1 and other match works" do + assert struct_or_map?(%Macro.Env{}) == true + assert struct_or_map?(%{}) == true + assert struct_or_map?(10) == false end - defmodule UserTuple do - def __struct__({ UserTuple, :ok }) do - %User{} + defp delegate_is_struct(arg, name), do: is_struct(arg, name) + + defp guarded_is_struct(arg, name) when is_struct(arg, name), do: true + defp guarded_is_struct(_arg, _name), do: false + + defp struct_or_map?(arg, name) when is_struct(arg, name) or is_map(arg), do: true + defp struct_or_map?(_arg, _name), do: false + + defp not_atom(), do: "not atom" + + test "is_struct/2" do + assert delegate_is_struct(%{}, Macro.Env) == false + assert delegate_is_struct([], Macro.Env) == false + assert delegate_is_struct(%Macro.Env{}, Macro.Env) == true + assert delegate_is_struct(%Macro.Env{}, URI) == false + assert guarded_is_struct(%Macro.Env{}, Macro.Env) == true + assert guarded_is_struct(%Macro.Env{}, URI) == false + assert guarded_is_struct(%{__struct__: "foo"}, "foo") == false + assert guarded_is_struct(%{__struct__: "foo"}, Macro.Env) == false + assert guarded_is_struct([], Macro.Env) == false + assert guarded_is_struct(%{}, Macro.Env) == false + + assert_raise ArgumentError, "argument error", fn -> + is_struct(%{}, not_atom()) end end - test "struct/1 and struct/2" do - assert struct(User) == %User{name: "jose"} + test "is_struct/2 and other match works" do + assert struct_or_map?(%{}, "foo") == false + assert struct_or_map?(%{}, Macro.Env) == true + assert struct_or_map?(%Macro.Env{}, Macro.Env) == true + end - user = struct(User, name: "eric") - assert user == %User{name: "eric"} + defp delegate_is_non_struct_map(arg), do: is_non_struct_map(arg) - assert struct(user, unknown: "key") == user - assert struct(user, %{name: "jose"}) == %User{name: "jose"} - assert struct(user, name: "other", __struct__: Post) == %User{name: "other"} + defp guarded_is_non_struct_map(arg) when is_non_struct_map(arg), do: true + defp guarded_is_non_struct_map(_arg), do: false + + defp non_struct_map_or_struct?(arg) when is_non_struct_map(arg) or is_struct(arg), do: true + defp non_struct_map_or_struct?(_arg), do: false + + test "is_non_struct_map/1" do + assert delegate_is_non_struct_map(%{}) == true + assert delegate_is_non_struct_map([]) == false + assert delegate_is_non_struct_map(%Macro.Env{}) == false + assert delegate_is_non_struct_map(%{__struct__: "foo"}) == true + assert guarded_is_non_struct_map(%Macro.Env{}) == false + assert guarded_is_non_struct_map(%{__struct__: "foo"}) == true + assert guarded_is_non_struct_map([]) == false + assert guarded_is_non_struct_map(%{}) == true + end - user_tuple = {UserTuple, :ok} - assert struct(user_tuple, name: "eric") == %User{name: "eric"} + test "is_non_struct_map/1 and other match works" do + assert non_struct_map_or_struct?(%Macro.Env{}) == true + assert non_struct_map_or_struct?(%{}) == true + assert non_struct_map_or_struct?(10) == false end - defdelegate my_flatten(list), to: List, as: :flatten - defdelegate [map(callback, list)], to: :lists, append_first: true + defp delegate_is_exception(arg), do: is_exception(arg) - dynamic = :dynamic_flatten - defdelegate unquote(dynamic)(list), to: List, as: :flatten + defp guarded_is_exception(arg) when is_exception(arg), do: true + defp guarded_is_exception(_arg), do: false - test "defdelefate/2" do - assert my_flatten([[1]]) == [1] + defp exception_or_map?(arg) when is_exception(arg) or is_map(arg), do: true + defp exception_or_map?(_arg), do: false + + test "is_exception/1" do + assert delegate_is_exception(%{}) == false + assert delegate_is_exception([]) == false + assert delegate_is_exception(%RuntimeError{}) == true + assert delegate_is_exception(%{__exception__: "foo"}) == false + assert guarded_is_exception(%RuntimeError{}) == true + assert guarded_is_exception(%{__exception__: "foo"}) == false + assert guarded_is_exception([]) == false + assert guarded_is_exception(%{}) == false end - test "defdelegate/2 with :append_first" do - assert map([1], fn(x) -> x + 1 end) == [2] + test "is_exception/1 and other match works" do + assert exception_or_map?(%RuntimeError{}) == true + assert exception_or_map?(%{}) == true + assert exception_or_map?(10) == false end - test "defdelegate/2 with unquote" do - assert dynamic_flatten([[1]]) == [1] + defp delegate_is_exception(arg, name), do: is_exception(arg, name) + + defp guarded_is_exception(arg, name) when is_exception(arg, name), do: true + defp guarded_is_exception(_arg, _name), do: false + + defp exception_or_map?(arg, name) when is_exception(arg, name) or is_map(arg), do: true + defp exception_or_map?(_arg, _name), do: false + + test "is_exception/2" do + assert delegate_is_exception(%{}, RuntimeError) == false + assert delegate_is_exception([], RuntimeError) == false + assert delegate_is_exception(%RuntimeError{}, RuntimeError) == true + assert delegate_is_exception(%RuntimeError{}, Macro.Env) == false + assert guarded_is_exception(%RuntimeError{}, RuntimeError) == true + assert guarded_is_exception(%RuntimeError{}, Macro.Env) == false + assert guarded_is_exception(%{__exception__: "foo"}, "foo") == false + assert guarded_is_exception(%{__exception__: "foo"}, RuntimeError) == false + assert guarded_is_exception([], RuntimeError) == false + assert guarded_is_exception(%{}, RuntimeError) == false + + assert_raise ArgumentError, "argument error", fn -> + delegate_is_exception(%{}, not_atom()) + end end - test "get_in/2" do - users = %{"josé" => %{age: 27}, "eric" => %{age: 23}} - assert get_in(users, ["josé", :age]) == 27 - assert get_in(users, ["dave", :age]) == nil - assert get_in(nil, ["josé", :age]) == nil + test "is_exception/2 and other match works" do + assert exception_or_map?(%{}, "foo") == false + assert exception_or_map?(%{}, RuntimeError) == true + assert exception_or_map?(%RuntimeError{}, RuntimeError) == true + end - assert_raise FunctionClauseError, fn -> - get_in(users, []) + test "then/2" do + assert 1 |> then(fn x -> x * 2 end) == 2 + end + + test "if/2 boolean optimization does not leak variables during expansion" do + if false do + :ok + else + assert Macro.Env.vars(__ENV__) == [] + end + end + + describe ".." do + test "returns 0..-1//1" do + assert (..) == 0..-1//1 end end - test "put_in/3" do - users = %{"josé" => %{age: 27}, "eric" => %{age: 23}} + describe "in/2" do + # This test may take a long time on machine with low resources + @tag timeout: 120_000 + test "too large list in guards" do + defmodule TooLargeList do + @list Enum.map(1..1024, & &1) + defguard is_value(value) when value in @list + end + end + + test "with literals on right side" do + assert 2 in [1, 2, 3] + assert 2 in 1..3 + refute 4 in [1, 2, 3] + refute 4 in 1..3 + refute 2 in [] + refute false in [] + refute true in [] + end - assert put_in(nil, ["josé", :age], 28) == - %{"josé" => %{age: 28}} + test "with expressions on right side" do + list = [1, 2, 3] + empty_list = [] + assert 2 in list + refute 4 in list - assert put_in(users, ["josé", :age], 28) == - %{"josé" => %{age: 28}, "eric" => %{age: 23}} + refute 4 in empty_list + refute false in empty_list + refute true in empty_list - assert put_in(users, ["dave", :age], 19) == - %{"josé" => %{age: 27}, "eric" => %{age: 23}, "dave" => %{age: 19}} + assert 2 in [1 | [2, 3]] + assert 3 in [1 | list] - assert_raise FunctionClauseError, fn -> - put_in(users, [], %{}) + some_call = & &1 + refute :x in [1, 2 | some_call.([3, 4])] + assert :x in [1, 2 | some_call.([3, :x])] + + assert_raise ArgumentError, fn -> + :x in [1, 2 | some_call.({3, 4})] + end end - end - test "put_in/2" do - users = %{"josé" => %{age: 27}, "eric" => %{age: 23}} + @at_list1 [4, 5] + @at_range 6..8 + @at_list2 [13, 14] + def fun_in(x) when x in [0], do: :list + def fun_in(x) when x in 1..3, do: :range + def fun_in(x) when x in @at_list1, do: :at_list + def fun_in(x) when x in @at_range, do: :at_range + def fun_in(x) when x in [9 | [10, 11]], do: :list_cons + def fun_in(x) when x in [12 | @at_list2], do: :list_cons_at + def fun_in(x) when x in 21..15//1, do: raise("oops positive") + def fun_in(x) when x in 15..21//-1, do: raise("oops negative") + def fun_in(x) when x in 15..21//2, do: :range_step_2 + def fun_in(x) when x in 15..21//1, do: :range_step_1 + def fun_in(_), do: :none + + test "in function guard" do + assert fun_in(0) == :list + assert fun_in(1) == :range + assert fun_in(2) == :range + assert fun_in(3) == :range + assert fun_in(5) == :at_list + assert fun_in(6) == :at_range + assert fun_in(7) == :at_range + assert fun_in(8) == :at_range + assert fun_in(9) == :list_cons + assert fun_in(10) == :list_cons + assert fun_in(11) == :list_cons + assert fun_in(12) == :list_cons_at + assert fun_in(13) == :list_cons_at + assert fun_in(14) == :list_cons_at + assert fun_in(15) == :range_step_2 + assert fun_in(16) == :range_step_1 + assert fun_in(17) == :range_step_2 + assert fun_in(22) == :none + + assert fun_in(0.0) == :none + assert fun_in(1.0) == :none + assert fun_in(2.0) == :none + assert fun_in(3.0) == :none + assert fun_in(6.0) == :none + assert fun_in(7.0) == :none + assert fun_in(8.0) == :none + assert fun_in(9.0) == :none + assert fun_in(10.0) == :none + assert fun_in(11.0) == :none + assert fun_in(12.0) == :none + assert fun_in(13.0) == :none + assert fun_in(14.0) == :none + assert fun_in(15.0) == :none + assert fun_in(16.0) == :none + assert fun_in(17.0) == :none + end - assert put_in(nil["josé"][:age], 28) == - %{"josé" => %{age: 28}} + def dynamic_step_in(x, y, z, w) when x in y..z//w, do: true + def dynamic_step_in(_x, _y, _z, _w), do: false - assert put_in(users["josé"][:age], 28) == - %{"josé" => %{age: 28}, "eric" => %{age: 23}} + test "in dynamic range with step function guard" do + assert dynamic_step_in(1, 1, 3, 1) + assert dynamic_step_in(2, 1, 3, 1) + assert dynamic_step_in(3, 1, 3, 1) - assert put_in(users["dave"][:age], 19) == - %{"josé" => %{age: 27}, "eric" => %{age: 23}, "dave" => %{age: 19}} + refute dynamic_step_in(1, 1, 3, -1) + refute dynamic_step_in(2, 1, 3, -1) + refute dynamic_step_in(3, 1, 3, -1) + assert dynamic_step_in(1, 3, 1, -1) + assert dynamic_step_in(2, 3, 1, -1) + assert dynamic_step_in(3, 3, 1, -1) - assert put_in(users["josé"].age, 28) == - %{"josé" => %{age: 28}, "eric" => %{age: 23}} + refute dynamic_step_in(1, 3, 1, 1) + refute dynamic_step_in(2, 3, 1, 1) + refute dynamic_step_in(3, 3, 1, 1) - assert_raise ArgumentError, fn -> - put_in(users["dave"].age, 19) + assert dynamic_step_in(1, 1, 3, 2) + refute dynamic_step_in(2, 1, 3, 2) + assert dynamic_step_in(3, 1, 3, 2) + assert dynamic_step_in(3, 1, 4, 2) + refute dynamic_step_in(4, 1, 4, 2) end - assert_raise KeyError, fn -> - put_in(users["eric"].unknown, "value") + defmacrop case_in(x, y) do + quote do + case 0 do + _ when unquote(x) in unquote(y) -> true + _ -> false + end + end end - end - test "update_in/3" do - users = %{"josé" => %{age: 27}, "eric" => %{age: 23}} + test "in case guard" do + assert case_in(1, [1, 2, 3]) == true + assert case_in(1, 1..3) == true + assert case_in(2, 1..3) == true + assert case_in(3, 1..3) == true + assert case_in(-3, -1..-3//-1) == true + end - assert update_in(nil, ["josé", :age], fn nil -> 28 end) == - %{"josé" => %{age: 28}} + def map_dot(map) when map.field, do: true + def map_dot(_other), do: false - assert update_in(users, ["josé", :age], &(&1 + 1)) == - %{"josé" => %{age: 28}, "eric" => %{age: 23}} + test "map dot guard" do + refute map_dot(:foo) + refute map_dot(%{}) + refute map_dot(%{field: false}) + assert map_dot(%{field: true}) + end - assert update_in(users, ["dave", :age], fn nil -> 19 end) == - %{"josé" => %{age: 27}, "eric" => %{age: 23}, "dave" => %{age: 19}} + test "performs all side-effects" do + assert 1 in [1, send(self(), 2)] + assert_received 2 - assert_raise FunctionClauseError, fn -> - update_in(users, [], fn _ -> %{} end) + assert 1 in [1 | send(self(), [2])] + assert_received [2] + + assert 2 in [1 | send(self(), [2])] + assert_received [2] + end + + test "has proper evaluation order" do + a = 1 + assert 1 in [a = 2, a] + # silence unused var warning + _ = a + end + + test "in module body" do + defmodule InSample do + @foo [:a, :b] + true = :a in @foo + end + after + purge(InSample) end - end - test "update_in/2" do - users = %{"josé" => %{age: 27}, "eric" => %{age: 23}} + test "inside and/2" do + response = %{code: 200} - assert update_in(nil["josé"][:age], fn nil -> 28 end) == - %{"josé" => %{age: 28}} + if is_map(response) and response.code in 200..299 do + :pass + end - assert update_in(users["josé"][:age], &(&1 + 1)) == - %{"josé" => %{age: 28}, "eric" => %{age: 23}} + # This module definition copies internal variable + # defined during in/2 expansion. + Module.create(InVarCopy, nil, __ENV__) + purge(InVarCopy) + end - assert update_in(users["dave"][:age], fn nil -> 19 end) == - %{"josé" => %{age: 27}, "eric" => %{age: 23}, "dave" => %{age: 19}} + test "with a non-literal non-escaped compile-time range in guards" do + message = ~r"found unescaped value on the right side of in/2: 1..3" - assert update_in(users["josé"].age, &(&1 + 1)) == - %{"josé" => %{age: 28}, "eric" => %{age: 23}} + assert_eval_raise(ArgumentError, message, """ + defmodule InErrors do + range = 1..3 + def foo(x) when x in unquote(range), do: :ok + end + """) + end + + test "with a non-compile-time range in guards" do + message = ~r/invalid right argument for operator "in", .* got: :hello/ - assert_raise ArgumentError, fn -> - update_in(users["dave"].age, &(&1 + 1)) + assert_eval_raise(ArgumentError, message, """ + defmodule InErrors do + def foo(x) when x in :hello, do: :ok + end + """) end - assert_raise KeyError, fn -> - put_in(users["eric"].unknown, &(&1 + 1)) + test "with a non-compile-time list cons in guards" do + message = ~r/invalid right argument for operator "in", .* got: \[1 | list\(\)\]/ + + assert_eval_raise(ArgumentError, message, """ + defmodule InErrors do + def list, do: [1] + def foo(x) when x in [1 | list()], do: :ok + end + """) end - end - test "get_and_update_in/3" do - users = %{"josé" => %{age: 27}, "eric" => %{age: 23}} + test "with a compile-time non-list in tail in guards" do + message = ~r/invalid right argument for operator "in", .* got: \[1 | 1..3\]/ - assert get_and_update_in(nil, ["josé", :age], fn nil -> {:ok, 28} end) == - {:ok, %{"josé" => %{age: 28}}} + assert_eval_raise(ArgumentError, message, """ + defmodule InErrors do + def foo(x) when x in [1 | 1..3], do: :ok + end + """) + end - assert get_and_update_in(users, ["josé", :age], &{&1, &1 + 1}) == - {27, %{"josé" => %{age: 28}, "eric" => %{age: 23}}} + test "with a non-integer range" do + message = "ranges (first..last) expect both sides to be integers, got: 0..5.0" - assert_raise FunctionClauseError, fn -> - update_in(users, [], fn _ -> %{} end) + assert_raise ArgumentError, message, fn -> + last = 5.0 + 1 in 0..last + end end - end - test "get_and_update_in/2" do - users = %{"josé" => %{age: 27}, "eric" => %{age: 23}} + test "hoists variables and keeps order" do + # Ranges + result = expand_to_string(quote(do: rand() in 1..2)) + assert result =~ "var = rand()" + + assert result =~ """ + :erlang.andalso( + :erlang.is_integer(var), + :erlang.andalso(:erlang.>=(var, 1), :erlang.\"=<\"(var, 2)) + )\ + """ - assert get_and_update_in(nil["josé"][:age], fn nil -> {:ok, 28} end) == - {:ok, %{"josé" => %{age: 28}}} + # Empty list + assert expand_to_string(quote(do: :x in [])) =~ "_ = :x\nfalse" + assert expand_to_string(quote(do: :x in []), :guard) == "false" - assert get_and_update_in(users["josé"].age, &{&1, &1 + 1}) == - {27, %{"josé" => %{age: 28}, "eric" => %{age: 23}}} + # Lists + result = expand_to_string(quote(do: rand() in [1, 2])) + assert result =~ "var = rand()" + assert result =~ ":erlang.orelse(:erlang.\"=:=\"(var, 1), :erlang.\"=:=\"(var, 2))" - assert_raise ArgumentError, fn -> - get_and_update_in(users["dave"].age, &{&1, &1 + 1}) + result = expand_to_string(quote(do: rand() in [1 | [2]])) + assert result =~ ":lists.member(rand(), [1 | [2]]" + + result = expand_to_string(quote(do: rand() in [1 | some_call()])) + assert result =~ ":lists.member(rand(), [1 | some_call()]" end - assert_raise KeyError, fn -> - get_and_update_in(users["eric"].unknown, &{&1, &1 + 1}) + test "is optimized" do + assert expand_to_string(quote(do: foo in [])) == + "_ = foo\nfalse" + + assert expand_to_string(quote(do: foo in [1, 2, 3])) == """ + :erlang.orelse( + :erlang.orelse(:erlang.\"=:=\"(foo, 1), :erlang.\"=:=\"(foo, 2)), + :erlang.\"=:=\"(foo, 3) + )\ + """ + + assert expand_to_string(quote(do: foo in 0..1)) == """ + :erlang.andalso( + :erlang.is_integer(foo), + :erlang.andalso(:erlang.>=(foo, 0), :erlang.\"=<\"(foo, 1)) + )\ + """ + + assert expand_to_string(quote(do: foo in -1..0)) == """ + :erlang.andalso( + :erlang.is_integer(foo), + :erlang.andalso(:erlang.>=(foo, -1), :erlang.\"=<\"(foo, 0)) + )\ + """ + + assert expand_to_string(quote(do: foo in 1..1)) == + ":erlang.\"=:=\"(foo, 1)" + + assert expand_to_string(quote(do: 2 in 1..3)) == + ":erlang.andalso(:erlang.is_integer(2), :erlang.andalso(:erlang.>=(2, 1), :erlang.\"=<\"(2, 3)))" + end + + defp expand_to_string(ast, environment_or_context \\ __ENV__) + + defp expand_to_string(ast, context) when is_atom(context) do + expand_to_string(ast, %{__ENV__ | context: context}) + end + + defp expand_to_string(ast, environment) do + ast + |> Macro.prewalk(&Macro.expand(&1, environment)) + |> Macro.to_string() end end - test "paths" do - map = empty_map() + describe "__info__" do + test ":macros" do + assert {:in, 2} in Kernel.__info__(:macros) + end - assert put_in(map[:foo], "bar") == %{foo: "bar"} - assert put_in(empty_map()[:foo], "bar") == %{foo: "bar"} - assert put_in(KernelTest.empty_map()[:foo], "bar") == %{foo: "bar"} - assert put_in(__MODULE__.empty_map()[:foo], "bar") == %{foo: "bar"} + test ":functions" do + refute {:__info__, 1} in Kernel.__info__(:functions) + end - assert_raise ArgumentError, ~r"access at least one field,", fn -> - Code.eval_quoted(quote(do: put_in(map, "bar")), []) + test ":struct" do + assert Kernel.__info__(:struct) == nil + assert [%{field: :scheme, default: nil} | _] = URI.__info__(:struct) end - assert_raise ArgumentError, ~r"must start with a variable, local or remote call", fn -> - Code.eval_quoted(quote(do: put_in(map.foo(1, 2)[:bar], "baz")), []) + test "others" do + assert Kernel.__info__(:module) == Kernel + assert is_list(Kernel.__info__(:compile)) + assert is_list(Kernel.__info__(:attributes)) end end - def empty_map, do: %{} + describe "@" do + test "setting attribute with do-block" do + exception = + catch_error( + defmodule UpcaseAttrSample do + @foo quote do + :ok + end + end + ) + + assert exception.message =~ "expected 0 or 1 argument for @foo, got 2" + assert exception.message =~ "You probably want to wrap the argument value in parentheses" + end - defmodule PipelineOp do - use ExUnit.Case, async: true + test "setting attribute with uppercase" do + message = "module attributes set via @ cannot start with an uppercase letter" - test "simple" do - assert [1, [2], 3] |> List.flatten == [1, 2, 3] + assert_raise ArgumentError, message, fn -> + defmodule UpcaseAttrSample do + @Upper + end + end end - test "nested pipelines" do - assert [1, [2], 3] |> List.flatten |> Enum.map(&(&1 * 2)) == [2, 4, 6] + test "matching attribute" do + assert_raise ArgumentError, ~r"invalid usage of module attributes", fn -> + defmodule MatchAttributeInModule do + @foo = 42 + end + end + + assert_raise ArgumentError, ~r"invalid usage of module attributes", fn -> + defmodule MatchAttributeInModule do + @foo 16 + <<_::@foo>> = "ab" + end + end + + assert_raise ArgumentError, ~r"invalid usage of module attributes", fn -> + defmodule MatchAttributeInModule do + @foo 16 + <<_::size(@foo)>> = "ab" + end + end end + end - test "local call" do - assert [1, [2], 3] |> List.flatten |> local == [2, 4, 6] + describe "defdelegate" do + defdelegate my_flatten(list), to: List, as: :flatten + + dynamic = :dynamic_flatten + defdelegate unquote(dynamic)(list), to: List, as: :flatten + + test "dispatches to delegated functions" do + assert my_flatten([[1]]) == [1] end - test "pipeline with capture" do - assert Enum.map([1, 2, 3], &(&1 |> twice |> twice)) == [4, 8, 12] + test "with unquote" do + assert dynamic_flatten([[1]]) == [1] end - test "non-call" do - assert 1 |> (&(&1*2)).() == 2 - assert [1] |> (&hd(&1)).() == 1 + test "raises with non-variable arguments" do + assert_raise ArgumentError, + "guards are not allowed in defdelegate/2, got: when is_list(term) or is_binary(term)", + fn -> + string = """ + defmodule IntDelegateWithGuards do + defdelegate foo(term) when is_list(term) or is_binary(term), to: List + end + """ + + Code.eval_string(string, [], __ENV__) + end + + msg = "defdelegate/2 only accepts function parameters, got: 1" + + assert_raise ArgumentError, msg, fn -> + string = """ + defmodule IntDelegate do + defdelegate foo(1), to: List + end + """ + + Code.eval_string(string, [], __ENV__) + end - import CompileAssertion - assert_compile_fail ArgumentError, "cannot pipe 1 into 2", "1 |> 2" + assert_raise ArgumentError, msg, fn -> + string = """ + defmodule IntOptionDelegate do + defdelegate foo(1 \\\\ 1), to: List + end + """ + + Code.eval_string(string, [], __ENV__) + end end - defp twice(a), do: a * 2 + test "raises when :to targeting the delegating module is given without the :as option" do + assert_raise ArgumentError, + ~r/defdelegate function is calling itself, which will lead to an infinite loop. You should either change the value of the :to option or specify the :as option/, + fn -> + defmodule ImplAttributes do + defdelegate foo(), to: __MODULE__ + end + end + end - defp local(list) do - Enum.map(list, &(&1 * 2)) + defdelegate my_reverse(list \\ []), to: :lists, as: :reverse + defdelegate my_get(map \\ %{}, key, default \\ ""), to: Map, as: :get + + test "accepts variable with optional arguments" do + assert my_reverse() == [] + assert my_reverse([1, 2, 3]) == [3, 2, 1] + + assert my_get("foo") == "" + assert my_get(%{}, "foo") == "" + assert my_get(%{"foo" => "bar"}, "foo") == "bar" + assert my_get(%{}, "foo", "not_found") == "not_found" end end - defmodule IfScope do - use ExUnit.Case, async: true + describe "defmodule" do + test "expects atoms as module names" do + msg = ~r"invalid module name: 3" - test "variables on nested if" do - if true do - a = 1 - if true do - b = 2 + assert_raise ArgumentError, msg, fn -> + defmodule 1 + 2, do: :ok + end + end + + test "does not accept special atoms as module names" do + special_atoms = [nil, true, false] + + Enum.each(special_atoms, fn special_atom -> + msg = ~r"invalid module name: #{inspect(special_atom)}" + + assert_raise ArgumentError, msg, fn -> + defmodule special_atom, do: :ok end + end) + end + + test "does not accept slashes in module names" do + assert_raise ArgumentError, ~r(invalid module name: :"foo/bar"), fn -> + defmodule :"foo/bar", do: :ok end - assert a == 1 - assert b == 2 + assert_raise ArgumentError, ~r(invalid module name: :"foo\\\\bar"), fn -> + defmodule :"foo\\bar", do: :ok + end end + end - test "variables on sibling if" do - if true do - a = 1 + describe "access" do + defmodule StructAccess do + defstruct [:foo, :bar] + end - if true do - b = 2 - end + test "get_in/1" do + users = %{"john" => %{age: 27}, :meg => %{age: 23}} + assert get_in(users["john"][:age]) == 27 + assert get_in(users["dave"][:age]) == nil + assert get_in(users["john"].age) == 27 + assert get_in(users["dave"].age) == nil + assert get_in(users.meg[:age]) == 23 + assert get_in(users.meg.age) == 23 - if true do - c = 3 - end + is_nil = nil + assert get_in(is_nil.age) == nil + + assert_raise KeyError, ~r"key :unknown not found", fn -> get_in(users.unknown) end + assert_raise KeyError, ~r"key :unknown not found", fn -> get_in(users.meg.unknown) end + end + + test "get_in/2" do + users = %{"john" => %{age: 27}, "meg" => %{age: 23}} + assert get_in(users, ["john", :age]) == 27 + assert get_in(users, ["dave", :age]) == nil + assert get_in(nil, ["john", :age]) == nil + + map = %{"fruits" => ["banana", "apple", "orange"]} + assert get_in(map, ["fruits", by_index(0)]) == "banana" + assert get_in(map, ["fruits", by_index(3)]) == nil + assert get_in(map, ["unknown", by_index(3)]) == nil + end + + test "put_in/3" do + users = %{"john" => %{age: 27}, "meg" => %{age: 23}} + + assert put_in(users, ["john", :age], 28) == %{"john" => %{age: 28}, "meg" => %{age: 23}} + + assert_raise ArgumentError, "could not put/update key \"john\" on a nil value", fn -> + put_in(nil, ["john", :age], 28) end + end - assert a == 1 - assert b == 2 - assert c == 3 + test "put_in/2" do + users = %{"john" => %{age: 27}, "meg" => %{age: 23}} + + assert put_in(users["john"][:age], 28) == %{"john" => %{age: 28}, "meg" => %{age: 23}} + + assert put_in(users["john"].age, 28) == %{"john" => %{age: 28}, "meg" => %{age: 23}} + + struct = %StructAccess{foo: %StructAccess{}} + + assert put_in(struct.foo.bar, :baz) == + %StructAccess{bar: nil, foo: %StructAccess{bar: :baz, foo: nil}} + + assert_raise BadMapError, fn -> + put_in(users["dave"].age, 19) + end + + assert_raise KeyError, fn -> + put_in(users["meg"].unknown, "value") + end + end + + test "update_in/3" do + users = %{"john" => %{age: 27}, "meg" => %{age: 23}} + + assert update_in(users, ["john", :age], &(&1 + 1)) == + %{"john" => %{age: 28}, "meg" => %{age: 23}} + + assert_raise ArgumentError, "could not put/update key \"john\" on a nil value", fn -> + update_in(nil, ["john", :age], fn _ -> %{} end) + end + + assert_raise UndefinedFunctionError, fn -> + pop_in(struct(Sample, []), [:name]) + end + end + + test "update_in/2" do + users = %{"john" => %{age: 27}, "meg" => %{age: 23}} + + assert update_in(users["john"][:age], &(&1 + 1)) == + %{"john" => %{age: 28}, "meg" => %{age: 23}} + + assert update_in(users["john"].age, &(&1 + 1)) == + %{"john" => %{age: 28}, "meg" => %{age: 23}} + + struct = %StructAccess{foo: %StructAccess{bar: 41}} + + assert update_in(struct.foo.bar, &(&1 + 1)) == + %StructAccess{bar: nil, foo: %StructAccess{bar: 42, foo: nil}} + + assert_raise BadMapError, fn -> + update_in(users["dave"].age, &(&1 + 1)) + end + + assert_raise KeyError, fn -> + put_in(users["meg"].unknown, &(&1 + 1)) + end + end + + test "get_and_update_in/3" do + users = %{"john" => %{age: 27}, "meg" => %{age: 23}} + + assert get_and_update_in(users, ["john", :age], &{&1, &1 + 1}) == + {27, %{"john" => %{age: 28}, "meg" => %{age: 23}}} + + map = %{"fruits" => ["banana", "apple", "orange"]} + + assert get_and_update_in(map, ["fruits", by_index(0)], &{&1, String.reverse(&1)}) == + {"banana", %{"fruits" => ["ananab", "apple", "orange"]}} + + assert get_and_update_in(map, ["fruits", by_index(3)], &{&1, &1}) == + {nil, %{"fruits" => ["banana", "apple", "orange"]}} + + assert get_and_update_in(map, ["unknown", by_index(3)], &{&1, []}) == + {:oops, %{"fruits" => ["banana", "apple", "orange"], "unknown" => []}} end - test "variables counter on nested ifs" do - r = (fn() -> 3 end).() # supresses warning at (if r < 0...) - r = r - 1 - r = r - 1 - r = r - 1 + test "get_and_update_in/2" do + users = %{"john" => %{age: 27}, "meg" => %{age: 23}} + + assert get_and_update_in(users["john"].age, &{&1, &1 + 1}) == + {27, %{"john" => %{age: 28}, "meg" => %{age: 23}}} + + struct = %StructAccess{foo: %StructAccess{bar: 41}} + + assert get_and_update_in(struct.foo.bar, &{&1, &1 + 1}) == + {41, %StructAccess{bar: nil, foo: %StructAccess{bar: 42, foo: nil}}} + + assert_raise ArgumentError, "could not put/update key \"john\" on a nil value", fn -> + get_and_update_in(nil["john"][:age], fn nil -> {:ok, 28} end) + end + + assert_raise BadMapError, fn -> + get_and_update_in(users["dave"].age, &{&1, &1 + 1}) + end - if true do - r = r - 1 - if r < 0, do: r = 0 + assert_raise KeyError, fn -> + get_and_update_in(users["meg"].unknown, &{&1, &1 + 1}) end + end + + test "pop_in/2" do + users = %{"john" => %{age: 27}, "meg" => %{age: 23}} + + assert pop_in(users, ["john", :age]) == + {27, %{"john" => %{}, "meg" => %{age: 23}}} + + assert pop_in(users, ["bob", :age]) == + {nil, %{"john" => %{age: 27}, "meg" => %{age: 23}}} + + assert pop_in([], [:foo, :bar]) == + {nil, []} + end + + test "pop_in/2 with paths" do + map = %{"fruits" => ["banana", "apple", "orange"]} + + assert pop_in(map, ["fruits", by_index(0)]) == + {"banana", %{"fruits" => ["apple", "orange"]}} + + assert pop_in(map, ["fruits", by_index(3)]) == {nil, map} + + map = %{"fruits" => [%{name: "banana"}, %{name: "apple"}]} + + assert pop_in(map, ["fruits", by_index(0), :name]) == + {"banana", %{"fruits" => [%{}, %{name: "apple"}]}} + + assert pop_in(map, ["fruits", by_index(3), :name]) == {nil, map} + end + + test "pop_in/1" do + users = %{"john" => %{age: 27}, "meg" => %{age: 23}} + + assert pop_in(users["john"]) == {%{age: 27}, %{"meg" => %{age: 23}}} + + assert pop_in(users["john"][:age]) == {27, %{"john" => %{}, "meg" => %{age: 23}}} + assert pop_in(users["john"][:name]) == {nil, %{"john" => %{age: 27}, "meg" => %{age: 23}}} + assert pop_in(users["bob"][:age]) == {nil, %{"john" => %{age: 27}, "meg" => %{age: 23}}} + + users = %{john: [age: 27], meg: [age: 23]} + + assert pop_in(users.john[:age]) == {27, %{john: [], meg: [age: 23]}} + assert pop_in(users.john[:name]) == {nil, %{john: [age: 27], meg: [age: 23]}} - assert r == 0 + assert pop_in([][:foo][:bar]) == {nil, []} + assert_raise KeyError, fn -> pop_in(users.bob[:age]) end + end + + test "pop_in/1,2 with nils" do + users = %{"john" => nil, "meg" => %{age: 23}} + assert pop_in(users["john"][:age]) == {nil, %{"meg" => %{age: 23}}} + assert pop_in(users, ["john", :age]) == {nil, %{"meg" => %{age: 23}}} + + users = %{john: nil, meg: %{age: 23}} + assert pop_in(users.john[:age]) == {nil, %{john: nil, meg: %{age: 23}}} + assert pop_in(users, [:john, :age]) == {nil, %{meg: %{age: 23}}} + + x = nil + assert_raise ArgumentError, fn -> pop_in(x["john"][:age]) end + assert_raise ArgumentError, fn -> pop_in(nil["john"][:age]) end + assert_raise ArgumentError, fn -> pop_in(nil, ["john", :age]) end + end + + test "with dynamic paths" do + map = empty_map() + + assert put_in(map[:foo], "bar") == %{foo: "bar"} + assert put_in(empty_map()[:foo], "bar") == %{foo: "bar"} + assert put_in(KernelTest.empty_map()[:foo], "bar") == %{foo: "bar"} + assert put_in(__MODULE__.empty_map()[:foo], "bar") == %{foo: "bar"} + + assert_raise ArgumentError, ~r"access at least one element,", fn -> + Code.eval_quoted(quote(do: put_in(map, "bar")), []) + end + + assert_raise ArgumentError, ~r"must start with a variable, local or remote call", fn -> + Code.eval_quoted(quote(do: put_in(map.foo(1, 2)[:bar], "baz")), []) + end + end + + def by_index(index) do + fn + :get, nil, _next -> + raise "won't be invoked" + + :get, data, next -> + next.(Enum.at(data, index)) + + :get_and_update, nil, next -> + next.(:oops) + + :get_and_update, data, next -> + current = Enum.at(data, index) + + case next.(current) do + {get, update} -> {get, List.replace_at(data, index, update)} + :pop -> {current, List.delete_at(data, index)} + end + end end end - defmodule Destructure do - use ExUnit.Case, async: true + describe "pipeline" do + test "simple" do + assert [1, [2], 3] |> List.flatten() == [1, 2, 3] + end + + test "nested" do + assert [1, [2], 3] |> List.flatten() |> Enum.map(&(&1 * 2)) == [2, 4, 6] + end + + test "local call" do + assert [1, [2], 3] |> List.flatten() |> local() == [2, 4, 6] + end + + test "with capture" do + assert Enum.map([1, 2, 3], &(&1 |> twice() |> twice())) == [4, 8, 12] + end + + test "with anonymous functions" do + assert 1 |> (&(&1 * 2)).() == 2 + assert [1] |> (&hd(&1)).() == 1 + end + + test "reverse associativity" do + assert [1, [2], 3] |> (List.flatten() |> Enum.map(&(&1 * 2))) == [2, 4, 6] + end + + defp twice(a), do: a * 2 + + defp local(list) do + Enum.map(list, &(&1 * 2)) + end + end + describe "destructure" do test "less args" do destructure [x, y, z], [1, 2, 3, 4, 5] assert x == 1 @@ -451,7 +1284,7 @@ defmodule KernelTest do end test "nil values" do - destructure [a, b, c], a_nil + destructure [a, b, c], a_nil() assert a == nil assert b == nil assert c == nil @@ -459,12 +1292,312 @@ defmodule KernelTest do test "invalid match" do a = List.first([3]) - assert_raise CaseClauseError, fn -> - destructure [^a, _b, _c], a_list + + assert_raise MatchError, fn -> + destructure [^a, _b, _c], a_list() end end defp a_list, do: [1, 2, 3] defp a_nil, do: nil end + + describe "use/2" do + import ExUnit.CaptureIO + + defmodule SampleA do + defmacro __using__(opts) do + prefix = Keyword.get(opts, :prefix, "") + IO.puts(prefix <> "A") + end + end + + defmodule SampleB do + defmacro __using__(_) do + IO.puts("B") + end + end + + test "invalid argument is literal" do + message = "invalid arguments for use, expected a compile time atom or alias, got: 42" + + assert_raise ArgumentError, message, fn -> + Code.eval_string("use 42") + end + end + + test "invalid argument is variable" do + message = "invalid arguments for use, expected a compile time atom or alias, got: variable" + + assert_raise ArgumentError, message, fn -> + Code.eval_string("use variable") + end + end + + test "multi-call" do + assert capture_io(fn -> + Code.eval_string("use KernelTest.{SampleA, SampleB,}", [], __ENV__) + end) == "A\nB\n" + end + + test "multi-call with options" do + assert capture_io(fn -> + Code.eval_string(~S|use KernelTest.{SampleA}, prefix: "-"|, [], __ENV__) + end) == "-A\n" + end + + test "multi-call with unquote" do + assert capture_io(fn -> + string = """ + defmodule TestMod do + def main() do + use KernelTest.{SampleB, unquote(:SampleA)} + end + end + """ + + Code.eval_string(string, [], __ENV__) + end) == "B\nA\n" + after + purge(KernelTest.TestMod) + end + end + + test "is_map_key/2" do + assert is_map_key(Map.new([]), :a) == false + assert is_map_key(Map.new(b: 1), :a) == false + assert is_map_key(Map.new(a: 1), :a) == true + + assert_raise BadMapError, fn -> + is_map_key(Process.get(:unused, []), :a) + end + + case Map.new(a: 1) do + map when is_map_key(map, :a) -> true + _ -> flunk("invalid guard") + end + end + + test "tap/1" do + import ExUnit.CaptureIO + + assert capture_io(fn -> + tap("foo", &IO.puts/1) + end) == "foo\n" + + assert 1 = tap(1, fn x -> x + 1 end) + end + + test "tl/1" do + assert tl([:one]) == [] + assert tl([1, 2, 3]) == [2, 3] + assert_raise ArgumentError, fn -> tl(Process.get(:unused, [])) end + + assert tl([:a | :b]) == :b + assert tl([:a, :b | :c]) == [:b | :c] + end + + test "hd/1" do + assert hd([1, 2, 3, 4]) == 1 + assert_raise ArgumentError, fn -> hd(Process.get(:unused, [])) end + assert hd([1 | 2]) == 1 + end + + test "floor/1" do + assert floor(1) === 1 + assert floor(1.0) === 1 + assert floor(0) === 0 + assert floor(0.0) === 0 + assert floor(-0.0) === 0 + assert floor(1.123) === 1 + assert floor(-10.123) === -11 + assert floor(-10) === -10 + assert floor(-10.0) === -10 + + assert match?(x when floor(x) == 0, 0.2) + end + + test "ceil/1" do + assert ceil(1) === 1 + assert ceil(1.0) === 1 + assert ceil(0) === 0 + assert ceil(0.0) === 0 + assert ceil(-0.0) === 0 + assert ceil(1.123) === 2 + assert ceil(-10.123) === -10 + assert ceil(-10) === -10 + assert ceil(-10.0) === -10 + + assert match?(x when ceil(x) == 1, 0.2) + end + + test "binary_slice/2" do + assert binary_slice("abc", -1..0) == "" + assert binary_slice("abc", -5..-5) == "" + assert binary_slice("x", 0..0//2) == "x" + assert binary_slice("abcde", 1..3//2) == "bd" + end + + test "sigil_U/2" do + assert ~U[2015-01-13 13:00:07.123Z] == %DateTime{ + calendar: Calendar.ISO, + day: 13, + hour: 13, + microsecond: {123_000, 3}, + minute: 0, + month: 1, + second: 7, + std_offset: 0, + time_zone: "Etc/UTC", + utc_offset: 0, + year: 2015, + zone_abbr: "UTC" + } + + assert_raise ArgumentError, ~r"reason: :invalid_format", fn -> + Code.eval_string(~s{~U[2015-01-13 13:00]}) + end + + assert_raise ArgumentError, ~r"reason: :invalid_format", fn -> + Code.eval_string(~s{~U[20150113 130007Z]}) + end + + assert_raise ArgumentError, ~r"reason: :missing_offset", fn -> + Code.eval_string(~s{~U[2015-01-13 13:00:07]}) + end + + assert_raise ArgumentError, ~r"reason: :non_utc_offset", fn -> + Code.eval_string(~s{~U[2015-01-13 13:00:07+00:30]}) + end + end + + describe "dbg/2" do + import ExUnit.CaptureIO + + test "prints the given expression and returns its value" do + output = capture_io(fn -> assert dbg(List.duplicate(:foo, 3)) == [:foo, :foo, :foo] end) + assert output =~ "kernel_test.exs" + assert output =~ "KernelTest" + assert output =~ "List" + assert output =~ "duplicate" + assert output =~ ":foo" + assert output =~ "3" + end + + test "prints the given expression with complex options" do + output = capture_io(fn -> assert dbg(123, [] ++ []) == 123 end) + assert output =~ "kernel_test.exs" + end + + test "doesn't print any colors if :syntax_colors is []" do + output = + capture_io(fn -> + assert dbg(List.duplicate(:foo, 3), syntax_colors: []) == [:foo, :foo, :foo] + end) + + assert output =~ "kernel_test.exs" + assert output =~ "KernelTest." + assert output =~ "List.duplicate(:foo, 3)" + assert output =~ "[:foo, :foo, :foo]" + refute output =~ "\\e[" + end + + test "prints binding() if no arguments are given" do + my_var = 1 + my_other_var = :foo + + output = capture_io(fn -> dbg() end) + + assert output =~ "binding()" + assert output =~ "my_var:" + assert output =~ "my_other_var:" + end + + test "is not allowed in guards" do + message = "invalid expression in guard, dbg is not allowed in guards" + + assert_raise ArgumentError, Regex.compile!(message), fn -> + defmodule DbgGuard do + def dbg_guard() when dbg(1), do: true + end + end + end + + test "is not allowed in pattern matches" do + message = "invalid expression in match, dbg is not allowed in patterns" + + assert_eval_raise(ArgumentError, Regex.compile!(message), """ + {:ok, dbg()} = make_ref() + """) + end + end + + describe "to_timeout/1" do + test "works with keyword lists" do + assert to_timeout(hour: 2) == 1000 * 60 * 60 * 2 + assert to_timeout(minute: 74) == 1000 * 60 * 74 + assert to_timeout(second: 1293) == 1_293_000 + assert to_timeout(millisecond: 1_234_123) == 1_234_123 + + assert to_timeout(hour: 2, minute: 30) == 1000 * 60 * 60 * 2 + 1000 * 60 * 30 + assert to_timeout(minute: 30, hour: 2) == 1000 * 60 * 60 * 2 + 1000 * 60 * 30 + assert to_timeout(minute: 74, second: 30) == 1000 * 60 * 74 + 1000 * 30 + end + + test "raises on invalid values with keyword lists" do + for unit <- [:hour, :minute, :second, :millisecond], + value <- [-1, 1.0, :not_an_int] do + message = + "timeout component #{inspect(unit)} must be a non-negative integer, " <> + "got: #{inspect(value)}" + + assert_raise ArgumentError, message, fn -> to_timeout([{unit, value}]) end + end + end + + test "raises on invalid keys with keyword lists" do + message = + "timeout component :not_a_unit is not a valid timeout component, valid values are: " <> + ":week, :day, :hour, :minute, :second, :millisecond" + + assert_raise ArgumentError, message, fn -> to_timeout(minute: 3, not_a_unit: 1) end + end + + test "raises on duplicated components with keyword lists" do + assert_raise ArgumentError, "timeout component :minute is duplicated", fn -> + to_timeout(minute: 3, hour: 2, minute: 1) + end + end + + test "works with durations" do + assert to_timeout(Duration.new!(hour: 2)) == 1000 * 60 * 60 * 2 + assert to_timeout(Duration.new!(minute: 74)) == 1000 * 60 * 74 + assert to_timeout(Duration.new!(second: 1293)) == 1_293_000 + assert to_timeout(Duration.new!(microsecond: {1_234_123, 4})) == 1_234 + + assert to_timeout(Duration.new!(hour: 2, minute: 30)) == 1000 * 60 * 60 * 2 + 1000 * 60 * 30 + assert to_timeout(Duration.new!(minute: 30, hour: 2)) == 1000 * 60 * 60 * 2 + 1000 * 60 * 30 + assert to_timeout(Duration.new!(minute: 74, second: 30)) == 1000 * 60 * 74 + 1000 * 30 + end + + test "raises on durations with non-zero months or days" do + message = "duration with a non-zero month cannot be reliably converted to timeouts" + + assert_raise ArgumentError, message, fn -> + to_timeout(Duration.new!(month: 3)) + end + + message = "duration with a non-zero year cannot be reliably converted to timeouts" + + assert_raise ArgumentError, message, fn -> + to_timeout(Duration.new!(year: 1)) + end + end + + test "works with timeouts" do + assert to_timeout(1_000) == 1_000 + assert to_timeout(0) == 0 + assert to_timeout(:infinity) == :infinity + end + end end diff --git a/lib/elixir/test/elixir/keyword_test.exs b/lib/elixir/test/elixir/keyword_test.exs index 4387b6026e3..2a133886cf3 100644 --- a/lib/elixir/test/elixir/keyword_test.exs +++ b/lib/elixir/test/elixir/keyword_test.exs @@ -1,8 +1,14 @@ -Code.require_file "test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("test_helper.exs", __DIR__) defmodule KeywordTest do use ExUnit.Case, async: true + doctest Keyword + test "has a literal syntax" do assert [B: 1] == [{:B, 1}] assert [foo?: :bar] == [{:foo?, :bar}] @@ -11,238 +17,229 @@ defmodule KeywordTest do end test "is a :: operator on ambiguity" do - assert [{:::, _, [{:a, _, _}, {:b, _, _}]}] = quote(do: [a::b]) + assert [{:"::", _, [{:a, _, _}, {:b, _, _}]}] = quote(do: [a :: b]) end test "supports optional comma" do - [a: 1, - b: 2, - c: 3, ] + assert Code.eval_string("[a: 1, b: 2, c: 3,]") == {[a: 1, b: 2, c: 3], []} end - test "keyword?/1" do - assert Keyword.keyword?([]) - assert Keyword.keyword?([a: 1]) - assert Keyword.keyword?([{Foo, 1}]) - refute Keyword.keyword?([{}]) - refute Keyword.keyword?(<<>>) + test "implements (almost) all functions in Map" do + assert Map.__info__(:functions) -- Keyword.__info__(:functions) == [from_struct: 1] end - test "new/0" do - assert Keyword.new == [] + test "get_and_update/3 removes duplicates from the input keyword list" do + assert Keyword.get_and_update([a: 1, b: 2, a: 3], :a, fn value -> {value, value + 10} end) == + {1, [a: 11, b: 2]} end - test "new/1" do - assert Keyword.new([{:second_key, 2}, {:first_key, 1}]) == - [first_key: 1, second_key: 2] - end + test "get_and_update/3 raises on bad return value from the argument function" do + message = "the given function must return a two-element tuple or :pop, got: 1" - test "new/2" do - assert Keyword.new([:a, :b], fn x -> {x, x} end) == - [b: :b, a: :a] - end - - test "get/2 and get/3" do - assert Keyword.get(create_keywords, :first_key) == 1 - assert Keyword.get(create_keywords, :second_key) == 2 - assert Keyword.get(create_keywords, :other_key) == nil - assert Keyword.get(create_empty_keywords, :first_key, "default") == "default" - end + assert_raise RuntimeError, message, fn -> + Keyword.get_and_update([a: 1], :a, fn value -> value end) + end - test "fetch!/2" do - assert Keyword.fetch!(create_keywords, :first_key) == 1 + message = "the given function must return a two-element tuple or :pop, got: nil" - error = assert_raise KeyError, fn -> - Keyword.fetch!(create_keywords, :unknown) + assert_raise RuntimeError, message, fn -> + Keyword.get_and_update([], :a, fn value -> value end) end + end - assert error.key == :unknown + test "get_and_update!/3 removes duplicates from the input keyword list" do + assert Keyword.get_and_update!([a: 1, b: 2, a: 3], :a, fn value -> {value, value + 10} end) == + {1, [a: 11, b: 2]} end - test "keys/1" do - assert Keyword.keys(create_keywords) == [:first_key, :second_key] - assert Keyword.keys(create_empty_keywords) == [] + test "get_and_update!/3 raises on bad return value from the argument function" do + message = "the given function must return a two-element tuple or :pop, got: 1" - assert_raise FunctionClauseError, fn -> - Keyword.keys([:foo]) + assert_raise RuntimeError, message, fn -> + Keyword.get_and_update!([a: 1], :a, fn value -> value end) end end - test "values/1" do - assert Keyword.values(create_keywords) == [1, 2] - assert Keyword.values(create_empty_keywords) == [] + test "update!" do + assert Keyword.update!([a: 1, b: 2, a: 3], :a, &(&1 * 2)) == [a: 2, b: 2] + assert Keyword.update!([a: 1, b: 2, c: 3], :b, &(&1 * 2)) == [a: 1, b: 4, c: 3] + end - assert_raise FunctionClauseError, fn -> - Keyword.values([:foo]) - end + test "replace" do + assert Keyword.replace([a: 1, b: 2, a: 3], :a, :new) == [a: :new, b: 2] + assert Keyword.replace([a: 1, b: 2, a: 3], :a, 1) == [a: 1, b: 2] + assert Keyword.replace([a: 1, b: 2, a: 3, b: 4], :a, 1) == [a: 1, b: 2, b: 4] + assert Keyword.replace([a: 1, b: 2, c: 3, b: 4], :b, :new) == [a: 1, b: :new, c: 3] + assert Keyword.replace([], :b, :new) == [] + assert Keyword.replace([a: 1, b: 2, a: 3], :c, :new) == [a: 1, b: 2, a: 3] end - test "delete/2" do - assert Keyword.delete(create_keywords, :second_key) == [first_key: 1] - assert Keyword.delete(create_keywords, :other_key) == [first_key: 1, second_key: 2] - assert Keyword.delete(create_empty_keywords, :other_key) == [] + test "replace!" do + assert Keyword.replace!([a: 1, b: 2, a: 3], :a, :new) == [a: :new, b: 2] + assert Keyword.replace!([a: 1, b: 2, a: 3], :a, 1) == [a: 1, b: 2] + assert Keyword.replace!([a: 1, b: 2, a: 3, b: 4], :a, 1) == [a: 1, b: 2, b: 4] + assert Keyword.replace!([a: 1, b: 2, c: 3, b: 4], :b, :new) == [a: 1, b: :new, c: 3] - assert_raise FunctionClauseError, fn -> - Keyword.delete([:foo], :foo) + assert_raise KeyError, "key :b not found in:\n\n []\n", fn -> + Keyword.replace!([], :b, :new) end - end - - test "delete/3" do - keywords = [a: 1, b: 2, c: 3, a: 2] - assert Keyword.delete(keywords, :a, 2) == [a: 1, b: 2, c: 3] - assert Keyword.delete(keywords, :a, 1) == [b: 2, c: 3, a: 2] - assert_raise FunctionClauseError, fn -> - Keyword.delete([:foo], :foo, 0) + assert_raise KeyError, "key :c not found in:\n\n [a: 1, b: 2, a: 3]\n", fn -> + Keyword.replace!([a: 1, b: 2, a: 3], :c, :new) end end - test "put/3" do - assert Keyword.put(create_empty_keywords, :first_key, 1) == [first_key: 1] - assert Keyword.put(create_keywords, :first_key, 3) == [first_key: 3, second_key: 2] - end + test "merge/2" do + assert Keyword.merge([a: 1, b: 2], c: 11, d: 12) == [a: 1, b: 2, c: 11, d: 12] + assert Keyword.merge([], c: 11, d: 12) == [c: 11, d: 12] + assert Keyword.merge([a: 1, b: 2], []) == [a: 1, b: 2] - test "put_new/3" do - assert Keyword.put_new(create_empty_keywords, :first_key, 1) == [first_key: 1] - assert Keyword.put_new(create_keywords, :first_key, 3) == [first_key: 1, second_key: 2] - end + message = "expected a keyword list as the first argument, got: [1, 2]" - test "merge/2" do - assert Keyword.merge(create_empty_keywords, create_keywords) == [first_key: 1, second_key: 2] - assert Keyword.merge(create_keywords, create_empty_keywords) == [first_key: 1, second_key: 2] - assert Keyword.merge(create_keywords, create_keywords) == [first_key: 1, second_key: 2] - assert Keyword.merge(create_empty_keywords, create_empty_keywords) == [] + assert_raise ArgumentError, message, fn -> + Keyword.merge([1, 2], c: 11, d: 12) + end - assert_raise FunctionClauseError, fn -> - Keyword.delete([:foo], [:foo]) + message = "expected a keyword list as the first argument, got: [1 | 2]" + + assert_raise ArgumentError, message, fn -> + Keyword.merge([1 | 2], c: 11, d: 12) end - end - test "merge/3" do - result = Keyword.merge [a: 1, b: 2], [a: 3, d: 4], fn _k, v1, v2 -> - v1 + v2 + message = "expected a keyword list as the second argument, got: [11, 12, 0]" + + assert_raise ArgumentError, message, fn -> + Keyword.merge([a: 1, b: 2], [11, 12, 0]) end - assert result == [a: 4, b: 2, d: 4] - end - test "has_key?/2" do - assert Keyword.has_key?([a: 1], :a) == true - assert Keyword.has_key?([a: 1], :b) == false - end + message = "expected a keyword list as the second argument, got: [11 | 12]" - test "update!/3" do - kw = [a: 1, b: 2, a: 3] - assert Keyword.update!(kw, :a, &(&1 * 2)) == [a: 2, b: 2] - assert_raise KeyError, fn -> - Keyword.update!([a: 1], :b, &(&1 * 2)) + assert_raise ArgumentError, message, fn -> + Keyword.merge([a: 1, b: 2], [11 | 12]) end - end - test "update/4" do - kw = [a: 1, b: 2, a: 3] - assert Keyword.update(kw, :a, 13, &(&1 * 2)) == [a: 2, b: 2] - assert Keyword.update([a: 1], :b, 11, &(&1 * 2)) == [a: 1, b: 11] - end + # duplicate keys in keywords1 are kept if key is not present in keywords2 + result = [a: 1, b: 2, a: 3, c: 11, d: 12] + assert Keyword.merge([a: 1, b: 2, a: 3], c: 11, d: 12) == result - defp create_empty_keywords, do: [] - defp create_keywords, do: [first_key: 1, second_key: 2] -end + result = [b: 2, a: 11] + assert Keyword.merge([a: 1, b: 2, a: 3], a: 11) == result -defmodule Keyword.DuplicatedTest do - use ExUnit.Case, async: true + # duplicate keys in keywords2 are always kept + result = [a: 1, b: 2, c: 11, c: 12, d: 13] + assert Keyword.merge([a: 1, b: 2], c: 11, c: 12, d: 13) == result - test "get/2" do - assert Keyword.get(create_keywords, :first_key) == 1 - assert Keyword.get(create_keywords, :second_key) == 2 - assert Keyword.get(create_keywords, :other_key) == nil - assert Keyword.get(create_empty_keywords, :first_key, "default") == "default" + # any key in keywords1 is removed if key is present in keyword2 + result = [a: 1, b: 2, c: 11, c: 12, d: 13] + assert Keyword.merge([a: 1, b: 2, c: 3, c: 4], c: 11, c: 12, d: 13) == result end - test "get_values/2" do - assert Keyword.get_values(create_keywords, :first_key) == [1, 2] - assert Keyword.get_values(create_keywords, :second_key) == [2] - assert Keyword.get_values(create_keywords, :other_key) == [] + test "merge/3" do + fun = fn _key, value1, value2 -> value1 + value2 end - assert_raise FunctionClauseError, fn -> - Keyword.get_values([:foo], :foo) - end - end + assert Keyword.merge([a: 1, b: 2], [c: 11, d: 12], fun) == [a: 1, b: 2, c: 11, d: 12] + assert Keyword.merge([], [c: 11, d: 12], fun) == [c: 11, d: 12] + assert Keyword.merge([a: 1, b: 2], [], fun) == [a: 1, b: 2] - test "keys/1" do - assert Keyword.keys(create_keywords) == [:first_key, :first_key, :second_key] - assert Keyword.keys(create_empty_keywords) == [] - end + message = "expected a keyword list as the first argument, got: [1, 2]" - test "equal?/2" do - assert Keyword.equal? [a: 1, b: 2], [b: 2, a: 1] - refute Keyword.equal? [a: 1, b: 2], [b: 2, c: 3] - end + assert_raise ArgumentError, message, fn -> + Keyword.merge([1, 2], [c: 11, d: 12], fun) + end - test "values/1" do - assert Keyword.values(create_keywords) == [1, 2, 2] - assert Keyword.values(create_empty_keywords) == [] - end + message = "expected a keyword list as the first argument, got: [1 | 2]" - test "delete/2" do - assert Keyword.delete(create_keywords, :first_key) == [second_key: 2] - assert Keyword.delete(create_keywords, :other_key) == create_keywords - assert Keyword.delete(create_empty_keywords, :other_key) == [] - end + assert_raise ArgumentError, message, fn -> + Keyword.merge([1 | 2], [c: 11, d: 12], fun) + end - test "delete_first/2" do - assert Keyword.delete_first(create_keywords, :first_key) == [first_key: 2, second_key: 2] - assert Keyword.delete_first(create_keywords, :other_key) == [first_key: 1, first_key: 2, second_key: 2] - assert Keyword.delete_first(create_empty_keywords, :other_key) == [] - end + message = "expected a keyword list as the second argument, got: [{:x, 1}, :y, :z]" - test "put/3" do - assert Keyword.put(create_empty_keywords, :first_key, 1) == [first_key: 1] - assert Keyword.put(create_keywords, :first_key, 1) == [first_key: 1, second_key: 2] - end + assert_raise ArgumentError, message, fn -> + Keyword.merge([a: 1, b: 2], [{:x, 1}, :y, :z], fun) + end - test "merge/2" do - assert Keyword.merge(create_empty_keywords, create_keywords) == create_keywords - assert Keyword.merge(create_keywords, create_empty_keywords) == create_keywords - assert Keyword.merge(create_keywords, create_keywords) == create_keywords - assert Keyword.merge(create_empty_keywords, create_empty_keywords) == [] - assert Keyword.merge(create_keywords, [first_key: 0]) == [first_key: 0, second_key: 2] - assert Keyword.merge(create_keywords, [first_key: 0, first_key: 3]) == [first_key: 0, first_key: 3, second_key: 2] - end + message = "expected a keyword list as the second argument, got: [:x | :y]" - test "merge/3" do - result = Keyword.merge [a: 1, b: 2], [a: 3, d: 4], fn _k, v1, v2 -> - v1 + v2 + assert_raise ArgumentError, message, fn -> + Keyword.merge([a: 1, b: 2], [:x | :y], fun) end - assert Keyword.equal?(result, [a: 4, b: 2, d: 4]) - end - test "has_key?/2" do - assert Keyword.has_key?([a: 1], :a) == true - assert Keyword.has_key?([a: 1], :b) == false - end + message = "expected a keyword list as the second argument, got: [{:x, 1} | :y]" - test "take/2" do - assert Keyword.take([], []) == [] - assert Keyword.take([a: 0, b: 1, a: 2], []) == [] - assert Keyword.take([a: 0, b: 1, a: 2], [:a]) == [a: 0, a: 2] - assert Keyword.take([a: 0, b: 1, a: 2], [:b]) == [b: 1] + assert_raise ArgumentError, message, fn -> + Keyword.merge([a: 1, b: 2], [{:x, 1} | :y], fun) + end - assert_raise FunctionClauseError, fn -> - Keyword.take([:foo], [:foo]) + # duplicate keys in keywords1 are left untouched if key is not present in keywords2 + result = [a: 1, b: 2, a: 3, c: 11, d: 12] + assert Keyword.merge([a: 1, b: 2, a: 3], [c: 11, d: 12], fun) == result + + result = [b: 2, a: 12] + assert Keyword.merge([a: 1, b: 2, a: 3], [a: 11], fun) == result + + # duplicate keys in keywords2 are always kept + result = [a: 1, b: 2, c: 11, c: 12, d: 13] + assert Keyword.merge([a: 1, b: 2], [c: 11, c: 12, d: 13], fun) == result + + # every key in keywords1 is replaced with fun result if key is present in keyword2 + result = [a: 1, b: 2, c: 14, c: 54, d: 13] + assert Keyword.merge([a: 1, b: 2, c: 3, c: 4], [c: 11, c: 50, d: 13], fun) == result + end + + test "merge/2 and merge/3 behave exactly the same way" do + fun = fn _key, _value1, value2 -> value2 end + + args = [ + {[a: 1, b: 2], [c: 11, d: 12]}, + {[], [c: 11, d: 12]}, + {[a: 1, b: 2], []}, + {[a: 1, b: 2, a: 3], [c: 11, d: 12]}, + {[a: 1, b: 2, a: 3], [a: 11]}, + {[a: 1, b: 2], [c: 11, c: 12, d: 13]}, + {[a: 1, b: 2, c: 3, c: 4], [c: 11, c: 12, d: 13]} + ] + + args_error = [ + {[1, 2], [c: 11, d: 12]}, + {[1 | 2], [c: 11, d: 12]}, + {[a: 1, b: 2], [11, 12, 0]}, + {[a: 1, b: 2], [11 | 12]}, + {[a: 1, b: 2], [{:x, 1}, :y, :z]}, + {[a: 1, b: 2], [:x | :y]}, + {[a: 1, b: 2], [{:x, 1} | :y]} + ] + + for {arg1, arg2} <- args do + assert Keyword.merge(arg1, arg2) == Keyword.merge(arg1, arg2, fun) + end + + for {arg1, arg2} <- args_error do + error = assert_raise ArgumentError, fn -> Keyword.merge(arg1, arg2) end + assert_raise ArgumentError, error.message, fn -> Keyword.merge(arg1, arg2, fun) end end end - test "drop/2" do - assert Keyword.drop([], []) == [] - assert Keyword.drop([a: 0, b: 1, a: 2], []) == [a: 0, b: 1, a: 2] - assert Keyword.drop([a: 0, b: 1, a: 2], [:a]) == [b: 1] - assert Keyword.drop([a: 0, b: 1, a: 2], [:b]) == [a: 0, a: 2] + test "validate/2 raises on invalid arguments" do + assert_raise ArgumentError, + "expected a keyword list as first argument, got invalid entry: :three", + fn -> Keyword.validate([:three], one: 1, two: 2) end - assert_raise FunctionClauseError, fn -> - Keyword.drop([:foo], [:foo]) - end + assert_raise ArgumentError, + "expected the second argument to be a list of atoms or tuples, got: 3", + fn -> Keyword.validate([three: 3], [:three, 3, :two]) end end - defp create_empty_keywords, do: [] - defp create_keywords, do: [first_key: 1, first_key: 2, second_key: 2] + test "split_with/2" do + assert Keyword.split_with([], fn {_k, v} -> rem(v, 2) == 0 end) == {[], []} + + assert Keyword.split_with([a: "1", a: 1, b: 2], fn {k, _v} -> k in [:a, :b] end) == + {[a: "1", a: 1, b: 2], []} + + assert Keyword.split_with([a: "1", a: 1, b: 2], fn {_k, v} -> v == 5 end) == + {[], [a: "1", a: 1, b: 2]} + + assert Keyword.split_with([a: "1", a: 1, b: 2], fn {k, v} -> k in [:a] and is_integer(v) end) == + {[a: 1], [a: "1", b: 2]} + end end diff --git a/lib/elixir/test/elixir/list/chars_test.exs b/lib/elixir/test/elixir/list/chars_test.exs index 94389bddb95..311496fe919 100644 --- a/lib/elixir/test/elixir/list/chars_test.exs +++ b/lib/elixir/test/elixir/list/chars_test.exs @@ -1,37 +1,47 @@ -Code.require_file "../test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) defmodule List.Chars.AtomTest do use ExUnit.Case, async: true - test :basic do - assert to_char_list(:foo) == 'foo' + test "basic" do + assert to_charlist(:foo) == ~c"foo" + end + + test "true false nil" do + assert to_charlist(false) == ~c"false" + assert to_charlist(true) == ~c"true" + assert to_charlist(nil) == ~c"" end end defmodule List.Chars.BitStringTest do use ExUnit.Case, async: true - test :basic do - assert to_char_list("foo") == 'foo' + test "basic" do + assert to_charlist("foo") == ~c"foo" end end defmodule List.Chars.NumberTest do use ExUnit.Case, async: true - test :integer do - assert to_char_list(1) == '1' + test "integer" do + assert to_charlist(1) == ~c"1" end - test :float do - assert to_char_list(1.0) == '1.0' + test "float" do + assert to_charlist(1.0) == ~c"1.0" end end defmodule List.Chars.ListTest do use ExUnit.Case, async: true - test :basic do - assert to_char_list([ 1, "b", 3 ]) == [1, "b", 3] + test "basic" do + assert to_charlist([1, "b", 3]) == [1, "b", 3] end end diff --git a/lib/elixir/test/elixir/list_test.exs b/lib/elixir/test/elixir/list_test.exs index 7da04556abc..2d1c283810d 100644 --- a/lib/elixir/test/elixir/list_test.exs +++ b/lib/elixir/test/elixir/list_test.exs @@ -1,121 +1,201 @@ -Code.require_file "test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("test_helper.exs", __DIR__) defmodule ListTest do use ExUnit.Case, async: true - test :cons_cell_precedence do - assert [1|:lists.flatten([2, 3])] == [1, 2, 3] + doctest List + + test "cons cell precedence" do + assert [1 | List.flatten([2, 3])] == [1, 2, 3] end - test :optional_comma do - assert [1] == [ 1, ] - assert [1, 2, 3] == [1, 2, 3, ] + test "optional comma" do + assert Code.eval_string("[1,]") == {[1], []} + assert Code.eval_string("[1, 2, 3,]") == {[1, 2, 3], []} end - test :partial_application do + test "partial application" do assert (&[&1, 2]).(1) == [1, 2] assert (&[&1, &2]).(1, 2) == [1, 2] assert (&[&2, &1]).(2, 1) == [1, 2] - assert (&[&1|&2]).(1, 2) == [1|2] - assert (&[&1, &2|&3]).(1, 2, 3) == [1, 2|3] + assert (&[&1 | &2]).(1, 2) == [1 | 2] + assert (&[&1, &2 | &3]).(1, 2, 3) == [1, 2 | 3] + end + + test "delete/2" do + assert List.delete([:a, :b, :c], :a) == [:b, :c] + assert List.delete([:a, :b, :c], :d) == [:a, :b, :c] + assert List.delete([:a, :b, :b, :c], :b) == [:a, :b, :c] + assert List.delete([], :b) == [] end - test :wrap do + test "wrap/1" do assert List.wrap([1, 2, 3]) == [1, 2, 3] assert List.wrap(1) == [1] assert List.wrap(nil) == [] end - test :flatten do + test "flatten/1" do assert List.flatten([1, 2, 3]) == [1, 2, 3] assert List.flatten([1, [2], 3]) == [1, 2, 3] assert List.flatten([[1, [2], 3]]) == [1, 2, 3] assert List.flatten([]) == [] assert List.flatten([[]]) == [] + assert List.flatten([[], [[], []]]) == [] end - test :flatten_with_tail do + test "flatten/2" do assert List.flatten([1, 2, 3], [4, 5]) == [1, 2, 3, 4, 5] assert List.flatten([1, [2], 3], [4, 5]) == [1, 2, 3, 4, 5] assert List.flatten([[1, [2], 3]], [4, 5]) == [1, 2, 3, 4, 5] + assert List.flatten([1, [], 2], [3, [], 4]) == [1, 2, 3, [], 4] end - test :foldl do + test "foldl/3" do assert List.foldl([1, 2, 3], 0, fn x, y -> x + y end) == 6 assert List.foldl([1, 2, 3], 10, fn x, y -> x + y end) == 16 assert List.foldl([1, 2, 3, 4], 0, fn x, y -> x - y end) == 2 end - test :foldr do + test "foldr/3" do assert List.foldr([1, 2, 3], 0, fn x, y -> x + y end) == 6 assert List.foldr([1, 2, 3], 10, fn x, y -> x + y end) == 16 assert List.foldr([1, 2, 3, 4], 0, fn x, y -> x - y end) == -2 end - test :reverse do - assert Enum.reverse([1, 2, 3]) == [3, 2, 1] - end - - test :duplicate do + test "duplicate/2" do + assert List.duplicate(1, 0) == [] assert List.duplicate(1, 3) == [1, 1, 1] assert List.duplicate([1], 1) == [[1]] end - test :last do + test "first/1" do + assert List.first([]) == nil + assert List.first([], 1) == 1 + assert List.first([1]) == 1 + assert List.first([1, 2, 3]) == 1 + end + + test "last/1" do assert List.last([]) == nil + assert List.last([], 1) == 1 assert List.last([1]) == 1 assert List.last([1, 2, 3]) == 3 end - test :zip do - assert List.zip([[1, 4], [2, 5], [3, 6]]) == [{1, 2, 3}, {4, 5, 6}] - assert List.zip([[1, 4], [2, 5, 0], [3, 6]]) == [{1, 2, 3}, {4, 5, 6}] - assert List.zip([[1], [2, 5], [3, 6]]) == [{1, 2, 3}] - assert List.zip([[1, 4], [2, 5], []]) == [] - end - - test :unzip do - assert List.unzip([{1, 2, 3}, {4, 5, 6}]) == [[1, 4], [2, 5], [3, 6]] - assert List.unzip([{1, 2, 3}, {4, 5}]) == [[1, 4], [2, 5]] - assert List.unzip([[1, 2, 3], [4, 5]]) == [[1, 4], [2, 5]] - assert List.unzip([]) == [] - end - - test :keyfind do + test "keyfind/4" do assert List.keyfind([a: 1, b: 2], :a, 0) == {:a, 1} assert List.keyfind([a: 1, b: 2], 2, 1) == {:b, 2} assert List.keyfind([a: 1, b: 2], :c, 0) == nil end - test :keyreplace do + test "keyreplace/4" do assert List.keyreplace([a: 1, b: 2], :a, 0, {:a, 3}) == [a: 3, b: 2] assert List.keyreplace([a: 1], :b, 0, {:b, 2}) == [a: 1] end - test :keysort do + test "keysort/2" do assert List.keysort([a: 4, b: 3, c: 5], 1) == [b: 3, a: 4, c: 5] assert List.keysort([a: 4, c: 1, b: 2], 0) == [a: 4, b: 2, c: 1] end - test :keystore do + test "keysort/3 with stable sorting" do + collection = [ + {2, 4}, + {1, 5}, + {2, 2}, + {3, 1}, + {4, 3} + ] + + # Stable sorting + assert List.keysort(collection, 0) == [ + {1, 5}, + {2, 4}, + {2, 2}, + {3, 1}, + {4, 3} + ] + + assert List.keysort(collection, 0) == + List.keysort(collection, 0, :asc) + + assert List.keysort(collection, 0, & - assert [] = List.delete_at([], i) + test "delete_at/2" do + for index <- [-1, 0, 1] do + assert List.delete_at([], index) == [] end + assert List.delete_at([1, 2, 3], 0) == [2, 3] assert List.delete_at([1, 2, 3], 2) == [1, 2] assert List.delete_at([1, 2, 3], 3) == [1, 2, 3] @@ -155,13 +236,193 @@ defmodule ListTest do assert List.delete_at([1, 2, 3], -4) == [1, 2, 3] end - test :to_string do + test "pop_at/3" do + for index <- [-1, 0, 1] do + assert List.pop_at([], index) == {nil, []} + end + + assert List.pop_at([1], 1, 2) == {2, [1]} + assert List.pop_at([1, 2, 3], 0) == {1, [2, 3]} + assert List.pop_at([1, 2, 3], 2) == {3, [1, 2]} + assert List.pop_at([1, 2, 3], 3) == {nil, [1, 2, 3]} + assert List.pop_at([1, 2, 3], -1) == {3, [1, 2]} + assert List.pop_at([1, 2, 3], -3) == {1, [2, 3]} + assert List.pop_at([1, 2, 3], -4) == {nil, [1, 2, 3]} + end + + describe "starts_with?/2" do + test "list and prefix are equal" do + assert List.starts_with?([], []) + assert List.starts_with?([1], [1]) + assert List.starts_with?([1, 2, 3], [1, 2, 3]) + end + + test "proper lists" do + refute List.starts_with?([1], [1, 2]) + assert List.starts_with?([1, 2, 3], [1, 2]) + refute List.starts_with?([1, 2, 3], [1, 2, 3, 4]) + end + + test "list is empty" do + refute List.starts_with?([], [1]) + refute List.starts_with?([], [1, 2]) + end + + test "prefix is empty" do + assert List.starts_with?([1], []) + assert List.starts_with?([1, 2], []) + assert List.starts_with?([1, 2, 3], []) + end + + test "only accepts proper lists" do + message = "no function clause matching in List.starts_with?/2" + + assert_raise FunctionClauseError, message, fn -> + List.starts_with?([1 | 2], [1 | 2]) + end + end + end + + describe "ends_with?/2" do + test "list and prefix are equal" do + assert List.ends_with?([], []) + assert List.ends_with?([1], [1]) + assert List.ends_with?([1, 2, 3], [1, 2, 3]) + end + + test "proper lists" do + refute List.ends_with?([2], [1, 2]) + assert List.ends_with?([1, 2, 3], [2, 3]) + refute List.ends_with?([2, 3, 4], [1, 2, 3, 4]) + end + + test "list is empty" do + refute List.ends_with?([], [1]) + refute List.ends_with?([], [1, 2]) + end + + test "prefix is empty" do + assert List.ends_with?([1], []) + assert List.ends_with?([1, 2], []) + assert List.ends_with?([1, 2, 3], []) + end + + test "only accepts proper lists" do + assert_raise ArgumentError, ~r/not a list/, fn -> + List.ends_with?([1 | 2], [1 | 2]) + end + end + end + + test "to_string/1" do assert List.to_string([?æ, ?ß]) == "æß" assert List.to_string([?a, ?b, ?c]) == "abc" + assert List.to_string([]) == "" + assert List.to_string([[], []]) == "" - assert_raise UnicodeConversionError, - "invalid code point 57343", fn -> + assert_raise UnicodeConversionError, "invalid code point 57343", fn -> List.to_string([0xDFFF]) end + + assert_raise UnicodeConversionError, "invalid encoding starting at <<216, 0>>", fn -> + List.to_string(["a", "b", <<0xD800::size(16)>>]) + end + + assert_raise ArgumentError, ~r"cannot convert the given list to a string", fn -> + List.to_string([:a, :b]) + end + end + + test "to_charlist/1" do + assert List.to_charlist([0x00E6, 0x00DF]) == ~c"æß" + assert List.to_charlist([0x0061, "bc"]) == ~c"abc" + assert List.to_charlist([0x0064, "ee", [~c"p"]]) == ~c"deep" + + assert_raise UnicodeConversionError, "invalid code point 57343", fn -> + List.to_charlist([0xDFFF]) + end + + assert_raise UnicodeConversionError, "invalid encoding starting at <<216, 0>>", fn -> + List.to_charlist(["a", "b", <<0xD800::size(16)>>]) + end + + assert_raise ArgumentError, ~r"cannot convert the given list to a charlist", fn -> + List.to_charlist([:a, :b]) + end + end + + describe "myers_difference/2" do + test "follows paper implementation" do + assert List.myers_difference([], []) == [] + assert List.myers_difference([], [1, 2, 3]) == [ins: [1, 2, 3]] + assert List.myers_difference([1, 2, 3], []) == [del: [1, 2, 3]] + assert List.myers_difference([1, 2, 3], [1, 2, 3]) == [eq: [1, 2, 3]] + assert List.myers_difference([1, 2, 3], [1, 4, 2, 3]) == [eq: [1], ins: [4], eq: [2, 3]] + assert List.myers_difference([1, 4, 2, 3], [1, 2, 3]) == [eq: [1], del: [4], eq: [2, 3]] + assert List.myers_difference([1], [[1]]) == [del: [1], ins: [[1]]] + assert List.myers_difference([[1]], [1]) == [del: [[1]], ins: [1]] + end + + test "rearranges inserts and equals for smaller diffs" do + assert List.myers_difference([3, 2, 0, 2], [2, 2, 0, 2]) == + [del: [3], ins: [2], eq: [2, 0, 2]] + + assert List.myers_difference([3, 2, 1, 0, 2], [2, 1, 2, 1, 0, 2]) == + [del: [3], ins: [2, 1], eq: [2, 1, 0, 2]] + + assert List.myers_difference([3, 2, 2, 1, 0, 2], [2, 2, 1, 2, 1, 0, 2]) == + [del: [3], eq: [2, 2, 1], ins: [2, 1], eq: [0, 2]] + + assert List.myers_difference([3, 2, 0, 2], [2, 2, 1, 0, 2]) == + [del: [3], eq: [2], ins: [2, 1], eq: [0, 2]] + end + end + + test "improper?/1" do + assert List.improper?([1 | 2]) + assert List.improper?([1, 2, 3 | 4]) + refute List.improper?([]) + refute List.improper?([1]) + refute List.improper?([[1]]) + refute List.improper?([1, 2]) + refute List.improper?([1, 2, 3]) + + assert_raise FunctionClauseError, fn -> + List.improper?(%{}) + end + end + + describe "ascii_printable?/2" do + test "proper lists without limit" do + assert List.ascii_printable?([]) + assert List.ascii_printable?(~c"abc") + refute(List.ascii_printable?(~c"abc" ++ [0])) + refute List.ascii_printable?(~c"mañana") + + printable_chars = ~c"\a\b\t\n\v\f\r\e" ++ Enum.to_list(32..126) + non_printable_chars = ~c"🌢áéíóúźç©¢🂭" + + assert List.ascii_printable?(printable_chars) + + for char <- printable_chars do + assert List.ascii_printable?([char]) + end + + refute List.ascii_printable?(non_printable_chars) + + for char <- non_printable_chars do + refute List.ascii_printable?([char]) + end + end + + test "proper lists with limit" do + assert List.ascii_printable?([], 100) + assert List.ascii_printable?(~c"abc" ++ [0], 2) + end + + test "improper lists" do + refute List.ascii_printable?(~c"abc" ++ ?d) + assert List.ascii_printable?(~c"abc" ++ ?d, 3) + end end end diff --git a/lib/elixir/test/elixir/macro/env_test.exs b/lib/elixir/test/elixir/macro/env_test.exs new file mode 100644 index 00000000000..1a62f3cab85 --- /dev/null +++ b/lib/elixir/test/elixir/macro/env_test.exs @@ -0,0 +1,261 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team + +Code.require_file("../test_helper.exs", __DIR__) + +defmodule MacroEnvMacros do + defmacro my_macro(arg) do + quote do: foo(unquote(arg)) + end + + @deprecated "this is deprecated" + defmacro my_deprecated_macro(arg), do: arg +end + +defmodule Macro.EnvTest do + use ExUnit.Case, async: true + + import Macro.Env + import ExUnit.CaptureIO + + alias List, as: CustomList, warn: false + import MacroEnvMacros, warn: false + + def trace(event, _env) do + send(self(), event) + :ok + end + + def meta, do: [file: "some_file.exs", line: 123] + def env, do: %{__ENV__ | tracers: [__MODULE__], line: 456} + + doctest Macro.Env + + test "inspect" do + assert inspect(__ENV__) =~ "#Macro.Env<" + end + + test "prune_compile_info" do + assert %Macro.Env{lexical_tracker: nil, tracers: []} = + Macro.Env.prune_compile_info(%{__ENV__ | lexical_tracker: self(), tracers: [Foo]}) + end + + test "stacktrace" do + env = %{__ENV__ | file: "foo", line: 12} + + assert Macro.Env.stacktrace(env) == + [{__MODULE__, :"test stacktrace", 1, [file: ~c"foo", line: 12]}] + + env = %{env | function: nil} + + assert Macro.Env.stacktrace(env) == [ + {__MODULE__, :__MODULE__, 0, [file: ~c"foo", line: 12]} + ] + + env = %{env | module: nil} + + assert Macro.Env.stacktrace(env) == + [{:elixir_compiler, :__FILE__, 1, [file: ~c"foo", line: 12]}] + end + + test "context modules" do + defmodule Foo.Bar do + assert __MODULE__ in __ENV__.context_modules + end + + assert Foo.Bar in __ENV__.context_modules + + Code.compile_string(""" + defmodule Foo.Bar.Compiled do + true = __MODULE__ in __ENV__.context_modules + end + """) + end + + test "to_match/1" do + quote = quote(do: x in []) + + assert {:__block__, [], [{:=, [], [{:_, [], Kernel}, {:x, [], Macro.EnvTest}]}, false]} = + Macro.expand_once(quote, __ENV__) + + assert Macro.expand_once(quote, Macro.Env.to_match(__ENV__)) == false + end + + test "prepend_tracer" do + assert %Macro.Env{tracers: [MyCustomTracer | _]} = + Macro.Env.prepend_tracer(__ENV__, MyCustomTracer) + end + + describe "define_import/4" do + test "with tracing" do + define_import(env(), meta(), List) + assert_received {:import, _, List, []} + + define_import(env(), meta(), Integer, only: :macros, trace: false) + refute_received {:import, _, Integer, _} + end + + test "with errors" do + message = + "invalid :only option for import, expected value to be an atom :functions, :macros, " <> + "or a literal keyword list of function names with arity as values, got: " + + assert define_import(env(), meta(), Integer, only: :unknown) == + {:error, message <> ":unknown"} + + assert define_import(env(), meta(), Integer, only: [:unknown]) == + {:error, message <> "[:unknown]"} + end + + test "with warnings" do + assert capture_io(:stderr, fn -> + define_import(env(), meta(), Integer, only: [is_odd: 1, is_odd: 1]) + end) =~ "invalid :only option for import, is_odd/1 is duplicated" + + assert {:ok, _env} = + define_import(env(), meta(), Integer, + only: [is_odd: 1, is_odd: 1], + emit_warnings: false + ) + end + end + + describe "expand_alias/4" do + test "with tracing" do + {:alias, List} = expand_alias(env(), meta(), [:CustomList]) + assert_received {:alias_expansion, _, Elixir.CustomList, List} + + {:alias, List} = expand_alias(env(), meta(), [:CustomList], trace: false) + refute_received {:alias_expansion, _, Elixir.CustomList, List} + + {:alias, List.Continues} = expand_alias(env(), meta(), [:CustomList, :Continues]) + assert_received {:alias_expansion, _, Elixir.CustomList, List} + end + end + + describe "expand_require/6" do + test "returns :error for functions and unknown modules" do + assert :error = expand_require(env(), meta(), List, :flatten, 1) + assert :error = expand_require(env(), meta(), Unknown, :flatten, 1) + end + + test "returns :error for unrequired modules" do + assert :error = expand_require(env(), meta(), Integer, :is_odd, 1) + end + + test "expands required modules" do + assert {:macro, Integer, _} = + expand_require(env(), [required: true] ++ meta(), Integer, :is_odd, 1) + + assert {:macro, Integer, _} = + expand_require(%{env() | requires: [Integer]}, meta(), Integer, :is_odd, 1) + + assert {:macro, Integer, _} = + expand_require(%{env() | module: Integer}, meta(), Integer, :is_odd, 1) + end + + test "expands with argument" do + {:macro, MacroEnvMacros, fun} = expand_require(env(), meta(), MacroEnvMacros, :my_macro, 1) + assert fun.([], [quote(do: hello())]) == quote(do: foo(hello())) + assert fun.([line: 789], [quote(do: hello())]) == quote(line: 789, do: foo(hello())) + + # do not propagate generated: true to arguments + assert {:foo, outer_meta, [{:hello, inner_meta, []}]} = + fun.([generated: true], [quote(do: hello())]) + + assert outer_meta[:generated] + refute inner_meta[:generated] + end + + test "with tracing and deprecations" do + message = "MacroEnvMacros.my_deprecated_macro/1 is deprecated" + + {:macro, MacroEnvMacros, fun} = + expand_require(env(), meta(), MacroEnvMacros, :my_deprecated_macro, 1) + + assert capture_io(:stderr, fn -> fun.([], [quote(do: hello())]) end) =~ message + assert_received {:remote_macro, _, MacroEnvMacros, :my_deprecated_macro, 1} + + {:macro, MacroEnvMacros, fun} = + expand_require(env(), meta(), MacroEnvMacros, :my_deprecated_macro, 1, + trace: false, + check_deprecations: false + ) + + refute capture_io(:stderr, fn -> fun.([], [quote(do: hello())]) end) =~ message + refute_received {:remote_macro, _, MacroEnvMacros, :my_deprecated_macro, 1} + end + end + + describe "expand_import/5" do + test "returns tagged :error for unknown imports" do + assert {:error, :not_found} = expand_import(env(), meta(), :flatten, 1) + end + + test "returns tagged :error for special forms" do + assert {:error, :not_found} = expand_import(env(), meta(), :case, 1) + end + + test "returns tagged :error for ambiguous" do + import Date, warn: false + import Time, warn: false + assert {:error, {:ambiguous, mods}} = expand_import(__ENV__, meta(), :new, 3) + assert Enum.sort(mods) == [Date, Time] + end + + test "returns :function tuple" do + assert {:function, ExUnit.CaptureIO, :capture_io} = + expand_import(env(), meta(), :capture_io, 1) + end + + test "expands with argument" do + {:macro, MacroEnvMacros, fun} = expand_import(env(), meta(), :my_macro, 1) + assert fun.([], [quote(do: hello())]) == quote(do: foo(hello())) + assert fun.([line: 789], [quote(do: hello())]) == quote(line: 789, do: foo(hello())) + + # do not propagate generated: true to arguments + assert {:foo, outer_meta, [{:hello, inner_meta, []}]} = + fun.([generated: true], [quote(do: hello())]) + + assert outer_meta[:generated] + refute inner_meta[:generated] + end + + defmacro allow_locals_example, do: :ok + + test "allow_locals" do + {:macro, Macro.EnvTest, fun} = + expand_import(env(), meta(), :allow_locals_example, 0) + + assert fun.([], []) == :ok + + assert expand_import(env(), meta(), :allow_locals_example, 0, allow_locals: false) == + {:error, :not_found} + + assert expand_import(env(), meta(), :allow_locals_example, 0, + allow_locals: fn -> send(self(), false) end + ) == + {:error, :not_found} + + assert_received false + end + + test "with tracing and deprecations" do + message = "MacroEnvMacros.my_deprecated_macro/1 is deprecated" + + {:macro, MacroEnvMacros, fun} = expand_import(env(), meta(), :my_deprecated_macro, 1) + + assert capture_io(:stderr, fn -> fun.([], [quote(do: hello())]) end) =~ message + assert_received {:imported_macro, _, MacroEnvMacros, :my_deprecated_macro, 1} + + {:macro, MacroEnvMacros, fun} = + expand_import(env(), meta(), :my_deprecated_macro, 1, + trace: false, + check_deprecations: false + ) + + refute capture_io(:stderr, fn -> fun.([], [quote(do: hello())]) end) =~ message + refute_received {:imported_macro, _, MacroEnvMacros, :my_deprecated_macro, 1} + end + end +end diff --git a/lib/elixir/test/elixir/macro_test.exs b/lib/elixir/test/elixir/macro_test.exs index 9a1de1c6130..1216af53224 100644 --- a/lib/elixir/test/elixir/macro_test.exs +++ b/lib/elixir/test/elixir/macro_test.exs @@ -1,4 +1,8 @@ -Code.require_file "test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("test_helper.exs", __DIR__) defmodule Macro.ExternalTest do defmacro external do @@ -11,520 +15,1899 @@ defmodule Macro.ExternalTest do end defmacro oror(left, right) do - quote do: unquote(left) || unquote(right) + quote(do: unquote(left) || unquote(right)) + end +end + +defmodule CustomIf do + def if(_cond, _expr) do + "custom if result" end end defmodule MacroTest do use ExUnit.Case, async: true + doctest Macro # Changing the lines above will make compilation # fail since we are asserting on the caller lines import Macro.ExternalTest - ## Escape + describe "escape/2" do + test "returns tuples with size equal to two" do + assert Macro.escape({:a, :b}) == {:a, :b} + end - test :escape_handle_tuples_with_size_different_than_two do - assert {:{}, [], [:a]} == Macro.escape({:a}) - assert {:{}, [], [:a, :b, :c]} == Macro.escape({:a, :b, :c}) - assert {:{}, [], [:a, {:{}, [], [1,2,3]}, :c]} == Macro.escape({:a, {1, 2, 3}, :c}) - end + test "returns lists" do + assert Macro.escape([1, 2, 3]) == [1, 2, 3] + end - test :escape_simply_returns_tuples_with_size_equal_to_two do - assert {:a, :b} == Macro.escape({:a, :b}) - end + test "escapes tuples with size different than two" do + assert Macro.escape({:a}) == {:{}, [], [:a]} + assert Macro.escape({:a, :b, :c}) == {:{}, [], [:a, :b, :c]} + assert Macro.escape({:a, {1, 2, 3}, :c}) == {:{}, [], [:a, {:{}, [], [1, 2, 3]}, :c]} - test :escape_simply_returns_any_other_structure do - assert [1, 2, 3] == Macro.escape([1, 2, 3]) - end + # False positives + assert Macro.escape({:quote, :foo, [:bar]}) == {:{}, [], [:quote, :foo, [:bar]]} + assert Macro.escape({:quote, :foo, [:bar, :baz]}) == {:{}, [], [:quote, :foo, [:bar, :baz]]} + end - test :escape_handles_maps do - assert {:%{}, [], [a: 1]} = Macro.escape(%{a: 1}) - end + test "escapes maps" do + assert Macro.escape(%{a: 1}) == {:%{}, [], [a: 1]} + end - test :escape_works_recursively do - assert [1,{:{}, [], [:a,:b,:c]}, 3] == Macro.escape([1, {:a, :b, :c}, 3]) - end + test "escapes bitstring" do + assert {:<<>>, [], args} = Macro.escape(<<300::12>>) + assert [{:"::", [], [1, {:size, [], [4]}]}, {:"::", [], [",", {:binary, [], nil}]}] = args + end - test :escape_improper do - assert [{:|, [], [1,2]}] == Macro.escape([1|2]) - assert [1,{:|, [], [2,3]}] == Macro.escape([1,2|3]) - end + test "escapes recursively" do + assert Macro.escape([1, {:a, :b, :c}, 3]) == [1, {:{}, [], [:a, :b, :c]}, 3] + end - test :escape_with_unquote do - contents = quote unquote: false, do: unquote(1) - assert Macro.escape(contents, unquote: true) == 1 + test "escapes improper lists" do + assert Macro.escape([1 | 2]) == [{:|, [], [1, 2]}] + assert Macro.escape([1, 2 | 3]) == [1, {:|, [], [2, 3]}] + end - contents = quote unquote: false, do: unquote(x) - assert Macro.escape(contents, unquote: true) == {:x, [], MacroTest} - end + test "prunes metadata" do + meta = [nothing: :important, counter: 1] + assert Macro.escape({:foo, meta, []}) == {:{}, [], [:foo, meta, []]} + assert Macro.escape({:foo, meta, []}, prune_metadata: true) == {:{}, [], [:foo, [], []]} + end - defp eval_escaped(contents) do - {eval, []} = Code.eval_quoted(Macro.escape(contents, unquote: true)) - eval - end + test "with unquote" do + contents = quote(unquote: false, do: unquote(1)) + assert Macro.escape(contents, unquote: true) == 1 - test :escape_with_remote_unquote do - contents = quote unquote: false, do: Kernel.unquote(:is_atom)(:ok) - assert eval_escaped(contents) == quote(do: Kernel.is_atom(:ok)) - end + contents = quote(unquote: false, do: unquote(x)) + assert Macro.escape(contents, unquote: true) == {:x, [], MacroTest} - test :escape_with_nested_unquote do - contents = quote do - quote do: unquote(x) + contents = %{foo: quote(unquote: false, do: unquote(1))} + assert Macro.escape(contents, unquote: true) == {:%{}, [], [foo: 1]} end - assert eval_escaped(contents) == quote do: (quote do: unquote(x)) - end - test :escape_with_alias_or_no_args_remote_unquote do - contents = quote unquote: false, do: Kernel.unquote(:self) - assert eval_escaped(contents) == quote(do: Kernel.self()) + test "with generated" do + assert Macro.escape(%{a: {}}, generated: true) == + {:%{}, [generated: true], [a: {:{}, [], []}]} + end - contents = quote unquote: false, do: x.unquote(Foo) - assert eval_escaped(contents) == quote(do: x.unquote(Foo)) - end + defp eval_escaped(contents) do + {eval, []} = Code.eval_quoted(Macro.escape(contents, unquote: true)) + eval + end - test :escape_with_splicing do - contents = quote unquote: false, do: [1, 2, 3, 4, 5] - assert Macro.escape(contents, unquote: true) == [1, 2, 3, 4, 5] + test "with remote unquote" do + contents = quote(unquote: false, do: Kernel.unquote(:is_atom)(:ok)) + assert eval_escaped(contents) == quote(do: Kernel.is_atom(:ok)) + assert eval_escaped(%{foo: contents}) == %{foo: quote(do: Kernel.is_atom(:ok))} + end - contents = quote unquote: false, do: [1, 2, unquote_splicing([3, 4, 5])] - assert eval_escaped(contents) == [1, 2, 3, 4, 5] + test "with nested unquote" do + contents = + quote do + quote(do: unquote(x)) + end - contents = quote unquote: false, do: [unquote_splicing([1, 2, 3]), 4, 5] - assert eval_escaped(contents) == [1, 2, 3, 4, 5] + assert eval_escaped(contents) == quote(do: quote(do: unquote(x))) + end - contents = quote unquote: false, do: [unquote_splicing([1, 2, 3]), unquote_splicing([4, 5])] - assert eval_escaped(contents) == [1, 2, 3, 4, 5] + test "with alias or no arguments remote unquote" do + contents = quote(unquote: false, do: Kernel.unquote(:self)()) + assert eval_escaped(contents) == quote(do: Kernel.self()) - contents = quote unquote: false, do: [1, unquote_splicing([2]), 3, unquote_splicing([4]), 5] - assert eval_escaped(contents) == [1, 2, 3, 4, 5] + contents = quote(unquote: false, do: x.unquote(Foo)) + assert eval_escaped(contents) == quote(do: x.unquote(Foo)) + end - contents = quote unquote: false, do: [1, unquote_splicing([2]), 3, unquote_splicing([4])|[5]] - assert eval_escaped(contents) == [1, 2, 3, 4, 5] - end + test "with splicing" do + contents = quote(unquote: false, do: [1, 2, 3, 4, 5]) + assert Macro.escape(contents, unquote: true) == [1, 2, 3, 4, 5] - ## Expansion + contents = quote(unquote: false, do: [1, 2, unquote_splicing([3, 4, 5])]) + assert eval_escaped(contents) == [1, 2, 3, 4, 5] - test :expand_once do - assert {:||, _, _} = Macro.expand_once(quote(do: oror(1, false)), __ENV__) - end + contents = quote(unquote: false, do: [unquote_splicing([1, 2, 3]), 4, 5]) + assert eval_escaped(contents) == [1, 2, 3, 4, 5] - test :expand_once_with_raw_atom do - assert Macro.expand_once(quote(do: :foo), __ENV__) == :foo - end + contents = + quote(unquote: false, do: [unquote_splicing([1, 2, 3]), unquote_splicing([4, 5])]) - test :expand_once_with_current_module do - assert Macro.expand_once(quote(do: __MODULE__), __ENV__) == __MODULE__ - end + assert eval_escaped(contents) == [1, 2, 3, 4, 5] - test :expand_once_with_main do - assert Macro.expand_once(quote(do: Elixir), __ENV__) == Elixir - end + contents = + quote(unquote: false, do: [1, unquote_splicing([2]), 3, unquote_splicing([4]), 5]) - test :expand_once_with_simple_alias do - assert Macro.expand_once(quote(do: Foo), __ENV__) == Foo - end + assert eval_escaped(contents) == [1, 2, 3, 4, 5] - test :expand_once_with_current_module_plus_alias do - assert Macro.expand_once(quote(do: __MODULE__.Foo), __ENV__) == __MODULE__.Foo - end + contents = + quote(unquote: false, do: [1, unquote_splicing([2]), 3, unquote_splicing([4]) | [5]]) - test :expand_once_with_main_plus_alias do - assert Macro.expand_once(quote(do: Elixir.Foo), __ENV__) == Foo - end + assert eval_escaped(contents) == [1, 2, 3, 4, 5] - test :expand_once_with_custom_alias do - alias Foo, as: Bar - assert Macro.expand_once(quote(do: Bar.Baz), __ENV__) == Foo.Baz - end + contents = %{foo: quote(unquote: false, do: [1, 2, unquote_splicing([3, 4, 5])])} + assert eval_escaped(contents) == %{foo: [1, 2, 3, 4, 5]} + end - test :expand_once_with_main_plus_custom_alias do - alias Foo, as: Bar, warn: false - assert Macro.expand_once(quote(do: Elixir.Bar.Baz), __ENV__) == Elixir.Bar.Baz - end + test "does not add context to quote" do + assert Macro.escape({:quote, [], [[do: :foo]]}) == {:{}, [], [:quote, [], [[do: :foo]]]} + end - test :expand_once_with_op do - assert Macro.expand_once(quote(do: Foo.bar.Baz), __ENV__) == (quote do - Foo.bar.Baz - end) - end + test "escapes the content of :quote tuples" do + assert Macro.escape({:quote, [%{}], [{}]}) == + {:{}, [], [:quote, [{:%{}, [], []}], [{:{}, [], []}]]} - test :expand_once_with_erlang do - assert Macro.expand_once(quote(do: :foo), __ENV__) == :foo - end + assert Macro.escape([:foo, {:quote, [%{}], [{}]}]) == + [:foo, {:{}, [], [:quote, [{:%{}, [], []}], [{:{}, [], []}]]}] + end - test :expand_once_env do - env = %{__ENV__ | line: 0} - assert Macro.expand_once(quote(do: __ENV__), env) == {:%{}, [], Map.to_list(env)} - assert Macro.expand_once(quote(do: __ENV__.file), env) == env.file - assert Macro.expand_once(quote(do: __ENV__.unknown), env) == quote(do: __ENV__.unknown) - end + @tag :re_import + test "escape regex will remove references and replace it by a call to :re.import/1" do + assert { + :%{}, + [], + [ + __struct__: Regex, + re_pattern: + {{:., [], [{:__aliases__, _, [:Regex]}, :__import_pattern__]}, + [required: true], [{:{}, [], [:re_exported_pattern | _]}]}, + source: "foo", + opts: [] + ] + } = Macro.escape(~r/foo/) + end - defmacro local_macro do - :local_macro - end + @tag :re_import + test "escape raises within structs fields" do + assert_raise ArgumentError, + ~r"Regex defines custom escaping rules which are not supported in struct defaults", + fn -> + defmodule Test do + defstruct my_regex: ~r/^hi$/ + end + end + end - test :expand_once_local_macro do - assert Macro.expand_once(quote(do: local_macro), __ENV__) == :local_macro - end + defmodule EscapedStruct do + defstruct [:ast, :ref] - test :expand_once_checks_vars do - local_macro = 1 - assert local_macro == 1 - quote = {:local_macro, [], nil} - assert Macro.expand_once(quote, __ENV__) == quote - end + def __escape__(%{ast: ast}), do: ast + end - defp expand_once_and_clean(quoted, env) do - cleaner = &Keyword.drop(&1, [:counter]) - quoted - |> Macro.expand_once(env) - |> Macro.prewalk(&Macro.update_meta(&1, cleaner)) - end + test "escape struct with custom __escape__ (valid AST)" do + struct = %EscapedStruct{ast: {:valid_ast, [], []}, ref: make_ref()} - test :expand_once_with_imported_macro do - temp_var = {:x, [], Kernel} - assert expand_once_and_clean(quote(do: 1 || false), __ENV__) == (quote context: Kernel do - case 1 do - unquote(temp_var) when unquote(temp_var) in [false, nil] -> false - unquote(temp_var) -> unquote(temp_var) - end - end) - end + assert {:valid_ast, [], []} = Macro.escape(struct) + end - test :expand_once_with_require_macro do - temp_var = {:x, [], Kernel} - assert expand_once_and_clean(quote(do: Kernel.||(1, false)), __ENV__) == (quote context: Kernel do - case 1 do - unquote(temp_var) when unquote(temp_var) in [false, nil] -> false - unquote(temp_var) -> unquote(temp_var) - end - end) - end + test "escape struct with custom __escape__ (shallow invalid AST)" do + struct = %EscapedStruct{ast: %{invalid: :ast}, ref: make_ref()} - test :expand_once_with_not_expandable_expression do - expr = quote(do: other(1, 2, 3)) - assert Macro.expand_once(expr, __ENV__) == expr + assert_raise ArgumentError, + "MacroTest.EscapedStruct.__escape__/1 returned invalid AST: %{invalid: :ast}", + fn -> Macro.escape(struct) end + end end - @foo 1 - @bar Macro.expand_once(quote(do: @foo), __ENV__) + describe "expand_once/2" do + test "with external macro" do + assert {:||, _, [1, false]} = Macro.expand_once(quote(do: oror(1, false)), __ENV__) + end + + test "with raw atom" do + assert Macro.expand_once(quote(do: :foo), __ENV__) == :foo + end + + test "with current module" do + assert Macro.expand_once(quote(do: __MODULE__), __ENV__) == __MODULE__ + end + + test "with main" do + assert Macro.expand_once(quote(do: Elixir), __ENV__) == Elixir + end + + test "with simple alias" do + assert Macro.expand_once(quote(do: Foo), __ENV__) == Foo + end + + test "with current module plus alias" do + assert Macro.expand_once(quote(do: __MODULE__.Foo), __ENV__) == __MODULE__.Foo + end + + test "with main plus alias" do + assert Macro.expand_once(quote(do: Elixir.Foo), __ENV__) == Foo + end - test :expand_once_with_module_at do - assert @bar == 1 + test "with custom alias" do + alias Foo, as: Bar + assert Macro.expand_once(quote(do: Bar.Baz), __ENV__) == Foo.Baz + end + + test "with main plus custom alias" do + alias Foo, as: Bar, warn: false + assert Macro.expand_once(quote(do: Elixir.Bar.Baz), __ENV__) == Elixir.Bar.Baz + end + + test "with call in alias" do + assert Macro.expand_once(quote(do: Foo.bar().Baz), __ENV__) == quote(do: Foo.bar().Baz) + end + + test "env" do + env = %{__ENV__ | line: 0, lexical_tracker: self()} + + expanded = Macro.expand_once(quote(do: __ENV__), env) + assert Macro.validate(expanded) == :ok + assert Code.eval_quoted(expanded) == {env, []} + + assert Macro.expand_once(quote(do: __ENV__.file), env) == env.file + assert Macro.expand_once(quote(do: __ENV__.unknown), env) == quote(do: __ENV__.unknown) + + expanded = Macro.expand_once(quote(do: __ENV__.versioned_vars), env) + assert Macro.validate(expanded) == :ok + assert Code.eval_quoted(expanded) == {env.versioned_vars, []} + end + + test "env in :match context does not expand" do + env = %{__ENV__ | line: 0, lexical_tracker: self(), context: :match} + + expanded = Macro.expand_once(quote(do: __ENV__), env) + assert expanded == quote(do: __ENV__) + + expanded = Macro.expand_once(quote(do: __ENV__.file), env) + assert expanded == quote(do: __ENV__.file) + end + + defmacro local_macro(), do: raise("ignored") + + test "vars" do + expr = {:local_macro, [], nil} + assert Macro.expand_once(expr, __ENV__) == expr + end + + defp expand_once_and_clean(quoted, env) do + cleaner = &Keyword.drop(&1, [:counter, :type_check]) + + quoted + |> Macro.expand_once(env) + |> Macro.prewalk(&Macro.update_meta(&1, cleaner)) + end + + test "with imported macro" do + temp_var = {:x, [], Kernel} + + quoted = + quote context: Kernel do + case 1 do + unquote(temp_var) when :"Elixir.Kernel".in(unquote(temp_var), [false, nil]) -> false + unquote(temp_var) -> unquote(temp_var) + end + end + + assert expand_once_and_clean(quote(do: 1 || false), __ENV__) == quoted + end + + test "with require macro" do + temp_var = {:x, [], Kernel} + + quoted = + quote context: Kernel do + case 1 do + unquote(temp_var) when :"Elixir.Kernel".in(unquote(temp_var), [false, nil]) -> false + unquote(temp_var) -> unquote(temp_var) + end + end + + assert expand_once_and_clean(quote(do: Kernel.||(1, false)), __ENV__) == quoted + end + + test "with not expandable expression" do + expr = quote(do: other(1, 2, 3)) + assert Macro.expand_once(expr, __ENV__) == expr + end + + test "propagates :generated" do + assert {:||, meta, [1, false]} = Macro.expand_once(quote(do: oror(1, false)), __ENV__) + refute meta[:generated] + + assert {:||, meta, [1, false]} = + Macro.expand_once(quote(generated: true, do: oror(1, false)), __ENV__) + + assert meta[:generated] + end + + test "does not propagate :generated to unquoted" do + non_generated = quote do: foo() + + assert {:||, outer_meta, [{:foo, inner_meta, []}, false]} = + Macro.expand_once( + quote generated: true do + oror(unquote(non_generated), false) + end, + __ENV__ + ) + + assert outer_meta[:generated] + refute inner_meta[:generated] + end + + defmacro foo_bar(x) do + y = quote do: bar(unquote(x)) + + quote do: foo(unquote(y)) + end + + test "propagates :generated to unquote within macro" do + non_generated = quote do: baz() + + assert {:foo, foo_meta, [{:bar, bar_meta, [{:baz, baz_meta, []}]}]} = + Macro.expand_once( + quote(generated: true, do: foo_bar(unquote(non_generated))), + __ENV__ + ) + + assert foo_meta[:generated] + assert bar_meta[:generated] + refute baz_meta[:generated] + end + + test "does not expand module attributes" do + message = + "could not call Module.get_attribute/2 because the module #{inspect(__MODULE__)} " <> + "is already compiled. Use the Module.__info__/1 callback or Code.fetch_docs/1 instead" + + assert_raise ArgumentError, message, fn -> + Macro.expand_once(quote(do: @foo), __ENV__) + end + end end defp expand_and_clean(quoted, env) do - cleaner = &Keyword.drop(&1, [:counter]) + cleaner = &Keyword.drop(&1, [:counter, :type_check]) + quoted |> Macro.expand(env) |> Macro.prewalk(&Macro.update_meta(&1, cleaner)) end - test :expand do + test "expand/2" do temp_var = {:x, [], Kernel} - assert expand_and_clean(quote(do: oror(1, false)), __ENV__) == (quote context: Kernel do - case 1 do - unquote(temp_var) when unquote(temp_var) in [false, nil] -> false - unquote(temp_var) -> unquote(temp_var) + + quoted = + quote context: Kernel do + case 1 do + unquote(temp_var) when :"Elixir.Kernel".in(unquote(temp_var), [false, nil]) -> false + unquote(temp_var) -> unquote(temp_var) + end end - end) + + assert expand_and_clean(quote(do: oror(1, false)), __ENV__) == quoted end - test :var do - assert Macro.var(:foo, nil) == {:foo, [], nil} - assert Macro.var(:foo, Other) == {:foo, [], Other} + test "expand_literals/2" do + assert Macro.expand_literals(quote(do: Foo), __ENV__) == Foo + assert Macro.expand_literals(quote(do: Foo + Bar), __ENV__) == quote(do: Foo + Bar) + assert Macro.expand_literals(quote(do: __MODULE__), __ENV__) == __MODULE__ + assert Macro.expand_literals(quote(do: __MODULE__.Foo), __ENV__) == __MODULE__.Foo + assert Macro.expand_literals(quote(do: [Foo, 1 + 2]), __ENV__) == [Foo, quote(do: 1 + 2)] end - ## to_string + test "expand_literals/3" do + fun = fn node, acc -> + expanded = Macro.expand(node, __ENV__) + {expanded, [expanded | acc]} + end - test :var_to_string do - assert Macro.to_string(quote do: foo) == "foo" - end + assert Macro.expand_literals(quote(do: Foo), [], fun) == {Foo, [Foo]} + assert Macro.expand_literals(quote(do: Foo + Bar), [], fun) == {quote(do: Foo + Bar), []} + assert Macro.expand_literals(quote(do: __MODULE__), [], fun) == {__MODULE__, [__MODULE__]} - test :local_call_to_string do - assert Macro.to_string(quote do: foo(1, 2, 3)) == "foo(1, 2, 3)" - assert Macro.to_string(quote do: foo([1, 2, 3])) == "foo([1, 2, 3])" + assert Macro.expand_literals(quote(do: __MODULE__.Foo), [], fun) == + {__MODULE__.Foo, [__MODULE__.Foo, __MODULE__]} end - test :remote_call_to_string do - assert Macro.to_string(quote do: foo.bar(1, 2, 3)) == "foo.bar(1, 2, 3)" - assert Macro.to_string(quote do: foo.bar([1, 2, 3])) == "foo.bar([1, 2, 3])" + test "var/2" do + assert Macro.var(:foo, nil) == {:foo, [], nil} + assert Macro.var(:foo, Other) == {:foo, [], Other} end - test :low_atom_remote_call_to_string do - assert Macro.to_string(quote do: :foo.bar(1, 2, 3)) == ":foo.bar(1, 2, 3)" - end + describe "dbg/3" do + defmacrop dbg_format_no_newline(ast, options \\ quote(do: [syntax_colors: []])) do + quote do + ExUnit.CaptureIO.with_io(fn -> + try do + unquote(Macro.dbg(ast, options, __CALLER__)) + rescue + e -> e + end + end) + end + end - test :big_atom_remote_call_to_string do - assert Macro.to_string(quote do: Foo.Bar.bar(1, 2, 3)) == "Foo.Bar.bar(1, 2, 3)" - end + defmacrop dbg_format(ast, options \\ quote(do: [syntax_colors: []])) do + quote do + {result, formatted} = + dbg_format_no_newline(unquote(ast), unquote(options)) - test :remote_and_fun_call_to_string do - assert Macro.to_string(quote do: foo.bar.(1, 2, 3)) == "foo.bar().(1, 2, 3)" - assert Macro.to_string(quote do: foo.bar.([1, 2, 3])) == "foo.bar().([1, 2, 3])" - end + # Make sure there's an empty line after the output. + assert String.ends_with?(formatted, "\n\n") or + String.ends_with?(formatted, "\n\n" <> IO.ANSI.reset()) - test :atom_call_to_string do - assert Macro.to_string(quote do: :foo.(1, 2, 3)) == ":foo.(1, 2, 3)" - end + {result, formatted} + end + end - test :aliases_call_to_string do - assert Macro.to_string(quote do: Foo.Bar.baz(1, 2, 3)) == "Foo.Bar.baz(1, 2, 3)" - assert Macro.to_string(quote do: Foo.Bar.baz([1, 2, 3])) == "Foo.Bar.baz([1, 2, 3])" - end + test "simple expression" do + {result, formatted} = dbg_format(1 + 1) + assert result == 2 + assert formatted =~ "1 + 1 #=> 2" + end - test :arrow_to_string do - assert Macro.to_string(quote do: foo(1, (2 -> 3))) == "foo(1, (2 -> 3))" - end + test "variables" do + my_var = 1 + 1 + {result, formatted} = dbg_format(my_var) + assert result == 2 + assert formatted =~ "my_var #=> 2" + end - test :blocks_to_string do - assert Macro.to_string(quote do: (1; 2; (:foo; :bar); 3)) <> "\n" == """ - ( - 1 - 2 - ( - :foo - :bar - ) - 3 - ) - """ + test "function call" do + {result, formatted} = dbg_format(Atom.to_string(:foo)) + + assert result == "foo" + assert formatted =~ ~s[Atom.to_string(:foo) #=> "foo"] + end + + test "multiline input" do + {result, formatted} = + dbg_format( + case 1 + 1 do + 2 -> :two + _other -> :math_is_broken + end + ) + + assert result == :two + + assert formatted =~ """ + case 1 + 1 do + 2 -> :two + _other -> :math_is_broken + end #=> :two + """ + end + + defp abc, do: [:a, :b, :c] + + test "pipeline on a single line" do + {result, formatted} = dbg_format(abc() |> tl() |> tl |> Kernel.hd()) + assert result == :c + + assert formatted =~ "macro_test.exs" + + assert formatted =~ """ + \nabc() #=> [:a, :b, :c] + |> tl() #=> [:b, :c] + |> tl #=> [:c] + |> Kernel.hd() #=> :c + """ + + # Regression for pipes sometimes erroneously ending with three newlines (one + # extra than needed). + assert formatted =~ ~r/[^\n]\n\n$/ + end + + test "pipeline on multiple lines" do + {result, formatted} = + dbg_format( + abc() + |> tl() + |> tl + |> Kernel.hd() + ) + + assert result == :c + + assert formatted =~ "macro_test.exs" + + assert formatted =~ """ + \nabc() #=> [:a, :b, :c] + |> tl() #=> [:b, :c] + |> tl #=> [:c] + |> Kernel.hd() #=> :c + """ + + # Regression for pipes sometimes erroneously ending with three newlines (one + # extra than needed). + assert formatted =~ ~r/[^\n]\n\n$/ + end + + test "pipeline on multiple lines that raises" do + {result, formatted} = + dbg_format_no_newline( + abc() + |> tl() + |> tl() + |> tl() + |> tl() + ) + + assert %ArgumentError{} = result + + assert formatted =~ "macro_test.exs" + + assert formatted =~ """ + abc() #=> [:a, :b, :c] + |> tl() #=> [:b, :c] + |> tl() #=> [:c] + |> tl() #=> [] + """ + end + + test "simple boolean expressions" do + {result, formatted} = dbg_format(:rand.uniform() < 0.0 and length([]) == 0) + assert result == false + + assert formatted =~ "macro_test.exs" + + assert formatted =~ """ + :rand.uniform() < 0.0 #=> false + :rand.uniform() < 0.0 and length([]) == 0 #=> false + """ + end + + test "left-associative operators" do + {result, formatted} = dbg_format(List.first([]) || "yes" || raise("foo")) + assert result == "yes" + + assert formatted =~ "macro_test.exs" + + assert formatted =~ """ + List.first([]) #=> nil + List.first([]) || "yes" #=> "yes" + List.first([]) || "yes" || raise "foo" #=> "yes" + """ + end + + test "composite boolean expressions" do + true1 = length([]) == 0 + true2 = length([]) == 0 + {result, formatted} = dbg_format((true1 and true2) or (List.first([]) || true1)) + + assert result == true + + assert formatted =~ "macro_test.exs" + + assert formatted =~ """ + true1 #=> true + true1 and true2 #=> true + (true1 and true2) or (List.first([]) || true1) #=> true + """ + end + + test "boolean expressions that raise" do + falsy = length([]) != 0 + x = 0 + + {result, formatted} = dbg_format_no_newline((falsy || x) && 1 / x) + + assert %ArithmeticError{} = result + + assert formatted =~ "macro_test.exs" + + assert formatted =~ """ + falsy #=> false + falsy || x #=> 0 + """ + end + + test "block of code" do + {result, formatted} = + dbg_format( + ( + a = 1 + b = a + 2 + a + b + ) + ) + + assert result == 4 + + assert formatted =~ "macro_test.exs" + + assert formatted =~ """ + a = 1 #=> 1 + b = a + 2 #=> 3 + a + b #=> 4 + """ + end + + test "block that raises" do + {result, formatted} = + dbg_format_no_newline( + ( + a = 1 + b = a - 1 + a / b + ) + ) + + assert result == %ArithmeticError{} + + assert formatted =~ "macro_test.exs" + + assert formatted =~ """ + a = 1 #=> 1 + b = a - 1 #=> 0 + """ + end + + test "case" do + list = List.flatten([1, 2, 3]) + + {result, formatted} = + dbg_format( + case list do + [] -> nil + _ -> Enum.sum(list) + end + ) + + assert result == 6 + + assert formatted =~ "macro_test.exs" + + assert formatted =~ """ + Case argument: + list #=> [1, 2, 3] + + Case expression (clause #2 matched): + case list do + [] -> nil + _ -> Enum.sum(list) + end #=> 6 + """ + end + + test "case that raises" do + x = true + + {result, formatted} = + dbg_format( + case true and x do + true -> raise "oops" + false -> :ok + end + ) + + assert %RuntimeError{} = result + + assert formatted =~ "macro_test.exs" + + assert formatted =~ """ + Case argument: + true and x #=> true + """ + end + + test "case with guards" do + {result, formatted} = + dbg_format( + case 0..100//5 do + %{first: first, last: last, step: step} when last > first -> + count = div(last - first, step) + {:ok, count} + + _ -> + :error + end + ) + + assert result == {:ok, 20} + + assert formatted =~ "macro_test.exs" + + assert formatted =~ """ + Case argument: + 0..100//5 #=> 0..100//5 + + Case expression (clause #1 matched): + case 0..100//5 do + %{first: first, last: last, step: step} when last > first -> + count = div(last - first, step) + {:ok, count} + + _ -> + :error + end #=> {:ok, 20} + """ + end + + test "cond" do + map = %{b: 5} + + {result, formatted} = + dbg_format( + cond do + a = map[:a] -> a + 1 + b = map[:b] -> b * 2 + true -> nil + end + ) + + assert result == 10 + + assert formatted =~ "macro_test.exs" + + assert formatted =~ """ + Cond clause (clause #2 matched): + b = map[:b] #=> 5 + + Cond expression: + cond do + a = map[:a] -> a + 1 + b = map[:b] -> b * 2 + true -> nil + end #=> 10 + """ + end + + test "if expression" do + x = true + map = %{a: 5, b: 1} + + {result, formatted} = + dbg_format( + if true and x do + map[:a] * 2 + else + map[:b] + end + ) + + assert result == 10 + + assert formatted =~ "macro_test.exs" + + assert formatted =~ """ + If condition: + true and x #=> true + + If expression: + if true and x do + map[:a] * 2 + else + map[:b] + end #=> 10 + """ + end + + test "if expression that raises" do + x = true + + {result, formatted} = + dbg_format( + if true and x do + raise "oops" + end + ) + + assert %RuntimeError{} = result + + assert formatted =~ "macro_test.exs" + + assert formatted =~ """ + If condition: + true and x #=> true + """ + end + + test "if expression without else" do + x = true + map = %{a: 5, b: 1} + + {result, formatted} = + dbg_format( + if false and x do + map[:a] * 2 + end + ) + + assert result == nil + + assert formatted =~ "macro_test.exs" + + assert formatted =~ """ + If condition: + false and x #=> false + + If expression: + if false and x do + map[:a] * 2 + end #=> nil + """ + end + + test "custom if definition" do + import Kernel, except: [if: 2] + import CustomIf, only: [if: 2] + + {result, formatted} = + dbg_format( + if true do + "something" + end + ) + + assert result == "custom if result" + + assert formatted =~ """ + if true do + "something" + end #=> "custom if result" + """ + end + + test "with/1 (all clauses match)" do + opts = %{width: 10, height: 15} + + {result, formatted} = + dbg_format( + with {:ok, width} <- Map.fetch(opts, :width), + double_width = width * 2, + IO.puts("just a side effect"), + {:ok, height} <- Map.fetch(opts, :height) do + {:ok, double_width * height} + end + ) + + assert result == {:ok, 300} + + assert formatted =~ "macro_test.exs" + + assert formatted =~ """ + With clauses: + Map.fetch(opts, :width) #=> {:ok, 10} + width * 2 #=> 20 + Map.fetch(opts, :height) #=> {:ok, 15} + + With expression: + with {:ok, width} <- Map.fetch(opts, :width), + double_width = width * 2, + IO.puts("just a side effect"), + {:ok, height} <- Map.fetch(opts, :height) do + {:ok, double_width * height} + end #=> {:ok, 300} + """ + end + + test "with/1 (no else)" do + opts = %{width: 10} + + {result, formatted} = + dbg_format( + with {:ok, width} <- Map.fetch(opts, :width), + {:ok, height} <- Map.fetch(opts, :height) do + {:ok, width * height} + end + ) + + assert result == :error + + assert formatted =~ "macro_test.exs" + + assert formatted =~ """ + With clauses: + Map.fetch(opts, :width) #=> {:ok, 10} + Map.fetch(opts, :height) #=> :error + + With expression: + with {:ok, width} <- Map.fetch(opts, :width), + {:ok, height} <- Map.fetch(opts, :height) do + {:ok, width * height} + end #=> :error + """ + end + + test "with/1 (else clause)" do + opts = %{width: 10} + + {result, formatted} = + dbg_format( + with {:ok, width} <- Map.fetch(opts, :width), + {:ok, height} <- Map.fetch(opts, :height) do + width * height + else + :error -> 0 + end + ) + + assert result == 0 + assert formatted =~ "macro_test.exs" + + assert formatted =~ """ + With clauses: + Map.fetch(opts, :width) #=> {:ok, 10} + Map.fetch(opts, :height) #=> :error + + With expression: + with {:ok, width} <- Map.fetch(opts, :width), + {:ok, height} <- Map.fetch(opts, :height) do + width * height + else + :error -> 0 + end #=> 0 + """ + end + + test "with/1 (guard)" do + opts = %{width: 10, height: 0.0} + + {result, formatted} = + dbg_format( + with {:ok, width} when is_integer(width) <- Map.fetch(opts, :width), + {:ok, height} when is_integer(height) <- Map.fetch(opts, :height) do + width * height + else + _ -> nil + end + ) + + assert result == nil + assert formatted =~ "macro_test.exs" + + assert formatted =~ """ + With clauses: + Map.fetch(opts, :width) #=> {:ok, 10} + Map.fetch(opts, :height) #=> {:ok, 0.0} + + With expression: + with {:ok, width} when is_integer(width) <- Map.fetch(opts, :width), + {:ok, height} when is_integer(height) <- Map.fetch(opts, :height) do + width * height + else + _ -> nil + end #=> nil + """ + end + + test "with/1 (guard in else)" do + opts = %{} + + {result, _formatted} = + dbg_format( + with {:ok, width} <- Map.fetch(opts, :width) do + width + else + other when is_integer(other) -> :int + other when is_atom(other) -> :atom + end + ) + + assert result == :atom + end + + test "with/1 respects the WithClauseError" do + value = Enum.random([:unexpected]) + + error = + assert_raise WithClauseError, fn -> + ExUnit.CaptureIO.capture_io(fn -> + dbg( + with :ok <- value do + true + else + :error -> false + end + ) + end) + end + + assert error.term == :unexpected + end + + test "with \"syntax_colors: []\" it doesn't print any color sequences" do + {_result, formatted} = dbg_format("hello") + refute formatted =~ "\e[" + end + + test "with \"syntax_colors: [...]\" it forces color sequences" do + {_result, formatted} = dbg_format("hello", syntax_colors: [string: :cyan]) + assert formatted =~ IO.iodata_to_binary(IO.ANSI.format([:cyan, ~s("hello")])) + end + + test "forwards options to the underlying inspect calls" do + value = ~c"hello" + assert {^value, formatted} = dbg_format(value, syntax_colors: [], charlists: :as_lists) + assert formatted =~ "value #=> [104, 101, 108, 108, 111]\n" + end + + test "with the :print_location option set to false, doesn't print any header" do + {result, formatted} = dbg_format("hello", print_location: false) + assert result == "hello" + refute formatted =~ Path.basename(__ENV__.file) + end end - test :if_else_to_string do - assert Macro.to_string(quote do: (if foo, do: bar, else: baz)) <> "\n" == """ - if(foo) do - bar - else - baz + describe "to_string/1" do + test "converts quoted to string" do + assert Macro.to_string(quote do: hello(world)) == "hello(world)" + end + + test "converts invalid AST with inspect" do + assert Macro.to_string(1..3) == "1..3" end - """ end - test :case_to_string do - assert Macro.to_string(quote do: (case foo do true -> 0; false -> (1; 2) end)) <> "\n" == """ - case(foo) do - true -> - 0 - false -> + describe "to_string/2" do + defp macro_to_string(var, fun \\ fn _ast, string -> string end) do + module = String.to_atom("Elixir.Macro") + module.to_string(var, fun) + end + + test "variable" do + assert macro_to_string(quote(do: foo)) == "foo" + end + + test "local call" do + assert macro_to_string(quote(do: foo(1, 2, 3))) == "foo(1, 2, 3)" + assert macro_to_string(quote(do: foo([1, 2, 3]))) == "foo([1, 2, 3])" + end + + test "remote call" do + assert macro_to_string(quote(do: foo.bar(1, 2, 3))) == "foo.bar(1, 2, 3)" + assert macro_to_string(quote(do: foo.bar([1, 2, 3]))) == "foo.bar([1, 2, 3])" + + quoted = + quote do + (foo do + :ok + end).bar([1, 2, 3]) + end + + assert macro_to_string(quoted) == "(foo do\n :ok\nend).bar([1, 2, 3])" + end + + test "nullary remote call" do + assert macro_to_string(quote do: foo.bar) == "foo.bar" + assert macro_to_string(quote do: foo.bar()) == "foo.bar()" + end + + test "atom remote call" do + assert macro_to_string(quote(do: :foo.bar(1, 2, 3))) == ":foo.bar(1, 2, 3)" + end + + test "remote and fun call" do + assert macro_to_string(quote(do: foo.bar().(1, 2, 3))) == "foo.bar().(1, 2, 3)" + assert macro_to_string(quote(do: foo.bar().([1, 2, 3]))) == "foo.bar().([1, 2, 3])" + end + + test "unusual remote atom fun call" do + assert macro_to_string(quote(do: Foo."42"())) == ~s/Foo."42"()/ + assert macro_to_string(quote(do: Foo."Bar"())) == ~s/Foo."Bar"()/ + assert macro_to_string(quote(do: Foo."bar baz"().""())) == ~s/Foo."bar baz"().""()/ + assert macro_to_string(quote(do: Foo."%{}"())) == ~s/Foo."%{}"()/ + assert macro_to_string(quote(do: Foo."..."())) == ~s/Foo."..."()/ + end + + test "atom fun call" do + assert macro_to_string(quote(do: :foo.(1, 2, 3))) == ":foo.(1, 2, 3)" + end + + test "aliases call" do + assert macro_to_string(quote(do: Elixir)) == "Elixir" + assert macro_to_string(quote(do: Foo)) == "Foo" + assert macro_to_string(quote(do: Foo.Bar.baz(1, 2, 3))) == "Foo.Bar.baz(1, 2, 3)" + assert macro_to_string(quote(do: Foo.Bar.baz([1, 2, 3]))) == "Foo.Bar.baz([1, 2, 3])" + assert macro_to_string(quote(do: Foo.bar(<<>>, []))) == "Foo.bar(<<>>, [])" + end + + test "keyword call" do + assert macro_to_string(quote(do: Foo.bar(foo: :bar))) == "Foo.bar(foo: :bar)" + assert macro_to_string(quote(do: Foo.bar("Elixir.Foo": :bar))) == "Foo.bar([{Foo, :bar}])" + end + + test "sigil call" do + assert macro_to_string(quote(do: ~r"123")) == ~S/~r"123"/ + assert macro_to_string(quote(do: ~r"\n123")) == ~S/~r"\n123"/ + assert macro_to_string(quote(do: ~r"12\"3")) == ~S/~r"12\"3"/ + assert macro_to_string(quote(do: ~r/12\/3/u)) == ~S"~r/12\/3/u" + assert macro_to_string(quote(do: ~r{\n123})) == ~S/~r{\n123}/ + assert macro_to_string(quote(do: ~r((1\)(2\)3))) == ~S/~r((1\)(2\)3)/ + assert macro_to_string(quote(do: ~r{\n1{1\}23})) == ~S/~r{\n1{1\}23}/ + assert macro_to_string(quote(do: ~r|12\|3|)) == ~S"~r|12\|3|" + + assert macro_to_string(quote(do: ~r[1#{two}3])) == ~S/~r[1#{two}3]/ + assert macro_to_string(quote(do: ~r[1[#{two}\]3])) == ~S/~r[1[#{two}\]3]/ + assert macro_to_string(quote(do: ~r'1#{two}3'u)) == ~S/~r'1#{two}3'u/ + + assert macro_to_string(quote(do: ~R"123")) == ~S/~R"123"/ + assert macro_to_string(quote(do: ~R"123"u)) == ~S/~R"123"u/ + assert macro_to_string(quote(do: ~R"\n123")) == ~S/~R"\n123"/ + + assert macro_to_string(quote(do: ~S["'(123)'"])) == ~S/~S["'(123)'"]/ + assert macro_to_string(quote(do: ~s"#{"foo"}")) == ~S/~s"#{"foo"}"/ + + assert macro_to_string(quote(do: ~HTML[hi])) == ~S/~HTML[hi]/ + + assert macro_to_string( + quote do + ~s""" + "\""foo"\"" + """ + end + ) == ~s[~s"""\n"\\""foo"\\""\n"""] + + assert macro_to_string( + quote do + ~s''' + '\''foo'\'' + ''' + end + ) == ~s[~s'''\n'\\''foo'\\''\n'''] + + assert macro_to_string( + quote do + ~s""" + "\"foo\"" + """ + end + ) == ~s[~s"""\n"\\"foo\\""\n"""] + + assert macro_to_string( + quote do + ~s''' + '\"foo\"' + ''' + end + ) == ~s[~s'''\n'\\"foo\\"'\n'''] + + assert macro_to_string( + quote do + ~S""" + "123" + """ + end + ) == ~s[~S"""\n"123"\n"""] + + assert macro_to_string( + quote do + ~HTML""" + "123" + """ + end + ) == ~s[~HTML"""\n"123"\n"""] + end + + test "tuple call" do + assert macro_to_string(quote(do: alias(Foo.{Bar, Baz, Bong}))) == + "alias(Foo.{Bar, Baz, Bong})" + + assert macro_to_string(quote(do: foo(Foo.{}))) == "foo(Foo.{})" + end + + test "arrow" do + assert macro_to_string(quote(do: foo(1, (2 -> 3)))) == "foo(1, (2 -> 3))" + end + + test "block" do + quoted = + quote do + 1 + 2 + + ( + :foo + :bar + ) + + 3 + end + + expected = """ + ( 1 2 + ( + :foo + :bar + ) + 3 + ) + """ + + assert macro_to_string(quoted) <> "\n" == expected + end + + test "not in" do + assert macro_to_string(quote(do: false not in [])) == "false not in []" end - """ - end - test :fn_to_string do - assert Macro.to_string(quote do: (fn -> 1 + 2 end)) == "fn -> 1 + 2 end" - assert Macro.to_string(quote do: (fn(x) -> x + 1 end)) == "fn x -> x + 1 end" + test "if else" do + expected = """ + if(foo) do + bar + else + baz + end + """ - assert Macro.to_string(quote do: (fn(x) -> y = x + 1; y end)) <> "\n" == """ - fn x -> - y = x + 1 - y + assert macro_to_string(quote(do: if(foo, do: bar, else: baz))) <> "\n" == expected end - """ - assert Macro.to_string(quote do: (fn(x) -> y = x + 1; y; (z) -> z end)) <> "\n" == """ - fn - x -> + test "case" do + quoted = + quote do + case foo do + true -> + 0 + + false -> + 1 + 2 + end + end + + expected = """ + case(foo) do + true -> + 0 + false -> + 1 + 2 + end + """ + + assert macro_to_string(quoted) <> "\n" == expected + end + + test "try" do + quoted = + quote do + try do + foo + catch + _, _ -> + 2 + rescue + ArgumentError -> + 1 + after + 4 + else + _ -> + 3 + end + end + + expected = """ + try do + foo + rescue + ArgumentError -> + 1 + catch + _, _ -> + 2 + else + _ -> + 3 + after + 4 + end + """ + + assert macro_to_string(quoted) <> "\n" == expected + end + + test "fn" do + assert macro_to_string(quote(do: fn -> 1 + 2 end)) == "fn -> 1 + 2 end" + assert macro_to_string(quote(do: fn x -> x + 1 end)) == "fn x -> x + 1 end" + + quoted = + quote do + fn x -> + y = x + 1 + y + end + end + + expected = """ + fn x -> y = x + 1 y - z -> - z + end + """ + + assert macro_to_string(quoted) <> "\n" == expected + + quoted = + quote do + fn + x -> + y = x + 1 + y + + z -> + z + end + end + + expected = """ + fn + x -> + y = x + 1 + y + z -> + z + end + """ + + assert macro_to_string(quoted) <> "\n" == expected + + assert macro_to_string(quote(do: (fn x -> x end).(1))) == "(fn x -> x end).(1)" + + quoted = + quote do + (fn + %{} -> :map + _ -> :other + end).(1) + end + + expected = """ + (fn + %{} -> + :map + _ -> + :other + end).(1) + """ + + assert macro_to_string(quoted) <> "\n" == expected end - """ - end - test :when do - assert Macro.to_string(quote do: (() -> x)) == "(() -> x)" - assert Macro.to_string(quote do: (x when y -> z)) == "(x when y -> z)" - assert Macro.to_string(quote do: (x, y when z -> w)) == "((x, y) when z -> w)" - assert Macro.to_string(quote do: ((x, y) when z -> w)) == "((x, y) when z -> w)" - end + test "range" do + assert macro_to_string(quote(do: -1..+2)) == "-1..+2" + assert macro_to_string(quote(do: Foo.integer()..3)) == "Foo.integer()..3" + assert macro_to_string(quote(do: -1..+2//-3)) == "-1..+2//-3" + + assert macro_to_string(quote(do: Foo.integer()..3//Bar.bat())) == + "Foo.integer()..3//Bar.bat()" + + # invalid AST + assert macro_to_string(-1..+2) == "-1..2" + assert macro_to_string(-1..+2//-3) == "-1..2//-3" + end + + test "when" do + assert macro_to_string(quote(do: (-> x))) == "(() -> x)" + assert macro_to_string(quote(do: (x when y -> z))) == "(x when y -> z)" + assert macro_to_string(quote(do: (x, y when z -> w))) == "((x, y) when z -> w)" + assert macro_to_string(quote(do: (x, y when z -> w))) == "((x, y) when z -> w)" + end - test :nested_to_string do - assert Macro.to_string(quote do: (defmodule Foo do def foo do 1 + 1 end end)) <> "\n" == """ - defmodule(Foo) do - def(foo) do - 1 + 1 + test "nested" do + quoted = + quote do + defmodule Foo do + def foo do + 1 + 1 + end + end + end + + expected = """ + defmodule(Foo) do + def(foo) do + 1 + 1 + end end + """ + + assert macro_to_string(quoted) <> "\n" == expected end - """ - end - test :op_precedence_to_string do - assert Macro.to_string(quote do: (1 + 2) * (3 - 4)) == "(1 + 2) * (3 - 4)" - assert Macro.to_string(quote do: ((1 + 2) * 3) - 4) == "(1 + 2) * 3 - 4" - assert Macro.to_string(quote do: (1 + 2 + 3) == "(1 + 2 + 3)") - assert Macro.to_string(quote do: (1 + 2 - 3) == "(1 + 2 - 3)") - end + test "operator precedence" do + assert macro_to_string(quote(do: (1 + 2) * (3 - 4))) == "(1 + 2) * (3 - 4)" + assert macro_to_string(quote(do: (1 + 2) * 3 - 4)) == "(1 + 2) * 3 - 4" + assert macro_to_string(quote(do: 1 + 2 + 3)) == "1 + 2 + 3" + assert macro_to_string(quote(do: 1 + 2 - 3)) == "1 + 2 - 3" + end - test :containers_to_string do - assert Macro.to_string(quote do: {}) == "{}" - assert Macro.to_string(quote do: []) == "[]" - assert Macro.to_string(quote do: {1, 2, 3}) == "{1, 2, 3}" - assert Macro.to_string(quote do: [ 1, 2, 3 ]) == "[1, 2, 3]" - assert Macro.to_string(quote do: %{}) == "%{}" - assert Macro.to_string(quote do: %{:foo => :bar}) == "%{foo: :bar}" - assert Macro.to_string(quote do: %{{1,2} => [1,2,3]}) == "%{{1, 2} => [1, 2, 3]}" - assert Macro.to_string(quote do: %{map | "a" => "b"}) == "%{map | \"a\" => \"b\"}" - assert Macro.to_string(quote do: [ 1, 2, 3 ]) == "[1, 2, 3]" - assert Macro.to_string(quote do: << 1, 2, 3 >>) == "<<1, 2, 3>>" - assert Macro.to_string(quote do: << <<1>> >>) == "<< <<1>> >>" - end + test "capture operator" do + assert macro_to_string(quote(do: &foo/0)) == "&foo/0" + assert macro_to_string(quote(do: &Foo.foo/0)) == "&Foo.foo/0" + assert macro_to_string(quote(do: &(&1 + &2))) == "&(&1 + &2)" + assert macro_to_string(quote(do: & &1)) == "&(&1)" + assert macro_to_string(quote(do: & &1.(:x))) == "&(&1.(:x))" + assert macro_to_string(quote(do: (& &1).(:x))) == "(&(&1)).(:x)" + end - test :struct_to_string do - assert Macro.to_string(quote do: %Test{}) == "%Test{}" - assert Macro.to_string(quote do: %Test{foo: 1, bar: 1}) == "%Test{foo: 1, bar: 1}" - assert Macro.to_string(quote do: %Test{struct | foo: 2}) == "%Test{struct | foo: 2}" - assert Macro.to_string(quote do: %Test{} + 1) == "%Test{} + 1" - end + test "containers" do + assert macro_to_string(quote(do: {})) == "{}" + assert macro_to_string(quote(do: [])) == "[]" + assert macro_to_string(quote(do: {1, 2, 3})) == "{1, 2, 3}" + assert macro_to_string(quote(do: [1, 2, 3])) == "[1, 2, 3]" + assert macro_to_string(quote(do: ["Elixir.Foo": :bar])) == "[{Foo, :bar}]" + assert macro_to_string(quote(do: %{})) == "%{}" + assert macro_to_string(quote(do: %{:foo => :bar})) == "%{foo: :bar}" + assert macro_to_string(quote(do: %{:"Elixir.Foo" => :bar})) == "%{Foo => :bar}" + assert macro_to_string(quote(do: %{{1, 2} => [1, 2, 3]})) == "%{{1, 2} => [1, 2, 3]}" + assert macro_to_string(quote(do: %{map | "a" => "b"})) == "%{map | \"a\" => \"b\"}" + assert macro_to_string(quote(do: [1, 2, 3])) == "[1, 2, 3]" + end - test :binary_ops_to_string do - assert Macro.to_string(quote do: 1 + 2) == "1 + 2" - assert Macro.to_string(quote do: [ 1, 2 | 3 ]) == "[1, 2 | 3]" - assert Macro.to_string(quote do: [h|t] = [1, 2, 3]) == "[h | t] = [1, 2, 3]" - assert Macro.to_string(quote do: (x ++ y) ++ z) == "(x ++ y) ++ z" - end + test "struct" do + assert macro_to_string(quote(do: %Test{})) == "%Test{}" + assert macro_to_string(quote(do: %Test{foo: 1, bar: 1})) == "%Test{foo: 1, bar: 1}" + assert macro_to_string(quote(do: %Test{struct | foo: 2})) == "%Test{struct | foo: 2}" + assert macro_to_string(quote(do: %Test{} + 1)) == "%Test{} + 1" + assert macro_to_string(quote(do: %Test{foo(1)} + 2)) == "%Test{foo(1)} + 2" + end - test :unary_ops_to_string do - assert Macro.to_string(quote do: not 1) == "not 1" - assert Macro.to_string(quote do: not foo) == "not foo" - assert Macro.to_string(quote do: -1) == "-1" - assert Macro.to_string(quote do: !(foo > bar)) == "!(foo > bar)" - assert Macro.to_string(quote do: @foo(bar)) == "@foo(bar)" - assert Macro.to_string(quote do: identity(&1)) == "identity(&1)" - assert Macro.to_string(quote do: identity(&foo)) == "identity(&foo)" - end + test "binary operators" do + assert macro_to_string(quote(do: 1 + 2)) == "1 + 2" + assert macro_to_string(quote(do: [1, 2 | 3])) == "[1, 2 | 3]" + assert macro_to_string(quote(do: [h | t] = [1, 2, 3])) == "[h | t] = [1, 2, 3]" + assert macro_to_string(quote(do: (x ++ y) ++ z)) == "(x ++ y) ++ z" + assert macro_to_string(quote(do: (x +++ y) +++ z)) == "(x +++ y) +++ z" + end - test :access_to_string do - assert Macro.to_string(quote do: a[b]) == "a[b]" - assert Macro.to_string(quote do: a[1 + 2]) == "a[1 + 2]" - end + test "unary operators" do + assert macro_to_string(quote(do: not 1)) == "not(1)" + assert macro_to_string(quote(do: not foo)) == "not(foo)" + assert macro_to_string(quote(do: -1)) == "-1" + assert macro_to_string(quote(do: +(+1))) == "+(+1)" + assert macro_to_string(quote(do: !(foo > bar))) == "!(foo > bar)" + assert macro_to_string(quote(do: @foo(bar))) == "@foo(bar)" + assert macro_to_string(quote(do: identity(&1))) == "identity(&1)" + end - test :kw_list do - assert Macro.to_string(quote do: [a: a, b: b]) == "[a: a, b: b]" - assert Macro.to_string(quote do: [a: 1, b: 1 + 2]) == "[a: 1, b: 1 + 2]" - assert Macro.to_string(quote do: ["a.b": 1, c: 1 + 2]) == "[\"a.b\": 1, c: 1 + 2]" - end + test "access" do + assert macro_to_string(quote(do: a[b])) == "a[b]" + assert macro_to_string(quote(do: a[1 + 2])) == "a[1 + 2]" + assert macro_to_string(quote(do: (a || [a: 1])[:a])) == "(a || [a: 1])[:a]" + assert macro_to_string(quote(do: Map.put(%{}, :a, 1)[:a])) == "Map.put(%{}, :a, 1)[:a]" + end - test :string_list do - assert Macro.to_string(quote do: []) == "[]" - assert Macro.to_string(quote do: 'abc') == "'abc'" - end + test "keyword list" do + assert macro_to_string(quote(do: [a: a, b: b])) == "[a: a, b: b]" + assert macro_to_string(quote(do: [a: 1, b: 1 + 2])) == "[a: 1, b: 1 + 2]" + assert macro_to_string(quote(do: ["a.b": 1, c: 1 + 2])) == "[\"a.b\": 1, c: 1 + 2]" + end - test :last_arg_kw_list do - assert Macro.to_string(quote do: foo([])) == "foo([])" - assert Macro.to_string(quote do: foo(x: y)) == "foo(x: y)" - assert Macro.to_string(quote do: foo(x: 1 + 2)) == "foo(x: 1 + 2)" - assert Macro.to_string(quote do: foo(x: y, p: q)) == "foo(x: y, p: q)" - assert Macro.to_string(quote do: foo(a, x: y, p: q)) == "foo(a, x: y, p: q)" + test "interpolation" do + assert macro_to_string(quote(do: "foo#{bar}baz")) == ~S["foo#{bar}baz"] + end - assert Macro.to_string(quote do: {[]}) == "{[]}" - assert Macro.to_string(quote do: {[a: b]}) == "{[a: b]}" - assert Macro.to_string(quote do: {x, a: b}) == "{x, [a: b]}" - end + test "bit syntax" do + ast = quote(do: <<1::8*4>>) + assert macro_to_string(ast) == "<<1::8*4>>" - test :to_string_with_fun do - assert Macro.to_string(quote(do: foo(1, 2, 3)), fn _, string -> ":#{string}:" end) == - ":foo(:1:, :2:, :3:):" + ast = quote(do: @type(foo :: <<_::8, _::_*4>>)) + assert macro_to_string(ast) == "@type(foo :: <<_::8, _::_*4>>)" - assert Macro.to_string(quote(do: Bar.foo(1, 2, 3)), fn _, string -> ":#{string}:" end) == - "::Bar:.foo(:1:, :2:, :3:):" - end + ast = quote(do: <<69 - 4::bits-size(8 - 4)-unit(1), 65>>) + assert macro_to_string(ast) == "<<69 - 4::bits-size(8 - 4)-unit(1), 65>>" - ## decompose_call - - test :decompose_call do - assert Macro.decompose_call(quote do: foo) == {:foo, []} - assert Macro.decompose_call(quote do: foo()) == {:foo, []} - assert Macro.decompose_call(quote do: foo(1, 2, 3)) == {:foo, [1, 2, 3]} - assert Macro.decompose_call(quote do: M.N.foo(1, 2, 3)) == - {{:__aliases__, [alias: false], [:M, :N]}, :foo, [1, 2, 3]} - assert Macro.decompose_call(quote do: :foo.foo(1, 2, 3)) == - {:foo, :foo, [1, 2, 3]} - assert Macro.decompose_call(quote do: 1.(1, 2, 3)) == :error - assert Macro.decompose_call(quote do: "some string") == :error - end + ast = quote(do: <<(<<65>>), 65>>) + assert macro_to_string(ast) == "<<(<<65>>), 65>>" + + ast = quote(do: <<65, (<<65>>)>>) + assert macro_to_string(ast) == "<<65, (<<65>>)>>" - ## env + ast = quote(do: for(<<(a::4 <- <<1, 2>>)>>, do: a)) + assert macro_to_string(ast) == "for(<<(a :: 4 <- <<1, 2>>)>>) do\n a\nend" + end + + test "charlist" do + assert macro_to_string(quote(do: [])) == "[]" + assert macro_to_string(quote(do: ~c"abc")) == ~S/~c"abc"/ + assert macro_to_string(quote(do: [?a, ?b, ?c])) == ~S/~c"abc"/ + end - test :env_stacktrace do - env = %{__ENV__ | file: "foo", line: 12} - assert Macro.Env.stacktrace(env) == - [{__MODULE__, :"test env_stacktrace", 1, [file: "foo", line: 12]}] + test "string" do + assert macro_to_string(quote(do: "")) == ~S/""/ + assert macro_to_string(quote(do: "abc")) == ~S/"abc"/ + assert macro_to_string(quote(do: "#{"abc"}")) == ~S/"#{"abc"}"/ + end - env = %{env | function: nil} - assert Macro.Env.stacktrace(env) == - [{__MODULE__, :__MODULE__, 0, [file: "foo", line: 12]}] + test "last arg keyword list" do + assert macro_to_string(quote(do: foo([]))) == "foo([])" + assert macro_to_string(quote(do: foo(x: y))) == "foo(x: y)" + assert macro_to_string(quote(do: foo(x: 1 + 2))) == "foo(x: 1 + 2)" + assert macro_to_string(quote(do: foo(x: y, p: q))) == "foo(x: y, p: q)" + assert macro_to_string(quote(do: foo(a, x: y, p: q))) == "foo(a, x: y, p: q)" + + assert macro_to_string(quote(do: {[]})) == "{[]}" + assert macro_to_string(quote(do: {[a: b]})) == "{[a: b]}" + assert macro_to_string(quote(do: {x, a: b})) == "{x, [a: b]}" + assert macro_to_string(quote(do: foo(else: a))) == "foo(else: a)" + assert macro_to_string(quote(do: foo(catch: a))) == "foo(catch: a)" + end - env = %{env | module: nil} - assert Macro.Env.stacktrace(env) == - [{:elixir_compiler, :__FILE__, 1, [file: "foo", line: 12]}] - end + test "with fun" do + assert macro_to_string(quote(do: foo(1, 2, 3)), fn _, string -> ":#{string}:" end) == + ":foo(:1:, :2:, :3:):" - test :context_modules do - defmodule Foo.Bar do - assert __MODULE__ in __ENV__.context_modules + assert macro_to_string(quote(do: Bar.foo(1, 2, 3)), fn _, string -> ":#{string}:" end) == + "::Bar:.foo(:1:, :2:, :3:):" end end + test "validate/1" do + ref = make_ref() + + assert Macro.validate(1) == :ok + assert Macro.validate(1.0) == :ok + assert Macro.validate(:foo) == :ok + assert Macro.validate("bar") == :ok + assert Macro.validate(<<0::8>>) == :ok + assert Macro.validate(self()) == :ok + assert Macro.validate({1, 2}) == :ok + assert Macro.validate({:foo, [], :baz}) == :ok + assert Macro.validate({:foo, [], []}) == :ok + assert Macro.validate([1, 2, 3]) == :ok + + assert Macro.validate(<<0::4>>) == {:error, <<0::4>>} + assert Macro.validate(ref) == {:error, ref} + assert Macro.validate({1, ref}) == {:error, ref} + assert Macro.validate({ref, 2}) == {:error, ref} + assert Macro.validate([1, ref, 3]) == {:error, ref} + assert Macro.validate({:foo, [], 0}) == {:error, {:foo, [], 0}} + assert Macro.validate({:foo, 0, []}) == {:error, {:foo, 0, []}} + end + + test "decompose_call/1" do + assert Macro.decompose_call(quote(do: foo)) == {:foo, []} + assert Macro.decompose_call(quote(do: foo())) == {:foo, []} + assert Macro.decompose_call(quote(do: foo(1, 2, 3))) == {:foo, [1, 2, 3]} + + assert Macro.decompose_call(quote(do: M.N.foo(1, 2, 3))) == + {{:__aliases__, [alias: false], [:M, :N]}, :foo, [1, 2, 3]} + + assert Macro.decompose_call(quote(do: :foo.foo(1, 2, 3))) == {:foo, :foo, [1, 2, 3]} + assert Macro.decompose_call(quote(do: 1.(1, 2, 3))) == :error + assert Macro.decompose_call(quote(do: "some string")) == :error + assert Macro.decompose_call(quote(do: {:foo, :bar, :baz})) == :error + assert Macro.decompose_call(quote(do: {:foo, :bar, :baz, 42})) == :error + end + ## pipe/unpipe - test :pipe do + test "pipe/3" do assert Macro.pipe(1, quote(do: foo), 0) == quote(do: foo(1)) assert Macro.pipe(1, quote(do: foo(2)), 0) == quote(do: foo(1, 2)) assert Macro.pipe(1, quote(do: foo), -1) == quote(do: foo(1)) assert Macro.pipe(2, quote(do: foo(1)), -1) == quote(do: foo(1, 2)) - assert_raise ArgumentError, "cannot pipe 1 into 2", fn -> + assert Macro.pipe(quote(do: %{foo: "bar"}), quote(do: Access.get(:foo)), 0) == + quote(do: Access.get(%{foo: "bar"}, :foo)) + + assert_raise ArgumentError, ~r"cannot pipe 1 into 2", fn -> Macro.pipe(1, 2, 0) end + + assert_raise ArgumentError, ~r"cannot pipe 1 into \{2, 3\}", fn -> + Macro.pipe(1, {2, 3}, 0) + end + + assert_raise ArgumentError, ~r"cannot pipe 1 into 1 \+ 1, the :\+ operator can", fn -> + Macro.pipe(1, quote(do: 1 + 1), 0) == quote(do: foo(1)) + end + + assert_raise ArgumentError, ~r"cannot pipe 1 into <<1>>", fn -> + Macro.pipe(1, quote(do: <<1>>), 0) + end + + assert_raise ArgumentError, ~r"cannot pipe 1 into the special form unquote/1", fn -> + Macro.pipe(1, quote(do: unquote()), 0) + end + + assert_raise ArgumentError, ~r"piping into a unary operator is not supported", fn -> + Macro.pipe(1, quote(do: +1), 0) + end + + assert_raise ArgumentError, ~r"cannot pipe Macro into Env", fn -> + Macro.pipe(Macro, quote(do: Env), 0) + end + + assert_raise ArgumentError, ~r"cannot pipe 1 into 2 && 3", fn -> + Macro.pipe(1, quote(do: 2 && 3), 0) + end + + message = ~r"cannot pipe :foo into an anonymous function without calling" + + assert_raise ArgumentError, message, fn -> + Macro.pipe(:foo, quote(do: fn x -> x end), 0) + end + + message = ~r"wrong operator precedence when piping into bracket-based access" + + assert_raise ArgumentError, message, fn -> + Macro.pipe(:foo, quote(do: %{foo: bar}[:foo]), 0) + end end - test :unpipe do + test "unpipe/1" do assert Macro.unpipe(quote(do: foo)) == quote(do: [{foo, 0}]) assert Macro.unpipe(quote(do: foo |> bar)) == quote(do: [{foo, 0}, {bar, 0}]) assert Macro.unpipe(quote(do: foo |> bar |> baz)) == quote(do: [{foo, 0}, {bar, 0}, {baz, 0}]) end - ## pre/postwalk - - test :prewalk do - assert prewalk({:foo, [], nil}) == - [{:foo, [], nil}] - - assert prewalk({:foo, [], [1, 2, 3]}) == - [{:foo, [], [1, 2, 3]}, 1, 2, 3] + ## traverse/pre/postwalk + + test "traverse/4" do + assert traverse({:foo, [], nil}) == [{:foo, [], nil}, {:foo, [], nil}] + + assert traverse({:foo, [], [1, 2, 3]}) == + [{:foo, [], [1, 2, 3]}, 1, 1, 2, 2, 3, 3, {:foo, [], [1, 2, 3]}] + + assert traverse({{:., [], [:foo, :bar]}, [], [1, 2, 3]}) == + [ + {{:., [], [:foo, :bar]}, [], [1, 2, 3]}, + {:., [], [:foo, :bar]}, + :foo, + :foo, + :bar, + :bar, + {:., [], [:foo, :bar]}, + 1, + 1, + 2, + 2, + 3, + 3, + {{:., [], [:foo, :bar]}, [], [1, 2, 3]} + ] + + assert traverse({[1, 2, 3], [4, 5, 6]}) == + [ + {[1, 2, 3], [4, 5, 6]}, + [1, 2, 3], + 1, + 1, + 2, + 2, + 3, + 3, + [1, 2, 3], + [4, 5, 6], + 4, + 4, + 5, + 5, + 6, + 6, + [4, 5, 6], + {[1, 2, 3], [4, 5, 6]} + ] + end + + defp traverse(ast) do + Macro.traverse(ast, [], &{&1, [&1 | &2]}, &{&1, [&1 | &2]}) |> elem(1) |> Enum.reverse() + end + + test "prewalk/3" do + assert prewalk({:foo, [], nil}) == [{:foo, [], nil}] + + assert prewalk({:foo, [], [1, 2, 3]}) == [{:foo, [], [1, 2, 3]}, 1, 2, 3] assert prewalk({{:., [], [:foo, :bar]}, [], [1, 2, 3]}) == - [{{:., [], [:foo, :bar]}, [], [1, 2, 3]}, {:., [], [:foo, :bar]}, :foo, :bar, 1, 2, 3] + [ + {{:., [], [:foo, :bar]}, [], [1, 2, 3]}, + {:., [], [:foo, :bar]}, + :foo, + :bar, + 1, + 2, + 3 + ] assert prewalk({[1, 2, 3], [4, 5, 6]}) == - [{[1, 2, 3], [4, 5, 6]}, [1, 2, 3], 1, 2, 3, [4, 5, 6], 4, 5, 6] + [{[1, 2, 3], [4, 5, 6]}, [1, 2, 3], 1, 2, 3, [4, 5, 6], 4, 5, 6] end defp prewalk(ast) do - Macro.prewalk(ast, [], &{&1, [&1|&2]}) |> elem(1) |> Enum.reverse + Macro.prewalk(ast, [], &{&1, [&1 | &2]}) |> elem(1) |> Enum.reverse() end - test :postwalk do - assert postwalk({:foo, [], nil}) == - [{:foo, [], nil}] + test "postwalk/3" do + assert postwalk({:foo, [], nil}) == [{:foo, [], nil}] - assert postwalk({:foo, [], [1, 2, 3]}) == - [1, 2, 3, {:foo, [], [1, 2, 3]}] + assert postwalk({:foo, [], [1, 2, 3]}) == [1, 2, 3, {:foo, [], [1, 2, 3]}] assert postwalk({{:., [], [:foo, :bar]}, [], [1, 2, 3]}) == - [:foo, :bar, {:., [], [:foo, :bar]}, 1, 2, 3, {{:., [], [:foo, :bar]}, [], [1, 2, 3]}] + [ + :foo, + :bar, + {:., [], [:foo, :bar]}, + 1, + 2, + 3, + {{:., [], [:foo, :bar]}, [], [1, 2, 3]} + ] assert postwalk({[1, 2, 3], [4, 5, 6]}) == - [1, 2, 3, [1, 2, 3], 4, 5, 6, [4, 5, 6], {[1, 2, 3], [4, 5, 6]}] + [1, 2, 3, [1, 2, 3], 4, 5, 6, [4, 5, 6], {[1, 2, 3], [4, 5, 6]}] + end + + test "generate_arguments/2" do + assert Macro.generate_arguments(0, __MODULE__) == [] + assert Macro.generate_arguments(1, __MODULE__) == [{:arg1, [], __MODULE__}] + assert Macro.generate_arguments(4, __MODULE__) |> length() == 4 end defp postwalk(ast) do - Macro.postwalk(ast, [], &{&1, [&1|&2]}) |> elem(1) |> Enum.reverse + Macro.postwalk(ast, [], &{&1, [&1 | &2]}) |> elem(1) |> Enum.reverse() + end + + test "struct_info!/2 expands structs multiple levels deep" do + defmodule StructBang do + @enforce_keys [:b] + defstruct [:a, :b] + + assert Macro.struct_info!(StructBang, __ENV__) == [ + %{field: :a, default: nil, required: false}, + %{field: :b, default: nil, required: true} + ] + + def within_function do + assert Macro.struct_info!(StructBang, __ENV__) == [ + %{field: :a, default: nil, required: false}, + %{field: :b, default: nil, required: true} + ] + end + + defmodule Nested do + assert Macro.struct_info!(StructBang, __ENV__) == [ + %{field: :a, default: nil, required: false}, + %{field: :b, default: nil, required: true} + ] + end + end + + assert Macro.struct_info!(StructBang, __ENV__) == [ + %{field: :a, default: nil, required: false}, + %{field: :b, default: nil, required: true} + ] + end + + test "prewalker/1" do + ast = quote do: :mod.foo(bar({1, 2}), [3, 4, five]) + map = Enum.map(Macro.prewalker(ast), & &1) + + assert map == [ + {{:., [], [:mod, :foo]}, [], [{:bar, [], [{1, 2}]}, [3, 4, {:five, [], MacroTest}]]}, + {:., [], [:mod, :foo]}, + :mod, + :foo, + {:bar, [], [{1, 2}]}, + {1, 2}, + 1, + 2, + [3, 4, {:five, [], MacroTest}], + 3, + 4, + {:five, [], MacroTest} + ] + + assert map == ast |> Macro.prewalk([], &{&1, [&1 | &2]}) |> elem(1) |> Enum.reverse() + assert Enum.zip(Macro.prewalker(ast), []) == Enum.zip(map, []) + + for i <- 0..(length(map) + 1) do + assert Enum.take(Macro.prewalker(ast), i) == Enum.take(map, i) + end + end + + test "postwalker/1" do + ast = quote do: :mod.foo(bar({1, 2}), [3, 4, five]) + map = Enum.map(Macro.postwalker(ast), & &1) + + assert map == [ + :mod, + :foo, + {:., [], [:mod, :foo]}, + 1, + 2, + {1, 2}, + {:bar, [], [{1, 2}]}, + 3, + 4, + {:five, [], MacroTest}, + [3, 4, {:five, [], MacroTest}], + {{:., [], [:mod, :foo]}, [], [{:bar, [], [{1, 2}]}, [3, 4, {:five, [], MacroTest}]]} + ] + + assert map == ast |> Macro.postwalk([], &{&1, [&1 | &2]}) |> elem(1) |> Enum.reverse() + assert Enum.zip(Macro.postwalker(ast), []) == Enum.zip(map, []) + + for i <- 0..(length(map) + 1) do + assert Enum.take(Macro.postwalker(ast), i) == Enum.take(map, i) + end + end + + test "operator?/2" do + assert Macro.operator?(:+, 2) + assert Macro.operator?(:+, 1) + refute Macro.operator?(:+, 0) + end + + test "quoted_literal?/1" do + assert Macro.quoted_literal?(quote(do: "foo")) + assert Macro.quoted_literal?(quote(do: {"foo", 1})) + assert Macro.quoted_literal?(quote(do: %{foo: "bar"})) + assert Macro.quoted_literal?(quote(do: %URI{path: "/"})) + assert Macro.quoted_literal?(quote(do: <<>>)) + assert Macro.quoted_literal?(quote(do: <<1, "foo", "bar"::utf16>>)) + assert Macro.quoted_literal?(quote(do: <<1000::size(8)-unit(4)>>)) + assert Macro.quoted_literal?(quote(do: <<1000::8*4>>)) + assert Macro.quoted_literal?(quote(do: <<102::unsigned-big-integer-size(8)>>)) + refute Macro.quoted_literal?(quote(do: {"foo", var})) + refute Macro.quoted_literal?(quote(do: <<"foo"::size(name_size)>>)) + refute Macro.quoted_literal?(quote(do: <<"foo"::binary-size(name_size)>>)) + refute Macro.quoted_literal?(quote(do: <<"foo"::custom_modifier()>>)) + refute Macro.quoted_literal?(quote(do: <<102, rest::binary>>)) + end + + test "underscore/1" do + assert Macro.underscore("foo") == "foo" + assert Macro.underscore("foo_bar") == "foo_bar" + assert Macro.underscore("Foo") == "foo" + assert Macro.underscore("FooBar") == "foo_bar" + assert Macro.underscore("FOOBar") == "foo_bar" + assert Macro.underscore("FooBAR") == "foo_bar" + assert Macro.underscore("FOO_BAR") == "foo_bar" + assert Macro.underscore("FoBaZa") == "fo_ba_za" + assert Macro.underscore("Foo10") == "foo10" + assert Macro.underscore("FOO10") == "foo10" + assert Macro.underscore("10Foo") == "10_foo" + assert Macro.underscore("FooBar10") == "foo_bar10" + assert Macro.underscore("FooBAR10") == "foo_bar10" + assert Macro.underscore("Foo10Bar") == "foo10_bar" + assert Macro.underscore("Foo.Bar") == "foo/bar" + assert Macro.underscore(Foo.Bar) == "foo/bar" + assert Macro.underscore("API.V1.User") == "api/v1/user" + assert Macro.underscore("") == "" + end + + test "camelize/1" do + assert Macro.camelize("Foo") == "Foo" + assert Macro.camelize("FooBar") == "FooBar" + assert Macro.camelize("foo") == "Foo" + assert Macro.camelize("foo_bar") == "FooBar" + assert Macro.camelize("foo_") == "Foo" + assert Macro.camelize("_foo") == "Foo" + assert Macro.camelize("foo10") == "Foo10" + assert Macro.camelize("_10foo") == "10foo" + assert Macro.camelize("foo_10") == "Foo10" + assert Macro.camelize("foo__10") == "Foo10" + assert Macro.camelize("foo__bar") == "FooBar" + assert Macro.camelize("foo/bar") == "Foo.Bar" + assert Macro.camelize("Foo.Bar") == "Foo.Bar" + assert Macro.camelize("foo1_0") == "Foo10" + assert Macro.camelize("foo_123_4_567") == "Foo1234567" + assert Macro.camelize("FOO_BAR") == "FOO_BAR" + assert Macro.camelize("FOO.BAR") == "FOO.BAR" + assert Macro.camelize("") == "" end end diff --git a/lib/elixir/test/elixir/map_set_test.exs b/lib/elixir/test/elixir/map_set_test.exs new file mode 100644 index 00000000000..7203cb79038 --- /dev/null +++ b/lib/elixir/test/elixir/map_set_test.exs @@ -0,0 +1,169 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("test_helper.exs", __DIR__) + +defmodule MapSetTest do + use ExUnit.Case, async: true + + doctest MapSet + + test "new/1" do + result = MapSet.new(1..5) + assert MapSet.equal?(result, Enum.into(1..5, MapSet.new())) + end + + test "new/2" do + result = MapSet.new(1..5, &(&1 + 2)) + assert MapSet.equal?(result, Enum.into(3..7, MapSet.new())) + end + + test "put/2" do + result = MapSet.put(MapSet.new(), 1) + assert MapSet.equal?(result, MapSet.new([1])) + + result = MapSet.put(MapSet.new([1, 3, 4]), 2) + assert MapSet.equal?(result, MapSet.new(1..4)) + + result = MapSet.put(MapSet.new(5..100), 10) + assert MapSet.equal?(result, MapSet.new(5..100)) + end + + test "union/2" do + result = MapSet.union(MapSet.new([1, 3, 4]), MapSet.new()) + assert MapSet.equal?(result, MapSet.new([1, 3, 4])) + + result = MapSet.union(MapSet.new(5..15), MapSet.new(10..25)) + assert MapSet.equal?(result, MapSet.new(5..25)) + + result = MapSet.union(MapSet.new(1..120), MapSet.new(1..100)) + assert MapSet.equal?(result, MapSet.new(1..120)) + end + + test "intersection/2" do + result = MapSet.intersection(MapSet.new(), MapSet.new(1..21)) + assert MapSet.equal?(result, MapSet.new()) + + result = MapSet.intersection(MapSet.new(1..21), MapSet.new(4..24)) + assert MapSet.equal?(result, MapSet.new(4..21)) + + result = MapSet.intersection(MapSet.new(2..100), MapSet.new(1..120)) + assert MapSet.equal?(result, MapSet.new(2..100)) + end + + test "difference/2" do + result = MapSet.difference(MapSet.new(2..20), MapSet.new()) + assert MapSet.equal?(result, MapSet.new(2..20)) + + result = MapSet.difference(MapSet.new(2..20), MapSet.new(1..21)) + assert MapSet.equal?(result, MapSet.new()) + + result = MapSet.difference(MapSet.new(1..101), MapSet.new(2..100)) + assert MapSet.equal?(result, MapSet.new([1, 101])) + end + + test "symmetric_difference/2" do + result = MapSet.symmetric_difference(MapSet.new(1..5), MapSet.new(3..8)) + assert MapSet.equal?(result, MapSet.new([1, 2, 6, 7, 8])) + + result = MapSet.symmetric_difference(MapSet.new(), MapSet.new()) + assert MapSet.equal?(result, MapSet.new()) + + result = MapSet.symmetric_difference(MapSet.new(1..5), MapSet.new(1..5)) + assert MapSet.equal?(result, MapSet.new()) + + result = MapSet.symmetric_difference(MapSet.new([1, 2, 3]), MapSet.new()) + assert MapSet.equal?(result, MapSet.new([1, 2, 3])) + + result = MapSet.symmetric_difference(MapSet.new(), MapSet.new([1, 2, 3])) + assert MapSet.equal?(result, MapSet.new([1, 2, 3])) + end + + test "disjoint?/2" do + assert MapSet.disjoint?(MapSet.new(), MapSet.new()) + assert MapSet.disjoint?(MapSet.new(1..6), MapSet.new(8..20)) + refute MapSet.disjoint?(MapSet.new(1..6), MapSet.new(5..15)) + refute MapSet.disjoint?(MapSet.new(1..120), MapSet.new(1..6)) + end + + test "subset?/2" do + assert MapSet.subset?(MapSet.new(), MapSet.new()) + assert MapSet.subset?(MapSet.new(1..6), MapSet.new(1..10)) + assert MapSet.subset?(MapSet.new(1..6), MapSet.new(1..120)) + refute MapSet.subset?(MapSet.new(1..120), MapSet.new(1..6)) + end + + test "equal?/2" do + assert MapSet.equal?(MapSet.new(), MapSet.new()) + refute MapSet.equal?(MapSet.new(1..20), MapSet.new(2..21)) + assert MapSet.equal?(MapSet.new(1..120), MapSet.new(1..120)) + end + + test "delete/2" do + result = MapSet.delete(MapSet.new(), 1) + assert MapSet.equal?(result, MapSet.new()) + + result = MapSet.delete(MapSet.new(1..4), 5) + assert MapSet.equal?(result, MapSet.new(1..4)) + + result = MapSet.delete(MapSet.new(1..4), 1) + assert MapSet.equal?(result, MapSet.new(2..4)) + + result = MapSet.delete(MapSet.new(1..4), 2) + assert MapSet.equal?(result, MapSet.new([1, 3, 4])) + end + + test "size/1" do + assert MapSet.size(MapSet.new()) == 0 + assert MapSet.size(MapSet.new(5..15)) == 11 + assert MapSet.size(MapSet.new(2..100)) == 99 + end + + test "to_list/1" do + assert MapSet.to_list(MapSet.new()) == [] + + list = MapSet.to_list(MapSet.new(1..20)) + assert Enum.sort(list) == Enum.to_list(1..20) + + list = MapSet.to_list(MapSet.new(5..120)) + assert Enum.sort(list) == Enum.to_list(5..120) + end + + test "filter/2" do + result = MapSet.filter(MapSet.new([1, nil, 2, false]), & &1) + assert MapSet.equal?(result, MapSet.new(1..2)) + + result = MapSet.filter(MapSet.new(1..10), &(&1 < 2 or &1 > 9)) + assert MapSet.equal?(result, MapSet.new([1, 10])) + + result = MapSet.filter(MapSet.new(~w(A a B b)), fn x -> String.downcase(x) == x end) + assert MapSet.equal?(result, MapSet.new(~w(a b))) + end + + test "reject/2" do + result = MapSet.reject(MapSet.new(1..10), &(&1 < 8)) + assert MapSet.equal?(result, MapSet.new(8..10)) + + result = MapSet.reject(MapSet.new(["a", :b, 1, 1.0]), &is_integer/1) + assert MapSet.equal?(result, MapSet.new(["a", :b, 1.0])) + + result = MapSet.reject(MapSet.new(1..3), fn x -> rem(x, 2) == 0 end) + assert MapSet.equal?(result, MapSet.new([1, 3])) + end + + test "split_with" do + assert MapSet.split_with(MapSet.new(), fn v -> rem(v, 2) == 0 end) == + {MapSet.new(), MapSet.new()} + + assert MapSet.split_with(MapSet.new([1, 2, 3]), fn v -> rem(v, 2) == 0 end) == + {MapSet.new([2]), MapSet.new([1, 3])} + + assert MapSet.split_with(MapSet.new([2, 4, 6]), fn v -> rem(v, 2) == 0 end) == + {MapSet.new([2, 4, 6]), MapSet.new([])} + end + + test "inspect" do + assert inspect(MapSet.new([?a])) == "MapSet.new([97])" + end +end diff --git a/lib/elixir/test/elixir/map_test.exs b/lib/elixir/test/elixir/map_test.exs index a005bf0c5ba..bf9bffd0ddc 100644 --- a/lib/elixir/test/elixir/map_test.exs +++ b/lib/elixir/test/elixir/map_test.exs @@ -1,52 +1,23 @@ -Code.require_file "test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("test_helper.exs", __DIR__) defmodule MapTest do use ExUnit.Case, async: true - defp empty_map do - %{} - end + doctest Map - defp two_items_map do - %{a: 1, b: 2} - end - - @map %{a: 1, b: 2} + @sample %{a: 1, b: 2} + defp sample, do: Process.get(:unused, %{a: 1, b: 2}) test "maps in attributes" do - assert @map == %{a: 1, b: 2} + assert @sample == %{a: 1, b: 2} end test "maps when quoted" do - assert (quote do - %{foo: 1} - end) == {:%{}, [], [{:foo, 1}]} - - assert (quote do - % - {foo: 1} - end) == {:%{}, [], [{:foo, 1}]} - end - - test "structs when quoted" do - assert (quote do - %User{foo: 1} - end) == {:%, [], [ - {:__aliases__, [alias: false], [:User]}, - {:%{}, [], [{:foo, 1}]} - ]} - - assert (quote do - % - User{foo: 1} - end) == {:%, [], [ - {:__aliases__, [alias: false], [:User]}, - {:%{}, [], [{:foo, 1}]} - ]} - - assert (quote do - %unquote(User){foo: 1} - end) == {:%, [], [User, {:%{}, [], [{:foo, 1}]}]} + assert quote(do: %{foo: 1}) == {:%{}, [], [{:foo, 1}]} end test "maps keywords and atoms" do @@ -60,76 +31,453 @@ defmodule MapTest do assert a == 1 end + test "maps with generated variables in key" do + assert %{"#{1}" => 1} == %{"1" => 1} + assert %{for(x <- 1..3, do: x) => 1} == %{[1, 2, 3] => 1} + assert %{with(x = 1, do: x) => 1} == %{1 => 1} + assert %{with({:ok, x} <- {:ok, 1}, do: x) => 1} == %{1 => 1} + + assert %{ + try do + raise "error" + rescue + _ -> 1 + end => 1 + } == %{1 => 1} + + assert %{ + try do + throw(1) + catch + x -> x + end => 1 + } == %{1 => 1} + + assert %{ + try do + a = 1 + a + rescue + _ -> 2 + end => 1 + } == %{1 => 1} + + assert %{ + try do + 1 + rescue + _exception -> flunk("should not be invoked") + else + a -> a + end => 1 + } == %{1 => 1} + end + + test "matching with map as a key" do + assert %{%{1 => 2} => x} = %{%{1 => 2} => 3} + assert x == 3 + end + test "is_map/1" do - assert is_map empty_map - refute is_map(Enum.to_list(empty_map)) + assert is_map(Map.new()) + refute is_map(Enum.to_list(%{})) end test "map_size/1" do - assert map_size(empty_map) == 0 - assert map_size(two_items_map) == 2 + assert map_size(%{}) == 0 + assert map_size(sample()) == 2 end - test "maps with optional comma" do - assert %{a: :b,} == %{a: :b} - assert %{1 => 2,} == %{1 => 2} - assert %{1 => 2, a: :b,} == %{1 => 2, a: :b} + test "new/1" do + assert Map.new(%{a: 1, b: 2}) == %{a: 1, b: 2} + assert Map.new(MapSet.new(a: 1, b: 2, a: 3)) == %{b: 2, a: 3} + end + + test "new/2" do + transformer = fn {key, value} -> {key, value * 2} end + assert Map.new(%{a: 1, b: 2}, transformer) == %{a: 2, b: 4} + assert Map.new(MapSet.new(a: 1, b: 2, a: 3), transformer) == %{b: 4, a: 6} + end + + test "take/2" do + assert Map.take(%{a: 1, b: 2, c: 3}, [:b, :c]) == %{b: 2, c: 3} + assert Map.take(%{a: 1, b: 2, c: 3}, []) == %{} + assert_raise BadMapError, fn -> Map.take(:foo, []) end + end + + test "drop/2" do + assert Map.drop(%{a: 1, b: 2, c: 3}, [:b, :c]) == %{a: 1} + assert_raise BadMapError, fn -> Map.drop(:foo, []) end + end + + test "split/2" do + assert Map.split(%{a: 1, b: 2, c: 3}, [:b, :c]) == {%{b: 2, c: 3}, %{a: 1}} + assert_raise BadMapError, fn -> Map.split(:foo, []) end + end + + test "split_with/2" do + assert Map.split_with(%{}, fn {_k, v} -> rem(v, 2) == 0 end) == {%{}, %{}} + + assert Map.split_with(%{a: 1, b: 2, c: 3}, fn {_k, v} -> rem(v, 2) == 0 end) == + {%{b: 2}, %{a: 1, c: 3}} + + assert Map.split_with(%{a: 2, b: 4, c: 6}, fn {_k, v} -> rem(v, 2) == 0 end) == + {%{a: 2, b: 4, c: 6}, %{}} + + assert Map.split_with(%{1 => 1, 2 => 2, 3 => 3}, fn {k, _v} -> rem(k, 2) == 0 end) == + {%{2 => 2}, %{1 => 1, 3 => 3}} + + assert Map.split_with(%{1 => 2, 3 => 4, 5 => 6}, fn {k, _v} -> rem(k, 2) == 0 end) == + {%{}, %{1 => 2, 3 => 4, 5 => 6}} end - test "maps with duplicate keys" do - assert %{a: :b, a: :c} == %{a: :c} - assert %{1 => 2, 1 => 3} == %{1 => 3} - assert %{:a => :b, a: :c} == %{a: :c} + test "get_and_update/3" do + message = "the given function must return a two-element tuple or :pop, got: 1" + + assert_raise RuntimeError, message, fn -> + Map.get_and_update(%{a: 1}, :a, fn value -> value end) + end + end + + test "get_and_update!/3" do + message = "the given function must return a two-element tuple or :pop, got: 1" + + assert_raise RuntimeError, message, fn -> + Map.get_and_update!(%{a: 1}, :a, fn value -> value end) + end + end + + test "maps with optional comma" do + assert Code.eval_string("%{a: :b,}") == {%{a: :b}, []} + assert Code.eval_string("%{1 => 2,}") == {%{1 => 2}, []} + assert Code.eval_string("%{1 => 2, a: :b,}") == {%{1 => 2, a: :b}, []} end test "update maps" do - assert %{two_items_map | a: 3} == %{a: 3, b: 2} + assert %{sample() | a: 3} == %{a: 3, b: 2} - assert_raise ArgumentError, fn -> - %{two_items_map | c: 3} + assert_raise KeyError, fn -> + %{sample() | c: 3} end end - test "map access" do - assert two_items_map.a == 1 + test "map dot access" do + assert sample().a == 1 assert_raise KeyError, fn -> - two_items_map.c + sample().c end end - defmodule ExternalUser do - def __struct__ do - %{__struct__: ThisDoesNotLeak, name: "josé", age: 27} + test "put/3 optimized by the compiler" do + map = %{a: 1, b: 2} + + assert Map.put(map, :a, 2) == %{a: 2, b: 2} + assert Map.put(map, :c, 3) == %{a: 1, b: 2, c: 3} + + assert Map.put(%{map | a: 2}, :a, 3) == %{a: 3, b: 2} + assert Map.put(%{map | a: 2}, :b, 3) == %{a: 2, b: 3} + + assert Map.put(map, :a, 2) |> Map.put(:a, 3) == %{a: 3, b: 2} + assert Map.put(map, :a, 2) |> Map.put(:c, 3) == %{a: 2, b: 2, c: 3} + assert Map.put(map, :c, 3) |> Map.put(:a, 2) == %{a: 2, b: 2, c: 3} + assert Map.put(map, :c, 3) |> Map.put(:c, 4) == %{a: 1, b: 2, c: 4} + end + + test "merge/2 with map literals optimized by the compiler" do + map = %{a: 1, b: 2} + + assert Map.merge(map, %{a: 2}) == %{a: 2, b: 2} + assert Map.merge(map, %{c: 3}) == %{a: 1, b: 2, c: 3} + assert Map.merge(%{a: 2}, map) == %{a: 1, b: 2} + assert Map.merge(%{c: 3}, map) == %{a: 1, b: 2, c: 3} + + assert Map.merge(%{map | a: 2}, %{a: 3}) == %{a: 3, b: 2} + assert Map.merge(%{map | a: 2}, %{b: 3}) == %{a: 2, b: 3} + assert Map.merge(%{a: 2}, %{map | a: 3}) == %{a: 3, b: 2} + assert Map.merge(%{a: 2}, %{map | b: 3}) == %{a: 1, b: 3} + + assert Map.merge(map, %{a: 2}) |> Map.merge(%{a: 3, c: 3}) == %{a: 3, b: 2, c: 3} + assert Map.merge(map, %{c: 3}) |> Map.merge(%{c: 4}) == %{a: 1, b: 2, c: 4} + assert Map.merge(map, %{a: 3, c: 3}) |> Map.merge(%{a: 2}) == %{a: 2, b: 2, c: 3} + end + + test "merge/3" do + # When first map is bigger + assert Map.merge(%{a: 1, b: 2, c: 3}, %{c: 4, d: 5}, fn :c, 3, 4 -> :x end) == + %{a: 1, b: 2, c: :x, d: 5} + + # When second map is bigger + assert Map.merge(%{b: 2, c: 3}, %{a: 1, c: 4, d: 5}, fn :c, 3, 4 -> :x end) == + %{a: 1, b: 2, c: :x, d: 5} + end + + test "replace/3" do + map = %{c: 3, b: 2, a: 1} + assert Map.replace(map, :b, 10) == %{c: 3, b: 10, a: 1} + assert Map.replace(map, :a, 1) == map + assert Map.replace(map, :x, 1) == map + assert Map.replace(%{}, :x, 1) == %{} + end + + test "replace!/3" do + map = %{c: 3, b: 2, a: 1} + assert Map.replace!(map, :b, 10) == %{c: 3, b: 10, a: 1} + assert Map.replace!(map, :a, 1) == map + + assert_raise KeyError, ~r/key :x not found in:\n\n %{.*a: 1.*}/, fn -> + Map.replace!(map, :x, 10) + end + + assert_raise KeyError, "key :x not found in:\n\n %{}\n", fn -> + Map.replace!(%{}, :x, 10) end end - test "structs" do - assert %ExternalUser{} == - %{__struct__: ExternalUser, name: "josé", age: 27} + test "intersect/2" do + map = %{a: 1, b: 2} - assert %ExternalUser{name: "valim"} == - %{__struct__: ExternalUser, name: "valim", age: 27} + assert Map.intersect(map, %{a: 2}) == %{a: 2} + assert Map.intersect(map, %{c: 3}) == %{} + assert Map.intersect(%{a: 2}, map) == %{a: 1} + assert Map.intersect(%{c: 3}, map) == %{} + + assert Map.intersect(map, %{a: 2}) |> Map.intersect(%{a: 3, c: 3}) == %{a: 3} + assert Map.intersect(map, %{c: 3}) |> Map.intersect(%{c: 4}) == %{} + assert Map.intersect(map, %{a: 3, c: 3}) |> Map.intersect(%{a: 2}) == %{a: 2} + end - user = %ExternalUser{} - assert %ExternalUser{user | name: "valim"} == - %{__struct__: ExternalUser, name: "valim", age: 27} + test "intersect/3" do + # When first map is bigger + assert Map.intersect(%{a: 1, b: 2, c: 3}, %{c: 4, d: 5}, fn :c, 3, 4 -> :x end) == + %{c: :x} + + # When second map is bigger + assert Map.intersect(%{b: 2, c: 3}, %{a: 1, c: 4, d: 5}, fn :c, 3, 4 -> :x end) == + %{c: :x} + end + + test "implements (almost) all functions in Keyword" do + assert Keyword.__info__(:functions) -- Map.__info__(:functions) == [ + delete: 3, + delete_first: 2, + get_values: 2, + keyword?: 1, + pop_first: 2, + pop_first: 3, + pop_values: 2, + validate: 2, + validate!: 2 + ] + end + + test "variable keys" do + x = :key + %{^x => :value} = %{x => :value} + assert %{x => :value} == %{key: :value} + assert (fn %{^x => :value} -> true end).(%{key: :value}) + + map = %{x => :value} + assert %{map | x => :new_value} == %{x => :new_value} + end + + defmodule ExternalUser do + defstruct name: "john", age: 27 + end + + test "structs" do + assert %ExternalUser{} == %{__struct__: ExternalUser, name: "john", age: 27} + assert %ExternalUser{name: "meg"} == %{__struct__: ExternalUser, name: "meg", age: 27} %ExternalUser{name: name} = %ExternalUser{} - assert name == "josé" + assert name == "john" + end + + describe "structs with variable name" do + test "extracts the struct module" do + %module{name: "john"} = %ExternalUser{name: "john", age: 27} + assert module == ExternalUser + end + + test "returns the struct on match" do + assert Code.eval_string("%struct{} = %ExternalUser{}", [], __ENV__) == + {%ExternalUser{}, [struct: ExternalUser]} + end + + test "supports the pin operator" do + module = ExternalUser + user = %ExternalUser{name: "john", age: 27} + %^module{name: "john"} = user + end + + test "is supported in case" do + user = %ExternalUser{name: "john", age: 27} + + case user do + %module{} = %{age: 27} -> assert module == ExternalUser + end + end + + defp destruct1(%module{}), do: module + defp destruct2(%_{}), do: :ok + + test "does not match" do + invalid_struct = Process.get(:unused, %{__struct__: "foo"}) + + assert_raise CaseClauseError, fn -> + case invalid_struct do + %module{} -> module + end + end + + assert_raise CaseClauseError, fn -> + case invalid_struct do + %_{} -> :ok + end + end + + assert_raise CaseClauseError, fn -> + foo = Process.get(:unused, "foo") - map = %{} - assert_raise BadStructError, "expected a struct named MapTest.ExternalUser, got: %{}", fn -> - %ExternalUser{map | name: "valim"} + case invalid_struct do + %^foo{} -> :ok + end + end + + assert_raise FunctionClauseError, fn -> + destruct1(invalid_struct) + end + + assert_raise FunctionClauseError, fn -> + destruct2(invalid_struct) + end + + assert_raise MatchError, fn -> + %module{} = invalid_struct + _ = module + end + + assert_raise MatchError, fn -> + %_{} = invalid_struct + end + + assert_raise MatchError, fn -> + foo = Process.get(:unused, "foo") + %^foo{} = invalid_struct + end end end + test "structs when using dynamic modules" do + defmodule Module.concat(MapTest, DynamicUser) do + defstruct [:name, :age] + + def sample do + %__MODULE__{} + end + end + end + + test "structs when quoted" do + quoted = + quote do + %User{foo: 1} + end + + assert {:%, [], [aliases, {:%{}, [], [{:foo, 1}]}]} = quoted + assert aliases == {:__aliases__, [alias: false], [:User]} + + quoted = + quote do + %unquote(User){foo: 1} + end + + assert quoted == {:%, [], [User, {:%{}, [], [{:foo, 1}]}]} + end + + test "structs with bitstring defaults" do + defmodule WithBitstring do + defstruct bitstring: <<255, 127::7>> + end + + info = Macro.struct_info!(WithBitstring, __ENV__) + assert info == [%{default: <<255, 127::7>>, field: :bitstring, required: false}] + end + + test "defstruct can only be used once in a module" do + message = + "defstruct has already been called for TestMod, " <> + "defstruct can only be called once per module" + + assert_raise ArgumentError, message, fn -> + Code.eval_string(""" + defmodule TestMod do + defstruct [:foo] + defstruct [:foo] + end + """) + end + end + + test "defstruct allows keys to be enforced" do + message = "the following keys must also be given when building struct TestMod: [:foo]" + + assert_raise ArgumentError, message, fn -> + Code.eval_string(""" + defmodule TestMod do + @enforce_keys :foo + defstruct [:foo] + + # Verify it remain set afterwards + :foo = @enforce_keys + + def foo do + %TestMod{} + end + end + """) + end + end + + test "defstruct raises on invalid enforce_keys" do + message = "keys given to @enforce_keys must be atoms, got: \"foo\"" + + assert_raise ArgumentError, message, fn -> + Code.eval_string(""" + defmodule TestMod do + @enforce_keys "foo" + defstruct [:foo] + end + """) + end + end + + test "struct always expands context module" do + Code.compiler_options(ignore_module_conflict: true) + + defmodule LocalPoint do + defstruct x: 0 + def new, do: %LocalPoint{} + end + + assert LocalPoint.new() == %{__struct__: LocalPoint, x: 0} + + defmodule LocalPoint do + defstruct x: 0, y: 0 + def new, do: %LocalPoint{} + end + + assert LocalPoint.new() == %{__struct__: LocalPoint, x: 0, y: 0} + after + Code.compiler_options(ignore_module_conflict: false) + end + defmodule LocalUser do defmodule NestedUser do defstruct [] end - defstruct name: "josé", nested: struct(NestedUser) + defstruct name: "john", nested: struct(NestedUser), context: %{} def new do %LocalUser{} @@ -142,16 +490,18 @@ defmodule MapTest do end end - test "local user" do - assert LocalUser.new == %LocalUser{name: "josé", nested: %LocalUser.NestedUser{}} - assert LocalUser.Context.new == %LocalUser{name: "josé", nested: %LocalUser.NestedUser{}} + test "local and nested structs" do + assert LocalUser.new() == %LocalUser{name: "john", nested: %LocalUser.NestedUser{}} + assert LocalUser.Context.new() == %LocalUser{name: "john", nested: %LocalUser.NestedUser{}} end - defmodule NilUser do - defstruct name: nil, contents: %{} + defmodule :elixir_struct_from_erlang_module do + defstruct [:hello] + def world(%:elixir_struct_from_erlang_module{} = struct), do: struct end - test "nil user" do - assert %NilUser{} == %{__struct__: NilUser, name: nil, contents: %{}} + test "struct from erlang module" do + struct = %:elixir_struct_from_erlang_module{} + assert :elixir_struct_from_erlang_module.world(struct) == struct end end diff --git a/lib/elixir/test/elixir/module/locals_tracker_test.exs b/lib/elixir/test/elixir/module/locals_tracker_test.exs deleted file mode 100644 index d24f7eb497d..00000000000 --- a/lib/elixir/test/elixir/module/locals_tracker_test.exs +++ /dev/null @@ -1,150 +0,0 @@ -Code.require_file "../test_helper.exs", __DIR__ - -defmodule Module.LocalsTrackerTest do - use ExUnit.Case, async: true - - alias Module.LocalsTracker, as: D - - setup do - {:ok, [pid: D.start_link]} - end - - ## Locals - - test "can add definitions", config do - D.add_definition(config[:pid], :def, {:foo, 1}) - D.add_definition(config[:pid], :defp, {:bar, 1}) - end - - test "can add locals", config do - D.add_definition(config[:pid], :def, {:foo, 1}) - D.add_local(config[:pid], {:foo, 1}, {:bar, 1}) - end - - test "public definitions are always reachable", config do - D.add_definition(config[:pid], :def, {:public, 1}) - assert {:public, 1} in D.reachable(config[:pid]) - - D.add_definition(config[:pid], :defmacro, {:public, 2}) - assert {:public, 2} in D.reachable(config[:pid]) - end - - test "private definitions are never reachable", config do - D.add_definition(config[:pid], :defp, {:private, 1}) - refute {:private, 1} in D.reachable(config[:pid]) - - D.add_definition(config[:pid], :defmacrop, {:private, 2}) - refute {:private, 2} in D.reachable(config[:pid]) - end - - test "private definitions are reachable when connected to local", config do - D.add_definition(config[:pid], :defp, {:private, 1}) - refute {:private, 1} in D.reachable(config[:pid]) - - D.add_local(config[:pid], {:private, 1}) - assert {:private, 1} in D.reachable(config[:pid]) - end - - test "private definitions are reachable when connected through a public one", config do - D.add_definition(config[:pid], :defp, {:private, 1}) - refute {:private, 1} in D.reachable(config[:pid]) - - D.add_definition(config[:pid], :def, {:public, 1}) - D.add_local(config[:pid], {:public, 1}, {:private, 1}) - assert {:private, 1} in D.reachable(config[:pid]) - end - - @unused [ - {{:private, 1}, :defp, 0} - ] - - test "unused private definitions are marked as so", config do - D.add_definition(config[:pid], :def, {:public, 1}) - - unused = D.collect_unused_locals(config[:pid], @unused) - assert unused == [{:unused_def, {:private, 1}, :defp}] - - D.add_local(config[:pid], {:public, 1}, {:private, 1}) - unused = D.collect_unused_locals(config[:pid], @unused) - refute unused == [{:unused_def, {:private, 1}, :defp}] - end - - @unused [ - {{:private, 3}, :defp, 3} - ] - - test "private definitions with unused default arguments", config do - D.add_definition(config[:pid], :def, {:public, 1}) - - unused = D.collect_unused_locals(config[:pid], @unused) - assert unused == [{:unused_def, {:private, 3}, :defp}] - - D.add_local(config[:pid], {:public, 1}, {:private, 3}) - unused = D.collect_unused_locals(config[:pid], @unused) - assert unused == [{:unused_args, {:private, 3}}] - end - - test "private definitions with some unused default arguments", config do - D.add_definition(config[:pid], :def, {:public, 1}) - D.add_local(config[:pid], {:public, 1}, {:private, 1}) - unused = D.collect_unused_locals(config[:pid], @unused) - assert unused == [{:unused_args, {:private, 3}, 1}] - end - - test "private definitions with all used default arguments", config do - D.add_definition(config[:pid], :def, {:public, 1}) - D.add_local(config[:pid], {:public, 1}, {:private, 0}) - unused = D.collect_unused_locals(config[:pid], @unused) - assert unused == [] - end - - ## Defaults - - test "can add defaults", config do - D.add_definition(config[:pid], :def, {:foo, 4}) - D.add_defaults(config[:pid], :def, {:foo, 4}, 2) - end - - test "defaults are reachable if public", config do - D.add_definition(config[:pid], :def, {:foo, 4}) - D.add_defaults(config[:pid], :def, {:foo, 4}, 2) - assert {:foo, 2} in D.reachable(config[:pid]) - assert {:foo, 3} in D.reachable(config[:pid]) - end - - test "defaults are not reachable if private", config do - D.add_definition(config[:pid], :defp, {:foo, 4}) - D.add_defaults(config[:pid], :defp, {:foo, 4}, 2) - refute {:foo, 2} in D.reachable(config[:pid]) - refute {:foo, 3} in D.reachable(config[:pid]) - end - - test "defaults are connected", config do - D.add_definition(config[:pid], :defp, {:foo, 4}) - D.add_defaults(config[:pid], :defp, {:foo, 4}, 2) - D.add_local(config[:pid], {:foo, 2}) - assert {:foo, 2} in D.reachable(config[:pid]) - assert {:foo, 3} in D.reachable(config[:pid]) - assert {:foo, 4} in D.reachable(config[:pid]) - end - - ## Imports - - test "find imports from dispatch", config do - D.add_import(config[:pid], nil, Module, {:concat, 1}) - assert Module in D.imports_with_dispatch(config[:pid], {:concat, 1}) - refute Module in D.imports_with_dispatch(config[:pid], {:unknown, 1}) - end - - test "find import conflicts", config do - refute {[Module], :conflict, 1} in D.collect_imports_conflicts(config[:pid], [conflict: 1]) - - # Calls outside local functions are not triggered - D.add_import(config[:pid], nil, Module, {:conflict, 1}) - refute {[Module], :conflict, 1} in D.collect_imports_conflicts(config[:pid], [conflict: 1]) - - D.add_local(config[:pid], {:foo, 2}) - D.add_import(config[:pid], {:foo, 2}, Module, {:conflict, 1}) - assert {[Module], :conflict, 1} in D.collect_imports_conflicts(config[:pid], [conflict: 1]) - end -end diff --git a/lib/elixir/test/elixir/module/types/descr_test.exs b/lib/elixir/test/elixir/module/types/descr_test.exs new file mode 100644 index 00000000000..49d3eed89b7 --- /dev/null +++ b/lib/elixir/test/elixir/module/types/descr_test.exs @@ -0,0 +1,2931 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team + +Code.require_file("type_helper.exs", __DIR__) + +defmodule NoFieldsStruct do + defstruct [] +end + +defmodule Decimal do + defstruct [:sign, :coef, :exp] +end + +defmodule Module.Types.DescrTest do + use ExUnit.Case, async: true + + import Module.Types.Descr, except: [fun: 1] + defmacro domain_key(arg) when is_atom(arg), do: [arg] + + defp number(), do: union(integer(), float()) + defp empty_tuple(), do: tuple([]) + defp tuple_of_size_at_least(n) when is_integer(n), do: open_tuple(List.duplicate(term(), n)) + defp tuple_of_size(n) when is_integer(n) and n >= 0, do: tuple(List.duplicate(term(), n)) + defp list(elem_type, tail_type), do: union(empty_list(), non_empty_list(elem_type, tail_type)) + defp map_with_default(descr), do: open_map([], if_set(descr)) + + describe "union" do + test "bitmap" do + assert union(integer(), float()) == union(float(), integer()) + end + + test "term" do + assert union(term(), float()) == term() + assert union(term(), binary()) == term() + assert union(term(), if_set(binary())) == if_set(term()) + end + + test "none" do + assert union(none(), float()) == float() + assert union(none(), binary()) == binary() + end + + test "atom" do + assert union(atom(), atom([:a])) == atom() + assert union(atom([:a]), atom([:b])) == atom([:a, :b]) + assert union(atom([:a]), negation(atom([:b]))) == negation(atom([:b])) + + assert union(negation(atom([:a, :b])), negation(atom([:b, :c]))) + |> equal?(negation(atom([:b]))) + end + + test "all primitive types" do + all = [ + atom(), + integer(), + float(), + binary(), + open_map(), + non_empty_list(term(), term()), + empty_list(), + tuple(), + fun(), + pid(), + port(), + reference() + ] + + assert Enum.reduce(all, &union/2) |> equal?(term()) + end + + test "dynamic" do + assert equal?(union(dynamic(), dynamic()), dynamic()) + assert equal?(union(dynamic(), term()), term()) + assert equal?(union(term(), dynamic()), term()) + + assert equal?(union(dynamic(atom()), atom()), atom()) + refute equal?(union(dynamic(atom()), atom()), dynamic(atom())) + + assert equal?(union(term(), dynamic(if_set(integer()))), union(term(), dynamic(not_set()))) + refute equal?(union(term(), dynamic(if_set(integer()))), dynamic(union(term(), not_set()))) + end + + test "tuple" do + assert equal?(union(tuple(), tuple()), tuple()) + + t = tuple([integer(), atom()]) + assert equal?(union(t, t), t) + + assert union(tuple([integer(), atom()]), tuple([float(), atom()])) + |> equal?(tuple([union(integer(), float()), atom()])) + + assert union(tuple([integer(), atom()]), tuple([integer(), binary()])) + |> equal?(tuple([integer(), union(atom(), binary())])) + + assert open_tuple([atom()]) + |> union(tuple([atom(), integer()])) + |> equal?(open_tuple([atom()])) + + assert tuple([union(integer(), atom())]) + |> difference(open_tuple([atom()])) + |> equal?(tuple([integer()])) + end + + test "map" do + assert equal?(union(open_map(), open_map()), open_map()) + assert equal?(union(closed_map(a: integer()), open_map()), open_map()) + assert equal?(union(closed_map(a: integer()), negation(closed_map(a: integer()))), term()) + + a_integer_open = open_map(a: integer()) + assert equal?(union(closed_map(a: integer()), a_integer_open), a_integer_open) + + # Domain key types + atom_to_atom = open_map([{domain_key(:atom), atom()}]) + atom_to_integer = open_map([{domain_key(:atom), integer()}]) + + # Test union identity and different type maps + assert union(atom_to_atom, atom_to_atom) == atom_to_atom + + # Test subtype relationships with domain key maps + refute open_map([{domain_key(:atom), union(atom(), integer())}]) + |> subtype?(union(atom_to_atom, atom_to_integer)) + + assert union(atom_to_atom, atom_to_integer) + |> subtype?(open_map([{domain_key(:atom), union(atom(), integer())}])) + + # Test unions with empty and open maps + assert union(empty_map(), open_map([{domain_key(:integer), atom()}])) + |> equal?(open_map([{domain_key(:integer), atom()}])) + + assert union(open_map(), open_map([{domain_key(:integer), atom()}])) + |> equal?(open_map()) + + # Test union of open map and map with domain key + assert union(open_map(), open_map([{domain_key(:integer), atom()}])) + |> equal?(open_map()) + + # Ensure no duplicate, no matter the order + assert union( + open_map(a: integer()), + open_map(a: number(), b: binary()) + ) + |> union(open_map(a: integer())) == + union( + open_map(a: number(), b: binary()), + open_map(a: integer()) + ) + |> union(open_map(a: integer())) + end + + test "list" do + assert union(list(term()), list(term())) |> equal?(list(term())) + assert union(list(integer()), list(term())) |> equal?(list(term())) + + assert union(difference(list(term()), list(integer())), list(integer())) + |> equal?(list(term())) + end + + test "fun" do + assert equal?(union(fun(), fun()), fun()) + assert equal?(union(fun(), none_fun(1)), fun()) + + dynamic_fun = intersection(fun(), dynamic()) + assert equal?(union(dynamic_fun, fun()), fun()) + end + + test "optimizations (maps)" do + # The tests are checking the actual implementation, not the semantics. + # This is why we are using structural comparisons. + # It's fine to remove these if the implementation changes, but breaking + # these might have an important impact on compile times. + + # Optimization one: same tags, all but one key are structurally equal + assert union( + open_map(a: float(), b: atom()), + open_map(a: integer(), b: atom()) + ) + |> equal?(open_map(a: union(float(), integer()), b: atom())) + + assert union( + closed_map(a: float(), b: atom()), + closed_map(a: integer(), b: atom()) + ) == closed_map(a: union(float(), integer()), b: atom()) + + # Optimization two: we can tell that one map is a subtype of the other: + + assert union( + closed_map(a: term(), b: term()), + closed_map(a: float(), b: binary()) + ) == closed_map(a: term(), b: term()) + + assert union( + open_map(a: term()), + closed_map(a: float(), b: binary()) + ) == open_map(a: term()) + + assert union( + closed_map(a: float(), b: binary()), + open_map(a: term()) + ) == open_map(a: term()) + + assert union( + closed_map(a: term(), b: tuple([term(), term()])), + closed_map(a: float(), b: tuple([atom(), binary()])) + ) == closed_map(a: term(), b: tuple([term(), term()])) + end + + test "optimizations (tuples)" do + # Optimization one: same tags, all but one key are structurally equal + assert union( + open_tuple([float(), atom()]), + open_tuple([integer(), atom()]) + ) == open_tuple([union(float(), integer()), atom()]) + + assert union( + tuple([float(), atom()]), + tuple([integer(), atom()]) + ) == tuple([union(float(), integer()), atom()]) + + # Optimization two: we can tell that one tuple is a subtype of the other: + + assert union( + tuple([term(), term()]), + tuple([float(), binary()]) + ) == tuple([term(), term()]) + + assert union( + open_tuple([term()]), + tuple([float(), binary()]) + ) == open_tuple([term()]) + + assert union( + tuple([float(), binary()]), + open_tuple([term()]) + ) == open_tuple([term()]) + end + end + + describe "intersection" do + test "bitmap" do + assert intersection(integer(), union(integer(), float())) == integer() + assert intersection(integer(), float()) == none() + end + + test "term" do + assert intersection(term(), term()) == term() + assert intersection(term(), float()) == float() + assert intersection(term(), binary()) == binary() + end + + test "none" do + assert intersection(none(), float()) == none() + assert intersection(none(), binary()) == none() + end + + test "atom" do + assert intersection(atom(), atom()) == atom() + assert intersection(atom(), atom([:a])) == atom([:a]) + assert intersection(atom([:a]), atom([:b])) == none() + assert intersection(atom([:a]), negation(atom([:b]))) == atom([:a]) + end + + test "dynamic" do + assert equal?(intersection(dynamic(), dynamic()), dynamic()) + assert equal?(intersection(dynamic(), term()), dynamic()) + assert equal?(intersection(term(), dynamic()), dynamic()) + assert empty?(intersection(dynamic(), none())) + assert empty?(intersection(intersection(dynamic(), atom()), integer())) + + assert empty?(intersection(dynamic(not_set()), term())) + refute empty?(intersection(dynamic(if_set(integer())), term())) + + # Check for structural equivalence + assert intersection(dynamic(not_set()), term()) == none() + end + + test "tuple" do + assert empty?(intersection(open_tuple([atom()]), open_tuple([integer()]))) + + assert intersection(open_tuple([atom()]), tuple([term(), integer()])) + |> equal?(tuple([atom(), integer()])) + + assert intersection(tuple([term(), integer()]), tuple([atom(), term()])) + |> equal?(tuple([atom(), integer()])) + end + + test "map" do + assert intersection(open_map(), open_map()) == open_map() + assert equal?(intersection(closed_map(a: integer()), open_map()), closed_map(a: integer())) + + assert equal?( + intersection(closed_map(a: integer()), open_map(a: integer())), + closed_map(a: integer()) + ) + + assert intersection(closed_map(a: integer()), open_map(b: not_set())) + |> equal?(closed_map(a: integer())) + + assert intersection(closed_map(a: integer()), open_map(b: if_set(integer()))) + |> equal?(closed_map(a: integer())) + + assert equal?( + intersection(closed_map(a: integer()), closed_map(a: if_set(integer()))), + closed_map(a: integer()) + ) + + assert empty?(intersection(closed_map(a: integer()), closed_map(a: atom()))) + end + + test "map with domain keys" do + # %{..., int => t1, atom => t2} and %{int => t3} + # intersection is %{int => t1 and t3, atom => none} + map1 = open_map([{domain_key(:integer), integer()}, {domain_key(:atom), atom()}]) + map2 = closed_map([{domain_key(:integer), number()}]) + + intersection = intersection(map1, map2) + + expected = + closed_map([{domain_key(:integer), integer()}, {domain_key(:atom), none()}]) + + assert equal?(intersection, expected) + + # %{..., int => t1, atom => t2} and %{int => t3, pid => t4} + # intersection is %{int =>t1 and t3, atom => none, pid => t4} + map1 = open_map([{domain_key(:integer), integer()}, {domain_key(:atom), atom()}]) + map2 = closed_map([{domain_key(:integer), float()}, {domain_key(:pid), binary()}]) + + intersection = intersection(map1, map2) + + expected = + closed_map([ + {domain_key(:integer), intersection(integer(), float())}, + {domain_key(:atom), none()}, + {domain_key(:pid), binary()} + ]) + + assert equal?(intersection, expected) + + # %{..., int => t1, string => t3} and %{int => t4} + # intersection is %{int => t1 and t4, string => none} + map1 = open_map([{domain_key(:integer), integer()}, {domain_key(:binary), binary()}]) + map2 = closed_map([{domain_key(:integer), float()}]) + + intersection = intersection(map1, map2) + + assert equal?( + intersection, + closed_map([ + {domain_key(:integer), intersection(integer(), float())}, + {domain_key(:binary), none()} + ]) + ) + + assert subtype?(empty_map(), closed_map([{domain_key(:integer), atom()}])) + + t1 = closed_map([{domain_key(:integer), atom()}]) + t2 = closed_map([{domain_key(:integer), binary()}]) + + assert equal?(intersection(t1, t2), empty_map()) + + t1 = closed_map([{domain_key(:integer), atom()}]) + t2 = closed_map([{domain_key(:atom), term()}]) + + # their intersection is the empty map + refute empty?(intersection(t1, t2)) + assert equal?(intersection(t1, t2), empty_map()) + end + + test "list" do + assert intersection(list(term()), list(term())) == list(term()) + assert intersection(list(integer()), list(integer())) == list(integer()) + assert intersection(list(integer()), list(number())) == list(integer()) + assert intersection(list(integer()), list(atom())) == empty_list() + + # Empty list intersections + assert intersection(empty_list(), list(term())) == empty_list() + assert intersection(empty_list(), list(integer())) == empty_list() + assert intersection(empty_list(), empty_list()) == empty_list() + + # List with any type + assert intersection(list(term()), list(integer())) == list(integer()) + assert intersection(list(term()), list(integer())) == list(integer()) + + # Intersection with more specific types + assert intersection(list(integer()), list(atom([:a, :b]))) == empty_list() + + # Intersection with union types + assert intersection(list(union(integer(), atom())), list(number())) == list(integer()) + + # Intersection with dynamic + assert equal?( + intersection(dynamic(list(term())), list(integer())), + dynamic(list(integer())) + ) + + assert equal?(intersection(dynamic(list(term())), list(term())), dynamic(list(term()))) + + # Nested list intersections + assert intersection(list(list(integer())), list(list(number()))) == list(list(integer())) + assert intersection(list(list(integer())), list(list(atom()))) == list(empty_list()) + + # Intersection with non-list types + assert intersection(list(integer()), integer()) == none() + + # Tests for list with last element + assert intersection(list(float(), atom()), list(number(), term())) == list(float(), atom()) + + assert intersection(list(number(), atom()), list(float(), boolean())) == + list(float(), boolean()) + + assert intersection(list(integer(), float()), list(number(), integer())) == empty_list() + + # Empty list with last element + assert intersection(empty_list(), list(integer(), atom())) == empty_list() + assert intersection(list(integer(), atom()), empty_list()) == empty_list() + + # List with any type and specific last element + assert intersection(list(term(), atom()), list(float(), boolean())) == + list(float(), boolean()) + + assert intersection(list(term(), term()), list(float(), atom())) == list(float(), atom()) + + # Nested lists with last element + assert intersection(list(list(integer()), atom()), list(list(number()), boolean())) == + list(list(integer()), boolean()) + + assert list(list(integer(), atom()), float()) + |> intersection(list(list(number(), boolean()), integer())) == empty_list() + + # Union types in last element + assert intersection(list(integer(), union(atom(), binary())), list(number(), atom())) == + list(integer(), atom()) + + # Dynamic with last element + assert intersection(dynamic(list(term(), atom())), list(integer(), boolean())) + |> equal?(dynamic(list(integer(), boolean()))) + + # Intersection with proper list (should result in empty list) + assert intersection(list(integer(), atom()), list(integer())) == empty_list() + end + + test "function" do + assert not empty?(intersection(negation(none_fun(2)), negation(none_fun(3)))) + end + end + + describe "difference" do + test "bitmap" do + assert difference(float(), integer()) == float() + assert difference(union(float(), integer()), integer()) == float() + assert difference(union(float(), integer()), binary()) == union(float(), integer()) + end + + test "term" do + assert difference(float(), term()) == none() + assert difference(integer(), term()) == none() + end + + test "none" do + assert difference(none(), integer()) == none() + assert difference(none(), float()) == none() + + assert difference(integer(), none()) == integer() + assert difference(float(), none()) == float() + end + + test "atom" do + assert difference(atom([:a]), atom()) == none() + assert difference(atom([:a]), atom([:b])) == atom([:a]) + end + + test "dynamic" do + assert equal?(dynamic(), difference(dynamic(), dynamic())) + assert equal?(dynamic(), difference(term(), dynamic())) + assert empty?(difference(dynamic(), term())) + assert empty?(difference(none(), dynamic())) + assert empty?(difference(dynamic(integer()), integer())) + end + + test "tuple" do + assert empty?(difference(open_tuple([atom()]), open_tuple([term()]))) + refute empty?(difference(tuple(), empty_tuple())) + refute tuple_of_size_at_least(2) |> difference(tuple_of_size(2)) |> empty?() + assert tuple_of_size_at_least(2) |> difference(tuple_of_size_at_least(1)) |> empty?() + assert tuple_of_size_at_least(3) |> difference(tuple_of_size_at_least(3)) |> empty?() + refute tuple_of_size_at_least(2) |> difference(tuple_of_size_at_least(3)) |> empty?() + refute tuple([term(), term()]) |> difference(tuple([atom(), term()])) |> empty?() + refute tuple([term(), term()]) |> difference(tuple([atom()])) |> empty?() + assert tuple([term(), term()]) |> difference(tuple([term(), term()])) |> empty?() + + # {term(), term(), ...} and not ({term(), term(), term(), ...} or {term(), term()}) + assert tuple_of_size_at_least(2) + |> difference(tuple_of_size(2)) + |> difference(tuple_of_size_at_least(3)) + |> empty?() + + assert tuple([term(), term()]) + |> difference(tuple([atom()])) + |> difference(open_tuple([term()])) + |> difference(empty_tuple()) + |> empty?() + + refute difference(tuple(), empty_tuple()) + |> difference(open_tuple([term(), term()])) + |> empty?() + + assert difference(open_tuple([term()]), open_tuple([term(), term()])) + |> difference(tuple([term()])) + |> empty?() + + assert open_tuple([atom()]) + |> difference(tuple([integer(), integer()])) + |> equal?(open_tuple([atom()])) + + assert tuple([union(atom(), integer()), term()]) + |> difference(open_tuple([atom(), term()])) + |> equal?(tuple([integer(), term()])) + + assert tuple([union(atom(), integer()), term()]) + |> difference(open_tuple([atom(), term()])) + |> difference(open_tuple([integer(), term()])) + |> empty?() + + assert tuple([term(), union(atom(), integer()), term()]) + |> difference(open_tuple([term(), integer()])) + |> equal?(tuple([term(), atom(), term()])) + + assert difference(tuple(), open_tuple([term(), term()])) + |> equal?(union(tuple([term()]), tuple([]))) + end + + test "map" do + assert empty?(difference(open_map(), open_map())) + assert empty?(difference(open_map(), term())) + assert equal?(difference(open_map(), none()), open_map()) + assert empty?(difference(closed_map(a: integer()), open_map())) + assert empty?(difference(closed_map(a: integer()), closed_map(a: integer()))) + assert empty?(difference(closed_map(a: integer()), open_map(a: integer()))) + assert empty?(difference(closed_map(a: integer()), open_map(b: if_set(integer())))) + + assert difference(closed_map(a: integer(), b: if_set(atom())), closed_map(a: integer())) + |> difference(closed_map(a: integer(), b: atom())) + |> empty?() + + assert difference(open_map(a: atom()), closed_map(b: integer())) + |> equal?(open_map(a: atom())) + + refute empty?(difference(open_map(), empty_map())) + + assert difference(open_map(a: integer()), closed_map(b: boolean())) + |> equal?(open_map(a: integer())) + end + + test "map with domain keys" do + # Non-overlapping domain keys + t1 = closed_map([{domain_key(:integer), atom()}]) + t2 = closed_map([{domain_key(:atom), binary()}]) + + assert equal?(difference(t1, t2) |> union(empty_map()), t1) + assert empty?(difference(t1, t1)) + + # %{atom() => t1} and not %{atom() => t2} is not %{atom() => t1 and not t2} + t3 = closed_map([{domain_key(:integer), atom()}]) + t4 = closed_map([{domain_key(:integer), atom([:ok])}]) + assert subtype?(difference(t3, t4), t3) + + refute difference(t3, t4) + |> equal?(closed_map([{domain_key(:integer), difference(atom(), atom([:ok]))}])) + + # Difference with a non-domain key map + t5 = closed_map([{domain_key(:integer), union(atom(), integer())}]) + t6 = closed_map(a: atom()) + assert equal?(difference(t5, t6), t5) + + # Removing atom keys from a map with defined atom keys + a_number = closed_map(a: number()) + a_number_and_pids = closed_map([{:a, number()}, {domain_key(:atom), pid()}]) + atom_to_float = closed_map([{domain_key(:atom), float()}]) + atom_to_term = closed_map([{domain_key(:atom), term()}]) + atom_to_pid = closed_map([{domain_key(:atom), pid()}]) + t_diff = difference(a_number, atom_to_float) + + # Removing atom keys that map to float, make the :a key point to integer only. + assert map_fetch_key(t_diff, :a) == {false, integer()} + # %{a => number, atom => pid} and not %{atom => float} gives numbers on :a + assert map_fetch_key(difference(a_number_and_pids, atom_to_float), :a) == {false, number()} + + assert map_fetch_key(t_diff, :foo) == :badkey + + assert subtype?(a_number, atom_to_term) + refute subtype?(a_number, atom_to_float) + + # Removing all atom keys from map %{:a => type} means there is nothing left. + assert empty?(difference(a_number, atom_to_term)) + refute empty?(intersection(atom_to_term, a_number)) + assert empty?(intersection(atom_to_pid, a_number)) + + # (%{:a => number} and not %{:a => float}) is %{:a => integer} + assert equal?(difference(a_number, atom_to_float), closed_map(a: integer())) + end + + test "list" do + # Basic list type differences + assert difference(list(term()), empty_list()) == non_empty_list(term()) + assert difference(list(integer()), list(term())) |> empty?() + + assert difference(list(integer()), list(float())) + |> equal?(non_empty_list(integer())) + + # All list of integers and floats, minus all lists of integers, is NOT all lists of floats + refute difference(list(union(integer(), float())), list(integer())) + |> equal?(non_empty_list(float())) + + # Interactions with empty_list() + assert difference(empty_list(), list(term())) == none() + assert difference(list(integer()), empty_list()) == non_empty_list(integer()) + + # Nested list structures + assert difference(list(list(integer())), list(list(float()))) + |> equal?(difference(list(list(integer())), list(empty_list()))) + + # Lists with union types + refute difference(list(union(integer(), float())), list(integer())) == list(float()) + refute difference(list(union(atom(), binary())), list(atom())) == list(binary()) + + # Tests for list with last element + assert difference(list(integer(), atom()), list(number(), term())) |> empty?() + + assert difference( + list(atom(), term()), + difference(list(atom(), term()), list(atom())) + ) + |> equal?(list(atom())) + + assert difference(list(integer(), float()), list(number(), integer())) + |> equal?(non_empty_list(integer(), difference(float(), integer()))) + + # Empty list with last element + assert difference(empty_list(), list(integer(), atom())) == none() + + assert difference(list(integer(), atom()), empty_list()) == + non_empty_list(integer(), atom()) + + # List with any type and specific last element + assert difference(list(term(), term()), list(term(), integer())) + |> equal?( + non_empty_list(term(), negation(union(integer(), non_empty_list(term(), term())))) + ) + + # Nested lists with last element + # "lists of (lists of integers), ending with atom" + # minus + # "lists of (lists of numbers), ending with boolean" + # gives: + # "non empty lists of (lists of integers), ending with (atom and not boolean)" + + assert difference(list(list(integer()), atom()), list(list(number()), boolean())) + |> equal?(non_empty_list(list(integer()), difference(atom(), boolean()))) + + # Union types in last element + assert difference(list(integer(), union(atom(), binary())), list(number(), atom())) + |> equal?( + union( + non_empty_list(integer(), binary()), + non_empty_list(difference(integer(), number()), union(atom(), binary())) + ) + ) + + # Dynamic with last element + assert equal?( + difference(dynamic(list(term(), atom())), list(integer(), term())), + dynamic(difference(list(term(), atom()), list(integer(), term()))) + ) + + # Difference with proper list + assert difference(list(integer(), atom()), list(integer())) + |> equal?(non_empty_list(integer(), atom())) + end + + test "fun" do + for arity <- [0, 1, 2, 3] do + assert empty?(difference(none_fun(arity), none_fun(arity))) + end + + assert empty?(difference(fun(), fun())) + assert empty?(difference(none_fun(3), fun())) + refute empty?(difference(fun(), none_fun(1))) + refute empty?(difference(none_fun(2), none_fun(3))) + assert empty?(intersection(none_fun(2), none_fun(3))) + + f1f2 = union(none_fun(1), none_fun(2)) + assert f1f2 |> difference(none_fun(1)) |> difference(none_fun(2)) |> empty?() + assert none_fun(1) |> difference(difference(f1f2, none_fun(2))) |> empty?() + assert f1f2 |> difference(none_fun(1)) |> equal?(none_fun(2)) + + assert fun([integer()], term()) |> difference(fun([none()], term())) |> empty?() + end + end + + describe "creation" do + test "map hoists dynamic" do + assert dynamic(open_map(a: integer())) == open_map(a: dynamic(integer())) + + assert dynamic(open_map(a: union(integer(), binary()))) == + open_map(a: dynamic(integer()) |> union(binary())) + + # For domains too + t1 = dynamic(open_map([{domain_key(:integer), integer()}])) + t2 = open_map([{domain_key(:integer), dynamic(integer())}]) + assert t1 == t2 + + # if_set on dynamic fields also must work + t1 = dynamic(open_map(a: if_set(integer()))) + t2 = open_map(a: if_set(dynamic(integer()))) + assert t1 == t2 + end + end + + describe "subtype" do + test "bitmap" do + assert subtype?(integer(), union(integer(), float())) + assert subtype?(integer(), integer()) + assert subtype?(integer(), term()) + assert subtype?(none(), integer()) + assert subtype?(integer(), negation(float())) + end + + test "atom" do + assert subtype?(atom([:a]), atom()) + assert subtype?(atom([:a]), atom([:a])) + assert subtype?(atom([:a]), term()) + assert subtype?(none(), atom([:a])) + assert subtype?(atom([:a]), atom([:a, :b])) + assert subtype?(atom([:a]), negation(atom([:b]))) + end + + test "dynamic" do + assert subtype?(dynamic(), term()) + assert subtype?(dynamic(), dynamic()) + refute subtype?(term(), dynamic()) + assert subtype?(intersection(dynamic(), integer()), integer()) + assert subtype?(integer(), union(dynamic(), integer())) + end + + test "tuple" do + assert subtype?(empty_tuple(), tuple()) + assert subtype?(tuple([integer(), atom()]), tuple()) + refute subtype?(empty_tuple(), open_tuple([term()])) + assert subtype?(tuple([integer(), atom()]), tuple([term(), term()])) + refute subtype?(tuple([integer(), atom()]), tuple([integer(), integer()])) + refute subtype?(tuple([integer(), atom()]), tuple([atom(), atom()])) + + assert subtype?(tuple([integer(), atom()]), open_tuple([integer(), atom()])) + refute subtype?(tuple([term()]), open_tuple([term(), term()])) + refute subtype?(tuple([integer(), atom()]), open_tuple([integer(), integer()])) + refute subtype?(open_tuple([integer(), atom()]), tuple([integer(), integer()])) + end + + test "map" do + assert subtype?(open_map(), term()) + assert subtype?(closed_map(a: integer()), open_map()) + assert subtype?(closed_map(a: integer()), closed_map(a: integer())) + assert subtype?(closed_map(a: integer()), open_map(a: integer())) + assert subtype?(closed_map(a: integer(), b: atom()), open_map(a: integer())) + assert subtype?(closed_map(a: integer()), closed_map(a: union(integer(), atom()))) + + # optional + refute subtype?(closed_map(a: if_set(integer())), closed_map(a: integer())) + assert subtype?(closed_map(a: integer()), closed_map(a: if_set(integer()))) + refute subtype?(closed_map(a: if_set(term())), closed_map(a: term())) + assert subtype?(closed_map(a: term()), closed_map(a: if_set(term()))) + + # With domains + t1 = closed_map([{domain_key(:integer), number()}]) + t2 = closed_map([{domain_key(:integer), integer()}]) + + assert subtype?(t2, t1) + + t1_minus_t2 = difference(t1, t2) + refute empty?(t1_minus_t2) + + assert subtype?(map_with_default(number()), open_map()) + t = difference(open_map(), map_with_default(number())) + refute empty?(t) + refute subtype?(open_map(), map_with_default(number())) + assert subtype?(map_with_default(integer()), map_with_default(number())) + refute subtype?(map_with_default(float()), map_with_default(atom())) + + assert equal?( + intersection(map_with_default(number()), map_with_default(float())), + map_with_default(float()) + ) + end + + test "optional" do + refute subtype?(if_set(none()), term()) + refute subtype?(if_set(term()), term()) + assert subtype?(if_set(term()), if_set(term())) + end + + test "list" do + refute subtype?(non_empty_list(integer()), difference(list(number()), list(integer()))) + assert subtype?(list(term(), boolean()), list(term(), atom())) + assert subtype?(list(integer()), list(term())) + assert subtype?(list(term()), list(term(), term())) + end + + test "fun" do + assert equal?(fun([], term()), fun([], term())) + refute equal?(fun([], integer()), fun([], atom())) + refute subtype?(fun([none()], term()), fun([integer()], integer())) + + # Difference with argument/return type variations + int_to_atom = fun([integer()], atom()) + num_to_atom = fun([number()], atom()) + int_to_bool = fun([integer()], boolean()) + + # number->atom is a subtype of int->atom + assert subtype?(num_to_atom, int_to_atom) + refute subtype?(int_to_atom, num_to_atom) + assert subtype?(int_to_bool, int_to_atom) + refute subtype?(int_to_bool, num_to_atom) + + # Multi-arity + f1 = fun([integer(), atom()], boolean()) + f2 = fun([number(), atom()], boolean()) + + # (int,atom)->boolean is a subtype of (number,atom)->boolean + # since number is a supertype of int + assert subtype?(f2, f1) + # f1 is not a subtype of f2 + refute subtype?(f1, f2) + + # Unary functions / Output covariance + assert subtype?(fun([], float()), fun([], term())) + refute subtype?(fun([], term()), fun([], float())) + + # Contravariance of domain + refute subtype?(fun([integer()], boolean()), fun([number()], boolean())) + assert subtype?(fun([number()], boolean()), fun([integer()], boolean())) + + # Nested function types + higher_order = fun([fun([integer()], atom())], boolean()) + specific = fun([fun([number()], atom())], boolean()) + + assert subtype?(higher_order, specific) + refute subtype?(specific, higher_order) + + ## Multi-arity + f = fun([none(), integer()], atom()) + assert subtype?(f, f) + assert subtype?(f, fun([none(), integer()], term())) + assert subtype?(fun([none(), number()], atom()), f) + assert subtype?(fun([tuple(), number()], atom()), f) + + # (none, float -> atom) is not a subtype of (none, integer -> atom) + # because float has an empty intersection with integer. + # it's only possible to find this out by doing the + # intersection one by one. + refute subtype?(fun([none(), float()], atom()), f) + refute subtype?(fun([pid(), float()], atom()), f) + # A function with the wrong arity is refused + refute subtype?(fun([none()], atom()), f) + end + end + + describe "compatible" do + test "intersection" do + assert compatible?(integer(), intersection(dynamic(), integer())) + refute compatible?(intersection(dynamic(), integer()), atom()) + refute compatible?(atom(), intersection(dynamic(), integer())) + refute compatible?(atom(), intersection(dynamic(), atom([:foo, :bar]))) + assert compatible?(intersection(dynamic(), atom()), atom([:foo, :bar])) + assert compatible?(atom([:foo, :bar]), intersection(dynamic(), atom())) + end + + test "static" do + refute compatible?(atom(), atom([:foo, :bar])) + refute compatible?(union(integer(), atom()), integer()) + refute compatible?(none(), integer()) + refute compatible?(union(atom(), dynamic()), integer()) + end + + test "dynamic" do + assert compatible?(dynamic(), term()) + assert compatible?(term(), dynamic()) + assert compatible?(dynamic(), integer()) + assert compatible?(integer(), dynamic()) + end + + test "tuple" do + assert compatible?(dynamic(tuple()), tuple([integer(), atom()])) + end + + test "map" do + assert compatible?(closed_map(a: integer()), open_map()) + assert compatible?(intersection(dynamic(), open_map()), closed_map(a: integer())) + end + + test "list" do + assert compatible?(dynamic(), list(term())) + assert compatible?(dynamic(list(term())), list(integer())) + assert compatible?(dynamic(list(term(), term())), list(integer(), integer())) + end + end + + describe "empty?" do + test "tuple" do + assert tuple([none()]) |> empty?() + assert open_tuple([integer(), none()]) |> empty?() + + assert intersection(tuple([integer(), atom()]), open_tuple([atom()])) |> empty?() + refute open_tuple([integer(), integer()]) |> difference(empty_tuple()) |> empty?() + refute open_tuple([integer(), integer()]) |> difference(open_tuple([atom()])) |> empty?() + refute open_tuple([term()]) |> difference(tuple([term()])) |> empty?() + assert difference(tuple(), empty_tuple()) |> difference(open_tuple([term()])) |> empty?() + assert difference(tuple(), open_tuple([term()])) |> difference(empty_tuple()) |> empty?() + + refute open_tuple([term()]) + |> difference(tuple([term()])) + |> difference(tuple([term()])) + |> empty?() + + assert tuple([integer(), union(integer(), atom())]) + |> difference(tuple([integer(), integer()])) + |> difference(tuple([integer(), atom()])) + |> empty?() + end + + test "map" do + assert open_map(a: none()) |> empty?() + assert closed_map(a: integer(), b: none()) |> empty?() + assert intersection(closed_map(b: atom()), open_map(a: integer())) |> empty?() + end + + test "fun" do + refute empty?(fun()) + refute empty?(none_fun(1)) + refute empty?(fun([integer()], atom())) + + assert empty?(intersection(none_fun(1), none_fun(2))) + refute empty?(intersection(fun(), none_fun(1))) + assert empty?(difference(none_fun(1), union(none_fun(1), none_fun(2)))) + end + end + + describe "function creation" do + test "fun_from_non_overlapping_clauses" do + assert fun_from_non_overlapping_clauses([{[integer()], atom()}, {[float()], binary()}]) == + intersection(fun([integer()], atom()), fun([float()], binary())) + end + + test "fun_from_inferred_clauses" do + # No overlap + assert fun_from_inferred_clauses([{[integer()], atom()}, {[float()], binary()}]) + |> equal?( + intersection( + fun_from_non_overlapping_clauses([{[integer()], atom()}, {[float()], binary()}]), + fun([number()], dynamic()) + ) + ) + + # Subsets + assert fun_from_inferred_clauses([{[integer()], atom()}, {[number()], binary()}]) + |> equal?( + fun_from_non_overlapping_clauses([ + {[integer()], dynamic(union(atom(), binary()))}, + {[number()], dynamic(union(atom(), binary()))} + ]) + ) + + assert fun_from_inferred_clauses([{[number()], binary()}, {[integer()], atom()}]) + |> equal?( + fun_from_non_overlapping_clauses([ + {[integer()], dynamic(union(atom(), binary()))}, + {[number()], dynamic(union(atom(), binary()))} + ]) + ) + + # Partial + assert fun_from_inferred_clauses([ + {[union(integer(), pid())], atom()}, + {[union(float(), pid())], binary()} + ]) + |> equal?( + fun_from_non_overlapping_clauses([ + {[union(integer(), pid())], dynamic(union(atom(), binary()))}, + {[union(float(), pid())], dynamic(union(atom(), binary()))} + ]) + ) + + # Difference + assert fun_from_inferred_clauses([ + {[integer(), union(pid(), atom())], atom()}, + {[number(), pid()], binary()} + ]) + |> equal?( + fun_from_non_overlapping_clauses([ + {[integer(), union(pid(), atom())], dynamic(union(atom(), binary()))}, + {[number(), pid()], dynamic(union(atom(), binary()))} + ]) + ) + end + end + + describe "function application" do + defp none_fun(arity), do: %{fun: {:union, %{arity => :bdd_top}}} + + test "non funs" do + assert fun_apply(term(), [integer()]) == :badfun + assert fun_apply(union(integer(), none_fun(1)), [integer()]) == :badfun + end + + test "static" do + # Full static + assert fun_apply(fun(), [integer()]) == {:badarg, [none()]} + assert fun_apply(difference(fun(), none_fun(2)), [integer()]) == {:badarg, [none()]} + + # Basic function application scenarios + assert fun_apply(fun([integer()], atom()), [integer()]) == {:ok, atom()} + assert fun_apply(fun([integer()], atom()), [float()]) == {:badarg, [integer()]} + assert fun_apply(fun([integer()], atom()), [term()]) == {:badarg, [integer()]} + + # Return types + assert fun_apply(fun([integer()], none()), [integer()]) == {:ok, none()} + assert fun_apply(fun([integer()], term()), [integer()]) == {:ok, term()} + + # Dynamic args + assert fun_apply(fun([term()], term()), [dynamic()]) == {:ok, term()} + + assert fun_apply(fun([integer()], atom()), [dynamic(integer())]) + |> elem(1) + |> equal?(atom()) + + assert fun_apply(fun([integer()], atom()), [dynamic(float())]) == {:badarg, [integer()]} + assert fun_apply(fun([integer()], atom()), [dynamic(term())]) == {:ok, dynamic()} + + # Arity mismatches + assert fun_apply(fun([integer()], integer()), [term(), term()]) == {:badarity, [1]} + assert fun_apply(fun([integer(), atom()], boolean()), [integer()]) == {:badarity, [2]} + + # Function intersection tests (no overlap) + fun0 = intersection(fun([integer()], atom()), fun([float()], binary())) + assert fun_apply(fun0, [integer()]) == {:ok, atom()} + assert fun_apply(fun0, [float()]) == {:ok, binary()} + assert fun_apply(fun0, [number()]) == {:ok, union(atom(), binary())} + + assert fun_apply(fun0, [dynamic(integer())]) |> elem(1) |> equal?(atom()) + assert fun_apply(fun0, [dynamic(float())]) |> elem(1) |> equal?(binary()) + assert fun_apply(fun0, [dynamic(number())]) |> elem(1) |> equal?(union(atom(), binary())) + assert fun_apply(fun0, [dynamic()]) == {:ok, dynamic()} + + # Function intersection tests (overlap) + fun1 = intersection(fun([integer()], atom()), fun([number()], term())) + assert fun_apply(fun1, [integer()]) == {:ok, atom()} + assert fun_apply(fun1, [float()]) == {:ok, term()} + + # Function intersection with unions + fun2 = + intersection( + fun([union(integer(), atom())], term()), + fun([union(integer(), pid())], atom()) + ) + + assert fun_apply(fun2, [integer()]) == {:ok, atom()} + assert fun_apply(fun2, [atom()]) == {:ok, term()} + assert fun_apply(fun2, [pid()]) == {:ok, atom()} + + # Function intersection with same domain, different codomains + assert fun([integer()], term()) + |> intersection(fun([integer()], atom())) + |> fun_apply([integer()]) == {:ok, atom()} + + # Function intersection with singleton atoms + fun3 = intersection(fun([atom([:ok])], atom([:success])), fun([atom([:ok])], atom([:done]))) + assert fun_apply(fun3, [atom([:ok])]) == {:ok, none()} + end + + test "static with dynamic signature" do + assert fun_apply(fun([dynamic()], term()), [dynamic()]) == {:ok, term()} + assert fun_apply(fun([integer()], dynamic()), [integer()]) == {:ok, dynamic()} + + assert fun_apply(fun([dynamic()], integer()), [dynamic()]) == + {:ok, union(integer(), dynamic())} + + assert fun_apply(fun([dynamic(), atom()], float()), [dynamic(), atom()]) == + {:ok, union(float(), dynamic())} + + fun = fun([dynamic(integer())], atom()) + assert fun_apply(fun, [dynamic(integer())]) == {:ok, union(atom(), dynamic())} + assert fun_apply(fun, [dynamic(number())]) == {:ok, dynamic()} + assert fun_apply(fun, [integer()]) == {:ok, dynamic()} + assert fun_apply(fun, [float()]) == {:badarg, [dynamic(integer())]} + end + + defp dynamic_fun(args, return), do: dynamic(fun(args, return)) + + test "dynamic" do + # Full dynamic + assert fun_apply(dynamic(), [integer()]) == {:ok, dynamic()} + assert fun_apply(dynamic(none_fun(1)), [integer()]) == {:ok, dynamic()} + assert fun_apply(difference(dynamic(), none_fun(2)), [integer()]) == {:ok, dynamic()} + + # Basic function application scenarios + assert fun_apply(dynamic_fun([integer()], atom()), [integer()]) == {:ok, dynamic(atom())} + assert fun_apply(dynamic_fun([integer()], atom()), [float()]) == {:ok, dynamic()} + assert fun_apply(dynamic_fun([integer()], atom()), [term()]) == {:ok, dynamic()} + assert fun_apply(dynamic_fun([integer()], none()), [integer()]) == {:ok, dynamic(none())} + assert fun_apply(dynamic_fun([integer()], term()), [integer()]) == {:ok, dynamic()} + + # Dynamic return and dynamic args + assert fun_apply(dynamic_fun([term()], term()), [dynamic()]) == {:ok, dynamic()} + + fun = dynamic_fun([integer()], binary()) + assert fun_apply(fun, [integer()]) == {:ok, dynamic(binary())} + assert fun_apply(fun, [dynamic(integer())]) == {:ok, dynamic(binary())} + assert fun_apply(fun, [dynamic(atom())]) == {:ok, dynamic()} + + # Arity mismatches + assert fun_apply(dynamic_fun([integer()], integer()), [term(), term()]) == {:badarity, [1]} + + assert fun_apply(dynamic_fun([integer(), atom()], boolean()), [integer()]) == + {:badarity, [2]} + + # Function intersection tests + fun0 = intersection(dynamic_fun([integer()], atom()), dynamic_fun([float()], binary())) + assert fun_apply(fun0, [integer()]) == {:ok, dynamic(atom())} + assert fun_apply(fun0, [float()]) == {:ok, dynamic(binary())} + assert fun_apply(fun0, [dynamic(integer())]) == {:ok, dynamic(atom())} + assert fun_apply(fun0, [dynamic(float())]) == {:ok, dynamic(binary())} + assert fun_apply(fun0, [dynamic(number())]) == {:ok, dynamic(union(binary(), atom()))} + + # Function intersection with subset domain + fun1 = intersection(dynamic_fun([integer()], atom()), dynamic_fun([number()], term())) + assert fun_apply(fun1, [integer()]) == {:ok, dynamic(atom())} + assert fun_apply(fun1, [float()]) == {:ok, dynamic()} + assert fun_apply(fun1, [dynamic(integer())]) == {:ok, dynamic(atom())} + assert fun_apply(fun1, [dynamic(float())]) == {:ok, dynamic()} + + # Function intersection with same domain, different codomains + assert dynamic_fun([integer()], term()) + |> intersection(dynamic_fun([integer()], atom())) + |> fun_apply([integer()]) == {:ok, dynamic(atom())} + + # Function intersection with overlapping domains + fun2 = + intersection( + dynamic_fun([union(integer(), atom())], term()), + dynamic_fun([union(integer(), pid())], atom()) + ) + + assert fun_apply(fun2, [integer()]) == {:ok, dynamic(atom())} + assert fun_apply(fun2, [atom()]) == {:ok, dynamic()} + assert fun_apply(fun2, [pid()]) |> elem(1) |> equal?(dynamic(atom())) + + assert fun_apply(fun2, [dynamic(integer())]) == {:ok, dynamic(atom())} + assert fun_apply(fun2, [dynamic(atom())]) == {:ok, dynamic()} + assert fun_apply(fun2, [dynamic(pid())]) |> elem(1) |> equal?(dynamic(atom())) + + # Function intersection with singleton atoms + fun3 = + intersection( + dynamic_fun([atom([:ok])], atom([:success])), + dynamic_fun([atom([:ok])], atom([:done])) + ) + + assert fun_apply(fun3, [atom([:ok])]) == {:ok, dynamic(none())} + end + + test "static and dynamic" do + # Bad arity + fun_arities = + union( + fun([atom()], integer()), + dynamic_fun([integer(), float()], binary()) + ) + + assert fun_arities + |> fun_apply([atom()]) + |> elem(1) + |> equal?(integer()) + + assert fun_arities |> fun_apply([integer(), float()]) == {:badarity, [1]} + + # Bad argument + fun_args = + union( + fun([atom()], integer()), + dynamic_fun([integer()], binary()) + ) + + assert fun_args |> fun_apply([atom()]) == {:ok, dynamic()} + assert fun_args |> fun_apply([integer()]) == {:badarg, [dynamic(atom())]} + + # Badfun + assert union( + fun([atom()], integer()), + dynamic_fun([integer()], binary()) |> intersection(none_fun(2)) + ) + |> fun_apply([atom()]) + |> elem(1) + |> equal?(integer()) + + assert union( + fun([atom()], integer()) |> intersection(none_fun(2)), + dynamic_fun([integer()], binary()) + ) + |> fun_apply([integer()]) == {:ok, dynamic(binary())} + end + end + + describe "projections" do + test "truthiness" do + for type <- [term(), none(), atom(), boolean(), union(atom([false]), integer())] do + assert truthiness(type) == :undefined + assert truthiness(dynamic(type)) == :undefined + end + + for type <- [atom([false]), atom([nil]), atom([nil, false]), atom([false, nil])] do + assert truthiness(type) == :always_false + assert truthiness(dynamic(type)) == :always_false + end + + for type <- + [negation(atom()), atom([true]), negation(atom([false, nil])), atom([:ok]), integer()] do + assert truthiness(type) == :always_true + assert truthiness(dynamic(type)) == :always_true + end + end + + test "atom_fetch" do + assert atom_fetch(term()) == :error + assert atom_fetch(union(term(), dynamic(atom([:foo, :bar])))) == :error + + assert atom_fetch(atom()) == {:infinite, []} + assert atom_fetch(dynamic()) == {:infinite, []} + + assert atom_fetch(atom([:foo, :bar])) == + {:finite, [:foo, :bar] |> :sets.from_list(version: 2) |> :sets.to_list()} + + assert atom_fetch(union(atom([:foo, :bar]), dynamic(atom()))) == {:infinite, []} + assert atom_fetch(union(atom([:foo, :bar]), dynamic(term()))) == {:infinite, []} + end + + test "list_hd" do + assert list_hd(none()) == :badnonemptylist + assert list_hd(term()) == :badnonemptylist + assert list_hd(list(term())) == :badnonemptylist + assert list_hd(empty_list()) == :badnonemptylist + assert list_hd(non_empty_list(term())) == {:ok, term()} + assert list_hd(non_empty_list(integer())) == {:ok, integer()} + assert list_hd(difference(list(number()), list(integer()))) == {:ok, number()} + + assert list_hd(dynamic()) == {:ok, dynamic()} + assert list_hd(dynamic(list(integer()))) == {:ok, dynamic(integer())} + assert list_hd(union(dynamic(), atom())) == :badnonemptylist + assert list_hd(union(dynamic(), list(term()))) == :badnonemptylist + + assert list_hd(difference(list(number()), list(number()))) == :badnonemptylist + assert list_hd(dynamic(difference(list(number()), list(number())))) == :badnonemptylist + + assert list_hd(union(dynamic(list(float())), non_empty_list(atom()))) == + {:ok, union(dynamic(float()), atom())} + + # If term() is in the tail, it means list(term()) is in the tail + # and therefore any term can be returned from hd. + assert list_hd(non_empty_list(atom(), term())) == {:ok, term()} + assert list_hd(non_empty_list(atom(), negation(list(term(), term())))) == {:ok, atom()} + end + + test "list_tl" do + assert list_tl(none()) == :badnonemptylist + assert list_tl(term()) == :badnonemptylist + assert list_tl(empty_list()) == :badnonemptylist + assert list_tl(list(integer())) == :badnonemptylist + assert list_tl(difference(list(number()), list(number()))) == :badnonemptylist + + assert list_tl(non_empty_list(integer())) == {:ok, list(integer())} + + assert list_tl(non_empty_list(integer(), atom())) == + {:ok, union(atom(), non_empty_list(integer(), atom()))} + + # The tail of either a (non empty) list of integers with an atom tail or a (non empty) list + # of tuples with a float tail is either an atom, or a float, or a (possibly empty) list of + # integers with an atom tail, or a (possibly empty) list of tuples with a float tail. + assert list_tl(union(non_empty_list(integer(), atom()), non_empty_list(tuple(), float()))) == + {:ok, + atom() + |> union(float()) + |> union( + union(non_empty_list(integer(), atom()), non_empty_list(tuple(), float())) + )} + + assert list_tl(dynamic()) == {:ok, dynamic()} + assert list_tl(dynamic(list(integer()))) == {:ok, dynamic(list(integer()))} + + assert list_tl(dynamic(list(integer(), atom()))) == + {:ok, dynamic(union(atom(), list(integer(), atom())))} + end + + test "tuple_fetch" do + assert tuple_fetch(term(), 0) == :badtuple + assert tuple_fetch(integer(), 0) == :badtuple + assert tuple_fetch(tuple([none(), atom()]), 1) == :badtuple + assert tuple_fetch(tuple([none()]), 0) == :badtuple + + assert tuple_fetch(tuple([integer(), atom()]), 0) == {false, integer()} + assert tuple_fetch(tuple([integer(), atom()]), 1) == {false, atom()} + assert tuple_fetch(tuple([integer(), atom()]), 2) == :badindex + + assert tuple_fetch(open_tuple([integer(), atom()]), 0) == {false, integer()} + assert tuple_fetch(open_tuple([integer(), atom()]), 1) == {false, atom()} + assert tuple_fetch(open_tuple([integer(), atom()]), 2) == :badindex + + assert tuple_fetch(tuple([integer(), atom()]), -1) == :badindex + assert tuple_fetch(empty_tuple(), 0) == :badindex + assert difference(tuple(), tuple()) |> tuple_fetch(0) == :badtuple + + assert tuple([atom()]) |> difference(empty_tuple()) |> tuple_fetch(0) == + {false, atom()} + + assert difference(tuple([union(integer(), atom())]), open_tuple([atom()])) + |> tuple_fetch(0) == {false, integer()} + + assert tuple_fetch(union(tuple([integer(), atom()]), dynamic(open_tuple([atom()]))), 1) + |> Kernel.then(fn {opt, ty} -> opt and equal?(ty, union(atom(), dynamic())) end) + + assert tuple_fetch(union(tuple([integer()]), tuple([atom()])), 0) == + {false, union(integer(), atom())} + + assert tuple([integer(), atom(), union(atom(), integer())]) + |> difference(tuple([integer(), term(), atom()])) + |> tuple_fetch(2) == {false, integer()} + + assert tuple([integer(), atom(), union(union(atom(), integer()), list(term()))]) + |> difference(tuple([integer(), term(), atom()])) + |> difference(open_tuple([term(), atom(), list(term())])) + |> tuple_fetch(2) == {false, integer()} + + assert tuple([integer(), atom(), integer()]) + |> difference(tuple([integer(), term(), integer()])) + |> tuple_fetch(1) == :badtuple + + assert tuple([integer(), atom(), integer()]) + |> difference(tuple([integer(), term(), atom()])) + |> tuple_fetch(2) == {false, integer()} + + assert tuple_fetch(tuple(), 0) == :badindex + end + + test "tuple_fetch with dynamic" do + assert tuple_fetch(dynamic(), 0) == {true, dynamic()} + assert tuple_fetch(dynamic(empty_tuple()), 0) == :badindex + assert tuple_fetch(dynamic(tuple([integer(), atom()])), 2) == :badindex + assert tuple_fetch(union(dynamic(), integer()), 0) == :badtuple + + assert tuple_fetch(dynamic(tuple()), 0) + |> Kernel.then(fn {opt, type} -> opt and equal?(type, dynamic()) end) + + assert tuple_fetch(union(dynamic(), open_tuple([atom()])), 0) == + {true, union(atom(), dynamic())} + end + + test "tuple_delete_at" do + assert tuple_delete_at(tuple([integer(), atom()]), 3) == :badindex + assert tuple_delete_at(tuple([integer(), atom()]), -1) == :badindex + assert tuple_delete_at(empty_tuple(), 0) == :badindex + assert tuple_delete_at(integer(), 0) == :badtuple + assert tuple_delete_at(term(), 0) == :badtuple + assert tuple_delete_at(tuple([none()]), 0) == :badtuple + + # Test deleting an element from a closed tuple + assert tuple_delete_at(tuple([integer(), atom(), boolean()]), 1) == + tuple([integer(), boolean()]) + + # Test deleting the last element from a closed tuple + assert tuple_delete_at(tuple([integer(), atom()]), 1) == + tuple([integer()]) + + # Test deleting from an open tuple + assert tuple_delete_at(open_tuple([integer(), atom(), boolean()]), 1) == + open_tuple([integer(), boolean()]) + + # Test deleting from a dynamic tuple + assert tuple_delete_at(dynamic(tuple([integer(), atom()])), 1) == + dynamic(tuple([integer()])) + + # Test deleting from a union of tuples + assert tuple_delete_at(union(tuple([integer(), atom()]), tuple([float(), binary()])), 1) + |> equal?(tuple([union(integer(), float())])) + + # Test deleting from an intersection of tuples + assert intersection(tuple([integer(), atom()]), tuple([term(), boolean()])) + |> tuple_delete_at(1) == tuple([integer()]) + + # Test deleting from a difference of tuples + assert difference(tuple([integer(), atom(), boolean()]), tuple([term(), term()])) + |> tuple_delete_at(1) + |> equal?(tuple([integer(), boolean()])) + + # Test deleting from a complex union involving dynamic + assert union(tuple([integer(), atom()]), dynamic(tuple([float(), binary()]))) + |> tuple_delete_at(1) + |> equal?(union(tuple([integer()]), dynamic(tuple([float()])))) + + # Successfully deleting at position `index` in a tuple means that the dynamic + # values that succeed are intersected with tuples of size at least `index` + assert dynamic(tuple()) |> tuple_delete_at(0) == dynamic(tuple()) + assert dynamic(term()) |> tuple_delete_at(0) == dynamic(tuple()) + + assert dynamic(union(tuple(), integer())) + |> tuple_delete_at(1) + |> equal?(dynamic(tuple_of_size_at_least(1))) + end + + test "tuple_insert_at" do + assert tuple_insert_at(tuple([integer(), atom()]), 3, boolean()) == :badindex + assert tuple_insert_at(tuple([integer(), atom()]), -1, boolean()) == :badindex + assert tuple_insert_at(integer(), 0, boolean()) == :badtuple + assert tuple_insert_at(term(), 0, boolean()) == :badtuple + + # Out-of-bounds in a union + assert union(tuple([integer(), atom()]), tuple([float()])) + |> tuple_insert_at(2, boolean()) == :badindex + + # Test inserting into a closed tuple + assert tuple_insert_at(tuple([integer(), atom()]), 1, boolean()) == + tuple([integer(), boolean(), atom()]) + + # Test inserting at the beginning of a tuple + assert tuple_insert_at(tuple([integer(), atom()]), 0, boolean()) == + tuple([boolean(), integer(), atom()]) + + # Test inserting at the end of a tuple + assert tuple_insert_at(tuple([integer(), atom()]), 2, boolean()) == + tuple([integer(), atom(), boolean()]) + + # Test inserting into an empty tuple + assert tuple_insert_at(empty_tuple(), 0, integer()) == tuple([integer()]) + + # Test inserting into an open tuple + assert tuple_insert_at(open_tuple([integer(), atom()]), 1, boolean()) == + open_tuple([integer(), boolean(), atom()]) + + # Test inserting a dynamic type + assert tuple_insert_at(tuple([integer(), atom()]), 1, dynamic()) == + dynamic(tuple([integer(), term(), atom()])) + + # Test inserting into a dynamic tuple + assert tuple_insert_at(dynamic(tuple([integer(), atom()])), 1, boolean()) == + dynamic(tuple([integer(), boolean(), atom()])) + + # Test inserting into a union of tuples + assert tuple_insert_at(union(tuple([integer()]), tuple([atom()])), 0, boolean()) == + union(tuple([boolean(), integer()]), tuple([boolean(), atom()])) + + # Test inserting into a difference of tuples + assert difference(tuple([integer(), atom(), boolean()]), tuple([term(), term()])) + |> tuple_insert_at(1, float()) + |> equal?(tuple([integer(), float(), atom(), boolean()])) + + # Test inserting into a complex union involving dynamic + assert union(tuple([integer(), atom()]), dynamic(tuple([float(), binary()]))) + |> tuple_insert_at(1, boolean()) + |> equal?( + union( + tuple([integer(), boolean(), atom()]), + dynamic(tuple([float(), boolean(), binary()])) + ) + ) + + # If you successfully intersect at position index in a type, then the dynamic values + # that succeed are intersected with tuples of size at least index + assert dynamic(union(tuple(), integer())) + |> tuple_insert_at(1, boolean()) + |> equal?(dynamic(open_tuple([term(), boolean()]))) + end + + test "tuple_values" do + assert tuple_values(integer()) == :badtuple + assert tuple_values(tuple([none()])) == :badtuple + assert tuple_values(tuple([])) == none() + assert tuple_values(tuple()) == term() + assert tuple_values(open_tuple([integer()])) == term() + assert tuple_values(tuple([integer(), atom()])) == union(integer(), atom()) + + assert tuple_values(union(tuple([float(), pid()]), tuple([reference()]))) == + union(float(), union(pid(), reference())) + + assert tuple_values(difference(tuple([number(), atom()]), tuple([float(), term()]))) == + union(integer(), atom()) + + assert union(tuple([atom([:ok])]), open_tuple([integer()])) + |> difference(open_tuple([term(), term()])) + |> tuple_values() == union(atom([:ok]), integer()) + + assert tuple_values(difference(tuple([number(), atom()]), tuple([float(), atom([:ok])]))) == + union(number(), atom()) + + assert tuple_values(dynamic(tuple())) == dynamic() + assert tuple_values(dynamic(tuple([integer()]))) == dynamic(integer()) + + assert tuple_values(union(dynamic(tuple([integer()])), tuple([atom()]))) == + union(dynamic(integer()), atom()) + + assert tuple_values(union(dynamic(tuple()), integer())) == :badtuple + assert tuple_values(dynamic(union(integer(), tuple([atom()])))) == dynamic(atom()) + + assert tuple_values(union(dynamic(tuple([integer()])), tuple([integer()]))) + |> equal?(integer()) + end + + test "map_to_list" do + assert map_to_list(:term) == :badmap + assert map_to_list(integer()) == :badmap + assert map_to_list(union(open_map(), integer())) == :badmap + assert map_to_list(none()) == :badmap + assert map_to_list(dynamic()) == {:ok, dynamic(list(tuple([term(), term()])))} + + # A non existent map type is refused + assert open_map() + |> difference(open_map(a: if_set(term()), c: if_set(term()))) + |> map_to_list() == :badmap + + assert map_to_list(empty_map()) == {:ok, empty_list()} + assert map_to_list(open_map()) == {:ok, list(tuple([term(), term()]))} + + assert map_to_list(closed_map(a: integer())) == + {:ok, non_empty_list(tuple([atom([:a]), integer()]))} + + assert map_to_list(closed_map(a: integer(), b: atom())) == + {:ok, + non_empty_list( + tuple([atom([:a]), integer()]) + |> union(tuple([atom([:b]), atom()])) + )} + + assert map_to_list(union(closed_map(a: float()), closed_map(b: pid()))) == + {:ok, + non_empty_list( + tuple([atom([:a]), float()]) + |> union(tuple([atom([:b]), pid()])) + )} + + # Test with domain keys + assert map_to_list(closed_map([{domain_key(:integer), binary()}])) == + {:ok, list(tuple([integer(), binary()]))} + + assert map_to_list(closed_map([{domain_key(:tuple), binary()}])) == + {:ok, list(tuple([tuple(), binary()]))} + + # Test with both atom keys and domain keys + map_with_both = + closed_map([ + {:a, atom([:ok])}, + {:b, float()}, + {domain_key(:integer), binary()}, + {domain_key(:tuple), pid()} + ]) + + assert map_to_list(map_with_both) == + {:ok, + non_empty_list( + tuple([atom([:a]), atom([:ok])]) + |> union(tuple([atom([:b]), float()])) + |> union(tuple([integer(), binary()])) + |> union(tuple([tuple(), pid()])) + )} + + # Test open maps - should return list of key-value tuples + assert map_to_list(open_map()) == {:ok, list(tuple([term(), term()]))} + assert map_to_list(open_map(a: integer())) == {:ok, non_empty_list(tuple([term(), term()]))} + + {:ok, list} = map_to_list(open_map([{domain_key(:integer), binary()}])) + + assert list( + Enum.reduce( + [binary(), float(), pid(), port(), reference()] ++ + [fun(), atom(), tuple(), open_map(), list(term(), term())], + tuple([integer(), binary()]), + fn domain, acc -> union(acc, tuple([domain, term()])) end + ) + ) + |> equal?(list) + + # Test with multiple domain keys + multiple_domains = + closed_map([ + {domain_key(:integer), atom([:int])}, + {domain_key(:float), atom([:float])}, + {domain_key(:atom), binary()}, + {domain_key(:binary), integer()}, + {domain_key(:tuple), float()} + ]) + + assert map_to_list(multiple_domains) == + {:ok, + list( + tuple([integer(), atom([:int])]) + |> union(tuple([float(), atom([:float])])) + |> union(tuple([atom(), binary()])) + |> union(tuple([binary(), integer()])) + |> union(tuple([tuple(), float()])) + )} + + # Test dynamic maps + assert map_to_list(dynamic(open_map())) == + {:ok, dynamic(list(tuple([term(), term()])))} + + assert map_to_list(dynamic(closed_map(a: integer()))) == + {:ok, dynamic(non_empty_list(tuple([atom([:a]), integer()])))} + + assert map_to_list(union(dynamic(closed_map(a: integer())), closed_map(b: atom()))) == + {:ok, + union( + non_empty_list(tuple([atom([:b]), atom()])), + dynamic( + non_empty_list( + union( + tuple([atom([:a]), integer()]), + tuple([atom([:b]), atom()]) + ) + ) + ) + )} + + # A static integer is refused + assert map_to_list(union(dynamic(open_map()), integer())) == :badmap + + # Test with negations + assert map_to_list( + difference(closed_map(a: integer(), b: atom()), closed_map(a: integer())) + ) == + {:ok, + non_empty_list( + tuple([atom([:a]), integer()]) + |> union(tuple([atom([:b]), atom()])) + )} + + # If a key is removed entirely by a negation, it should not appear in the result + assert closed_map(a: if_set(integer()), b: atom()) + |> difference(closed_map(a: integer(), b: term())) + |> map_to_list() == + {:ok, non_empty_list(tuple([atom([:b]), atom()]))} + end + + test "domain_to_args" do + # take complex tuples, normalize them, and check if they are still equal + complex_tuples = [ + tuple([term(), atom(), number()]) + |> difference(tuple([atom(), atom(), float()])), + # overlapping union and difference producing multiple variants + difference( + tuple([union(atom(), pid()), union(integer(), float())]), + tuple([union(atom(), pid()), float()]) + ) + ] + + Enum.each(complex_tuples, fn domain -> + args = domain_to_args(domain) + + assert Enum.reduce(args, none(), &union(args_to_domain(&1), &2)) + |> equal?(domain) + end) + end + + test "map_fetch_key" do + assert map_fetch_key(term(), :a) == :badmap + assert map_fetch_key(union(open_map(), integer()), :a) == :badmap + assert map_fetch_key(difference(open_map(), open_map()), :a) == :badmap + + assert map_fetch_key(difference(closed_map(a: integer()), closed_map(a: term())), :a) == + :badmap + + assert map_fetch_key(open_map(), :a) == :badkey + assert map_fetch_key(open_map(a: not_set()), :a) == :badkey + assert map_fetch_key(union(closed_map(a: integer()), closed_map(b: atom())), :a) == :badkey + + assert map_fetch_key(closed_map(a: integer()), :a) == {false, integer()} + + assert map_fetch_key(union(closed_map(a: integer()), closed_map(a: atom())), :a) == + {false, union(integer(), atom())} + + {false, value_type} = + open_map(my_map: open_map(foo: integer())) + |> intersection(open_map(my_map: open_map(bar: boolean()))) + |> map_fetch_key(:my_map) + + assert equal?(value_type, open_map(foo: integer(), bar: boolean())) + + {false, value_type} = + closed_map(a: union(integer(), atom())) + |> difference(open_map(a: integer())) + |> map_fetch_key(:a) + + assert equal?(value_type, atom()) + + {false, value_type} = + closed_map(a: integer(), b: atom()) + |> difference(closed_map(a: integer(), b: atom([:foo]))) + |> map_fetch_key(:a) + + assert equal?(value_type, integer()) + + {false, value_type} = + closed_map(a: integer()) + |> difference(closed_map(a: atom())) + |> map_fetch_key(:a) + + assert equal?(value_type, integer()) + + {false, value_type} = + open_map(a: integer(), b: atom()) + |> union(closed_map(a: tuple())) + |> map_fetch_key(:a) + + assert equal?(value_type, union(integer(), tuple())) + + {false, value_type} = + closed_map(a: atom()) + |> difference(closed_map(a: atom([:foo, :bar]))) + |> difference(closed_map(a: atom([:bar]))) + |> map_fetch_key(:a) + + assert equal?(value_type, intersection(atom(), negation(atom([:foo, :bar])))) + + assert closed_map(a: union(atom([:ok]), pid()), b: integer(), c: tuple()) + |> difference(open_map(a: atom([:ok]), b: integer())) + |> difference(open_map(a: atom(), c: tuple())) + |> map_fetch_key(:a) == {false, pid()} + + assert closed_map(a: union(atom([:foo]), pid()), b: integer(), c: tuple()) + |> difference(open_map(a: atom([:foo]), b: integer())) + |> difference(open_map(a: atom(), c: tuple())) + |> map_fetch_key(:a) == {false, pid()} + + assert closed_map(a: union(atom([:foo, :bar, :baz]), integer())) + |> difference(open_map(a: atom([:foo, :bar]))) + |> difference(open_map(a: atom([:foo, :baz]))) + |> map_fetch_key(:a) == {false, integer()} + end + + test "map_fetch_key with dynamic" do + assert map_fetch_key(dynamic(), :a) == {true, dynamic()} + assert map_fetch_key(union(dynamic(), integer()), :a) == :badmap + assert map_fetch_key(union(dynamic(open_map(a: integer())), integer()), :a) == :badmap + assert map_fetch_key(union(dynamic(integer()), integer()), :a) == :badmap + + assert intersection(dynamic(), open_map(a: integer())) + |> map_fetch_key(:a) == {false, intersection(integer(), dynamic())} + + {false, type} = union(dynamic(integer()), open_map(a: integer())) |> map_fetch_key(:a) + assert equal?(type, integer()) + + assert union(dynamic(integer()), open_map(a: if_set(integer()))) |> map_fetch_key(:a) == + :badkey + + assert union(dynamic(open_map(a: atom())), open_map(a: integer())) + |> map_fetch_key(:a) == {false, union(dynamic(atom()), integer())} + end + + test "map_fetch_key with domain keys" do + integer_to_atom = open_map([{domain_key(:integer), atom()}]) + assert map_fetch_key(integer_to_atom, :foo) == :badkey + + # the key :a is for sure of type pid and exists in type + # %{atom() => pid()} and not %{:a => not_set()} + t1 = closed_map([{domain_key(:atom), pid()}]) + t2 = closed_map(a: not_set()) + t3 = open_map(a: not_set()) + + # Indeed, t2 is equivalent to the empty map + assert map_fetch_key(difference(t1, t2), :a) == :badkey + assert map_fetch_key(difference(t1, t3), :a) == {false, pid()} + + t4 = closed_map([{domain_key(:pid), atom()}]) + assert map_fetch_key(difference(t1, t4) |> difference(t3), :a) == {false, pid()} + + assert map_fetch_key(closed_map([{domain_key(:atom), pid()}]), :a) == :badkey + + assert map_fetch_key(dynamic(closed_map([{domain_key(:atom), pid()}])), :a) == + {true, dynamic(pid())} + + assert closed_map([{domain_key(:atom), number()}]) + |> difference(open_map(a: if_set(integer()))) + |> map_fetch_key(:a) == {false, float()} + + assert closed_map([{domain_key(:atom), number()}]) + |> difference(closed_map(b: if_set(integer()))) + |> map_fetch_key(:a) == :badkey + end + end + + describe "map_get" do + test "with domain keys" do + assert map_get(term(), term()) == :badmap + + map_type = closed_map([{domain_key(:tuple), binary()}]) + assert map_get(map_type, tuple()) == {true, binary()} + + # Type with all domain types + # %{:bar => :ok, integer() => :int, float() => :float, atom() => binary(), binary() => integer(), tuple() => float(), map() => pid(), reference() => port(), pid() => boolean()} + all_domains = + closed_map([ + {:bar, atom([:ok])}, + {domain_key(:integer), atom([:int])}, + {domain_key(:float), atom([:float])}, + {domain_key(:atom), binary()}, + {domain_key(:binary), integer()}, + {domain_key(:tuple), float()}, + {domain_key(:map), pid()}, + {domain_key(:reference), port()}, + {domain_key(:pid), reference()}, + {domain_key(:port), boolean()} + ]) + + assert map_get(all_domains, atom([:bar])) == {false, atom([:ok])} + + assert map_get(all_domains, integer()) == {true, atom([:int])} + assert map_get(all_domains, number()) == {true, atom([:int, :float])} + + assert map_get(all_domains, empty_list()) == :error + assert map_get(all_domains, atom([:foo])) == {true, binary()} + assert map_get(all_domains, binary()) == {true, integer()} + assert map_get(all_domains, tuple([integer(), atom()])) == {true, float()} + assert map_get(all_domains, empty_map()) == {true, pid()} + + # Union + assert map_get(all_domains, union(tuple(), empty_map())) == + {true, union(float(), pid())} + + # Removing all maps with tuple keys + t_no_tuple = difference(all_domains, closed_map([{domain_key(:tuple), float()}])) + t_really_no_tuple = difference(all_domains, open_map([{domain_key(:tuple), float()}])) + assert subtype?(all_domains, open_map()) + # It's only closed maps, so it should not change + assert map_get(t_no_tuple, tuple()) == {true, float()} + # This time we actually removed all tuple to float keys + assert map_get(t_really_no_tuple, tuple()) == :error + + t1 = closed_map([{domain_key(:tuple), integer()}]) + t2 = closed_map([{domain_key(:tuple), float()}]) + t3 = union(t1, t2) + assert map_get(t3, tuple()) == {true, number()} + end + + test "with dynamic" do + {_answer, type_selected} = map_get(dynamic(), term()) + assert equal?(type_selected, dynamic()) + end + + test "with atom fall back" do + map = closed_map([{:a, atom([:a])}, {:b, atom([:b])}, {domain_key(:atom), pid()}]) + + assert map_get(map, atom([:a, :b])) == + {false, atom([:a, :b])} + + assert map_get(map, atom([:a, :c])) == + {true, union(atom([:a]), pid())} + + assert map_get(map, atom() |> difference(atom([:a, :b]))) == + {true, pid()} + + assert map_get(map, atom() |> difference(atom([:a]))) == + {true, union(atom([:b]), pid())} + + assert map_get(closed_map(a: atom([:a]), b: atom([:b])), atom()) == + {true, atom([:a, :b])} + + assert map_get(closed_map([{domain_key(:atom), integer()}]), atom([:a, :b])) == + {true, integer()} + end + + test "with lists" do + # Verify that empty_list() bitmap type maps to :list domain (not :empty_list domain) + map_with_list_domain = closed_map([{domain_key(:list), atom([:empty])}]) + + # empty_list() should access the :list domain + assert map_get(map_with_list_domain, empty_list()) == {true, atom([:empty])} + + # non_empty_list() should also access the :list domain + assert map_get(map_with_list_domain, non_empty_list(integer())) == + {true, atom([:empty])} + + # list() should also access the :list domain + assert map_get(map_with_list_domain, list(integer())) == + {true, atom([:empty])} + + # If I create a map and instantiate both empty_list() and non_empty_list(integer()), it should return the union of the two types + map = + closed_map([{domain_key(:list), atom([:empty])}, {domain_key(:list), atom([:non_empty])}]) + + assert map_get(map, empty_list()) == {true, atom([:empty, :non_empty])} + + assert map_get(map, non_empty_list(integer())) == + {true, atom([:empty, :non_empty])} + + assert map_get(map, list(integer())) == {true, atom([:empty, :non_empty])} + end + end + + describe "map_update" do + test "with static atom keys" do + assert map_update(open_map(key: binary()), atom([:key]), integer()) == + {binary(), open_map(key: integer()), []} + + assert map_update(dynamic(open_map(key: binary())), atom([:key]), integer()) == + {dynamic(binary()), dynamic(open_map(key: integer())), []} + + # Optional fail for static maps + assert map_update(open_map(key: if_set(atom([:value]))), atom([:key]), integer()) == + {:error, [badkey: :key]} + + # But optional does not fail for dynamic ones + assert map_update(dynamic(open_map(key: if_set(atom([:value])))), atom([:key]), integer()) == + {dynamic(atom([:value])), dynamic(open_map(key: integer())), []} + + {type, descr, []} = map_update(dynamic(), atom([:key]), integer()) + assert equal?(type, dynamic()) + assert descr == dynamic(open_map(key: integer())) + + # Empty value fails for static maps + assert map_update(closed_map(key: not_set()), atom([:key]), integer()) == + {:error, [badkey: :key]} + + # When putting multiple keys, we don't know which one will be set + assert map_update(open_map(key1: atom(), key2: binary()), atom([:key1, :key2]), integer()) == + {union(atom(), binary()), + union( + open_map(key1: atom(), key2: integer()), + open_map(key1: integer(), key2: binary()) + ), []} + + # When putting multiple keys, all have to be set + assert map_update(open_map(key1: atom(), key2: binary()), atom([:key1, :key3]), integer()) == + {atom(), open_map(key1: integer(), key2: binary()), [badkey: :key3]} + + {type, descr, []} = map_update(dynamic(open_map()), atom([:key1, :key2]), integer()) + assert equal?(type, dynamic()) + assert descr == dynamic(union(open_map(key1: integer()), open_map(key2: integer()))) + + # A non-existing map + assert open_map() + |> difference(open_map(a: if_set(term()), c: if_set(term()))) + |> map_update(atom([:b]), integer()) == {:error, [badkey: :b]} + end + + test "with dynamic atom keys" do + assert map_update(closed_map(key: atom([:value])), dynamic(), atom([:new_value])) == + {atom([:value]), closed_map(key: atom([:value, :new_value])), []} + + assert map_update(dynamic(closed_map(key: atom([:value]))), dynamic(), atom([:new_value])) == + {dynamic(atom([:value])), dynamic(closed_map(key: atom([:value, :new_value]))), []} + + # When precise dynamic keys are given, at least one must succeed + assert map_update( + open_map(key1: atom(), key2: binary()), + dynamic(atom([:key1, :key3])), + integer() + ) == {atom(), open_map(key1: integer(), key2: binary()), []} + + assert map_update( + open_map(key1: atom(), key2: binary()), + dynamic(atom([:key3, :key4])), + integer() + ) == {:error, []} + end + + test "with domain keys" do + map = + closed_map([ + {domain_key(:integer), binary()}, + {domain_key(:pid), binary()}, + {domain_key(:port), binary()} + ]) + + assert map_update(map, none(), integer()) == + {:error, []} + + assert map_update(map, integer(), integer()) == + {binary(), + closed_map([ + {domain_key(:integer), union(integer(), binary())}, + {domain_key(:pid), binary()}, + {domain_key(:port), binary()} + ]), []} + + assert map_update(map, union(pid(), integer()), integer()) == + {binary(), + closed_map([ + {domain_key(:integer), union(integer(), binary())}, + {domain_key(:pid), union(integer(), binary())}, + {domain_key(:port), binary()} + ]), []} + + assert map_update(map, union(pid(), reference()), integer()) == + {binary(), + closed_map([ + {domain_key(:integer), binary()}, + {domain_key(:pid), union(integer(), binary())}, + {domain_key(:port), binary()} + ]), [baddomain: reference()]} + + assert map_update(map, union(pid(), dynamic(union(reference(), integer()))), integer()) == + {binary(), + closed_map([ + {domain_key(:integer), union(integer(), binary())}, + {domain_key(:pid), union(integer(), binary())}, + {domain_key(:port), binary()} + ]), []} + + assert map_update(map, union(pid(), dynamic(union(reference(), binary()))), integer()) == + {binary(), + closed_map([ + {domain_key(:integer), binary()}, + {domain_key(:pid), union(integer(), binary())}, + {domain_key(:port), binary()} + ]), []} + + assert map_update(map, dynamic(union(reference(), binary())), integer()) == + {:error, []} + + # Putting dynamic atom over record keys + assert map_update(closed_map(key1: binary(), key2: pid()), atom(), integer()) == + {union(binary(), pid()), + closed_map(key1: union(integer(), binary()), key2: union(integer(), pid())), + [baddomain: atom()]} + + assert map_update(closed_map(key1: binary(), key2: pid()), dynamic(atom()), integer()) == + {union(binary(), pid()), + closed_map(key1: union(integer(), binary()), key2: union(integer(), pid())), []} + + # A non-existing map + assert open_map() + |> difference(open_map(a: if_set(term()), c: if_set(term()))) + |> map_update(binary(), integer()) == {:error, [baddomain: binary()]} + end + + test "with mixed keys" do + assert map_update(dynamic(), union(atom([:key]), binary()), integer()) == + {dynamic(), dynamic(open_map()), []} + + # When precise dynamic keys are given, at least one must succeed + assert map_update( + closed_map([{:key, atom()}, {domain_key(:integer), binary()}]), + dynamic(union(atom([:key]), integer())), + integer() + ) == + {union(atom(), binary()), + union( + closed_map([{:key, integer()}, {domain_key(:integer), binary()}]), + closed_map([{:key, atom()}, {domain_key(:integer), union(binary(), integer())}]) + ), []} + + assert map_update( + closed_map([{:key, atom()}, {domain_key(:integer), binary()}]), + dynamic(union(atom([:other_key]), pid())), + integer() + ) == {:error, []} + + # Negated keys + assert map_update( + closed_map(key1: binary(), key2: binary()), + difference(atom(), atom([:key1])), + integer() + ) == + {binary(), closed_map(key1: binary(), key2: union(integer(), binary())), + [baddomain: atom()]} + + assert map_update( + closed_map([key1: binary(), key2: binary()] ++ [{domain_key(:atom), pid()}]), + difference(atom(), atom([:key1])), + integer() + ) == + {union(binary(), pid()), + closed_map( + [key1: binary(), key2: union(integer(), binary())] ++ + [{domain_key(:atom), union(integer(), pid())}] + ), []} + end + end + + describe "map_put" do + test "with static atom keys" do + assert map_put(open_map(key: binary()), atom([:key]), integer()) == + {:ok, open_map(key: integer())} + + assert map_put(dynamic(open_map(key: binary())), atom([:key]), integer()) == + {:ok, dynamic(open_map(key: integer()))} + + # Optional does not fail on put keys + assert map_put(open_map(key: if_set(atom([:value]))), atom([:key]), integer()) == + {:ok, open_map(key: integer())} + + # But optional does not fail for dynamic ones + assert map_put(dynamic(open_map(key: if_set(atom([:value])))), atom([:key]), integer()) == + {:ok, dynamic(open_map(key: integer()))} + + assert map_put(dynamic(), atom([:key]), integer()) == + {:ok, dynamic(open_map(key: integer()))} + + # Empty value does not fail for put + assert map_put(closed_map(key: not_set()), atom([:key]), integer()) == + {:ok, closed_map(key: integer())} + + # When putting multiple keys, we don't know which one will be set + assert map_put(open_map(key1: atom(), key2: binary()), atom([:key1, :key2]), integer()) == + {:ok, + union( + open_map(key1: atom(), key2: integer()), + open_map(key1: integer(), key2: binary()) + )} + + # When putting multiple keys, set even missing keys + assert map_put(open_map(key1: atom(), key2: binary()), atom([:key1, :key3]), integer()) == + {:ok, + union( + open_map(key1: atom(), key2: binary(), key3: integer()), + open_map(key1: integer(), key2: binary()) + )} + + assert map_put(dynamic(open_map()), atom([:key1, :key2]), integer()) == + {:ok, dynamic(union(open_map(key1: integer()), open_map(key2: integer())))} + end + + test "with dynamic/term as key-value" do + assert map_put(closed_map(key: atom([:value])), dynamic(), dynamic()) == + {:ok, dynamic(open_map())} + + assert map_put(closed_map(key: atom([:value])), dynamic(), term()) == + {:ok, open_map()} + + assert map_put(closed_map(key: atom([:value])), term(), dynamic()) == + {:ok, dynamic(open_map())} + + assert map_put(closed_map(key: atom([:value])), term(), term()) == + {:ok, open_map()} + + assert map_put(dynamic(closed_map(key: atom([:value]))), term(), term()) == + {:ok, dynamic(open_map())} + end + + test "with dynamic atom keys" do + assert map_put( + open_map(key1: atom(), key2: binary()), + dynamic(atom([:key1, :key3])), + integer() + ) == + {:ok, + union( + open_map(key1: atom(), key2: binary(), key3: integer()), + open_map(key1: integer(), key2: binary()) + )} + + assert map_put( + open_map(key1: atom(), key2: binary()), + dynamic(atom([:key3, :key4])), + integer() + ) == + {:ok, + union( + open_map(key1: atom(), key2: binary(), key3: integer()), + open_map(key1: atom(), key2: binary(), key4: integer()) + )} + end + + test "with domain keys" do + map = + closed_map([ + {domain_key(:integer), binary()}, + {domain_key(:pid), binary()}, + {domain_key(:port), binary()} + ]) + + assert map_put(map, integer(), integer()) == + {:ok, + closed_map([ + {domain_key(:integer), union(integer(), binary())}, + {domain_key(:pid), binary()}, + {domain_key(:port), binary()} + ])} + + assert map_put(map, union(pid(), integer()), integer()) == + {:ok, + closed_map([ + {domain_key(:integer), union(integer(), binary())}, + {domain_key(:pid), union(integer(), binary())}, + {domain_key(:port), binary()} + ])} + + assert map_put(map, union(pid(), reference()), integer()) == + {:ok, + closed_map([ + {domain_key(:integer), binary()}, + {domain_key(:pid), union(integer(), binary())}, + {domain_key(:port), binary()}, + {domain_key(:reference), integer()} + ])} + + assert map_put(map, union(pid(), dynamic(union(reference(), integer()))), integer()) == + {:ok, + closed_map([ + {domain_key(:integer), union(integer(), binary())}, + {domain_key(:pid), union(integer(), binary())}, + {domain_key(:port), binary()}, + {domain_key(:reference), integer()} + ])} + + assert map_put(map, dynamic(union(reference(), binary())), integer()) == + {:ok, + closed_map([ + {domain_key(:integer), binary()}, + {domain_key(:pid), binary()}, + {domain_key(:port), binary()}, + {domain_key(:reference), integer()}, + {domain_key(:binary), integer()} + ])} + + # Putting dynamic atom over record keys + assert map_put(closed_map(key1: binary(), key2: binary()), atom(), integer()) == + {:ok, + closed_map( + [key1: union(integer(), binary()), key2: union(integer(), binary())] ++ + [{domain_key(:atom), integer()}] + )} + end + + test "with mixed keys" do + assert map_put(dynamic(), union(atom([:key]), binary()), integer()) == + {:ok, dynamic(open_map())} + + # When precise dynamic keys are given, at least one must succeed + assert map_put( + closed_map([{:key, atom()}, {domain_key(:integer), binary()}]), + dynamic(union(atom([:key]), integer())), + integer() + ) == + {:ok, + union( + closed_map([{:key, integer()}, {domain_key(:integer), binary()}]), + closed_map([{:key, atom()}, {domain_key(:integer), union(binary(), integer())}]) + )} + + assert map_put( + closed_map([{:key, atom()}, {domain_key(:integer), binary()}]), + dynamic(union(atom([:other_key]), pid())), + integer() + ) == + {:ok, + union( + closed_map([ + {:key, atom()}, + {:other_key, integer()}, + {domain_key(:integer), binary()} + ]), + closed_map([ + {:key, atom()}, + {domain_key(:integer), binary()}, + {domain_key(:pid), integer()} + ]) + )} + + # Negated keys + assert map_put( + closed_map(key1: binary(), key2: binary()), + difference(atom(), atom([:key1])), + integer() + ) == + {:ok, + closed_map( + [key1: binary(), key2: union(binary(), integer())] ++ + [{domain_key(:atom), integer()}] + )} + + assert map_put( + closed_map([key1: binary(), key2: binary()] ++ [{domain_key(:atom), pid()}]), + difference(atom(), atom([:key1])), + integer() + ) == + {:ok, + closed_map( + [key1: binary(), key2: union(integer(), binary())] ++ + [{domain_key(:atom), union(integer(), pid())}] + )} + end + end + + describe "disjoint" do + test "optional" do + assert disjoint?(term(), if_set(none())) + assert disjoint?(term(), if_set(none()) |> union(non_empty_list(none()))) + end + + test "map" do + refute disjoint?(open_map(), open_map(a: integer())) + end + end + + describe "to_quoted" do + test "bitmap" do + assert union(integer(), union(float(), binary())) |> to_quoted_string() == + "binary() or float() or integer()" + end + + test "none" do + assert none() |> to_quoted_string() == "none()" + assert dynamic(none()) |> to_quoted_string() == "none()" + end + + test "negation" do + assert negation(negation(integer())) |> to_quoted_string() == "integer()" + assert negation(negation(atom([:foo, :bar]))) |> to_quoted_string() == ":bar or :foo" + assert negation(negation(list(term()))) |> to_quoted_string() == "list(term())" + end + + test "atom" do + assert atom() |> to_quoted_string() == "atom()" + assert atom([:a]) |> to_quoted_string() == ":a" + assert atom([:a, :b]) |> to_quoted_string() == ":a or :b" + assert difference(atom(), atom([:a])) |> to_quoted_string() == "atom() and not :a" + + assert atom([Elixir]) |> to_quoted_string() == "Elixir" + assert atom([Foo.Bar]) |> to_quoted_string() == "Foo.Bar" + end + + test "boolean" do + assert boolean() |> to_quoted_string() == "boolean()" + assert atom([true, false, :a]) |> to_quoted_string() == ":a or boolean()" + assert atom([true, :a]) |> to_quoted_string() == ":a or true" + assert difference(atom(), boolean()) |> to_quoted_string() == "atom() and not boolean()" + end + + test "dynamic" do + assert dynamic() |> to_quoted_string() == "dynamic()" + + assert dynamic(union(atom(), integer())) |> union(integer()) |> to_quoted_string() == + "dynamic(atom()) or integer()" + + assert intersection(binary(), dynamic()) |> to_quoted_string() == "binary()" + + assert intersection(union(binary(), pid()), dynamic()) |> to_quoted_string() == + "dynamic(binary() or pid())" + + assert intersection(atom(), dynamic()) |> to_quoted_string() == "dynamic(atom())" + + assert union(atom([:foo, :bar]), dynamic()) |> to_quoted_string() == + "dynamic() or :bar or :foo" + + assert intersection(dynamic(), closed_map(a: integer())) |> to_quoted_string() == + "dynamic(%{a: integer()})" + end + + test "lists" do + assert list(term()) |> to_quoted_string() == "list(term())" + assert list(integer()) |> to_quoted_string() == "list(integer())" + + assert list(term()) |> difference(empty_list()) |> to_quoted_string() == + "non_empty_list(term())" + + assert list(term()) |> difference(list(integer())) |> to_quoted_string() == + "non_empty_list(term()) and not non_empty_list(integer())" + + assert list(term()) + |> difference(list(integer())) + |> difference(list(atom())) + |> to_quoted_string() == + "non_empty_list(term()) and not (non_empty_list(integer()) or non_empty_list(atom()))" + + assert list(term(), integer()) |> to_quoted_string() == + "empty_list() or non_empty_list(term(), integer())" + + assert difference(list(term(), atom()), list(term(), boolean())) |> to_quoted_string() == + "non_empty_list(term(), atom() and not boolean())" + + assert list(term(), term()) |> to_quoted_string() == + "empty_list() or non_empty_list(term(), term())" + + # Test normalization + + # Remove duplicates + assert union(list(integer()), list(integer())) |> to_quoted_string() == "list(integer())" + + # Merge subtypes + assert union(list(float(), pid()), list(number(), pid())) |> to_quoted_string() == + "empty_list() or non_empty_list(float() or integer(), pid())" + + # Merge last element types + assert union(list(atom([:ok]), integer()), list(atom([:ok]), float())) + |> to_quoted_string() == + "empty_list() or non_empty_list(:ok, float() or integer())" + + assert union(dynamic(list(integer(), float())), dynamic(list(integer(), pid()))) + |> to_quoted_string() == + "dynamic(empty_list() or non_empty_list(integer(), float() or pid()))" + end + + test "tuples" do + assert tuple([integer(), atom()]) |> to_quoted_string() == "{integer(), atom()}" + + assert tuple([integer(), dynamic(atom())]) |> to_quoted_string() == + "dynamic({integer(), atom()})" + + assert open_tuple([integer(), atom()]) |> to_quoted_string() == "{integer(), atom(), ...}" + + assert union(tuple([integer(), atom()]), open_tuple([atom()])) |> to_quoted_string() == + "{atom(), ...} or {integer(), atom()}" + + assert difference(tuple([integer(), atom()]), open_tuple([atom()])) |> to_quoted_string() == + "{integer(), atom()}" + + assert tuple([closed_map(a: integer()), open_map()]) |> to_quoted_string() == + "{%{a: integer()}, map()}" + + assert union(tuple([integer(), atom()]), tuple([integer(), atom()])) |> to_quoted_string() == + "{integer(), atom()}" + + assert union(tuple([integer(), atom()]), tuple([float(), atom()])) |> to_quoted_string() == + "{float() or integer(), atom()}" + + assert union(tuple([integer(), atom()]), tuple([float(), atom()])) + |> union(tuple([pid(), pid(), port()])) + |> union(tuple([pid(), pid(), atom()])) + |> to_quoted_string() == + "{float() or integer(), atom()} or {pid(), pid(), atom() or port()}" + + assert union(open_tuple([integer()]), open_tuple([float()])) |> to_quoted_string() == + "{float() or integer(), ...}" + + # {:ok, {term(), integer()}} or {:ok, {term(), float()}} or {:exit, :kill} or {:exit, :timeout} + assert tuple([atom([:ok]), tuple([term(), empty_list()])]) + |> union(tuple([atom([:ok]), tuple([term(), open_map()])])) + |> union(tuple([atom([:exit]), atom([:kill])])) + |> union(tuple([atom([:exit]), atom([:timeout])])) + |> to_quoted_string() == + "{:exit, :kill or :timeout} or {:ok, {term(), empty_list() or map()}}" + + # Detection of duplicates + assert tuple([atom([:ok]), term()]) + |> union(tuple([atom([:ok]), term()])) + |> to_quoted_string() == "{:ok, term()}" + + assert tuple([closed_map(a: integer(), b: atom()), open_map()]) + |> union(tuple([closed_map(a: integer(), b: atom()), open_map()])) + |> to_quoted_string() == + "{%{a: integer(), b: atom()}, map()}" + + # Nested fusion + assert tuple([closed_map(a: integer(), b: atom()), open_map()]) + |> union(tuple([closed_map(a: float(), b: atom()), open_map()])) + |> to_quoted_string() == + "{%{a: float() or integer(), b: atom()}, map()}" + + # Complex simplification of map/tuple combinations. Initial type is: + # ``` + # dynamic( + # :error or + # ({%Decimal{coef: :inf, exp: integer(), sign: integer()}, binary()} or + # {%Decimal{coef: :NaN, exp: integer(), sign: integer()}, binary()} or + # {%Decimal{coef: integer(), exp: integer(), sign: integer()}, term()} or + # {%Decimal{coef: :inf, exp: integer(), sign: integer()} or + # %Decimal{coef: :NaN, exp: integer(), sign: integer()} or + # %Decimal{coef: integer(), exp: integer(), sign: integer()}, term()}) + # ) + # ``` + decimal_inf = + closed_map( + __struct__: atom([Decimal]), + coef: atom([:inf]), + exp: integer(), + sign: integer() + ) + + decimal_nan = + closed_map( + __struct__: atom([Decimal]), + coef: atom([:NaN]), + exp: integer(), + sign: integer() + ) + + decimal_int = + closed_map( + __struct__: atom([Decimal]), + coef: integer(), + exp: integer(), + sign: integer() + ) + + assert atom([:error]) + |> union( + tuple([decimal_inf, binary()]) + |> union( + tuple([decimal_nan, binary()]) + |> union( + tuple([decimal_int, term()]) + |> union(tuple([union(decimal_inf, union(decimal_nan, decimal_int)), term()])) + ) + ) + ) + |> dynamic() + |> to_quoted_string() == + """ + dynamic( + :error or {%Decimal{sign: integer(), coef: :NaN or :inf or integer(), exp: integer()}, term()} + )\ + """ + end + + test "function" do + assert fun() |> to_quoted_string() == "fun()" + assert none_fun(1) |> to_quoted_string() == "(none() -> term())" + + assert none_fun(1) + |> intersection(none_fun(2)) + |> to_quoted_string() == "none()" + + assert fun([integer(), float()], boolean()) |> to_quoted_string() == + "(integer(), float() -> boolean())" + + assert fun([integer()], boolean()) + |> union(fun([float()], boolean())) + |> to_quoted_string() == + "(integer() -> boolean()) or (float() -> boolean())" + + assert fun([integer()], boolean()) + |> intersection(fun([float()], boolean())) + |> to_quoted_string() == + "(integer() -> boolean()) and (float() -> boolean())" + + # Thanks to lazy BDDs, consecutive union of functions come out as the original union + assert fun([integer()], integer()) + |> union(fun([float()], float())) + |> union(fun([pid()], pid())) + |> to_quoted_string() == + "(integer() -> integer()) or (float() -> float()) or (pid() -> pid())" + end + + test "function with optimized intersections" do + assert fun([integer()], atom()) |> intersection(none_fun(1)) |> to_quoted_string() == + "(integer() -> atom())" + + assert fun([integer()], atom()) + |> difference(none_fun(2)) + |> intersection(none_fun(1)) + |> to_quoted_string() == + "(integer() -> atom())" + end + + test "function with dynamic signatures" do + assert fun([dynamic(integer())], float()) |> to_quoted_string() == + "(dynamic(integer()) -> float())" + + assert fun([dynamic(atom())], float()) |> to_quoted_string() == + "(dynamic(atom()) -> float())" + + assert fun([integer(), float()], dynamic(atom())) |> to_quoted_string() == + "(integer(), float() -> dynamic(atom()))" + + domain_part = fun([dynamic(atom()) |> union(integer()), binary()], float()) + + assert domain_part |> to_quoted_string() == + "(dynamic(atom()) or integer(), binary() -> float())" + + codomain_part = fun([pid(), float()], dynamic(atom()) |> union(integer())) + + assert codomain_part |> to_quoted_string() == + "(pid(), float() -> dynamic(atom()) or integer())" + + assert union(domain_part, codomain_part) |> to_quoted_string() == + """ + (dynamic(atom()) or integer(), binary() -> float()) or + (pid(), float() -> dynamic(atom()) or integer())\ + """ + + assert intersection(domain_part, codomain_part) |> to_quoted_string() == + """ + (dynamic(atom()) or integer(), binary() -> float()) and + (pid(), float() -> dynamic(atom()) or integer())\ + """ + end + + test "map as records" do + assert empty_map() |> to_quoted_string() == "empty_map()" + assert open_map() |> to_quoted_string() == "map()" + + assert closed_map(a: integer()) |> to_quoted_string() == "%{a: integer()}" + assert open_map(a: float()) |> to_quoted_string() == "%{..., a: float()}" + + assert closed_map("Elixir.Foo.Bar": integer()) |> to_quoted_string() == + "%{Foo.Bar => integer()}" + + assert open_map("Elixir.Foo.Bar": float()) |> to_quoted_string() == + "%{..., Foo.Bar => float()}" + + assert difference(open_map(), open_map(a: term())) |> to_quoted_string() == + "%{..., a: not_set()}" + + assert closed_map(a: integer(), b: atom()) |> to_quoted_string() == + "%{a: integer(), b: atom()}" + + assert open_map(a: float()) + |> difference(closed_map(a: float())) + |> to_quoted_string() == "%{..., a: float()} and not %{a: float()}" + + assert difference(open_map(), empty_map()) |> to_quoted_string() == + "map() and not empty_map()" + + assert closed_map(foo: union(integer(), not_set())) |> to_quoted_string() == + "%{foo: if_set(integer())}" + + # Test normalization + assert open_map(a: integer(), b: atom()) + |> difference(open_map(b: atom())) + |> union(open_map(a: integer())) + |> to_quoted_string() == "%{..., a: integer()}" + + assert union(open_map(a: integer()), open_map(a: integer())) |> to_quoted_string() == + "%{..., a: integer()}" + + assert difference(open_map(a: number(), b: atom()), open_map(a: integer())) + |> to_quoted_string() == "%{..., a: float(), b: atom()}" + + # Basic map fusion + assert union(closed_map(a: integer()), closed_map(a: integer())) |> to_quoted_string() == + "%{a: integer()}" + + assert union(closed_map(a: integer()), closed_map(a: float())) |> to_quoted_string() == + "%{a: float() or integer()}" + + # Nested fusion + assert union(closed_map(a: integer(), b: atom()), closed_map(a: float(), b: atom())) + |> union(closed_map(x: pid(), y: pid(), z: port())) + |> union(closed_map(x: pid(), y: pid(), z: atom())) + |> to_quoted_string() == + "%{a: float() or integer(), b: atom()} or %{x: pid(), y: pid(), z: atom() or port()}" + + # Open map fusion + assert union(open_map(a: integer()), open_map(a: float())) |> to_quoted_string() == + "%{..., a: float() or integer()}" + + # Fusing complex nested maps with unions + assert closed_map( + status: atom([:ok]), + data: closed_map(value: term(), count: empty_list()) + ) + |> union( + closed_map( + status: atom([:ok]), + data: closed_map(value: term(), count: open_map()) + ) + ) + |> union(closed_map(status: atom([:error]), reason: atom([:timeout]))) + |> union(closed_map(status: atom([:error]), reason: atom([:crash]))) + |> to_quoted_string() == + "%{data: %{count: empty_list() or map(), value: term()}, status: :ok} or\n %{reason: :crash or :timeout, status: :error}" + + # Difference and union tests + assert closed_map(status: atom([:ok]), value: term()) + |> difference(closed_map(status: atom([:ok]), value: float())) + |> union( + closed_map(status: atom([:ok]), value: term()) + |> difference(closed_map(status: atom([:ok]), value: integer())) + ) + |> to_quoted_string() == + "%{status: :ok, value: term()}" + + # Nested map fusion + assert closed_map(data: closed_map(x: integer(), y: atom()), meta: open_map()) + |> union(closed_map(data: closed_map(x: float(), y: atom()), meta: open_map())) + |> to_quoted_string() == + "%{data: %{x: float() or integer(), y: atom()}, meta: map()}" + + # Test complex combinations + assert intersection( + open_map(a: number(), b: atom()), + open_map(a: integer(), c: boolean()) + ) + |> union(difference(open_map(x: atom()), open_map(x: boolean()))) + |> to_quoted_string() == + "%{..., a: integer(), b: atom(), c: boolean()} or %{..., x: atom() and not boolean()}" + + assert closed_map(a: number(), b: atom(), c: pid()) + |> difference(closed_map(a: integer(), b: atom(), c: pid())) + |> to_quoted_string() == "%{a: float(), b: atom(), c: pid()}" + + # No simplification compared to above, as it is an open map + assert open_map(a: number(), b: atom()) + |> difference(closed_map(a: integer(), b: atom())) + |> to_quoted_string() == + "%{..., a: float() or integer(), b: atom()} and not %{a: integer(), b: atom()}" + + # Remark: this simplification is order dependent. Having the first difference + # after the second gives a different result. + assert open_map(a: number(), b: atom(), c: union(pid(), port())) + |> difference(open_map(a: integer(), b: atom(), c: union(pid(), port()))) + |> difference(open_map(a: float(), b: atom(), c: pid())) + |> to_quoted_string() == "%{..., a: float(), b: atom(), c: port()}" + + assert open_map(a: number(), b: atom(), c: union(pid(), port())) + |> difference(open_map(a: float(), b: atom(), c: pid())) + |> difference(open_map(a: integer(), b: atom(), c: union(pid(), port()))) + |> to_quoted_string() == "%{..., a: float(), b: atom(), c: port()}" + end + + test "maps as dictionaries" do + assert closed_map([{domain_key(:integer), integer()}]) + |> to_quoted_string() == "%{integer() => integer()}" + + assert closed_map([{domain_key(:integer), not_set()}, {:float, float()}]) + |> to_quoted_string() == "%{integer() => not_set(), float: float()}" + end + + test "structs" do + assert open_map(__struct__: atom([URI])) |> to_quoted_string() == + "%{..., __struct__: URI}" + + assert closed_map(__struct__: atom([URI])) |> to_quoted_string() == + "%{__struct__: URI}" + + assert closed_map(__struct__: atom([NoFieldsStruct])) |> to_quoted_string() == + "%NoFieldsStruct{}" + + assert closed_map(__struct__: atom([URI, Another])) |> to_quoted_string() == + "%{__struct__: Another or URI}" + + assert closed_map(__struct__: atom([Decimal]), coef: term(), exp: term(), sign: term()) + |> to_quoted_string(collapse_structs: false) == + "%Decimal{sign: term(), coef: term(), exp: term()}" + + assert closed_map(__struct__: atom([Decimal]), coef: term(), exp: term(), sign: term()) + |> to_quoted_string() == + "%Decimal{}" + + assert closed_map(__struct__: atom([Decimal]), coef: term(), exp: term(), sign: integer()) + |> to_quoted_string() == + "%Decimal{sign: integer()}" + + # Does not fuse structs + assert union(closed_map(__struct__: atom([Foo])), closed_map(__struct__: atom([Bar]))) + |> to_quoted_string() == + "%{__struct__: Bar} or %{__struct__: Foo}" + + # Properly format non_struct_map + assert open_map(__struct__: if_set(negation(atom()))) |> to_quoted_string() == + "non_struct_map()" + end + end + + describe "performance" do + test "tuple difference" do + # Large difference with no duplicates + descr1 = + union( + atom([:ignored, :reset]), + tuple([atom([:font_style]), atom([:italic])]) + ) + + descr2 = + union( + atom([:ignored, :reset]), + union( + tuple([atom([:font_style]), atom([:italic])]), + Enum.reduce( + for elem1 <- 1..5, elem2 <- 1..5 do + tuple([atom([:"f#{elem1}"]), atom([:"s#{elem2}"])]) + end, + &union/2 + ) + ) + ) + + assert subtype?(descr1, descr2) + refute subtype?(descr2, descr1) + end + + test "map difference" do + # Create a large map with various types + map1 = + open_map([ + {:id, integer()}, + {:name, binary()}, + {:age, union(integer(), atom())}, + {:email, binary()}, + {:active, boolean()}, + {:tags, list(atom())} + ]) + + # Create another large map with some differences and many more entries + map2 = + open_map( + [ + {:id, integer()}, + {:name, binary()}, + {:age, integer()}, + {:email, binary()}, + {:active, boolean()}, + {:tags, non_empty_list(atom())}, + {:meta, + open_map([ + {:created_at, binary()}, + {:updated_at, binary()}, + {:status, atom()} + ])}, + {:permissions, tuple([atom(), integer(), atom()])}, + {:profile, + open_map([ + {:bio, binary()}, + {:interests, non_empty_list(binary())}, + {:social_media, + open_map([ + {:twitter, binary()}, + {:instagram, binary()}, + {:linkedin, binary()} + ])} + ])}, + {:notifications, boolean()} + ] ++ + Enum.map(1..50, fn i -> + {:"field_#{i}", atom([:"value_#{i}"])} + end) + ) + + refute subtype?(map1, map2) + assert subtype?(map2, map1) + end + + test "map intersection and then difference" do + actual = open_map(__struct__: atom(), __exception__: atom([true])) + + expected = + for i <- 1..50 do + name = :"name_#{i}" + closed_map([__struct__: atom([name])] ++ [{name, binary()}]) + end + |> Enum.reduce(&union/2) + + common = intersection(actual, expected) + difference(actual, common) + end + + test "struct difference" do + entries = + [ + closed_map(__struct__: atom([MapSet]), map: term()), + closed_map(__struct__: atom([Jason.OrderedObject]), values: term()), + closed_map(__struct__: atom([GenEvent.Stream]), timeout: term(), manager: term()), + closed_map(__struct__: atom([HashDict]), size: term(), root: term()), + closed_map(__struct__: atom([HashSet]), size: term(), root: term()), + closed_map( + __struct__: atom([IO.Stream]), + raw: term(), + device: term(), + line_or_bytes: term() + ), + closed_map(__struct__: atom([Range]), first: term(), last: term(), step: term()), + closed_map( + __struct__: atom([Stream]), + enum: term(), + done: term(), + funs: term(), + accs: term() + ), + closed_map( + __struct__: atom([Req.Response.Async]), + pid: term(), + ref: term(), + stream_fun: term(), + cancel_fun: term() + ), + closed_map( + __struct__: atom([Postgrex.Stream]), + options: term(), + params: term(), + query: term(), + conn: term() + ), + closed_map( + __struct__: atom([DBConnection.PrepareStream]), + opts: term(), + params: term(), + query: term(), + conn: term() + ), + closed_map( + __struct__: atom([DBConnection.Stream]), + opts: term(), + params: term(), + query: term(), + conn: term() + ), + closed_map( + __struct__: atom([Ecto.Adapters.SQL.Stream]), + meta: term(), + opts: term(), + params: term(), + statement: term() + ), + closed_map( + __struct__: atom([Date.Range]), + first: term(), + last: term(), + step: term(), + first_in_iso_days: term(), + last_in_iso_days: term() + ), + closed_map( + __struct__: atom([File.Stream]), + node: term(), + raw: term(), + path: term(), + modes: term(), + line_or_bytes: term() + ), + closed_map( + __struct__: atom([Phoenix.LiveView.LiveStream]), + name: term(), + ref: term(), + inserts: term(), + deletes: term(), + reset?: term(), + dom_id: term(), + consumable?: term() + ) + ] + + range = + closed_map(__struct__: atom([Range]), first: integer(), last: integer(), step: integer()) + + assert subtype?(range, Enum.reduce(entries, &union/2)) + end + end +end diff --git a/lib/elixir/test/elixir/module/types/expr_test.exs b/lib/elixir/test/elixir/module/types/expr_test.exs new file mode 100644 index 00000000000..69fe585da73 --- /dev/null +++ b/lib/elixir/test/elixir/module/types/expr_test.exs @@ -0,0 +1,2211 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("type_helper.exs", __DIR__) + +defmodule Module.Types.ExprTest do + use ExUnit.Case, async: true + + import TypeHelper + import Module.Types.Descr + defmacro domain_key(arg) when is_atom(arg), do: [arg] + + defmacro generated(x) do + quote generated: true do + unquote(x).foo() + end + end + + test "literal" do + assert typecheck!(true) == atom([true]) + assert typecheck!(false) == atom([false]) + assert typecheck!(:foo) == atom([:foo]) + assert typecheck!(0) == integer() + assert typecheck!(0.0) == float() + assert typecheck!("foo") == binary() + assert typecheck!([]) == empty_list() + assert typecheck!(%{}) == closed_map([]) + end + + test "generated" do + assert typecheck!([x = 1], generated(x)) == dynamic() + end + + describe "lists" do + test "creating lists" do + assert typecheck!([1, 2]) == non_empty_list(integer()) + assert typecheck!([1, 2 | 3]) == non_empty_list(integer(), integer()) + assert typecheck!([1, 2 | [3, 4]]) == non_empty_list(integer()) + + assert typecheck!([:ok, 123]) == non_empty_list(union(atom([:ok]), integer())) + assert typecheck!([:ok | 123]) == non_empty_list(atom([:ok]), integer()) + assert typecheck!([x], [:ok, x]) == dynamic(non_empty_list(term())) + assert typecheck!([x], [:ok | x]) == dynamic(non_empty_list(term(), term())) + end + + test "inference" do + assert typecheck!( + [x, y, z], + ( + List.to_integer([x, y | z]) + {x, y, z} + ) + ) == dynamic(tuple([integer(), integer(), list(integer())])) + end + + test "hd" do + assert typecheck!([x = [123, :foo]], hd(x)) == dynamic(union(atom([:foo]), integer())) + assert typecheck!([x = [123 | :foo]], hd(x)) == dynamic(integer()) + + assert typeerror!(hd([])) |> strip_ansi() == + ~l""" + incompatible types given to Kernel.hd/1: + + hd([]) + + given types: + + empty_list() + + but expected one of: + + non_empty_list(term(), term()) + """ + + assert typeerror!(hd(123)) |> strip_ansi() == + ~l""" + incompatible types given to Kernel.hd/1: + + hd(123) + + given types: + + integer() + + but expected one of: + + non_empty_list(term(), term()) + """ + end + + test "tl" do + assert typecheck!([x = [123, :foo]], tl(x)) == dynamic(list(union(atom([:foo]), integer()))) + + assert typecheck!([x = [123 | :foo]], tl(x)) == + dynamic(union(atom([:foo]), non_empty_list(integer(), atom([:foo])))) + + assert typeerror!(tl([])) |> strip_ansi() == + ~l""" + incompatible types given to Kernel.tl/1: + + tl([]) + + given types: + + empty_list() + + but expected one of: + + non_empty_list(term(), term()) + """ + + assert typeerror!(tl(123)) |> strip_ansi() == + ~l""" + incompatible types given to Kernel.tl/1: + + tl(123) + + given types: + + integer() + + but expected one of: + + non_empty_list(term(), term()) + """ + end + end + + describe "funs" do + test "infers calls" do + assert typecheck!( + [x], + ( + x.(1, 2) + x + ) + ) == dynamic(fun(2)) + end + + test "infers functions" do + assert typecheck!(& &1) |> equal?(fun([term()], dynamic())) + + assert typecheck!(fn -> :ok end) |> equal?(fun([], dynamic(atom([:ok])))) + + assert typecheck!(fn + <<"ok">>, {} -> :ok + <<"error">>, {} -> :error + [_ | _], %{} -> :list + end) + |> equal?( + intersection( + fun( + [non_empty_list(term(), term()), open_map()], + dynamic(atom([:list])) + ), + fun( + [binary(), tuple([])], + dynamic(atom([:ok, :error])) + ) + ) + ) + end + + test "application" do + assert typecheck!( + [map], + (fn + %{a: a} = data -> %{data | b: a} + data -> data + end).(map) + ) == dynamic() + + assert typecheck!( + [], + [true, false] + |> Enum.random() + |> then(fn + true -> :ok + _ -> :error + end) + ) == dynamic(atom([:ok, :error])) + end + + test "bad function" do + assert typeerror!([%x{}, a1, a2], x.(a1, a2)) == ~l""" + expected a 2-arity function on call: + + x.(a1, a2) + + but got type: + + dynamic(atom()) + + where "x" was given the type: + + # type: dynamic(atom()) + # from: types_test.ex:LINE + %x{} + """ + end + + test "bad arity" do + assert typeerror!([a1, a2], (&String.to_integer/1).(a1, a2)) == ~l""" + expected a 2-arity function on call: + + (&String.to_integer/1).(a1, a2) + + but got function with arity 1: + + (binary() -> integer()) + """ + end + + test "bad argument" do + assert typeerror!([], (&String.to_integer/1).(:foo)) + |> strip_ansi() == ~l""" + incompatible types given on function application: + + (&String.to_integer/1).(:foo) + + given types: + + :foo + + but function has type: + + (binary() -> integer()) + """ + + assert typeerror!( + [x], + (if x do + &String.to_integer/1 + else + &List.to_integer/1 + end).(:foo) + ) + |> strip_ansi() == ~l""" + incompatible types given on function application: + + (if x do + &String.to_integer/1 + else + &List.to_integer/1 + end).(:foo) + + given types: + + :foo + + but function has type: + + (binary() -> integer()) or (non_empty_list(integer()) -> integer()) + + hint: the function has an empty domain and therefore cannot be applied to any argument. \ + This may happen when you have a union of functions, which means the only valid argument \ + to said function are types that satisfy all sides of the union (which may be none) + """ + end + + test "bad arguments from inferred type" do + assert typeerror!( + ( + fun = fn %{} -> :map end + fun.(:error) + ) + ) + |> strip_ansi() == """ + incompatible types given on function application: + + fun.(:error) + + given types: + + :error + + but function has type: + + (map() -> dynamic(:map)) + """ + end + + test "capture printing" do + assert typeerror!(123 = &{:ok, &1}) == """ + the following pattern will never match: + + 123 = &{:ok, &1} + + because the right-hand side has type: + + (term() -> dynamic({:ok, term()})) + """ + end + + test "works when there are multiple clauses with lists and maps" do + type = + typecheck!(fn + [:oban, :job, _event], _measure, _meta, _opts -> + :ok + + [:oban, :notifier, :switch], _measure, %{status: _status}, _opts -> + :ok + + [:oban, :peer, :election, :stop], _measure, _meta, _opts -> + :ok + + [:oban, :plugin, :exception], _measure, _meta, _opts -> + :ok + + [:oban, :plugin, :stop], _measure, _meta, _opts -> + :ok + + [:oban, :queue, :shutdown], _measure, %{orphaned: [_ | _]}, _opts -> + :ok + + [:oban, :stager, :switch], _measure, %{mode: _mode}, _opts -> + :ok + + _event, _measure, _meta, _opts -> + :ok + end) + + assert subtype?(type, fun([term(), term(), term(), term()], atom([:ok]))) + end + end + + describe "remotes" do + test "dynamic calls" do + assert typecheck!([%x{}], x.foo_bar()) == dynamic() + end + + test "infers atoms" do + assert typecheck!( + [x], + ( + x.foo_bar() + x + ) + ) == dynamic(atom()) + + assert typecheck!( + [x], + ( + x.foo_bar(123) + x + ) + ) == dynamic(atom()) + + assert typecheck!( + [x], + ( + &x.foo_bar/1 + x + ) + ) == dynamic(atom()) + end + + test "infers maps" do + assert typecheck!( + [x], + ( + :foo = x.foo_bar + 123 = x.baz_bat + x + ) + ) == dynamic(open_map(foo_bar: atom([:foo]), baz_bat: integer())) + end + + test "infers args" do + assert typecheck!( + [x, y], + ( + z = Integer.to_string(x + y) + {x, y, z} + ) + ) == dynamic(tuple([integer(), integer(), binary()])) + end + + test "undefined function warnings" do + assert typewarn!(URI.unknown("foo")) == + {dynamic(), "URI.unknown/1 is undefined or private"} + + assert typewarn!(if(:rand.uniform() > 0.5, do: URI.unknown("foo"))) == + {dynamic() |> union(atom([nil])), "URI.unknown/1 is undefined or private"} + + assert typewarn!(try(do: :ok, after: URI.unknown("foo"))) == + {atom([:ok]), "URI.unknown/1 is undefined or private"} + + # Check it also emits over a union + assert typewarn!( + [x = Atom, y = GenServer, z], + ( + mod = + cond do + z -> x + true -> y + end + + mod.to_string(:atom) + ) + ) == + {union(dynamic(), binary()), "GenServer.to_string/1 is undefined or private"} + end + + test "calling a function with none()" do + assert typeerror!(Integer.to_string(raise "oops")) |> strip_ansi() == + ~l""" + incompatible types given to Integer.to_string/1: + + Integer.to_string(raise RuntimeError.exception("oops")) + + given types: + + none() + + the 1st argument is empty (often represented as none()), \ + most likely because it is the result of an expression that \ + always fails, such as a `raise` or a previous invalid call. \ + This causes any function called with this value to fail + """ + end + + test "calling a nullary function on non atoms" do + assert typeerror!([<>], x.foo_bar()) == + ~l""" + expected a module (an atom) when invoking foo_bar/0 in expression: + + x.foo_bar() + + where "x" was given the type: + + # type: integer() + # from: types_test.ex:LINE-1 + <> + + #{hints(:dot)} + """ + end + + test "calling a function on non atoms with arguments" do + assert typeerror!([<>], x.foo_bar(1, 2)) == + ~l""" + expected a module (an atom) when invoking foo_bar/2 in expression: + + x.foo_bar(1, 2) + + where "x" was given the type: + + # type: integer() + # from: types_test.ex:LINE-1 + <> + """ + + assert typeerror!( + [<>, y = SomeMod, z], + ( + mod = + cond do + z -> x + true -> y + end + + mod.to_string(:atom) + ) + ) == + ~l""" + expected a module (an atom) when invoking to_string/1 in expression: + + mod.to_string(:atom) + + where "mod" was given the type: + + # type: dynamic(SomeMod) or integer() + # from: types_test.ex:LINE-9 + mod = + cond do + z -> x + true -> y + end + """ + end + + test "calling a function with invalid arguments on variables" do + assert typeerror!( + ( + x = List + x.to_tuple(123) + ) + ) + |> strip_ansi() == + ~l""" + incompatible types given to List.to_tuple/1: + + x.to_tuple(123) + + given types: + + integer() + + but expected one of: + + list(term()) + + where "x" was given the type: + + # type: List + # from: types_test.ex:LINE-5 + x = List + """ + end + + test "capture a function with non atoms" do + assert typeerror!([<>], &x.foo_bar/2) == + ~l""" + expected a module (an atom) when invoking foo_bar/2 in expression: + + &x.foo_bar/2 + + but got type: + + integer() + + where "x" was given the type: + + # type: integer() + # from: types_test.ex:LINE-1 + <> + """ + end + + test "requires all combinations to be compatible (except refinements)" do + assert typecheck!( + [condition, arg], + ( + # While the code below may raise, it may also always succeed + # if condition and arg are passed in tandem. Therefore, we + # turn off refinement on dynamic calls. + mod = if condition, do: String, else: List + res = mod.to_integer(arg) + {arg, res} + ) + ) == tuple([dynamic(), integer()]) + + assert typeerror!( + [condition], + ( + arg = if condition, do: "foo", else: [?f, ?o, ?o] + mod = if condition, do: String, else: List + mod.to_integer(arg) + ) + ) + |> strip_ansi() == ~l""" + incompatible types given to List.to_integer/1: + + mod.to_integer(arg) + #=> invoked as List.to_integer/1 + + given types: + + binary() or non_empty_list(integer()) + + but expected one of: + + non_empty_list(integer()) + + where "arg" was given the type: + + # type: binary() or non_empty_list(integer()) + # from: types_test.ex:LINE-5 + arg = + if condition do + "foo" + else + ~c"foo" + end + + where "mod" was given the type: + + # type: List or String + # from: types_test.ex:LINE-4 + mod = + if condition do + String + else + List + end + """ + end + end + + describe "remote capture" do + test "strong" do + assert typecheck!(&String.to_atom/1) == fun([binary()], atom()) + assert typecheck!(&:erlang.element/2) == fun([integer(), open_tuple([])], dynamic()) + end + + test "unknown" do + assert typecheck!(&Module.Types.ExprTest.__ex_unit__/1) == dynamic(fun(1)) + assert typecheck!([x], &x.something/1) == dynamic(fun(1)) + end + end + + describe "binaries" do + test "inference" do + assert typecheck!( + [x, y], + ( + <> + {x, y} + ) + ) == dynamic(tuple([union(float(), integer()), integer()])) + end + + test "warnings" do + assert typeerror!([<>], <>) == + ~l""" + incompatible types in binary construction: + + <> + + got type: + + binary() + + but expected type: + + float() or integer() + + where "x" was given the type: + + # type: binary() + # from: types_test.ex:LINE-1 + <> + """ + + assert typeerror!([<>], <>) == + ~l""" + incompatible types in binary construction: + + <> + + got type: + + binary() + + but expected type: + + integer() + + where "x" was given the type: + + # type: binary() + # from: types_test.ex:LINE-1 + <> + + #{hints(:inferred_bitstring_spec)} + """ + + assert typeerror!([<>], <>) == + ~l""" + incompatible types in binary construction: + + <> + + got type: + + integer() + + but expected type: + + binary() + + where "x" was given the type: + + # type: integer() + # from: types_test.ex:LINE-1 + <> + + #{hints(:inferred_bitstring_spec)} + """ + end + + test "size ok" do + assert typecheck!([<>, z], <>) == binary() + end + + test "size error" do + assert typeerror!([<>, y], <>) == + ~l""" + expected an integer in binary size: + + size(x) + + got type: + + binary() + + where "x" was given the type: + + # type: binary() + # from: types_test.ex:LINE-1 + <> + """ + end + end + + describe "tuples" do + test "creating tuples" do + assert typecheck!({:ok, 123}) == tuple([atom([:ok]), integer()]) + assert typecheck!([x], {:ok, x}) == dynamic(tuple([atom([:ok]), term()])) + end + + test "inference" do + assert typecheck!( + [x, y], + ( + {:ok, :error} = {x, y} + {x, y} + ) + ) == dynamic(tuple([atom([:ok]), atom([:error])])) + end + + test "elem/2" do + assert typecheck!(elem({:ok, 123}, 0)) == atom([:ok]) + assert typecheck!(elem({:ok, 123}, 1)) == integer() + assert typecheck!([x], elem({:ok, x}, 0)) == dynamic(atom([:ok])) + assert typecheck!([x], elem({:ok, x}, 1)) == dynamic(term()) + + assert typeerror!([<>], elem(x, 0)) |> strip_ansi() == + ~l""" + incompatible types given to Kernel.elem/2: + + elem(x, 0) + + given types: + + float(), integer() + + but expected one of: + + {...}, integer() + + where "x" was given the type: + + # type: float() + # from: types_test.ex:LINE-1 + <> + """ + + assert typeerror!(elem({:ok, 123}, 2)) == + ~l""" + expected a tuple with at least 3 elements in Kernel.elem/2: + + elem({:ok, 123}, 2) + + the given type does not have the given index: + + {:ok, integer()} + """ + end + + test "Tuple.insert_at/3" do + assert typecheck!(Tuple.insert_at({}, 0, "foo")) == tuple([binary()]) + + assert typecheck!(Tuple.insert_at({:ok, 123}, 0, "foo")) == + tuple([binary(), atom([:ok]), integer()]) + + assert typecheck!(Tuple.insert_at({:ok, 123}, 1, "foo")) == + tuple([atom([:ok]), binary(), integer()]) + + assert typecheck!(Tuple.insert_at({:ok, 123}, 2, "foo")) == + tuple([atom([:ok]), integer(), binary()]) + + assert typeerror!([<>], Tuple.insert_at(x, 0, "foo")) |> strip_ansi() == + ~l""" + incompatible types given to Tuple.insert_at/3: + + Tuple.insert_at(x, 0, "foo") + + given types: + + float(), integer(), binary() + + but expected one of: + + {...}, integer(), term() + + where "x" was given the type: + + # type: float() + # from: types_test.ex:LINE-1 + <> + """ + + assert typeerror!(Tuple.insert_at({:ok, 123}, 3, "foo")) == + ~l""" + expected a tuple with at least 3 elements in Tuple.insert_at/3: + + Tuple.insert_at({:ok, 123}, 3, "foo") + + the given type does not have the given index: + + {:ok, integer()} + """ + end + + test "Tuple.delete_at/2" do + assert typecheck!(Tuple.delete_at({:ok, 123}, 0)) == tuple([integer()]) + assert typecheck!(Tuple.delete_at({:ok, 123}, 1)) == tuple([atom([:ok])]) + assert typecheck!([x], Tuple.delete_at({:ok, x}, 0)) == dynamic(tuple([term()])) + assert typecheck!([x], Tuple.delete_at({:ok, x}, 1)) == dynamic(tuple([atom([:ok])])) + + assert typeerror!([<>], Tuple.delete_at(x, 0)) |> strip_ansi() == + ~l""" + incompatible types given to Tuple.delete_at/2: + + Tuple.delete_at(x, 0) + + given types: + + float(), integer() + + but expected one of: + + {...}, integer() + + where "x" was given the type: + + # type: float() + # from: types_test.ex:LINE-1 + <> + """ + + assert typeerror!(Tuple.delete_at({:ok, 123}, 2)) == + ~l""" + expected a tuple with at least 3 elements in Tuple.delete_at/2: + + Tuple.delete_at({:ok, 123}, 2) + + the given type does not have the given index: + + {:ok, integer()} + """ + end + + test "Tuple.duplicate/2" do + assert typecheck!(Tuple.duplicate(123, 0)) == tuple([]) + assert typecheck!(Tuple.duplicate(123, 1)) == tuple([integer()]) + assert typecheck!(Tuple.duplicate(123, 2)) == tuple([integer(), integer()]) + assert typecheck!([x], Tuple.duplicate(x, 2)) == dynamic(tuple([term(), term()])) + end + end + + describe "maps/structs" do + test "creating maps as records" do + assert typecheck!(%{foo: :bar}) == closed_map(foo: atom([:bar])) + assert typecheck!([x], %{key: x}) == dynamic(closed_map(key: term())) + end + + test "creating maps as records with dynamic keys" do + assert typecheck!( + ( + foo = :foo + %{foo => :first, foo => :second} + ) + ) == closed_map(foo: atom([:second])) + + assert typecheck!( + ( + foo_or_bar = + cond do + :rand.uniform() > 0.5 -> :foo + true -> :bar + end + + %{foo_or_bar => :first, foo_or_bar => :second} + ) + ) + |> equal?( + closed_map(foo: atom([:second])) + |> union(closed_map(bar: atom([:second]))) + |> union(closed_map(foo: atom([:first]), bar: atom([:second]))) + |> union(closed_map(bar: atom([:first]), foo: atom([:second]))) + ) + end + + test "creating maps as dictionaries" do + assert typecheck!(%{123 => 456}) == closed_map([{domain_key(:integer), integer()}]) + + # Since key cannot override :foo based on position, we preserve it + assert typecheck!([key], %{key => 456, foo: :bar}) == + dynamic( + closed_map([ + {to_domain_keys(:term), integer()}, + {:foo, atom([:bar])} + ]) + ) + + # Since key can override :foo based on position, we union it + assert typecheck!([key], %{:foo => :bar, key => :baz}) == + dynamic( + closed_map([ + {to_domain_keys(:term), atom([:baz])}, + {:foo, atom([:bar, :baz])} + ]) + ) + + # Since key cannot override :foo based on domain, we preserve it + assert typecheck!( + [arg], + ( + key = String.to_integer(arg) + %{:foo => :bar, key => :baz} + ) + ) == + closed_map([ + {domain_key(:integer), atom([:baz])}, + {:foo, atom([:bar])} + ]) + + # Multiple keys are fully overridden for simplicity + assert typecheck!( + [arg], + ( + foo_or_bar = if String.starts_with?(arg, "0"), do: :foo, else: :bar + key = String.to_integer(arg) + %{foo_or_bar => :old, key => :new} + ) + ) == + union( + closed_map([ + {domain_key(:integer), atom([:new])}, + {:foo, atom([:old])} + ]), + closed_map([ + {domain_key(:integer), atom([:new])}, + {:bar, atom([:old])} + ]) + ) + end + + test "creating structs" do + assert typecheck!(%Point{}) == + closed_map( + __struct__: atom([Point]), + x: atom([nil]), + y: atom([nil]), + z: integer() + ) + + assert typecheck!(%Point{x: :zero}) == + closed_map( + __struct__: atom([Point]), + x: atom([:zero]), + y: atom([nil]), + z: integer() + ) + end + + test "updating to maps as records" do + assert typecheck!([x], %{x | x: :zero}) == + dynamic(open_map(x: atom([:zero]))) + + assert typecheck!([x], %{%{x | x: :zero} | y: :one}) == + dynamic(open_map(x: atom([:zero]), y: atom([:one]))) + + assert typecheck!( + ( + foo_or_bar = + cond do + :rand.uniform() > 0.5 -> :key1 + true -> :key2 + end + + x = %{key1: :one, key2: :two} + %{x | foo_or_bar => :one!, foo_or_bar => :two!} + ) + ) + |> equal?( + closed_map(key1: atom([:one]), key2: atom([:two!])) + |> union(closed_map(key1: atom([:two!]), key2: atom([:one!]))) + |> union(closed_map(key1: atom([:one!]), key2: atom([:two!]))) + |> union(closed_map(key1: atom([:two!]), key2: atom([:two]))) + ) + + assert typeerror!([x = :foo], %{x | x: :zero}) == ~l""" + expected a map within map update syntax: + + %{x | x: :zero} + + but got type: + + dynamic(:foo) + + where "x" was given the type: + + # type: dynamic(:foo) + # from: types_test.ex:LINE + x = :foo + """ + + assert typeerror!( + ( + x = %{} + %{x | x: :zero} + ) + ) == ~l""" + expected a map with key :x in map update syntax: + + %{x | x: :zero} + + but got type: + + empty_map() + + where "x" was given the type: + + # type: empty_map() + # from: types_test.ex:LINE-3 + x = %{} + """ + + # Assert we check all possible combinations + assert typeerror!( + ( + foo_or_bar = + cond do + :rand.uniform() > 0.5 -> :foo + true -> :bar + end + + x = %{foo: :baz} + %{x | foo_or_bar => :bat} + ) + ) == ~l""" + expected a map with key :bar in map update syntax: + + %{x | foo_or_bar => :bat} + + but got type: + + %{foo: :baz} + + where "foo_or_bar" was given the type: + + # type: :bar or :foo + # from: types_test.ex:LINE-9 + foo_or_bar = + cond do + :rand.uniform() > 0.5 -> :foo + true -> :bar + end + + where "x" was given the type: + + # type: %{foo: :baz} + # from: types_test.ex:LINE-3 + x = %{foo: :baz} + """ + + # The goal of this assertion is to verify we assert keys, + # even if they may be overridden later. + assert typeerror!( + [key], + ( + x = %{key: :value} + %{x | :foo => :baz, key => :bat} + ) + ) == ~l""" + expected a map with key :foo in map update syntax: + + %{x | :foo => :baz, key => :bat} + + but got type: + + %{key: :value} + + where "key" was given the type: + + # type: dynamic() + # from: types_test.ex:LINE-5 + key + + where "x" was given the type: + + # type: %{key: :value} + # from: types_test.ex:LINE-3 + x = %{key: :value} + """ + end + + test "updating structs" do + integer_date_type = + dynamic( + closed_map( + __struct__: atom([Date]), + day: integer(), + calendar: atom(), + month: term(), + year: term() + ) + ) + + # When we know the type + assert typecheck!([], %Date{Date.new!(1, 1, 1) | day: 31}) == + integer_date_type + + assert typecheck!([], %Date{%Date{Date.new!(1, 1, 1) | day: 13} | day: 31}) == + integer_date_type + + # When we don't know the type of var + assert typeerror!([x], %Date{x | day: 31}) == ~l""" + a struct for Date is expected on struct update: + + %Date{x | day: 31} + + but got type: + + dynamic() + + where "x" was given the type: + + # type: dynamic() + # from: types_test.ex:LINE + x + + when defining the variable "x", you must also pattern match on "%Date{}". + + hint: given pattern matching is enough to catch typing errors, you may optionally convert the struct update into a map update. For example, instead of: + + user = some_function() + %User{user | name: "John Doe"} + + it is enough to write: + + %User{} = user = some_function() + %{user | name: "John Doe"} + """ + + # When we don't know the type of capture + assert typeerror!([], &%Date{&1 | day: 31}) =~ ~l""" + a struct for Date is expected on struct update: + + %Date{&1 | day: 31} + + but got type: + + dynamic() + + where "capture" was given the type: + + # type: dynamic() + # from: types_test.ex:LINE + &1 + + instead of using &1, you must define an anonymous function, define a variable and pattern match on "%Date{}". + """ + + # When we don't know the type of expression + assert typeerror!([], %Date{SomeMod.fun() | day: 31}) =~ """ + a struct for Date is expected on struct update: + + %Date{SomeMod.fun() | day: 31} + + but got type: + + dynamic() + + you must assign "SomeMod.fun()" to variable and pattern match on "%Date{}". + """ + end + + test "updating to maps as dictionaries" do + assert typecheck!( + [key], + ( + x = %{foo: :bar} + %{x | key => :baz} + ) + ) == closed_map(foo: atom([:bar, :baz])) + + # Override based on position + assert typecheck!( + [key], + ( + x = %{foo: :bar, baz: :bat} + %{x | key => :old, foo: :new} + ) + ) == closed_map(foo: atom([:new]), baz: atom([:old, :bat])) + + assert typeerror!( + [key], + ( + x = %{String.to_integer(key) => :old} + %{x | String.to_atom(key) => :new} + ) + ) == ~l""" + expected a map with key of type atom() in map update syntax: + + %{x | String.to_atom(key) => :new} + + but got type: + + %{integer() => :old} + + where "key" was given the type: + + # type: binary() + # from: types_test.ex:LINE-3 + String.to_integer(key) + + where "x" was given the type: + + # type: %{integer() => :old} + # from: types_test.ex:LINE-3 + x = %{String.to_integer(key) => :old} + """ + + assert typeerror!( + [key], + ( + x = %{key: :old} + %{x | String.to_atom(key) => :new} + ) + ) == ~l""" + expected a map with key of type atom() in map update syntax: + + %{x | String.to_atom(key) => :new} + + but got type: + + %{key: :old} + + where "key" was given the type: + + # type: binary() + # from: types_test.ex:LINE-2 + String.to_atom(key) + + where "x" was given the type: + + # type: %{key: :old} + # from: types_test.ex:LINE-3 + x = %{key: :old} + """ + end + + test "nested map" do + assert typecheck!([x = %{}], x.foo.bar) == dynamic() + end + + test "accessing a field on not a map" do + assert typeerror!([<>], x.foo_bar) == + ~l""" + expected a map or struct when accessing .foo_bar in expression: + + x.foo_bar + + where "x" was given the type: + + # type: integer() + # from: types_test.ex:LINE-1 + <> + + #{hints(:dot)} + """ + end + + test "accessing an unknown field on struct with diagnostic" do + {type, [diagnostic]} = typediag!(%Point{}.foo_bar) + assert type == dynamic() + assert diagnostic.span == {__ENV__.line - 2, 56} + + assert diagnostic.message == ~l""" + unknown key .foo_bar in expression: + + %Point{x: nil, y: nil, z: 0}.foo_bar + + the given type does not have the given key: + + %Point{x: nil, y: nil, z: integer()} + """ + end + + test "accessing an unknown field on struct in a var with diagnostic" do + {type, [diagnostic]} = typediag!([x = %URI{}], x.foo_bar) + assert type == dynamic() + assert diagnostic.span == {__ENV__.line - 2, 63} + + assert diagnostic.message == ~l""" + unknown key .foo_bar in expression: + + x.foo_bar + + the given type does not have the given key: + + dynamic(%URI{ + scheme: term(), + authority: term(), + userinfo: term(), + host: term(), + port: term(), + path: term(), + query: term(), + fragment: term() + }) + + where "x" was given the type: + + # type: dynamic(%URI{}) + # from: types_test.ex:LINE-4:43 + x = %URI{} + """ + + assert [%{type: :variable, name: :x}] = diagnostic.details.typing_traces + end + + test "inspect struct definition" do + assert typeerror!( + ( + p = %Point{x: 123} + Integer.to_string(p) + ) + ) + |> strip_ansi() == ~l""" + incompatible types given to Integer.to_string/1: + + Integer.to_string(p) + + given types: + + %Point{x: integer(), y: nil, z: integer()} + + but expected one of: + + integer() + + where "p" was given the type: + + # type: %Point{x: integer(), y: nil, z: integer()} + # from: types_test.ex:LINE-4 + p = %Point{..., x: 123} + """ + end + end + + describe "comparison" do + test "in static mode" do + assert typecheck!([x = 123, y = 456.0], x < y) == boolean() + assert typecheck!([x = 123, y = 456.0], x == y) == boolean() + end + + test "in dynamic mode" do + assert typedyn!([x = 123, y = 456.0], x < y) == dynamic(boolean()) + assert typedyn!([x = 123, y = 456.0], x == y) == dynamic(boolean()) + assert typedyn!([x = 123, y = 456], x == y) == dynamic(boolean()) + end + + test "using literals" do + assert typecheck!(:foo == :bar) == boolean() + end + + test "min/max" do + assert typecheck!(min(123, 456.0)) == union(integer(), float()) + # min/max uses parametric types, which will carry dynamic regardless of being a strong arrow + assert typecheck!([x = 123, y = 456.0], min(x, y)) == dynamic(union(integer(), float())) + assert typedyn!([x = 123, y = 456.0], min(x, y)) == dynamic(union(integer(), float())) + end + + test "warns when comparison is constant" do + assert typeerror!([x = :foo, y = 321], min(x, y)) == + ~l""" + comparison between distinct types found: + + min(x, y) + + given types: + + min(dynamic(:foo), integer()) + + where "x" was given the type: + + # type: dynamic(:foo) + # from: types_test.ex:LINE-1 + x = :foo + + where "y" was given the type: + + # type: integer() + # from: types_test.ex:LINE-1 + y = 321 + + While Elixir can compare across all types, you are comparing across types \ + which are always disjoint, and the result is either always true or always false + """ + + assert typeerror!([x = 123, y = 456.0], x === y) == + ~l""" + comparison between distinct types found: + + x === y + + given types: + + integer() === float() + + where "x" was given the type: + + # type: integer() + # from: types_test.ex:LINE-1 + x = 123 + + where "y" was given the type: + + # type: float() + # from: types_test.ex:LINE-1 + y = 456.0 + + While Elixir can compare across all types, you are comparing across types \ + which are always disjoint, and the result is either always true or always false + """ + end + + test "warns on comparison with struct across dynamic call" do + assert typeerror!([x = %Point{}, y = %Point{}, mod = Kernel], mod.<=(x, y)) == + ~l""" + comparison with structs found: + + mod.<=(x, y) + + given types: + + dynamic(%Point{}) <= dynamic(%Point{}) + + where "mod" was given the type: + + # type: dynamic(Kernel) + # from: types_test.ex:LINE-1 + mod = Kernel + + where "x" was given the type: + + # type: dynamic(%Point{}) + # from: types_test.ex:LINE-1 + x = %Point{} + + where "y" was given the type: + + # type: dynamic(%Point{}) + # from: types_test.ex:LINE-1 + y = %Point{} + + Comparison operators (>, <, >=, <=, min, and max) perform structural and not semantic comparison. Comparing with a struct won't give meaningful results. Structs that can be compared typically define a compare/2 function within their modules that can be used for semantic comparison. + """ + + assert typeerror!( + [x = %Point{}, mod = Kernel, condition], + ( + y = if condition, do: 456, else: %Point{} + mod.<=(x, y) + ) + ) =~ "comparison with structs found:" + + assert typecheck!( + [x = 123, mod = Kernel, condition], + ( + y = if condition, do: 456, else: %Point{} + mod.<=(x, y) + ) + ) == boolean() + + assert typeerror!( + [mod = Kernel, condition], + ( + x = if condition, do: 123, else: %Point{} + y = if condition, do: 456, else: %Point{} + mod.<=(x, y) + ) + ) =~ "comparison with structs found:" + end + end + + describe ":erlang rewrites" do + test "Kernel.not/1" do + assert typecheck!([x], not is_list(x)) == boolean() + assert typedyn!([x], not is_list(x)) == dynamic(boolean()) + end + + test "Kernel.+/2" do + assert typeerror!([x = :foo, y = 123], x + y) |> strip_ansi() == + ~l""" + incompatible types given to Kernel.+/2: + + x + y + + given types: + + dynamic(:foo), integer() + + but expected one of: + + #1 + integer(), integer() + + #2 + integer(), float() + + #3 + float(), integer() + + #4 + float(), float() + + where "x" was given the type: + + # type: dynamic(:foo) + # from: types_test.ex:LINE-1 + x = :foo + + where "y" was given the type: + + # type: integer() + # from: types_test.ex:LINE-1 + y = 123 + """ + end + + test "Integer.to_string/1" do + assert typecheck!([x = 123], Integer.to_string(x)) == binary() + assert typedyn!([x = 123], Integer.to_string(x)) == dynamic(binary()) + + assert typeerror!([x = :foo], Integer.to_string(x)) |> strip_ansi() == + ~l""" + incompatible types given to Integer.to_string/1: + + Integer.to_string(x) + + given types: + + dynamic(:foo) + + but expected one of: + + integer() + + where "x" was given the type: + + # type: dynamic(:foo) + # from: types_test.ex:LINE-1 + x = :foo + """ + end + + test "Bitwise.bnot/1" do + assert typecheck!([x = 123], Bitwise.bnot(x)) == integer() + assert typedyn!([x = 123], Bitwise.bnot(x)) == dynamic(integer()) + + assert typeerror!([x = :foo], Bitwise.bnot(x)) |> strip_ansi() == + ~l""" + incompatible types given to Bitwise.bnot/1: + + Bitwise.bnot(x) + + given types: + + dynamic(:foo) + + but expected one of: + + integer() + + where "x" was given the type: + + # type: dynamic(:foo) + # from: types_test.ex:LINE-1 + x = :foo + """ + end + end + + describe "case" do + test "does not type check literals" do + assert typecheck!( + case :dev do + :dev -> :ok + :prod -> :error + end + ) == atom([:ok, :error]) + end + + test "resets branches" do + assert typecheck!( + [x], + ( + case :rand.uniform() do + y when y < 0.5 -> x.foo + y when y > 0.5 -> x.bar() + end + + x + ) + ) == dynamic() + end + + test "returns unions of all clauses" do + assert typecheck!( + [x], + case x do + :ok -> :ok + :error -> :error + end + ) == atom([:ok, :error]) + + assert typedyn!( + [x], + case x do + :ok -> :ok + :error -> :error + end + ) == dynamic(atom([:ok, :error])) + end + + test "reports error from clause that will never match" do + assert typeerror!( + [x], + case Atom.to_string(x) do + :error -> :error + x -> x + end + ) == ~l""" + the following clause will never match: + + :error + + because it attempts to match on the result of: + + Atom.to_string(x) + + which has type: + + binary() + """ + end + + test "reports errors from multiple clauses" do + {type, [_, _]} = + typediag!( + [x], + case Atom.to_string(x) do + :ok -> :ok + :error -> :error + end + ) + + assert type == atom([:ok, :error]) + end + end + + describe "conditionals" do + test "if does not report on literals" do + assert typecheck!( + if true do + :ok + end + ) == atom([:ok, nil]) + end + + test "and does not report on literals" do + assert typecheck!(false and true) == boolean() + end + + test "and reports violations" do + assert typeerror!([x = 123], x and true) =~ """ + the following conditional expression will always evaluate to integer(): + + x + """ + end + end + + describe "receive" do + test "returns unions of all clauses" do + assert typecheck!( + receive do + :ok -> :ok + :error -> :error + after + 0 -> :timeout + end + ) == atom([:ok, :error, :timeout]) + + assert typedyn!( + receive do + :ok -> :ok + :error -> :error + after + 0 -> :timeout + end + ) == dynamic(atom([:ok, :error, :timeout])) + end + + test "infers type for timeout" do + assert typecheck!( + [x], + receive do + after + x -> x + end + ) == dynamic(union(integer(), atom([:infinity]))) + end + + test "resets branches" do + assert typecheck!( + [x, timeout = :infinity], + ( + receive do + y when y > 0.5 -> x.foo + _ -> x.bar() + after + timeout -> <<^x::integer>> = :crypto.strong_rand_bytes(1) + end + + x + ) + ) == dynamic() + end + + test "errors on bad timeout" do + assert typeerror!( + [x = :timeout], + receive do + after + x -> :ok + end + ) == ~l""" + expected "after" timeout given to receive to be an integer: + + x + + but got type: + + dynamic(:timeout) + + where "x" was given the type: + + # type: dynamic(:timeout) + # from: types_test.ex:LINE-5 + x = :timeout + """ + + # Check for compatibility, not subtyping + assert typeerror!( + [<>], + receive do + after + if(:rand.uniform(), do: x, else: y) -> :ok + end + ) =~ "expected " + after + " timeout given to receive to be an integer" + end + end + + describe "try" do + test "returns unions of all clauses" do + assert typecheck!( + try do + :do + rescue + _ -> :rescue + catch + :caught -> :caught1 + :throw, :caught -> :caught2 + after + :not_used + end + ) == atom([:do, :caught1, :caught2, :rescue]) + + assert typecheck!( + [x], + try do + x + rescue + _ -> :rescue + catch + :caught -> :caught1 + :throw, :caught -> :caught2 + after + :not_used + else + :match -> :else1 + _ -> :else2 + end + ) == atom([:caught1, :caught2, :rescue, :else1, :else2]) + end + + test "resets branches (except after)" do + assert typecheck!( + [x], + ( + try do + <<^x::float>> = :crypto.strong_rand_bytes(8) + rescue + ArgumentError -> x.foo + catch + _, _ -> x.bar() + after + <<^x::integer>> = :crypto.strong_rand_bytes(8) + end + + x + ) + ) == dynamic(integer()) + end + + test "reports error from clause that will never match" do + assert typeerror!( + [x], + try do + Atom.to_string(x) + rescue + _ -> :ok + else + :error -> :error + x -> x + end + ) == ~l""" + the following clause will never match: + + :error + + it attempts to match on the result of the try do-block which has incompatible type: + + binary() + """ + end + + test "warns on undefined exceptions" do + assert typewarn!( + try do + :ok + rescue + e in UnknownError -> e + end + ) == + {dynamic() |> union(atom([:ok])), + "struct UnknownError is undefined (module UnknownError is not available or is yet to be defined). " <> + "Make sure the module name is correct and has been specified in full (or that an alias has been defined)"} + + assert typewarn!( + try do + :ok + rescue + e in Enumerable -> e + end + ) == + {dynamic() |> union(atom([:ok])), + "struct Enumerable is undefined (there is such module but it does not define a struct)"} + end + + test "defines unions of exceptions in rescue" do + assert typecheck!( + try do + raise "oops" + rescue + e in [RuntimeError, ArgumentError] -> + e + end + ) == + dynamic( + union( + closed_map( + __struct__: atom([ArgumentError]), + __exception__: atom([true]), + message: term() + ), + closed_map( + __struct__: atom([RuntimeError]), + __exception__: atom([true]), + message: term() + ) + ) + ) + end + + test "generates custom traces" do + assert typeerror!( + try do + raise "oops" + rescue + e -> + Integer.to_string(e) + end + ) + |> strip_ansi() == ~l""" + incompatible types given to Integer.to_string/1: + + Integer.to_string(e) + + given types: + + %{..., __exception__: true, __struct__: atom()} + + but expected one of: + + integer() + + where "e" was given the type: + + # type: %{..., __exception__: true, __struct__: atom()} + # from: types_test.ex + rescue e + + hint: when you rescue without specifying exception names, the variable is assigned a type of a struct but all of its fields are unknown. If you are trying to access an exception's :message key, either specify the exception names or use `Exception.message/1`. + """ + end + + test "defines an open map of two fields in anonymous rescue" do + assert typecheck!( + try do + raise "oops" + rescue + e -> e + end + ) == + open_map( + __struct__: atom(), + __exception__: atom([true]) + ) + end + + test "matches on stacktrace" do + assert typeerror!( + try do + :ok + rescue + _ -> + [{_, _, args_or_arity, _} | _] = __STACKTRACE__ + args_or_arity.fun() + end + ) =~ ~l""" + expected a module (an atom) when invoking fun/0 in expression: + + args_or_arity.fun() + + where "args_or_arity" was given the type: + + # type: integer() or list(term()) + # from: types_test.ex:LINE-3 + [{_, _, args_or_arity, _} | _] = __STACKTRACE__ + """ + end + end + + describe "cond" do + test "always true" do + assert typecheck!( + cond do + true -> :ok + end + ) == atom([:ok]) + + assert typecheck!( + [x, y], + cond do + y -> :y + x -> :x + end + ) == atom([:x, :y]) + + assert typedyn!( + [x, y], + cond do + y -> :y + x -> :x + end + ) == dynamic(atom([:x, :y])) + + assert typewarn!( + [x, y = {:foo, :bar}], + cond do + y -> :y + x -> :x + end + ) == + {atom([:x, :y]), + ~l""" + this clause in cond will always match: + + y + + since it has type: + + dynamic({:foo, :bar}) + + where "y" was given the type: + + # type: dynamic({:foo, :bar}) + # from: types_test.ex:LINE-7 + y = {:foo, :bar} + """} + end + + test "always false" do + assert typewarn!( + [x, y = false], + cond do + y -> :y + x -> :x + end + ) == + {atom([:x, :y]), + ~l""" + this clause in cond will never match: + + y + + since it has type: + + dynamic(false) + + where "y" was given the type: + + # type: dynamic(false) + # from: types_test.ex:LINE-7 + y = false + """} + end + + test "resets branches" do + assert typecheck!( + [x], + ( + cond do + :rand.uniform() > 0.5 -> x.foo + true -> x.bar() + end + + x + ) + ) == dynamic() + end + end + + describe "comprehensions" do + test "binary generators" do + assert typeerror!([<>], for(<>, do: y)) == + ~l""" + expected the right side of <- in a binary generator to be a binary: + + x + + but got type: + + integer() + + where "x" was given the type: + + # type: integer() + # from: types_test.ex:LINE-1 + <> + + #{hints(:inferred_bitstring_spec)} + """ + + # Check for compatibility, not subtyping + assert typeerror!( + [<>], + for(< 0.5, do: x, else: y)>>, do: i) + ) =~ + ~l""" + expected the right side of <- in a binary generator to be a binary: + + if :rand.uniform() > 0.5 do + x + else + y + end + + but got type: + + binary() or integer() + + where "x" was given the type: + + # type: integer() + # from: types_test.ex:LINE-3 + <> + + where "y" was given the type: + + # type: binary() + # from: types_test.ex:LINE-3 + <<..., y::binary>> + """ + end + + test "infers binary generators" do + assert typecheck!( + [x], + ( + for <<_ <- x>>, do: :ok + x + ) + ) == dynamic(binary()) + end + + test ":into" do + assert typecheck!([binary], for(<>, do: x)) == list(integer()) + assert typecheck!([binary], for(<>, do: x, into: [])) == list(integer()) + assert typecheck!([binary], for(<>, do: x, into: "")) == binary() + assert typecheck!([binary, other], for(<>, do: x, into: other)) == dynamic() + + assert typecheck!([enum], for(x <- enum, do: x)) == list(dynamic()) + assert typecheck!([enum], for(x <- enum, do: x, into: [])) == list(dynamic()) + assert typecheck!([enum], for(x <- enum, do: x, into: "")) == binary() + assert typecheck!([enum, other], for(x <- enum, do: x, into: other)) == dynamic() + + assert typecheck!( + [binary], + ( + into = if :rand.uniform() > 0.5, do: [], else: "0" + for(<>, do: x, into: into) + ) + ) == union(binary(), list(float())) + + assert typecheck!( + [binary, empty_list = []], + ( + into = if :rand.uniform() > 0.5, do: empty_list, else: "0" + for(<>, do: x, into: into) + ) + ) == dynamic(union(binary(), list(float()))) + end + + test ":reduce checks" do + assert typecheck!( + [list], + for _ <- list, reduce: :ok do + :ok -> 1 + _ -> 2.0 + end + ) == union(atom([:ok]), union(integer(), float())) + end + + test ":reduce inference" do + assert typecheck!( + [list, x], + ( + 123 = + for _ <- list, reduce: x do + x -> x + end + + x + ) + ) == dynamic(integer()) + end + end + + describe "info" do + test "__info__/1" do + assert typecheck!(GenServer.__info__(:functions)) == list(tuple([atom(), integer()])) + + assert typewarn!(:string.__info__(:functions)) == + {dynamic(), ":string.__info__/1 is undefined or private"} + + assert typecheck!([x], x.__info__(:functions)) == list(tuple([atom(), integer()])) + + assert typeerror!([x], x.__info__(:whatever)) |> strip_ansi() =~ """ + incompatible types given to __info__/1: + + x.__info__(:whatever) + + given types: + + :whatever + """ + end + + test "__info__/1 for struct information" do + assert typecheck!(GenServer.__info__(:struct)) == atom([nil]) + + assert typecheck!(URI.__info__(:struct)) == + list(closed_map(default: if_set(term()), field: atom())) + + assert typecheck!([x], x.__info__(:struct)) == + list(closed_map(default: if_set(term()), field: atom())) |> union(atom([nil])) + end + + test "behaviour_info/1" do + assert typecheck!([x], x.behaviour_info(:callbacks)) == list(tuple([atom(), integer()])) + + assert typecheck!(GenServer.behaviour_info(:callbacks)) == list(tuple([atom(), integer()])) + + assert typewarn!(String.behaviour_info(:callbacks)) == + {dynamic(), "String.behaviour_info/1 is undefined or private"} + end + + test "module_info/1" do + assert typecheck!([x], x.module_info(:exports)) == list(tuple([atom(), integer()])) + assert typecheck!(GenServer.module_info(:exports)) == list(tuple([atom(), integer()])) + end + + test "module_info/0" do + assert typecheck!([x], x.module_info()) |> subtype?(list(tuple([atom(), term()]))) + assert typecheck!(GenServer.module_info()) |> subtype?(list(tuple([atom(), term()]))) + end + end +end diff --git a/lib/elixir/test/elixir/module/types/helpers_test.exs b/lib/elixir/test/elixir/module/types/helpers_test.exs new file mode 100644 index 00000000000..96fe6509ee4 --- /dev/null +++ b/lib/elixir/test/elixir/module/types/helpers_test.exs @@ -0,0 +1,21 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team + +Code.require_file("type_helper.exs", __DIR__) + +defmodule Module.Types.HelpersTest do + use ExUnit.Case, async: true + import Module.Types.Helpers + + test "expr_to_string/1" do + assert expr_to_string({1, 2}) == "{1, 2}" + assert expr_to_string(quote(do: Foo.bar(arg))) == "Foo.bar(arg)" + assert expr_to_string(quote(do: :erlang.band(a, b))) == "Bitwise.band(a, b)" + assert expr_to_string(quote(do: :erlang.orelse(a, b))) == "a or b" + assert expr_to_string(quote(do: :erlang."=:="(a, b))) == "a === b" + assert expr_to_string(quote(do: :erlang.list_to_atom(a))) == "List.to_atom(a)" + assert expr_to_string(quote(do: :maps.remove(a, b))) == "Map.delete(b, a)" + assert expr_to_string(quote(do: :erlang.element(1, a))) == "elem(a, 0)" + assert expr_to_string(quote(do: :erlang.element(:erlang.+(a, 1), b))) == "elem(b, a)" + end +end diff --git a/lib/elixir/test/elixir/module/types/infer_test.exs b/lib/elixir/test/elixir/module/types/infer_test.exs new file mode 100644 index 00000000000..ba51903b2d9 --- /dev/null +++ b/lib/elixir/test/elixir/module/types/infer_test.exs @@ -0,0 +1,133 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team + +Code.require_file("type_helper.exs", __DIR__) + +defmodule Module.Types.InferTest do + use ExUnit.Case, async: true + + import Module.Types.Descr + + defmacro infer(config, do: block) do + quote do + runtime_infer(unquote(config).test, unquote(Macro.escape(block))) + end + end + + defp runtime_infer(module, block) do + {{:module, _, binary, _}, []} = + Code.eval_quoted( + quote do + defmodule unquote(module), do: unquote(block) + end, + [] + ) + + version = :elixir_erl.checker_version() + {:ok, {_, [{~c"ExCk", chunk}]}} = :beam_lib.chunks(binary, [~c"ExCk"]) + {^version, data} = :erlang.binary_to_term(chunk) + for {fun, %{sig: sig}} <- data.exports, into: %{}, do: {fun, sig} + end + + test "infer types from patterns", config do + types = + infer config do + def fun1(%y{}, %x{}, x = y, x = Point), do: :ok + def fun2(%x{}, %y{}, x = y, x = Point), do: :ok + def fun3(%y{}, %x{}, x = y, y = Point), do: :ok + def fun4(%x{}, %y{}, x = y, y = Point), do: :ok + end + + args = [ + dynamic(open_map(__struct__: atom([Point]))), + dynamic(open_map(__struct__: atom([Point]))), + dynamic(atom([Point])), + dynamic(atom([Point])) + ] + + assert types[{:fun1, 4}] == {:infer, nil, [{args, atom([:ok])}]} + assert types[{:fun2, 4}] == {:infer, nil, [{args, atom([:ok])}]} + assert types[{:fun3, 4}] == {:infer, nil, [{args, atom([:ok])}]} + assert types[{:fun4, 4}] == {:infer, nil, [{args, atom([:ok])}]} + end + + test "infer types from expressions", config do + types = + infer config do + def fun(x) do + x.foo + x.bar + end + end + + number = union(integer(), float()) + + assert types[{:fun, 1}] == + {:infer, nil, [{[dynamic(open_map(foo: number, bar: number))], dynamic(number)}]} + end + + test "infer with Elixir built-in", config do + types = + infer config do + def parse(string), do: Integer.parse(string) + end + + assert types[{:parse, 1}] == + {:infer, nil, + [{[dynamic()], dynamic(union(atom([:error]), tuple([integer(), term()])))}]} + end + + test "merges patterns", config do + types = + infer config do + def fun(:ok), do: :one + def fun("two"), do: :two + def fun("three"), do: :three + def fun("four"), do: :four + def fun(:error), do: :five + end + + assert types[{:fun, 1}] == + {:infer, [dynamic(union(atom([:ok, :error]), binary()))], + [ + {[dynamic(atom([:ok]))], atom([:one])}, + {[dynamic(binary())], atom([:two, :three, :four])}, + {[dynamic(atom([:error]))], atom([:five])} + ]} + end + + test "infers return types from private functions", config do + types = + infer config do + def pub(x), do: priv(x) + defp priv(:ok), do: :ok + defp priv(:error), do: :error + end + + assert types[{:pub, 1}] == + {:infer, nil, [{[dynamic(atom([:ok, :error]))], dynamic(atom([:ok, :error]))}]} + + assert types[{:priv, 1}] == nil + end + + test "infers return types from super functions", config do + types = + infer config do + def pub(:ok), do: :ok + def pub(:error), do: :error + defoverridable pub: 1 + def pub(x), do: super(x) + end + + assert types[{:pub, 1}] == + {:infer, nil, [{[dynamic(atom([:ok, :error]))], dynamic(atom([:ok, :error]))}]} + end + + test "infers return types even with loops", config do + types = + infer config do + def pub(x), do: pub(x) + end + + assert types[{:pub, 1}] == {:infer, nil, [{[dynamic()], dynamic()}]} + end +end diff --git a/lib/elixir/test/elixir/module/types/integration_test.exs b/lib/elixir/test/elixir/module/types/integration_test.exs new file mode 100644 index 00000000000..6d2b18017f0 --- /dev/null +++ b/lib/elixir/test/elixir/module/types/integration_test.exs @@ -0,0 +1,1585 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("type_helper.exs", __DIR__) + +defmodule Module.Types.IntegrationTest do + use ExUnit.Case + + import ExUnit.CaptureIO + import Module.Types.Descr + + defp builtin_protocols do + [ + Collectable, + Enumerable, + IEx.Info, + Inspect, + JSON.Encoder, + List.Chars, + String.Chars + ] + end + + test "built-in protocols" do + builtin_protocols = + for app <- ~w[eex elixir ex_unit iex logger mix]a, + Application.ensure_loaded(app), + module <- Application.spec(app, :modules), + Code.ensure_loaded(module), + function_exported?(module, :__protocol__, 1), + do: module + + # If this test fails, update: + # * lib/elixir/scripts/elixir_docs.ex + assert Enum.sort(builtin_protocols) == builtin_protocols() + end + + setup_all do + Application.put_env(:elixir, :ansi_enabled, false) + + on_exit(fn -> + Application.put_env(:elixir, :ansi_enabled, true) + end) + end + + describe "ExCk chunk" do + test "writes exports" do + files = %{ + "a.ex" => """ + defmodule A do + defp a, do: :ok + defmacrop b, do: a() + def c, do: b() + defmacro d, do: b() + @deprecated "oops" + def e, do: :ok + end + """, + "b.ex" => """ + defmodule B do + @callback f() :: :ok + end + """, + "c.ex" => """ + defmodule C do + @macrocallback g() :: :ok + end + """ + } + + modules = compile_modules(files) + + assert [ + {{:c, 0}, %{}}, + {{:e, 0}, %{deprecated: "oops", sig: {:infer, _, _}}} + ] = read_chunk(modules[A]).exports + + assert read_chunk(modules[B]).exports == [ + {{:behaviour_info, 1}, %{sig: :none}} + ] + + assert read_chunk(modules[C]).exports == [ + {{:behaviour_info, 1}, %{sig: :none}} + ] + end + + test "writes exports with inferred map types" do + files = %{ + "a.ex" => """ + defmodule A do + defstruct [:x, :y, :z] + + def struct_create_with_atom_keys(x) do + infer(y = %A{x: x}) + {x, y} + end + + def map_create_with_atom_keys(x) do + infer(%{__struct__: A, x: x, y: nil, z: nil}) + x + end + + def map_update_with_atom_keys(x) do + infer(%{x | y: nil}) + x + end + + def map_update_with_unknown_keys(x, key) do + infer(%{x | key => 123}) + x + end + + defp infer(%A{x: <<_::binary>>, y: nil}) do + :ok + end + end + """ + } + + modules = compile_modules(files) + exports = read_chunk(modules[A]).exports |> Map.new() + + return = fn name, arity -> + pair = {name, arity} + %{^pair => %{sig: {:infer, nil, [{_, return}]}}} = exports + return + end + + assert return.(:struct_create_with_atom_keys, 1) == + dynamic( + tuple([ + binary(), + closed_map( + __struct__: atom([A]), + x: binary(), + y: atom([nil]), + z: atom([nil]) + ) + ]) + ) + + assert return.(:map_create_with_atom_keys, 1) == dynamic(binary()) + + assert return.(:map_update_with_atom_keys, 1) == + dynamic( + closed_map( + __struct__: atom([A]), + x: binary(), + y: atom([nil]), + z: term() + ) + ) + + assert return.(:map_update_with_unknown_keys, 2) == + dynamic( + closed_map( + __struct__: atom([A]), + x: binary(), + y: atom([nil]), + z: term() + ) + ) + end + + test "writes exports with inferred function types" do + files = %{ + "a.ex" => """ + defmodule A do + def captured, do: &to_capture/1 + defp to_capture(<<"ok">>), do: :ok + defp to_capture(<<"error">>), do: :error + defp to_capture([_ | _]), do: :list + end + """ + } + + modules = compile_modules(files) + exports = read_chunk(modules[A]).exports |> Map.new() + + return = fn name, arity -> + pair = {name, arity} + %{^pair => %{sig: {:infer, nil, [{_, return}]}}} = exports + return + end + + assert return.(:captured, 0) + |> equal?( + fun_from_non_overlapping_clauses([ + {[binary()], dynamic(atom([:ok, :error]))}, + {[non_empty_list(term(), term())], dynamic(atom([:list]))} + ]) + ) + end + + test "writes exports for implementations" do + files = %{ + "pi.ex" => """ + defprotocol Itself do + @fallback_to_any true + def itself(data) + end + + defimpl Itself, + for: [ + Atom, + BitString, + Float, + Function, + Integer, + List, + Map, + Port, + PID, + Reference, + Tuple, + Any, + Range, + Unknown + ] do + def itself(data), do: data + def this_wont_warn(:ok), do: :ok + end + """ + } + + {modules, stderr} = with_io(:stderr, fn -> compile_modules(files) end) + + assert stderr =~ + "you are implementing a protocol for Unknown but said module is not available" + + refute stderr =~ "this_wont_warn" + + itself_arg = fn mod -> + {_, %{sig: {:infer, nil, [{[value], value}]}}} = + List.keyfind(read_chunk(modules[mod]).exports, {:itself, 1}, 0) + + value + end + + assert itself_arg.(Itself.Atom) == dynamic(atom()) + assert itself_arg.(Itself.BitString) == dynamic(binary()) + assert itself_arg.(Itself.Float) == dynamic(float()) + assert itself_arg.(Itself.Function) == dynamic(fun()) + assert itself_arg.(Itself.Integer) == dynamic(integer()) + + assert itself_arg.(Itself.List) == + dynamic(union(empty_list(), non_empty_list(term(), term()))) + + assert itself_arg.(Itself.Map) == dynamic(open_map(__struct__: if_set(negation(atom())))) + assert itself_arg.(Itself.Port) == dynamic(port()) + assert itself_arg.(Itself.PID) == dynamic(pid()) + assert itself_arg.(Itself.Reference) == dynamic(reference()) + assert itself_arg.(Itself.Tuple) == dynamic(tuple()) + assert itself_arg.(Itself.Any) == dynamic(term()) + + assert itself_arg.(Itself.Range) == + dynamic( + closed_map(__struct__: atom([Range]), first: term(), last: term(), step: term()) + ) + + assert itself_arg.(Itself.Unknown) == dynamic(open_map(__struct__: atom([Unknown]))) + end + end + + describe "type checking" do + test "inferred remote calls" do + files = %{ + "a.ex" => """ + defmodule A do + def fun(:ok), do: :doki + def fun(:error), do: :bad + end + """, + "b.ex" => """ + defmodule B do + def badarg do + A.fun(:unknown) + end + + def badmatch do + :doki = A.fun(:error) + end + end + """ + } + + warnings = [ + """ + warning: incompatible types given to A.fun/1: + + A.fun(:unknown) + """, + """ + but expected one of: + + #1 + dynamic(:ok) + + #2 + dynamic(:error) + """, + """ + warning: the following pattern will never match: + + :doki = A.fun(:error) + + because the right-hand side has type: + + dynamic(:bad) + """ + ] + + assert_warnings(files, warnings) + end + + test "mismatched locals" do + files = %{ + "a.ex" => """ + defmodule A do + def error(), do: private(raise "oops") + def public(x), do: private(List.to_tuple(x)) + defp private(:ok), do: nil + end + """ + } + + warnings = [ + """ + warning: incompatible types given to private/1: + + private(raise RuntimeError.exception("oops")) + + """, + "the 1st argument is empty (often represented as none())", + """ + typing violation found at: + │ + 2 │ def error(), do: private(raise "oops") + │ ~ + │ + └─ a.ex:2:20: A.error/0 + """, + """ + warning: incompatible types given to private/1: + + private(List.to_tuple(x)) + """, + """ + typing violation found at: + │ + 3 │ def public(x), do: private(List.to_tuple(x)) + │ ~ + │ + └─ a.ex:3:22: A.public/1 + """ + ] + + assert_warnings(files, warnings) + end + + test "unused private clauses" do + files = %{ + "a.ex" => """ + defmodule A do + def public(x) do + private(List.to_tuple(x)) + end + + defp private(nil), do: nil + defp private("foo"), do: "foo" + defp private({:ok, ok}), do: ok + defp private({:error, error}), do: error + defp private("bar"), do: "bar" + end + """ + } + + warnings = [ + """ + warning: this clause of defp private/1 is never used + │ + 6 │ defp private(nil), do: nil + │ ~ + │ + └─ a.ex:6:8: A.private/1 + """, + """ + warning: this clause of defp private/1 is never used + │ + 7 │ defp private("foo"), do: "foo" + │ ~ + │ + └─ a.ex:7:8: A.private/1 + """, + """ + warning: this clause of defp private/1 is never used + │ + 10 │ defp private("bar"), do: "bar" + │ ~ + │ + └─ a.ex:10:8: A.private/1 + """ + ] + + assert_warnings(files, warnings) + end + + test "unused overridable private clauses" do + files = %{ + "a.ex" => """ + defmodule A do + use B + def public(x), do: private(x) + defp private(x), do: super(List.to_tuple(x)) + end + """, + "b.ex" => """ + defmodule B do + defmacro __using__(_) do + quote do + defp private({:ok, ok}), do: ok + defp private(:error), do: :error + defoverridable private: 1 + end + end + end + """ + } + + assert_no_warnings(files) + end + + test "unused private clauses without warnings" do + files = %{ + "a.ex" => """ + defmodule A do + use B + + # Not all clauses are invoked, but do not warn since they are generated + def public1(x), do: generated(List.to_tuple(x)) + + # Avoid false positives caused by inference + def public2(x), do: (:ok = raising_private(x)) + + defp raising_private(true), do: :ok + defp raising_private(false), do: raise "oops" + end + """, + "b.ex" => """ + defmodule B do + defmacro __using__(_) do + quote generated: true do + defp generated({:ok, ok}), do: ok + defp generated(:error), do: :error + end + end + end + """ + } + + assert_no_warnings(files) + end + + test "mismatched implementation" do + files = %{ + "a.ex" => """ + defprotocol Itself do + def itself(data) + end + + defimpl Itself, for: Range do + def itself(nil), do: nil + def itself(range), do: range + end + """ + } + + warnings = [ + """ + warning: the 1st pattern in clause will never match: + + nil + + because it is expected to receive type: + + dynamic(%Range{}) + + hint: defimpl for Range requires its callbacks to match exclusively on %Range{} + + typing violation found at: + │ + 6 │ def itself(nil), do: nil + │ ~~~~~~~~~~~~~~~~~~~~~~~~ + │ + └─ a.ex:6: Itself.Range.itself/1 + """ + ] + + assert_warnings(files, warnings) + end + + @tag :require_ast + test "no implementation" do + files = %{ + "a.ex" => """ + defprotocol NoImplProtocol do + def callback(data) + end + """, + "b.ex" => """ + defmodule NoImplProtocol.Caller do + def run do + NoImplProtocol.callback(:hello) + end + end + """ + } + + warnings = [ + """ + warning: incompatible types given to NoImplProtocol.callback/1: + + NoImplProtocol.callback(:hello) + + given types: + + -:hello- + + but the NoImplProtocol protocol was not yet implemented for any type and therefore will always fail. + + This warning will disappear once you define a implementation. If the protocol is part of a library, you may define a dummy implementation for development/test. + + typing violation found at: + │ + 3 │ NoImplProtocol.callback(:hello) + │ ~ + │ + └─ b.ex:3:20: NoImplProtocol.Caller.run/0 + """ + ] + + assert_warnings(files, warnings, consolidate_protocols: true) + end + + @tag :require_ast + test "String.Chars protocol dispatch" do + files = %{ + "a.ex" => """ + defmodule FooBar do + def example1(_.._//_ = data), do: to_string(data) + def example2(_.._//_ = data), do: "hello \#{data} world" + end + """ + } + + warnings = [ + """ + warning: incompatible types given to String.Chars.to_string/1: + + to_string(data) + + given types: + + -dynamic(%Range{})- + + but expected a type that implements the String.Chars protocol. + You either passed the wrong value or you must: + + 1. convert the given value to a string explicitly + (use inspect/1 if you want to convert any data structure to a string) + 2. implement the String.Chars protocol + + where "data" was given the type: + + # type: dynamic(%Range{}) + # from: a.ex:2:24 + _.._//_ = data + + hint: the String.Chars protocol is implemented for the following types: + + dynamic( + %Date{} or %DateTime{} or %NaiveDateTime{} or %Time{} or %URI{} or %Version{} or + %Version.Requirement{} + ) or atom() or binary() or empty_list() or float() or integer() or non_empty_list(term(), term()) + """, + """ + warning: incompatible value given to string interpolation: + + data + + it has type: + + -dynamic(%Range{})- + + but expected a type that implements the String.Chars protocol. + You either passed the wrong value or you must: + + 1. convert the given value to a string explicitly + (use inspect/1 if you want to convert any data structure to a string) + 2. implement the String.Chars protocol + + where "data" was given the type: + + # type: dynamic(%Range{}) + # from: a.ex:3:24 + _.._//_ = data + """ + ] + + assert_warnings(files, warnings, consolidate_protocols: true) + end + + @tag :require_ast + test "Enumerable protocol dispatch" do + files = %{ + "a.ex" => """ + defmodule FooBar do + def example1(%Date{} = date), do: for(x <- date, do: x) + def example2(), do: for(i <- [1, 2, 3], into: Date.utc_today(), do: i * 2) + def example3(), do: for(i <- [1, 2, 3], into: 456, do: i * 2) + end + """ + } + + warnings = [ + """ + warning: incompatible value given to for-comprehension: + + x <- date + + it has type: + + -dynamic(%Date{})- + + but expected a type that implements the Enumerable protocol. + You either passed the wrong value or you must: + + 1. convert the given value to an Enumerable explicitly + 2. implement the Enumerable protocol + + where "date" was given the type: + + # type: dynamic(%Date{}) + # from: a.ex:2:24 + %Date{} = date + + hint: the Enumerable protocol is implemented for the following types: + + dynamic( + %Date.Range{} or %File.Stream{} or %GenEvent.Stream{} or %HashDict{} or %HashSet{} or + %IO.Stream{} or %MapSet{} or %Range{} or %Stream{} + ) or empty_list() or fun() or non_empty_list(term(), term()) or non_struct_map() + """, + """ + warning: incompatible value given to :into option in for-comprehension: + + into: Date.utc_today() + + it has type: + + -dynamic(%Date{})- + + but expected a type that implements the Collectable protocol. + You either passed the wrong value or you forgot to implement the protocol. + + hint: the Collectable protocol is implemented for the following types: + + dynamic(%File.Stream{} or %HashDict{} or %HashSet{} or %IO.Stream{} or %MapSet{}) or binary() or + empty_list() or non_empty_list(term(), term()) or non_struct_map() + """, + """ + warning: incompatible value given to :into option in for-comprehension: + + into: 456 + + it has type: + + -integer()- + + but expected a type that implements the Collectable protocol. + You either passed the wrong value or you forgot to implement the protocol. + + hint: the Collectable protocol is implemented for the following types: + + dynamic(%File.Stream{} or %HashDict{} or %HashSet{} or %IO.Stream{} or %MapSet{}) or binary() or + empty_list() or non_empty_list(term(), term()) or non_struct_map() + """ + ] + + assert_warnings(files, warnings, consolidate_protocols: true) + end + + test "incompatible default argument" do + files = %{ + "a.ex" => """ + defmodule A do + def ok(x = :ok \\\\ nil) do + x + end + end + """ + } + + warnings = [ + ~S""" + warning: incompatible types given as default arguments to ok/1: + + -nil- + + but expected one of: + + dynamic(:ok) + + typing violation found at: + │ + 2 │ def ok(x = :ok \\ nil) do + │ ~ + │ + └─ a.ex:2:18: A.ok/0 + """ + ] + + assert_warnings(files, warnings) + end + + test "returns diagnostics with source and file" do + files = %{ + "a.ex" => """ + defmodule A do + @file "generated.ex" + def fun(arg) do + :ok = List.to_tuple(arg) + end + end + """ + } + + {_modules, warnings} = with_compile_warnings(files) + + assert [ + %{ + message: "the following pattern will never match" <> _, + file: file, + source: source + } + ] = warnings.runtime_warnings + + assert String.ends_with?(source, "a.ex") + assert Path.type(source) == :absolute + assert String.ends_with?(file, "generated.ex") + assert Path.type(file) == :absolute + after + purge(A) + end + + @tag :require_ast + test "regressions" do + files = %{ + # do not emit false positives from defguard + "a.ex" => """ + defmodule A do + defguard is_non_nil_arity_function(fun, arity) + when arity != nil and is_function(fun, arity) + + def check(fun, args) do + is_non_nil_arity_function(fun, length(args)) + end + end + """, + # do not parse binary segments as variables + "b.ex" => """ + defmodule B do + def decode(byte) do + case byte do + enc when enc in [<<0x00>>, <<0x01>>] -> :ok + end + end + end + """, + # String.Chars protocol dispatch on improper lists + "c.ex" => """ + defmodule C do + def example, do: to_string([?a, ?b | "!"]) + end + """ + } + + assert_no_warnings(files, consolidate_protocols: true) + end + end + + describe "undefined warnings" do + test "handles Erlang modules" do + files = %{ + "a.ex" => """ + defmodule A do + def a, do: :not_a_module.no_module() + def b, do: :lists.no_func() + end + """ + } + + warnings = [ + ":not_a_module.no_module/0 is undefined (module :not_a_module is not available or is yet to be defined)", + "a.ex:2:28: A.a/0", + ":lists.no_func/0 is undefined or private", + "a.ex:3:21: A.b/0" + ] + + assert_warnings(files, warnings) + end + + test "handles built in functions" do + files = %{ + "a.ex" => """ + defmodule A do + def a, do: Kernel.module_info() + def b, do: Kernel.module_info(:functions) + def c, do: Kernel.__info__(:functions) + def d, do: GenServer.behaviour_info(:callbacks) + def e, do: Kernel.behaviour_info(:callbacks) + end + """ + } + + warnings = [ + "Kernel.behaviour_info/1 is undefined or private", + "a.ex:6:21: A.e/0" + ] + + assert_warnings(files, warnings) + end + + test "handles module body conditionals" do + files = %{ + "a.ex" => """ + defmodule A do + if function_exported?(List, :flatten, 1) do + List.flatten([1, 2, 3]) + else + List.old_flatten([1, 2, 3]) + end + + if function_exported?(List, :flatten, 1) do + def flatten(arg), do: List.flatten(arg) + else + def flatten(arg), do: List.old_flatten(arg) + end + + if function_exported?(List, :flatten, 1) do + def flatten2(arg), do: List.old_flatten(arg) + else + def flatten2(arg), do: List.flatten(arg) + end + end + """ + } + + warnings = [ + "List.old_flatten/1 is undefined or private. Did you mean:", + "* flatten/1", + "* flatten/2", + "a.ex:15:33: A.flatten2/1" + ] + + assert_warnings(files, warnings) + end + + test "reports missing functions" do + files = %{ + "a.ex" => """ + defmodule A do + def a, do: A.no_func() + def b, do: A.a() + + @file "external_source.ex" + def c, do: &A.no_func/1 + end + """ + } + + warnings = [ + "A.no_func/0 is undefined or private", + "a.ex:2:16: A.a/0", + "A.no_func/1 is undefined or private", + "external_source.ex:6:17: A.c/0" + ] + + assert_warnings(files, warnings) + end + + test "reports missing functions respecting arity" do + files = %{ + "a.ex" => """ + defmodule A do + def a, do: :ok + def b, do: A.a(1) + + @file "external_source.ex" + def c, do: A.b(1) + end + """ + } + + warnings = [ + "A.a/1 is undefined or private. Did you mean:", + "* a/0", + "a.ex:3:16: A.b/0", + "A.b/1 is undefined or private. Did you mean:", + "* b/0", + "external_source.ex:6:16: A.c/0" + ] + + assert_warnings(files, warnings) + end + + test "reports missing modules" do + files = %{ + "a.ex" => """ + defmodule A do + def a, do: D.no_module() + + @file "external_source.ex" + def c, do: E.no_module() + + def i, do: Io.puts "hello" + end + """ + } + + warnings = [ + "D.no_module/0 is undefined (module D is not available or is yet to be defined)", + "a.ex:2:16: A.a/0", + "E.no_module/0 is undefined (module E is not available or is yet to be defined)", + "external_source.ex:5:16: A.c/0", + "Io.puts/1 is undefined (module Io is not available or is yet to be defined)", + "a.ex:7:17: A.i/0" + ] + + assert_warnings(files, warnings) + end + + test "reports missing captures" do + files = %{ + "a.ex" => """ + defmodule A do + def a, do: &A.no_func/0 + + @file "external_source.ex" + def c, do: &A.no_func/1 + end + """ + } + + warnings = [ + "A.no_func/0 is undefined or private", + "a.ex:2:17: A.a/0", + "A.no_func/1 is undefined or private", + "external_source.ex:5:17: A.c/0" + ] + + assert_warnings(files, warnings) + end + + test "doesn't report missing functions at compile time" do + files = %{ + "a.ex" => """ + Enum.map([], fn _ -> BadReferencer.no_func4() end) + + if function_exported?(List, :flatten, 1) do + List.flatten([1, 2, 3]) + else + List.old_flatten([1, 2, 3]) + end + """ + } + + assert_no_warnings(files) + end + + test "handles multiple modules in one file" do + files = %{ + "a.ex" => """ + defmodule A do + def a, do: B.no_func() + def b, do: B.a() + end + """, + "b.ex" => """ + defmodule B do + def a, do: A.no_func() + def b, do: A.b() + end + """ + } + + warnings = [ + "B.no_func/0 is undefined or private", + "a.ex:2:16: A.a/0", + "A.no_func/0 is undefined or private", + "b.ex:2:16: B.a/0" + ] + + assert_warnings(files, warnings) + end + + test "groups multiple warnings in one file" do + files = %{ + "a.ex" => """ + defmodule A do + def a, do: A.no_func() + + @file "external_source.ex" + def b, do: A2.no_func() + + def c, do: A.no_func() + def d, do: A2.no_func() + end + """ + } + + warnings = [ + "A2.no_func/0 is undefined (module A2 is not available or is yet to be defined)", + "└─ a.ex:8:17: A.d/0", + "└─ external_source.ex:5:17: A.b/0", + "A.no_func/0 is undefined or private", + "└─ a.ex:2:16: A.a/0", + "└─ a.ex:7:16: A.c/0" + ] + + assert_warnings(files, warnings) + end + + test "hints exclude deprecated functions" do + files = %{ + "a.ex" => """ + defmodule A do + def to_charlist(a), do: a + + @deprecated "Use String.to_charlist/1 instead" + def to_char_list(a), do: a + + def c(a), do: A.to_list(a) + end + """ + } + + warnings = [ + "A.to_list/1 is undefined or private. Did you mean:", + "* to_charlist/1", + "a.ex:7:19: A.c/1" + ] + + assert_warnings(files, warnings) + end + + test "do not warn of module defined in local (runtime) context" do + files = %{ + "a.ex" => """ + defmodule A do + def a() do + defmodule B do + def b(), do: :ok + end + + B.b() + end + end + """ + } + + assert_no_warnings(files) + end + + test "warn of unrequired module" do + files = %{ + "ab.ex" => """ + defmodule A do + def a(), do: B.b() + end + + defmodule B do + defmacro b(), do: :ok + end + """ + } + + warnings = [ + "Be sure to require B if you intend to invoke this macro", + "ab.ex:2:18: A.a/0" + ] + + assert_warnings(files, warnings) + end + + test "excludes local no_warn_undefined" do + files = %{ + "a.ex" => """ + defmodule A do + @compile {:no_warn_undefined, [MissingModule, {MissingModule2, :func, 2}]} + @compile {:no_warn_undefined, {B, :func, 2}} + + def a, do: MissingModule.func(1) + def b, do: MissingModule2.func(1, 2) + def c, do: MissingModule2.func(1) + def d, do: MissingModule3.func(1, 2) + def e, do: B.func(1) + def f, do: B.func(1, 2) + def g, do: B.func(1, 2, 3) + end + """, + "b.ex" => """ + defmodule B do + def func(_), do: :ok + end + """ + } + + warnings = [ + "MissingModule2.func/1 is undefined (module MissingModule2 is not available or is yet to be defined)", + "a.ex:7:29: A.c/0", + "MissingModule3.func/2 is undefined (module MissingModule3 is not available or is yet to be defined)", + "a.ex:8:29: A.d/0", + "B.func/3 is undefined or private. Did you mean:", + "* func/1", + "a.ex:11:16: A.g/0" + ] + + assert_warnings(files, warnings) + end + + test "warn of external nested module" do + files = %{ + "a.ex" => """ + defmodule A.B do + def a, do: :ok + end + defmodule A do + alias A.B + def a, do: B.a() + def b, do: B.a(1) + def c, do: B.no_func() + end + """ + } + + warnings = [ + "A.B.a/1 is undefined or private. Did you mean:", + "* a/0", + " def b, do: B.a(1)", + "a.ex:7:16: A.b/0", + "A.B.no_func/0 is undefined or private", + "def c, do: B.no_func()", + "a.ex:8:16: A.c/0" + ] + + assert_warnings(files, warnings) + end + + test "warn of compile time context module defined before calls" do + files = %{ + "a.ex" => """ + defmodule A do + defmodule B do + def a, do: :ok + end + def a, do: B.a() + def b, do: B.a(1) + def c, do: B.no_func() + end + """ + } + + warnings = [ + "A.B.a/1 is undefined or private. Did you mean:", + "* a/0", + " def b, do: B.a(1)", + "a.ex:6:16: A.b/0", + "A.B.no_func/0 is undefined or private", + "def c, do: B.no_func()", + "a.ex:7:16: A.c/0" + ] + + assert_warnings(files, warnings) + end + + test "warn of compile time context module defined after calls and aliased" do + files = %{ + "a.ex" => """ + defmodule A do + alias A.B + def a, do: B.a() + def b, do: B.a(1) + def c, do: B.no_func() + defmodule B do + def a, do: :ok + end + end + """ + } + + warnings = [ + "A.B.a/1 is undefined or private. Did you mean:", + "* a/0", + " def b, do: B.a(1)", + "a.ex:4:16: A.b/0", + "A.B.no_func/0 is undefined or private", + "def c, do: B.no_func()", + "a.ex:5:16: A.c/0" + ] + + assert_warnings(files, warnings) + end + + test "excludes global no_warn_undefined" do + no_warn_undefined = Code.get_compiler_option(:no_warn_undefined) + + try do + Code.compiler_options( + no_warn_undefined: [MissingModule, {MissingModule2, :func, 2}, {B, :func, 2}] + ) + + files = %{ + "a.ex" => """ + defmodule A do + @compile {:no_warn_undefined, [MissingModule, {MissingModule2, :func, 2}]} + @compile {:no_warn_undefined, {B, :func, 2}} + + def a, do: MissingModule.func(1) + def b, do: MissingModule2.func(1, 2) + def c, do: MissingModule2.func(1) + def d, do: MissingModule3.func(1, 2) + def e, do: B.func(1) + def f, do: B.func(1, 2) + def g, do: B.func(1, 2, 3) + end + """, + "b.ex" => """ + defmodule B do + def func(_), do: :ok + end + """ + } + + warnings = [ + "MissingModule2.func/1 is undefined (module MissingModule2 is not available or is yet to be defined)", + "a.ex:7:29: A.c/0", + "MissingModule3.func/2 is undefined (module MissingModule3 is not available or is yet to be defined)", + "a.ex:8:29: A.d/0", + "B.func/3 is undefined or private. Did you mean:", + "* func/1", + "a.ex:11:16: A.g/0" + ] + + assert_warnings(files, warnings) + after + Code.compiler_options(no_warn_undefined: no_warn_undefined) + end + end + + test "global no_warn_undefined :all" do + no_warn_undefined = Code.get_compiler_option(:no_warn_undefined) + + try do + Code.compiler_options(no_warn_undefined: :all) + + files = %{ + "a.ex" => """ + defmodule A do + def a, do: MissingModule.func(1) + end + """ + } + + assert_no_warnings(files) + after + Code.compiler_options(no_warn_undefined: no_warn_undefined) + end + end + + test "global no_warn_undefined :all and local exclude" do + no_warn_undefined = Code.get_compiler_option(:no_warn_undefined) + + try do + Code.compiler_options(no_warn_undefined: :all) + + files = %{ + "a.ex" => """ + defmodule A do + @compile {:no_warn_undefined, MissingModule} + + def a, do: MissingModule.func(1) + def b, do: MissingModule2.func(1, 2) + end + """ + } + + assert_no_warnings(files) + after + Code.compiler_options(no_warn_undefined: no_warn_undefined) + end + end + end + + describe "after_verify" do + test "reports functions" do + files = %{ + "a.ex" => """ + defmodule A do + @after_verify __MODULE__ + + def __after_verify__(__MODULE__) do + IO.warn "from after_verify", [] + end + end + """ + } + + warning = [ + "warning: ", + "from after_verify" + ] + + assert_warnings(files, warning) + end + end + + describe "deprecated" do + test "reports functions" do + files = %{ + "a.ex" => """ + defmodule A do + @deprecated "oops" + def a, do: A.a() + end + """ + } + + warnings = [ + "A.a/0 is deprecated. oops", + "a.ex:3:16: A.a/0" + ] + + assert_warnings(files, warnings) + end + + test "reports imported functions" do + files = %{ + "a.ex" => """ + defmodule A do + @deprecated "oops" + def a, do: :ok + end + """, + "b.ex" => """ + defmodule B do + import A + def b, do: a() + end + """ + } + + warnings = [ + "A.a/0 is deprecated. oops", + "b.ex:3:14: B.b/0" + ] + + assert_warnings(files, warnings) + end + + test "reports structs" do + files = %{ + "a.ex" => """ + defmodule A do + @deprecated "oops" + defstruct [:x, :y] + def match(%A{}), do: :ok + def build(:ok), do: %A{} + end + """, + "b.ex" => """ + defmodule B do + def match(%A{}), do: :ok + def build(:ok), do: %A{} + end + """ + } + + warnings = [ + "A.__struct__/0 is deprecated. oops", + "└─ a.ex:4:13: A.match/1", + "└─ a.ex:5:23: A.build/1", + "A.__struct__/0 is deprecated. oops", + "└─ b.ex:2:13: B.match/1", + "└─ b.ex:3:23: B.build/1" + ] + + assert_warnings(files, warnings) + end + + test "reports module body" do + files = %{ + "a.ex" => """ + defmodule A do + @deprecated "oops" + def a, do: :ok + end + """, + "b.ex" => """ + defmodule B do + require A + A.a() + end + """ + } + + warnings = [ + "A.a/0 is deprecated. oops", + "b.ex:3:5: B (module)" + ] + + assert_warnings(files, warnings) + end + + test "reports macro" do + files = %{ + "a.ex" => """ + defmodule A do + @deprecated "oops" + defmacro a, do: :ok + end + """, + "b.ex" => """ + defmodule B do + require A + def b, do: A.a() + end + """ + } + + warnings = [ + "A.a/0 is deprecated. oops", + "b.ex:3:16: B.b/0" + ] + + assert_warnings(files, warnings) + end + + test "reports unquote functions" do + files = %{ + "a.ex" => """ + defmodule A do + @deprecated "oops" + def a, do: :ok + end + """, + "b.ex" => """ + defmodule B do + def b, do: unquote(&A.a/0) + end + """ + } + + warnings = [ + "A.a/0 is deprecated. oops", + "b.ex: B.b/0" + ] + + assert_warnings(files, warnings) + end + end + + defp assert_warnings(files, expected, opts \\ []) + + defp assert_warnings(files, expected, opts) when is_binary(expected) do + assert capture_compile_warnings(files, opts) == expected + end + + defp assert_warnings(files, expecteds, opts) when is_list(expecteds) do + output = capture_compile_warnings(files, opts) + + Enum.each(expecteds, fn expected -> + assert output =~ expected + end) + end + + defp assert_no_warnings(files, opts \\ []) do + assert capture_compile_warnings(files, opts) == "" + end + + defp capture_compile_warnings(files, opts) do + in_tmp(fn -> + paths = generate_files(files) + capture_io(:stderr, fn -> compile_to_path(paths, opts) end) + end) + end + + defp with_compile_warnings(files) do + in_tmp(fn -> + paths = generate_files(files) + with_io(:stderr, fn -> compile_to_path(paths, []) end) |> elem(0) + end) + end + + defp compile_modules(files) do + in_tmp(fn -> + paths = generate_files(files) + {modules, _warnings} = compile_to_path(paths, []) + + Map.new(modules, fn module -> + {^module, binary, _filename} = :code.get_object_code(module) + {module, binary} + end) + end) + end + + defp compile_to_path(paths, opts) do + if opts[:consolidate_protocols] do + Code.prepend_path(".") + + result = + compile_to_path_with_after_compile(paths, fn -> + if Keyword.get(opts, :consolidate_protocols, false) do + paths = [".", Application.app_dir(:elixir, "ebin")] + protocols = Protocol.extract_protocols(paths) + + for protocol <- protocols do + impls = Protocol.extract_impls(protocol, paths) + {:ok, binary} = Protocol.consolidate(protocol, impls) + File.write!(Atom.to_string(protocol) <> ".beam", binary) + purge(protocol) + end + end + end) + + Code.delete_path(".") + Enum.each(builtin_protocols(), &purge/1) + + result + else + compile_to_path_with_after_compile(paths, fn -> :ok end) + end + end + + defp compile_to_path_with_after_compile(paths, callback) do + {:ok, modules, warnings} = + Kernel.ParallelCompiler.compile_to_path(paths, ".", + return_diagnostics: true, + after_compile: callback + ) + + for module <- modules do + purge(module) + end + + {modules, warnings} + end + + defp generate_files(files) do + for {file, contents} <- files do + File.write!(file, contents) + file + end + end + + defp read_chunk(binary) do + assert {:ok, {_module, [{~c"ExCk", chunk}]}} = :beam_lib.chunks(binary, [~c"ExCk"]) + assert {:elixir_checker_v4, map} = :erlang.binary_to_term(chunk) + map + end + + defp purge(mod) do + :code.delete(mod) + :code.purge(mod) + end + + defp in_tmp(fun) do + path = PathHelpers.tmp_path("checker") + + File.rm_rf!(path) + File.mkdir_p!(path) + File.cd!(path, fun) + end +end diff --git a/lib/elixir/test/elixir/module/types/map_test.exs b/lib/elixir/test/elixir/module/types/map_test.exs new file mode 100644 index 00000000000..ef7f8778ddc --- /dev/null +++ b/lib/elixir/test/elixir/module/types/map_test.exs @@ -0,0 +1,181 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("type_helper.exs", __DIR__) + +defmodule Module.Types.MapTest do + # Tests for the Map module + use ExUnit.Case, async: true + + import TypeHelper + import Module.Types.Descr + defmacro domain_key(arg) when is_atom(arg), do: [arg] + + describe "Map.to_list/1" do + test "checking" do + assert typecheck!([x = %{}], Map.to_list(x)) == dynamic(list(tuple([term(), term()]))) + + assert typecheck!( + ( + x = %{} + Map.to_list(x) + ) + ) == empty_list() + + assert typecheck!( + ( + x = %{"c" => :three} + Map.to_list(x) + ) + ) == + list(tuple([binary(), atom([:three])])) + + assert typecheck!( + ( + x = %{a: 1, b: "two"} + Map.to_list(x) + ) + ) == + non_empty_list( + union(tuple([atom([:a]), integer()]), tuple([atom([:b]), binary()])) + ) + + assert typecheck!( + ( + x = %{"c" => :three, a: 1, b: "two"} + Map.to_list(x) + ) + ) == + non_empty_list( + tuple([atom([:a]), integer()]) + |> union(tuple([atom([:b]), binary()])) + |> union(tuple([binary(), atom([:three])])) + ) + end + + test "inference" do + assert typecheck!( + [x], + ( + _ = Map.to_list(x) + x + ) + ) == dynamic(open_map()) + end + + test "errors" do + assert typeerror!([x = []], Map.to_list(x)) =~ "incompatible types given to Map.to_list/1" + end + end + + describe "Map.keys/1" do + test "checking" do + assert typecheck!([x = %{}], Map.keys(x)) == dynamic(list(term())) + + assert typecheck!( + ( + x = %{} + Map.keys(x) + ) + ) == empty_list() + + assert typecheck!( + ( + x = %{"c" => :three} + Map.keys(x) + ) + ) == + list(binary()) + + assert typecheck!( + ( + x = %{a: 1, b: "two"} + Map.keys(x) + ) + ) == + non_empty_list(union(atom([:a]), atom([:b]))) + + assert typecheck!( + ( + x = %{"c" => :three, a: 1, b: "two"} + Map.keys(x) + ) + ) == + non_empty_list( + atom([:a]) + |> union(atom([:b])) + |> union(binary()) + ) + end + + test "inference" do + assert typecheck!( + [x], + ( + _ = Map.keys(x) + x + ) + ) == dynamic(open_map()) + end + + test "errors" do + assert typeerror!([x = []], Map.keys(x)) =~ "incompatible types given to Map.keys/1" + end + end + + describe "Map.values/1" do + test "checking" do + assert typecheck!([x = %{}], Map.values(x)) == dynamic(list(term())) + + assert typecheck!( + ( + x = %{} + Map.values(x) + ) + ) == empty_list() + + assert typecheck!( + ( + x = %{"c" => :three} + Map.values(x) + ) + ) == + list(atom([:three])) + + assert typecheck!( + ( + x = %{a: 1, b: "two"} + Map.values(x) + ) + ) == + non_empty_list(union(integer(), binary())) + + assert typecheck!( + ( + x = %{"c" => :three, a: 1, b: "two"} + Map.values(x) + ) + ) == + non_empty_list( + integer() + |> union(binary()) + |> union(atom([:three])) + ) + end + + test "inference" do + assert typecheck!( + [x], + ( + _ = Map.values(x) + x + ) + ) == dynamic(open_map()) + end + + test "errors" do + assert typeerror!([x = []], Map.values(x)) =~ "incompatible types given to Map.values/1" + end + end +end diff --git a/lib/elixir/test/elixir/module/types/pattern_test.exs b/lib/elixir/test/elixir/module/types/pattern_test.exs new file mode 100644 index 00000000000..626c4c91478 --- /dev/null +++ b/lib/elixir/test/elixir/module/types/pattern_test.exs @@ -0,0 +1,346 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team + +Code.require_file("type_helper.exs", __DIR__) + +defmodule Module.Types.PatternTest do + use ExUnit.Case, async: true + + import TypeHelper + import Module.Types.Descr + + describe "variables" do + test "captures variables from simple assignment in head" do + assert typecheck!([x = :foo], x) == dynamic(atom([:foo])) + assert typecheck!([:foo = x], x) == dynamic(atom([:foo])) + end + + test "captures variables from simple assignment in =" do + assert typecheck!( + ( + x = :foo + x + ) + ) == atom([:foo]) + end + + test "refines information across patterns" do + assert typecheck!([%y{}, %x{}, x = y, x = Point], y) == dynamic(atom([Point])) + end + + test "repeated refinements are ignored on reporting" do + assert typeerror!([{name, arity}, arity = 123], hd(Atom.to_charlist(name))) |> strip_ansi() == + ~l""" + incompatible types given to Kernel.hd/1: + + hd(Atom.to_charlist(name)) + + given types: + + empty_list() or non_empty_list(integer()) + + but expected one of: + + non_empty_list(term(), term()) + + where "name" was given the type: + + # type: dynamic() + # from: types_test.ex + {name, arity} + """ + end + + test "errors on conflicting refinements" do + assert typeerror!([a = b, a = :foo, b = :bar], {a, b}) == + ~l""" + the following pattern will never match: + + a = b + + where "a" was given the type: + + # type: dynamic(:foo) + # from: types_test.ex:LINE-1 + a = :foo + + where "b" was given the type: + + # type: dynamic(:bar) + # from: types_test.ex:LINE-1 + b = :bar + """ + end + + test "can be accessed even if they don't match" do + assert typeerror!( + ( + # This will never match, info should not be "corrupted" + [info | _] = __ENV__.function + info + ) + ) =~ "the following pattern will never match" + end + + test "does not check underscore" do + assert typecheck!(_ = raise("oops")) == none() + end + end + + describe "=" do + test "precedence does not matter" do + uri_type = typecheck!([x = %URI{}], x) + + assert typecheck!( + ( + x = %URI{} = URI.new!("/") + x + ) + ) == uri_type + + assert typecheck!( + ( + %URI{} = x = URI.new!("/") + x + ) + ) == uri_type + end + + test "refines types" do + assert typecheck!( + [x, foo = :foo, bar = 123], + ( + {^foo, ^bar} = x + x + ) + ) == dynamic(tuple([atom([:foo]), integer()])) + end + + test "reports incompatible types" do + assert typeerror!([x = {:ok, _}], [_ | _] = x) == ~l""" + the following pattern will never match: + + [_ | _] = x + + because the right-hand side has type: + + dynamic({:ok, term()}) + + where "x" was given the type: + + # type: dynamic({:ok, term()}) + # from: types_test.ex:LINE + x = {:ok, _} + """ + end + end + + describe "structs" do + test "variable name" do + assert typecheck!([%x{}], x) == dynamic(atom()) + end + + test "variable name fields" do + assert typecheck!([x = %_{}], x.__struct__) == dynamic(atom()) + assert typecheck!([x = %_{}], x) == dynamic(open_map(__struct__: atom())) + + assert typecheck!([x = %m{}, m = Point], x) == + dynamic(open_map(__struct__: atom([Point]))) + + assert typecheck!([m = Point, x = %m{}], x) == + dynamic(open_map(__struct__: atom([Point]))) + + assert typeerror!([m = 123], %^m{} = %Point{}) == + ~l""" + expected an atom as struct name: + + %^m{} + + got type: + + integer() + + where "m" was given the type: + + # type: integer() + # from: types_test.ex:LINE-1 + m = 123 + """ + end + + test "fields in guards" do + assert typeerror!([x = %Point{}], x.foo_bar, :ok) == + ~l""" + unknown key .foo_bar in expression: + + x.foo_bar + + the given type does not have the given key: + + dynamic(%Point{x: term(), y: term(), z: term()}) + + where "x" was given the type: + + # type: dynamic(%Point{}) + # from: types_test.ex:LINE-1 + x = %Point{} + """ + end + end + + describe "maps" do + test "fields in patterns" do + assert typecheck!([x = %{foo: :bar}], x) == dynamic(open_map(foo: atom([:bar]))) + assert typecheck!([x = %{123 => 456}], x) == dynamic(open_map()) + assert typecheck!([x = %{123 => 456, foo: :bar}], x) == dynamic(open_map(foo: atom([:bar]))) + end + + test "fields in guards" do + assert typecheck!([x = %{foo: :bar}], x.bar, x) == dynamic(open_map(foo: atom([:bar]))) + end + end + + describe "tuples" do + test "in patterns" do + assert typecheck!([x = {:ok, 123}], x) == dynamic(tuple([atom([:ok]), integer()])) + assert typecheck!([{:x, y} = {x, :y}], {x, y}) == dynamic(tuple([atom([:x]), atom([:y])])) + end + end + + describe "lists" do + test "in patterns" do + assert typecheck!([x = [1, 2, 3]], x) == + dynamic(non_empty_list(integer())) + + assert typecheck!([x = [1, 2, 3 | y], y = :foo], x) == + dynamic(non_empty_list(integer(), atom([:foo]))) + + assert typecheck!([x = [1, 2, 3 | y], y = [1.0, 2.0, 3.0]], x) == + dynamic(non_empty_list(union(integer(), float()))) + + assert typecheck!([x = [:ok | z]], {x, z}) == + dynamic(tuple([non_empty_list(term(), term()), term()])) + + assert typecheck!([x = [y | z]], {x, y, z}) == + dynamic(tuple([non_empty_list(term(), term()), term(), term()])) + end + + test "in patterns through ++" do + assert typecheck!([x = [] ++ []], x) == dynamic(empty_list()) + + assert typecheck!([x = [] ++ y, y = :foo], x) == + dynamic(atom([:foo])) + + assert typecheck!([x = [1, 2, 3] ++ y, y = :foo], x) == + dynamic(non_empty_list(integer(), atom([:foo]))) + + assert typecheck!([x = [1, 2, 3] ++ y, y = [1.0, 2.0, 3.0]], x) == + dynamic(non_empty_list(union(integer(), float()))) + end + + test "with lists inside tuples inside lists" do + assert typecheck!([[node_1 = {[arg]}, node_2 = {[arg]}]], {node_1, node_2, arg}) + |> equal?( + dynamic( + tuple([ + tuple([non_empty_list(term())]), + tuple([non_empty_list(term())]), + term() + ]) + ) + ) + end + end + + describe "binaries" do + test "ok" do + assert typecheck!([<>], x) == integer() + assert typecheck!([<>], x) == float() + assert typecheck!([<>], x) == binary() + assert typecheck!([<>], x) == integer() + end + + test "nested" do + assert typecheck!([<<0, <>::binary>>], x) == binary() + end + + test "error" do + assert typeerror!([<>], x) == ~l""" + incompatible types assigned to "x": + + binary() !~ float() + + where "x" was given the types: + + # type: binary() + # from: types_test.ex:LINE + <> + + # type: float() + # from: types_test.ex:LINE + <<..., x::float>> + """ + + assert typeerror!([<>], x) == ~l""" + incompatible types assigned to "x": + + float() !~ integer() + + where "x" was given the types: + + # type: float() + # from: types_test.ex:LINE + <> + + # type: integer() + # from: types_test.ex:LINE + <<..., x>> + + #{hints(:inferred_bitstring_spec)} + """ + end + + test "pin inference" do + assert typecheck!( + [x, y], + ( + <<^x>> = y + x + ) + ) == dynamic(integer()) + end + + test "size ok" do + assert typecheck!([<>], :ok) == atom([:ok]) + end + + test "size error" do + assert typeerror!([<>], :ok) == + ~l""" + expected an integer in binary size: + + size(x) + + got type: + + float() + + where "x" was given the type: + + # type: float() + # from: types_test.ex:LINE-1 + <> + """ + end + + test "size pin inference" do + assert typecheck!( + [x, y], + ( + <<_::size(^x)>> = y + x + ) + ) == dynamic(integer()) + end + end +end diff --git a/lib/elixir/test/elixir/module/types/type_helper.exs b/lib/elixir/test/elixir/module/types/type_helper.exs new file mode 100644 index 00000000000..79c4997ebff --- /dev/null +++ b/lib/elixir/test/elixir/module/types/type_helper.exs @@ -0,0 +1,213 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../../test_helper.exs", __DIR__) + +defmodule Point do + defstruct [:x, :y, z: 0] +end + +defmodule TypeHelper do + alias Module.Types + alias Module.Types.{Pattern, Expr, Descr} + + @doc """ + Main helper for checking the given AST type checks without warnings. + """ + defmacro typedyn!(patterns \\ [], guards \\ true, body) do + quote do + unquote(typecheck(:dynamic, patterns, guards, body, __CALLER__)) + |> TypeHelper.__typecheck__!() + end + end + + @doc """ + Main helper for checking the given AST type checks without warnings. + """ + defmacro typecheck!(patterns \\ [], guards \\ true, body) do + quote do + unquote(typecheck(:static, patterns, guards, body, __CALLER__)) + |> TypeHelper.__typecheck__!() + end + end + + @doc """ + Main helper for checking the given AST type checks errors. + """ + defmacro typeerror!(patterns \\ [], guards \\ true, body) do + [patterns, guards, body] = prune_columns([patterns, guards, body]) + + quote do + unquote(typecheck(:static, patterns, guards, body, __CALLER__)) + |> TypeHelper.__typeerror__!() + end + end + + @doc """ + Main helper for checking the given AST type warns. + """ + defmacro typewarn!(patterns \\ [], guards \\ true, body) do + [patterns, guards, body] = prune_columns([patterns, guards, body]) + + quote do + unquote(typecheck(:static, patterns, guards, body, __CALLER__)) + |> TypeHelper.__typewarn__!() + end + end + + @doc """ + Main helper for checking the diagnostic of a given AST. + """ + defmacro typediag!(patterns \\ [], guards \\ true, body) do + quote do + unquote(typecheck(:static, patterns, guards, body, __CALLER__)) + |> TypeHelper.__typediag__!() + end + end + + @doc false + def __typecheck__!({type, %{warnings: []}}), do: type + + def __typecheck__!({_type, %{warnings: warnings, failed: false}}), + do: raise("type checking ok but with warnings: #{inspect(warnings)}") + + def __typecheck__!({_type, %{warnings: warnings, failed: true}}), + do: raise("type checking errored with warnings: #{inspect(warnings)}") + + @doc false + def __typeerror__!({_type, %{warnings: [{module, warning, _locs} | _], failed: true}}), + do: module.format_diagnostic(warning).message + + def __typeerror__!({_type, %{warnings: warnings, failed: false}}), + do: raise("type checking with warnings but expected error: #{inspect(warnings)}") + + def __typeerror__!({type, _}), + do: raise("type checking ok but expected error: #{Descr.to_quoted_string(type)}") + + @doc false + def __typediag__!({type, %{warnings: [_ | _] = warnings}}), + do: {type, for({module, arg, _} <- warnings, do: module.format_diagnostic(arg))} + + def __typediag__!({type, %{warnings: []}}), + do: raise("type checking without diagnostics: #{Descr.to_quoted_string(type)}") + + @doc false + def __typewarn__!({type, %{warnings: [{module, warning, _locs}], failed: false}}), + do: {type, module.format_diagnostic(warning).message} + + def __typewarn__!({type, %{warnings: []}}), + do: raise("type checking ok without warnings: #{Descr.to_quoted_string(type)}") + + def __typewarn__!({_type, %{warnings: warnings, failed: false}}), + do: raise("type checking ok but many warnings: #{inspect(warnings)}") + + def __typewarn__!({_type, %{warnings: warnings, failed: true}}), + do: raise("type checking errored with warnings: #{inspect(warnings)}") + + defp typecheck(mode, patterns, guards, body, env) do + {patterns, guards, body} = expand_and_unpack(patterns, guards, body, env) + + quote do + TypeHelper.__typecheck__( + unquote(mode), + unquote(Macro.escape(patterns)), + unquote(Macro.escape(guards)), + unquote(Macro.escape(body)) + ) + end + end + + def __typecheck__(mode, patterns, guards, body) do + stack = new_stack(mode) + expected = Enum.map(patterns, fn _ -> Descr.dynamic() end) + + {_trees, context} = + Pattern.of_head(patterns, guards, expected, :default, [], stack, new_context()) + + Expr.of_expr(body, Descr.term(), :ok, stack, context) + end + + defp expand_and_unpack(patterns, guards, body, env) do + fun = + quote do + fn unquote_splicing(patterns) when unquote(guards) -> unquote(body) end + end + + {ast, _, _} = :elixir_expand.expand(fun, :elixir_env.env_to_ex(env), env) + {:fn, _, [{:->, _, [[{:when, _, args}], body]}]} = ast + {patterns, guards} = Enum.split(args, -1) + {patterns, guards, body} + end + + defp new_stack(mode) do + cache = if mode == :infer, do: :none, else: Module.ParallelChecker.test_cache() + handler = fn _, fun_arity, _, _ -> raise "no local lookup for: #{inspect(fun_arity)}" end + Types.stack(mode, "types_test.ex", TypesTest, {:test, 0}, [], cache, handler) + end + + defp new_context() do + Types.context() + end + + @doc """ + Interpolate the given hints. + """ + def hints(hints) do + hints + |> List.wrap() + |> Module.Types.Helpers.format_hints() + |> IO.iodata_to_binary() + |> String.trim() + end + + @doc """ + A string-like sigil that replaces LINE references by actual line. + """ + defmacro sigil_l({:<<>>, meta, parts}, []) do + parts = + for part <- parts do + if is_binary(part) do + part + |> replace_line(__CALLER__.line) + |> :elixir_interpolation.unescape_string() + else + part + end + end + + {:<<>>, meta, parts} + end + + @strip_ansi [IO.ANSI.green(), IO.ANSI.red(), IO.ANSI.reset()] + + @doc """ + Strip ansi escapes from message. + """ + def strip_ansi(doc) do + String.replace(doc, @strip_ansi, "") + end + + defp replace_line(string, line) do + [head | rest] = String.split(string, "LINE") + + rest = + for part <- rest do + case part do + <> when num in ?0..?9 -> + [Integer.to_string(line - num + ?0), part] + + part -> + [Integer.to_string(line), part] + end + end + + IO.iodata_to_binary([head | rest]) + end + + defp prune_columns(ast) do + Macro.prewalk(ast, fn node -> + Macro.update_meta(node, &Keyword.delete(&1, :column)) + end) + end +end diff --git a/lib/elixir/test/elixir/module_test.exs b/lib/elixir/test/elixir/module_test.exs index a750e93becd..a167086ad4b 100644 --- a/lib/elixir/test/elixir/module_test.exs +++ b/lib/elixir/test/elixir/module_test.exs @@ -1,4 +1,8 @@ -Code.require_file "test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("test_helper.exs", __DIR__) defmodule ModuleTest.ToBeUsed do def value, do: 1 @@ -9,19 +13,24 @@ defmodule ModuleTest.ToBeUsed do Module.put_attribute(target, :before_compile, __MODULE__) Module.put_attribute(target, :after_compile, __MODULE__) Module.put_attribute(target, :before_compile, {__MODULE__, :callback}) - quote do: (def line, do: __ENV__.line) + quote(do: def(line, do: __ENV__.line)) end defmacro __before_compile__(env) do - quote do: (def before_compile, do: unquote(env.vars)) + quote(do: def(before_compile, do: unquote(Macro.Env.vars(env)))) end - defmacro __after_compile__(%Macro.Env{module: ModuleTest.ToUse, vars: []}, bin) when is_binary(bin) do - # IO.puts "HELLO" + defmacro __after_compile__(%Macro.Env{module: ModuleTest.ToUse} = env, bin) + when is_binary(bin) do + # Ensure module is not longer tracked as being loaded + false = __MODULE__ in :elixir_module.compiler_modules() + [] = Macro.Env.vars(env) + :ok end defmacro callback(env) do value = Module.get_attribute(env.module, :has_callback) + quote do def callback_value(true), do: unquote(value) end @@ -29,9 +38,11 @@ defmodule ModuleTest.ToBeUsed do end defmodule ModuleTest.ToUse do - 32 = __ENV__.line # Moving the next line around can make tests fail + # Moving the next line around can make tests fail + 42 = __ENV__.line var = 1 - var # Not available in callbacks + # Not available in callbacks + _ = var def callback_value(false), do: false use ModuleTest.ToBeUsed end @@ -39,77 +50,117 @@ end defmodule ModuleTest do use ExUnit.Case, async: true - Module.register_attribute __MODULE__, :register_example, accumulate: true, persist: true + doctest Module + + Module.register_attribute(__MODULE__, :register_unset_example, persist: true) + Module.register_attribute(__MODULE__, :register_empty_example, accumulate: true, persist: true) + Module.register_attribute(__MODULE__, :register_example, accumulate: true, persist: true) @register_example :it_works @register_example :still_works - contents = quote do: (def eval_quoted_info, do: {__MODULE__, __ENV__.file, __ENV__.line}) - Module.eval_quoted __MODULE__, contents, [], file: "sample.ex", line: 13 + defp purge(module) do + :code.purge(module) + :code.delete(module) + end defmacrop in_module(block) do quote do - defmodule Temp, unquote(block) - :code.purge(Temp) - :code.delete(Temp) + defmodule(Temp, unquote(block)) + purge(Temp) end end - ## Eval - - test :eval_quoted do - assert eval_quoted_info() == {ModuleTest, "sample.ex", 13} + test "module attributes returns value" do + in_module do + assert @return([:foo, :bar]) == :ok + _ = @return + end end - test :line_from_macro do - assert ModuleTest.ToUse.line == 36 - end + test "raises on write access attempts from __after_compile__/2" do + contents = + quote do + @after_compile __MODULE__ - ## Callbacks + def __after_compile__(%Macro.Env{module: module}, bin) when is_binary(bin) do + Module.put_attribute(module, :foo, 42) + end + end - test :compile_callback_hook do - assert ModuleTest.ToUse.callback_value(true) == true - assert ModuleTest.ToUse.callback_value(false) == false + assert_raise ArgumentError, + "could not call Module.put_attribute/3 because the module ModuleTest.Raise is in read-only mode (@after_compile)", + fn -> + Module.create(ModuleTest.Raise, contents, __ENV__) + end end - test :before_compile_callback_hook do - assert ModuleTest.ToUse.before_compile == [] + test "supports read access to module from __after_compile__/2" do + defmodule ModuleTest.NoRaise do + @after_compile __MODULE__ + @foo 42 + + def __after_compile__(%Macro.Env{module: module}, bin) when is_binary(bin) do + send(self(), Module.get_attribute(module, :foo)) + end + end + + assert_received 42 end - test :on_definition do - defmodule OnDefinition do - @on_definition ModuleTest + test "supports @after_verify for inlined modules" do + defmodule ModuleTest.AfterVerify do + @after_verify __MODULE__ - def hello(foo, bar) do - foo + bar + def __after_verify__(ModuleTest.AfterVerify) do + send(self(), ModuleTest.AfterVerify) end end - assert Process.get(ModuleTest.OnDefinition) == :called + assert_received ModuleTest.AfterVerify + end + + test "in memory modules are tagged as so" do + assert :code.which(__MODULE__) == ~c"" + end + + ## Callbacks + + test "retrieves line from use callsite" do + assert ModuleTest.ToUse.line() == 47 + end + + test "executes custom before_compile callback" do + assert ModuleTest.ToUse.callback_value(true) == true + assert ModuleTest.ToUse.callback_value(false) == false + end + + test "executes default before_compile callback" do + assert ModuleTest.ToUse.before_compile() == [] end def __on_definition__(env, kind, name, args, guards, expr) do - Process.put(env.module, :called) + Process.put(env.module, {args, guards, expr}) assert env.module == ModuleTest.OnDefinition assert kind == :def assert name == :hello - assert [{:foo, _, _}, {:bar, _ , _}] = args - assert [] = guards - assert {{:., _, [:erlang, :+]}, _, [{:foo, _, nil}, {:bar, _, nil}]} = expr + assert Module.defines?(env.module, {:hello, 2}) end - test :overridable_inside_before_compile do - defmodule OverridableWithBeforeCompile do - @before_compile ModuleTest - end - assert OverridableWithBeforeCompile.constant == 1 - end + test "executes on definition callback" do + defmodule OnDefinition do + @on_definition ModuleTest - test :alias_with_raw_atom do - defmodule :"Elixir.ModuleTest.RawModule" do - def hello, do: :world - end + def hello(foo, bar) + + assert {[{:foo, _, _}, {:bar, _, _}], [], nil} = Process.get(ModuleTest.OnDefinition) + + def hello(foo, bar) do + foo + bar + end - assert RawModule.hello == :world + assert {[{:foo, _, _}, {:bar, _, _}], [], [do: {:+, _, [{:foo, _, nil}, {:bar, _, nil}]}]} = + Process.get(ModuleTest.OnDefinition) + end end defmacro __before_compile__(_) do @@ -119,29 +170,43 @@ defmodule ModuleTest do end end - ## Attributes + test "may set overridable inside before_compile callback" do + defmodule OverridableWithBeforeCompile do + @before_compile ModuleTest + end - test :reserved_attributes do - assert List.keyfind(ExUnit.Server.__info__(:attributes), :behaviour, 0) == {:behaviour, [:gen_server]} + assert OverridableWithBeforeCompile.constant() == 1 end - test :registered_attributes do - assert [{:register_example, [:it_works]}, {:register_example, [:still_works]}] == - Enum.filter __MODULE__.__info__(:attributes), &match?({:register_example, _}, &1) + describe "__info__(:attributes)" do + test "reserved attributes" do + assert List.keyfind(ExUnit.Server.__info__(:attributes), :behaviour, 0) == + {:behaviour, [GenServer]} + end + + test "registered attributes" do + assert Enum.filter(__MODULE__.__info__(:attributes), &match?({:register_example, _}, &1)) == + [{:register_example, [:it_works]}, {:register_example, [:still_works]}] + end + + test "registered attributes with no values are not present" do + refute List.keyfind(__MODULE__.__info__(:attributes), :register_unset_example, 0) + refute List.keyfind(__MODULE__.__info__(:attributes), :register_empty_example, 0) + end end - @some_attribute [1] + @some_attribute [1] @other_attribute [3, 2, 1] - test :inside_function_attributes do - assert [1] = @some_attribute - assert [3, 2, 1] = @other_attribute + test "inside function attributes" do + assert @some_attribute == [1] + assert @other_attribute == [3, 2, 1] end ## Naming - test :concat do - assert Module.concat(Foo, Bar) == Foo.Bar + test "concat" do + assert Module.concat(Foo, Bar) == Foo.Bar assert Module.concat(Foo, :Bar) == Foo.Bar assert Module.concat(Foo, "Bar") == Foo.Bar assert Module.concat(Foo, Bar.Baz) == Foo.Bar.Baz @@ -149,96 +214,472 @@ defmodule ModuleTest do assert Module.concat(Bar, nil) == Elixir.Bar end - test :safe_concat do + test "safe concat" do assert Module.safe_concat(Foo, :Bar) == Foo.Bar + assert_raise ArgumentError, fn -> - Module.safe_concat SafeConcat, Doesnt.Exist + Module.safe_concat(SafeConcat, Doesnt.Exist) end end - test :split do + test "split" do module = Very.Long.Module.Name.And.Even.Longer assert Module.split(module) == ["Very", "Long", "Module", "Name", "And", "Even", "Longer"] assert Module.split("Elixir.Very.Long") == ["Very", "Long"] + + assert_raise ArgumentError, "expected an Elixir module, got: :just_an_atom", fn -> + Module.split(:just_an_atom) + end + + assert_raise ArgumentError, "expected an Elixir module, got: \"Foo\"", fn -> + Module.split("Foo") + end + assert Module.concat(Module.split(module)) == module end - test :__MODULE__ do + test "__MODULE__" do assert Code.eval_string("__MODULE__.Foo") |> elem(0) == Foo end + test "__ENV__.file" do + assert Path.basename(__ENV__.file) == "module_test.exs" + end + + @file "sample.ex" + test "@file sets __ENV__.file" do + assert __ENV__.file == Path.absname("sample.ex") + end + + test "@file raises when invalid" do + assert_raise ArgumentError, ~r"@file is a built-in module attribute", fn -> + defmodule BadFile do + @file :oops + def my_fun, do: :ok + end + end + end + ## Creation - test :defmodule do - assert match?({:module, Defmodule, binary, 3} when is_binary(binary), defmodule Defmodule do - 1 + 2 - end) + test "defmodule" do + result = + defmodule Defmodule do + 1 + 2 + end + + assert {:module, Defmodule, binary, 3} = result + assert is_binary(binary) + end + + test "defmodule with atom" do + result = + defmodule :root_defmodule do + :ok + end + + assert {:module, :root_defmodule, _, _} = result + end + + test "does not leak alias from atom" do + defmodule :"Elixir.ModuleTest.RawModule" do + def hello, do: :world + end + + refute __ENV__.aliases[Elixir.ModuleTest] + refute __ENV__.aliases[Elixir.RawModule] + assert ModuleTest.RawModule.hello() == :world + end + + test "does not leak alias from non-atom alias" do + defmodule __MODULE__.NonAtomAlias do + def hello, do: :world + end + + refute __ENV__.aliases[Elixir.ModuleTest] + refute __ENV__.aliases[Elixir.NonAtomAlias] + assert Elixir.ModuleTest.NonAtomAlias.hello() == :world end - test :defmodule_with_atom do - assert match?({:module, :root_defmodule, _, _}, defmodule :root_defmodule do - :ok - end) + test "does not leak alias from Elixir root alias" do + defmodule Elixir.ModuleTest.ElixirRootAlias do + def hello, do: :world + end + + refute __ENV__.aliases[Elixir.ModuleTest] + refute __ENV__.aliases[Elixir.ElixirRootAlias] + assert Elixir.ModuleTest.ElixirRootAlias.hello() == :world + end + + test "does not warn on captured underscored vars" do + _unused = 123 + + defmodule __MODULE__.NoVarWarning do + end end - test :create do + @compile {:no_warn_undefined, ModuleCreateSample} + + test "create" do contents = quote do def world, do: true end - {:module, ModuleCreateSample, _, _} = - Module.create(ModuleCreateSample, contents, __ENV__) - assert ModuleCreateSample.world + + {:module, ModuleCreateSample, _, _} = Module.create(ModuleCreateSample, contents, __ENV__) + assert ModuleCreateSample.world() end - test :create_with_elixir_as_a_name do + test "create with a reserved module name" do contents = quote do def world, do: true end - assert_raise CompileError, fn -> - {:module, Elixir, _, _} = + + assert_raise CompileError, ~r/cannot compile module Elixir/, fn -> + Code.with_diagnostics(fn -> Module.create(Elixir, contents, __ENV__) + end) end end - test :no_function_in_module_body do + @compile {:no_warn_undefined, ModuleTracersSample} + + test "create with propagated tracers" do + contents = + quote do + def world, do: true + end + + env = %{__ENV__ | tracers: [:invalid]} + {:module, ModuleTracersSample, _, _} = Module.create(ModuleTracersSample, contents, env) + assert ModuleTracersSample.world() + end + + @compile {:no_warn_undefined, ModuleHygiene} + + test "create with aliases/var hygiene" do + contents = + quote do + alias List, as: L + + def test do + L.flatten([1, [2], 3]) + end + end + + Module.create(ModuleHygiene, contents, __ENV__) + assert ModuleHygiene.test() == [1, 2, 3] + end + + test "ensure function clauses are sorted (to avoid non-determinism in module vsn)" do + {_, _, binary, _} = + defmodule Ordered do + def foo(:foo), do: :bar + def baz(:baz), do: :bat + end + + {:ok, {ModuleTest.Ordered, [abstract_code: {:raw_abstract_v1, abstract_code}]}} = + :beam_lib.chunks(binary, [:abstract_code]) + + # We need to traverse functions instead of using :exports as exports are sorted + funs = for {:function, _, name, arity, _} <- abstract_code, do: {name, arity} + assert funs == [__info__: 1, baz: 1, foo: 1] + end + + @compile {:no_warn_undefined, ModuleCreateGenerated} + + test "create with generated true does not emit warnings" do + contents = + quote generated: true do + def world, do: true + def world, do: false + end + + {:module, ModuleCreateGenerated, _, _} = + Module.create(ModuleCreateGenerated, contents, __ENV__) + + assert ModuleCreateGenerated.world() + end + + test "uses the debug_info chunk" do + {:module, ModuleCreateDebugInfo, binary, _} = + Module.create(ModuleCreateDebugInfo, :ok, __ENV__) + + {:ok, {_, [debug_info: {:debug_info_v1, backend, data}]}} = + :beam_lib.chunks(binary, [:debug_info]) + + {:ok, map} = backend.debug_info(:elixir_v1, ModuleCreateDebugInfo, data, []) + assert map.module == ModuleCreateDebugInfo + end + + test "uses the debug_info chunk when explicitly set to true" do + {:module, ModuleCreateDebugInfoTrue, binary, _} = + Module.create(ModuleCreateDebugInfoTrue, quote(do: @compile({:debug_info, true})), __ENV__) + + {:ok, {_, [debug_info: {:debug_info_v1, backend, data}]}} = + :beam_lib.chunks(binary, [:debug_info]) + + {:ok, map} = backend.debug_info(:elixir_v1, ModuleCreateDebugInfoTrue, data, []) + assert map.module == ModuleCreateDebugInfoTrue + end + + test "uses the debug_info chunk even if debug_info is set to false" do + {:module, ModuleCreateNoDebugInfo, binary, _} = + Module.create(ModuleCreateNoDebugInfo, quote(do: @compile({:debug_info, false})), __ENV__) + + {:ok, {_, [debug_info: {:debug_info_v1, backend, data}]}} = + :beam_lib.chunks(binary, [:debug_info]) + + assert backend.debug_info(:elixir_v1, ModuleCreateNoDebugInfo, data, []) == {:error, :missing} + end + + test "compiles to core" do + import PathHelpers + + write_beam( + defmodule ExampleModule do + end + ) + + {:ok, {ExampleModule, [{~c"Dbgi", dbgi}]}} = + ExampleModule |> :code.which() |> :beam_lib.chunks([~c"Dbgi"]) + + {:debug_info_v1, backend, data} = :erlang.binary_to_term(dbgi) + {:ok, core} = backend.debug_info(:core_v1, ExampleModule, data, []) + assert is_tuple(core) + end + + test "no function in module body" do in_module do assert __ENV__.function == nil end end + test "does not use ETS tables named after the module" do + in_module do + assert :ets.info(__MODULE__) == :undefined + end + end + ## Definitions - test :defines? do + test "defines?" do in_module do - refute Module.defines? __MODULE__, {:foo, 0} + refute Module.defines?(__MODULE__, {:foo, 0}) def foo(), do: bar() - assert Module.defines? __MODULE__, {:foo, 0} - assert Module.defines? __MODULE__, {:foo, 0}, :def + assert Module.defines?(__MODULE__, {:foo, 0}) + assert Module.defines?(__MODULE__, {:foo, 0}, :def) - refute Module.defines? __MODULE__, {:bar, 0}, :defp + refute Module.defines?(__MODULE__, {:bar, 0}, :defp) defp bar(), do: :ok - assert Module.defines? __MODULE__, {:bar, 0}, :defp + assert Module.defines?(__MODULE__, {:bar, 0}, :defp) - refute Module.defines? __MODULE__, {:baz, 0}, :defmacro + refute Module.defines?(__MODULE__, {:baz, 0}, :defmacro) defmacro baz(), do: :ok - assert Module.defines? __MODULE__, {:baz, 0}, :defmacro + assert Module.defines?(__MODULE__, {:baz, 0}, :defmacro) end end - test :definitions_in do + test "definitions in" do in_module do - def foo(1, 2, 3), do: 4 + defp bar(), do: :ok + def foo(1, 2, 3), do: bar() + + defmacrop macro_bar(), do: 4 + defmacro macro_foo(1, 2, 3), do: macro_bar() + + assert Module.definitions_in(__MODULE__) |> Enum.sort() == + [{:bar, 0}, {:foo, 3}, {:macro_bar, 0}, {:macro_foo, 3}] + + assert Module.definitions_in(__MODULE__, :def) == [foo: 3] + assert Module.definitions_in(__MODULE__, :defp) == [bar: 0] + assert Module.definitions_in(__MODULE__, :defmacro) == [macro_foo: 3] + assert Module.definitions_in(__MODULE__, :defmacrop) == [macro_bar: 0] + + defoverridable foo: 3 + + assert Module.definitions_in(__MODULE__) |> Enum.sort() == + [{:bar, 0}, {:macro_bar, 0}, {:macro_foo, 3}] + + assert Module.definitions_in(__MODULE__, :def) == [] + end + end + + test "get_definition/2 and delete_definition/2" do + in_module do + def foo(a, b), do: a + b + + assert {:v1, :def, def_meta, + [ + {clause_meta, [{:a, _, nil}, {:b, _, nil}], [], + {{:., _, [:erlang, :+]}, _, [{:a, _, nil}, {:b, _, nil}]}} + ]} = Module.get_definition(__MODULE__, {:foo, 2}) + + assert [line: _, column: _] = Keyword.take(def_meta, [:line, :column]) + assert [line: _, column: _] = Keyword.take(clause_meta, [:line, :column]) + assert {:v1, :def, _, []} = Module.get_definition(__MODULE__, {:foo, 2}, skip_clauses: true) - assert Module.definitions_in(__MODULE__) == [foo: 3] - assert Module.definitions_in(__MODULE__, :def) == [foo: 3] - assert Module.definitions_in(__MODULE__, :defp) == [] + assert Module.delete_definition(__MODULE__, {:foo, 2}) + assert Module.get_definition(__MODULE__, {:foo, 2}) == nil + refute Module.delete_definition(__MODULE__, {:foo, 2}) end end - test :function do - assert Module.function(:erlang, :atom_to_list, 1).(:hello) == 'hello' - assert is_function Module.function(This, :also_works, 0) + test "make_overridable/2 with invalid arguments" do + contents = + quote do + Module.make_overridable(__MODULE__, [{:foo, 256}]) + end + + message = + "each element in tuple list has to be a {function_name :: atom, arity :: 0..255} " <> + "tuple, got: {:foo, 256}" + + assert_raise ArgumentError, message, fn -> + Module.create(MakeOverridable, contents, __ENV__) + end + after + purge(MakeOverridable) + end + + test "raise when called with already compiled module" do + message = + "could not call Module.get_attribute/2 because the module Enum is already compiled. " <> + "Use the Module.__info__/1 callback or Code.fetch_docs/1 instead" + + assert_raise ArgumentError, message, fn -> + Module.get_attribute(Enum, :moduledoc) + end + end + + describe "get_attribute/3" do + test "returns a list when the attribute is marked as `accumulate: true`" do + in_module do + Module.register_attribute(__MODULE__, :value, accumulate: true) + assert Module.get_attribute(__MODULE__, :value) == [] + Module.put_attribute(__MODULE__, :value, 1) + assert Module.get_attribute(__MODULE__, :value) == [1] + Module.put_attribute(__MODULE__, :value, 2) + assert Module.get_attribute(__MODULE__, :value) == [2, 1] + end + end + + test "returns the value of the attribute if it exists" do + in_module do + Module.put_attribute(__MODULE__, :attribute, 1) + assert Module.get_attribute(__MODULE__, :attribute) == 1 + assert Module.get_attribute(__MODULE__, :attribute, :default) == 1 + Module.put_attribute(__MODULE__, :attribute, nil) + assert Module.get_attribute(__MODULE__, :attribute, :default) == nil + end + end + + test "returns the value of the attribute if persisted" do + in_module do + Module.register_attribute(__MODULE__, :value, persist: true) + assert Module.get_attribute(__MODULE__, :value, 123) == 123 + Module.put_attribute(__MODULE__, :value, 1) + assert Module.get_attribute(__MODULE__, :value) == 1 + Module.put_attribute(__MODULE__, :value, 2) + assert Module.get_attribute(__MODULE__, :value) == 2 + Module.delete_attribute(__MODULE__, :value) + assert Module.get_attribute(__MODULE__, :value, 123) == 123 + end + end + + test "returns the passed default if the attribute does not exist" do + in_module do + assert Module.get_attribute(__MODULE__, :attribute, :default) == :default + end + end + end + + describe "get_last_attribute/3" do + test "returns the last set value when the attribute is marked as `accumulate: true`" do + in_module do + Module.register_attribute(__MODULE__, :value, accumulate: true) + Module.put_attribute(__MODULE__, :value, 1) + assert Module.get_last_attribute(__MODULE__, :value) == 1 + Module.put_attribute(__MODULE__, :value, 2) + assert Module.get_last_attribute(__MODULE__, :value) == 2 + end + end + + test "returns the value of the non-accumulate attribute if it exists" do + in_module do + Module.put_attribute(__MODULE__, :attribute, 1) + assert Module.get_last_attribute(__MODULE__, :attribute) == 1 + Module.put_attribute(__MODULE__, :attribute, nil) + assert Module.get_last_attribute(__MODULE__, :attribute, :default) == nil + end + end + + test "returns the passed default if the accumulate attribute has not yet been set" do + in_module do + Module.register_attribute(__MODULE__, :value, accumulate: true) + assert Module.get_last_attribute(__MODULE__, :value) == nil + assert Module.get_last_attribute(__MODULE__, :value, :default) == :default + end + end + + test "returns the passed default if the non-accumulate attribute does not exist" do + in_module do + assert Module.get_last_attribute(__MODULE__, :value, :default) == :default + end + end + end + + describe "has_attribute?/2 and attributes_in/2" do + test "returns true when attribute has been defined" do + in_module do + @foo 1 + Module.register_attribute(__MODULE__, :bar, []) + Module.register_attribute(__MODULE__, :baz, accumulate: true) + Module.put_attribute(__MODULE__, :qux, 2) + + # silence warning + _ = @foo + + assert Module.has_attribute?(__MODULE__, :foo) + assert :foo in Module.attributes_in(__MODULE__) + assert Module.has_attribute?(__MODULE__, :bar) + assert :bar in Module.attributes_in(__MODULE__) + assert Module.has_attribute?(__MODULE__, :baz) + assert :baz in Module.attributes_in(__MODULE__) + assert Module.has_attribute?(__MODULE__, :qux) + assert :qux in Module.attributes_in(__MODULE__) + end + end + + test "returns false when attribute has not been defined" do + in_module do + refute Module.has_attribute?(__MODULE__, :foo) + end + end + + test "returns false when attribute has been deleted" do + in_module do + @foo 1 + Module.delete_attribute(__MODULE__, :foo) + + refute Module.has_attribute?(__MODULE__, :foo) + end + end + end + + test "@on_load" do + Process.register(self(), :on_load_test_process) + + defmodule OnLoadTest do + @on_load :on_load + + defp on_load do + send(:on_load_test_process, :on_loaded) + :ok + end + end + + assert_received :on_loaded end end diff --git a/lib/elixir/test/elixir/node_test.exs b/lib/elixir/test/elixir/node_test.exs deleted file mode 100644 index ff612e4a8cb..00000000000 --- a/lib/elixir/test/elixir/node_test.exs +++ /dev/null @@ -1,11 +0,0 @@ -Code.require_file "test_helper.exs", __DIR__ - -defmodule NodeTest do - use ExUnit.Case - - test "start/3 and stop/0" do - assert Node.stop == {:error, :not_found} - assert {:ok, _} = Node.start(:hello, :shortnames, 15000) - assert Node.stop() == :ok - end -end diff --git a/lib/elixir/test/elixir/option_parser_test.exs b/lib/elixir/test/elixir/option_parser_test.exs index e4f648f0d97..737a3832e26 100644 --- a/lib/elixir/test/elixir/option_parser_test.exs +++ b/lib/elixir/test/elixir/option_parser_test.exs @@ -1,293 +1,658 @@ -Code.require_file "test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("test_helper.exs", __DIR__) defmodule OptionParserTest do use ExUnit.Case, async: true - test "parses boolean option" do - assert OptionParser.parse(["--docs"]) == {[docs: true], [], []} - end + doctest OptionParser - test "parses alias boolean option as the alias key" do - assert OptionParser.parse(["-d"], aliases: [d: :docs]) - == {[docs: true], [], []} + test "parses --key value option" do + assert OptionParser.parse(["--source", "form_docs/", "other"], switches: [source: :string]) == + {[source: "form_docs/"], ["other"], []} end - test "parses more than one boolean option" do - assert OptionParser.parse(["--docs", "--compile"]) - == {[docs: true, compile: true], [], []} + test "parses --key=value option" do + assert OptionParser.parse(["--source=form_docs/", "other"], switches: [source: :string]) == + {[source: "form_docs/"], ["other"], []} end - test "parses more than one boolean options as the alias" do - assert OptionParser.parse(["-d", "--compile"], aliases: [d: :docs]) - == {[docs: true, compile: true], [], []} + test "parses overrides options by default" do + assert OptionParser.parse( + ["--require", "foo", "--require", "bar", "baz"], + switches: [require: :string] + ) == {[require: "bar"], ["baz"], []} end - test "parses --key value option" do - assert OptionParser.parse(["--source", "form_docs/"]) - == {[source: "form_docs/"], [], []} - end + test "parses multi-word option" do + config = [switches: [hello_world: :boolean]] + assert OptionParser.next(["--hello-world"], config) == {:ok, :hello_world, true, []} + assert OptionParser.next(["--no-hello-world"], config) == {:ok, :hello_world, false, []} - test "parses --key=value option" do - assert OptionParser.parse(["--source=form_docs/", "other"]) - == {[source: "form_docs/"], ["other"], []} - end + assert OptionParser.next(["--no-hello-world"], strict: []) == + {:undefined, "--no-hello-world", nil, []} - test "parses alias --key value option as the alias" do - assert OptionParser.parse(["-s", "from_docs/"], aliases: [s: :source]) - == {[source: "from_docs/"], [], []} - end + assert OptionParser.next(["--no-hello_world"], strict: []) == + {:undefined, "--no-hello_world", nil, []} - test "parses alias --key=value option as the alias" do - assert OptionParser.parse(["-s=from_docs/", "other"], aliases: [s: :source]) - == {[source: "from_docs/"], ["other"], []} - end + config = [strict: [hello_world: :boolean]] + assert OptionParser.next(["--hello-world"], config) == {:ok, :hello_world, true, []} + assert OptionParser.next(["--no-hello-world"], config) == {:ok, :hello_world, false, []} + assert OptionParser.next(["--hello_world"], config) == {:undefined, "--hello_world", nil, []} - test "does not interpret undefined options with value as boolean" do - assert OptionParser.parse(["--no-bool"]) - == {[no_bool: true], [], []} - assert OptionParser.parse(["--no-bool"], strict: []) - == {[], [], [{"--no-bool", nil}]} - assert OptionParser.parse(["--no-bool=...", "other"]) - == {[], ["other"], [{"--no-bool", "..."}]} + assert OptionParser.next(["--no-hello_world"], config) == + {:undefined, "--no-hello_world", nil, []} end - test "does not parse -- as an alias" do - assert OptionParser.parse(["--s=from_docs/"], aliases: [s: :source]) - == {[s: "from_docs/"], [], []} - end + test "parses more than one key-value pair options using switches" do + opts = [switches: [source: :string, docs: :string]] - test "does not parse - as a switch" do - assert OptionParser.parse(["-source=from_docs/"], aliases: [s: :source]) - == {[], [], [{"-source", "from_docs/"}]} - end + assert OptionParser.parse(["--source", "from_docs/", "--docs", "show"], opts) == + {[source: "from_docs/", docs: "show"], [], []} - test "parses configured booleans" do - assert OptionParser.parse(["--docs=false"], switches: [docs: :boolean]) - == {[docs: false], [], []} - assert OptionParser.parse(["--docs=true"], switches: [docs: :boolean]) - == {[docs: true], [], []} - assert OptionParser.parse(["--docs=other"], switches: [docs: :boolean]) - == {[], [], [{"--docs", "other"}]} - assert OptionParser.parse(["--docs="], switches: [docs: :boolean]) - == {[], [], [{"--docs", ""}]} - - assert OptionParser.parse(["--docs", "foo"], switches: [docs: :boolean]) - == {[docs: true], ["foo"], []} - assert OptionParser.parse(["--no-docs", "foo"], switches: [docs: :boolean]) - == {[docs: false], ["foo"], []} - assert OptionParser.parse(["--no-docs=foo", "bar"], switches: [docs: :boolean]) - == {[], ["bar"], [{"--no-docs", "foo"}]} - assert OptionParser.parse(["--no-docs=", "bar"], switches: [docs: :boolean]) - == {[], ["bar"], [{"--no-docs", ""}]} - end + assert OptionParser.parse(["--source", "from_docs/", "--doc", "show"], opts) == + {[source: "from_docs/", doc: "show"], [], []} + + assert OptionParser.parse(["--source", "from_docs/", "--doc=show"], opts) == + {[source: "from_docs/", doc: "show"], [], []} - test "does not set unparsed booleans" do - assert OptionParser.parse(["foo"], switches: [docs: :boolean]) - == {[], ["foo"], []} + assert OptionParser.parse(["--no-bool"], strict: []) == {[], [], [{"--no-bool", nil}]} end - test "keeps options on configured keep" do - args = ["--require", "foo", "--require", "bar", "baz"] - assert OptionParser.parse(args, switches: [require: :keep]) - == {[require: "foo", require: "bar"], ["baz"], []} + test "parses more than one key-value pair options using strict" do + opts = [strict: [source: :string, docs: :string]] - assert OptionParser.parse(["--require"], switches: [require: :keep]) - == {[], [], [{"--require", nil}]} - end + assert OptionParser.parse(["--source", "from_docs/", "--docs", "show"], opts) == + {[source: "from_docs/", docs: "show"], [], []} - test "parses configured strings" do - assert OptionParser.parse(["--value", "1", "foo"], switches: [value: :string]) - == {[value: "1"], ["foo"], []} - assert OptionParser.parse(["--value=1", "foo"], switches: [value: :string]) - == {[value: "1"], ["foo"], []} - assert OptionParser.parse(["--value"], switches: [value: :string]) - == {[], [], [{"--value", nil}]} - assert OptionParser.parse(["--no-value"], switches: [value: :string]) - == {[], [], [{"--no-value", nil}]} - end + assert OptionParser.parse(["--source", "from_docs/", "--doc", "show"], opts) == + {[source: "from_docs/"], ["show"], [{"--doc", nil}]} + + assert OptionParser.parse(["--source", "from_docs/", "--doc=show"], opts) == + {[source: "from_docs/"], [], [{"--doc", nil}]} - test "parses configured integers" do - assert OptionParser.parse(["--value", "1", "foo"], switches: [value: :integer]) - == {[value: 1], ["foo"], []} - assert OptionParser.parse(["--value=1", "foo"], switches: [value: :integer]) - == {[value: 1], ["foo"], []} - assert OptionParser.parse(["--value", "WAT", "foo"], switches: [value: :integer]) - == {[], ["foo"], [{"--value", "WAT"}]} + assert OptionParser.parse(["--no-bool"], strict: []) == {[], [], [{"--no-bool", nil}]} end - test "parses configured integers with keep" do - args = ["--value", "1", "--value", "2", "foo"] - assert OptionParser.parse(args, switches: [value: [:integer, :keep]]) - == {[value: 1, value: 2], ["foo"], []} + test "collects multiple invalid options" do + argv = ["--bad", "opt", "foo", "-o", "bad", "bar"] + + assert OptionParser.parse(argv, switches: [bad: :integer]) == + {[], ["foo", "bar"], [{"--bad", "opt"}]} + end + + test "parse/2 raises when using both options: switches and strict" do + assert_raise ArgumentError, ":switches and :strict cannot be given together", fn -> + OptionParser.parse(["--elixir"], switches: [ex: :string], strict: [elixir: :string]) + end + end + + test "parse/2 raises an exception on invalid switch types/modifiers" do + assert_raise ArgumentError, "invalid switch types/modifiers: :bad", fn -> + OptionParser.parse(["--elixir"], switches: [ex: :bad]) + end + + assert_raise ArgumentError, "invalid switch types/modifiers: :bad, :bad_modifier", fn -> + OptionParser.parse(["--elixir"], switches: [ex: [:bad, :bad_modifier]]) + end + end + + test "parse!/2 raises an exception for an unknown option using strict" do + msg = + """ + 1 error found! + --doc-bar : Unknown option. Did you mean --docs-bar? + + Supported options: + --docs-bar STRING + --source STRING\ + """ + + assert_raise OptionParser.ParseError, msg, fn -> + argv = ["--source", "from_docs/", "--doc-bar", "show"] + OptionParser.parse!(argv, strict: [source: :string, docs_bar: :string]) + end + + assert_raise OptionParser.ParseError, + """ + 1 error found! + --foo : Unknown option + + Supported options: + --docs STRING + --source STRING\ + """, + fn -> + argv = ["--source", "from_docs/", "--foo", "show"] + OptionParser.parse!(argv, strict: [source: :string, docs: :string]) + end + end + + test "parse!/2 raises an exception for an unknown option using strict when it is only off by underscores" do + msg = + """ + 1 error found! + --docs_bar : Unknown option. Did you mean --docs-bar? + + Supported options: + --docs-bar STRING + --source STRING\ + """ + + assert_raise OptionParser.ParseError, msg, fn -> + argv = ["--source", "from_docs/", "--docs_bar", "show"] + OptionParser.parse!(argv, strict: [source: :string, docs_bar: :string]) + end + end + + test "parse!/2 raises an exception when an option is of the wrong type" do + assert_raise OptionParser.ParseError, + """ + 1 error found! + --bad : Expected type integer, got "opt" + + Supported options: + --bad INTEGER\ + """, + fn -> + argv = ["--bad", "opt", "foo", "-o", "bad", "bar"] + OptionParser.parse!(argv, switches: [bad: :integer]) + end + end + + test "parse!/2 lists all supported options and aliases" do + expected_suggestion = + """ + 1 error found! + --verbos : Unknown option. Did you mean --verbose? + + Supported options: + --count INTEGER (alias: -c) + --debug, --no-debug (alias: -d) + --files STRING (alias: -f) (may be given more than once) + --name STRING (alias: -n) + --verbose, --no-verbose (alias: -v)\ + """ + + assert_raise OptionParser.ParseError, expected_suggestion, fn -> + OptionParser.parse!(["--verbos"], + strict: [ + name: :string, + count: :integer, + verbose: :boolean, + debug: :boolean, + files: :keep + ], + aliases: [n: :name, c: :count, v: :verbose, d: :debug, f: :files] + ) + end + end + + test "parse_head!/2 raises an exception when an option is of the wrong type" do + message = + """ + 1 error found! + --number : Expected type integer, got "lib" + + Supported options: + --number INTEGER\ + """ + + assert_raise OptionParser.ParseError, message, fn -> + argv = ["--number", "lib", "test/enum_test.exs"] + OptionParser.parse_head!(argv, strict: [number: :integer]) + end + end + + describe "arguments" do + test "parses until --" do + assert OptionParser.parse( + ["--source", "foo", "--", "1", "2", "3"], + switches: [source: :string] + ) == {[source: "foo"], ["1", "2", "3"], []} + + assert OptionParser.parse_head( + ["--source", "foo", "--", "1", "2", "3"], + switches: [source: :string] + ) == {[source: "foo"], ["1", "2", "3"], []} + + assert OptionParser.parse( + ["--source", "foo", "bar", "--", "-x"], + switches: [source: :string] + ) == {[source: "foo"], ["bar", "-x"], []} + + assert OptionParser.parse_head( + ["--source", "foo", "bar", "--", "-x"], + switches: [source: :string] + ) == {[source: "foo"], ["bar", "--", "-x"], []} + end + + test "return separators" do + assert OptionParser.parse_head(["--", "foo"], + switches: [], + return_separator: true + ) == {[], ["--", "foo"], []} + + assert OptionParser.parse_head(["--no-halt", "--", "foo"], + switches: [halt: :boolean], + return_separator: true + ) == {[halt: false], ["--", "foo"], []} + + assert OptionParser.parse_head(["foo.exs", "--no-halt", "--", "foo"], + switches: [halt: :boolean], + return_separator: true + ) == {[], ["foo.exs", "--no-halt", "--", "foo"], []} + end + + test "parses - as argument" do + argv = ["--foo", "-", "-b", "-"] + opts = [strict: [foo: :boolean, boo: :string], aliases: [b: :boo]] + assert OptionParser.parse(argv, opts) == {[foo: true, boo: "-"], ["-"], []} + end + + test "parses until first non-option arguments" do + argv = ["--source", "from_docs/", "test/enum_test.exs", "--verbose"] + + assert OptionParser.parse_head(argv, switches: [source: :string]) == + {[source: "from_docs/"], ["test/enum_test.exs", "--verbose"], []} + end + end + + describe "aliases" do + test "supports boolean aliases" do + assert OptionParser.parse(["-d"], aliases: [d: :docs], switches: [docs: :boolean]) == + {[docs: true], [], []} + end + + test "supports non-boolean aliases" do + assert OptionParser.parse( + ["-s", "from_docs/"], + aliases: [s: :source], + switches: [source: :string] + ) == {[source: "from_docs/"], [], []} + end + + test "supports --key=value aliases" do + assert OptionParser.parse( + ["-s=from_docs/", "other"], + aliases: [s: :source], + switches: [source: :string] + ) == {[source: "from_docs/"], ["other"], []} + end + + test "parses -ab as -a -b" do + opts = [aliases: [a: :first, b: :second], switches: [second: :integer]] + assert OptionParser.parse(["-ab=1"], opts) == {[first: true, second: 1], [], []} + assert OptionParser.parse(["-ab", "1"], opts) == {[first: true, second: 1], [], []} + + opts = [aliases: [a: :first, b: :second], switches: [first: :boolean, second: :boolean]] + assert OptionParser.parse(["-ab"], opts) == {[first: true, second: true], [], []} + assert OptionParser.parse(["-ab3"], opts) == {[first: true], [], [{"-b", "3"}]} + assert OptionParser.parse(["-ab=bar"], opts) == {[first: true], [], [{"-b", "bar"}]} + assert OptionParser.parse(["-ab3=bar"], opts) == {[first: true], [], [{"-b", "3=bar"}]} + assert OptionParser.parse(["-3ab"], opts) == {[], ["-3ab"], []} + end + end + + describe "types" do + test "parses configured booleans" do + assert OptionParser.parse(["--docs=false"], switches: [docs: :boolean]) == + {[docs: false], [], []} + + assert OptionParser.parse(["--docs=true"], switches: [docs: :boolean]) == + {[docs: true], [], []} + + assert OptionParser.parse(["--docs=other"], switches: [docs: :boolean]) == + {[], [], [{"--docs", "other"}]} + + assert OptionParser.parse(["--docs="], switches: [docs: :boolean]) == + {[], [], [{"--docs", ""}]} + + assert OptionParser.parse(["--docs", "foo"], switches: [docs: :boolean]) == + {[docs: true], ["foo"], []} + + assert OptionParser.parse(["--no-docs", "foo"], switches: [docs: :boolean]) == + {[docs: false], ["foo"], []} + + assert OptionParser.parse(["--no-docs=foo", "bar"], switches: [docs: :boolean]) == + {[], ["bar"], [{"--no-docs", "foo"}]} + + assert OptionParser.parse(["--no-docs=", "bar"], switches: [docs: :boolean]) == + {[], ["bar"], [{"--no-docs", ""}]} + end - args = ["--value=1", "foo", "--value=2", "bar"] - assert OptionParser.parse(args, switches: [value: [:integer, :keep]]) - == {[value: 1, value: 2], ["foo", "bar"], []} - end + test "does not set unparsed booleans" do + assert OptionParser.parse(["foo"], switches: [docs: :boolean]) == {[], ["foo"], []} + end - test "parses configured floats" do - assert OptionParser.parse(["--value", "1.0", "foo"], switches: [value: :float]) - == {[value: 1.0], ["foo"], []} - assert OptionParser.parse(["--value=1.0", "foo"], switches: [value: :float]) - == {[value: 1.0], ["foo"], []} - assert OptionParser.parse(["--value", "WAT", "foo"], switches: [value: :float]) - == {[], ["foo"], [{"--value", "WAT"}]} - end + test "keeps options on configured keep" do + argv = ["--require", "foo", "--require", "bar", "baz"] - test "parses no switches as flags" do - assert OptionParser.parse(["--no-docs", "foo"]) - == {[no_docs: true], ["foo"], []} - end + assert OptionParser.parse(argv, switches: [require: :keep]) == + {[require: "foo", require: "bar"], ["baz"], []} - test "overrides options by default" do - assert OptionParser.parse(["--require", "foo", "--require", "bar", "baz"]) - == {[require: "bar"], ["baz"], []} - end + assert OptionParser.parse(["--require"], switches: [require: :keep]) == + {[], [], [{"--require", nil}]} + end - test "parses mixed options" do - args = ["--source", "from_docs/", "--compile", "-x"] - assert OptionParser.parse(args, aliases: [x: :x]) - == {[source: "from_docs/", compile: true, x: true], [], []} - end + test "parses configured strings" do + assert OptionParser.parse(["--value", "1", "foo"], switches: [value: :string]) == + {[value: "1"], ["foo"], []} - test "stops on first non option arguments" do - args = ["--source", "from_docs/", "test/enum_test.exs", "--verbose"] - assert OptionParser.parse_head(args) - == {[source: "from_docs/"], ["test/enum_test.exs", "--verbose"], []} - end + assert OptionParser.parse(["--value=1", "foo"], switches: [value: :string]) == + {[value: "1"], ["foo"], []} - test "stops on --" do - options = OptionParser.parse(["--source", "from_docs/", "--", "1", "2", "3"]) - assert options == {[source: "from_docs/"], ["1", "2", "3"], []} + assert OptionParser.parse(["--value"], switches: [value: :string]) == + {[], [], [{"--value", nil}]} - options = OptionParser.parse_head(["--source", "from_docs/", "--", "1", "2", "3"]) - assert options == {[source: "from_docs/"], ["1", "2", "3"], []} + assert OptionParser.parse(["--no-value"], switches: [value: :string]) == + {[no_value: true], [], []} + end - options = OptionParser.parse(["--no-dash", "foo", "bar", "--", "-x"]) - assert options == {[no_dash: true], ["foo", "bar", "-x"], []} + test "parses configured counters" do + assert OptionParser.parse(["--verbose"], switches: [verbose: :count]) == + {[verbose: 1], [], []} - options = OptionParser.parse_head(["--no-dash", "foo", "bar", "--", "-x"]) - assert options == {[no_dash: true], ["foo", "bar", "--", "-x"], []} - end + assert OptionParser.parse(["--verbose", "--verbose"], switches: [verbose: :count]) == + {[verbose: 2], [], []} - test "goes beyond the first non option arguments" do - args = ["--source", "from_docs/", "test/enum_test.exs", "--verbose"] - assert OptionParser.parse(args) - == {[source: "from_docs/", verbose: true], ["test/enum_test.exs"], []} - end + argv = ["--verbose", "-v", "-v", "--", "bar"] + opts = [aliases: [v: :verbose], strict: [verbose: :count]] + assert OptionParser.parse(argv, opts) == {[verbose: 3], ["bar"], []} + end + + test "parses configured integers" do + assert OptionParser.parse(["--value", "1", "foo"], switches: [value: :integer]) == + {[value: 1], ["foo"], []} + + assert OptionParser.parse(["--value=1", "foo"], switches: [value: :integer]) == + {[value: 1], ["foo"], []} + + assert OptionParser.parse(["--value", "WAT", "foo"], switches: [value: :integer]) == + {[], ["foo"], [{"--value", "WAT"}]} + end + + test "parses configured integers with keep" do + argv = ["--value", "1", "--value", "2", "foo"] + + assert OptionParser.parse(argv, switches: [value: [:integer, :keep]]) == + {[value: 1, value: 2], ["foo"], []} + + argv = ["--value=1", "foo", "--value=2", "bar"] + + assert OptionParser.parse(argv, switches: [value: [:integer, :keep]]) == + {[value: 1, value: 2], ["foo", "bar"], []} + end + + test "parses configured floats" do + assert OptionParser.parse(["--value", "1.0", "foo"], switches: [value: :float]) == + {[value: 1.0], ["foo"], []} + + assert OptionParser.parse(["--value=1.0", "foo"], switches: [value: :float]) == + {[value: 1.0], ["foo"], []} + + assert OptionParser.parse(["--value", "WAT", "foo"], switches: [value: :float]) == + {[], ["foo"], [{"--value", "WAT"}]} + end + + test "correctly handles negative integers" do + opts = [switches: [option: :integer], aliases: [o: :option]] + assert OptionParser.parse(["arg1", "-o43"], opts) == {[option: 43], ["arg1"], []} + assert OptionParser.parse(["arg1", "-o", "-43"], opts) == {[option: -43], ["arg1"], []} + assert OptionParser.parse(["arg1", "--option=-43"], opts) == {[option: -43], ["arg1"], []} + + assert OptionParser.parse(["arg1", "--option", "-43"], opts) == + {[option: -43], ["arg1"], []} + end + + test "correctly handles negative floating-point numbers" do + opts = [switches: [option: :float], aliases: [o: :option]] + assert OptionParser.parse(["arg1", "-o43.2"], opts) == {[option: 43.2], ["arg1"], []} + assert OptionParser.parse(["arg1", "-o", "-43.2"], opts) == {[option: -43.2], ["arg1"], []} + + assert OptionParser.parse(["arg1", "--option=-43.2"], switches: [option: :float]) == + {[option: -43.2], ["arg1"], []} - test "parses more than one key/value options" do - assert OptionParser.parse(["--source", "from_docs/", "--docs", "show"]) - == {[source: "from_docs/", docs: "show"], [], []} + assert OptionParser.parse(["arg1", "--option", "-43.2"], opts) == + {[option: -43.2], ["arg1"], []} + end + + test "parses configured regexes" do + assert {[pattern: regex], ["foo"], []} = + OptionParser.parse(["--pattern", "a.*b", "foo"], switches: [pattern: :regex]) + + assert Regex.match?(regex, "aXXXb") + refute Regex.match?(regex, "xyz") + + assert {[pattern: regex], ["foo"], []} = + OptionParser.parse(["--pattern=a.*b", "foo"], switches: [pattern: :regex]) + + assert Regex.match?(regex, "aXXXb") + + # Test Unicode support + assert {[pattern: regex], ["foo"], []} = + OptionParser.parse(["--pattern", "café.*résumé", "foo"], + switches: [pattern: :regex] + ) + + assert Regex.match?(regex, "café test résumé") + refute Regex.match?(regex, "ascii only") + + # Test invalid regex + assert OptionParser.parse(["--pattern", "[invalid", "foo"], switches: [pattern: :regex]) == + {[], ["foo"], [{"--pattern", "[invalid"}]} + end + + test "parses configured regexes with keep" do + argv = ["--pattern", "a.*", "--pattern", "b.*", "foo"] + + assert {[pattern: regex1, pattern: regex2], ["foo"], []} = + OptionParser.parse(argv, switches: [pattern: [:regex, :keep]]) + + assert Regex.match?(regex1, "aXXX") + assert Regex.match?(regex2, "bXXX") + refute Regex.match?(regex1, "bXXX") + refute Regex.match?(regex2, "aXXX") + + argv = ["--pattern=a.*", "foo", "--pattern=b.*", "bar"] + + assert {[pattern: regex1, pattern: regex2], ["foo", "bar"], []} = + OptionParser.parse(argv, switches: [pattern: [:regex, :keep]]) + + assert Regex.match?(regex1, "aXXX") + assert Regex.match?(regex2, "bXXX") + end + + test "correctly handles regex compilation errors" do + opts = [switches: [pattern: :regex]] + + # Invalid regex patterns should be treated as errors + assert OptionParser.parse(["--pattern", "*invalid"], opts) == + {[], [], [{"--pattern", "*invalid"}]} + + assert OptionParser.parse(["--pattern", "[unclosed"], opts) == + {[], [], [{"--pattern", "[unclosed"}]} + + assert OptionParser.parse(["--pattern", "(?invalid)"], opts) == + {[], [], [{"--pattern", "(?invalid)"}]} + end + + test "parse! raises an exception for invalid regex patterns" do + assert_raise OptionParser.ParseError, + ~r/Invalid regular expression \"\[invalid\": missing terminating \] for character class at position \d/, + fn -> + OptionParser.parse!(["--pattern", "[invalid"], switches: [pattern: :regex]) + end + + assert_raise OptionParser.ParseError, + ~r/Invalid regular expression \"\(\?invalid\)\": unrecognized character after \(\? or \(\?\- at position \d/, + fn -> + OptionParser.parse!(["--pattern", "(?invalid)"], switches: [pattern: :regex]) + end + end end - test "collects multiple invalid options" do - args = ["--bad", "opt", "foo", "-o", "bad", "bar"] - assert OptionParser.parse(args, switches: [bad: :integer]) - == {[], ["foo", "bar"], [{"--bad", "opt"}, {"-o", "bad"}]} + describe "next" do + test "with strict good options" do + config = [strict: [str: :string, int: :integer, bool: :boolean]] + assert OptionParser.next(["--str", "hello", "..."], config) == {:ok, :str, "hello", ["..."]} + assert OptionParser.next(["--int=13", "..."], config) == {:ok, :int, 13, ["..."]} + assert OptionParser.next(["--bool=false", "..."], config) == {:ok, :bool, false, ["..."]} + assert OptionParser.next(["--no-bool", "..."], config) == {:ok, :bool, false, ["..."]} + assert OptionParser.next(["--bool", "..."], config) == {:ok, :bool, true, ["..."]} + assert OptionParser.next(["..."], config) == {:error, ["..."]} + end + + test "with strict unknown options" do + config = [strict: [bool: :boolean]] + + assert OptionParser.next(["--str", "13", "..."], config) == + {:undefined, "--str", nil, ["13", "..."]} + + assert OptionParser.next(["--int=hello", "..."], config) == + {:undefined, "--int", "hello", ["..."]} + + assert OptionParser.next(["-no-bool=other", "..."], config) == + {:undefined, "-no-bool", "other", ["..."]} + end + + test "with strict bad type" do + config = [strict: [str: :string, int: :integer, bool: :boolean]] + assert OptionParser.next(["--str", "13", "..."], config) == {:ok, :str, "13", ["..."]} + + assert OptionParser.next(["--int=hello", "..."], config) == + {:invalid, "--int", "hello", ["..."]} + + assert OptionParser.next(["--int", "hello", "..."], config) == + {:invalid, "--int", "hello", ["..."]} + + assert OptionParser.next(["--bool=other", "..."], config) == + {:invalid, "--bool", "other", ["..."]} + end + + test "with strict missing value" do + config = [strict: [str: :string, int: :integer, bool: :boolean]] + assert OptionParser.next(["--str"], config) == {:invalid, "--str", nil, []} + assert OptionParser.next(["--int"], config) == {:invalid, "--int", nil, []} + assert OptionParser.next(["--bool=", "..."], config) == {:invalid, "--bool", "", ["..."]} + + assert OptionParser.next(["--no-bool=", "..."], config) == + {:invalid, "--no-bool", "", ["..."]} + end + end + + test "split" do + assert OptionParser.split(~S[]) == [] + assert OptionParser.split(~S[foo]) == ["foo"] + assert OptionParser.split(~S[foo bar]) == ["foo", "bar"] + assert OptionParser.split(~S[ foo bar ]) == ["foo", "bar"] + assert OptionParser.split(~S[foo\ bar]) == ["foo bar"] + assert OptionParser.split(~S[foo" bar"]) == ["foo bar"] + assert OptionParser.split(~S[foo\" bar\"]) == ["foo\"", "bar\""] + assert OptionParser.split(~S[foo "\ bar\""]) == ["foo", "\\ bar\""] + assert OptionParser.split(~S[foo '\"bar"\'\ ']) == ["foo", "\\\"bar\"'\\ "] + end + + describe "to_argv" do + test "converts options back to switches" do + assert OptionParser.to_argv(foo_bar: "baz") == ["--foo-bar", "baz"] + + assert OptionParser.to_argv(bool: true, bool: false, discarded: nil) == + ["--bool", "--no-bool"] + end + + test "handles :count switch type" do + original = ["--counter", "--counter"] + {opts, [], []} = OptionParser.parse(original, switches: [counter: :count]) + assert original == OptionParser.to_argv(opts, switches: [counter: :count]) + end end +end - test "parses more than one key/value options using strict" do - assert OptionParser.parse(["--source", "from_docs/", "--docs", "show"], - strict: [source: :string, docs: :string]) - == {[source: "from_docs/", docs: "show"], [], []} +defmodule OptionsParserDeprecationsTest do + use ExUnit.Case, async: true - assert OptionParser.parse(["--source", "from_docs/", "--doc", "show"], - strict: [source: :string, docs: :string]) - == {[source: "from_docs/"], ["show"], [{"--doc", nil}]} + def assert_deprecated(fun) do + assert ExUnit.CaptureIO.capture_io(:stderr, fun) =~ + "not passing the :switches or :strict option to OptionParser is deprecated" + end - assert OptionParser.parse(["--source", "from_docs/", "--doc=show"], - strict: [source: :string, docs: :string]) - == {[source: "from_docs/"], [], [{"--doc", nil}]} + test "parses boolean option" do + assert_deprecated(fn -> + assert OptionParser.parse(["--docs"]) == {[docs: true], [], []} + end) end - test "parses - as argument" do - assert OptionParser.parse(["-a", "-", "-", "-b", "-"], aliases: [b: :boo]) - == {[boo: "-"], ["-"], [{"-a", "-"}]} + test "parses more than one boolean option" do + assert_deprecated(fn -> + assert OptionParser.parse(["--docs", "--compile"]) == {[docs: true, compile: true], [], []} + end) + end - assert OptionParser.parse(["--foo", "-", "-b", "-"], strict: [foo: :boolean, boo: :string], aliases: [b: :boo]) - == {[foo: true, boo: "-"], ["-"], []} + test "parses more than one boolean options as the alias" do + assert_deprecated(fn -> + assert OptionParser.parse(["-d", "--compile"], aliases: [d: :docs]) == + {[docs: true, compile: true], [], []} + end) end - test "multi-word option" do - config = [switches: [hello_world: :boolean]] - assert OptionParser.next(["--hello-world"], config) - == {:ok, :hello_world, true, []} - assert OptionParser.next(["--no-hello-world"], config) - == {:ok, :hello_world, false, []} - - assert OptionParser.next(["--hello-world"], []) - == {:ok, :hello_world, true, []} - assert OptionParser.next(["--no-hello-world"], []) - == {:ok, :no_hello_world, true, []} - assert OptionParser.next(["--hello_world"], []) - == {:invalid, "--hello_world", nil, []} - assert OptionParser.next(["--no-hello_world"], []) - == {:invalid, "--no-hello_world", nil, []} - - assert OptionParser.next(["--no-hello-world"], strict: []) - == {:undefined, "--no-hello-world", nil, []} - assert OptionParser.next(["--no-hello_world"], strict: []) - == {:undefined, "--no-hello_world", nil, []} + test "parses --key value option" do + assert_deprecated(fn -> + assert OptionParser.parse(["--source", "form_docs/"]) == {[source: "form_docs/"], [], []} + end) + end - config = [strict: [hello_world: :boolean]] - assert OptionParser.next(["--hello-world"], config) - == {:ok, :hello_world, true, []} - assert OptionParser.next(["--no-hello-world"], config) - == {:ok, :hello_world, false, []} - assert OptionParser.next(["--hello_world"], config) - == {:undefined, "--hello_world", nil, []} - assert OptionParser.next(["--no-hello_world"], config) - == {:undefined, "--no-hello_world", nil, []} + test "does not interpret undefined options with value as boolean" do + assert_deprecated(fn -> + assert OptionParser.parse(["--no-bool"]) == {[no_bool: true], [], []} + end) + + assert_deprecated(fn -> + assert OptionParser.parse(["--no-bool=...", "other"]) == {[no_bool: "..."], ["other"], []} + end) end - test "next strict: good options" do - config = [strict: [str: :string, int: :integer, bool: :boolean]] - assert OptionParser.next(["--str", "hello", "..."], config) - == {:ok, :str, "hello", ["..."]} - assert OptionParser.next(["--int=13", "..."], config) - == {:ok, :int, 13, ["..."]} - assert OptionParser.next(["--bool=false", "..."], config) - == {:ok, :bool, false, ["..."]} - assert OptionParser.next(["--no-bool", "..."], config) - == {:ok, :bool, false, ["..."]} - assert OptionParser.next(["--bool", "..."], config) - == {:ok, :bool, true, ["..."]} - assert OptionParser.next(["..."], config) - == {:error, ["..."]} + test "parses -ab as -a -b" do + assert_deprecated(fn -> + assert OptionParser.parse(["-ab"], aliases: [a: :first, b: :second]) == + {[first: true, second: true], [], []} + end) end - test "next strict: unknown options" do - config = [strict: [bool: :boolean]] - assert OptionParser.next(["--str", "13", "..."], config) - == {:undefined, "--str", nil, ["13", "..."]} - assert OptionParser.next(["--int=hello", "..."], config) - == {:undefined, "--int", "hello", ["..."]} - assert OptionParser.next(["-no-bool=other", "..."], config) - == {:undefined, "-no-bool", "other", ["..."]} + test "parses mixed options" do + argv = ["--source", "from_docs/", "--compile", "-x"] + + assert_deprecated(fn -> + assert OptionParser.parse(argv, aliases: [x: :x]) == + {[source: "from_docs/", compile: true, x: true], [], []} + end) end - test "next strict: bad type" do - config = [strict: [str: :string, int: :integer, bool: :boolean]] - assert OptionParser.next(["--str", "13", "..."], config) - == {:ok, :str, "13", ["..."]} - assert OptionParser.next(["--int=hello", "..."], config) - == {:invalid, "--int", "hello", ["..."]} - assert OptionParser.next(["--int", "hello", "..."], config) - == {:invalid, "--int", "hello", ["..."]} - assert OptionParser.next(["--bool=other", "..."], config) - == {:invalid, "--bool", "other", ["..."]} + test "parses more than one key-value pair options" do + assert_deprecated(fn -> + assert OptionParser.parse(["--source", "from_docs/", "--docs", "show"]) == + {[source: "from_docs/", docs: "show"], [], []} + end) end - test "next strict: missing value" do - config = [strict: [str: :string, int: :integer, bool: :boolean]] - assert OptionParser.next(["--str"], config) - == {:invalid, "--str", nil, []} - assert OptionParser.next(["--int"], config) - == {:invalid, "--int", nil, []} - assert OptionParser.next(["--bool=", "..."], config) - == {:invalid, "--bool", "", ["..."]} - assert OptionParser.next(["--no-bool=", "..."], config) - == {:undefined, "--no-bool", "", ["..."]} + test "multi-word option" do + assert_deprecated(fn -> + assert OptionParser.next(["--hello-world"], []) == {:ok, :hello_world, true, []} + end) + + assert_deprecated(fn -> + assert OptionParser.next(["--no-hello-world"], []) == {:ok, :no_hello_world, true, []} + end) + + assert_deprecated(fn -> + assert OptionParser.next(["--hello_world"], []) == {:undefined, "--hello_world", nil, []} + end) + + assert_deprecated(fn -> + assert OptionParser.next(["--no-hello_world"], []) == + {:undefined, "--no-hello_world", nil, []} + end) end end diff --git a/lib/elixir/test/elixir/partition_supervisor_test.exs b/lib/elixir/test/elixir/partition_supervisor_test.exs new file mode 100644 index 00000000000..36c1d34b15a --- /dev/null +++ b/lib/elixir/test/elixir/partition_supervisor_test.exs @@ -0,0 +1,276 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team + +Code.require_file("test_helper.exs", __DIR__) + +defmodule PartitionSupervisorTest do + use ExUnit.Case, async: true + + describe "child_spec" do + test "uses the atom name as id" do + assert Supervisor.child_spec({PartitionSupervisor, name: Foo}, []) == %{ + id: Foo, + start: {PartitionSupervisor, :start_link, [[name: Foo]]}, + type: :supervisor + } + end + + test "uses the via value as id" do + via = {:via, Foo, {:bar, :baz}} + + assert Supervisor.child_spec({PartitionSupervisor, name: via}, []) == %{ + id: {:bar, :baz}, + start: {PartitionSupervisor, :start_link, [[name: via]]}, + type: :supervisor + } + end + end + + describe "start_link/1" do + test "on success with atom name", config do + {:ok, _} = + PartitionSupervisor.start_link( + child_spec: DynamicSupervisor, + name: config.test + ) + + assert PartitionSupervisor.partitions(config.test) == System.schedulers_online() + + refs = + for _ <- 1..100 do + ref = make_ref() + + DynamicSupervisor.start_child( + {:via, PartitionSupervisor, {config.test, ref}}, + {Agent, fn -> ref end} + ) + + ref + end + + agents = + for {_, pid, _, _} <- PartitionSupervisor.which_children(config.test), + {_, pid, _, _} <- DynamicSupervisor.which_children(pid), + do: Agent.get(pid, & &1) + + assert Enum.sort(refs) == Enum.sort(agents) + end + + test "on success with via name", config do + {:ok, _} = Registry.start_link(keys: :unique, name: PartitionRegistry) + + name = {:via, Registry, {PartitionRegistry, config.test}} + + {:ok, _} = PartitionSupervisor.start_link(child_spec: {Agent, fn -> :hello end}, name: name) + + assert PartitionSupervisor.partitions(name) == System.schedulers_online() + + assert Agent.get({:via, PartitionSupervisor, {name, 0}}, & &1) == :hello + end + + test "with_arguments", config do + {:ok, _} = + PartitionSupervisor.start_link( + child_spec: {Agent, fn -> raise "unused" end}, + with_arguments: fn [_fun], partition -> [fn -> partition end] end, + partitions: 3, + name: config.test + ) + + assert PartitionSupervisor.partitions(config.test) == 3 + + assert Agent.get({:via, PartitionSupervisor, {config.test, 0}}, & &1) == 0 + assert Agent.get({:via, PartitionSupervisor, {config.test, 1}}, & &1) == 1 + assert Agent.get({:via, PartitionSupervisor, {config.test, 2}}, & &1) == 2 + + assert Agent.get({:via, PartitionSupervisor, {config.test, 3}}, & &1) == 0 + assert Agent.get({:via, PartitionSupervisor, {config.test, -1}}, & &1) == 1 + end + + test "raises without name" do + assert_raise ArgumentError, + "the :name option must be given to PartitionSupervisor", + fn -> PartitionSupervisor.start_link(child_spec: DynamicSupervisor) end + end + + test "raises without child_spec" do + assert_raise ArgumentError, + "the :child_spec option must be given to PartitionSupervisor", + fn -> PartitionSupervisor.start_link(name: Foo) end + end + + test "raises on bad partitions" do + assert_raise ArgumentError, + "the :partitions option must be a positive integer, got: 0", + fn -> + PartitionSupervisor.start_link( + name: Foo, + child_spec: DynamicSupervisor, + partitions: 0 + ) + end + end + + test "raises on bad with_arguments" do + assert_raise ArgumentError, + ~r"the :with_arguments option must be a function that receives two arguments", + fn -> + PartitionSupervisor.start_link( + name: Foo, + child_spec: DynamicSupervisor, + with_arguments: 123 + ) + end + end + + test "raises with bad auto_shutdown" do + assert_raise ArgumentError, + "the :auto_shutdown option must be :never, got: :any_significant", + fn -> + PartitionSupervisor.start_link( + child_spec: DynamicSupervisor, + name: Foo, + auto_shutdown: :any_significant + ) + end + end + end + + describe "stop/1" do + test "is synchronous", config do + {:ok, _} = + PartitionSupervisor.start_link( + child_spec: {Agent, fn -> %{} end}, + name: config.test + ) + + assert PartitionSupervisor.stop(config.test) == :ok + assert Process.whereis(config.test) == nil + end + end + + describe "partitions/1" do + test "raises noproc for unknown atom partition supervisor" do + assert {:noproc, _} = catch_exit(PartitionSupervisor.partitions(:unknown)) + end + + test "raises noproc for unknown via partition supervisor", config do + {:ok, _} = Registry.start_link(keys: :unique, name: config.test) + via = {:via, Registry, {config.test, :unknown}} + assert {:noproc, _} = catch_exit(PartitionSupervisor.partitions(via)) + end + end + + describe "resize!/1" do + test "resizes the number of children", config do + {:ok, _} = + PartitionSupervisor.start_link( + child_spec: {Agent, fn -> %{} end}, + name: config.test, + partitions: 8 + ) + + for range <- [8..0//1, 0..8//1, Enum.shuffle(0..8)], i <- range do + PartitionSupervisor.resize!(config.test, i) + assert PartitionSupervisor.partitions(config.test) == i + + assert PartitionSupervisor.count_children(config.test) == + %{active: i, specs: 8, supervisors: 0, workers: 8} + + # Assert that we can still query across all range, + # but they are routed properly, as long as we have + # a single partition. + children = + for partition <- 0..7, i != 0, uniq: true do + GenServer.whereis({:via, PartitionSupervisor, {config.test, partition}}) + end + + assert length(children) == i + end + end + + test "raises on lookup after resizing to zero", config do + {:ok, _} = + PartitionSupervisor.start_link( + child_spec: {Agent, fn -> %{} end}, + name: config.test, + partitions: 8 + ) + + assert PartitionSupervisor.resize!(config.test, 0) == 8 + + assert_raise ArgumentError, ~r"has zero partitions", fn -> + GenServer.whereis({:via, PartitionSupervisor, {config.test, 0}}) + end + + assert PartitionSupervisor.resize!(config.test, 8) == 0 + end + + test "raises if trying to increase the number of partitions", config do + {:ok, _} = + PartitionSupervisor.start_link( + child_spec: {Agent, fn -> %{} end}, + name: config.test, + partitions: 8 + ) + + assert_raise ArgumentError, + "the number of partitions to resize to must be a number between 0 and 8, got: 9", + fn -> PartitionSupervisor.resize!(config.test, 9) end + end + end + + describe "which_children/1" do + test "returns all partitions", config do + {:ok, _} = + PartitionSupervisor.start_link( + child_spec: {Agent, fn -> %{} end}, + name: config.test + ) + + assert PartitionSupervisor.partitions(config.test) == System.schedulers_online() + + children = + config.test + |> PartitionSupervisor.which_children() + |> Enum.sort() + + for {child, partition} <- Enum.zip(children, 0..(System.schedulers_online() - 1)) do + via = {:via, PartitionSupervisor, {config.test, partition}} + assert child == {partition, GenServer.whereis(via), :worker, [Agent]} + end + end + end + + describe "count_children/1" do + test "with workers", config do + {:ok, _} = + PartitionSupervisor.start_link( + child_spec: {Agent, fn -> %{} end}, + name: config.test + ) + + partitions = System.schedulers_online() + + assert PartitionSupervisor.count_children(config.test) == + %{active: partitions, specs: partitions, supervisors: 0, workers: partitions} + end + + test "with supervisors", config do + {:ok, _} = + PartitionSupervisor.start_link( + child_spec: DynamicSupervisor, + name: config.test + ) + + partitions = System.schedulers_online() + + assert PartitionSupervisor.count_children(config.test) == + %{active: partitions, specs: partitions, supervisors: partitions, workers: 0} + end + + test "raises noproc for unknown partition supervisor" do + assert {:noproc, _} = catch_exit(PartitionSupervisor.count_children(:unknown)) + end + end +end diff --git a/lib/elixir/test/elixir/path_test.exs b/lib/elixir/test/elixir/path_test.exs index 4fa577c9fa8..a95aaea74fd 100644 --- a/lib/elixir/test/elixir/path_test.exs +++ b/lib/elixir/test/elixir/path_test.exs @@ -1,91 +1,257 @@ -Code.require_file "test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("test_helper.exs", __DIR__) defmodule PathTest do use ExUnit.Case, async: true - import PathHelpers + doctest Path + + if :file.native_name_encoding() == :utf8 do + @tag :tmp_dir + test "wildcard with UTF-8", config do + File.mkdir_p(Path.join(config.tmp_dir, "héllò")) - if :file.native_name_encoding == :utf8 do - test :wildcard_with_utf8 do - File.mkdir_p(tmp_path("héllò")) - assert Path.wildcard(tmp_path("héllò")) == [tmp_path("héllò")] + assert Path.wildcard(Path.join(config.tmp_dir, "héllò")) == + [Path.join(config.tmp_dir, "héllò")] after - File.rm_rf tmp_path("héllò") + File.rm_rf(Path.join(config.tmp_dir, "héllò")) end end - test :wildcard do - hello = tmp_path("wildcard/.hello") - world = tmp_path("wildcard/.hello/world") + @tag :tmp_dir + test "wildcard/2", config do + hello = Path.join(config.tmp_dir, "wildcard/.hello") + world = Path.join(config.tmp_dir, "wildcard/.hello/world") File.mkdir_p(world) - assert Path.wildcard(tmp_path("wildcard/*/*")) == [] - assert Path.wildcard(tmp_path("wildcard/**/*")) == [] - assert Path.wildcard(tmp_path("wildcard/?hello/world")) == [] + assert Path.wildcard(Path.join(config.tmp_dir, "wildcard/*/*")) == [] + assert Path.wildcard(Path.join(config.tmp_dir, "wildcard/**/*")) == [] + assert Path.wildcard(Path.join(config.tmp_dir, "wildcard/?hello/world")) == [] + + assert Path.wildcard(Path.join(config.tmp_dir, "wildcard/*/*"), match_dot: true) == + [world] + + assert Path.wildcard(Path.join(config.tmp_dir, "wildcard/**/*"), match_dot: true) == + [hello, world] + + assert Path.wildcard(Path.join(config.tmp_dir, "wildcard/?hello/world"), match_dot: true) == + [world] + after + File.rm_rf(Path.join(config.tmp_dir, "wildcard")) + end + + @tag :tmp_dir + test "wildcard/2 follows ..", config do + hello = Path.join(config.tmp_dir, "wildcard/hello") + world = Path.join(config.tmp_dir, "wildcard/world") + File.mkdir_p(hello) + File.touch(world) + + assert Path.wildcard(Path.join(config.tmp_dir, "wildcard/w*/../h*")) == [] - assert Path.wildcard(tmp_path("wildcard/*/*"), match_dot: true) == [world] - assert Path.wildcard(tmp_path("wildcard/**/*"), match_dot: true) == [hello, world] - assert Path.wildcard(tmp_path("wildcard/?hello/world"), match_dot: true) == [world] + assert Path.wildcard(Path.join(config.tmp_dir, "wildcard/h*/../w*")) == + [Path.join(config.tmp_dir, "wildcard/hello/../world")] after - File.rm_rf tmp_path("wildcard") + File.rm_rf(Path.join(config.tmp_dir, "wildcard")) end - if is_win? do - test :relative do - assert Path.relative("C:/usr/local/bin") == "usr/local/bin" + test "wildcard/2 raises on null byte" do + assert_raise ArgumentError, ~r/null byte/, fn -> Path.wildcard("foo\0bar") end + end + + describe "Windows" do + @describetag :windows + + test "absname/1" do + assert Path.absname("//host/path") == "//host/path" + assert Path.absname("\\\\host\\path") == "//host/path" + assert Path.absname("\\/host\\path") == "//host/path" + assert Path.absname("/\\host\\path") == "//host/path" + + assert Path.absname("c:/") == "c:/" + assert Path.absname("c:/host/path") == "c:/host/path" + + cwd = File.cwd!() + assert Path.absname(cwd |> String.split("/") |> hd()) == cwd + + <> = cwd + random = Enum.random(Enum.to_list(?c..?z) -- [letter]) + assert Path.absname(<>) == <> + end + + test "relative/1" do + assert Path.relative("C:/usr/local/bin") == "usr/local/bin" assert Path.relative("C:\\usr\\local\\bin") == "usr\\local\\bin" - assert Path.relative("C:usr\\local\\bin") == "usr\\local\\bin" + assert Path.relative("C:usr\\local\\bin") == "usr\\local\\bin" - assert Path.relative("/usr/local/bin") == "usr/local/bin" - assert Path.relative("usr/local/bin") == "usr/local/bin" + assert Path.relative("/usr/local/bin") == "usr/local/bin" + assert Path.relative("usr/local/bin") == "usr/local/bin" assert Path.relative("../usr/local/bin") == "../usr/local/bin" end - test :type do - assert Path.type("C:/usr/local/bin") == :absolute - assert Path.type('C:\\usr\\local\\bin') == :absolute - assert Path.type("C:usr\\local\\bin") == :volumerelative + test "relative_to/3" do + # should give same relative paths for both force true and false + for force <- [true, false] do + assert Path.relative_to("//usr/local/foo", "//usr/", force: force) == "local/foo" + + assert Path.relative_to("D:/usr/local/foo", "D:/usr/", force: force) == "local/foo" + assert Path.relative_to("D:/usr/local/foo", "d:/usr/", force: force) == "local/foo" + assert Path.relative_to("d:/usr/local/foo", "D:/usr/", force: force) == "local/foo" + assert Path.relative_to("D:/usr/local/foo", "d:/", force: force) == "usr/local/foo" + assert Path.relative_to("D:/usr/local/foo", "D:/", force: force) == "usr/local/foo" + + assert Path.relative_to("d:/usr/local/foo/..", "d:/usr/local", force: force) == "." + assert Path.relative_to("d:/usr/local/../foo", "d:/usr/foo", force: force) == "." + assert Path.relative_to("d:/usr/local/../foo/bar", "d:/usr/foo", force: force) == "bar" + assert Path.relative_to("d:/usr/local/../foo/./bar", "d:/usr/foo", force: force) == "bar" + + assert Path.relative_to("d:/usr/local/../foo/bar/..", "d:/usr/foo", force: force) == "." + assert Path.relative_to("d:/usr/local/foo/..", "d:/usr/local/..", force: force) == "local" + assert Path.relative_to("d:/usr/local/foo/..", "d:/usr/local/.", force: force) == "." + end + + # different results for force: true + assert Path.relative_to("d:/usr/local/../foo", "d:/usr/local") == "d:/usr/foo" + assert Path.relative_to("d:/usr/local/../foo", "d:/usr/local", force: true) == "../foo" + + assert Path.relative_to("d:/usr/local/../foo/../bar", "d:/usr/foo") == "d:/usr/bar" + assert Path.relative_to("d:/usr/local/../foo/../bar", "d:/usr/foo", force: true) == "../bar" + + # on different volumes with force: true it should return the original path + assert Path.relative_to("d:/usr/local", "c:/usr/local", force: true) == "d:/usr/local" + assert Path.relative_to("d:/usr/local", "c:/another/local", force: true) == "d:/usr/local" + end + + test "type/1" do + assert Path.type("C:/usr/local/bin") == :absolute + assert Path.type(~c"C:\\usr\\local\\bin") == :absolute + assert Path.type("C:usr\\local\\bin") == :volumerelative - assert Path.type("/usr/local/bin") == :volumerelative - assert Path.type('usr/local/bin') == :relative + assert Path.type("/usr/local/bin") == :volumerelative + assert Path.type(~c"usr/local/bin") == :relative assert Path.type("../usr/local/bin") == :relative + + assert Path.type("//host/path") == :absolute + assert Path.type("\\\\host\\path") == :absolute + assert Path.type("/\\host\\path") == :absolute + assert Path.type("\\/host\\path") == :absolute end - else - test :relative do - assert Path.relative("/usr/local/bin") == "usr/local/bin" - assert Path.relative("usr/local/bin") == "usr/local/bin" + + test "split/1" do + assert Path.split("C:\\foo\\bar") == ["c:/", "foo", "bar"] + assert Path.split("C:/foo/bar") == ["c:/", "foo", "bar"] + end + + test "safe_relative/1" do + assert Path.safe_relative("local/foo") == {:ok, "local/foo"} + assert Path.safe_relative("D:/usr/local/foo") == :error + assert Path.safe_relative("d:/usr/local/foo") == :error + assert Path.safe_relative("foo/../..") == :error + end + + test "safe_relative/2" do + assert Path.safe_relative("local/foo/bar", "local") == {:ok, "local/foo/bar"} + assert Path.safe_relative("foo/..", "local") == {:ok, ""} + assert Path.safe_relative("..", "local/foo") == :error + assert Path.safe_relative("d:/usr/local/foo", "D:/") == :error + assert Path.safe_relative("D:/usr/local/foo", "d:/") == :error + end + end + + describe "Unix" do + @describetag :unix + + test "relative/1" do + assert Path.relative("/usr/local/bin") == "usr/local/bin" + assert Path.relative("usr/local/bin") == "usr/local/bin" assert Path.relative("../usr/local/bin") == "../usr/local/bin" - assert Path.relative(['/usr', ?/, "local/bin"]) == "usr/local/bin" + assert Path.relative("/") == "." + assert Path.relative(~c"/") == "." + assert Path.relative([~c"/usr", ?/, "local/bin"]) == "usr/local/bin" end - test :type do - assert Path.type("/usr/local/bin") == :absolute - assert Path.type("usr/local/bin") == :relative + test "relative_to/3" do + # subpaths of cwd, should give the same result for both force true and false + for force <- [false, true] do + assert Path.relative_to("/usr/local/foo", "/usr/local", force: force) == "foo" + assert Path.relative_to("/usr/local/foo", "/", force: force) == "usr/local/foo" + assert Path.relative_to("/usr/local/foo", "/usr/local/foo", force: force) == "." + assert Path.relative_to("/usr/local/foo/", "/usr/local/foo", force: force) == "." + assert Path.relative_to("/usr/local/foo", "/usr/local/foo/", force: force) == "." + + assert Path.relative_to("/usr/local/foo/..", "/usr/local", force: force) == "." + assert Path.relative_to("/usr/local/../foo", "/usr/foo", force: force) == "." + assert Path.relative_to("/usr/local/../foo/bar", "/usr/foo", force: force) == "bar" + assert Path.relative_to("/usr/local/../foo/./bar", "/usr/foo", force: force) == "bar" + assert Path.relative_to("/usr/local/../foo/bar/..", "/usr/foo", force: force) == "." + + assert Path.relative_to("/usr/local/foo/..", "/usr/local/..", force: force) == "local" + assert Path.relative_to("/usr/local/foo/..", "/usr/local/.", force: force) == "." + end + + # With relative second argument + assert Path.relative_to("/usr/local/foo", "etc") == "/usr/local/foo" + assert Path.relative_to("/usr/local/foo", "etc", force: true) == "/usr/local/foo" + + # Different relative paths for force true/false + assert Path.relative_to("/usr/local/foo", "/etc") == "/usr/local/foo" + assert Path.relative_to("/usr/local/foo", "/etc", force: true) == "../usr/local/foo" + + assert Path.relative_to("/usr/local/../foo", "/usr/local") == "/usr/foo" + assert Path.relative_to("/usr/local/../foo", "/usr/local", force: true) == "../foo" + + assert Path.relative_to("/usr/local/../foo/../bar", "/usr/foo") == "/usr/bar" + assert Path.relative_to("/usr/local/../foo/../bar", "/usr/foo", force: true) == "../bar" + + # More tests with force: true + assert Path.relative_to("/etc", "/usr/local/foo", force: true) == "../../../etc" + assert Path.relative_to(~c"/usr/local/foo", "/etc", force: true) == "../usr/local/foo" + assert Path.relative_to("/usr/local", "/usr/local/foo", force: true) == ".." + assert Path.relative_to("/usr/local/..", "/usr/local", force: true) == ".." + + assert Path.relative_to("/usr/../etc/foo/../../bar", "/log/foo/../../usr/", force: true) == + "../bar" + end + + test "type/1" do + assert Path.type("/usr/local/bin") == :absolute + assert Path.type("usr/local/bin") == :relative assert Path.type("../usr/local/bin") == :relative - assert Path.type('/usr/local/bin') == :absolute - assert Path.type('usr/local/bin') == :relative - assert Path.type('../usr/local/bin') == :relative + assert Path.type(~c"/usr/local/bin") == :absolute + assert Path.type(~c"usr/local/bin") == :relative + assert Path.type(~c"../usr/local/bin") == :relative - assert Path.type(['/usr/', 'local/bin']) == :absolute - assert Path.type(['usr/', 'local/bin']) == :relative - assert Path.type(['../usr', '/local/bin']) == :relative + assert Path.type([~c"/usr/", ~c"local/bin"]) == :absolute + assert Path.type([~c"usr/", ~c"local/bin"]) == :relative + assert Path.type([~c"../usr", ~c"/local/bin"]) == :relative end end - test :relative_to_cwd do - assert Path.relative_to_cwd(__ENV__.file) == - Path.relative_to(__ENV__.file, System.cwd!) + test "relative_to_cwd/2" do + assert Path.relative_to_cwd(__ENV__.file) == Path.relative_to(__ENV__.file, File.cwd!()) + + assert Path.relative_to_cwd(to_charlist(__ENV__.file)) == + Path.relative_to(to_charlist(__ENV__.file), to_charlist(File.cwd!())) - assert Path.relative_to_cwd(to_char_list(__ENV__.file)) == - Path.relative_to(to_char_list(__ENV__.file), to_char_list(System.cwd!)) + assert Path.relative_to_cwd(Path.dirname(File.cwd!()), force: true) == ".." + + [slash | split_cwd] = Path.split(File.cwd!()) + relative_to_root = List.duplicate("..", length(split_cwd)) + + assert Path.relative_to_cwd(slash) == slash + assert Path.relative_to_cwd(slash, force: true) == Path.join(relative_to_root) end - test :absname do - assert (Path.absname("/") |> strip_drive_letter_if_windows) == "/" - assert (Path.absname("/foo") |> strip_drive_letter_if_windows) == "/foo" - assert (Path.absname("/foo/bar") |> strip_drive_letter_if_windows) == "/foo/bar" - assert (Path.absname("/foo/bar/") |> strip_drive_letter_if_windows) == "/foo/bar" - assert (Path.absname("/foo/bar/../bar") |> strip_drive_letter_if_windows) == "/foo/bar/../bar" + test "absname/1,2" do + assert Path.absname("/") |> strip_drive_letter_if_windows() == "/" + assert Path.absname("/foo") |> strip_drive_letter_if_windows() == "/foo" + assert Path.absname("/./foo") |> strip_drive_letter_if_windows() == "/foo" + assert Path.absname("/foo/bar") |> strip_drive_letter_if_windows() == "/foo/bar" + assert Path.absname("/foo/bar/") |> strip_drive_letter_if_windows() == "/foo/bar" + assert Path.absname("/foo/bar/../bar") |> strip_drive_letter_if_windows() == "/foo/bar/../bar" assert Path.absname("bar", "/foo") == "/foo/bar" assert Path.absname("bar/", "/foo") == "/foo/bar" @@ -95,82 +261,116 @@ defmodule PathTest do assert Path.absname(["bar/", ?., ?., ["/bar"]], "/foo") == "/foo/bar/../bar" end - test :expand_path_with_user_home do - home = System.user_home! + test "expand/1,2 with user home" do + home = System.user_home!() |> Path.absname() assert home == Path.expand("~") - assert home == Path.expand('~') - assert is_binary Path.expand("~/foo") - assert is_binary Path.expand('~/foo') + assert home == Path.expand(~c"~") + assert is_binary(Path.expand("~/foo")) + assert is_binary(Path.expand(~c"~/foo")) assert Path.expand("~/file") == Path.join(home, "file") assert Path.expand("~/file", "whatever") == Path.join(home, "file") - assert Path.expand("file", Path.expand("~")) == Path.expand("~/file") + assert Path.expand("file", Path.expand("~")) == Path.join(home, "file") assert Path.expand("file", "~") == Path.join(home, "file") + assert Path.expand("~file") == Path.join(File.cwd!(), "~file") end - test :expand_path do - assert (Path.expand("/") |> strip_drive_letter_if_windows) == "/" - assert (Path.expand("/foo") |> strip_drive_letter_if_windows) == "/foo" - assert (Path.expand("/foo/bar") |> strip_drive_letter_if_windows) == "/foo/bar" - assert (Path.expand("/foo/bar/") |> strip_drive_letter_if_windows) == "/foo/bar" - assert (Path.expand("/foo/bar/.") |> strip_drive_letter_if_windows)== "/foo/bar" - assert (Path.expand("/foo/bar/../bar") |> strip_drive_letter_if_windows) == "/foo/bar" + test "expand/1,2" do + assert Path.expand("/") |> strip_drive_letter_if_windows() == "/" + assert Path.expand("/foo/../..") |> strip_drive_letter_if_windows() == "/" + assert Path.expand("/foo") |> strip_drive_letter_if_windows() == "/foo" + assert Path.expand("/./foo") |> strip_drive_letter_if_windows() == "/foo" + assert Path.expand("/../foo") |> strip_drive_letter_if_windows() == "/foo" + assert Path.expand("/foo/bar") |> strip_drive_letter_if_windows() == "/foo/bar" + assert Path.expand("/foo/bar/") |> strip_drive_letter_if_windows() == "/foo/bar" + assert Path.expand("/foo/bar/.") |> strip_drive_letter_if_windows() == "/foo/bar" + assert Path.expand("/foo/bar/../bar") |> strip_drive_letter_if_windows() == "/foo/bar" + + assert Path.expand("bar", "/foo") |> strip_drive_letter_if_windows() == "/foo/bar" + assert Path.expand("bar/", "/foo") |> strip_drive_letter_if_windows() == "/foo/bar" + assert Path.expand("bar/.", "/foo") |> strip_drive_letter_if_windows() == "/foo/bar" + assert Path.expand("bar/../bar", "/foo") |> strip_drive_letter_if_windows() == "/foo/bar" - assert (Path.expand("bar", "/foo") |> strip_drive_letter_if_windows)== "/foo/bar" - assert (Path.expand("bar/", "/foo") |> strip_drive_letter_if_windows)== "/foo/bar" - assert (Path.expand("bar/.", "/foo") |> strip_drive_letter_if_windows)== "/foo/bar" - assert (Path.expand("bar/../bar", "/foo") |> strip_drive_letter_if_windows)== "/foo/bar" - assert (Path.expand("../bar/../bar", "/foo/../foo/../foo") |> strip_drive_letter_if_windows) == "/bar" + drive_letter = + Path.expand("../bar/../bar", "/foo/../foo/../foo") |> strip_drive_letter_if_windows() - assert (Path.expand(['..', ?/, "bar/../bar"], '/foo/../foo/../foo') |> - strip_drive_letter_if_windows) == "/bar" + assert drive_letter == "/bar" + + drive_letter = + Path.expand([~c"..", ?/, "bar/../bar"], ~c"/foo/../foo/../foo") + |> strip_drive_letter_if_windows() + + assert "/bar" == drive_letter + + assert Path.expand("/..") |> strip_drive_letter_if_windows() == "/" assert Path.expand("bar/../bar", "foo") == Path.expand("foo/bar") + end - assert (Path.expand("/..") |> strip_drive_letter_if_windows) == "/" + test "relative_to/3 (with relative paths)" do + # on cwd + assert Path.relative_to("foo", File.cwd!()) == "foo" + assert Path.relative_to("./foo", File.cwd!()) == "foo" + assert Path.relative_to("./foo/.", File.cwd!()) == "foo" + assert Path.relative_to("./foo/./bar/.", File.cwd!()) == "foo/bar" + assert Path.relative_to("../foo/./bar/.", File.cwd!()) == "../foo/bar" + assert Path.relative_to("../foo/./bar/..", File.cwd!()) == "../foo" + assert Path.relative_to("../foo/../bar/..", File.cwd!()) == ".." + assert Path.relative_to("./foo/../bar/..", File.cwd!()) == "." + + # both relative + assert Path.relative_to("usr/local/foo", ".") == "usr/local/foo" + assert Path.relative_to(".", "usr/local/foo") == "." + assert Path.relative_to("usr/local/foo", "usr/local") == "foo" + assert Path.relative_to("usr/local/foo", "etc") == "../usr/local/foo" + assert Path.relative_to(~c"usr/local/foo", "etc") == "../usr/local/foo" + assert Path.relative_to("usr/local/foo", "usr/local") == "foo" + assert Path.relative_to(["usr", ?/, ~c"local/foo"], ~c"usr/local") == "foo" end - test :relative_to do - assert Path.relative_to("/usr/local/foo", "/usr/local") == "foo" - assert Path.relative_to("/usr/local/foo", "/") == "usr/local/foo" - assert Path.relative_to("/usr/local/foo", "/etc") == "/usr/local/foo" - assert Path.relative_to("/usr/local/foo", "/usr/local/foo") == "/usr/local/foo" + test "safe_relative/1" do + assert Path.safe_relative("foo/bar") == {:ok, "foo/bar"} + assert Path.safe_relative("foo/..") == {:ok, ""} + assert Path.safe_relative("./foo") == {:ok, "foo"} - assert Path.relative_to("usr/local/foo", "usr/local") == "foo" - assert Path.relative_to("usr/local/foo", "etc") == "usr/local/foo" - assert Path.relative_to('usr/local/foo', "etc") == "usr/local/foo" + assert Path.safe_relative("/usr/local/foo") == :error + assert Path.safe_relative("foo/../..") == :error + end - assert Path.relative_to("usr/local/foo", "usr/local") == "foo" - assert Path.relative_to(["usr", ?/, 'local/foo'], 'usr/local') == "foo" + test "safe_relative/2" do + assert Path.safe_relative("/usr/local/foo", "/usr/local") == :error + assert Path.safe_relative("../../..", "foo/bar") == :error + assert Path.safe_relative("../../..", "foo/bar") == :error + assert Path.safe_relative("/usr/local/foo", "/") == :error end - test :rootname do + test "rootname/2" do assert Path.rootname("~/foo/bar.ex", ".ex") == "~/foo/bar" assert Path.rootname("~/foo/bar.exs", ".ex") == "~/foo/bar.exs" assert Path.rootname("~/foo/bar.old.ex", ".ex") == "~/foo/bar.old" - assert Path.rootname([?~, '/foo/bar', ".old.ex"], '.ex') == "~/foo/bar.old" + assert Path.rootname([?~, ~c"/foo/bar", ".old.ex"], ~c".ex") == "~/foo/bar.old" end - test :extname do + test "extname/1" do assert Path.extname("foo.erl") == ".erl" assert Path.extname("~/foo/bar") == "" - assert Path.extname('foo.erl') == ".erl" - assert Path.extname('~/foo/bar') == "" + assert Path.extname(~c"foo.erl") == ".erl" + assert Path.extname(~c"~/foo/bar") == "" end - test :dirname do + test "dirname/1" do assert Path.dirname("/foo/bar.ex") == "/foo" assert Path.dirname("foo/bar.ex") == "foo" assert Path.dirname("~/foo/bar.ex") == "~/foo" assert Path.dirname("/foo/bar/baz/") == "/foo/bar/baz" - assert Path.dirname([?~, "/foo", '/bar.ex']) == "~/foo" + assert Path.dirname([?~, "/foo", ~c"/bar.ex"]) == "~/foo" end - test :basename do + test "basename/1,2" do assert Path.basename("foo") == "foo" assert Path.basename("/foo/bar") == "bar" assert Path.basename("/") == "" @@ -179,38 +379,53 @@ defmodule PathTest do assert Path.basename("~/foo/bar.exs", ".ex") == "bar.exs" assert Path.basename("~/for/bar.old.ex", ".ex") == "bar.old" - assert Path.basename([?~, "/for/bar", '.old.ex'], ".ex") == "bar.old" + assert Path.basename([?~, "/for/bar", ~c".old.ex"], ".ex") == "bar.old" end - test :join do + test "join/1" do assert Path.join([""]) == "" assert Path.join(["foo"]) == "foo" assert Path.join(["/", "foo", "bar"]) == "/foo/bar" + assert Path.join(["/", "foo", "bar", "/"]) == "/foo/bar" assert Path.join(["~", "foo", "bar"]) == "~/foo/bar" - assert Path.join(['/foo/', "/bar/"]) == "/foo/bar" + assert Path.join([~c"/foo/", "/bar/"]) == "/foo/bar" + assert Path.join(["/", ""]) == "/" + assert Path.join(["/", "", "bar"]) == "/bar" + assert Path.join([~c"foo", [?b, "a", ?r]]) == "foo/bar" + assert Path.join([[?f, ~c"o", "o"]]) == "foo" end - test :join_two do + test "join/2" do assert Path.join("/foo", "bar") == "/foo/bar" assert Path.join("~", "foo") == "~/foo" - assert Path.join("", "bar") == "/bar" + assert Path.join("", "bar") == "bar" + assert Path.join("bar", "") == "bar" + assert Path.join("", "/bar") == "bar" + assert Path.join("/bar", "") == "/bar" + + assert Path.join("foo", "/bar") == "foo/bar" + assert Path.join("/foo", "/bar") == "/foo/bar" assert Path.join("/foo", "/bar") == "/foo/bar" - assert Path.join("/foo", "./bar") == "/foo/bar" + assert Path.join("/foo", "./bar") == "/foo/./bar" + + assert Path.join("/foo", "/") == "/foo" + assert Path.join("/foo", "/bar/zar/") == "/foo/bar/zar" - assert Path.join([?/, "foo"], "./bar") == "/foo/bar" + assert Path.join([?/, "foo"], "./bar") == "/foo/./bar" + assert Path.join(["/foo", "bar"], ["fiz", "buz"]) == "/foobar/fizbuz" end - test :split_with_binary do + test "split/1" do assert Path.split("") == [] assert Path.split("foo") == ["foo"] assert Path.split("/foo/bar") == ["/", "foo", "bar"] assert Path.split([?/, "foo/bar"]) == ["/", "foo", "bar"] end - if is_win? do - defp strip_drive_letter_if_windows([_d,?:|rest]), do: rest - defp strip_drive_letter_if_windows(<<_d,?:,rest::binary>>), do: rest + if PathHelpers.windows?() do + defp strip_drive_letter_if_windows([_d, ?: | rest]), do: rest + defp strip_drive_letter_if_windows(<<_d, ?:, rest::binary>>), do: rest else defp strip_drive_letter_if_windows(path), do: path end diff --git a/lib/elixir/test/elixir/port_test.exs b/lib/elixir/test/elixir/port_test.exs new file mode 100644 index 00000000000..dcefc4dc6bf --- /dev/null +++ b/lib/elixir/test/elixir/port_test.exs @@ -0,0 +1,38 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("test_helper.exs", __DIR__) + +defmodule PortTest do + use ExUnit.Case, async: true + + test "info/1,2 with registered name" do + {:ok, port} = :gen_udp.open(0) + + assert Port.info(port, :links) == {:links, [self()]} + assert Port.info(port, :registered_name) == {:registered_name, []} + + Process.register(port, __MODULE__) + + assert Port.info(port, :registered_name) == {:registered_name, __MODULE__} + + :ok = :gen_udp.close(port) + + assert Port.info(port, :registered_name) == nil + assert Port.info(port) == nil + end + + # In contrast with other inlined functions, + # it is important to test that monitor/1 is inlined, + # this way we gain the monitor receive optimisation. + test "monitor/1 is inlined" do + assert expand(quote(do: Port.monitor(port())), __ENV__) == + quote(do: :erlang.monitor(:port, port())) + end + + defp expand(expr, env) do + {expr, _, _} = :elixir_expand.expand(expr, :elixir_env.env_to_ex(env), env) + expr + end +end diff --git a/lib/elixir/test/elixir/process_test.exs b/lib/elixir/test/elixir/process_test.exs index be5b984f2f2..dcca57ddfc9 100644 --- a/lib/elixir/test/elixir/process_test.exs +++ b/lib/elixir/test/elixir/process_test.exs @@ -1,41 +1,201 @@ -Code.require_file "test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("test_helper.exs", __DIR__) defmodule ProcessTest do use ExUnit.Case, async: true + doctest Process + + test "dictionary" do + assert Process.put(:foo, :bar) == nil + assert Process.put(:foo, :baz) == :bar + + assert Enum.member?(Process.get_keys(), :foo) + refute Enum.member?(Process.get_keys(), :bar) + refute Enum.member?(Process.get_keys(), :baz) + assert Process.get_keys(:bar) == [] + assert Process.get_keys(:baz) == [:foo] + + assert Process.get(:foo) == :baz + assert Process.delete(:foo) == :baz + assert Process.get(:foo) == nil + end + test "group_leader/2 and group_leader/0" do - another = spawn_link(fn -> :timer.sleep(1000) end) - assert Process.group_leader(self, another) - assert Process.group_leader == another + another = spawn_link(fn -> Process.sleep(1000) end) + assert Process.group_leader(self(), another) + assert Process.group_leader() == another end - test "monitoring functions are inlined by the compiler" do + # In contrast with other inlined functions, + # it is important to test that monitor/1,2 are inlined, + # this way we gain the monitor receive optimisation. + test "monitor/1 and monitor/2 are inlined" do assert expand(quote(do: Process.monitor(pid())), __ENV__) == - quote(do: :erlang.monitor(:process, pid())) + quote(do: :erlang.monitor(:process, pid())) + + assert expand(quote(do: Process.monitor(pid(), alias: :demonitor)), __ENV__) == + quote(do: :erlang.monitor(:process, pid(), alias: :demonitor)) + end + + test "monitor/2 with monitor options" do + pid = + spawn(fn -> + receive do + {:ping, source_alias} -> send(source_alias, :pong) + end + end) + + ref_and_alias = Process.monitor(pid, alias: :explicit_unalias) + + send(pid, {:ping, ref_and_alias}) + + assert_receive :pong + assert_receive {:DOWN, ^ref_and_alias, _, _, _} + end + + test "sleep/1" do + assert Process.sleep(0) == :ok + end + + test "sleep/1 with 2^32" do + {pid, monitor_ref} = spawn_monitor(fn -> Process.sleep(2 ** 32) end) + refute_receive {:DOWN, ^monitor_ref, :process, ^pid, {:timeout_value, _trace}}, 100 + Process.exit(pid, :kill) end test "info/2" do - pid = spawn fn -> end + pid = spawn(fn -> Process.sleep(1000) end) + assert Process.info(pid, :priority) == {:priority, :normal} + assert Process.info(pid, [:priority]) == [priority: :normal] + Process.exit(pid, :kill) assert Process.info(pid, :backtrace) == nil + assert Process.info(pid, [:backtrace, :status]) == nil end test "info/2 with registered name" do - pid = spawn fn -> end + pid = spawn(fn -> nil end) Process.exit(pid, :kill) - assert Process.info(pid, :registered_name) == - nil + assert Process.info(pid, :registered_name) == nil + assert Process.info(pid, [:registered_name]) == nil + + assert Process.info(self(), :registered_name) == {:registered_name, []} + assert Process.info(self(), [:registered_name]) == [registered_name: []] + + Process.register(self(), __MODULE__) + assert Process.info(self(), :registered_name) == {:registered_name, __MODULE__} + assert Process.info(self(), [:registered_name]) == [registered_name: __MODULE__] + end + + test "send_after/3 sends messages once expired" do + Process.send_after(self(), :hello, 10) + assert_receive :hello + end + + test "send_after/4 with absolute time sends message once expired" do + time = System.monotonic_time(:millisecond) + 10 + Process.send_after(self(), :hello, time, abs: true) + assert_receive :hello + end + + test "send_after/3 returns a timer reference that can be read or cancelled" do + timer = Process.send_after(self(), :hello, 100_000) + refute_received :hello + assert is_integer(Process.read_timer(timer)) + assert is_integer(Process.cancel_timer(timer)) + + timer = Process.send_after(self(), :hello, 0) + assert_receive :hello + assert Process.read_timer(timer) == false + assert Process.cancel_timer(timer) == false + + timer = Process.send_after(self(), :hello, 100_000) + assert Process.cancel_timer(timer, async: true) + assert_receive {:cancel_timer, ^timer, result} + assert is_integer(result) + end + + test "exit(pid, :normal) does not cause the target process to exit" do + Process.flag(:trap_exit, true) + + pid = + spawn_link(fn -> + receive do + :done -> nil + end + end) + + true = Process.exit(pid, :normal) + refute_receive {:EXIT, ^pid, :normal}, 100 + assert Process.alive?(pid) + + # now exit the process for real so it doesn't hang around + true = Process.exit(pid, :abnormal) + assert_receive {:EXIT, ^pid, :abnormal} + refute Process.alive?(pid) + end + + test "exit(self(), :normal) causes the calling process to exit" do + Process.flag(:trap_exit, true) + pid = spawn_link(fn -> Process.exit(self(), :normal) end) + assert_receive {:EXIT, ^pid, :normal} + refute Process.alive?(pid) + end + + describe "alias/0, alias/1, and unalias/1" do + test "simple alias + unalias flow" do + server = + spawn(fn -> + receive do + {:ping, alias} -> send(alias, :pong) + end + end) + + alias = Process.alias() + Process.unalias(alias) + + send(server, {:ping, alias}) + refute_receive :pong, 20 + end + + test "with :reply option when aliasing" do + server = + spawn(fn -> + receive do + {:ping, alias} -> + send(alias, :pong) + send(alias, :extra_pong) + end + end) + + alias = Process.alias([:reply]) + + send(server, {:ping, alias}) + assert_receive :pong + refute_receive :extra_pong, 20 + end + end + + describe "set_label/1" do + @compile {:no_warn_undefined, :proc_lib} - assert Process.info(self, :registered_name) == - {:registered_name, []} + test "sets a process label, compatible with OTP 27+ `:proc_lib.get_label/1`" do + label = {:some_label, :rand.uniform(99999)} + assert :ok = Process.set_label(label) - Process.register(self, __MODULE__) - assert Process.info(self, :registered_name) == - {:registered_name, __MODULE__} + # TODO: Remove this when we require Erlang/OTP 27+ + if System.otp_release() >= "27" do + assert :proc_lib.get_label(self()) == label + end + end end defp expand(expr, env) do - {expr, _env} = :elixir_exp.expand(expr, env) + {expr, _, _} = :elixir_expand.expand(expr, :elixir_env.env_to_ex(env), env) expr end end diff --git a/lib/elixir/test/elixir/protocol/consolidation_test.exs b/lib/elixir/test/elixir/protocol/consolidation_test.exs new file mode 100644 index 00000000000..902645a975b --- /dev/null +++ b/lib/elixir/test/elixir/protocol/consolidation_test.exs @@ -0,0 +1,292 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) + +path = Path.expand("../../ebin", __DIR__) +File.mkdir_p!(path) + +files = Path.wildcard(PathHelpers.fixture_path("consolidation/*")) +Kernel.ParallelCompiler.compile_to_path(files, path, return_diagnostics: true) + +defmodule Protocol.ConsolidationTest do + use ExUnit.Case, async: true + alias Protocol.ConsolidationTest.{Sample, WithAny, NoImpl} + + defimpl WithAny, for: Map do + def ok(map, _opts) do + {:ok, map} + end + end + + defimpl WithAny, for: Any do + def ok(any, _opts) do + {:ok, any} + end + end + + defmodule NoImplStruct do + defstruct a: 0, b: 0 + end + + defmodule ImplStruct do + @derive [WithAny] + defstruct a: 0, b: 0 + + defimpl Sample do + @compile {:no_warn_undefined, Unknown} + + def ok(struct) do + Unknown.undefined(struct) + end + end + end + + Code.append_path(path) + + # Any is ignored because there is no fallback + :code.purge(Sample) + :code.delete(Sample) + {:ok, binary} = Protocol.consolidate(Sample, [Any, ImplStruct]) + :code.load_binary(Sample, ~c"protocol_test.exs", binary) + + defp sample_binary, do: unquote(binary) + + # Any should be moved to the end + :code.purge(WithAny) + :code.delete(WithAny) + {:ok, binary} = Protocol.consolidate(WithAny, [Any, ImplStruct, Map]) + :code.load_binary(WithAny, ~c"protocol_test.exs", binary) + + defp with_any_binary, do: unquote(binary) + + # No Any + :code.purge(NoImpl) + :code.delete(NoImpl) + {:ok, binary} = Protocol.consolidate(NoImpl, []) + :code.load_binary(NoImpl, ~c"protocol_test.exs", binary) + + defp no_impl_binary, do: unquote(binary) + + test "consolidated?/1" do + assert Protocol.consolidated?(WithAny) + refute Protocol.consolidated?(Enumerable) + end + + test "consolidation warns on new implementations" do + output = + ExUnit.CaptureIO.capture_io(:stderr, fn -> + defimpl WithAny, for: Integer do + def ok(_any, _opts), do: :ok + end + end) + + assert output =~ ~r"the .+WithAny protocol has already been consolidated" + after + :code.purge(WithAny.Integer) + :code.delete(WithAny.Integer) + end + + test "consolidation warns on new implementations unless disabled" do + Code.put_compiler_option(:ignore_already_consolidated, true) + + defimpl WithAny, for: Integer do + def ok(_any), do: :ok + end + after + Code.put_compiler_option(:ignore_already_consolidated, false) + :code.purge(WithAny.Integer) + :code.delete(WithAny.Integer) + end + + test "consolidated implementations without fallback to any" do + assert is_nil(Sample.impl_for(:foo)) + assert is_nil(Sample.impl_for(fn x -> x end)) + assert is_nil(Sample.impl_for(1)) + assert is_nil(Sample.impl_for(1.1)) + assert is_nil(Sample.impl_for([])) + assert is_nil(Sample.impl_for([1, 2, 3])) + assert is_nil(Sample.impl_for({})) + assert is_nil(Sample.impl_for({1, 2, 3})) + assert is_nil(Sample.impl_for("foo")) + assert is_nil(Sample.impl_for(<<1>>)) + assert is_nil(Sample.impl_for(self())) + assert is_nil(Sample.impl_for(%{})) + assert is_nil(Sample.impl_for(hd(:erlang.ports()))) + assert is_nil(Sample.impl_for(make_ref())) + + assert Sample.impl_for(%ImplStruct{}) == Sample.Protocol.ConsolidationTest.ImplStruct + assert Sample.impl_for(%NoImplStruct{}) == nil + end + + test "consolidated implementations with fallback to any" do + assert WithAny.impl_for(%NoImplStruct{}) == WithAny.Any + + # Derived + assert WithAny.impl_for(%ImplStruct{}) == + Protocol.ConsolidationTest.WithAny.Protocol.ConsolidationTest.ImplStruct + + assert WithAny.impl_for(%{__struct__: "foo"}) == WithAny.Map + assert WithAny.impl_for(%{}) == WithAny.Map + assert WithAny.impl_for(self()) == WithAny.Any + end + + test "consolidation keeps docs" do + {:ok, {Sample, [{~c"Docs", docs_bin}]}} = :beam_lib.chunks(sample_binary(), [~c"Docs"]) + {:docs_v1, _, _, _, _, _, docs} = :erlang.binary_to_term(docs_bin) + ok_doc = List.keyfind(docs, {:function, :ok, 1}, 0) + + assert {{:function, :ok, 1}, _, ["ok(term)"], %{"en" => "Ok"}, _} = ok_doc + end + + @tag :requires_source + test "consolidation keeps source" do + assert Sample.__info__(:compile)[:source] + end + + test "consolidated keeps callbacks" do + {:ok, callbacks} = Code.Typespec.fetch_callbacks(sample_binary()) + assert callbacks != [] + end + + test "consolidation updates attributes" do + assert Sample.__protocol__(:consolidated?) + assert Sample.__protocol__(:impls) == {:consolidated, [ImplStruct]} + assert WithAny.__protocol__(:consolidated?) + assert WithAny.__protocol__(:impls) == {:consolidated, [Any, Map, ImplStruct]} + assert NoImpl.__protocol__(:consolidated?) + assert NoImpl.__protocol__(:impls) == {:consolidated, []} + end + + describe "exports" do + import Module.Types.Descr + alias Module.Types.Of + + defp exports(binary) do + {:ok, {_, [{~c"ExCk", check_bin}]}} = :beam_lib.chunks(binary, [~c"ExCk"]) + assert {:elixir_checker_v4, contents} = :erlang.binary_to_term(check_bin) + Map.new(contents.exports) + end + + test "keeps deprecations" do + deprecated = [{{:ok, 1}, "Reason"}] + assert deprecated == Sample.__info__(:deprecated) + + assert %{{:ok, 1} => %{deprecated: "Reason", sig: _}} = exports(sample_binary()) + end + + test "defines signatures without fallback to any" do + exports = exports(sample_binary()) + + assert %{{:impl_for, 1} => %{sig: {:strong, domain, clauses}}} = exports + assert domain == [term()] + + assert clauses == [ + {[Of.impl(ImplStruct)], atom([Sample.Protocol.ConsolidationTest.ImplStruct])}, + {[negation(Of.impl(ImplStruct))], atom([nil])} + ] + + assert %{{:impl_for!, 1} => %{sig: {:strong, domain, clauses}}} = exports + assert domain == [Of.impl(ImplStruct)] + + assert clauses == [ + {[Of.impl(ImplStruct)], atom([Sample.Protocol.ConsolidationTest.ImplStruct])} + ] + + assert %{{:ok, 1} => %{sig: {:strong, nil, clauses}}} = exports + + assert clauses == [ + {[Of.impl(ImplStruct)], dynamic()} + ] + end + + test "defines signatures with fallback to any" do + exports = exports(with_any_binary()) + + assert %{ + {:impl_for, 1} => %{sig: {:strong, domain, clauses}}, + {:impl_for!, 1} => %{sig: {:strong, domain, clauses}} + } = exports + + assert domain == [term()] + + assert clauses == [ + {[Of.impl(Map)], atom([WithAny.Map])}, + {[Of.impl(ImplStruct)], atom([WithAny.Protocol.ConsolidationTest.ImplStruct])}, + {[negation(union(Of.impl(ImplStruct), Of.impl(Map)))], atom([WithAny.Any])} + ] + + assert %{{:ok, 2} => %{sig: {:strong, nil, clauses}}} = exports + + assert clauses == [ + {[term(), term()], dynamic()} + ] + end + + test "defines signatures without implementation" do + exports = exports(no_impl_binary()) + + assert %{{:impl_for, 1} => %{sig: {:strong, domain, clauses}}} = exports + assert domain == [term()] + assert clauses == [{[term()], atom([nil])}] + + assert %{{:impl_for!, 1} => %{sig: {:strong, domain, clauses}}} = exports + assert domain == [none()] + assert clauses == [{[none()], none()}] + + assert %{{:ok, 1} => %{sig: {:strong, nil, clauses}}} = exports + assert clauses == [{[none()], dynamic()}] + end + end + + test "consolidation errors on missing BEAM files" do + import PathHelpers + + write_beam( + defmodule ExampleModule do + end + ) + + defprotocol NoBeam do + def example(arg) + end + + assert Protocol.consolidate(ExampleModule, []) == {:error, :not_a_protocol} + assert Protocol.consolidate(NoBeam, []) == {:error, :no_beam_info} + end + + test "protocol not implemented" do + message = + "protocol Protocol.ConsolidationTest.Sample not implemented for Atom. " <> + "This protocol is implemented for: Protocol.ConsolidationTest.ImplStruct" <> + "\n\nGot value:\n\n :foo\n" + + assert_raise Protocol.UndefinedError, message, fn -> + sample = String.to_atom("Elixir.Protocol.ConsolidationTest.Sample") + sample.ok(:foo) + end + end + + describe "extraction" do + test "protocols" do + protos = Protocol.extract_protocols([Application.app_dir(:elixir, "ebin")]) + assert Enumerable in protos + assert Inspect in protos + end + + test "implementations with charlist path" do + protos = + Protocol.extract_impls(Enumerable, [to_charlist(Application.app_dir(:elixir, "ebin"))]) + + assert List in protos + assert Function in protos + end + + test "implementations with binary path" do + protos = Protocol.extract_impls(Enumerable, [Application.app_dir(:elixir, "ebin")]) + assert List in protos + assert Function in protos + end + end +end diff --git a/lib/elixir/test/elixir/protocol_test.exs b/lib/elixir/test/elixir/protocol_test.exs index 4392b189946..4319d69c610 100644 --- a/lib/elixir/test/elixir/protocol_test.exs +++ b/lib/elixir/test/elixir/protocol_test.exs @@ -1,27 +1,41 @@ -Code.require_file "test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("test_helper.exs", __DIR__) defmodule ProtocolTest do use ExUnit.Case, async: true - defprotocol Sample do - @type t :: any - @doc "Ok" - @spec ok(t) :: boolean - def ok(thing) - end + doctest Protocol - defprotocol WithAny do - @fallback_to_any true - @doc "Ok" - def ok(thing) - end + {_, _, sample_binary, _} = + defprotocol Sample do + @type t :: any + @doc "Ok" + @deprecated "Reason" + @spec ok(t) :: boolean + def ok(term) + end + + @sample_binary sample_binary + + {_, _, with_any_binary, _} = + defprotocol WithAny do + @fallback_to_any true + @doc "Ok" + def ok(term) + end + + @with_any_binary with_any_binary defprotocol Derivable do - def ok(a) - end + @undefined_impl_description "you should try harder" + + @impl true + defmacro __deriving__(module, options) do + struct = Macro.struct!(module, __CALLER__) - defimpl Derivable, for: Map do - defmacro __deriving__(module, struct, options) do quote do defimpl Derivable, for: unquote(module) do def ok(arg) do @@ -31,6 +45,10 @@ defmodule ProtocolTest do end end + def ok(a) + end + + defimpl Derivable, for: Any do def ok(arg) do {:ok, arg} end @@ -57,63 +75,115 @@ defmodule ProtocolTest do defstruct a: 0, b: 0 defimpl Sample do + @compile {:no_warn_undefined, Unknown} + def ok(struct) do Unknown.undefined(struct) end end end - test "protocol implementations without any" do - assert nil? Sample.impl_for(:foo) - assert nil? Sample.impl_for(fn(x) -> x end) - assert nil? Sample.impl_for(1) - assert nil? Sample.impl_for(1.1) - assert nil? Sample.impl_for([]) - assert nil? Sample.impl_for([1, 2, 3]) - assert nil? Sample.impl_for({}) - assert nil? Sample.impl_for({1, 2, 3}) - assert nil? Sample.impl_for("foo") - assert nil? Sample.impl_for(<<1>>) - assert nil? Sample.impl_for(%{}) - assert nil? Sample.impl_for(self) - assert nil? Sample.impl_for(hd(:erlang.ports)) - assert nil? Sample.impl_for(make_ref) - - assert Sample.impl_for(%ImplStruct{}) == - Sample.ProtocolTest.ImplStruct - assert Sample.impl_for(%NoImplStruct{}) == - nil + defmodule ImplStructExplicitFor do + defstruct a: 0, b: 0 + + defimpl Sample, for: __MODULE__ do + def ok(_struct), do: true + end end - test "protocol implementation with any and structs fallback" do - assert WithAny.impl_for(%NoImplStruct{}) == WithAny.Any - assert WithAny.impl_for(%ImplStruct{}) == WithAny.Map # Derived + test "protocol implementations without any" do + assert is_nil(Sample.impl_for(:foo)) + assert is_nil(Sample.impl_for(fn x -> x end)) + assert is_nil(Sample.impl_for(1)) + assert is_nil(Sample.impl_for(1.1)) + assert is_nil(Sample.impl_for([])) + assert is_nil(Sample.impl_for([1, 2, 3])) + assert is_nil(Sample.impl_for({})) + assert is_nil(Sample.impl_for({1, 2, 3})) + assert is_nil(Sample.impl_for("foo")) + assert is_nil(Sample.impl_for(<<1>>)) + assert is_nil(Sample.impl_for(%{})) + assert is_nil(Sample.impl_for(self())) + assert is_nil(Sample.impl_for(hd(:erlang.ports()))) + assert is_nil(Sample.impl_for(make_ref())) + + assert Sample.impl_for(%ImplStruct{}) == Sample.ProtocolTest.ImplStruct + assert Sample.impl_for(%ImplStructExplicitFor{}) == Sample.ProtocolTest.ImplStructExplicitFor + assert Sample.impl_for(%NoImplStruct{}) == nil + assert is_nil(Sample.impl_for(%{__struct__: nil})) + end + + test "protocol implementation with Any and struct fallbacks" do + assert WithAny.impl_for(%NoImplStruct{}) == WithAny.Any + assert WithAny.impl_for(%{__struct__: nil}) == WithAny.Any assert WithAny.impl_for(%{__struct__: "foo"}) == WithAny.Map - assert WithAny.impl_for(%{}) == WithAny.Map - assert WithAny.impl_for(self) == WithAny.Any + assert WithAny.impl_for(%{}) == WithAny.Map + assert WithAny.impl_for(self()) == WithAny.Any + + # Derived + assert WithAny.impl_for(%ImplStruct{}) == ProtocolTest.WithAny.ProtocolTest.ImplStruct end test "protocol not implemented" do - assert_raise Protocol.UndefinedError, "protocol ProtocolTest.Sample not implemented for :foo", fn -> - Sample.ok(:foo) + message = + """ + protocol ProtocolTest.Sample not implemented for Atom + + Got value: + + :foo + """ + + assert_raise Protocol.UndefinedError, message, fn -> + sample = String.to_atom("Elixir.ProtocolTest.Sample") + sample.ok(:foo) end end - test "protocol documentation" do + test "protocol documentation and deprecated" do import PathHelpers - write_beam(defprotocol SampleDocsProto do - @type t :: any - @doc "Ok" - @spec ok(t) :: boolean - def ok(thing) - end) + write_beam( + defprotocol SampleDocsProto do + @doc "Ok" + @deprecated "Reason" + @spec ok(t) :: boolean + def ok(term) + end + ) + + write_beam( + defimpl SampleDocsProto, for: List do + def ok(_), do: true + end + ) + + write_beam( + defimpl SampleDocsProto, for: Map do + @moduledoc "for map" + + def ok(_), do: true + end + ) - docs = Code.get_docs(SampleDocsProto, :docs) - assert {{:ok, 1}, _, :def, [{:thing, _, nil}], "Ok"} = - List.keyfind(docs, {:ok, 1}, 0) + {:docs_v1, _, _, _, _, _, docs} = Code.fetch_docs(SampleDocsProto) + + assert {{:type, :t, 0}, _, [], %{"en" => type_doc}, _} = List.keyfind(docs, {:type, :t, 0}, 0) + assert type_doc =~ "All the types that implement this protocol" + + assert {{:function, :ok, 1}, _, ["ok(term)"], %{"en" => "Ok"}, _} = + List.keyfind(docs, {:function, :ok, 1}, 0) + + deprecated = SampleDocsProto.__info__(:deprecated) + assert [{{:ok, 1}, "Reason"}] = deprecated + + {:docs_v1, _, _, _, :hidden, _, _} = Code.fetch_docs(SampleDocsProto.List) + {:docs_v1, _, _, _, moduledoc, _, _} = Code.fetch_docs(SampleDocsProto.Map) + assert moduledoc == %{"en" => "for map"} end + @compile {:no_warn_undefined, WithAll} + test "protocol keeps underlying UndefinedFunctionError" do assert_raise UndefinedFunctionError, fn -> WithAll.ok(%ImplStruct{}) @@ -121,34 +191,54 @@ defmodule ProtocolTest do end test "protocol defines callbacks" do - assert get_callbacks(Sample, :ok, 1) == - [{:type, 9, :fun, [{:type, 9, :product, [{:type, 9, :t, []}]}, {:type, 9, :boolean, []}]}] + assert [{:type, {17, 13}, :fun, args}] = get_callbacks(@sample_binary, :ok, 1) + + assert args == [ + {:type, {17, 13}, :product, [{:user_type, {17, 16}, :t, []}]}, + {:type, {17, 22}, :boolean, []} + ] + + assert [{:type, 27, :fun, args}] = get_callbacks(@with_any_binary, :ok, 1) + assert args == [{:type, 27, :product, [{:user_type, 27, :t, []}]}, {:type, 27, :term, []}] + end - assert get_callbacks(WithAny, :ok, 1) == - [{:type, 16, :fun, [{:type, 16, :product, [{:type, 16, :t, []}]}, {:type, 16, :term, []}]}] + test "protocol defines t/0 type with documentation" do + assert {:type, {:t, {_, _, :any, []}, []}} = get_type(@sample_binary, :t, 0) end - test "protocol defines attributes" do - assert Sample.__info__(:attributes)[:protocol] == [fallback_to_any: false, consolidated: false] - assert WithAny.__info__(:attributes)[:protocol] == [fallback_to_any: true, consolidated: false] + test "protocol defines functions and attributes" do + assert Sample.__protocol__(:module) == Sample + assert Sample.__protocol__(:functions) == [ok: 1] + refute Sample.__protocol__(:consolidated?) + assert Sample.__protocol__(:impls) == :not_consolidated + assert Sample.__info__(:attributes)[:__protocol__] == [fallback_to_any: false] + + assert WithAny.__protocol__(:module) == WithAny + assert WithAny.__protocol__(:functions) == [ok: 1] + refute WithAny.__protocol__(:consolidated?) + assert WithAny.__protocol__(:impls) == :not_consolidated + assert WithAny.__info__(:attributes)[:__protocol__] == [fallback_to_any: true] end test "defimpl" do - defprotocol Attribute do - def test(thing) - end + module = Module.concat(Sample, ImplStruct) + assert module.__impl__(:for) == ImplStruct + assert module.__impl__(:protocol) == Sample + assert module.__info__(:attributes)[:__impl__] == [protocol: Sample, for: ImplStruct] + end - defimpl Attribute, for: ImplStruct do - def test(_) do - {@protocol, @for} - end - end + test "defimpl with implicit derive" do + module = Module.concat(WithAny, ImplStruct) + assert module.__impl__(:for) == ImplStruct + assert module.__impl__(:protocol) == WithAny + assert module.__info__(:attributes)[:__impl__] == [protocol: WithAny, for: ImplStruct] + end - assert Attribute.test(%ImplStruct{}) == {Attribute, ImplStruct} - assert Attribute.ProtocolTest.ImplStruct.__impl__(:protocol) == Attribute - assert Attribute.ProtocolTest.ImplStruct.__impl__(:for) == ImplStruct - assert Attribute.ProtocolTest.ImplStruct.__info__(:attributes)[:impl] == - [protocol: Attribute, for: ImplStruct] + test "defimpl with explicit derive" do + module = Module.concat(Derivable, ImplStruct) + assert module.__impl__(:for) == ImplStruct + assert module.__impl__(:protocol) == Derivable + assert module.__info__(:attributes)[:__impl__] == [protocol: Derivable, for: ImplStruct] end test "defimpl with multiple for" do @@ -164,47 +254,67 @@ defmodule ProtocolTest do assert Multi.test(:a) == :a end - defp get_callbacks(module, name, arity) do - callbacks = for {:callback, info} <- module.__info__(:attributes), do: hd(info) - List.keyfind(callbacks, {name, arity}, 0) |> elem(1) + test "defimpl without :for option when outside a module" do + msg = "defimpl/3 expects a :for option when declared outside a module" + + assert_raise ArgumentError, msg, fn -> + ast = + quote do + defimpl Sample do + def ok(_term), do: true + end + end + + Code.eval_quoted(ast, [], %{__ENV__ | module: nil}) + end end - test "derives protocol" do - struct = %ImplStruct{a: 1, b: 1} - assert WithAny.ok(struct) == {:ok, struct} + defp get_callbacks(beam, name, arity) do + {:ok, callbacks} = Code.Typespec.fetch_callbacks(beam) + List.keyfind(callbacks, {name, arity}, 0) |> elem(1) end - test "derived protocol keeps local file/line info" do - assert ProtocolTest.WithAny.ProtocolTest.ImplStruct.__info__(:compile)[:source] == - String.to_char_list(__ENV__.file) + defp get_type(beam, name, arity) do + {:ok, types} = Code.Typespec.fetch_types(beam) + + assert {:value, value} = + :lists.search(&match?({_, {^name, _, args}} when length(args) == arity, &1), types) + + value end - test "custom derive implementation" do + test "derives protocol implicitly" do struct = %ImplStruct{a: 1, b: 1} - assert Derivable.ok(struct) == {:ok, struct, %ImplStruct{}, []} + assert WithAny.ok(struct) == {:ok, struct} + struct = %NoImplStruct{a: 1, b: 1} + assert WithAny.ok(struct) == {:ok, struct} + end + + test "derives protocol explicitly" do struct = %ImplStruct{a: 1, b: 1} assert Derivable.ok(struct) == {:ok, struct, %ImplStruct{}, []} - assert_raise Protocol.UndefinedError, fn -> - struct = %NoImplStruct{a: 1, b: 1} - Derivable.ok(struct) - end + assert_raise Protocol.UndefinedError, + ~r"protocol ProtocolTest.Derivable not implemented for ProtocolTest.NoImplStruct \(a struct\), you should try harder", + fn -> + struct = %NoImplStruct{a: 1, b: 1} + Derivable.ok(struct) + end end - test "custom derive implementation with options" do + test "derives protocol explicitly with options" do defmodule AnotherStruct do @derive [{Derivable, :ok}] @derive [WithAny] defstruct a: 0, b: 0 end - struct = struct AnotherStruct, a: 1, b: 1 - assert Derivable.ok(struct) == - {:ok, struct, struct(AnotherStruct), :ok} + struct = struct(AnotherStruct, a: 1, b: 1) + assert Derivable.ok(struct) == {:ok, struct, struct(AnotherStruct), :ok} end - test "custom derive implementation via API" do + test "derive protocol explicitly via API" do defmodule InlineStruct do defstruct a: 0, b: 0 end @@ -212,156 +322,132 @@ defmodule ProtocolTest do require Protocol assert Protocol.derive(Derivable, InlineStruct, :oops) == :ok - struct = struct InlineStruct, a: 1, b: 1 - assert Derivable.ok(struct) == - {:ok, struct, struct(InlineStruct), :oops} + struct = struct(InlineStruct, a: 1, b: 1) + assert Derivable.ok(struct) == {:ok, struct, struct(InlineStruct), :oops} end - test "cannot derive without a map implementation" do - assert_raise ArgumentError, - ~r"#{inspect Sample.Map} is not available, cannot derive #{inspect Sample}", fn -> - defmodule NotCompiled do - @derive [Sample] - defstruct hello: :world - end - end + @tag :requires_source + test "derived implementation keeps local file/line info" do + assert ProtocolTest.WithAny.ProtocolTest.ImplStruct.__info__(:compile)[:source] == + String.to_charlist(__ENV__.file) end -end -path = Path.expand("../ebin", __DIR__) -File.mkdir_p!(path) + describe "warnings" do + import ExUnit.CaptureIO -compile = fn {:module, module, binary, _} -> - File.write!("#{path}/#{module}.beam", binary) -end + test "with no definitions" do + assert capture_io(:stderr, fn -> + defprotocol SampleWithNoDefinitions do + end + end) =~ "protocols must define at least one function, but none was defined" + end -defmodule Protocol.ConsolidationTest do - use ExUnit.Case, async: true + test "when @callbacks and friends are defined inside a protocol" do + message = + capture_io(:stderr, fn -> + defprotocol SampleWithCallbacks do + @spec with_specs(any(), keyword()) :: tuple() + def with_specs(term, options \\ []) - compile.( - defprotocol Sample do - @type t :: any - @doc "Ok" - @spec ok(t) :: boolean - def ok(thing) - end - ) + @spec with_specs_and_when(any(), opts) :: tuple() when opts: keyword + def with_specs_and_when(term, options \\ []) - compile.( - defprotocol WithAny do - @fallback_to_any true - @doc "Ok" - def ok(thing) - end - ) + def without_specs(term, options \\ []) - defimpl WithAny, for: Map do - def ok(map) do - {:ok, map} - end - end + @callback foo :: {:ok, term} + @callback foo(term) :: {:ok, term} + @callback foo(term, keyword) :: {:ok, term, keyword} - defimpl WithAny, for: Any do - def ok(any) do - {:ok, any} - end - end + @callback foo_when :: {:ok, x} when x: term + @callback foo_when(x) :: {:ok, x} when x: term + @callback foo_when(x, opts) :: {:ok, x, opts} when x: term, opts: keyword - defmodule NoImplStruct do - defstruct a: 0, b: 0 - end + @macrocallback bar(term) :: {:ok, term} + @macrocallback bar(term, keyword) :: {:ok, term, keyword} - defmodule ImplStruct do - @derive [WithAny] - defstruct a: 0, b: 0 + @optional_callbacks [foo: 1, foo: 2] + @optional_callbacks [without_specs: 2] + end + end) - defimpl Sample do - def ok(struct) do - Unknown.undefined(struct) - end - end - end + assert message =~ + "cannot define @callback foo/0 inside protocol, use def/1 to outline your protocol definition" - Code.append_path(path) + assert message =~ + "cannot define @callback foo/1 inside protocol, use def/1 to outline your protocol definition" - # Any is ignored because there is no fallback - :code.purge(Sample) - :code.delete(Sample) - {:ok, binary} = Protocol.consolidate(Sample, [Any, ImplStruct]) - :code.load_binary(Sample, 'protocol_test.exs', binary) + assert message =~ + "cannot define @callback foo/2 inside protocol, use def/1 to outline your protocol definition" - # Any should be moved to the end - :code.purge(WithAny) - :code.delete(WithAny) - {:ok, binary} = Protocol.consolidate(WithAny, [Any, ImplStruct, Map]) - :code.load_binary(WithAny, 'protocol_test.exs', binary) + assert message =~ + "cannot define @callback foo_when/0 inside protocol, use def/1 to outline your protocol definition" - test "consolidated?/1" do - assert Protocol.consolidated?(WithAny) - refute Protocol.consolidated?(Enumerable) - end + assert message =~ + "cannot define @callback foo_when/1 inside protocol, use def/1 to outline your protocol definition" - test "consolidated implementations without any" do - assert nil? Sample.impl_for(:foo) - assert nil? Sample.impl_for(fn(x) -> x end) - assert nil? Sample.impl_for(1) - assert nil? Sample.impl_for(1.1) - assert nil? Sample.impl_for([]) - assert nil? Sample.impl_for([1, 2, 3]) - assert nil? Sample.impl_for({}) - assert nil? Sample.impl_for({1, 2, 3}) - assert nil? Sample.impl_for("foo") - assert nil? Sample.impl_for(<<1>>) - assert nil? Sample.impl_for(self) - assert nil? Sample.impl_for(%{}) - assert nil? Sample.impl_for(hd(:erlang.ports)) - assert nil? Sample.impl_for(make_ref) - - assert Sample.impl_for(%ImplStruct{}) == - Sample.Protocol.ConsolidationTest.ImplStruct - assert Sample.impl_for(%NoImplStruct{}) == - nil - end + assert message =~ + "cannot define @callback foo_when/2 inside protocol, use def/1 to outline your protocol definition" - test "consolidated implementations with any and tuple fallback" do - assert WithAny.impl_for(%NoImplStruct{}) == WithAny.Any - assert WithAny.impl_for(%ImplStruct{}) == WithAny.Map # Derived - assert WithAny.impl_for(%{__struct__: "foo"}) == WithAny.Map - assert WithAny.impl_for(%{}) == WithAny.Map - assert WithAny.impl_for(self) == WithAny.Any - end + assert message =~ + "cannot define @macrocallback bar/1 inside protocol, use def/1 to outline your protocol definition" - test "consolidation keeps docs" do - docs = Code.get_docs(Sample, :docs) - assert {{:ok, 1}, _, :def, [{:thing, _, nil}], "Ok"} = - List.keyfind(docs, {:ok, 1}, 0) - end + assert message =~ + "cannot define @macrocallback bar/2 inside protocol, use def/1 to outline your protocol definition" - test "consolidated keeps callbacks" do - callbacks = for {:callback, info} <- Sample.__info__(:attributes), do: hd(info) - assert callbacks != [] - end + assert message =~ + "cannot define @optional_callbacks inside protocol, all of the protocol definitions are required" + end - test "consolidation errors on missing beams" do - defprotocol NoBeam, do: nil - assert Protocol.consolidate(String, []) == {:error, :not_a_protocol} - assert Protocol.consolidate(NoBeam, []) == {:error, :no_beam_info} - end + test "when deriving after struct" do + assert capture_io(:stderr, fn -> + defmodule DeriveTooLate do + defstruct [] + @derive [{Derivable, :ok}] + end + end) =~ + "module attribute @derive was set after defstruct, all @derive calls must come before defstruct" + end - test "consolidation updates attributes" do - assert Sample.__info__(:attributes)[:protocol] == [fallback_to_any: false, consolidated: true] - assert WithAny.__info__(:attributes)[:protocol] == [fallback_to_any: true, consolidated: true] + test "when deriving with no struct" do + assert capture_io(:stderr, fn -> + defmodule DeriveNeverUsed do + @derive [{Derivable, :ok}] + end + end) =~ + "module attribute @derive was set but never used (it must come before defstruct)" + end end - test "consolidation extracts protocols" do - protos = Protocol.extract_protocols([:code.lib_dir(:elixir, :ebin)]) - assert Enumerable in protos - assert Inspect in protos + describe "errors" do + test "cannot derive without any implementation" do + assert_raise ArgumentError, + ~r"could not load module #{inspect(Sample.Any)} due to reason :nofile, cannot derive #{inspect(Sample)}", + fn -> + defmodule NotCompiled do + @derive [Sample] + defstruct hello: :world + end + end + end end +end + +defmodule Protocol.DebugInfoTest do + use ExUnit.Case + + test "protocols always keep debug_info" do + Code.compiler_options(debug_info: false) + + {:module, _, binary, _} = + defprotocol DebugInfoProto do + def example(info) + end + + assert {:ok, {DebugInfoProto, [debug_info: debug_info]}} = + :beam_lib.chunks(binary, [:debug_info]) - test "consolidation extracts implementations" do - protos = Protocol.extract_impls(Enumerable, [:code.lib_dir(:elixir, :ebin)]) - assert List in protos - assert Function in protos + assert {:debug_info_v1, :elixir_erl, {:elixir_v1, _, _}} = debug_info + after + Code.compiler_options(debug_info: true) end end diff --git a/lib/elixir/test/elixir/range_test.exs b/lib/elixir/test/elixir/range_test.exs index cde88cc2fb5..9c961343872 100644 --- a/lib/elixir/test/elixir/range_test.exs +++ b/lib/elixir/test/elixir/range_test.exs @@ -1,41 +1,246 @@ -Code.require_file "test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("test_helper.exs", __DIR__) defmodule RangeTest do use ExUnit.Case, async: true - test :precedence do - assert Enum.to_list(1..3+2) == [1, 2, 3, 4, 5] - assert 1..3 |> Enum.to_list == [1, 2, 3] + doctest Range + + defp reverse(first..last//step) do + last..first//-step end - test :op do - assert (1..3).first == 1 - assert (1..3).last == 3 + defp assert_disjoint(r1, r2) do + disjoint_assertions(r1, r2, true) end - test :range? do - assert Range.range?(1..3) - refute Range.range?(0) + defp assert_overlap(r1, r2) do + disjoint_assertions(r1, r2, false) end - test :enum do - refute Enum.empty?(1..1) + defp disjoint_assertions(r1, r2, expected) do + # The caller should choose pairs of representative ranges, and we take care + # here of commuting them. + Enum.each([[r1, r2], [r2, r1]], fn [a, b] -> + assert Range.disjoint?(a, b) == expected + assert Range.disjoint?(reverse(a), b) == expected + assert Range.disjoint?(a, reverse(b)) == expected + assert Range.disjoint?(reverse(a), reverse(b)) == expected + end) + end - assert Enum.member?(1..3, 2) - refute Enum.member?(1..3, 0) - refute Enum.member?(1..3, 4) - refute Enum.member?(3..1, 0) - refute Enum.member?(3..1, 4) + test "new" do + assert Range.new(1, 3) == 1..3//1 + assert Range.new(1, 3, 2) == 1..3//2 + assert Range.new(3, 1, -2) == 3..1//-2 - assert Enum.count(1..3) == 3 - assert Enum.count(3..1) == 3 + assert ExUnit.CaptureIO.capture_io(:stderr, fn -> + assert Range.new(3, 1) == 3..1//-1 + end) =~ "default to a step of -1" + end - assert Enum.map(1..3, &(&1 * 2)) == [2, 4, 6] - assert Enum.map(3..1, &(&1 * 2)) == [6, 4, 2] + test "fields" do + assert (1..3).first == 1 + assert (1..3).last == 3 + assert (1..3).step == 1 + assert (3..1//-1).step == -1 + assert (1..3//2).step == 2 end - test :inspect do + test "inspect" do assert inspect(1..3) == "1..3" - assert inspect(3..1) == "3..1" + assert inspect(1..3//2) == "1..3//2" + + assert inspect(3..1//-1) == "3..1//-1" + assert inspect(3..1//1) == "3..1//1" + end + + test "shift" do + assert Range.shift(0..10//2, 2) == 4..14//2 + assert Range.shift(0..10//2, 0) == 0..10//2 + assert Range.shift(10..0//-2, 2) == 6..-4//-2 + assert Range.shift(10..0//-2, -2) == 14..4//-2 + end + + test "in guard equality" do + case {1, 1..1} do + {n, range} when range == n..n//1 -> true + end + end + + test "limits are integer only" do + first = 1.0 + last = 3.0 + message = "ranges (first..last) expect both sides to be integers, got: 1.0..3.0" + assert_raise ArgumentError, message, fn -> first..last end + + first = [] + last = [] + message = "ranges (first..last) expect both sides to be integers, got: []..[]" + assert_raise ArgumentError, message, fn -> first..last end + end + + test "step is a non-zero integer" do + step = 1.0 + message = ~r"the step to be a non-zero integer" + assert_raise ArgumentError, message, fn -> 1..3//step end + + step = 0 + message = ~r"the step to be a non-zero integer" + assert_raise ArgumentError, message, fn -> 1..3//step end + end + + describe "disjoint?" do + test "returns true for disjoint ranges" do + assert_disjoint(1..5, 6..9) + assert_disjoint(-3..1, 2..3) + assert_disjoint(-7..-5, -3..-1) + + assert Range.disjoint?(1..1, 2..2) == true + assert Range.disjoint?(2..2, 1..1) == true + end + + test "returns false for ranges with common endpoints" do + assert_overlap(1..5, 5..9) + assert_overlap(-1..0, 0..1) + assert_overlap(-7..-5, -5..-1) + end + + test "returns false for ranges that overlap" do + assert_overlap(1..5, 3..7) + assert_overlap(-3..1, -1..3) + assert_overlap(-7..-5, -5..-1) + + assert Range.disjoint?(1..1, 1..1) == false + end + end + + describe "split" do + @times 10 + + test "increasing ranges" do + for _ <- 1..@times do + left = Enum.random(-10..10) + right = Enum.random(-10..10) + input = min(left, right)..max(left, right)//Enum.random(1..3) + + for split <- -3..3 do + {left, right} = Range.split(input, split) + assert input.first == left.first + assert input.last == right.last + assert input.step == left.step + assert input.step == right.step + + assert Range.size(input) == Range.size(left) + Range.size(right), + "size mismatch: Range.split(#{inspect(input)}, #{split})" + end + end + end + + test "decreasing ranges" do + for _ <- 1..@times do + left = Enum.random(-10..10) + right = Enum.random(-10..10) + input = max(left, right)..min(left, right)//-Enum.random(1..3) + + for split <- -3..3 do + {left, right} = Range.split(input, split) + assert input.first == left.first + assert input.last == right.last + assert input.step == left.step + assert input.step == right.step + + assert Range.size(input) == Range.size(left) + Range.size(right), + "size mismatch: Range.split(#{inspect(input)}, #{split})" + end + end + end + + test "empty increasing ranges" do + for _ <- 1..@times, + left = Enum.random(-10..10), + right = Enum.random(-10..10), + left != right do + input = min(left, right)..max(left, right)//-Enum.random(1..3) + + for split <- -3..3 do + {left, right} = Range.split(input, split) + assert input.first == left.first + assert input.last == right.last + assert input.step == left.step + assert input.step == right.step + + assert Range.size(input) == Range.size(left) + Range.size(right), + "size mismatch: Range.split(#{inspect(input)}, #{split})" + end + end + end + + test "empty decreasing ranges" do + for _ <- 1..@times, + left = Enum.random(-10..10), + right = Enum.random(-10..10), + left != right do + input = max(left, right)..min(left, right)//Enum.random(1..3) + + for split <- -3..3 do + {left, right} = Range.split(input, split) + assert input.first == left.first + assert input.last == right.last + assert input.step == left.step + assert input.step == right.step + + assert Range.size(input) == Range.size(left) + Range.size(right), + "size mismatch: Range.split(#{inspect(input)}, #{split})" + end + end + end + end + + describe "old ranges" do + test "enum" do + asc = %{__struct__: Range, first: 1, last: 3} + desc = %{__struct__: Range, first: 3, last: 1} + + assert Enum.to_list(asc) == [1, 2, 3] + assert Enum.member?(asc, 2) + assert Enum.count(asc) == 3 + assert Enum.drop(asc, 1) == [2, 3] + assert Enum.slice([1, 2, 3, 4, 5, 6], asc) == [2, 3, 4] + # testing private Enum.aggregate + assert Enum.max(asc) == 3 + assert Enum.sum(asc) == 6 + assert Enum.min_max(asc) == {1, 3} + assert Enum.reduce(asc, 0, fn a, b -> a + b end) == 6 + + assert Enum.to_list(desc) == [3, 2, 1] + assert Enum.member?(desc, 2) + assert Enum.count(desc) == 3 + assert Enum.drop(desc, 1) == [2, 1] + + assert ExUnit.CaptureIO.capture_io(:stderr, fn -> + assert Enum.slice([1, 2, 3, 4, 5, 6], desc) == [] + end) =~ "negative steps are not supported in Enum.slice/2, pass 3..1//1 instead" + + # testing private Enum.aggregate + assert Enum.max(desc) == 3 + assert Enum.sum(desc) == 6 + assert Enum.min_max(desc) == {1, 3} + assert Enum.reduce(desc, 0, fn a, b -> a + b end) == 6 + end + + test "string" do + asc = %{__struct__: Range, first: 1, last: 3} + desc = %{__struct__: Range, first: 3, last: 1} + + assert String.slice("elixir", asc) == "lix" + + assert ExUnit.CaptureIO.capture_io(:stderr, fn -> + assert String.slice("elixir", desc) == "" + end) =~ "negative steps are not supported in String.slice/2, pass 3..1//1 instead" + end end end diff --git a/lib/elixir/test/elixir/record_test.exs b/lib/elixir/test/elixir/record_test.exs index 4916b01d56b..90685d5d4bb 100644 --- a/lib/elixir/test/elixir/record_test.exs +++ b/lib/elixir/test/elixir/record_test.exs @@ -1,16 +1,31 @@ -Code.require_file "test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("test_helper.exs", __DIR__) defmodule RecordTest do use ExUnit.Case, async: true require Record + doctest Record test "extract/2 extracts information from an Erlang file" do - assert Record.extract(:file_info, from_lib: "kernel/include/file.hrl") == - [size: :undefined, type: :undefined, access: :undefined, atime: :undefined, - mtime: :undefined, ctime: :undefined, mode: :undefined, links: :undefined, - major_device: :undefined, minor_device: :undefined, inode: :undefined, - uid: :undefined, gid: :undefined] + assert Record.extract(:file_info, from_lib: "kernel/include/file.hrl") == [ + size: :undefined, + type: :undefined, + access: :undefined, + atime: :undefined, + mtime: :undefined, + ctime: :undefined, + mode: :undefined, + links: :undefined, + major_device: :undefined, + minor_device: :undefined, + inode: :undefined, + uid: :undefined, + gid: :undefined + ] end test "extract/2 handles nested records too" do @@ -24,58 +39,251 @@ defmodule RecordTest do defstruct Record.extract(:file_info, from_lib: "kernel/include/file.hrl") end - assert %{__struct__: StructExtract, size: :undefined} = - StructExtract.__struct__ + assert %{__struct__: StructExtract, size: :undefined} = StructExtract.__struct__() + end + + test "extract_all/1 extracts all records information from an Erlang file" do + all_extract = Record.extract_all(from_lib: "kernel/include/file.hrl") + # has been stable over the very long time + assert length(all_extract) == 2 + assert all_extract[:file_info] + assert all_extract[:file_descriptor] end # We need indirection to avoid warnings defp record?(data, kind) do - Record.record?(data, kind) + Record.is_record(data, kind) end - test "record?/2" do - assert record?({User, "jose", 27}, User) - refute record?({User, "jose", 27}, Author) + test "is_record/2" do + assert record?({User, "meg", 27}, User) + refute record?({User, "meg", 27}, Author) refute record?(13, Author) + refute record?({"user", "meg", 27}, "user") + refute record?({}, User) + refute record?([], User) end # We need indirection to avoid warnings defp record?(data) do - Record.record?(data) + Record.is_record(data) end - test "record?/1" do - assert record?({User, "jose", 27}) - refute record?({"jose", 27}) + test "is_record/1" do + assert record?({User, "john", 27}) + refute record?({"john", 27}) refute record?(13) + refute record?({}) + end + + def record_in_guard?(term) when Record.is_record(term), do: true + def record_in_guard?(_), do: false + + def record_in_guard?(term, kind) when Record.is_record(term, kind), do: true + def record_in_guard?(_, _), do: false + + test "is_record/1,2 (in guard)" do + assert record_in_guard?({User, "john", 27}) + refute record_in_guard?({"user", "john", 27}) + + assert record_in_guard?({User, "john", 27}, User) + refute record_in_guard?({"user", "john", 27}, "user") end - Record.defrecord :timestamp, [:date, :time] - Record.defrecord :user, __MODULE__, name: "José", age: 25 - Record.defrecordp :file_info, Record.extract(:file_info, from_lib: "kernel/include/file.hrl") + Record.defrecord(:timestamp, [:date, :time]) + Record.defrecord(:user, __MODULE__, name: "john", age: 25) + + Record.defrecordp(:file_info, Record.extract(:file_info, from_lib: "kernel/include/file.hrl")) + + Record.defrecordp( + :certificate, + :OTPCertificate, + Record.extract(:OTPCertificate, from_lib: "public_key/include/public_key.hrl") + ) - test "records generates macros that generates tuples" do + test "records are tagged" do + assert elem(file_info(), 0) == :file_info + end + + test "records macros" do record = user() - assert user(record, :name) == "José" - assert user(record, :age) == 25 + assert user(record, :name) == "john" + assert user(record, :age) == 25 - record = user(record, name: "Eric") - assert user(record, :name) == "Eric" + record = user(record, name: "meg") + assert user(record, :name) == "meg" - assert elem(record, user(:name)) == "Eric" + assert elem(record, user(:name)) == "meg" assert elem(record, 0) == RecordTest user(name: name) = record - assert name == "Eric" + assert name == "meg" + + assert user(:name) == 1 end - test "records with no tag" do - assert elem(file_info(), 0) == :file_info + test "records with default values" do + record = user(_: :_, name: "meg") + assert user(record, :name) == "meg" + assert user(record, :age) == :_ + assert match?(user(_: _), user()) + end + + test "records preserve side-effects order" do + user = + user( + age: send(self(), :age), + name: send(self(), :name) + ) + + assert Process.info(self(), :messages) == {:messages, [:age, :name]} + + _ = + user(user, + age: send(self(), :update_age), + name: send(self(), :update_name) + ) + + assert Process.info(self(), :messages) == + {:messages, [:age, :name, :update_age, :update_name]} + end + + test "nested records preserve side-effects order" do + user = + user( + age: + user( + age: send(self(), :inner_age), + name: send(self(), :inner_name) + ), + name: send(self(), :name) + ) + + assert user == {RecordTest, :name, {RecordTest, :inner_name, :inner_age}} + assert for(_ <- 1..3, do: assert_receive(_)) == [:inner_age, :inner_name, :name] + + user = + user( + name: send(self(), :name), + age: + user( + age: send(self(), :inner_age), + name: send(self(), :inner_name) + ) + ) + + assert user == {RecordTest, :name, {RecordTest, :inner_name, :inner_age}} + assert for(_ <- 1..3, do: assert_receive(_)) == [:name, :inner_age, :inner_name] + end + + Record.defrecord( + :defaults, + struct: ~D[2016-01-01], + map: %{}, + tuple_zero: {}, + tuple_one: {1}, + tuple_two: {1, 2}, + tuple_three: {1, 2, 3}, + list: [1, 2, 3], + call: MapSet.new(), + string: "abc", + binary: <<1, 2, 3>>, + charlist: ~c"abc" + ) + + test "records with literal defaults and on-the-fly record" do + assert defaults(defaults()) == [ + struct: ~D[2016-01-01], + map: %{}, + tuple_zero: {}, + tuple_one: {1}, + tuple_two: {1, 2}, + tuple_three: {1, 2, 3}, + list: [1, 2, 3], + call: MapSet.new(), + string: "abc", + binary: <<1, 2, 3>>, + charlist: ~c"abc" + ] + + assert defaults(defaults(), :struct) == ~D[2016-01-01] + assert defaults(defaults(), :map) == %{} + assert defaults(defaults(), :tuple_zero) == {} + assert defaults(defaults(), :tuple_one) == {1} + assert defaults(defaults(), :tuple_two) == {1, 2} + assert defaults(defaults(), :tuple_three) == {1, 2, 3} + assert defaults(defaults(), :list) == [1, 2, 3] + assert defaults(defaults(), :call) == MapSet.new() + assert defaults(defaults(), :string) == "abc" + assert defaults(defaults(), :binary) == <<1, 2, 3>> + assert defaults(defaults(), :charlist) == ~c"abc" + end + + test "records with literal defaults and record in a variable" do + defaults = defaults() + + assert defaults(defaults) == [ + struct: ~D[2016-01-01], + map: %{}, + tuple_zero: {}, + tuple_one: {1}, + tuple_two: {1, 2}, + tuple_three: {1, 2, 3}, + list: [1, 2, 3], + call: MapSet.new(), + string: "abc", + binary: <<1, 2, 3>>, + charlist: ~c"abc" + ] + + assert defaults(defaults, :struct) == ~D[2016-01-01] + assert defaults(defaults, :map) == %{} + assert defaults(defaults, :tuple_zero) == {} + assert defaults(defaults, :tuple_one) == {1} + assert defaults(defaults, :tuple_two) == {1, 2} + assert defaults(defaults, :tuple_three) == {1, 2, 3} + assert defaults(defaults, :list) == [1, 2, 3] + assert defaults(defaults, :call) == MapSet.new() + assert defaults(defaults, :string) == "abc" + assert defaults(defaults, :binary) == <<1, 2, 3>> + assert defaults(defaults, :charlist) == ~c"abc" end test "records with dynamic arguments" do record = file_info() assert file_info(record, :size) == :undefined + + record = user() + assert user(record) == [name: "john", age: 25] + assert user(user()) == [name: "john", age: 25] + + msg = + "expected argument to be a literal atom, literal keyword or a :file_info record, " <> + "got runtime: {RecordTest, \"john\", 25}" + + assert_raise ArgumentError, msg, fn -> + file_info(record) + end + + pretender = {RecordTest, "john"} + + msg = + "expected argument to be a RecordTest record with 2 fields, " <> + "got: {RecordTest, \"john\"}" + + assert_raise ArgumentError, msg, fn -> + user(pretender) + end + + pretender = {RecordTest, "john", 25, []} + + msg = + "expected argument to be a RecordTest record with 2 fields, " <> + "got: {RecordTest, \"john\", 25, []}" + + assert_raise ArgumentError, msg, fn -> + user(pretender) + end end test "records visibility" do @@ -83,6 +291,12 @@ defmodule RecordTest do refute macro_exported?(__MODULE__, :file_info, 1) end + test "records reflection" do + assert %{fields: [:name, :age], kind: :defrecord, name: :user, tag: RecordTest} in @__records__ + + assert %{fields: [:date, :time], kind: :defrecord, name: :timestamp, tag: :timestamp} in @__records__ + end + test "records with no defaults" do record = timestamp() assert timestamp(record, :date) == nil @@ -92,4 +306,94 @@ defmodule RecordTest do assert timestamp(record, :date) == :foo assert timestamp(record, :time) == :bar end + + test "records defined multiple times" do + msg = "cannot define record :r because a definition r/0 already exists" + + assert_raise ArgumentError, msg, fn -> + defmodule M do + import Record + defrecord :r, [:a] + defrecord :r, [:a] + end + end + end + + test "macro and record with the same name defined" do + msg = "cannot define record :a because a definition a/1 already exists" + + assert_raise ArgumentError, msg, fn -> + defmodule M do + defmacro a(_) do + end + + require Record + Record.defrecord(:a, [:a]) + end + end + + msg = "cannot define record :a because a definition a/2 already exists" + + assert_raise ArgumentError, msg, fn -> + defmodule M do + defmacro a(_, _) do + end + + require Record + Record.defrecord(:a, [:a]) + end + end + end + + test "docs metadata" do + import PathHelpers + + write_beam( + defmodule Metadata do + Record.defrecord(:user, foo: 0, bar: "baz") + end + ) + + {:docs_v1, 352, :elixir, "text/markdown", _, %{}, docs} = Code.fetch_docs(RecordTest.Metadata) + {{:macro, :user, 1}, _meta, _sig, _docs, metadata} = List.keyfind(docs, {:macro, :user, 1}, 0) + assert %{record: {:user, [foo: 0, bar: "baz"]}} = metadata + end + + describe "warnings" do + import ExUnit.CaptureIO + + test "warns on bad record update input" do + assert capture_io(:stderr, fn -> + defmodule RecordSample do + require Record + Record.defrecord(:user, __MODULE__, name: "john", age: 25) + + def fun do + user(user(), _: :_, name: "meg") + end + end + end) =~ + "updating a record with a default (:_) is equivalent to creating a new record" + after + purge(RecordSample) + end + + test "defrecord warns with duplicate keys" do + assert capture_io(:stderr, fn -> + Code.eval_string(""" + defmodule RecordSample do + import Record + defrecord :r, [:foo, :bar, foo: 1] + end + """) + end) =~ "duplicate key :foo found in record" + after + purge(RecordSample) + end + + defp purge(module) when is_atom(module) do + :code.purge(module) + :code.delete(module) + end + end end diff --git a/lib/elixir/test/elixir/regex_test.exs b/lib/elixir/test/elixir/regex_test.exs index 702c7594f66..926c53435c1 100644 --- a/lib/elixir/test/elixir/regex_test.exs +++ b/lib/elixir/test/elixir/regex_test.exs @@ -1,47 +1,80 @@ -Code.require_file "test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("test_helper.exs", __DIR__) defmodule RegexTest do use ExUnit.Case, async: true - test :multiline do - refute Regex.match?(~r/^b$/, "a\nb\nc") - assert Regex.match?(~r/^b$/m, "a\nb\nc") + doctest Regex + + test "module attribute" do + defmodule ModAttr do + @regex ~r/example/ + def regex, do: @regex + + @bare_regex :erlang.term_to_binary(@regex) + def bare_regex, do: :erlang.binary_to_term(@bare_regex) + + # We don't rewrite outside of functions + assert @regex.re_pattern == :erlang.binary_to_term(@bare_regex).re_pattern + end + + if System.otp_release() >= "28" do + assert ModAttr.regex().re_pattern != ModAttr.bare_regex().re_pattern + else + assert ModAttr.regex().re_pattern == ModAttr.bare_regex().re_pattern + end end - test :precedence do - assert {"aa", :unknown} |> elem(0) =~ ~r/(a)\1/ + @tag :re_import + test "module attribute in match context" do + assert_raise( + ArgumentError, + ~r/escaped Regex structs are not allowed in match or guards/, + fn -> + Code.eval_quoted( + quote do + defmodule ModAttrGuard do + @regex ~r/example/ + def example?(@regex), do: true + def example?(_), do: false + end + end + ) + end + ) end - test :backreference do - assert "aa" =~ ~r/(a)\1/ + test "multiline" do + refute Regex.match?(~r/^b$/, "a\nb\nc") + assert Regex.match?(~r/^b$/m, "a\nb\nc") end - test :compile! do - assert Regex.regex?(Regex.compile!("foo")) + @tag :re_import + test "export" do + # exported patterns have no structs, so these are structurally equal + assert ~r/foo/E == Regex.compile!("foo", [:export]) - assert_raise Regex.CompileError, ~r/position 0$/, fn -> - Regex.compile!("*foo") - end - end + assert Regex.match?(~r/foo/E, "foo") + refute Regex.match?(~r/foo/E, "Foo") - test :compile do - {:ok, regex} = Regex.compile("foo") - assert Regex.regex?(regex) - assert {:error, _} = Regex.compile("*foo") - assert {:error, _} = Regex.compile("foo", "y") + assert Regex.run(~r/c(d)/E, "abcd") == ["cd", "d"] + assert Regex.run(~r/e/E, "abcd") == nil + + assert Regex.names(~r/(?foo)/E) == ["FOO"] end - test :compile_with_erl_opts do - {:ok, regex} = Regex.compile("foo\\sbar", [:dotall, {:newline, :anycrlf}]) - assert "foo\nbar" =~ regex + test "precedence" do + assert {"aa", :unknown} |> elem(0) =~ ~r/(a)\1/ end - test :regex? do - assert Regex.regex?(~r/foo/) - refute Regex.regex?(0) + test "backreference" do + assert "aa" =~ ~r/(a)\1/ end - test :source do + test "source" do src = "foo" assert Regex.source(Regex.compile!(src)) == src assert Regex.source(~r/#{src}/) == src @@ -55,53 +88,85 @@ defmodule RegexTest do assert Regex.source(~r/#{src}/) == src end - test :literal_source do + test "literal source" do assert Regex.source(Regex.compile!("foo")) == "foo" assert Regex.source(~r"foo") == "foo" - assert Regex.re_pattern(Regex.compile!("foo")) - == Regex.re_pattern(~r"foo") assert Regex.source(Regex.compile!("\a\b\d\e\f\n\r\s\t\v")) == "\a\b\d\e\f\n\r\s\t\v" - assert Regex.source(~r<\a\b\d\e\f\n\r\s\t\v>) == "\a\\b\\d\\e\f\n\r\\s\t\v" - assert Regex.re_pattern(Regex.compile!("\a\b\d\e\f\n\r\s\t\v")) - == Regex.re_pattern(~r"\a\010\177\033\f\n\r \t\v") + assert Regex.source(~r<\a\b\d\e\f\n\r\s\t\v>) == "\\a\\b\\d\\e\\f\\n\\r\\s\\t\\v" + end + + test "Unicode" do + assert "olá" =~ ~r"\p{Latin}$"u + refute "£" =~ ~r/\p{Lu}/u - assert Regex.source(Regex.compile!("\a\\b\\d\e\f\n\r\\s\t\v")) == "\a\\b\\d\e\f\n\r\\s\t\v" - assert Regex.source(~r<\a\\b\\d\\e\f\n\r\\s\t\v>) == "\a\\\\b\\\\d\\\\e\f\n\r\\\\s\t\v" - assert Regex.re_pattern(Regex.compile!("\a\\b\\d\e\f\n\r\\s\t\v")) - == Regex.re_pattern(~r"\a\b\d\e\f\n\r\s\t\v") + # Non breaking space matches [[:space:]] with Unicode + assert <<0xA0::utf8>> =~ ~r/[[:space:]]/u + assert <<0xA0::utf8>> =~ ~r/\s/u + assert <>> =~ ~r/<.>/ end - test :opts do - assert Regex.opts(Regex.compile!("foo", "i")) == "i" + test "ungreedy" do + assert Regex.run(~r/[\d ]+/, "1 2 3 4 5"), ["1 2 3 4 5"] + assert Regex.run(~r/[\d ]?+/, "1 2 3 4 5"), ["1"] + assert Regex.run(~r/[\d ]+/U, "1 2 3 4 5"), ["1"] end - test :unicode do - assert "josé" =~ ~r"\p{Latin}$"u - refute "£" =~ ~r/\p{Lu}/u + test "compile/1" do + {:ok, %Regex{}} = Regex.compile("foo") + assert {:error, _} = Regex.compile("*foo") + assert {:error, _} = Regex.compile("foo", "y") + assert {:error, _} = Regex.compile("foo", "uy") + end - assert <>> =~ ~r/<.>/ - refute <>> =~ ~r/<.>/u + test "compile/1 with Erlang options" do + {:ok, regex} = Regex.compile("foo\\sbar", [:dotall, {:newline, :anycrlf}]) + assert "foo\nbar" =~ regex + end + + test "compile!/1" do + assert %Regex{} = Regex.compile!("foo") + + assert_raise Regex.CompileError, ~r/position 0$/, fn -> + Regex.compile!("*foo") + end + end + + test "import/1" do + # no-op for non-exported regexes + regex = ~r/foo/ + assert Regex.import(regex) == regex + + imported = Regex.import(~r/foo/E) + + assert imported.opts == [] + assert "foo" =~ imported + assert {:re_pattern, _, _, _, _} = imported.re_pattern end - test :names do + test "opts/1" do + assert Regex.opts(Regex.compile!("foo", "i")) == [:caseless] + assert Regex.opts(Regex.compile!("foo", [:ucp])) == [:ucp] + end + + test "names/1" do assert Regex.names(~r/(?foo)/) == ["FOO"] end - test :match? do + test "match?/2" do assert Regex.match?(~r/foo/, "foo") refute Regex.match?(~r/foo/, "FOO") assert Regex.match?(~r/foo/i, "FOO") assert Regex.match?(~r/\d{1,3}/i, "123") - assert Regex.match?(~r/foo/, "afooa") + assert Regex.match?(~r/foo/, "afooa") refute Regex.match?(~r/^foo/, "afooa") - assert Regex.match?(~r/^foo/, "fooa") + assert Regex.match?(~r/^foo/, "fooa") refute Regex.match?(~r/foo$/, "afooa") - assert Regex.match?(~r/foo$/, "afoo") + assert Regex.match?(~r/foo$/, "afoo") end - test :named_captures do + test "named_captures/2" do assert Regex.named_captures(~r/(?c)(?d)/, "abcd") == %{"bar" => "d", "foo" => "c"} assert Regex.named_captures(~r/c(?d)/, "abcd") == %{"foo" => "d"} assert Regex.named_captures(~r/c(?d)/, "no_match") == nil @@ -109,61 +174,142 @@ defmodule RegexTest do assert Regex.named_captures(~r/c(.)/, "cat") == %{} end - test :sigil_R do - assert Regex.match?(~R/f#{1,3}o/, "f#o") - end - - test :run do + test "run/2" do assert Regex.run(~r"c(d)", "abcd") == ["cd", "d"] assert Regex.run(~r"e", "abcd") == nil end - test :run_with_all_names do + test "run/3 with :all_names as the value of the :capture option" do assert Regex.run(~r/c(?d)/, "abcd", capture: :all_names) == ["d"] assert Regex.run(~r/c(?d)/, "no_match", capture: :all_names) == nil assert Regex.run(~r/c(?d|e)/, "abcd abce", capture: :all_names) == ["d"] end - test :run_with_indexes do + test "run/3 with :index as the value of the :return option" do assert Regex.run(~r"c(d)", "abcd", return: :index) == [{2, 2}, {3, 1}] assert Regex.run(~r"e", "abcd", return: :index) == nil end - test :scan do + test "run/3 with :offset" do + assert Regex.run(~r"^foo", "foobar", offset: 0) == ["foo"] + assert Regex.run(~r"^foo", "foobar", offset: 2) == nil + assert Regex.run(~r"^foo", "foobar", offset: 2, return: :index) == nil + assert Regex.run(~r"bar", "foobar", offset: 2, return: :index) == [{3, 3}] + end + + test "scan/2" do assert Regex.scan(~r"c(d|e)", "abcd abce") == [["cd", "d"], ["ce", "e"]] assert Regex.scan(~r"c(?:d|e)", "abcd abce") == [["cd"], ["ce"]] assert Regex.scan(~r"e", "abcd") == [] end - test :scan_with_all_names do + test "scan/2 with :all_names as the value of the :capture option" do assert Regex.scan(~r/cd/, "abcd", capture: :all_names) == [] assert Regex.scan(~r/c(?d)/, "abcd", capture: :all_names) == [["d"]] assert Regex.scan(~r/c(?d)/, "no_match", capture: :all_names) == [] assert Regex.scan(~r/c(?d|e)/, "abcd abce", capture: :all_names) == [["d"], ["e"]] end - test :split do + test "scan/2 with :offset" do + assert Regex.scan(~r"^foo", "foobar", offset: 0) == [["foo"]] + assert Regex.scan(~r"^foo", "foobar", offset: 1) == [] + end + + test "split/2,3" do assert Regex.split(~r",", "") == [""] + assert Regex.split(~r",", "", trim: true) == [] + assert Regex.split(~r",", "", trim: true, parts: 2) == [] + + assert Regex.split(~r"=", "key=") == ["key", ""] + assert Regex.split(~r"=", "=value") == ["", "value"] + assert Regex.split(~r" ", "foo bar baz") == ["foo", "bar", "baz"] - assert Regex.split(~r" ", "foo bar baz", parts: 0) == ["foo", "bar", "baz"] assert Regex.split(~r" ", "foo bar baz", parts: :infinity) == ["foo", "bar", "baz"] assert Regex.split(~r" ", "foo bar baz", parts: 10) == ["foo", "bar", "baz"] assert Regex.split(~r" ", "foo bar baz", parts: 2) == ["foo", "bar baz"] - assert Regex.split(~r"\s", "foobar") == ["foobar"] + assert Regex.split(~r" ", " foo bar baz ") == ["", "foo", "bar", "baz", ""] assert Regex.split(~r" ", " foo bar baz ", trim: true) == ["foo", "bar", "baz"] - assert Regex.split(~r"=", "key=") == ["key", ""] - assert Regex.split(~r"=", "=value") == ["", "value"] + assert Regex.split(~r" ", " foo bar baz ", parts: 2) == ["", "foo bar baz "] + assert Regex.split(~r" ", " foo bar baz ", trim: true, parts: 2) == ["foo", "bar baz "] + + assert Regex.split(~r/b\K/, "ababab") == ["ab", "ab", "ab", ""] + end + + test "split/3 with the :on option" do + assert Regex.split(~r/()abc()/, "xabcxabcx", on: :none) == ["xabcxabcx"] + + parts = ["x", "abc", "x", "abc", "x"] + assert Regex.split(~r/()abc()/, "xabcxabcx", on: :all_but_first) == parts + + assert Regex.split(~r/(?)abc(?)/, "xabcxabcx", on: [:first, :last]) == parts + + parts = ["xabc", "xabc", "x"] + assert Regex.split(~r/(?)abc(?)/, "xabcxabcx", on: [:last, :first]) == parts + + assert Regex.split(~r/a(?b)c/, "abc", on: [:second]) == ["a", "c"] + + parts = ["a", "c adc a", "c"] + assert Regex.split(~r/a(?b)c|a(?d)c/, "abc adc abc", on: [:second]) == parts + + assert Regex.split(~r/a(?b)c|a(?d)c/, "abc adc abc", on: [:second, :fourth]) == + ["a", "c a", "c a", "c"] end - test :replace do - assert Regex.replace(~r(d), "abc", "d") == "abc" - assert Regex.replace(~r(b), "abc", "d") == "adc" - assert Regex.replace(~r(b), "abc", "[\\0]") == "a[b]c" + test "split/3 with the :include_captures option" do + assert Regex.split(~r/([ln])/, "Erlang", include_captures: true) == ["Er", "l", "a", "n", "g"] + assert Regex.split(~r/([kw])/, "Elixir", include_captures: true) == ["Elixir"] + + assert Regex.split(~r/([Ee]lixir)/, "Elixir", include_captures: true, trim: true) == + ["Elixir"] + + assert Regex.split(~r/([Ee]lixir)/, "Elixir", include_captures: true, trim: false) == + ["", "Elixir", ""] + + assert Regex.split(~r//, "abc", include_captures: true) == + ["", "", "a", "", "b", "", "c", "", ""] + + assert Regex.split(~r/a/, "abc", include_captures: true) == ["", "a", "bc"] + assert Regex.split(~r/c/, "abc", include_captures: true) == ["ab", "c", ""] + + assert Regex.split(~r/[Ei]/, "Elixir", include_captures: true, parts: 2) == + ["", "E", "lixir"] + + assert Regex.split(~r/[Ei]/, "Elixir", include_captures: true, parts: 3) == + ["", "E", "l", "i", "xir"] + + assert Regex.split(~r/[Ei]/, "Elixir", include_captures: true, parts: 2, trim: true) == + ["E", "lixir"] + + assert Regex.split(~r/[Ei]/, "Elixir", include_captures: true, parts: 3, trim: true) == + ["E", "l", "i", "xir"] + + assert Regex.split(~r/b\Kc/, "abcabc", include_captures: true) == ["ab", "c", "ab", "c", ""] + assert Regex.split(~r/(b\K)/, "abab", include_captures: true) == ["ab", "", "ab", "", ""] + + assert Regex.split(~r/(b\K)/, "abab", include_captures: true, trim: true) == [ + "ab", + "", + "ab", + "" + ] + end + + test "replace/3,4" do + assert Regex.replace(~r/d/, "abc", "d") == "abc" + assert Regex.replace(~r/b/, "abc", "d") == "adc" + assert Regex.replace(~r/b/, "abc", "[\\0]") == "a[b]c" assert Regex.replace(~r[(b)], "abc", "[\\1]") == "a[b]c" + assert Regex.replace(~r[(b)], "abc", "[\\2]") == "a[]c" + assert Regex.replace(~r[(b)], "abc", "[\\3]") == "a[]c" + assert Regex.replace(~r/b/, "abc", "[\\g{0}]") == "a[b]c" + assert Regex.replace(~r[(b)], "abc", "[\\g{1}]") == "a[b]c" + + assert Regex.replace(~r/b/, "abcbe", "d") == "adcde" + assert Regex.replace(~r/b/, "abcbe", "d", global: false) == "adcbe" - assert Regex.replace(~r(b), "abcbe", "d") == "adcde" - assert Regex.replace(~r(b), "abcbe", "d", global: false) == "adcbe" + assert Regex.replace(~r/ /, "first third", "\\second\\") == "first\\second\\third" + assert Regex.replace(~r/ /, "first third", "\\\\second\\\\") == "first\\second\\third" assert Regex.replace(~r[a(b)c], "abcabc", fn -> "ac" end) == "acac" assert Regex.replace(~r[a(b)c], "abcabc", fn "abc" -> "ac" end) == "acac" @@ -172,7 +318,7 @@ defmodule RegexTest do assert Regex.replace(~r[a(b)c], "abcabc", fn "abc", "b" -> "ac" end, global: false) == "acabc" end - test :escape do + test "escape" do assert matches_escaped?(".") refute matches_escaped?(".", "x") @@ -190,10 +336,18 @@ defmodule RegexTest do assert matches_escaped?("\\A \\z") assert matches_escaped?(" x ") - assert matches_escaped?("  x    x ") # unicode spaces here + # Unicode spaces here + assert matches_escaped?("  x    x ") assert matches_escaped?("# lol") - assert matches_escaped?("\\A.^$*+?()[{\\| \t\n\xff\\z #hello\x{202F}\x{205F}") + assert matches_escaped?("\\A.^$*+?()[{\\| \t\n\x20\\z #hello\u202F\u205F") + assert Regex.match?(Regex.compile!("[" <> Regex.escape("!-#") <> "]"), "-") + + assert Regex.escape("{}") == "\\{\\}" + assert Regex.escape("[]") == "\\[\\]" + + assert Regex.escape("{foo}") == "\\{foo\\}" + assert Regex.escape("[foo]") == "\\[foo\\]" end defp matches_escaped?(string) do @@ -201,6 +355,6 @@ defmodule RegexTest do end defp matches_escaped?(string, match) do - Regex.match? ~r/#{Regex.escape(string)}/simxu, match + Regex.match?(~r/#{Regex.escape(string)}/simx, match) end end diff --git a/lib/elixir/test/elixir/registry/duplicate_test.exs b/lib/elixir/test/elixir/registry/duplicate_test.exs new file mode 100644 index 00000000000..00f1c6191a5 --- /dev/null +++ b/lib/elixir/test/elixir/registry/duplicate_test.exs @@ -0,0 +1,504 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Registry.DuplicateTest do + use ExUnit.Case, + async: true, + parameterize: + for( + keys <- [:duplicate, {:duplicate, :pid}, {:duplicate, :key}], + partitions <- [1, 8], + do: %{keys: keys, partitions: partitions} + ) + + setup config do + keys = config.keys + partitions = config.partitions + + listeners = + List.wrap(config[:base_listener]) |> Enum.map(&:"#{&1}_#{partitions}_#{inspect(keys)}") + + name = :"#{config.test}_#{partitions}_#{inspect(keys)}" + opts = [keys: config.keys, name: name, partitions: partitions, listeners: listeners] + {:ok, _} = start_supervised({Registry, opts}) + %{registry: name, listeners: listeners} + end + + test "starts configured number of partitions", %{registry: registry, partitions: partitions} do + assert length(Supervisor.which_children(registry)) == partitions + end + + test "counts 0 keys in an empty registry", %{registry: registry} do + assert 0 == Registry.count(registry) + end + + test "counts the number of keys in a registry", %{registry: registry} do + {:ok, _} = Registry.register(registry, "hello", :value) + {:ok, _} = Registry.register(registry, "hello", :value) + + assert 2 == Registry.count(registry) + end + + test "has duplicate registrations", %{registry: registry} do + {:ok, pid} = Registry.register(registry, "hello", :value) + assert is_pid(pid) + assert Registry.keys(registry, self()) == ["hello"] + assert Registry.values(registry, "hello", self()) == [:value] + + assert {:ok, pid} = Registry.register(registry, "hello", :value) + assert is_pid(pid) + assert Registry.keys(registry, self()) == ["hello", "hello"] + assert Registry.values(registry, "hello", self()) == [:value, :value] + + {:ok, pid} = Registry.register(registry, "world", :value) + assert is_pid(pid) + assert Registry.keys(registry, self()) |> Enum.sort() == ["hello", "hello", "world"] + end + + test "has duplicate registrations across processes", %{registry: registry} do + {_, task} = register_task(registry, "hello", :world) + assert Registry.keys(registry, self()) == [] + assert Registry.keys(registry, task) == ["hello"] + assert Registry.values(registry, "hello", self()) == [] + assert Registry.values(registry, "hello", task) == [:world] + + assert {:ok, _pid} = Registry.register(registry, "hello", :value) + assert Registry.keys(registry, self()) == ["hello"] + assert Registry.values(registry, "hello", self()) == [:value] + end + + test "compares using matches", %{registry: registry} do + {:ok, _} = Registry.register(registry, 1.0, :value) + {:ok, _} = Registry.register(registry, 1, :value) + assert Registry.keys(registry, self()) |> Enum.sort() == [1, 1.0] + end + + test "dispatches to multiple keys in serial", %{registry: registry} do + Process.flag(:trap_exit, true) + parent = self() + + fun = fn _ -> raise "will never be invoked" end + assert Registry.dispatch(registry, "hello", fun, parallel: false) == :ok + + {:ok, _} = Registry.register(registry, "hello", :value1) + {:ok, _} = Registry.register(registry, "hello", :value2) + {:ok, _} = Registry.register(registry, "world", :value3) + + fun = fn entries -> + assert parent == self() + for {pid, value} <- entries, do: send(pid, {:dispatch, value}) + end + + assert Registry.dispatch(registry, "hello", fun, parallel: false) + + assert_received {:dispatch, :value1} + assert_received {:dispatch, :value2} + refute_received {:dispatch, :value3} + + fun = fn entries -> + assert parent == self() + for {pid, value} <- entries, do: send(pid, {:dispatch, value}) + end + + assert Registry.dispatch(registry, "world", fun, parallel: false) + + refute_received {:dispatch, :value1} + refute_received {:dispatch, :value2} + assert_received {:dispatch, :value3} + + refute_received {:EXIT, _, _} + end + + test "dispatches to multiple keys in parallel", context do + %{registry: registry, partitions: partitions} = context + Process.flag(:trap_exit, true) + parent = self() + + fun = fn _ -> raise "will never be invoked" end + assert Registry.dispatch(registry, "hello", fun, parallel: true) == :ok + + {:ok, _} = Registry.register(registry, "hello", :value1) + {:ok, _} = Registry.register(registry, "hello", :value2) + {:ok, _} = Registry.register(registry, "world", :value3) + + fun = fn entries -> + if partitions == 8 do + assert parent != self() + else + assert parent == self() + end + + for {pid, value} <- entries, do: send(pid, {:dispatch, value}) + end + + assert Registry.dispatch(registry, "hello", fun, parallel: true) + + assert_received {:dispatch, :value1} + assert_received {:dispatch, :value2} + refute_received {:dispatch, :value3} + + fun = fn entries -> + if partitions == 8 do + assert parent != self() + else + assert parent == self() + end + + for {pid, value} <- entries, do: send(pid, {:dispatch, value}) + end + + assert Registry.dispatch(registry, "world", fun, parallel: true) + + refute_received {:dispatch, :value1} + refute_received {:dispatch, :value2} + assert_received {:dispatch, :value3} + + refute_received {:EXIT, _, _} + end + + test "unregisters by key", %{registry: registry} do + {:ok, _} = Registry.register(registry, "hello", :value) + {:ok, _} = Registry.register(registry, "hello", :value) + {:ok, _} = Registry.register(registry, "world", :value) + assert Registry.keys(registry, self()) |> Enum.sort() == ["hello", "hello", "world"] + + :ok = Registry.unregister(registry, "hello") + assert Registry.keys(registry, self()) == ["world"] + + :ok = Registry.unregister(registry, "world") + assert Registry.keys(registry, self()) == [] + end + + test "unregisters with no entries", %{registry: registry} do + assert Registry.unregister(registry, "hello") == :ok + end + + test "unregisters with tricky keys", %{registry: registry} do + {:ok, _} = Registry.register(registry, :_, :foo) + {:ok, _} = Registry.register(registry, :_, :bar) + {:ok, _} = Registry.register(registry, "hello", "a") + {:ok, _} = Registry.register(registry, "hello", "b") + + Registry.unregister(registry, :_) + assert Registry.keys(registry, self()) |> Enum.sort() == ["hello", "hello"] + end + + test "supports match patterns", %{registry: registry} do + value1 = {1, :atom, 1} + value2 = {2, :atom, 2} + + {:ok, _} = Registry.register(registry, "hello", value1) + {:ok, _} = Registry.register(registry, "hello", value2) + + assert Registry.match(registry, "hello", {1, :_, :_}) == [{self(), value1}] + assert Registry.match(registry, "hello", {1.0, :_, :_}) == [] + + assert Registry.match(registry, "hello", {:_, :atom, :_}) |> Enum.sort() == + [{self(), value1}, {self(), value2}] + + assert Registry.match(registry, "hello", {:"$1", :_, :"$1"}) |> Enum.sort() == + [{self(), value1}, {self(), value2}] + + assert Registry.match(registry, "hello", {2, :_, :_}) == [{self(), value2}] + assert Registry.match(registry, "hello", {2.0, :_, :_}) == [] + end + + test "supports guards", %{registry: registry} do + value1 = {1, :atom, 1} + value2 = {2, :atom, 2} + + {:ok, _} = Registry.register(registry, "hello", value1) + {:ok, _} = Registry.register(registry, "hello", value2) + + assert Registry.match(registry, "hello", {:"$1", :_, :_}, [{:<, :"$1", 2}]) == + [{self(), value1}] + + assert Registry.match(registry, "hello", {:"$1", :_, :_}, [{:>, :"$1", 3}]) == [] + + assert Registry.match(registry, "hello", {:"$1", :_, :_}, [{:<, :"$1", 3}]) |> Enum.sort() == + [{self(), value1}, {self(), value2}] + + assert Registry.match(registry, "hello", {:_, :"$1", :_}, [{:is_atom, :"$1"}]) + |> Enum.sort() == [{self(), value1}, {self(), value2}] + end + + test "count_match supports match patterns", %{registry: registry} do + value = {1, :atom, 1} + {:ok, _} = Registry.register(registry, "hello", value) + assert 1 == Registry.count_match(registry, "hello", {1, :_, :_}) + assert 0 == Registry.count_match(registry, "hello", {1.0, :_, :_}) + assert 1 == Registry.count_match(registry, "hello", {:_, :atom, :_}) + assert 1 == Registry.count_match(registry, "hello", {:"$1", :_, :"$1"}) + assert 1 == Registry.count_match(registry, "hello", :_) + assert 0 == Registry.count_match(registry, :_, :_) + + value2 = %{a: "a", b: "b"} + {:ok, _} = Registry.register(registry, "world", value2) + assert 1 == Registry.count_match(registry, "world", %{b: "b"}) + end + + test "count_match supports guard conditions", %{registry: registry} do + value = {1, :atom, 2} + {:ok, _} = Registry.register(registry, "hello", value) + + assert 1 == Registry.count_match(registry, "hello", {:_, :_, :"$1"}, [{:>, :"$1", 1}]) + assert 0 == Registry.count_match(registry, "hello", {:_, :_, :"$1"}, [{:>, :"$1", 2}]) + assert 1 == Registry.count_match(registry, "hello", {:_, :"$1", :_}, [{:is_atom, :"$1"}]) + end + + test "unregister_match supports patterns", %{registry: registry} do + value1 = {1, :atom, 1} + value2 = {2, :atom, 2} + + {:ok, _} = Registry.register(registry, "hello", value1) + {:ok, _} = Registry.register(registry, "hello", value2) + + Registry.unregister_match(registry, "hello", {2, :_, :_}) + assert Registry.lookup(registry, "hello") == [{self(), value1}] + + {:ok, _} = Registry.register(registry, "hello", value2) + Registry.unregister_match(registry, "hello", {2.0, :_, :_}) + assert Registry.lookup(registry, "hello") == [{self(), value1}, {self(), value2}] + Registry.unregister_match(registry, "hello", {:_, :atom, :_}) + assert Registry.lookup(registry, "hello") == [] + end + + test "unregister_match supports guards", %{registry: registry} do + value1 = {1, :atom, 1} + value2 = {2, :atom, 2} + + {:ok, _} = Registry.register(registry, "hello", value1) + {:ok, _} = Registry.register(registry, "hello", value2) + + Registry.unregister_match(registry, "hello", {:"$1", :_, :_}, [{:<, :"$1", 2}]) + assert Registry.lookup(registry, "hello") == [{self(), value2}] + end + + test "unregister_match supports tricky keys", %{registry: registry} do + {:ok, _} = Registry.register(registry, :_, :foo) + {:ok, _} = Registry.register(registry, :_, :bar) + {:ok, _} = Registry.register(registry, "hello", "a") + {:ok, _} = Registry.register(registry, "hello", "b") + + Registry.unregister_match(registry, :_, :foo) + assert Registry.lookup(registry, :_) == [{self(), :bar}] + + assert Registry.keys(registry, self()) |> Enum.sort() == [:_, "hello", "hello"] + end + + @tag base_listener: :unique_listener + test "allows listeners", %{registry: registry, listeners: [listener]} do + Process.register(self(), listener) + {_, task} = register_task(registry, "hello", :world) + assert_received {:register, ^registry, "hello", ^task, :world} + + self = self() + {:ok, _} = Registry.register(registry, "hello", :value) + assert_received {:register, ^registry, "hello", ^self, :value} + + :ok = Registry.unregister(registry, "hello") + assert_received {:unregister, ^registry, "hello", ^self} + after + Process.unregister(listener) + end + + test "links and unlinks on register/unregister", %{registry: registry} do + {:ok, pid} = Registry.register(registry, "hello", :value) + {:links, links} = Process.info(self(), :links) + assert pid in links + + {:ok, pid} = Registry.register(registry, "world", :value) + {:links, links} = Process.info(self(), :links) + assert pid in links + + :ok = Registry.unregister(registry, "hello") + {:links, links} = Process.info(self(), :links) + assert pid in links + + :ok = Registry.unregister(registry, "world") + {:links, links} = Process.info(self(), :links) + refute pid in links + end + + test "raises on unknown registry name" do + assert_raise ArgumentError, ~r/unknown registry/, fn -> + Registry.register(:unknown, "hello", :value) + end + end + + test "raises if attempt to be used on via", %{registry: registry} do + assert_raise ArgumentError, ":via is not supported for duplicate registries", fn -> + name = {:via, Registry, {registry, "hello"}} + Agent.start_link(fn -> 0 end, name: name) + end + end + + test "empty list for empty registry", %{registry: registry} do + assert Registry.select(registry, [{{:_, :_, :_}, [], [:"$_"]}]) == [] + end + + test "select all", %{registry: registry} do + {:ok, _} = Registry.register(registry, "hello", :value) + {:ok, _} = Registry.register(registry, "hello", :value) + + assert Registry.select(registry, [{{:"$1", :"$2", :"$3"}, [], [{{:"$1", :"$2", :"$3"}}]}]) + |> Enum.sort() == + [{"hello", self(), :value}, {"hello", self(), :value}] + end + + test "select supports full match specs", %{registry: registry} do + value = {1, :atom, 1} + {:ok, _} = Registry.register(registry, "hello", value) + + assert [{"hello", self(), value}] == + Registry.select(registry, [ + {{"hello", :"$2", :"$3"}, [], [{{"hello", :"$2", :"$3"}}]} + ]) + + assert [{"hello", self(), value}] == + Registry.select(registry, [ + {{:"$1", self(), :"$3"}, [], [{{:"$1", self(), :"$3"}}]} + ]) + + assert [{"hello", self(), value}] == + Registry.select(registry, [ + {{:"$1", :"$2", value}, [], [{{:"$1", :"$2", {value}}}]} + ]) + + assert [] == + Registry.select(registry, [ + {{"world", :"$2", :"$3"}, [], [{{"world", :"$2", :"$3"}}]} + ]) + + assert [] == Registry.select(registry, [{{:"$1", :"$2", {1.0, :_, :_}}, [], [:"$_"]}]) + + assert [{"hello", self(), value}] == + Registry.select(registry, [ + {{:"$1", :"$2", {:"$3", :atom, :"$4"}}, [], + [{{:"$1", :"$2", {{:"$3", :atom, :"$4"}}}}]} + ]) + + assert [{"hello", self(), {1, :atom, 1}}] == + Registry.select(registry, [ + {{:"$1", :"$2", {:"$3", :"$4", :"$3"}}, [], + [{{:"$1", :"$2", {{:"$3", :"$4", :"$3"}}}}]} + ]) + + value2 = %{a: "a", b: "b"} + {:ok, _} = Registry.register(registry, "world", value2) + + assert [:match] == + Registry.select(registry, [{{"world", self(), %{b: "b"}}, [], [:match]}]) + + assert ["hello", "world"] == + Registry.select(registry, [{{:"$1", :_, :_}, [], [:"$1"]}]) |> Enum.sort() + end + + test "select supports guard conditions", %{registry: registry} do + value = {1, :atom, 2} + {:ok, _} = Registry.register(registry, "hello", value) + + assert [{"hello", self(), {1, :atom, 2}}] == + Registry.select(registry, [ + {{:"$1", :"$2", {:"$3", :"$4", :"$5"}}, [{:>, :"$5", 1}], + [{{:"$1", :"$2", {{:"$3", :"$4", :"$5"}}}}]} + ]) + + assert [] == + Registry.select(registry, [ + {{:_, :_, {:_, :_, :"$1"}}, [{:>, :"$1", 2}], [:"$_"]} + ]) + + assert ["hello"] == + Registry.select(registry, [ + {{:"$1", :_, {:_, :"$2", :_}}, [{:is_atom, :"$2"}], [:"$1"]} + ]) + end + + test "select allows multiple specs", %{registry: registry} do + {:ok, _} = Registry.register(registry, "hello", :value) + {:ok, _} = Registry.register(registry, "world", :value) + + assert ["hello", "world"] == + Registry.select(registry, [ + {{"hello", :_, :_}, [], [{:element, 1, :"$_"}]}, + {{"world", :_, :_}, [], [{:element, 1, :"$_"}]} + ]) + |> Enum.sort() + end + + test "count_select supports match specs", %{registry: registry} do + value = {1, :atom, 1} + {:ok, _} = Registry.register(registry, "hello", value) + assert 1 == Registry.count_select(registry, [{{:_, :_, value}, [], [true]}]) + assert 1 == Registry.count_select(registry, [{{"hello", :_, :_}, [], [true]}]) + assert 1 == Registry.count_select(registry, [{{:_, :_, {1, :atom, :_}}, [], [true]}]) + assert 1 == Registry.count_select(registry, [{{:_, :_, {:"$1", :_, :"$1"}}, [], [true]}]) + assert 0 == Registry.count_select(registry, [{{"hello", :_, nil}, [], [true]}]) + + value2 = %{a: "a", b: "b"} + {:ok, _} = Registry.register(registry, "world", value2) + assert 1 == Registry.count_select(registry, [{{"world", :_, :_}, [], [true]}]) + end + + test "count_select supports guard conditions", %{registry: registry} do + value = {1, :atom, 2} + {:ok, _} = Registry.register(registry, "hello", value) + + assert 1 == + Registry.count_select(registry, [ + {{:_, :_, {:_, :"$1", :_}}, [{:is_atom, :"$1"}], [true]} + ]) + + assert 1 == + Registry.count_select(registry, [ + {{:_, :_, {:_, :_, :"$1"}}, [{:>, :"$1", 1}], [true]} + ]) + + assert 0 == + Registry.count_select(registry, [ + {{:_, :_, {:_, :_, :"$1"}}, [{:>, :"$1", 2}], [true]} + ]) + end + + test "count_select allows multiple specs", %{registry: registry} do + {:ok, _} = Registry.register(registry, "hello", :value) + {:ok, _} = Registry.register(registry, "world", :value) + + assert 2 == + Registry.count_select(registry, [ + {{"hello", :_, :_}, [], [true]}, + {{"world", :_, :_}, [], [true]} + ]) + end + + test "rejects invalid tuple syntax", %{partitions: partitions} do + name = :"test_invalid_tuple_#{partitions}" + + assert_raise ArgumentError, ~r/expected :keys to be given and be one of/, fn -> + Registry.start_link(keys: {:duplicate, :invalid}, name: name, partitions: partitions) + end + end + + test "update_value is not supported", %{registry: registry} do + assert_raise ArgumentError, ~r/Registry.update_value\/3 is not supported/, fn -> + Registry.update_value(registry, "hello", fn val -> val end) + end + end + + defp register_task(registry, key, value) do + parent = self() + + {:ok, task} = + Task.start(fn -> + send(parent, Registry.register(registry, key, value)) + Process.sleep(:infinity) + end) + + assert_receive {:ok, owner} + {owner, task} + end +end diff --git a/lib/elixir/test/elixir/registry/unique_test.exs b/lib/elixir/test/elixir/registry/unique_test.exs new file mode 100644 index 00000000000..3cd37b82120 --- /dev/null +++ b/lib/elixir/test/elixir/registry/unique_test.exs @@ -0,0 +1,507 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Registry.UniqueTest do + use ExUnit.Case, + async: true, + parameterize: [ + %{partitions: 1}, + %{partitions: 8} + ] + + @keys :unique + + setup config do + partitions = config.partitions + listeners = List.wrap(config[:base_listener]) |> Enum.map(&:"#{&1}_#{partitions}") + name = :"#{config.test}_#{partitions}" + opts = [keys: @keys, name: name, partitions: partitions, listeners: listeners] + {:ok, _} = start_supervised({Registry, opts}) + %{registry: name, listeners: listeners} + end + + test "starts configured number of partitions", %{registry: registry, partitions: partitions} do + assert length(Supervisor.which_children(registry)) == partitions + end + + test "counts 0 keys in an empty registry", %{registry: registry} do + assert 0 == Registry.count(registry) + end + + test "counts the number of keys in a registry", %{registry: registry} do + {:ok, _} = Registry.register(registry, "hello", :value) + {:ok, _} = Registry.register(registry, "world", :value) + + assert 2 == Registry.count(registry) + end + + test "has unique registrations", %{registry: registry} do + {:ok, pid} = Registry.register(registry, "hello", :value) + assert is_pid(pid) + assert Registry.keys(registry, self()) == ["hello"] + assert Registry.values(registry, "hello", self()) == [:value] + + assert {:error, {:already_registered, pid}} = Registry.register(registry, "hello", :value) + assert pid == self() + assert Registry.keys(registry, self()) == ["hello"] + assert Registry.values(registry, "hello", self()) == [:value] + + {:ok, pid} = Registry.register(registry, "world", :value) + assert is_pid(pid) + assert Registry.keys(registry, self()) |> Enum.sort() == ["hello", "world"] + end + + test "has unique registrations across processes", %{registry: registry} do + {_, task} = register_task(registry, "hello", :value) + Process.link(Process.whereis(registry)) + assert Registry.keys(registry, task) == ["hello"] + assert Registry.values(registry, "hello", task) == [:value] + + assert {:error, {:already_registered, ^task}} = + Registry.register(registry, "hello", :recent) + + assert Registry.keys(registry, self()) == [] + assert Registry.values(registry, "hello", self()) == [] + + {:links, links} = Process.info(self(), :links) + assert Process.whereis(registry) in links + end + + test "has unique registrations even if partition is delayed", %{registry: registry} do + {owner, task} = register_task(registry, "hello", :value) + + assert Registry.register(registry, "hello", :other) == + {:error, {:already_registered, task}} + + :sys.suspend(owner) + kill_and_assert_down(task) + Registry.register(registry, "hello", :other) + assert Registry.lookup(registry, "hello") == [{self(), :other}] + end + + test "supports match patterns", %{registry: registry} do + value = {1, :atom, 1} + {:ok, _} = Registry.register(registry, "hello", value) + assert Registry.match(registry, "hello", {1, :_, :_}) == [{self(), value}] + assert Registry.match(registry, "hello", {1.0, :_, :_}) == [] + assert Registry.match(registry, "hello", {:_, :atom, :_}) == [{self(), value}] + assert Registry.match(registry, "hello", {:"$1", :_, :"$1"}) == [{self(), value}] + assert Registry.match(registry, "hello", :_) == [{self(), value}] + assert Registry.match(registry, :_, :_) == [] + + value2 = %{a: "a", b: "b"} + {:ok, _} = Registry.register(registry, "world", value2) + assert Registry.match(registry, "world", %{b: "b"}) == [{self(), value2}] + end + + test "supports guard conditions", %{registry: registry} do + value = {1, :atom, 2} + {:ok, _} = Registry.register(registry, "hello", value) + + assert Registry.match(registry, "hello", {:_, :_, :"$1"}, [{:>, :"$1", 1}]) == + [{self(), value}] + + assert Registry.match(registry, "hello", {:_, :_, :"$1"}, [{:>, :"$1", 2}]) == [] + + assert Registry.match(registry, "hello", {:_, :"$1", :_}, [{:is_atom, :"$1"}]) == + [{self(), value}] + end + + test "count_match supports match patterns", %{registry: registry} do + value = {1, :atom, 1} + {:ok, _} = Registry.register(registry, "hello", value) + assert 1 == Registry.count_match(registry, "hello", {1, :_, :_}) + assert 0 == Registry.count_match(registry, "hello", {1.0, :_, :_}) + assert 1 == Registry.count_match(registry, "hello", {:_, :atom, :_}) + assert 1 == Registry.count_match(registry, "hello", {:"$1", :_, :"$1"}) + assert 1 == Registry.count_match(registry, "hello", :_) + assert 0 == Registry.count_match(registry, :_, :_) + + value2 = %{a: "a", b: "b"} + {:ok, _} = Registry.register(registry, "world", value2) + assert 1 == Registry.count_match(registry, "world", %{b: "b"}) + end + + test "count_match supports guard conditions", %{registry: registry} do + value = {1, :atom, 2} + {:ok, _} = Registry.register(registry, "hello", value) + + assert 1 == Registry.count_match(registry, "hello", {:_, :_, :"$1"}, [{:>, :"$1", 1}]) + assert 0 == Registry.count_match(registry, "hello", {:_, :_, :"$1"}, [{:>, :"$1", 2}]) + assert 1 == Registry.count_match(registry, "hello", {:_, :"$1", :_}, [{:is_atom, :"$1"}]) + end + + test "unregister_match supports patterns", %{registry: registry} do + value = {1, :atom, 1} + {:ok, _} = Registry.register(registry, "hello", value) + + Registry.unregister_match(registry, "hello", {2, :_, :_}) + assert Registry.lookup(registry, "hello") == [{self(), value}] + Registry.unregister_match(registry, "hello", {1.0, :_, :_}) + assert Registry.lookup(registry, "hello") == [{self(), value}] + Registry.unregister_match(registry, "hello", {:_, :atom, :_}) + assert Registry.lookup(registry, "hello") == [] + end + + test "unregister_match supports guards", %{registry: registry} do + value = {1, :atom, 1} + {:ok, _} = Registry.register(registry, "hello", value) + + Registry.unregister_match(registry, "hello", {:"$1", :_, :_}, [{:<, :"$1", 2}]) + assert Registry.lookup(registry, "hello") == [] + end + + test "unregister_match supports tricky keys", %{registry: registry} do + {:ok, _} = Registry.register(registry, :_, :foo) + {:ok, _} = Registry.register(registry, "hello", "b") + + Registry.unregister_match(registry, :_, :foo) + assert Registry.lookup(registry, :_) == [] + assert Registry.keys(registry, self()) |> Enum.sort() == ["hello"] + end + + test "compares using ===", %{registry: registry} do + {:ok, _} = Registry.register(registry, 1.0, :value) + {:ok, _} = Registry.register(registry, 1, :value) + assert Registry.keys(registry, self()) |> Enum.sort() == [1, 1.0] + end + + test "updates current process value", %{registry: registry} do + assert Registry.update_value(registry, "hello", &raise/1) == :error + register_task(registry, "hello", :value) + assert Registry.update_value(registry, "hello", &raise/1) == :error + + Registry.register(registry, "world", 1) + assert Registry.lookup(registry, "world") == [{self(), 1}] + assert Registry.update_value(registry, "world", &(&1 + 1)) == {2, 1} + assert Registry.lookup(registry, "world") == [{self(), 2}] + end + + test "dispatches to a single key", %{registry: registry} do + fun = fn _ -> raise "will never be invoked" end + assert Registry.dispatch(registry, "hello", fun) == :ok + + {:ok, _} = Registry.register(registry, "hello", :value) + + fun = fn [{pid, value}] -> send(pid, {:dispatch, value}) end + assert Registry.dispatch(registry, "hello", fun) + + assert_received {:dispatch, :value} + end + + test "unregisters process by key", %{registry: registry} do + :ok = Registry.unregister(registry, "hello") + + {:ok, _} = Registry.register(registry, "hello", :value) + {:ok, _} = Registry.register(registry, "world", :value) + assert Registry.keys(registry, self()) |> Enum.sort() == ["hello", "world"] + + :ok = Registry.unregister(registry, "hello") + assert Registry.keys(registry, self()) == ["world"] + + :ok = Registry.unregister(registry, "world") + assert Registry.keys(registry, self()) == [] + end + + test "unregisters with no entries", %{registry: registry} do + assert Registry.unregister(registry, "hello") == :ok + end + + test "unregisters with tricky keys", %{registry: registry} do + {:ok, _} = Registry.register(registry, :_, :foo) + {:ok, _} = Registry.register(registry, "hello", "b") + + Registry.unregister(registry, :_) + assert Registry.lookup(registry, :_) == [] + assert Registry.keys(registry, self()) |> Enum.sort() == ["hello"] + end + + @tag base_listener: :unique_listener + test "allows listeners", %{registry: registry, listeners: [listener]} do + Process.register(self(), listener) + {_, task} = register_task(registry, "hello", :world) + assert_received {:register, ^registry, "hello", ^task, :world} + + self = self() + {:ok, _} = Registry.register(registry, "world", :value) + assert_received {:register, ^registry, "world", ^self, :value} + + :ok = Registry.unregister(registry, "world") + assert_received {:unregister, ^registry, "world", ^self} + after + Process.unregister(listener) + end + + test "links and unlinks on register/unregister", %{registry: registry} do + {:ok, pid} = Registry.register(registry, "hello", :value) + {:links, links} = Process.info(self(), :links) + assert pid in links + + {:ok, pid} = Registry.register(registry, "world", :value) + {:links, links} = Process.info(self(), :links) + assert pid in links + + :ok = Registry.unregister(registry, "hello") + {:links, links} = Process.info(self(), :links) + assert pid in links + + :ok = Registry.unregister(registry, "world") + {:links, links} = Process.info(self(), :links) + refute pid in links + end + + test "raises on unknown registry name" do + assert_raise ArgumentError, ~r/unknown registry/, fn -> + Registry.register(:unknown, "hello", :value) + end + end + + test "via callbacks", %{registry: registry} do + name = {:via, Registry, {registry, "hello"}} + + # register_name + {:ok, pid} = Agent.start_link(fn -> 0 end, name: name) + + # send + assert Agent.update(name, &(&1 + 1)) == :ok + + # whereis_name + assert Agent.get(name, & &1) == 1 + + # unregister_name + assert {:error, _} = Agent.start(fn -> raise "oops" end) + + # errors + assert {:error, {:already_started, ^pid}} = Agent.start(fn -> 0 end, name: name) + end + + test "uses value provided in via", %{registry: registry} do + name = {:via, Registry, {registry, "hello", :value}} + {:ok, pid} = Agent.start_link(fn -> 0 end, name: name) + assert Registry.lookup(registry, "hello") == [{pid, :value}] + end + + test "empty list for empty registry", %{registry: registry} do + assert Registry.select(registry, [{{:_, :_, :_}, [], [:"$_"]}]) == [] + end + + test "select all", %{registry: registry} do + name = {:via, Registry, {registry, "hello"}} + {:ok, pid} = Agent.start_link(fn -> 0 end, name: name) + {:ok, _} = Registry.register(registry, "world", :value) + + assert Registry.select(registry, [{{:"$1", :"$2", :"$3"}, [], [{{:"$1", :"$2", :"$3"}}]}]) + |> Enum.sort() == + [{"hello", pid, nil}, {"world", self(), :value}] + end + + test "select supports full match specs", %{registry: registry} do + value = {1, :atom, 1} + {:ok, _} = Registry.register(registry, "hello", value) + + assert [{"hello", self(), value}] == + Registry.select(registry, [ + {{"hello", :"$2", :"$3"}, [], [{{"hello", :"$2", :"$3"}}]} + ]) + + assert [{"hello", self(), value}] == + Registry.select(registry, [ + {{:"$1", self(), :"$3"}, [], [{{:"$1", self(), :"$3"}}]} + ]) + + assert [{"hello", self(), value}] == + Registry.select(registry, [ + {{:"$1", :"$2", value}, [], [{{:"$1", :"$2", {value}}}]} + ]) + + assert [] == + Registry.select(registry, [ + {{"world", :"$2", :"$3"}, [], [{{"world", :"$2", :"$3"}}]} + ]) + + assert [] == Registry.select(registry, [{{:"$1", :"$2", {1.0, :_, :_}}, [], [:"$_"]}]) + + assert [{"hello", self(), value}] == + Registry.select(registry, [ + {{:"$1", :"$2", {:"$3", :atom, :"$4"}}, [], + [{{:"$1", :"$2", {{:"$3", :atom, :"$4"}}}}]} + ]) + + assert [{"hello", self(), {1, :atom, 1}}] == + Registry.select(registry, [ + {{:"$1", :"$2", {:"$3", :"$4", :"$3"}}, [], + [{{:"$1", :"$2", {{:"$3", :"$4", :"$3"}}}}]} + ]) + + value2 = %{a: "a", b: "b"} + {:ok, _} = Registry.register(registry, "world", value2) + + assert [:match] == + Registry.select(registry, [{{"world", self(), %{b: "b"}}, [], [:match]}]) + + assert ["hello", "world"] == + Registry.select(registry, [{{:"$1", :_, :_}, [], [:"$1"]}]) |> Enum.sort() + end + + test "select supports guard conditions", %{registry: registry} do + value = {1, :atom, 2} + {:ok, _} = Registry.register(registry, "hello", value) + + assert [{"hello", self(), {1, :atom, 2}}] == + Registry.select(registry, [ + {{:"$1", :"$2", {:"$3", :"$4", :"$5"}}, [{:>, :"$5", 1}], + [{{:"$1", :"$2", {{:"$3", :"$4", :"$5"}}}}]} + ]) + + assert [] == + Registry.select(registry, [ + {{:_, :_, {:_, :_, :"$1"}}, [{:>, :"$1", 2}], [:"$_"]} + ]) + + assert ["hello"] == + Registry.select(registry, [ + {{:"$1", :_, {:_, :"$2", :_}}, [{:is_atom, :"$2"}], [:"$1"]} + ]) + end + + test "select allows multiple specs", %{registry: registry} do + {:ok, _} = Registry.register(registry, "hello", :value) + {:ok, _} = Registry.register(registry, "world", :value) + + assert ["hello", "world"] == + Registry.select(registry, [ + {{"hello", :_, :_}, [], [{:element, 1, :"$_"}]}, + {{"world", :_, :_}, [], [{:element, 1, :"$_"}]} + ]) + |> Enum.sort() + end + + test "select raises on incorrect shape of match spec", %{registry: registry} do + assert_raise ArgumentError, fn -> + Registry.select(registry, [{:_, [], []}]) + end + end + + test "count_select supports match specs", %{registry: registry} do + value = {1, :atom, 1} + {:ok, _} = Registry.register(registry, "hello", value) + assert 1 == Registry.count_select(registry, [{{:_, :_, value}, [], [true]}]) + assert 1 == Registry.count_select(registry, [{{"hello", :_, :_}, [], [true]}]) + assert 1 == Registry.count_select(registry, [{{:_, :_, {1, :atom, :_}}, [], [true]}]) + assert 1 == Registry.count_select(registry, [{{:_, :_, {:"$1", :_, :"$1"}}, [], [true]}]) + assert 0 == Registry.count_select(registry, [{{"hello", :_, nil}, [], [true]}]) + + value2 = %{a: "a", b: "b"} + {:ok, _} = Registry.register(registry, "world", value2) + assert 1 == Registry.count_select(registry, [{{"world", :_, :_}, [], [true]}]) + end + + test "count_select supports guard conditions", %{registry: registry} do + value = {1, :atom, 2} + {:ok, _} = Registry.register(registry, "hello", value) + + assert 1 == + Registry.count_select(registry, [ + {{:_, :_, {:_, :"$1", :_}}, [{:is_atom, :"$1"}], [true]} + ]) + + assert 1 == + Registry.count_select(registry, [ + {{:_, :_, {:_, :_, :"$1"}}, [{:>, :"$1", 1}], [true]} + ]) + + assert 0 == + Registry.count_select(registry, [ + {{:_, :_, {:_, :_, :"$1"}}, [{:>, :"$1", 2}], [true]} + ]) + end + + test "count_select allows multiple specs", %{registry: registry} do + {:ok, _} = Registry.register(registry, "hello", :value) + {:ok, _} = Registry.register(registry, "world", :value) + + assert 2 == + Registry.count_select(registry, [ + {{"hello", :_, :_}, [], [true]}, + {{"world", :_, :_}, [], [true]} + ]) + end + + test "count_select raises on incorrect shape of match spec", %{registry: registry} do + assert_raise ArgumentError, fn -> + Registry.count_select(registry, [{:_, [], []}]) + end + end + + test "doesn't grow ets on already_registered", + %{registry: registry, partitions: partitions} do + assert sum_pid_entries(registry, partitions) == 0 + + {:ok, pid} = Registry.register(registry, "hello", :value) + assert is_pid(pid) + assert sum_pid_entries(registry, partitions) == 1 + + {:ok, pid} = Registry.register(registry, "world", :value) + assert is_pid(pid) + assert sum_pid_entries(registry, partitions) == 2 + + assert {:error, {:already_registered, _pid}} = + Registry.register(registry, "hello", :value) + + assert sum_pid_entries(registry, partitions) == 2 + end + + test "doesn't grow ets on already_registered across processes", + %{registry: registry, partitions: partitions} do + assert sum_pid_entries(registry, partitions) == 0 + + {_, task} = register_task(registry, "hello", :value) + Process.link(Process.whereis(registry)) + + assert sum_pid_entries(registry, partitions) == 1 + + {:ok, pid} = Registry.register(registry, "world", :value) + assert is_pid(pid) + assert sum_pid_entries(registry, partitions) == 2 + + assert {:error, {:already_registered, ^task}} = + Registry.register(registry, "hello", :recent) + + assert sum_pid_entries(registry, partitions) == 2 + end + + defp register_task(registry, key, value) do + parent = self() + + {:ok, task} = + Task.start(fn -> + send(parent, Registry.register(registry, key, value)) + Process.sleep(:infinity) + end) + + assert_receive {:ok, owner} + {owner, task} + end + + defp kill_and_assert_down(pid) do + ref = Process.monitor(pid) + Process.exit(pid, :kill) + assert_receive {:DOWN, ^ref, _, _, _} + end + + defp sum_pid_entries(registry, partitions) do + Enum.sum_by(0..(partitions - 1), fn partition -> + registry + |> Module.concat("PIDPartition#{partition}") + |> ets_entries() + end) + end + + defp ets_entries(table_name) do + :ets.all() + |> Enum.find_value(fn id -> :ets.info(id, :name) == table_name and :ets.info(id, :size) end) + end +end diff --git a/lib/elixir/test/elixir/registry_test.exs b/lib/elixir/test/elixir/registry_test.exs new file mode 100644 index 00000000000..71129cd4c7f --- /dev/null +++ b/lib/elixir/test/elixir/registry_test.exs @@ -0,0 +1,251 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("test_helper.exs", __DIR__) + +defmodule Registry.CommonTest do + use ExUnit.Case, async: true + doctest Registry, except: [:moduledoc] +end + +defmodule Registry.Test do + use ExUnit.Case, + async: true, + parameterize: + for( + keys <- [:unique, :duplicate, {:duplicate, :pid}, {:duplicate, :key}], + partitions <- [1, 8], + do: %{keys: keys, partitions: partitions} + ) + + setup config do + keys = config.keys || :unique + partitions = config.partitions + + listeners = + List.wrap(config[:base_listener]) |> Enum.map(&:"#{&1}_#{partitions}_#{inspect(keys)}") + + name = :"#{config.test}_#{partitions}_#{inspect(keys)}" + opts = [keys: keys, name: name, partitions: partitions, listeners: listeners] + {:ok, _} = start_supervised({Registry, opts}) + %{registry: name, listeners: listeners} + end + + # Note: those tests relies on internals + test "clean up registry on process crash", + %{registry: registry, partitions: partitions} do + {_, task1} = register_task(registry, "hello", :value) + {_, task2} = register_task(registry, "world", :value) + + kill_and_assert_down(task1) + kill_and_assert_down(task2) + + # pid might be in different partition to key so need to sync with all + # partitions before checking ETS tables are empty. + if partitions > 1 do + for i <- 0..(partitions - 1) do + [{_, _, {partition, _}}] = :ets.lookup(registry, i) + GenServer.call(partition, :sync) + end + + for i <- 0..(partitions - 1) do + [{_, key, {_, pid}}] = :ets.lookup(registry, i) + assert :ets.tab2list(key) == [] + assert :ets.tab2list(pid) == [] + end + else + [{-1, {_, _, key, {partition, pid}, _}}] = :ets.lookup(registry, -1) + GenServer.call(partition, :sync) + assert :ets.tab2list(key) == [] + assert :ets.tab2list(pid) == [] + end + end + + defp register_task(registry, key, value) do + parent = self() + + {:ok, task} = + Task.start(fn -> + send(parent, Registry.register(registry, key, value)) + Process.sleep(:infinity) + end) + + assert_receive {:ok, owner} + {owner, task} + end + + defp kill_and_assert_down(pid) do + ref = Process.monitor(pid) + Process.exit(pid, :kill) + assert_receive {:DOWN, ^ref, _, _, _} + end +end + +defmodule Registry.LockTest do + use ExUnit.Case, + async: true, + parameterize: [ + %{keys: :unique, partitions: 1}, + %{keys: :unique, partitions: 8}, + %{keys: :duplicate, partitions: 1}, + %{keys: :duplicate, partitions: 8} + ] + + setup config do + keys = config.keys + partitions = config.partitions + name = :"#{config.test}_#{keys}_#{partitions}" + opts = [keys: keys, name: name, partitions: partitions] + {:ok, _} = start_supervised({Registry, opts}) + %{registry: name} + end + + test "does not lock when using different keys", config do + parent = self() + + task1 = + Task.async(fn -> + Registry.lock(config.registry, 1, fn -> + send(parent, :locked1) + assert_receive :unlock + :done + end) + end) + + assert_receive :locked1 + + task2 = + Task.async(fn -> + Registry.lock(config.registry, 2, fn -> + send(parent, :locked2) + assert_receive :unlock + :done + end) + end) + + assert_receive :locked2 + + send(task1.pid, :unlock) + send(task2.pid, :unlock) + assert Task.await(task1) == :done + assert Task.await(task2) == :done + assert Registry.lock(config.registry, 1, fn -> :done end) == :done + assert Registry.lock(config.registry, 2, fn -> :done end) == :done + end + + test "locks when using the same key", config do + parent = self() + + task1 = + Task.async(fn -> + Registry.lock(config.registry, :ok, fn -> + send(parent, :locked1) + assert_receive :unlock + :done + end) + end) + + assert_receive :locked1 + + task2 = + Task.async(fn -> + Registry.lock(config.registry, :ok, fn -> + send(parent, :locked2) + :done + end) + end) + + refute_receive :locked2, 100 + + send(task1.pid, :unlock) + assert Task.await(task1) == :done + assert_receive :locked2 + assert Task.await(task2) == :done + assert Registry.lock(config.registry, :ok, fn -> :done end) == :done + end + + @tag :capture_log + test "locks when the one holding the lock raises", config do + parent = self() + + task1 = + Task.async(fn -> + Registry.lock(config.registry, :ok, fn -> + send(parent, :locked) + assert_receive :unlock + raise "oops" + end) + end) + + Process.unlink(task1.pid) + assert_receive :locked + + task2 = + Task.async(fn -> + Registry.lock(config.registry, :ok, fn -> + :done + end) + end) + + send(task1.pid, :unlock) + assert {:exit, {%RuntimeError{message: "oops"}, [_ | _]}} = Task.yield(task1) + assert Task.await(task2) == :done + assert Registry.lock(config.registry, :ok, fn -> :done end) == :done + end + + test "locks when the one holding the lock terminates", config do + parent = self() + + task1 = + Task.async(fn -> + Registry.lock(config.registry, :ok, fn -> + send(parent, :locked) + assert_receive :unlock + :done + end) + end) + + assert_receive :locked + + task2 = + Task.async(fn -> + Registry.lock(config.registry, :ok, fn -> + :done + end) + end) + + assert Task.shutdown(task1, :brutal_kill) == nil + assert Task.await(task2) == :done + assert Registry.lock(config.registry, :ok, fn -> :done end) == :done + end + + test "locks when the one waiting for the lock terminates", config do + parent = self() + + task1 = + Task.async(fn -> + Registry.lock(config.registry, :ok, fn -> + send(parent, :locked) + assert_receive :unlock + :done + end) + end) + + assert_receive :locked + + task2 = + Task.async(fn -> + Registry.lock(config.registry, :ok, fn -> + :done + end) + end) + + :erlang.yield() + assert Task.shutdown(task2, :brutal_kill) == nil + + send(task1.pid, :unlock) + assert Task.await(task1) == :done + assert Registry.lock(config.registry, :ok, fn -> :done end) == :done + end +end diff --git a/lib/elixir/test/elixir/set_test.exs b/lib/elixir/test/elixir/set_test.exs deleted file mode 100644 index 52545b8a6fd..00000000000 --- a/lib/elixir/test/elixir/set_test.exs +++ /dev/null @@ -1,189 +0,0 @@ -Code.require_file "test_helper.exs", __DIR__ - -# A TestSet implementation used only for testing. -defmodule TestSet do - defstruct list: [] - def new(list \\ []) when is_list(list) do - %TestSet{list: list} - end - - def reduce(%TestSet{list: list}, acc, fun) do - Enumerable.reduce(list, acc, fun) - end - - def member?(%TestSet{list: list}, v) do - v in list - end - - def size(%TestSet{list: list}) do - length(list) - end -end - -defmodule SetTest.Common do - defmacro __using__(_) do - quote location: :keep do - defp new_set(list \\ []) do - Enum.into list, set_impl.new - end - - defp new_set(list, fun) do - Enum.into list, set_impl.new, fun - end - - defp int_set() do - Enum.into [1, 2, 3], set_impl.new - end - - test "delete/2" do - result = Set.delete(new_set([1, 2, 3]), 2) - assert Set.equal?(result, new_set([1, 3])) - end - - test "delete/2 with match" do - refute Set.member?(Set.delete(int_set, 1), 1) - assert Set.member?(Set.delete(int_set, 1.0), 1) - end - - test "difference/2" do - result = Set.difference(new_set([1, 2, 3]), new_set([3])) - assert Set.equal?(result, new_set([1, 2])) - end - - test "difference/2 with match" do - refute Set.member?(Set.difference(int_set, new_set([1])), 1) - assert Set.member?(Set.difference(int_set, new_set([1.0])), 1) - end - - test "difference/2 with other set" do - result = Set.difference(new_set([1, 2, 3]), TestSet.new([3])) - assert Set.equal?(result, new_set([1, 2])) - end - - test "disjoint?/2" do - assert Set.disjoint?(new_set([1, 2, 3]), new_set([4, 5 ,6])) - refute Set.disjoint?(new_set([1, 2, 3]), new_set([3, 4 ,5])) - end - - test "disjoint/2 with other set" do - assert Set.disjoint?(new_set([1, 2, 3]), TestSet.new([4, 5 ,6])) - refute Set.disjoint?(new_set([1, 2, 3]), TestSet.new([3, 4 ,5])) - end - - test "equal?/2" do - assert Set.equal?(new_set([1, 2, 3]), new_set([3, 2, 1])) - refute Set.equal?(new_set([1, 2, 3]), new_set([3.0, 2.0, 1.0])) - end - - test "equal?/2 with other set" do - assert Set.equal?(new_set([1, 2, 3]), TestSet.new([3, 2, 1])) - refute Set.equal?(new_set([1, 2, 3]), TestSet.new([3.0, 2.0, 1.0])) - end - - test "intersection/2" do - result = Set.intersection(new_set([1, 2, 3]), new_set([2, 3, 4])) - assert Set.equal?(result, new_set([2, 3])) - end - - test "intersection/2 with match" do - assert Set.member?(Set.intersection(int_set, new_set([1])), 1) - refute Set.member?(Set.intersection(int_set, new_set([1.0])), 1) - end - - test "intersection/2 with other set" do - result = Set.intersection(new_set([1, 2, 3]), TestSet.new([2, 3, 4])) - assert Set.equal?(result, new_set([2, 3])) - end - - test "member?/2" do - assert Set.member?(new_set([1, 2, 3]), 2) - refute Set.member?(new_set([1, 2, 3]), 4) - refute Set.member?(new_set([1, 2, 3]), 1.0) - end - - test "put/2" do - result = Set.put(new_set([1, 2]), 3) - assert Set.equal?(result, new_set([1, 2, 3])) - end - - test "put/2 with match" do - assert Set.size(Set.put(int_set, 1)) == 3 - assert Set.size(Set.put(int_set, 1.0)) == 4 - end - - test "size/1" do - assert Set.size(new_set([1, 2, 3])) == 3 - end - - test "subset?/2" do - assert Set.subset?(new_set([1, 2]), new_set([1, 2, 3])) - refute Set.subset?(new_set([1, 2, 3]), new_set([1, 2])) - end - - test "subset/2 with match?" do - assert Set.subset?(new_set([1]), int_set) - refute Set.subset?(new_set([1.0]), int_set) - end - - test "subset?/2 with other set" do - assert Set.subset?(new_set([1, 2]), TestSet.new([1, 2, 3])) - refute Set.subset?(new_set([1, 2, 3]), TestSet.new([1, 2])) - end - - test "to_list/1" do - assert Set.to_list(new_set([1, 2, 3])) |> Enum.sort == [1, 2, 3] - end - - test "union/2" do - result = Set.union(new_set([1, 2, 3]), new_set([2, 3, 4])) - assert Set.equal?(result, new_set([1, 2, 3, 4])) - end - - test "union/2 with match" do - assert Set.size(Set.union(int_set, new_set([1]))) == 3 - assert Set.size(Set.union(int_set, new_set([1.0]))) == 4 - end - - test "union/2 with other set" do - result = Set.union(new_set([1, 2, 3]), TestSet.new([2, 3, 4])) - assert Set.equal?(result, new_set([1, 2, 3, 4])) - end - - test "is enumerable" do - assert Enum.member?(int_set, 1) - refute Enum.member?(int_set, 1.0) - assert Enum.sort(int_set) == [1,2,3] - end - - test "is collectable" do - assert Set.equal?(new_set([1, 1, 2, 3, 3, 3]), new_set([1, 2, 3])) - assert Set.equal?(new_set([1, 1, 2, 3, 3, 3], &(&1 * 2)), new_set([2, 4, 6])) - assert Collectable.empty(new_set([1, 2, 3])) == new_set - end - - test "is zippable" do - set = new_set(1..8) - list = Dict.to_list(set) - assert Enum.zip(list, list) == Enum.zip(set, set) - - set = new_set(1..100) - list = Dict.to_list(set) - assert Enum.zip(list, list) == Enum.zip(set, set) - end - - test "unsupported set" do - assert_raise ArgumentError, "unsupported set: :bad_set", fn -> - Set.to_list :bad_set - end - end - end - end -end - -defmodule Set.HashSetTest do - use ExUnit.Case, async: true - use SetTest.Common - - doctest Set - def set_impl, do: HashSet -end diff --git a/lib/elixir/test/elixir/stream_test.exs b/lib/elixir/test/elixir/stream_test.exs index bc1e3ad64b5..a7444f7f8a8 100644 --- a/lib/elixir/test/elixir/stream_test.exs +++ b/lib/elixir/test/elixir/stream_test.exs @@ -1,13 +1,51 @@ -Code.require_file "test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("test_helper.exs", __DIR__) defmodule StreamTest do use ExUnit.Case, async: true + doctest Stream + + defmodule Pdict do + defstruct [] + + defimpl Collectable do + def into(struct) do + fun = fn + _, {:cont, x} -> Process.put(:stream_cont, [x | Process.get(:stream_cont)]) + _, :done -> Process.put(:stream_done, true) + _, :halt -> Process.put(:stream_halt, true) + end + + {struct, fun} + end + end + end + + defmodule HaltAcc do + defstruct [:acc] + + defimpl Enumerable do + def count(_lazy), do: {:error, __MODULE__} + + def member?(_lazy, _value), do: {:error, __MODULE__} + + def slice(_lazy), do: {:error, __MODULE__} + + def reduce(lazy, _acc, _fun) do + {:halted, Enum.to_list(lazy.acc)} + end + end + end + test "streams as enumerables" do - stream = Stream.map([1,2,3], &(&1 * 2)) + stream = Stream.map([1, 2, 3], &(&1 * 2)) # Reduce - assert Enum.map(stream, &(&1 + 1)) == [3,5,7] + assert Enum.map(stream, &(&1 + 1)) == [3, 5, 7] # Member assert Enum.member?(stream, 4) refute Enum.member?(stream, 1) @@ -16,203 +54,377 @@ defmodule StreamTest do end test "streams are composable" do - stream = Stream.map([1,2,3], &(&1 * 2)) - assert is_lazy(stream) + stream = Stream.map([1, 2, 3], &(&1 * 2)) + assert lazy?(stream) stream = Stream.map(stream, &(&1 + 1)) - assert is_lazy(stream) - - assert Enum.to_list(stream) == [3,5,7] - end - - test "chunk/2, chunk/3 and chunk/4" do - assert Stream.chunk([1, 2, 3, 4, 5], 2) |> Enum.to_list == - [[1, 2], [3, 4]] - assert Stream.chunk([1, 2, 3, 4, 5], 2, 2, [6]) |> Enum.to_list == - [[1, 2], [3, 4], [5, 6]] - assert Stream.chunk([1, 2, 3, 4, 5, 6], 3, 2) |> Enum.to_list == - [[1, 2, 3], [3, 4, 5]] - assert Stream.chunk([1, 2, 3, 4, 5, 6], 2, 3) |> Enum.to_list == - [[1, 2], [4, 5]] - assert Stream.chunk([1, 2, 3, 4, 5, 6], 3, 2, []) |> Enum.to_list == - [[1, 2, 3], [3, 4, 5], [5, 6]] - assert Stream.chunk([1, 2, 3, 4, 5, 6], 3, 3, []) |> Enum.to_list == - [[1, 2, 3], [4, 5, 6]] - assert Stream.chunk([1, 2, 3, 4, 5], 4, 4, 6..10) |> Enum.to_list == - [[1, 2, 3, 4], [5, 6, 7, 8]] - end - - test "chunk/4 is zippable" do - stream = Stream.chunk([1, 2, 3, 4, 5, 6], 3, 2, []) - list = Enum.to_list(stream) + assert lazy?(stream) + + assert Enum.to_list(stream) == [3, 5, 7] + end + + test "chunk_every/2, chunk_every/3 and chunk_every/4" do + assert Stream.chunk_every([1, 2, 3, 4, 5], 2) |> Enum.to_list() == [[1, 2], [3, 4], [5]] + + assert Stream.chunk_every([1, 2, 3, 4, 5], 2, 2, [6]) |> Enum.to_list() == + [[1, 2], [3, 4], [5, 6]] + + assert Stream.chunk_every([1, 2, 3, 4, 5, 6], 3, 2, :discard) |> Enum.to_list() == + [[1, 2, 3], [3, 4, 5]] + + assert Stream.chunk_every([1, 2, 3, 4, 5, 6], 2, 3, :discard) |> Enum.to_list() == + [[1, 2], [4, 5]] + + assert Stream.chunk_every([1, 2, 3, 4, 5, 6], 3, 2, []) |> Enum.to_list() == + [[1, 2, 3], [3, 4, 5], [5, 6]] + + assert Stream.chunk_every([1, 2, 3, 4, 5, 6], 3, 3, []) |> Enum.to_list() == + [[1, 2, 3], [4, 5, 6]] + + assert Stream.chunk_every([1, 2, 3, 4, 5], 4, 4, 6..10) |> Enum.to_list() == + [[1, 2, 3, 4], [5, 6, 7, 8]] + end + + test "chunk_every/4 is zippable" do + stream = Stream.chunk_every([1, 2, 3, 4, 5, 6], 3, 2, []) + list = Enum.to_list(stream) assert Enum.zip(list, list) == Enum.zip(stream, stream) end + test "chunk_every/4 is haltable" do + assert 1..10 |> Stream.take(6) |> Stream.chunk_every(4, 4, [7, 8]) |> Enum.to_list() == + [[1, 2, 3, 4], [5, 6, 7, 8]] + + assert 1..10 + |> Stream.take(6) + |> Stream.chunk_every(4, 4, [7, 8]) + |> Stream.take(3) + |> Enum.to_list() == [[1, 2, 3, 4], [5, 6, 7, 8]] + + assert 1..10 + |> Stream.take(6) + |> Stream.chunk_every(4, 4, [7, 8]) + |> Stream.take(2) + |> Enum.to_list() == [[1, 2, 3, 4], [5, 6, 7, 8]] + + assert 1..10 + |> Stream.take(6) + |> Stream.chunk_every(4, 4, [7, 8]) + |> Stream.take(1) + |> Enum.to_list() == [[1, 2, 3, 4]] + + assert 1..6 |> Stream.take(6) |> Stream.chunk_every(4, 4, [7, 8]) |> Enum.to_list() == + [[1, 2, 3, 4], [5, 6, 7, 8]] + end + test "chunk_by/2" do stream = Stream.chunk_by([1, 2, 2, 3, 4, 4, 6, 7, 7], &(rem(&1, 2) == 1)) - assert is_lazy(stream) - assert Enum.to_list(stream) == - [[1], [2, 2], [3], [4, 4, 6], [7, 7]] - assert stream |> Stream.take(3) |> Enum.to_list == - [[1], [2, 2], [3]] + assert lazy?(stream) + assert Enum.to_list(stream) == [[1], [2, 2], [3], [4, 4, 6], [7, 7]] + assert stream |> Stream.take(3) |> Enum.to_list() == [[1], [2, 2], [3]] + assert 1..10 |> Stream.chunk_every(2) |> Enum.take(2) == [[1, 2], [3, 4]] end test "chunk_by/2 is zippable" do stream = Stream.chunk_by([1, 2, 2, 3], &(rem(&1, 2) == 1)) - list = Enum.to_list(stream) + list = Enum.to_list(stream) assert Enum.zip(list, list) == Enum.zip(stream, stream) end + test "chunk_while/4" do + chunk_fun = fn i, acc -> + cond do + i > 10 -> {:halt, acc} + rem(i, 2) == 0 -> {:cont, Enum.reverse([i | acc]), []} + true -> {:cont, [i | acc]} + end + end + + after_fun = fn + [] -> {:cont, []} + acc -> {:cont, Enum.reverse(acc), []} + end + + assert Stream.chunk_while([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], [], chunk_fun, after_fun) + |> Enum.to_list() == [[1, 2], [3, 4], [5, 6], [7, 8], [9, 10]] + + assert Stream.chunk_while(0..9, [], chunk_fun, after_fun) |> Enum.to_list() == + [[0], [1, 2], [3, 4], [5, 6], [7, 8], [9]] + + assert Stream.chunk_while(0..10, [], chunk_fun, after_fun) |> Enum.to_list() == + [[0], [1, 2], [3, 4], [5, 6], [7, 8], [9, 10]] + + assert Stream.chunk_while(0..11, [], chunk_fun, after_fun) |> Enum.to_list() == + [[0], [1, 2], [3, 4], [5, 6], [7, 8], [9, 10]] + + assert Stream.chunk_while([5, 7, 9, 11], [], chunk_fun, after_fun) |> Enum.to_list() == + [[5, 7, 9]] + end + + test "chunk_while/4 with inner halt" do + chunk_fun = fn + i, [] -> + {:cont, [i]} + + i, chunk -> + if rem(i, 2) == 0 do + {:cont, Enum.reverse(chunk), [i]} + else + {:cont, [i | chunk]} + end + end + + after_fun = fn + [] -> {:cont, []} + chunk -> {:cont, Enum.reverse(chunk), []} + end + + assert Stream.chunk_while([1, 2, 3, 4, 5], [], chunk_fun, after_fun) |> Enum.at(0) == [1] + end + + test "chunk_while/4 regression case with concat" do + result = + ["WrongHeader\nJohn Doe", "skipped"] + |> Stream.take(1) + |> Stream.chunk_while( + "", + fn element, acc -> + {acc, elements} = String.split(acc <> element, "\n") |> List.pop_at(-1) + {:cont, elements, acc} + end, + &{:cont, [&1], []} + ) + |> Stream.concat() + |> Enum.to_list() + + assert result == ["WrongHeader", "John Doe"] + end + test "concat/1" do stream = Stream.concat([1..3, [], [4, 5, 6], [], 7..9]) assert is_function(stream) - assert Enum.to_list(stream) == [1,2,3,4,5,6,7,8,9] - assert Enum.take(stream, 5) == [1,2,3,4,5] + assert Enum.to_list(stream) == [1, 2, 3, 4, 5, 6, 7, 8, 9] + assert Enum.take(stream, 5) == [1, 2, 3, 4, 5] stream = Stream.concat([1..3, [4, 5, 6], Stream.cycle(7..100)]) assert is_function(stream) - assert Enum.take(stream, 13) == [1,2,3,4,5,6,7,8,9,10,11,12,13] + assert Enum.take(stream, 13) == [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13] end test "concat/2" do stream = Stream.concat(1..3, 4..6) assert is_function(stream) - assert Stream.cycle(stream) |> Enum.take(16) == [1,2,3,4,5,6,1,2,3,4,5,6,1,2,3,4] + + assert Stream.cycle(stream) |> Enum.take(16) == + [1, 2, 3, 4, 5, 6, 1, 2, 3, 4, 5, 6, 1, 2, 3, 4] stream = Stream.concat(1..3, []) assert is_function(stream) - assert Stream.cycle(stream) |> Enum.take(5) == [1,2,3,1,2] + assert Stream.cycle(stream) |> Enum.take(5) == [1, 2, 3, 1, 2] stream = Stream.concat(1..6, Stream.cycle(7..9)) assert is_function(stream) - assert Stream.drop(stream, 3) |> Enum.take(13) == [4,5,6,7,8,9,7,8,9,7,8,9,7] + assert Stream.drop(stream, 3) |> Enum.take(13) == [4, 5, 6, 7, 8, 9, 7, 8, 9, 7, 8, 9, 7] stream = Stream.concat(Stream.cycle(1..3), Stream.cycle(4..6)) assert is_function(stream) - assert Enum.take(stream, 13) == [1,2,3,1,2,3,1,2,3,1,2,3,1] + assert Enum.take(stream, 13) == [1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3, 1] + end + + test "concat/2 is zippable" do + stream = 1..2 |> Stream.take(2) |> Stream.concat(3..4) + assert Enum.zip(1..4, [1, 2, 3, 4]) == Enum.zip(1..4, stream) end test "concat/2 does not intercept wrapped lazy enumeration" do # concat returns a lazy enumeration that does not halt assert Stream.concat([[0], Stream.map([1, 2, 3], & &1), [4]]) |> Stream.take_while(fn x -> x <= 4 end) - |> Enum.to_list == [0, 1, 2, 3, 4] + |> Enum.to_list() == [0, 1, 2, 3, 4] # concat returns a lazy enumeration that does halts assert Stream.concat([[0], Stream.take_while(1..6, &(&1 <= 3)), [4]]) |> Stream.take_while(fn x -> x <= 4 end) - |> Enum.to_list == [0, 1, 2, 3, 4] + |> Enum.to_list() == [0, 1, 2, 3, 4] end test "cycle/1" do - stream = Stream.cycle([1,2,3]) + stream = Stream.cycle([1, 2, 3]) assert is_function(stream) - assert Stream.cycle([1,2,3]) |> Stream.take(5) |> Enum.to_list == [1,2,3,1,2] - assert Enum.take(stream, 5) == [1,2,3,1,2] + assert_raise ArgumentError, "cannot cycle over an empty enumerable", fn -> + Stream.cycle([]) + end + + assert_raise ArgumentError, "cannot cycle over an empty enumerable", fn -> + Stream.cycle(%{}) |> Enum.to_list() + end + + assert Stream.cycle([1, 2, 3]) |> Stream.take(5) |> Enum.to_list() == [1, 2, 3, 1, 2] + assert Enum.take(stream, 5) == [1, 2, 3, 1, 2] end test "cycle/1 is zippable" do - stream = Stream.cycle([1,2,3]) - assert Enum.zip(1..6, [1,2,3,1,2,3]) == Enum.zip(1..6, stream) + stream = Stream.cycle([1, 2, 3]) + assert Enum.zip(1..6, [1, 2, 3, 1, 2, 3]) == Enum.zip(1..6, stream) end test "cycle/1 with inner stream" do - assert [1,2,3] |> Stream.take(2) |> Stream.cycle |> Enum.take(4) == - [1,2,1,2] + assert [1, 2, 3] |> Stream.take(2) |> Stream.cycle() |> Enum.take(4) == [1, 2, 1, 2] + end + + test "cycle/1 with cycle/1 with cycle/1" do + assert [1] |> Stream.cycle() |> Stream.cycle() |> Stream.cycle() |> Enum.take(5) == + [1, 1, 1, 1, 1] + end + + test "dedup/1 is lazy" do + assert lazy?(Stream.dedup([1, 2, 3])) + end + + test "dedup/1" do + assert Stream.dedup([1, 1, 2, 1, 1, 2, 1]) |> Enum.to_list() == [1, 2, 1, 2, 1] + assert Stream.dedup([2, 1, 1, 2, 1]) |> Enum.to_list() == [2, 1, 2, 1] + assert Stream.dedup([1, 2, 3, 4]) |> Enum.to_list() == [1, 2, 3, 4] + assert Stream.dedup([1, 1.0, 2.0, 2]) |> Enum.to_list() == [1, 1.0, 2.0, 2] + assert Stream.dedup([]) |> Enum.to_list() == [] + + assert Stream.dedup([nil, nil, true, {:value, true}]) |> Enum.to_list() == + [nil, true, {:value, true}] + + assert Stream.dedup([nil]) |> Enum.to_list() == [nil] + end + + test "dedup_by/2" do + assert Stream.dedup_by([{1, :x}, {2, :y}, {2, :z}, {1, :x}], fn {x, _} -> x end) + |> Enum.to_list() == [{1, :x}, {2, :y}, {1, :x}] end test "drop/2" do stream = Stream.drop(1..10, 5) - assert is_lazy(stream) - assert Enum.to_list(stream) == [6,7,8,9,10] + assert lazy?(stream) + assert Enum.to_list(stream) == [6, 7, 8, 9, 10] - assert Enum.to_list(Stream.drop(1..5, 0)) == [1,2,3,4,5] + assert Enum.to_list(Stream.drop(1..5, 0)) == [1, 2, 3, 4, 5] assert Enum.to_list(Stream.drop(1..3, 5)) == [] nats = Stream.iterate(1, &(&1 + 1)) - assert Stream.drop(nats, 2) |> Enum.take(5) == [3,4,5,6,7] + assert Stream.drop(nats, 2) |> Enum.take(5) == [3, 4, 5, 6, 7] end test "drop/2 with negative count" do stream = Stream.drop(1..10, -5) - assert is_lazy(stream) - assert Enum.to_list(stream) == [1,2,3,4,5] + assert lazy?(stream) + assert Enum.to_list(stream) == [1, 2, 3, 4, 5] stream = Stream.drop(1..10, -5) - list = Enum.to_list(stream) + list = Enum.to_list(stream) assert Enum.zip(list, list) == Enum.zip(stream, stream) end test "drop/2 with negative count stream entries" do - par = self - pid = spawn_link fn -> - Enum.each Stream.drop(&inbox_stream/2, -3), - fn x -> send par, {:stream, x} end - end + par = self() - send pid, {:stream, 1} - send pid, {:stream, 2} - send pid, {:stream, 3} - refute_receive {:stream, 1} + pid = + spawn_link(fn -> + Enum.each(Stream.drop(&inbox_stream/2, -3), fn x -> send(par, {:stream, x}) end) + end) - send pid, {:stream, 4} + send(pid, {:stream, 1}) + send(pid, {:stream, 2}) + send(pid, {:stream, 3}) + refute_receive {:stream, 1}, 100 + + send(pid, {:stream, 4}) assert_receive {:stream, 1} - send pid, {:stream, 5} + send(pid, {:stream, 5}) assert_receive {:stream, 2} - refute_receive {:stream, 3} + refute_receive {:stream, 3}, 100 + end + + test "drop_every/2" do + assert 1..10 + |> Stream.drop_every(2) + |> Enum.to_list() == [2, 4, 6, 8, 10] + + assert 1..10 + |> Stream.drop_every(3) + |> Enum.to_list() == [2, 3, 5, 6, 8, 9] + + assert 1..10 + |> Stream.drop(2) + |> Stream.drop_every(2) + |> Stream.drop(1) + |> Enum.to_list() == [6, 8, 10] + + assert 1..10 + |> Stream.drop_every(0) + |> Enum.to_list() == [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + + assert [] + |> Stream.drop_every(10) + |> Enum.to_list() == [] + end + + test "drop_every/2 without non-negative integer" do + assert_raise FunctionClauseError, fn -> + Stream.drop_every(1..10, -1) + end + + assert_raise FunctionClauseError, fn -> + Stream.drop_every(1..10, 3.33) + end end test "drop_while/2" do stream = Stream.drop_while(1..10, &(&1 <= 5)) - assert is_lazy(stream) - assert Enum.to_list(stream) == [6,7,8,9,10] + assert lazy?(stream) + assert Enum.to_list(stream) == [6, 7, 8, 9, 10] - assert Enum.to_list(Stream.drop_while(1..5, &(&1 <= 0))) == [1,2,3,4,5] + assert Enum.to_list(Stream.drop_while(1..5, &(&1 <= 0))) == [1, 2, 3, 4, 5] assert Enum.to_list(Stream.drop_while(1..3, &(&1 <= 5))) == [] nats = Stream.iterate(1, &(&1 + 1)) - assert Stream.drop_while(nats, &(&1 <= 5)) |> Enum.take(5) == [6,7,8,9,10] + assert Stream.drop_while(nats, &(&1 <= 5)) |> Enum.take(5) == [6, 7, 8, 9, 10] + end + + test "duplicate/2" do + stream = Stream.duplicate(7, 7) + + assert is_function(stream) + assert stream |> Stream.take(5) |> Enum.to_list() == [7, 7, 7, 7, 7] + assert Enum.to_list(stream) == [7, 7, 7, 7, 7, 7, 7] end test "each/2" do Process.put(:stream_each, []) - stream = Stream.each([1,2,3], fn x -> - Process.put(:stream_each, [x|Process.get(:stream_each)]) - end) + stream = + Stream.each([1, 2, 3], fn x -> + Process.put(:stream_each, [x | Process.get(:stream_each)]) + end) - assert is_lazy(stream) - assert Enum.to_list(stream) == [1,2,3] - assert Process.get(:stream_each) == [3,2,1] + assert lazy?(stream) + assert Enum.to_list(stream) == [1, 2, 3] + assert Process.get(:stream_each) == [3, 2, 1] end test "filter/2" do - stream = Stream.filter([1,2,3], fn(x) -> rem(x, 2) == 0 end) - assert is_lazy(stream) + stream = Stream.filter([1, 2, 3], fn x -> rem(x, 2) == 0 end) + assert lazy?(stream) assert Enum.to_list(stream) == [2] nats = Stream.iterate(1, &(&1 + 1)) - assert Stream.filter(nats, &(rem(&1, 2) == 0)) |> Enum.take(5) == [2,4,6,8,10] - end - - test "filter_map/3" do - stream = Stream.filter_map([1,2,3], fn(x) -> rem(x, 2) == 0 end, &(&1 * 2)) - assert is_lazy(stream) - assert Enum.to_list(stream) == [4] - - nats = Stream.iterate(1, &(&1 + 1)) - assert Stream.filter_map(nats, &(rem(&1, 2) == 0), &(&1 * 2)) - |> Enum.take(5) == [4,8,12,16,20] + assert Stream.filter(nats, &(rem(&1, 2) == 0)) |> Enum.take(5) == [2, 4, 6, 8, 10] end test "flat_map/2" do stream = Stream.flat_map([1, 2, 3], &[&1, &1 * 2]) - assert is_lazy(stream) + assert lazy?(stream) assert Enum.to_list(stream) == [1, 2, 2, 4, 3, 6] nats = Stream.iterate(1, &(&1 + 1)) @@ -222,59 +434,67 @@ defmodule StreamTest do test "flat_map/2 does not intercept wrapped lazy enumeration" do # flat_map returns a lazy enumeration that does not halt assert [1, 2, 3, -1, -2] - |> Stream.flat_map(fn x -> Stream.map([x, x+1], & &1) end) + |> Stream.flat_map(fn x -> Stream.map([x, x + 1], & &1) end) |> Stream.take_while(fn x -> x >= 0 end) - |> Enum.to_list == [1, 2, 2, 3, 3, 4] + |> Enum.to_list() == [1, 2, 2, 3, 3, 4] # flat_map returns a lazy enumeration that does halts assert [1, 2, 3, -1, -2] - |> Stream.flat_map(fn x -> Stream.take_while([x, x+1, x+2], &(&1 <= x + 1)) end) + |> Stream.flat_map(fn x -> Stream.take_while([x, x + 1, x + 2], &(&1 <= x + 1)) end) |> Stream.take_while(fn x -> x >= 0 end) - |> Enum.to_list == [1, 2, 2, 3, 3, 4] + |> Enum.to_list() == [1, 2, 2, 3, 3, 4] # flat_map returns a lazy enumeration that does halts wrapped in an enumerable assert [1, 2, 3, -1, -2] - |> Stream.flat_map(fn x -> Stream.concat([x], Stream.take_while([x+1, x+2], &(&1 <= x + 1))) end) + |> Stream.flat_map(fn x -> + Stream.concat([x], Stream.take_while([x + 1, x + 2], &(&1 <= x + 1))) + end) |> Stream.take_while(fn x -> x >= 0 end) - |> Enum.to_list == [1, 2, 2, 3, 3, 4] + |> Enum.to_list() == [1, 2, 2, 3, 3, 4] end test "flat_map/2 is zippable" do - stream = [1, 2, 3, -1, -2] - |> Stream.flat_map(fn x -> Stream.map([x, x+1], & &1) end) - |> Stream.take_while(fn x -> x >= 0 end) - list = Enum.to_list(stream) + stream = + [1, 2, 3, -1, -2] + |> Stream.flat_map(fn x -> Stream.map([x, x + 1], & &1) end) + |> Stream.take_while(fn x -> x >= 0 end) + + list = Enum.to_list(stream) assert Enum.zip(list, list) == Enum.zip(stream, stream) end test "flat_map/2 does not leave inner stream suspended" do - stream = Stream.flat_map [1,2,3], - fn i -> - Stream.resource(fn -> i end, - fn acc -> {acc, acc + 1} end, - fn _ -> Process.put(:stream_flat_map, true) end) - end + stream = + Stream.flat_map([1, 2, 3], fn i -> + Stream.resource(fn -> i end, fn acc -> {[acc], acc + 1} end, fn _ -> + Process.put(:stream_flat_map, true) + end) + end) Process.put(:stream_flat_map, false) - assert stream |> Enum.take(3) == [1,2,3] + assert stream |> Enum.take(3) == [1, 2, 3] assert Process.get(:stream_flat_map) end test "flat_map/2 does not leave outer stream suspended" do - stream = Stream.resource(fn -> 1 end, - fn acc -> {acc, acc + 1} end, - fn _ -> Process.put(:stream_flat_map, true) end) + stream = + Stream.resource(fn -> 1 end, fn acc -> {[acc], acc + 1} end, fn _ -> + Process.put(:stream_flat_map, true) + end) + stream = Stream.flat_map(stream, fn i -> [i, i + 1, i + 2] end) Process.put(:stream_flat_map, false) - assert stream |> Enum.take(3) == [1,2,3] + assert stream |> Enum.take(3) == [1, 2, 3] assert Process.get(:stream_flat_map) end test "flat_map/2 closes on error" do - stream = Stream.resource(fn -> 1 end, - fn acc -> {acc, acc + 1} end, - fn _ -> Process.put(:stream_flat_map, true) end) + stream = + Stream.resource(fn -> 1 end, fn acc -> {[acc], acc + 1} end, fn _ -> + Process.put(:stream_flat_map, true) + end) + stream = Stream.flat_map(stream, fn _ -> throw(:error) end) Process.put(:stream_flat_map, false) @@ -282,20 +502,58 @@ defmodule StreamTest do assert Process.get(:stream_flat_map) end + test "flat_map/2 with inner flat_map/2" do + stream = + Stream.flat_map(1..5, fn x -> + Stream.flat_map([x], fn x -> + x..(x * x) + end) + |> Stream.map(&(&1 * 1)) + end) + + assert Enum.take(stream, 5) == [1, 2, 3, 4, 3] + end + + test "flat_map/2 properly halts both inner and outer stream when inner stream is halted" do + # Fixes a bug that, when the inner stream was done, + # sending it a halt would cause it to return the + # inner stream was halted, forcing flat_map to get + # the next value from the outer stream, evaluate it, + # get another inner stream, just to halt it. + # 2 should never be used + assert [1, 2] + |> Stream.flat_map(fn 1 -> Stream.repeatedly(fn -> 1 end) end) + |> Stream.flat_map(fn 1 -> Stream.repeatedly(fn -> 1 end) end) + |> Enum.take(1) == [1] + end + + test "interval/1" do + stream = Stream.interval(10) + {time_us, value} = :timer.tc(fn -> Enum.take(stream, 5) end) + + assert value == [0, 1, 2, 3, 4] + assert time_us >= 50000 + end + + test "interval/1 with infinity" do + stream = Stream.interval(:infinity) + spawn(Stream, :run, [stream]) + end + test "into/2 and run/1" do Process.put(:stream_cont, []) Process.put(:stream_done, false) Process.put(:stream_halt, false) - stream = Stream.into([1, 2, 3], collectable_pdict) + stream = Stream.into([1, 2, 3], %Pdict{}) - assert is_lazy(stream) + assert lazy?(stream) assert Stream.run(stream) == :ok - assert Process.get(:stream_cont) == [3,2,1] + assert Process.get(:stream_cont) == [3, 2, 1] assert Process.get(:stream_done) refute Process.get(:stream_halt) - stream = Stream.into(fn _, _ -> raise "error" end, collectable_pdict) + stream = Stream.into(fn _, _ -> raise "error" end, %Pdict{}) catch_error(Stream.run(stream)) assert Process.get(:stream_halt) end @@ -305,9 +563,9 @@ defmodule StreamTest do Process.put(:stream_done, false) Process.put(:stream_halt, false) - stream = Stream.into([1, 2, 3], collectable_pdict, fn x -> x*2 end) + stream = Stream.into([1, 2, 3], %Pdict{}, fn x -> x * 2 end) - assert is_lazy(stream) + assert lazy?(stream) assert Enum.to_list(stream) == [1, 2, 3] assert Process.get(:stream_cont) == [6, 4, 2] assert Process.get(:stream_done) @@ -319,9 +577,9 @@ defmodule StreamTest do Process.put(:stream_done, false) Process.put(:stream_halt, false) - stream = Stream.into([1, 2, 3], collectable_pdict) + stream = Stream.into([1, 2, 3], %Pdict{}) - assert is_lazy(stream) + assert lazy?(stream) assert Enum.take(stream, 1) == [1] assert Process.get(:stream_cont) == [1] assert Process.get(:stream_done) @@ -330,29 +588,60 @@ defmodule StreamTest do test "transform/3" do stream = Stream.transform([1, 2, 3], 0, &{[&1, &2], &1 + &2}) - assert is_lazy(stream) + assert lazy?(stream) assert Enum.to_list(stream) == [1, 0, 2, 1, 3, 3] nats = Stream.iterate(1, &(&1 + 1)) assert Stream.transform(nats, 0, &{[&1, &2], &1 + &2}) |> Enum.take(6) == [1, 0, 2, 1, 3, 3] end + test "transform/3 with early halt" do + stream = + fn -> throw(:error) end + |> Stream.repeatedly() + |> Stream.transform(nil, &{[&1, &2], &1}) + + assert {:halted, nil} = Enumerable.reduce(stream, {:halt, nil}, fn _, _ -> throw(:error) end) + end + + test "transform/3 with early suspend" do + stream = + Stream.repeatedly(fn -> throw(:error) end) + |> Stream.transform(nil, &{[&1, &2], &1}) + + assert {:suspended, nil, _} = + Enumerable.reduce(stream, {:suspend, nil}, fn _, _ -> throw(:error) end) + end + test "transform/3 with halt" do - stream = Stream.resource(fn -> 1 end, - fn acc -> {acc, acc + 1} end, - fn _ -> Process.put(:stream_transform, true) end) - stream = Stream.transform(stream, 0, fn i, acc -> if acc < 3, do: {[i], acc + 1}, else: {:halt, acc} end) + stream = + Stream.resource(fn -> 1 end, fn acc -> {[acc], acc + 1} end, fn _ -> + Process.put(:stream_transform, true) + end) + + stream = + Stream.transform(stream, 0, fn i, acc -> + if acc < 3, do: {[i], acc + 1}, else: {:halt, acc} + end) Process.put(:stream_transform, false) - assert Enum.to_list(stream) == [1,2,3] + assert Enum.to_list(stream) == [1, 2, 3] assert Process.get(:stream_transform) end + test "transform/3 (via flat_map) handles multiple returns from suspension" do + assert [false] + |> Stream.take(1) + |> Stream.concat([true]) + |> Stream.flat_map(&[&1]) + |> Enum.to_list() == [false, true] + end + test "iterate/2" do - stream = Stream.iterate(0, &(&1+2)) - assert Enum.take(stream, 5) == [0,2,4,6,8] - stream = Stream.iterate(5, &(&1+2)) - assert Enum.take(stream, 5) == [5,7,9,11,13] + stream = Stream.iterate(0, &(&1 + 2)) + assert Enum.take(stream, 5) == [0, 2, 4, 6, 8] + stream = Stream.iterate(5, &(&1 + 2)) + assert Enum.take(stream, 5) == [5, 7, 9, 11, 13] # Only calculate values if needed stream = Stream.iterate("HELLO", &raise/1) @@ -360,49 +649,173 @@ defmodule StreamTest do end test "map/2" do - stream = Stream.map([1,2,3], &(&1 * 2)) - assert is_lazy(stream) - assert Enum.to_list(stream) == [2,4,6] + stream = Stream.map([1, 2, 3], &(&1 * 2)) + assert lazy?(stream) + assert Enum.to_list(stream) == [2, 4, 6] nats = Stream.iterate(1, &(&1 + 1)) - assert Stream.map(nats, &(&1 * 2)) |> Enum.take(5) == [2,4,6,8,10] + assert Stream.map(nats, &(&1 * 2)) |> Enum.take(5) == [2, 4, 6, 8, 10] assert Stream.map(nats, &(&1 - 2)) |> Stream.map(&(&1 * 2)) |> Enum.take(3) == [-2, 0, 2] end + test "map_every/3" do + assert 1..10 + |> Stream.map_every(2, &(&1 * 2)) + |> Enum.to_list() == [2, 2, 6, 4, 10, 6, 14, 8, 18, 10] + + assert 1..10 + |> Stream.map_every(3, &(&1 * 2)) + |> Enum.to_list() == [2, 2, 3, 8, 5, 6, 14, 8, 9, 20] + + assert 1..10 + |> Stream.drop(2) + |> Stream.map_every(2, &(&1 * 2)) + |> Stream.drop(1) + |> Enum.to_list() == [4, 10, 6, 14, 8, 18, 10] + + assert 1..5 + |> Stream.map_every(0, &(&1 * 2)) + |> Enum.to_list() == [1, 2, 3, 4, 5] + + assert [] + |> Stream.map_every(10, &(&1 * 2)) + |> Enum.to_list() == [] + + assert_raise FunctionClauseError, fn -> + Stream.map_every(1..10, -1, &(&1 * 2)) + end + + assert_raise FunctionClauseError, fn -> + Stream.map_every(1..10, 3.33, &(&1 * 2)) + end + end + test "reject/2" do - stream = Stream.reject([1,2,3], fn(x) -> rem(x, 2) == 0 end) - assert is_lazy(stream) - assert Enum.to_list(stream) == [1,3] + stream = Stream.reject([1, 2, 3], fn x -> rem(x, 2) == 0 end) + assert lazy?(stream) + assert Enum.to_list(stream) == [1, 3] nats = Stream.iterate(1, &(&1 + 1)) - assert Stream.reject(nats, &(rem(&1, 2) == 0)) |> Enum.take(5) == [1,3,5,7,9] + assert Stream.reject(nats, &(rem(&1, 2) == 0)) |> Enum.take(5) == [1, 3, 5, 7, 9] end test "repeatedly/1" do stream = Stream.repeatedly(fn -> 1 end) - assert Enum.take(stream, 5) == [1,1,1,1,1] - stream = Stream.repeatedly(&:random.uniform/0) - [r1,r2] = Enum.take(stream, 2) + assert Enum.take(stream, 5) == [1, 1, 1, 1, 1] + stream = Stream.repeatedly(&:rand.uniform/0) + [r1, r2] = Enum.take(stream, 2) assert r1 != r2 end - test "resource/3 closes on errors" do - stream = Stream.resource(fn -> 1 end, - fn acc -> {acc, acc + 1} end, - fn _ -> Process.put(:stream_resource, true) end) + test "resource/3 closes on outer errors" do + stream = + Stream.resource( + fn -> 1 end, + fn + 2 -> throw(:error) + acc -> {[acc], acc + 1} + end, + fn 2 -> Process.put(:stream_resource, true) end + ) Process.put(:stream_resource, false) - stream = Stream.map(stream, fn x -> if x > 2, do: throw(:error), else: x end) assert catch_throw(Enum.to_list(stream)) == :error assert Process.get(:stream_resource) end + test "resource/3 closes with correct accumulator on outer errors with inner single-element list" do + stream = + Stream.resource( + fn -> :start end, + fn _ -> {[:error], :end} end, + fn acc -> Process.put(:stream_resource, acc) end + ) + |> Stream.map(fn :error -> throw(:error) end) + + Process.put(:stream_resource, nil) + assert catch_throw(Enum.to_list(stream)) == :error + assert Process.get(:stream_resource) == :end + end + + test "resource/3 closes with correct accumulator on outer errors with inner list" do + stream = + Stream.resource( + fn -> :start end, + fn _ -> {[:ok, :error], :end} end, + fn acc -> Process.put(:stream_resource, acc) end + ) + |> Stream.map(fn acc -> if acc == :error, do: throw(:error), else: acc end) + + Process.put(:stream_resource, nil) + assert catch_throw(Enum.to_list(stream)) == :error + assert Process.get(:stream_resource) == :end + end + + test "resource/3 closes with correct accumulator on outer errors with inner enum" do + stream = + Stream.resource( + fn -> 1 end, + fn acc -> {acc..(acc + 2), acc + 1} end, + fn acc -> Process.put(:stream_resource, acc) end + ) + |> Stream.map(fn x -> if x > 2, do: throw(:error), else: x end) + + Process.put(:stream_resource, nil) + assert catch_throw(Enum.to_list(stream)) == :error + assert Process.get(:stream_resource) == 2 + end + test "resource/3 is zippable" do - stream = Stream.resource(fn -> 1 end, - fn 10 -> nil - acc -> {acc, acc + 1} - end, - fn _ -> Process.put(:stream_resource, true) end) + transform_fun = fn + 10 -> {:halt, 10} + acc -> {[acc], acc + 1} + end + + after_fun = fn _ -> Process.put(:stream_resource, true) end + stream = Stream.resource(fn -> 1 end, transform_fun, after_fun) + + list = Enum.to_list(stream) + Process.put(:stream_resource, false) + assert Enum.zip(list, list) == Enum.zip(stream, stream) + assert Process.get(:stream_resource) + end + + test "resource/3 returning inner empty list" do + transform_fun = fn acc -> if rem(acc, 2) == 0, do: {[], acc + 1}, else: {[acc], acc + 1} end + stream = Stream.resource(fn -> 1 end, transform_fun, fn _ -> :ok end) + + assert Enum.take(stream, 5) == [1, 3, 5, 7, 9] + end + + test "resource/3 halts with inner list" do + transform_fun = fn acc -> {[acc, acc + 1, acc + 2], acc + 1} end + after_fun = fn _ -> Process.put(:stream_resource, true) end + stream = Stream.resource(fn -> 1 end, transform_fun, after_fun) + + Process.put(:stream_resource, false) + assert Enum.take(stream, 5) == [1, 2, 3, 2, 3] + assert Process.get(:stream_resource) + end + + test "resource/3 closes on errors with inner list" do + transform_fun = fn acc -> {[acc, acc + 1, acc + 2], acc + 1} end + after_fun = fn _ -> Process.put(:stream_resource, true) end + stream = Stream.resource(fn -> 1 end, transform_fun, after_fun) + + Process.put(:stream_resource, false) + stream = Stream.map(stream, fn x -> if x > 2, do: throw(:error), else: x end) + assert catch_throw(Enum.to_list(stream)) == :error + assert Process.get(:stream_resource) + end + + test "resource/3 is zippable with inner list" do + transform_fun = fn + 10 -> {:halt, 10} + acc -> {[acc, acc + 1, acc + 2], acc + 1} + end + + after_fun = fn _ -> Process.put(:stream_resource, true) end + stream = Stream.resource(fn -> 1 end, transform_fun, after_fun) list = Enum.to_list(stream) Process.put(:stream_resource, false) @@ -410,159 +823,625 @@ defmodule StreamTest do assert Process.get(:stream_resource) end + test "resource/3 halts with inner enum" do + transform_fun = fn acc -> {acc..(acc + 2), acc + 1} end + after_fun = fn _ -> Process.put(:stream_resource, true) end + stream = Stream.resource(fn -> 1 end, transform_fun, after_fun) + + Process.put(:stream_resource, false) + assert Enum.take(stream, 5) == [1, 2, 3, 2, 3] + assert Process.get(:stream_resource) + end + + test "resource/3 closes on errors with inner enum" do + transform_fun = fn acc -> {acc..(acc + 2), acc + 1} end + after_fun = fn _ -> Process.put(:stream_resource, true) end + stream = Stream.resource(fn -> 1 end, transform_fun, after_fun) + + Process.put(:stream_resource, false) + stream = Stream.map(stream, fn x -> if x > 2, do: throw(:error), else: x end) + assert catch_throw(Enum.to_list(stream)) == :error + assert Process.get(:stream_resource) + end + + test "resource/3 is zippable with inner enum" do + transform_fun = fn + 10 -> {:halt, 10} + acc -> {acc..(acc + 2), acc + 1} + end + + after_fun = fn _ -> Process.put(:stream_resource, true) end + stream = Stream.resource(fn -> 1 end, transform_fun, after_fun) + + list = Enum.to_list(stream) + Process.put(:stream_resource, false) + assert Enum.zip(list, list) == Enum.zip(stream, stream) + assert Process.get(:stream_resource) + end + + test "transform/4" do + transform_fun = fn x, acc -> {[x, x + acc], x} end + after_fun = fn 10 -> Process.put(:stream_transform, true) end + stream = Stream.transform(1..10, fn -> 0 end, transform_fun, after_fun) + Process.put(:stream_transform, false) + + assert Enum.to_list(stream) == + [1, 1, 2, 3, 3, 5, 4, 7, 5, 9, 6, 11, 7, 13, 8, 15, 9, 17, 10, 19] + + assert Process.get(:stream_transform) + end + + test "transform/4 with early halt" do + after_fun = fn nil -> Process.put(:stream_transform, true) end + + stream = + fn -> throw(:error) end + |> Stream.repeatedly() + |> Stream.transform(fn -> nil end, &{[&1, &2], &1}, after_fun) + + Process.put(:stream_transform, false) + assert {:halted, nil} = Enumerable.reduce(stream, {:halt, nil}, fn _, _ -> throw(:error) end) + assert Process.get(:stream_transform) + end + + test "transform/4 with early suspend" do + after_fun = fn nil -> Process.put(:stream_transform, true) end + + stream = + fn -> throw(:error) end + |> Stream.repeatedly() + |> Stream.transform(fn -> nil end, &{[&1, &2], &1}, after_fun) + + refute Process.get(:stream_transform) + + assert {:suspended, nil, _} = + Enumerable.reduce(stream, {:suspend, nil}, fn _, _ -> throw(:error) end) + end + + test "transform/4 closes on outer errors" do + transform_fun = fn + 3, _ -> throw(:error) + x, acc -> {[x + acc], x} + end + + after_fun = fn 2 -> Process.put(:stream_transform, true) end + + stream = Stream.transform(1..10, fn -> 0 end, transform_fun, after_fun) + + Process.put(:stream_transform, false) + assert catch_throw(Enum.to_list(stream)) == :error + assert Process.get(:stream_transform) + end + + test "transform/4 closes on nested errors" do + transform_fun = fn + 3, _ -> throw(:error) + x, acc -> {[x + acc], x} + end + + after_fun = fn _ -> Process.put(:stream_transform_inner, true) end + outer_after_fun = fn 0 -> Process.put(:stream_transform_outer, true) end + + stream = + 1..10 + |> Stream.transform(fn -> 0 end, transform_fun, after_fun) + |> Stream.transform(fn -> 0 end, fn x, acc -> {[x], acc} end, outer_after_fun) + + Process.put(:stream_transform_inner, false) + Process.put(:stream_transform_outer, false) + assert catch_throw(Enum.to_list(stream)) == :error + assert Process.get(:stream_transform_inner) + assert Process.get(:stream_transform_outer) + end + + test "transform/4 is zippable" do + transform_fun = fn + 10, acc -> {:halt, acc} + x, acc -> {[x + acc], x} + end + + after_fun = fn 9 -> Process.put(:stream_transform, true) end + stream = Stream.transform(1..20, fn -> 0 end, transform_fun, after_fun) + + list = Enum.to_list(stream) + Process.put(:stream_transform, false) + assert Enum.zip(list, list) == Enum.zip(stream, stream) + assert Process.get(:stream_transform) + end + + test "transform/4 halts with inner list" do + transform_fun = fn x, acc -> {[x, x + 1, x + 2], acc} end + after_fun = fn :acc -> Process.put(:stream_transform, true) end + stream = Stream.transform(1..10, fn -> :acc end, transform_fun, after_fun) + + Process.put(:stream_transform, false) + assert Enum.take(stream, 5) == [1, 2, 3, 2, 3] + assert Process.get(:stream_transform) + end + + test "transform/4 closes on errors with inner list" do + transform_fun = fn x, acc -> {[x, x + 1, x + 2], acc} end + after_fun = fn :acc -> Process.put(:stream_transform, true) end + stream = Stream.transform(1..10, fn -> :acc end, transform_fun, after_fun) + + Process.put(:stream_transform, false) + stream = Stream.map(stream, fn x -> if x > 2, do: throw(:error), else: x end) + assert catch_throw(Enum.to_list(stream)) == :error + assert Process.get(:stream_transform) + end + + test "transform/4 is zippable with inner list" do + transform_fun = fn + 10, acc -> {:halt, acc} + x, acc -> {[x, x + 1, x + 2], acc} + end + + after_fun = fn :inner -> Process.put(:stream_transform, true) end + + stream = Stream.transform(1..20, fn -> :inner end, transform_fun, after_fun) + + list = Enum.to_list(stream) + Process.put(:stream_transform, false) + assert Enum.zip(list, list) == Enum.zip(stream, stream) + assert Process.get(:stream_transform) + end + + test "transform/4 halts with inner enum" do + transform_fun = fn x, acc -> {x..(x + 2), acc} end + after_fun = fn :acc -> Process.put(:stream_transform, true) end + stream = Stream.transform(1..10, fn -> :acc end, transform_fun, after_fun) + + Process.put(:stream_transform, false) + assert Enum.take(stream, 5) == [1, 2, 3, 2, 3] + assert Process.get(:stream_transform) + end + + test "transform/4 closes on errors with inner enum" do + transform_fun = fn x, acc -> {x..(x + 2), acc} end + after_fun = fn :acc -> Process.put(:stream_transform, true) end + stream = Stream.transform(1..10, fn -> :acc end, transform_fun, after_fun) + + Process.put(:stream_transform, false) + stream = Stream.map(stream, fn x -> if x > 2, do: throw(:error), else: x end) + assert catch_throw(Enum.to_list(stream)) == :error + assert Process.get(:stream_transform) + end + + test "transform/4 is zippable with inner enum" do + transform_fun = fn + 10, acc -> {:halt, acc} + x, acc -> {x..(x + 2), acc} + end + + after_fun = fn :inner -> Process.put(:stream_transform, true) end + stream = Stream.transform(1..20, fn -> :inner end, transform_fun, after_fun) + + list = Enum.to_list(stream) + Process.put(:stream_transform, false) + assert Enum.zip(list, list) == Enum.zip(stream, stream) + assert Process.get(:stream_transform) + end + + test "transform/5 emits last elements on done" do + stream = + Stream.transform( + 1..5//2, + fn -> 0 end, + fn i, _acc -> {i..(i + 1), i + 1} end, + fn 6 -> {7..10, 10} end, + fn i when is_integer(i) -> Process.put(__MODULE__, i) end + ) + + assert Enum.to_list(stream) == [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + assert Process.get(__MODULE__) == 10 + + assert Enum.take(stream, 3) == [1, 2, 3] + assert Process.get(__MODULE__) == 4 + + assert Enum.take(stream, 4) == [1, 2, 3, 4] + assert Process.get(__MODULE__) == 4 + + assert Enum.take(stream, 7) == [1, 2, 3, 4, 5, 6, 7] + assert Process.get(__MODULE__) == 10 + end + + test "transform/5 emits last elements on inner halt done" do + stream = + Stream.transform( + Stream.take(1..15//2, 3), + fn -> 0 end, + fn i, _acc -> {i..(i + 1), i + 1} end, + fn 6 -> {7..10, 10} end, + fn i when is_integer(i) -> Process.put(__MODULE__, i) end + ) + + assert Enum.to_list(stream) == [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + assert Process.get(__MODULE__) == 10 + + assert Enum.take(stream, 3) == [1, 2, 3] + assert Process.get(__MODULE__) == 4 + + assert Enum.take(stream, 4) == [1, 2, 3, 4] + assert Process.get(__MODULE__) == 4 + + assert Enum.take(stream, 7) == [1, 2, 3, 4, 5, 6, 7] + assert Process.get(__MODULE__) == 10 + end + + test "transform/5 does not halt twice" do + resource_start = fn -> 0 end + + resource_next = fn current -> + if current < 5 do + {[current], current + 1} + else + {:halt, current} + end + end + + resource_after = fn _ -> + send(self(), {:halted, :resource}) + end + + transform_next = fn current, index -> {[current + 1], index} end + transform_last = fn index -> {:halt, index} end + + transform_after = fn _ -> + send(self(), {:halted, :transform}) + end + + Stream.resource(resource_start, resource_next, resource_after) + |> Stream.transform(fn -> 1 end, transform_next, transform_last, transform_after) + |> Stream.run() + + assert_received {:halted, :resource} + assert_received {:halted, :transform} + refute_received {:halted, :resource} + refute_received {:halted, :transform} + end + test "scan/2" do stream = Stream.scan(1..5, &(&1 + &2)) - assert is_lazy(stream) - assert Enum.to_list(stream) == [1,3,6,10,15] - assert Stream.scan([], &(&1 + &2)) |> Enum.to_list == [] + assert lazy?(stream) + assert Enum.to_list(stream) == [1, 3, 6, 10, 15] + assert Stream.scan([], &(&1 + &2)) |> Enum.to_list() == [] end test "scan/3" do stream = Stream.scan(1..5, 0, &(&1 + &2)) - assert is_lazy(stream) - assert Enum.to_list(stream) == [1,3,6,10,15] - assert Stream.scan([], 0, &(&1 + &2)) |> Enum.to_list == [] + assert lazy?(stream) + assert Enum.to_list(stream) == [1, 3, 6, 10, 15] + assert Stream.scan([], 0, &(&1 + &2)) |> Enum.to_list() == [] end test "take/2" do stream = Stream.take(1..1000, 5) - assert is_lazy(stream) - assert Enum.to_list(stream) == [1,2,3,4,5] + assert lazy?(stream) + assert Enum.to_list(stream) == [1, 2, 3, 4, 5] assert Enum.to_list(Stream.take(1..1000, 0)) == [] - assert Enum.to_list(Stream.take(1..3, 5)) == [1,2,3] + assert Enum.to_list(Stream.take([], 5)) == [] + assert Enum.to_list(Stream.take(1..3, 5)) == [1, 2, 3] nats = Stream.iterate(1, &(&1 + 1)) - assert Enum.to_list(Stream.take(nats, 5)) == [1,2,3,4,5] + assert Enum.to_list(Stream.take(nats, 5)) == [1, 2, 3, 4, 5] stream = Stream.drop(1..100, 5) - assert Stream.take(stream, 5) |> Enum.to_list == [6,7,8,9,10] + assert Stream.take(stream, 5) |> Enum.to_list() == [6, 7, 8, 9, 10] stream = 1..5 |> Stream.take(10) |> Stream.drop(15) assert {[], []} = Enum.split(stream, 5) stream = 1..20 |> Stream.take(10 + 5) |> Stream.drop(4) - assert Enum.to_list(stream) == [5,6,7,8,9,10,11,12,13,14,15] + assert Enum.to_list(stream) == [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] + end + + test "take/2 does not consume next element on halt" do + assert [false, true] + |> Stream.each(&(&1 && raise("oops"))) + |> Stream.take(1) + |> Stream.take_while(& &1) + |> Enum.to_list() == [] + end + + test "take/2 does not consume next element on suspend" do + assert [false, true] + |> Stream.each(&(&1 && raise("oops"))) + |> Stream.take(1) + |> Stream.flat_map(&[&1]) + |> Enum.to_list() == [false] end test "take/2 with negative count" do Process.put(:stream_each, []) stream = Stream.take(1..100, -5) - assert is_lazy(stream) + assert lazy?(stream) - stream = Stream.each(stream, &Process.put(:stream_each, [&1|Process.get(:stream_each)])) - assert Enum.to_list(stream) == [96,97,98,99,100] - assert Process.get(:stream_each) == [100,99,98,97,96] + stream = Stream.each(stream, &Process.put(:stream_each, [&1 | Process.get(:stream_each)])) + assert Enum.to_list(stream) == [96, 97, 98, 99, 100] + assert Process.get(:stream_each) == [100, 99, 98, 97, 96] end test "take/2 is zippable" do stream = Stream.take(1..1000, 5) - list = Enum.to_list(stream) + list = Enum.to_list(stream) assert Enum.zip(list, list) == Enum.zip(stream, stream) end test "take_every/2" do assert 1..10 |> Stream.take_every(2) - |> Enum.to_list == [1, 3, 5, 7, 9] + |> Enum.to_list() == [1, 3, 5, 7, 9] + + assert 1..10 + |> Stream.take_every(3) + |> Enum.to_list() == [1, 4, 7, 10] assert 1..10 |> Stream.drop(2) |> Stream.take_every(2) |> Stream.drop(1) - |> Enum.to_list == [5, 7, 9] + |> Enum.to_list() == [5, 7, 9] + + assert 1..10 + |> Stream.take_every(0) + |> Enum.to_list() == [] + + assert [] + |> Stream.take_every(10) + |> Enum.to_list() == [] + end + + test "take_every/2 without non-negative integer" do + assert_raise FunctionClauseError, fn -> + Stream.take_every(1..10, -1) + end + + assert_raise FunctionClauseError, fn -> + Stream.take_every(1..10, 3.33) + end end test "take_while/2" do stream = Stream.take_while(1..1000, &(&1 <= 5)) - assert is_lazy(stream) - assert Enum.to_list(stream) == [1,2,3,4,5] + assert lazy?(stream) + assert Enum.to_list(stream) == [1, 2, 3, 4, 5] assert Enum.to_list(Stream.take_while(1..1000, &(&1 <= 0))) == [] - assert Enum.to_list(Stream.take_while(1..3, &(&1 <= 5))) == [1,2,3] + assert Enum.to_list(Stream.take_while(1..3, &(&1 <= 5))) == [1, 2, 3] nats = Stream.iterate(1, &(&1 + 1)) - assert Enum.to_list(Stream.take_while(nats, &(&1 <= 5))) == [1,2,3,4,5] + assert Enum.to_list(Stream.take_while(nats, &(&1 <= 5))) == [1, 2, 3, 4, 5] stream = Stream.drop(1..100, 5) - assert Stream.take_while(stream, &(&1 < 11)) |> Enum.to_list == [6,7,8,9,10] + assert Stream.take_while(stream, &(&1 < 11)) |> Enum.to_list() == [6, 7, 8, 9, 10] + end + + test "timer/1" do + stream = Stream.timer(10) + + {time_us, value} = :timer.tc(fn -> Enum.to_list(stream) end) + + assert value == [0] + # We check for >= 5000 (us) instead of >= 10000 (us) + # because the resolution on Windows system is not high + # enough and we would get a difference of 9000 from + # time to time. So a value halfway is good enough. + assert time_us >= 5000 + end + + test "timer/1 with infinity" do + stream = Stream.timer(:infinity) + spawn(Stream, :run, [stream]) end test "unfold/2" do - stream = Stream.unfold(10, fn x -> if x > 0, do: {x, x-1}, else: nil end) + stream = Stream.unfold(10, fn x -> if x > 0, do: {x, x - 1} end) assert Enum.take(stream, 5) == [10, 9, 8, 7, 6] - stream = Stream.unfold(5, fn x -> if x > 0, do: {x, x-1}, else: nil end) + stream = Stream.unfold(5, fn x -> if x > 0, do: {x, x - 1} end) assert Enum.to_list(stream) == [5, 4, 3, 2, 1] end test "unfold/2 only calculates values if needed" do - stream = Stream.unfold(1, fn x -> if x > 0, do: {x, x-1}, else: throw(:boom) end) + stream = Stream.unfold(1, fn x -> if x > 0, do: {x, x - 1}, else: throw(:boom) end) assert Enum.take(stream, 1) == [1] - stream = Stream.unfold(5, fn x -> if x > 0, do: {x, x-1}, else: nil end) + stream = Stream.unfold(5, fn x -> if x > 0, do: {x, x - 1} end) assert Enum.to_list(Stream.take(stream, 2)) == [5, 4] end test "unfold/2 is zippable" do - stream = Stream.unfold(10, fn x -> if x > 0, do: {x, x-1}, else: nil end) - list = Enum.to_list(stream) + stream = Stream.unfold(10, fn x -> if x > 0, do: {x, x - 1} end) + list = Enum.to_list(stream) assert Enum.zip(list, list) == Enum.zip(stream, stream) end - test "uniq/1" do - assert Stream.uniq([1, 2, 3, 2, 1]) |> Enum.to_list == - [1, 2, 3] + test "uniq/1 & uniq/2" do + assert Stream.uniq([1, 2, 3, 2, 1]) |> Enum.to_list() == [1, 2, 3] + end - assert Stream.uniq([{1, :x}, {2, :y}, {1, :z}], fn {x, _} -> x end) |> Enum.to_list == - [{1,:x}, {2,:y}] + test "uniq_by/2" do + assert Stream.uniq_by([{1, :x}, {2, :y}, {1, :z}], fn {x, _} -> x end) |> Enum.to_list() == + [{1, :x}, {2, :y}] + + assert Stream.uniq_by([a: {:tea, 2}, b: {:tea, 2}, c: {:coffee, 1}], fn {_, y} -> y end) + |> Enum.to_list() == [a: {:tea, 2}, c: {:coffee, 1}] end test "zip/2" do concat = Stream.concat(1..3, 4..6) - cycle = Stream.cycle([:a, :b, :c]) - assert Stream.zip(concat, cycle) |> Enum.to_list == - [{1,:a},{2,:b},{3,:c},{4,:a},{5,:b},{6,:c}] + cycle = Stream.cycle([:a, :b, :c]) + + assert Stream.zip(concat, cycle) |> Enum.to_list() == + [{1, :a}, {2, :b}, {3, :c}, {4, :a}, {5, :b}, {6, :c}] end - test "zip/2 does not leave streams suspended" do - stream = Stream.resource(fn -> 1 end, - fn acc -> {acc, acc + 1} end, - fn _ -> Process.put(:stream_zip, true) end) + test "zip_with/3" do + concat = Stream.concat(1..3, 4..6) + cycle = Stream.cycle([:a, :b, :c]) + zip_fun = &List.to_tuple([&1, &2]) + + assert Stream.zip_with(concat, cycle, zip_fun) |> Enum.to_list() == + [{1, :a}, {2, :b}, {3, :c}, {4, :a}, {5, :b}, {6, :c}] + + stream = Stream.concat(1..3, 4..6) + other_stream = fn _, _ -> {:cont, [1, 2]} end + result = Stream.zip_with(stream, other_stream, fn a, b -> a + b end) |> Enum.to_list() + assert result == [2, 4] + end + + test "zip_with/2" do + concat = Stream.concat(1..3, 4..6) + cycle = Stream.cycle([:a, :b, :c]) + zip_fun = &List.to_tuple/1 + + assert Stream.zip_with([concat, cycle], zip_fun) |> Enum.to_list() == + [{1, :a}, {2, :b}, {3, :c}, {4, :a}, {5, :b}, {6, :c}] + + assert Stream.chunk_every([0, 1, 2, 3], 2) |> Stream.zip_with(zip_fun) |> Enum.to_list() == + [{0, 2}, {1, 3}] + + stream = %HaltAcc{acc: 1..3} + assert Stream.zip_with([1..3, stream], zip_fun) |> Enum.to_list() == [{1, 1}, {2, 2}, {3, 3}] + + range_cycle = Stream.cycle(1..2) + + assert Stream.zip_with([1..3, range_cycle], zip_fun) |> Enum.to_list() == [ + {1, 1}, + {2, 2}, + {3, 1} + ] + end + + test "zip_with/2 does not leave streams suspended" do + zip_with_fun = &List.to_tuple/1 + + stream = + Stream.resource(fn -> 1 end, fn acc -> {[acc], acc + 1} end, fn _ -> + Process.put(:stream_zip_with, true) + end) + + Process.put(:stream_zip_with, false) + + assert Stream.zip_with([[:a, :b, :c], stream], zip_with_fun) |> Enum.to_list() == [ + a: 1, + b: 2, + c: 3 + ] + + assert Process.get(:stream_zip_with) + + Process.put(:stream_zip_with, false) + + assert Stream.zip_with([stream, [:a, :b, :c]], zip_with_fun) |> Enum.to_list() == [ + {1, :a}, + {2, :b}, + {3, :c} + ] + + assert Process.get(:stream_zip_with) + end + + test "zip_with/2 does not leave streams suspended on halt" do + zip_with_fun = &List.to_tuple/1 + + stream = + Stream.resource(fn -> 1 end, fn acc -> {[acc], acc + 1} end, fn _ -> + Process.put(:stream_zip_with, :done) + end) + + assert Stream.zip_with([[:a, :b, :c, :d, :e], stream], zip_with_fun) |> Enum.take(3) == [ + a: 1, + b: 2, + c: 3 + ] + + assert Process.get(:stream_zip_with) == :done + end + + test "zip_with/2 closes on inner error" do + zip_with_fun = &List.to_tuple/1 + stream = Stream.into([1, 2, 3], %Pdict{}) + + stream = + Stream.zip_with([stream, Stream.map([:a, :b, :c], fn _ -> throw(:error) end)], zip_with_fun) + + Process.put(:stream_done, false) + assert catch_throw(Enum.to_list(stream)) == :error + assert Process.get(:stream_done) + end + + test "zip_with/2 closes on outer error" do + zip_with_fun = &List.to_tuple/1 + + stream = + Stream.zip_with([Stream.into([1, 2, 3], %Pdict{}), [:a, :b, :c]], zip_with_fun) + |> Stream.map(fn _ -> throw(:error) end) + + Process.put(:stream_done, false) + assert catch_throw(Enum.to_list(stream)) == :error + assert Process.get(:stream_done) + end + + test "zip/1" do + concat = Stream.concat(1..3, 4..6) + cycle = Stream.cycle([:a, :b, :c]) + + assert Stream.zip([concat, cycle]) |> Enum.to_list() == + [{1, :a}, {2, :b}, {3, :c}, {4, :a}, {5, :b}, {6, :c}] + + assert Stream.chunk_every([0, 1, 2, 3], 2) |> Stream.zip() |> Enum.to_list() == + [{0, 2}, {1, 3}] + + assert Stream.zip([]) |> Enum.to_list() == [] + + stream = %HaltAcc{acc: 1..3} + assert Stream.zip([1..3, stream]) |> Enum.to_list() == [{1, 1}, {2, 2}, {3, 3}] + + range_cycle = Stream.cycle(1..2) + assert Stream.zip([1..3, range_cycle]) |> Enum.to_list() == [{1, 1}, {2, 2}, {3, 1}] + end + + test "zip/1 does not leave streams suspended" do + stream = + Stream.resource(fn -> 1 end, fn acc -> {[acc], acc + 1} end, fn _ -> + Process.put(:stream_zip, true) + end) Process.put(:stream_zip, false) - assert Stream.zip([:a, :b, :c], stream) |> Enum.to_list == [a: 1, b: 2, c: 3] + assert Stream.zip([[:a, :b, :c], stream]) |> Enum.to_list() == [a: 1, b: 2, c: 3] assert Process.get(:stream_zip) Process.put(:stream_zip, false) - assert Stream.zip(stream, [:a, :b, :c]) |> Enum.to_list == [{1, :a}, {2, :b}, {3, :c}] + assert Stream.zip([stream, [:a, :b, :c]]) |> Enum.to_list() == [{1, :a}, {2, :b}, {3, :c}] assert Process.get(:stream_zip) end - test "zip/2 does not leave streams suspended on halt" do - stream = Stream.resource(fn -> 1 end, - fn acc -> {acc, acc + 1} end, - fn _ -> Process.put(:stream_zip, :done) end) + test "zip/1 does not leave streams suspended on halt" do + stream = + Stream.resource(fn -> 1 end, fn acc -> {[acc], acc + 1} end, fn _ -> + Process.put(:stream_zip, :done) + end) - assert Stream.zip([:a, :b, :c, :d, :e], stream) |> Enum.take(3) == - [a: 1, b: 2, c: 3] + assert Stream.zip([[:a, :b, :c, :d, :e], stream]) |> Enum.take(3) == [a: 1, b: 2, c: 3] assert Process.get(:stream_zip) == :done end - test "zip/2 closes on inner error" do - stream = Stream.into([1, 2, 3], collectable_pdict) - stream = Stream.zip(stream, Stream.map([:a, :b, :c], fn _ -> throw(:error) end)) + test "zip/1 closes on inner error" do + stream = Stream.into([1, 2, 3], %Pdict{}) + stream = Stream.zip([stream, Stream.map([:a, :b, :c], fn _ -> throw(:error) end)]) Process.put(:stream_done, false) assert catch_throw(Enum.to_list(stream)) == :error assert Process.get(:stream_done) end - test "zip/2 closes on outer error" do - stream = Stream.into([1, 2, 3], collectable_pdict) - |> Stream.zip([:a, :b, :c]) - |> Stream.map(fn _ -> throw(:error) end) + test "zip/1 closes on outer error" do + stream = + Stream.zip([Stream.into([1, 2, 3], %Pdict{}), [:a, :b, :c]]) + |> Stream.map(fn _ -> throw(:error) end) Process.put(:stream_done, false) assert catch_throw(Enum.to_list(stream)) == :error @@ -570,24 +1449,45 @@ defmodule StreamTest do end test "with_index/2" do - stream = Stream.with_index([1,2,3]) - assert is_lazy(stream) - assert Enum.to_list(stream) == [{1,0},{2,1},{3,2}] + stream = Stream.with_index([1, 2, 3]) + assert lazy?(stream) + assert Enum.to_list(stream) == [{1, 0}, {2, 1}, {3, 2}] + + stream = Stream.with_index([1, 2, 3], 10) + assert Enum.to_list(stream) == [{1, 10}, {2, 11}, {3, 12}] nats = Stream.iterate(1, &(&1 + 1)) - assert Stream.with_index(nats) |> Enum.take(3) == [{1,0},{2,1},{3,2}] + assert Stream.with_index(nats) |> Enum.take(3) == [{1, 0}, {2, 1}, {3, 2}] end - defp is_lazy(stream) do - match?(%Stream{}, stream) or is_function(stream, 2) + test "intersperse/2 is lazy" do + assert lazy?(Stream.intersperse([], 0)) end - defp collectable_pdict do - fn - _, {:cont, x} -> Process.put(:stream_cont, [x|Process.get(:stream_cont)]) - _, :done -> Process.put(:stream_done, true) - _, :halt -> Process.put(:stream_halt, true) - end + test "intersperse/2 on an empty list" do + assert Enum.to_list(Stream.intersperse([], 0)) == [] + end + + test "intersperse/2 on a single element list" do + assert Enum.to_list(Stream.intersperse([1], 0)) == [1] + end + + test "intersperse/2 on a multiple elements list" do + assert Enum.to_list(Stream.intersperse(1..3, 0)) == [1, 0, 2, 0, 3] + end + + test "intersperse/2 is zippable" do + stream = Stream.intersperse(1..10, 0) + list = Enum.to_list(stream) + assert Enum.zip(list, list) == Enum.zip(stream, stream) + end + + test "inspect/1" do + "#Stream<[enum: 1..10, funs: " <> _ = Stream.map(1..10, & &1) |> inspect() + end + + defp lazy?(stream) do + match?(%Stream{}, stream) or is_function(stream, 2) end defp inbox_stream({:suspend, acc}, f) do @@ -600,8 +1500,8 @@ defmodule StreamTest do defp inbox_stream({:cont, acc}, f) do receive do - {:stream, item} -> - inbox_stream(f.(item, acc), f) + {:stream, element} -> + inbox_stream(f.(element, acc), f) end end end diff --git a/lib/elixir/test/elixir/string/chars_test.exs b/lib/elixir/test/elixir/string/chars_test.exs index 173439287bb..875820e86b8 100644 --- a/lib/elixir/test/elixir/string/chars_test.exs +++ b/lib/elixir/test/elixir/string/chars_test.exs @@ -1,28 +1,34 @@ -Code.require_file "../test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) defmodule String.Chars.AtomTest do use ExUnit.Case, async: true - test :basic do + doctest String.Chars + + test "basic" do assert to_string(:foo) == "foo" end - test :empty do + test "empty" do assert to_string(:"") == "" end - test :true_false_nil do + test "true false nil" do assert to_string(false) == "false" assert to_string(true) == "true" assert to_string(nil) == "" end - test :with_uppercase do + test "with uppercase" do assert to_string(:fOO) == "fOO" assert to_string(:FOO) == "FOO" end - test :alias_atom do + test "alias atom" do assert to_string(Foo.Bar) == "Elixir.Foo.Bar" end end @@ -30,7 +36,7 @@ end defmodule String.Chars.BitStringTest do use ExUnit.Case, async: true - test :binary do + test "binary" do assert to_string("foo") == "foo" assert to_string(<>) == "abc" assert to_string("我今天要学习.") == "我今天要学习." @@ -40,11 +46,11 @@ end defmodule String.Chars.NumberTest do use ExUnit.Case, async: true - test :integer do + test "integer" do assert to_string(100) == "100" end - test :float do + test "float" do assert to_string(1.0) == "1.0" assert to_string(1.0e10) == "1.0e10" end @@ -53,66 +59,127 @@ end defmodule String.Chars.ListTest do use ExUnit.Case, async: true - test :basic do - assert to_string([ 1, "b", 3 ]) == <<1, 98, 3>> + test "basic" do + assert to_string([1, "b", 3]) == <<1, 98, 3>> end - test :printable do - assert to_string('abc') == "abc" + test "printable" do + assert to_string(~c"abc") == "abc" end - test :char_list do - assert to_string([0, 1, 2, 3, 255]) == - <<0, 1, 2, 3, 195, 191>> + test "charlist" do + assert to_string([0, 1, 2, 3, 255]) == <<0, 1, 2, 3, 195, 191>> assert to_string([0, [1, "hello"], 2, [["bye"]]]) == - <<0, 1, 104, 101, 108, 108, 111, 2, 98, 121, 101>> + <<0, 1, 104, 101, 108, 108, 111, 2, 98, 121, 101>> end - test :empty do + test "empty" do assert to_string([]) == "" end end +defmodule String.Chars.Version.RequirementTest do + use ExUnit.Case, async: true + + test "version requirement" do + {:ok, requirement} = Version.parse_requirement("== 2.0.1") + assert String.Chars.to_string(requirement) == "== 2.0.1" + end +end + +defmodule String.Chars.URITest do + use ExUnit.Case, async: true + + test "uri" do + uri = URI.parse("http://google.com") + assert String.Chars.to_string(uri) == "http://google.com" + + uri_no_host = URI.parse("/foo/bar") + assert String.Chars.to_string(uri_no_host) == "/foo/bar" + end +end + defmodule String.Chars.ErrorsTest do use ExUnit.Case, async: true - test :bitstring do - assert_raise Protocol.UndefinedError, - "protocol String.Chars not implemented for <<0, 1::size(4)>>, " <> - "cannot convert a bitstring to a string", fn -> - to_string(<<1 :: [size(12), integer, signed]>>) + defmodule Foo do + defstruct foo: "bar" + end + + test "bitstring" do + message = + """ + protocol String.Chars not implemented for BitString, cannot convert a bitstring to a string + + Got value: + + <<0, 1::size(4)>> + """ + + assert_raise Protocol.UndefinedError, message, fn -> + to_string(<<1::size(12)-integer-signed>>) end end - test :tuple do - assert_raise Protocol.UndefinedError, "protocol String.Chars not implemented for {1, 2, 3}", fn -> + test "tuple" do + message = """ + protocol String.Chars not implemented for Tuple + + Got value: + + {1, 2, 3} + """ + + assert_raise Protocol.UndefinedError, message, fn -> to_string({1, 2, 3}) end end - test :pid do - assert_raise Protocol.UndefinedError, ~r"^protocol String\.Chars not implemented for #PID<.+?>$", fn -> + test "PID" do + message = + ~r"^protocol String\.Chars not implemented for PID\n\nGot value:\n\n #PID<.+?>$" + + assert_raise Protocol.UndefinedError, message, fn -> to_string(self()) end end - test :ref do - assert_raise Protocol.UndefinedError, ~r"^protocol String\.Chars not implemented for #Reference<.+?>$", fn -> + test "ref" do + message = + ~r"^protocol String\.Chars not implemented for Reference\n\nGot value:\n\n #Reference<.+?>$" + + assert_raise Protocol.UndefinedError, message, fn -> to_string(make_ref()) == "" end end - test :function do - assert_raise Protocol.UndefinedError, ~r"^protocol String\.Chars not implemented for #Function<.+?>$", fn -> - to_string(fn -> end) + test "function" do + message = + ~r"^protocol String\.Chars not implemented for Function\n\nGot value:\n\n #Function<.+?>$" + + assert_raise Protocol.UndefinedError, message, fn -> + to_string(fn -> nil end) end end - test :port do - [port|_] = Port.list - assert_raise Protocol.UndefinedError, ~r"^protocol String\.Chars not implemented for #Port<.+?>$", fn -> + test "port" do + [port | _] = Port.list() + + message = + ~r"^protocol String\.Chars not implemented for Port\n\nGot value:\n\n #Port<.+?>$" + + assert_raise Protocol.UndefinedError, message, fn -> to_string(port) end end + + test "user-defined struct" do + message = + "protocol String\.Chars not implemented for String.Chars.ErrorsTest.Foo (a struct)\n\nGot value:\n\n %String.Chars.ErrorsTest.Foo{foo: \"bar\"}\n" + + assert_raise Protocol.UndefinedError, message, fn -> + to_string(%Foo{}) + end + end end diff --git a/lib/elixir/test/elixir/string_io_test.exs b/lib/elixir/test/elixir/string_io_test.exs index da589631d29..46d889ecd39 100644 --- a/lib/elixir/test/elixir/string_io_test.exs +++ b/lib/elixir/test/elixir/string_io_test.exs @@ -1,233 +1,404 @@ -Code.require_file "test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("test_helper.exs", __DIR__) defmodule StringIOTest do use ExUnit.Case, async: true - test "start and stop" do - {:ok, pid} = StringIO.open("") - assert StringIO.close(pid) == {:ok, {"", ""}} - end + doctest StringIO - test "start_link and stop" do + test "open and close" do {:ok, pid} = StringIO.open("") assert StringIO.close(pid) == {:ok, {"", ""}} end - test "peek" do + test "contents" do {:ok, pid} = StringIO.open("abc") IO.write(pid, "edf") assert StringIO.contents(pid) == {"abc", "edf"} end - ## IO module - - def start(string, opts \\ []) do - StringIO.open(string, opts) |> elem(1) + test "flush" do + {:ok, pid} = StringIO.open("") + IO.write(pid, "edf") + assert StringIO.flush(pid) == "edf" + assert StringIO.contents(pid) == {"", ""} end - def contents(pid) do - StringIO.contents(pid) - end + ## IO module test "IO.read :line with \\n" do - pid = start("abc\n") + {:ok, pid} = StringIO.open("abc\n") assert IO.read(pid, :line) == "abc\n" assert IO.read(pid, :line) == :eof - assert contents(pid) == {"", ""} + assert StringIO.contents(pid) == {"", ""} end test "IO.read :line with \\rn" do - pid = start("abc\r\n") + {:ok, pid} = StringIO.open("abc\r\n") assert IO.read(pid, :line) == "abc\n" assert IO.read(pid, :line) == :eof - assert contents(pid) == {"", ""} + assert StringIO.contents(pid) == {"", ""} end test "IO.read :line without line break" do - pid = start("abc") + {:ok, pid} = StringIO.open("abc") assert IO.read(pid, :line) == "abc" assert IO.read(pid, :line) == :eof - assert contents(pid) == {"", ""} + assert StringIO.contents(pid) == {"", ""} + end + + test "IO.read :line with UTF-8" do + {:ok, pid} = StringIO.open("⼊\n") + assert IO.read(pid, :line) == "⼊\n" + assert IO.read(pid, :line) == :eof + assert StringIO.contents(pid) == {"", ""} end - test "IO.read :line with invalid utf8" do - pid = start(<< 130, 227, 129, 132, 227, 129, 134 >>) + test "IO.read :line with invalid UTF-8" do + {:ok, pid} = StringIO.open(<<130, 227, 129, 132, 227, 129, 134>>) assert IO.read(pid, :line) == {:error, :collect_line} - assert contents(pid) == {<< 130, 227, 129, 132, 227, 129, 134 >>, ""} + assert StringIO.contents(pid) == {<<130, 227, 129, 132, 227, 129, 134>>, ""} end test "IO.read count" do - pid = start("abc") + {:ok, pid} = StringIO.open("abc") assert IO.read(pid, 2) == "ab" assert IO.read(pid, 8) == "c" assert IO.read(pid, 1) == :eof - assert contents(pid) == {"", ""} + assert StringIO.contents(pid) == {"", ""} end - test "IO.read count with utf8" do - pid = start("あいう") + test "IO.read count with UTF-8" do + {:ok, pid} = StringIO.open("あいう") assert IO.read(pid, 2) == "あい" assert IO.read(pid, 8) == "う" assert IO.read(pid, 1) == :eof - assert contents(pid) == {"", ""} + assert StringIO.contents(pid) == {"", ""} end - test "IO.read count with invalid utf8" do - pid = start(<< 130, 227, 129, 132, 227, 129, 134 >>) + test "IO.read count with invalid UTF-8" do + {:ok, pid} = StringIO.open(<<130, 227, 129, 132, 227, 129, 134>>) assert IO.read(pid, 2) == {:error, :invalid_unicode} - assert contents(pid) == {<< 130, 227, 129, 132, 227, 129, 134 >>, ""} + assert StringIO.contents(pid) == {<<130, 227, 129, 132, 227, 129, 134>>, ""} end test "IO.binread :line with \\n" do - pid = start("abc\n") + {:ok, pid} = StringIO.open("abc\n") assert IO.binread(pid, :line) == "abc\n" assert IO.binread(pid, :line) == :eof - assert contents(pid) == {"", ""} + assert StringIO.contents(pid) == {"", ""} end test "IO.binread :line with \\r\\n" do - pid = start("abc\r\n") + {:ok, pid} = StringIO.open("abc\r\n") assert IO.binread(pid, :line) == "abc\n" assert IO.binread(pid, :line) == :eof - assert contents(pid) == {"", ""} + assert StringIO.contents(pid) == {"", ""} end test "IO.binread :line without line break" do - pid = start("abc") + {:ok, pid} = StringIO.open("abc") assert IO.binread(pid, :line) == "abc" assert IO.binread(pid, :line) == :eof - assert contents(pid) == {"", ""} + assert StringIO.contents(pid) == {"", ""} + end + + test "IO.binread :line with raw bytes" do + {:ok, pid} = StringIO.open(<<181, 255, 194, ?\n>>) + assert IO.binread(pid, :line) == <<181, 255, 194, ?\n>> + assert IO.binread(pid, :line) == :eof + assert StringIO.contents(pid) == {"", ""} end test "IO.binread count" do - pid = start("abc") + {:ok, pid} = StringIO.open("abc") assert IO.binread(pid, 2) == "ab" assert IO.binread(pid, 8) == "c" assert IO.binread(pid, 1) == :eof - assert contents(pid) == {"", ""} + assert StringIO.contents(pid) == {"", ""} end - test "IO.binread count with utf8" do - pid = start("あいう") - assert IO.binread(pid, 2) == << 227, 129 >> - assert IO.binread(pid, 8) == << 130, 227, 129, 132, 227, 129, 134 >> + test "IO.binread count with UTF-8" do + {:ok, pid} = StringIO.open("あいう") + assert IO.binread(pid, 2) == <<227, 129>> + assert IO.binread(pid, 8) == <<130, 227, 129, 132, 227, 129, 134>> assert IO.binread(pid, 1) == :eof - assert contents(pid) == {"", ""} + assert StringIO.contents(pid) == {"", ""} end test "IO.write" do - pid = start("") + {:ok, pid} = StringIO.open("") assert IO.write(pid, "foo") == :ok - assert contents(pid) == {"", "foo"} + assert StringIO.contents(pid) == {"", "foo"} end - test "IO.write with utf8" do - pid = start("") + test "IO.write with UTF-8" do + {:ok, pid} = StringIO.open("") assert IO.write(pid, "あいう") == :ok - assert contents(pid) == {"", "あいう"} + assert StringIO.contents(pid) == {"", "あいう"} + end + + test "IO.write with non-printable arguments" do + {:ok, pid} = StringIO.open("") + + assert_raise ArgumentError, fn -> + IO.write(pid, [<<1::1>>]) + end + + assert_raise ErlangError, ~r/no_translation/, fn -> + IO.write(pid, <<222>>) + end end test "IO.binwrite" do - pid = start("") + {:ok, pid} = StringIO.open("") assert IO.binwrite(pid, "foo") == :ok - assert contents(pid) == {"", "foo"} + assert StringIO.contents(pid) == {"", "foo"} end - test "IO.binwrite with utf8" do - pid = start("") + test "IO.binwrite with UTF-8" do + {:ok, pid} = StringIO.open("") assert IO.binwrite(pid, "あいう") == :ok - assert contents(pid) == {"", "あいう"} + + binary = + <<195, 163, 194, 129, 194, 130, 195, 163>> <> + <<194, 129, 194, 132, 195, 163, 194, 129, 194, 134>> + + assert StringIO.contents(pid) == {"", binary} + end + + test "IO.binwrite with bytes" do + {:ok, pid} = StringIO.open("") + assert IO.binwrite(pid, <<127, 128>>) == :ok + assert StringIO.contents(pid) == {"", <<127, 194, 128>>} + end + + test "IO.binwrite with bytes and latin1 encoding" do + {:ok, pid} = StringIO.open("", encoding: :latin1) + assert IO.binwrite(pid, <<127, 128>>) == :ok + assert StringIO.contents(pid) == {"", <<127, 128>>} end test "IO.puts" do - pid = start("") + {:ok, pid} = StringIO.open("") assert IO.puts(pid, "abc") == :ok - assert contents(pid) == {"", "abc\n"} + assert StringIO.contents(pid) == {"", "abc\n"} + end + + test "IO.puts with non-printable arguments" do + {:ok, pid} = StringIO.open("") + + assert_raise ArgumentError, fn -> + IO.puts(pid, [<<1::1>>]) + end end test "IO.inspect" do - pid = start("") + {:ok, pid} = StringIO.open("") assert IO.inspect(pid, {}, []) == {} - assert contents(pid) == {"", "{}\n"} + assert StringIO.contents(pid) == {"", "{}\n"} end test "IO.getn" do - pid = start("abc") + {:ok, pid} = StringIO.open("abc") assert IO.getn(pid, ">", 2) == "ab" - assert contents(pid) == {"c", ""} + assert StringIO.contents(pid) == {"c", ""} end - test "IO.getn with utf8" do - pid = start("あいう") + test "IO.getn with UTF-8" do + {:ok, pid} = StringIO.open("あいう") assert IO.getn(pid, ">", 2) == "あい" - assert contents(pid) == {"う", ""} + assert StringIO.contents(pid) == {"う", ""} end - test "IO.getn with invalid utf8" do - pid = start(<< 130, 227, 129, 132, 227, 129, 134 >>) + test "IO.getn with invalid UTF-8" do + {:ok, pid} = StringIO.open(<<130, 227, 129, 132, 227, 129, 134>>) assert IO.getn(pid, ">", 2) == {:error, :invalid_unicode} - assert contents(pid) == {<< 130, 227, 129, 132, 227, 129, 134 >>, ""} + assert StringIO.contents(pid) == {<<130, 227, 129, 132, 227, 129, 134>>, ""} end test "IO.getn with capture_prompt" do - pid = start("abc", capture_prompt: true) + {:ok, pid} = StringIO.open("abc", capture_prompt: true) assert IO.getn(pid, ">", 2) == "ab" - assert contents(pid) == {"c", ">"} + assert StringIO.contents(pid) == {"c", ">"} end test "IO.gets with \\n" do - pid = start("abc\nd") + {:ok, pid} = StringIO.open("abc\nd") assert IO.gets(pid, ">") == "abc\n" - assert contents(pid) == {"d", ""} + assert StringIO.contents(pid) == {"d", ""} end test "IO.gets with \\r\\n" do - pid = start("abc\r\nd") + {:ok, pid} = StringIO.open("abc\r\nd") assert IO.gets(pid, ">") == "abc\n" - assert contents(pid) == {"d", ""} + assert StringIO.contents(pid) == {"d", ""} end test "IO.gets without line breaks" do - pid = start("abc") + {:ok, pid} = StringIO.open("abc") assert IO.gets(pid, ">") == "abc" - assert contents(pid) == {"", ""} + assert StringIO.contents(pid) == {"", ""} end - test "IO.gets with invalid utf8" do - pid = start(<< 130, 227, 129, 132, 227, 129, 134 >>) + test "IO.gets with invalid UTF-8" do + {:ok, pid} = StringIO.open(<<130, 227, 129, 132, 227, 129, 134>>) assert IO.gets(pid, ">") == {:error, :collect_line} - assert contents(pid) == {<< 130, 227, 129, 132, 227, 129, 134 >>, ""} + assert StringIO.contents(pid) == {<<130, 227, 129, 132, 227, 129, 134>>, ""} end test "IO.gets with capture_prompt" do - pid = start("abc\n", capture_prompt: true) + {:ok, pid} = StringIO.open("abc\n", capture_prompt: true) assert IO.gets(pid, ">") == "abc\n" - assert contents(pid) == {"", ">"} + assert StringIO.contents(pid) == {"", ">"} end test ":io.get_password" do - pid = start("abc\n") + {:ok, pid} = StringIO.open("abc\n") assert :io.get_password(pid) == "abc\n" - assert contents(pid) == {"", ""} + assert StringIO.contents(pid) == {"", ""} end test "IO.stream" do - pid = start("abc") - assert IO.stream(pid, 2) |> Enum.to_list == ["ab", "c"] - assert contents(pid) == {"", ""} + {:ok, pid} = StringIO.open("abc") + assert IO.stream(pid, 2) |> Enum.to_list() == ["ab", "c"] + assert StringIO.contents(pid) == {"", ""} end - test "IO.stream with invalid utf8" do - pid = start(<< 130, 227, 129, 132, 227, 129, 134 >>) - assert_raise IO.StreamError, fn-> - IO.stream(pid, 2) |> Enum.to_list + test "IO.stream with invalid UTF-8" do + {:ok, pid} = StringIO.open(<<130, 227, 129, 132, 227, 129, 134>>) + + assert_raise IO.StreamError, "error during streaming: :invalid_unicode", fn -> + IO.stream(pid, 2) |> Enum.to_list() end - assert contents(pid) == {<< 130, 227, 129, 132, 227, 129, 134 >>, ""} + + assert StringIO.contents(pid) == {<<130, 227, 129, 132, 227, 129, 134>>, ""} end test "IO.binstream" do - pid = start("abc") - assert IO.stream(pid, 2) |> Enum.to_list == ["ab", "c"] - assert contents(pid) == {"", ""} + {:ok, pid} = StringIO.open("abc") + assert IO.stream(pid, 2) |> Enum.to_list() == ["ab", "c"] + assert StringIO.contents(pid) == {"", ""} + end + + defp get_until(pid, encoding, prompt, module, function) do + :io.request(pid, {:get_until, encoding, prompt, module, function, []}) + end + + defmodule GetUntilCallbacks do + def until_eof(continuation, :eof) do + {:done, continuation, :eof} + end + + def until_eof(continuation, content) do + {:more, continuation ++ content} + end + + def until_eof_then_try_more(~c"magic-stop-prefix" ++ continuation, :eof) do + {:done, continuation, :eof} + end + + def until_eof_then_try_more(continuation, :eof) do + {:more, ~c"magic-stop-prefix" ++ continuation} + end + + def until_eof_then_try_more(continuation, content) do + {:more, continuation ++ content} + end + + def up_to_3_bytes(continuation, :eof) do + {:done, continuation, :eof} + end + + def up_to_3_bytes(continuation, content) do + case continuation ++ content do + [a, b, c | tail] -> {:done, [a, b, c], tail} + str -> {:more, str} + end + end + + def up_to_3_bytes_discard_rest(continuation, :eof) do + {:done, continuation, :eof} + end + + def up_to_3_bytes_discard_rest(continuation, content) do + case continuation ++ content do + [a, b, c | _tail] -> {:done, [a, b, c], :eof} + str -> {:more, str} + end + end + end + + test "get_until with up_to_3_bytes" do + {:ok, pid} = StringIO.open("abcdefg") + result = get_until(pid, :unicode, "", GetUntilCallbacks, :up_to_3_bytes) + assert result == "abc" + assert IO.read(pid, :eof) == "defg" + end + + test "get_until with up_to_3_bytes_discard_rest" do + {:ok, pid} = StringIO.open("abcdefg") + result = get_until(pid, :unicode, "", GetUntilCallbacks, :up_to_3_bytes_discard_rest) + assert result == "abc" + assert IO.read(pid, :eof) == :eof + end + + test "get_until with until_eof" do + {:ok, pid} = StringIO.open("abc\nd") + result = get_until(pid, :unicode, "", GetUntilCallbacks, :until_eof) + assert result == "abc\nd" + end + + test "get_until with until_eof and \\r\\n" do + {:ok, pid} = StringIO.open("abc\r\nd") + result = get_until(pid, :unicode, "", GetUntilCallbacks, :until_eof) + assert result == "abc\r\nd" + end + + test "get_until with until_eof capturing prompt" do + {:ok, pid} = StringIO.open("abc\nd", capture_prompt: true) + result = get_until(pid, :unicode, ">", GetUntilCallbacks, :until_eof) + assert result == "abc\nd" + assert StringIO.contents(pid) == {"", ">>>"} + end + + test "get_until with until_eof_then_try_more" do + {:ok, pid} = StringIO.open("abc\nd") + result = get_until(pid, :unicode, "", GetUntilCallbacks, :until_eof_then_try_more) + assert result == "abc\nd" + end + + test "get_until with invalid UTF-8" do + {:ok, pid} = StringIO.open(<<130, 227, 129, 132, 227, 129, 134>>) + result = get_until(pid, :unicode, "", GetUntilCallbacks, :until_eof) + assert result == :error + end + + test "get_until with raw bytes (latin1)" do + {:ok, pid} = StringIO.open(<<181, 255, 194, ?\n>>) + result = get_until(pid, :latin1, "", GetUntilCallbacks, :until_eof) + assert result == <<181, 255, 194, ?\n>> + end + + test ":io.erl_scan_form/2" do + {:ok, pid} = StringIO.open("1.") + result = :io.scan_erl_form(pid, ~c"p>") + assert result == {:ok, [{:integer, 1, 1}, {:dot, 1}], 1} + assert StringIO.contents(pid) == {"", ""} + end + + test ":io.erl_scan_form/2 with capture_prompt" do + {:ok, pid} = StringIO.open("1.", capture_prompt: true) + result = :io.scan_erl_form(pid, ~c"p>") + assert result == {:ok, [{:integer, 1, 1}, {:dot, 1}], 1} + assert StringIO.contents(pid) == {"", "p>p>"} + end + + test "returns enotsup for files" do + {:ok, pid} = StringIO.open("123") + assert :file.position(pid, :cur) == {:error, :enotsup} end end diff --git a/lib/elixir/test/elixir/string_test.exs b/lib/elixir/test/elixir/string_test.exs index f6f38e7624d..278d753b3e8 100644 --- a/lib/elixir/test/elixir/string_test.exs +++ b/lib/elixir/test/elixir/string_test.exs @@ -1,22 +1,22 @@ -Code.require_file "test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("test_helper.exs", __DIR__) defmodule StringTest do use ExUnit.Case, async: true - test :integer_codepoints do - assert ?é == 233 - assert ?\xE9 == 233 - assert ?\351 == 233 - end + doctest String - test :next_codepoint do + test "next_codepoint/1" do assert String.next_codepoint("ésoj") == {"é", "soj"} assert String.next_codepoint(<<255>>) == {<<255>>, ""} assert String.next_codepoint("") == nil end - # test cases described in http://mortoray.com/2013/11/27/the-string-type-is-broken/ - test :unicode do + # test cases described in https://mortoray.com/2013/11/27/the-string-type-is-broken/ + test "Unicode" do assert String.reverse("noël") == "lëon" assert String.slice("noël", 0..2) == "noë" assert String.length("noël") == 4 @@ -26,41 +26,75 @@ defmodule StringTest do assert String.reverse("") == "" assert String.upcase("baffle") == "BAFFLE" + + assert String.equivalent?("noël", "noël") end - test :split do - assert String.split("") == [""] + test "split/1,2,3" do + assert String.split("") == [] assert String.split("foo bar") == ["foo", "bar"] assert String.split(" foo bar") == ["foo", "bar"] assert String.split("foo bar ") == ["foo", "bar"] assert String.split(" foo bar ") == ["foo", "bar"] assert String.split("foo\t\n\v\f\r\sbar\n") == ["foo", "bar"] - assert String.split("foo" <> <<31>> <> "bar") == ["foo", "bar"] assert String.split("foo" <> <<194, 133>> <> "bar") == ["foo", "bar"] + # information separators are not considered whitespace + assert String.split("foo\u001Fbar") == ["foo\u001Fbar"] + # no-break space is excluded + assert String.split("foo\00A0bar") == ["foo\00A0bar"] + assert String.split("foo\u202Fbar") == ["foo\u202Fbar"] - assert String.split("", ",") == [""] assert String.split("a,b,c", ",") == ["a", "b", "c"] assert String.split("a,b", ".") == ["a,b"] assert String.split("1,2 3,4", [" ", ","]) == ["1", "2", "3", "4"] + + assert String.split("", ",") == [""] assert String.split(" a b c ", " ") == ["", "a", "b", "c", ""] + assert String.split(" a b c ", " ", parts: :infinity) == ["", "a", "b", "c", ""] + assert String.split(" a b c ", " ", parts: 1) == [" a b c "] + assert String.split(" a b c ", " ", parts: 2) == ["", "a b c "] + assert String.split("", ",", trim: true) == [] assert String.split(" a b c ", " ", trim: true) == ["a", "b", "c"] - assert String.split(" a b c ", " ", trim: true, parts: 0) == ["a", "b", "c"] assert String.split(" a b c ", " ", trim: true, parts: :infinity) == ["a", "b", "c"] assert String.split(" a b c ", " ", trim: true, parts: 1) == [" a b c "] + assert String.split(" a b c ", " ", trim: true, parts: 2) == ["a", "b c "] - assert String.split("abé", "") == ["a", "b", "é", ""] - assert String.split("abé", "", parts: 0) == ["a", "b", "é", ""] + assert String.split("abé", "") == ["", "a", "b", "é", ""] + assert String.split("abé", "", parts: :infinity) == ["", "a", "b", "é", ""] assert String.split("abé", "", parts: 1) == ["abé"] - assert String.split("abé", "", parts: 2) == ["a", "bé"] - assert String.split("abé", "", parts: 10) == ["a", "b", "é", ""] + assert String.split("abé", "", parts: 2) == ["", "abé"] + assert String.split("abé", "", parts: 3) == ["", "a", "bé"] + assert String.split("abé", "", parts: 4) == ["", "a", "b", "é"] + assert String.split("abé", "", parts: 5) == ["", "a", "b", "é", ""] + assert String.split("abé", "", parts: 10) == ["", "a", "b", "é", ""] assert String.split("abé", "", trim: true) == ["a", "b", "é"] - assert String.split("abé", "", trim: true, parts: 0) == ["a", "b", "é"] + assert String.split("abé", "", trim: true, parts: :infinity) == ["a", "b", "é"] assert String.split("abé", "", trim: true, parts: 2) == ["a", "bé"] + assert String.split("abé", "", trim: true, parts: 3) == ["a", "b", "é"] + assert String.split("abé", "", trim: true, parts: 4) == ["a", "b", "é"] + + assert String.split("noël", "") == ["", "n", "o", "ë", "l", ""] + assert String.split("x-", "-", parts: 2, trim: true) == ["x"] + assert String.split("x-x-", "-", parts: 3, trim: true) == ["x", "x"] + + assert String.split("hello", []) == ["hello"] + assert String.split("hello", [], trim: true) == ["hello"] + assert String.split("", []) == [""] + assert String.split("", [], trim: true) == [] + + assert_raise ArgumentError, fn -> + String.split("a,b,c", [""]) + end + + assert_raise ArgumentError, fn -> + String.split("a,b,c", [""]) + end end - test :split_with_regex do + test "split/2,3 with regex" do assert String.split("", ~r{,}) == [""] + assert String.split("", ~r{,}, trim: true) == [] assert String.split("a,b", ~r{,}) == ["a", "b"] assert String.split("a,b,c", ~r{,}) == ["a", "b", "c"] assert String.split("a,b,c", ~r{,}, parts: 2) == ["a", "b,c"] @@ -69,7 +103,52 @@ defmodule StringTest do assert String.split("a,b", ~r{\.}) == ["a,b"] end - test :split_at do + test "split/2,3 with compiled pattern" do + pattern = :binary.compile_pattern("-") + + assert String.split("x-", pattern) == ["x", ""] + assert String.split("x-", pattern, parts: 2, trim: true) == ["x"] + assert String.split("x-x-", pattern, parts: 3, trim: true) == ["x", "x"] + end + + test "split/2,3 with malformed" do + assert String.split(<<225, 158, 128, 225, 158, 185, 225>>, "", parts: 1) == + [<<225, 158, 128, 225, 158, 185, 225>>] + + assert String.split(<<225, 158, 128, 225, 158, 185, 225>>, "", parts: 2) == + ["", <<225, 158, 128, 225, 158, 185, 225>>] + + assert String.split(<<225, 158, 128, 225, 158, 185, 225>>, "", parts: 3) == + ["", "កឹ", <<225>>] + + assert String.split(<<225, 158, 128, 225, 158, 185, 225>>, "", parts: 4) == + ["", "កឹ", <<225>>, ""] + end + + test "splitter/2,3" do + assert String.splitter("a,b,c", ",") |> Enum.to_list() == ["a", "b", "c"] + assert String.splitter("a,b", ".") |> Enum.to_list() == ["a,b"] + assert String.splitter("1,2 3,4", [" ", ","]) |> Enum.to_list() == ["1", "2", "3", "4"] + assert String.splitter("", ",") |> Enum.to_list() == [""] + + assert String.splitter("", ",", trim: true) |> Enum.to_list() == [] + assert String.splitter(" a b c ", " ", trim: true) |> Enum.to_list() == ["a", "b", "c"] + assert String.splitter(" a b c ", " ", trim: true) |> Enum.take(1) == ["a"] + assert String.splitter(" a b c ", " ", trim: true) |> Enum.take(2) == ["a", "b"] + + assert String.splitter("hello", []) |> Enum.to_list() == ["hello"] + assert String.splitter("hello", [], trim: true) |> Enum.to_list() == ["hello"] + assert String.splitter("", []) |> Enum.to_list() == [""] + assert String.splitter("", [], trim: true) |> Enum.to_list() == [] + + assert String.splitter("1,2 3,4 5", "") |> Enum.take(4) == ["", "1", ",", "2"] + + assert_raise ArgumentError, fn -> + String.splitter("a", [""]) + end + end + + test "split_at/2" do assert String.split_at("", 0) == {"", ""} assert String.split_at("", -1) == {"", ""} assert String.split_at("", 1) == {"", ""} @@ -84,47 +163,108 @@ defmodule StringTest do assert String.split_at("abc", -3) == {"", "abc"} assert String.split_at("abc", -4) == {"", "abc"} assert String.split_at("abc", -1000) == {"", "abc"} + + assert_raise FunctionClauseError, fn -> + String.split_at("abc", 0.1) + end + + assert_raise FunctionClauseError, fn -> + String.split_at("abc", -0.1) + end + end + + test "split_at/2 with malformed" do + assert String.split_at(<>, 2) == {<>, <<10, ?a>>} + assert String.split_at(<<107, 205, 135, 184>>, 1) == {<<107, 205, 135>>, <<184>>} + + assert String.split_at(<<225, 158, 128, 225, 158, 185, 225>>, 0) == + {"", <<225, 158, 128, 225, 158, 185, 225>>} + + assert String.split_at(<<225, 158, 128, 225, 158, 185, 225>>, 1) == + {"កឹ", <<225>>} + + assert String.split_at(<<225, 158, 128, 225, 158, 185, 225>>, 2) == + {<<225, 158, 128, 225, 158, 185, 225>>, ""} end - test :upcase do - assert String.upcase("123 abcd 456 efg hij ( %$#) kl mnop @ qrst = -_ uvwxyz") == "123 ABCD 456 EFG HIJ ( %$#) KL MNOP @ QRST = -_ UVWXYZ" + test "upcase/1" do + assert String.upcase("123 abcd 456 efg hij ( %$#) kl mnop @ qrst = -_ uvwxyz") == + "123 ABCD 456 EFG HIJ ( %$#) KL MNOP @ QRST = -_ UVWXYZ" + assert String.upcase("") == "" assert String.upcase("abcD") == "ABCD" end - test :upcase_utf8 do + test "upcase/1 with UTF-8" do assert String.upcase("& % # àáâ ãäå 1 2 ç æ") == "& % # ÀÁ ÃÄÅ 1 2 Ç Æ" assert String.upcase("àáâãäåæçèéêëìíîïðñòóôõöøùúûüýþ") == "ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞ" end - test :upcase_utf8_multibyte do + test "upcase/1 with UTF-8 multibyte" do assert String.upcase("straße") == "STRASSE" assert String.upcase("áüÈß") == "ÁÜÈSS" end - test :downcase do - assert String.downcase("123 ABcD 456 EfG HIJ ( %$#) KL MNOP @ QRST = -_ UVWXYZ") == "123 abcd 456 efg hij ( %$#) kl mnop @ qrst = -_ uvwxyz" + test "upcase/1 with ascii" do + assert String.upcase("olá", :ascii) == "OLá" + end + + test "upcase/1 with turkic" do + assert String.upcase("ıi", :turkic) == "Iİ" + assert String.upcase("Iİ", :turkic) == "Iİ" + end + + test "downcase/1" do + assert String.downcase("123 ABcD 456 EfG HIJ ( %$#) KL MNOP @ QRST = -_ UVWXYZ") == + "123 abcd 456 efg hij ( %$#) kl mnop @ qrst = -_ uvwxyz" + assert String.downcase("abcD") == "abcd" assert String.downcase("") == "" end - test :downcase_utf8 do + test "downcase/1 with UTF-8" do assert String.downcase("& % # ÀÁ ÃÄÅ 1 2 Ç Æ") == "& % # àáâ ãäå 1 2 ç æ" assert String.downcase("ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞ") == "àáâãäåæçèéêëìíîïðñòóôõöøùúûüýþ" assert String.downcase("áüÈß") == "áüèß" end - test :capitalize do + test "downcase/1 with greek final sigma" do + assert String.downcase("Σ") == "σ" + assert String.downcase("ΣΣ") == "σσ" + assert String.downcase("Σ ΣΣ") == "σ σσ" + assert String.downcase("ΜΕΣ'ΑΠΟ") == "μεσ'απο" + assert String.downcase("ΑΣ'ΤΟΥΣ") == "ασ'τουσ" + + assert String.downcase("Σ", :greek) == "σ" + assert String.downcase("Σ ΣΣ", :greek) == "σ σς" + assert String.downcase("Σ ΣΑΣ Σ", :greek) == "σ σας σ" + assert String.downcase("ΜΕΣ'ΑΠΟ", :greek) == "μεσ'απο" + assert String.downcase("ΑΣ'ΤΟΥΣ", :greek) == "ασ'τους" + end + + test "downcase/1 with ascii" do + assert String.downcase("OLÁ", :ascii) == "olÁ" + end + + test "downcase/1 with turkic" do + assert String.downcase("Iİ", :turkic) == "ıi" + assert String.downcase("İ", :turkic) == "i" + + assert String.downcase("ıi", :turkic) == "ıi" + assert String.downcase("i", :turkic) == "i" + + assert String.downcase("İ") == "i̇" + end + + test "capitalize/1" do assert String.capitalize("") == "" + assert String.capitalize("1") == "1" assert String.capitalize("abc") == "Abc" assert String.capitalize("ABC") == "Abc" assert String.capitalize("c b a") == "C b a" assert String.capitalize("1ABC") == "1abc" assert String.capitalize("_aBc1") == "_abc1" assert String.capitalize(" aBc1") == " abc1" - end - - test :capitalize_utf8 do assert String.capitalize("àáâ") == "Àáâ" assert String.capitalize("ÀÁÂ") == "Àáâ" assert String.capitalize("âáà") == "Âáà" @@ -132,128 +272,408 @@ defmodule StringTest do assert String.capitalize("òóôõö") == "Òóôõö" assert String.capitalize("ÒÓÔÕÖ") == "Òóôõö" assert String.capitalize("fin") == "Fin" + + assert String.capitalize("ABC", :ascii) == "Abc" + assert String.capitalize("àáâ", :ascii) == "àáâ" + assert String.capitalize("aáA", :ascii) == "Aáa" + + assert String.capitalize("iii", :turkic) == "İii" + assert String.capitalize("ııı", :turkic) == "Iıı" + assert String.capitalize("İii", :turkic) == "İii" + assert String.capitalize("Iıı", :turkic) == "Iıı" + + assert String.capitalize(<<138, ?B, ?C>>) == <<138, ?b, ?c>> + + assert String.capitalize(<<225, 158, 128, 225, 158, 185, 225>>) == + <<225, 158, 128, 225, 158, 185, 225>> end - test :rstrip do - assert String.rstrip("") == "" - assert String.rstrip(" abc ") == " abc" - assert String.rstrip(" abc a") == " abc a" - assert String.rstrip("a abc a\n\n") == "a abc a" - assert String.rstrip("a abc a\t\n\v\f\r\s") == "a abc a" - assert String.rstrip("a abc a " <> <<31>>) == "a abc a" - assert String.rstrip("a abc a" <> <<194, 133>>) == "a abc a" - assert String.rstrip(" abc aa", ?a) == " abc " - assert String.rstrip(" abc __", ?_) == " abc " - assert String.rstrip(" cat 猫猫", ?猫) == " cat " - end - - test :lstrip do - assert String.lstrip("") == "" - assert String.lstrip(" abc ") == "abc " - assert String.lstrip("a abc a") == "a abc a" - assert String.lstrip("\n\na abc a") == "a abc a" - assert String.lstrip("\t\n\v\f\r\sa abc a") == "a abc a" - assert String.lstrip(<<31>> <> " a abc a") == "a abc a" - assert String.lstrip(<<194, 133>> <> "a abc a") == "a abc a" - assert String.lstrip("__ abc _", ?_) == " abc _" - assert String.lstrip("猫猫 cat ", ?猫) == " cat " - end - - test :strip do - assert String.strip("") == "" - assert String.strip(" abc ") == "abc" - assert String.strip("a abc a\n\n") == "a abc a" - assert String.strip("a abc a\t\n\v\f\r\s") == "a abc a" - assert String.strip("___ abc ___", ?_) == " abc " - assert String.strip("猫猫猫 cat 猫猫猫", ?猫) == " cat " - end - - test :rjust do - assert String.rjust("", 5) == " " - assert String.rjust("abc", 5) == " abc" - assert String.rjust(" abc ", 9) == " abc " - assert String.rjust("猫", 5) == " 猫" - assert String.rjust("abc", 5, ?-) == "--abc" - assert String.rjust("abc", 5, ?猫) == "猫猫abc" - end - - test :ljust do - assert String.ljust("", 5) == " " - assert String.ljust("abc", 5) == "abc " - assert String.ljust(" abc ", 9) == " abc " - assert String.ljust("猫", 5) == "猫 " - assert String.ljust("abc", 5, ?-) == "abc--" - assert String.ljust("abc", 5, ?猫) == "abc猫猫" - end - - test :reverse do + test "replace_leading/3" do + assert String.replace_leading("aa abc ", "a", "b") == "bb abc " + assert String.replace_leading("__ abc ", "_", "b") == "bb abc " + assert String.replace_leading("aaaaaaaa ", "a", "b") == "bbbbbbbb " + assert String.replace_leading("aaaaaaaa ", "aaa", "b") == "bbaa " + assert String.replace_leading("aaaaaaaaa", "a", "b") == "bbbbbbbbb" + assert String.replace_leading("]]]]]]", "]", "[]") == "[][][][][][]" + assert String.replace_leading("]]]]]]]]", "]", "") == "" + assert String.replace_leading("]]]]]] ]", "]", "") == " ]" + assert String.replace_leading("猫猫 cat ", "猫", "й") == "йй cat " + assert String.replace_leading("test", "t", "T") == "Test" + assert String.replace_leading("t", "t", "T") == "T" + assert String.replace_leading("aaa", "b", "c") == "aaa" + + message = ~r/cannot use an empty string/ + + assert_raise ArgumentError, message, fn -> + String.replace_leading("foo", "", "bar") + end + + assert_raise ArgumentError, message, fn -> + String.replace_leading("", "", "bar") + end + end + + test "replace_trailing/3" do + assert String.replace_trailing(" abc aa", "a", "b") == " abc bb" + assert String.replace_trailing(" abc __", "_", "b") == " abc bb" + assert String.replace_trailing(" aaaaaaaa", "a", "b") == " bbbbbbbb" + assert String.replace_trailing(" aaaaaaaa", "aaa", "b") == " aabb" + assert String.replace_trailing("aaaaaaaaa", "a", "b") == "bbbbbbbbb" + assert String.replace_trailing("]]]]]]", "]", "[]") == "[][][][][][]" + assert String.replace_trailing("]]]]]]]]", "]", "") == "" + assert String.replace_trailing("] ]]]]]]", "]", "") == "] " + assert String.replace_trailing(" cat 猫猫", "猫", "й") == " cat йй" + assert String.replace_trailing("test", "t", "T") == "tesT" + assert String.replace_trailing("t", "t", "T") == "T" + assert String.replace_trailing("aaa", "b", "c") == "aaa" + + message = ~r/cannot use an empty string/ + + assert_raise ArgumentError, message, fn -> + String.replace_trailing("foo", "", "bar") + end + + assert_raise ArgumentError, message, fn -> + String.replace_trailing("", "", "bar") + end + end + + test "trim/1,2" do + assert String.trim("") == "" + assert String.trim(" abc ") == "abc" + assert String.trim("a abc a\n\n") == "a abc a" + assert String.trim("a abc a\t\n\v\f\r\s") == "a abc a" + + assert String.trim("___ abc ___", "_") == " abc " + assert String.trim("猫猫猫cat猫猫猫", "猫猫") == "猫cat猫" + # no-break space + assert String.trim("\u00A0a abc a\u00A0") == "a abc a" + # whitespace defined as a range + assert String.trim("\u2008a abc a\u2005") == "a abc a" + end + + test "trim_leading/1,2" do + assert String.trim_leading("") == "" + assert String.trim_leading(" abc ") == "abc " + assert String.trim_leading("a abc a") == "a abc a" + assert String.trim_leading("\n\na abc a") == "a abc a" + assert String.trim_leading("\t\n\v\f\r\sa abc a") == "a abc a" + assert String.trim_leading(<<194, 133, "a abc a">>) == "a abc a" + # information separators are not whitespace + assert String.trim_leading("\u001F a abc a") == "\u001F a abc a" + # no-break space + assert String.trim_leading("\u00A0 a abc a") == "a abc a" + + assert String.trim_leading("aa aaa", "aaa") == "aa aaa" + assert String.trim_leading("aaa aaa", "aa") == "a aaa" + assert String.trim_leading("aa abc ", "a") == " abc " + assert String.trim_leading("__ abc ", "_") == " abc " + assert String.trim_leading("aaaaaaaaa ", "a") == " " + assert String.trim_leading("aaaaaaaaaa", "a") == "" + assert String.trim_leading("]]]]]] ]", "]") == " ]" + assert String.trim_leading("猫猫 cat ", "猫") == " cat " + assert String.trim_leading("test", "t") == "est" + assert String.trim_leading("t", "t") == "" + assert String.trim_leading("", "t") == "" + end + + test "trim_trailing/1,2" do + assert String.trim_trailing("") == "" + assert String.trim_trailing("1\n") == "1" + assert String.trim_trailing("\r\n") == "" + assert String.trim_trailing(" abc ") == " abc" + assert String.trim_trailing(" abc a") == " abc a" + assert String.trim_trailing("a abc a\n\n") == "a abc a" + assert String.trim_trailing("a abc a\t\n\v\f\r\s") == "a abc a" + assert String.trim_trailing(<<"a abc a", 194, 133>>) == "a abc a" + # information separators are not whitespace + assert String.trim_trailing("a abc a \u001F") == "a abc a \u001F" + # no-break space + assert String.trim_trailing("a abc a \u00A0") == "a abc a" + + assert String.trim_trailing("aaa aa", "aaa") == "aaa aa" + assert String.trim_trailing("aaa aaa", "aa") == "aaa a" + assert String.trim_trailing(" abc aa", "a") == " abc " + assert String.trim_trailing(" abc __", "_") == " abc " + assert String.trim_trailing(" aaaaaaaaa", "a") == " " + assert String.trim_trailing("aaaaaaaaaa", "a") == "" + assert String.trim_trailing("] ]]]]]]", "]") == "] " + assert String.trim_trailing(" cat 猫猫", "猫") == " cat " + assert String.trim_trailing("test", "t") == "tes" + assert String.trim_trailing("t", "t") == "" + assert String.trim_trailing("", "t") == "" + end + + test "pad_leading/2,3" do + assert String.pad_leading("", 5) == " " + assert String.pad_leading("abc", 5) == " abc" + assert String.pad_leading(" abc ", 9) == " abc " + assert String.pad_leading("猫", 5) == " 猫" + assert String.pad_leading("-", 0) == "-" + assert String.pad_leading("-", 1) == "-" + + assert String.pad_leading("---", 5, "abc") == "ab---" + assert String.pad_leading("---", 9, "abc") == "abcabc---" + + assert String.pad_leading("---", 5, ["abc"]) == "abcabc---" + assert String.pad_leading("--", 6, ["a", "bc"]) == "abcabc--" + + assert_raise FunctionClauseError, fn -> + String.pad_leading("-", -1) + end + + assert_raise FunctionClauseError, fn -> + String.pad_leading("-", 1, []) + end + + message = "expected a string padding element, got: 10" + + assert_raise ArgumentError, message, fn -> + String.pad_leading("-", 3, ["-", 10]) + end + end + + test "pad_trailing/2,3" do + assert String.pad_trailing("", 5) == " " + assert String.pad_trailing("abc", 5) == "abc " + assert String.pad_trailing(" abc ", 9) == " abc " + assert String.pad_trailing("猫", 5) == "猫 " + assert String.pad_trailing("-", 0) == "-" + assert String.pad_trailing("-", 1) == "-" + + assert String.pad_trailing("---", 5, "abc") == "---ab" + assert String.pad_trailing("---", 9, "abc") == "---abcabc" + + assert String.pad_trailing("---", 5, ["abc"]) == "---abcabc" + assert String.pad_trailing("--", 6, ["a", "bc"]) == "--abcabc" + + assert_raise FunctionClauseError, fn -> + String.pad_trailing("-", -1) + end + + assert_raise FunctionClauseError, fn -> + String.pad_trailing("-", 1, []) + end + + message = "expected a string padding element, got: 10" + + assert_raise ArgumentError, message, fn -> + String.pad_trailing("-", 3, ["-", 10]) + end + end + + test "reverse/1" do assert String.reverse("") == "" assert String.reverse("abc") == "cba" assert String.reverse("Hello World") == "dlroW olleH" assert String.reverse("Hello ∂og") == "go∂ olleH" assert String.reverse("Ā̀stute") == "etutsĀ̀" assert String.reverse(String.reverse("Hello World")) == "Hello World" + assert String.reverse(String.reverse("Hello \r\n World")) == "Hello \r\n World" end - test :replace do - assert String.replace("a,b,c", ",", "-") == "a-b-c" - assert String.replace("a,b,c", [",", "b"], "-") == "a---c" + describe "replace/3" do + test "with empty string and string replacement" do + assert String.replace("elixir", "", "") == "elixir" + assert String.replace("ELIXIR", "", ".") == ".E.L.I.X.I.R." + assert String.replace("ELIXIR", "", ".", global: true) == ".E.L.I.X.I.R." + assert String.replace("ELIXIR", "", ".", global: false) == ".ELIXIR" - assert String.replace("a,b,c", ",", "-", global: false) == "a-b,c" - assert String.replace("a,b,c", [",", "b"], "-", global: false) == "a-b,c" - assert String.replace("ãéã", "é", "e", global: false) == "ãeã" + assert_raise ArgumentError, fn -> + String.replace("elixir", [""], "") + end + end + + test "with empty string and string replacement with malformed" do + assert String.replace(<<225, 158, 128, 225, 158, 185, 225>>, "", ".") == ".កឹ.\xE1." + end + + test "with empty pattern list" do + assert String.replace("elixir", [], "anything") == "elixir" + end - assert String.replace("a,b,c", ",", "[]", insert_replaced: 2) == "a[],b[],c" - assert String.replace("a,b,c", ",", "[]", insert_replaced: [1, 1]) == "a[,,]b[,,]c" - assert String.replace("a,b,c", "b", "[]", insert_replaced: 1, global: false) == "a,[b],c" + test "with match pattern and string replacement" do + assert String.replace("a,b,c", ",", "-") == "a-b-c" + assert String.replace("a,b,c", [",", "b"], "-") == "a---c" - assert String.replace("a,b,c", ~r/,(.)/, ",\\1\\1") == "a,bb,cc" - assert String.replace("a,b,c", ~r/,(.)/, ",\\1\\1", global: false) == "a,bb,c" + assert String.replace("a,b,c", ",", "-", global: false) == "a-b,c" + assert String.replace("a,b,c", [",", "b"], "-", global: false) == "a-b,c" + assert String.replace("ãéã", "é", "e", global: false) == "ãeã" + end + + test "with regex and string replacement" do + assert String.replace("a,b,c", ~r/,(.)/, ",\\1\\1") == "a,bb,cc" + assert String.replace("a,b,c", ~r/,(.)/, ",\\1\\1", global: false) == "a,bb,c" + end + + test "with empty string and function replacement" do + assert String.replace("elixir", "", fn "" -> "" end) == "elixir" + assert String.replace("ELIXIR", "", fn "" -> "." end) == ".E.L.I.X.I.R." + assert String.replace("ELIXIR", "", fn "" -> "." end, global: true) == ".E.L.I.X.I.R." + assert String.replace("ELIXIR", "", fn "" -> "." end, global: false) == ".ELIXIR" + + assert String.replace("elixir", "", fn "" -> [""] end) == "elixir" + assert String.replace("ELIXIR", "", fn "" -> ["."] end) == ".E.L.I.X.I.R." + assert String.replace("ELIXIR", "", fn "" -> ["."] end, global: true) == ".E.L.I.X.I.R." + assert String.replace("ELIXIR", "", fn "" -> ["."] end, global: false) == ".ELIXIR" + end + + test "with match pattern and function replacement" do + assert String.replace("a,b,c", ",", fn "," -> "-" end) == "a-b-c" + assert String.replace("a,b,c", [",", "b"], fn x -> "[#{x}]" end) == "a[,][b][,]c" + assert String.replace("a,b,c", [",", "b"], fn x -> [?[, x, ?]] end) == "a[,][b][,]c" + + assert String.replace("a,b,c", ",", fn "," -> "-" end, global: false) == "a-b,c" + assert String.replace("a,b,c", [",", "b"], fn x -> "[#{x}]" end, global: false) == "a[,]b,c" + assert String.replace("ãéã", "é", fn "é" -> "e" end, global: false) == "ãeã" + end + + test "with regex and function replacement" do + assert String.replace("a,b,c", ~r/,(.)/, fn x -> "#{x}#{x}" end) == "a,b,b,c,c" + assert String.replace("a,b,c", ~r/,(.)/, fn x -> [x, x] end) == "a,b,b,c,c" + assert String.replace("a,b,c", ~r/,(.)/, fn x -> "#{x}#{x}" end, global: false) == "a,b,b,c" + assert String.replace("a,b,c", ~r/,(.)/, fn x -> [x, x] end, global: false) == "a,b,b,c" + end end - test :duplicate do + describe "replace/4" do + test "with incorrect params" do + assert_raise FunctionClauseError, "no function clause matching in String.replace/4", fn -> + String.replace("a,b,c", "a,b,c", ",", "") + end + end + end + + test "duplicate/2" do assert String.duplicate("abc", 0) == "" assert String.duplicate("abc", 1) == "abc" assert String.duplicate("abc", 2) == "abcabc" assert String.duplicate("&ã$", 2) == "&ã$&ã$" + + assert_raise ArgumentError, fn -> + String.duplicate("abc", -1) + end end - test :codepoints do + test "codepoints/1" do assert String.codepoints("elixir") == ["e", "l", "i", "x", "i", "r"] - assert String.codepoints("elixír") == ["e", "l", "i", "x", "í", "r"] # slovak - assert String.codepoints("ոգելից ըմպելիք") == ["ո", "գ", "ե", "լ", "ի", "ց", " ", "ը", "մ", "պ", "ե", "լ", "ի", "ք"] # armenian - assert String.codepoints("эліксір") == ["э", "л", "і", "к", "с", "і", "р"] # belarussian - assert String.codepoints("ελιξήριο") == ["ε", "λ", "ι", "ξ", "ή", "ρ", "ι", "ο"] # greek - assert String.codepoints("סם חיים") == ["ס", "ם", " ", "ח", "י", "י", "ם"] # hebraic - assert String.codepoints("अमृत") == ["अ", "म", "ृ", "त"] # hindi - assert String.codepoints("স্পর্শমণি") == ["স", "্", "প", "র", "্", "শ", "ম", "ণ", "ি"] # bengali - assert String.codepoints("સર્વશ્રેષ્ઠ ઇલાજ") == ["સ", "ર", "્", "વ", "શ", "્", "ર", "ે", "ષ", "્", "ઠ", " ", "ઇ", "લ", "ા", "જ"] # gujarati - assert String.codepoints("世界中の一番") == ["世", "界", "中", "の", "一", "番"] # japanese + # slovak + assert String.codepoints("elixír") == ["e", "l", "i", "x", "í", "r"] + # armenian + assert String.codepoints("ոգելից ըմպելիք") == + ["ո", "գ", "ե", "լ", "ի", "ց", " ", "ը", "մ", "պ", "ե", "լ", "ի", "ք"] + + # belarussian + assert String.codepoints("эліксір") == ["э", "л", "і", "к", "с", "і", "р"] + # greek + assert String.codepoints("ελιξήριο") == ["ε", "λ", "ι", "ξ", "ή", "ρ", "ι", "ο"] + # hebraic + assert String.codepoints("סם חיים") == ["ס", "ם", " ", "ח", "י", "י", "ם"] + # hindi + assert String.codepoints("अमृत") == ["अ", "म", "ृ", "त"] + # bengali + assert String.codepoints("স্পর্শমণি") == ["স", "্", "প", "র", "্", "শ", "ম", "ণ", "ি"] + # gujarati + assert String.codepoints("સર્વશ્રેષ્ઠ ઇલાજ") == + ["સ", "ર", "્", "વ", "શ", "્", "ર", "ે", "ષ", "્", "ઠ", " ", "ઇ", "લ", "ા", "જ"] + + # japanese + assert String.codepoints("世界中の一番") == ["世", "界", "中", "の", "一", "番"] assert String.codepoints("がガちゃ") == ["が", "ガ", "ち", "ゃ"] assert String.codepoints("") == [] + assert String.codepoints("ϖͲϥЫݎߟΈټϘለДШव׆ש؇؊صلټܗݎޥޘ߉ऌ૫ሏᶆ℆ℙℱ ⅚Ⅷ↠∈⌘①ffi") == - ["ϖ", "Ͳ", "ϥ", "Ы", "ݎ", "ߟ", "Έ", "ټ", "Ϙ", "ለ", "Д", "Ш", "व", "׆", "ש", "؇", "؊", "ص", "ل", "ټ", "ܗ", "ݎ", "ޥ", "ޘ", "߉", "ऌ", "૫", "ሏ", "ᶆ", "℆", "ℙ", "ℱ", " ", "⅚", "Ⅷ", "↠", "∈", "⌘", "①", "ffi"] + ["ϖ", "Ͳ", "ϥ", "Ы", "ݎ", "ߟ", "Έ"] ++ + ["ټ", "Ϙ", "ለ", "Д", "Ш", "व"] ++ + ["׆", "ש", "؇", "؊", "ص", "ل", "ټ"] ++ + ["ܗ", "ݎ", "ޥ", "ޘ", "߉", "ऌ", "૫"] ++ + ["ሏ", "ᶆ", "℆", "ℙ", "ℱ", " ", "⅚"] ++ ["Ⅷ", "↠", "∈", "⌘", "①", "ffi"] + end + + test "equivalent?/2" do + assert String.equivalent?("", "") + assert String.equivalent?("elixir", "elixir") + assert String.equivalent?("뢴", "뢴") + assert String.equivalent?("ṩ", "ṩ") + refute String.equivalent?("ELIXIR", "elixir") + refute String.equivalent?("døge", "dóge") end - test :graphemes do + test "graphemes/1" do # Extended assert String.graphemes("Ā̀stute") == ["Ā̀", "s", "t", "u", "t", "e"] # CLRF - assert String.graphemes("\n\r\f") == ["\n\r", "\f"] + assert String.graphemes("\r\n\f") == ["\r\n", "\f"] # Regional indicator - assert String.graphemes("\x{1F1E6}\x{1F1E7}\x{1F1E8}") == ["\x{1F1E6}\x{1F1E7}\x{1F1E8}"] + assert String.graphemes("\u{1F1E6}\u{1F1E7}") == ["\u{1F1E6}\u{1F1E7}"] + assert String.graphemes("\u{1F1E6}\u{1F1E7}\u{1F1E8}") == ["\u{1F1E6}\u{1F1E7}", "\u{1F1E8}"] # Hangul - assert String.graphemes("\x{1100}\x{115D}\x{B4A4}") == ["ᄀᅝ뒤"] + assert String.graphemes("\u1100\u115D\uB4A4") == ["ᄀᅝ뒤"] # Special Marking with Extended - assert String.graphemes("a\x{0300}\x{0903}") == ["a\x{0300}\x{0903}"] + assert String.graphemes("a\u0300\u0903") == ["a\u0300\u0903"] end - test :next_grapheme do + test "next_grapheme/1" do assert String.next_grapheme("Ā̀stute") == {"Ā̀", "stute"} + assert String.next_grapheme(<<225, 158, 128, 225, 158, 185, 225>>) == {"កឹ", <<225>>} assert String.next_grapheme("") == nil end - test :first do + describe "randomized" do + test "next_grapheme" do + for _ <- 1..10 do + bin = :crypto.strong_rand_bytes(20) + + try do + bin |> Stream.unfold(&String.next_grapheme/1) |> Enum.to_list() + rescue + # Ignore malformed pictographic sequences + _ -> :ok + else + list -> + assert Enum.all?(list, &is_binary/1), "cannot build graphemes for #{inspect(bin)}" + end + end + end + + test "split empty" do + for _ <- 1..10 do + bin = :crypto.strong_rand_bytes(20) + + try do + String.split(bin, "") + rescue + # Ignore malformed pictographic sequences + _ -> :ok + else + split -> + assert Enum.all?(split, &is_binary/1), "cannot split #{inspect(bin)}" + assert IO.iodata_to_binary(split) == bin + end + end + end + + test "graphemes" do + for _ <- 1..10 do + bin = :crypto.strong_rand_bytes(20) + + try do + String.graphemes(bin) + rescue + # Ignore malformed pictographic sequences + _ -> :ok + else + graphemes -> + assert Enum.all?(graphemes, &is_binary/1), + "cannot build graphemes for #{inspect(bin)}" + + assert IO.iodata_to_binary(graphemes) == bin + end + end + end + end + + test "first/1" do assert String.first("elixir") == "e" assert String.first("íelixr") == "í" assert String.first("եոգլից ըմպելիք") == "ե" @@ -265,7 +685,7 @@ defmodule StringTest do assert String.first("") == nil end - test :last do + test "last/1" do assert String.last("elixir") == "r" assert String.last("elixrí") == "í" assert String.last("եոգլից ըմպելիքե") == "ե" @@ -277,7 +697,7 @@ defmodule StringTest do assert String.last("") == nil end - test :length do + test "length/1" do assert String.length("elixir") == 6 assert String.length("elixrí") == 6 assert String.length("եոգլից") == 6 @@ -286,10 +706,11 @@ defmodule StringTest do assert String.length("סם ייםח") == 7 assert String.length("がガちゃ") == 4 assert String.length("Ā̀stute") == 6 + assert String.length("👨‍👩‍👧‍👦") == 1 assert String.length("") == 0 end - test :at do + test "at/2" do assert String.at("л", 0) == "л" assert String.at("elixir", 1) == "l" assert String.at("がガちゃ", 2) == "ち" @@ -299,9 +720,17 @@ defmodule StringTest do assert String.at("л", -3) == nil assert String.at("Ā̀stute", 1) == "s" assert String.at("elixir", 6) == nil + + assert_raise FunctionClauseError, fn -> + String.at("elixir", 0.1) + end + + assert_raise FunctionClauseError, fn -> + String.at("elixir", -0.1) + end end - test :slice do + test "slice/3" do assert String.slice("elixir", 1, 3) == "lix" assert String.slice("あいうえお", 2, 2) == "うえ" assert String.slice("ειξήριολ", 2, 3) == "ξήρ" @@ -311,9 +740,9 @@ defmodule StringTest do assert String.slice("elixir", -3, 2) == "xi" assert String.slice("あいうえお", -4, 3) == "いうえ" assert String.slice("ειξήριολ", -5, 3) == "ήρι" - assert String.slice("elixir", -10, 1) == "" - assert String.slice("あいうえお", -10, 2) == "" - assert String.slice("ειξήριολ", -10, 3) == "" + assert String.slice("elixir", -10, 1) == "e" + assert String.slice("あいうえお", -10, 2) == "あい" + assert String.slice("ειξήριολ", -10, 3) == "ειξ" assert String.slice("elixir", 8, 2) == "" assert String.slice("あいうえお", 6, 2) == "" assert String.slice("ειξήριολ", 8, 1) == "" @@ -321,13 +750,17 @@ defmodule StringTest do assert String.slice("elixir", 0, 0) == "" assert String.slice("elixir", 5, 0) == "" assert String.slice("elixir", -5, 0) == "" + assert String.slice("elixir", -10, 10) == "elixir" assert String.slice("", 0, 1) == "" assert String.slice("", 1, 1) == "" + end - assert String.slice("elixir", 0..-2) == "elixi" + test "slice/2" do + assert String.slice("elixir", 0..-2//1) == "elixi" assert String.slice("elixir", 1..3) == "lix" assert String.slice("elixir", -5..-3) == "lix" assert String.slice("elixir", -5..3) == "lix" + assert String.slice("elixir", -10..10) == "elixir" assert String.slice("あいうえお", 2..3) == "うえ" assert String.slice("ειξήριολ", 2..4) == "ξήρ" assert String.slice("elixir", 3..6) == "xir" @@ -342,141 +775,372 @@ defmodule StringTest do assert String.slice("ειξήριολ", 9..9) == "" assert String.slice("", 0..0) == "" assert String.slice("", 1..1) == "" - assert String.slice("あいうえお", -2..-4) == "" - assert String.slice("あいうえお", -10..-15) == "" + assert String.slice("あいうえお", -2..-4//1) == "" + assert String.slice("あいうえお", -10..-15//1) == "" + assert String.slice("hello あいうえお Unicode", 8..-1//1) == "うえお Unicode" + assert String.slice("abc", -1..14) == "c" + assert String.slice("a·̀ͯ‿.⁀:", 0..-2//1) == "a·̀ͯ‿.⁀" + + assert_raise FunctionClauseError, fn -> + String.slice(nil, 0..1) + end + + assert ExUnit.CaptureIO.capture_io(:stderr, fn -> + assert String.slice("elixir", 0..-2//-1) == "elixi" + end) =~ "negative steps are not supported in String.slice/2, pass 0..-2//1 instead" + end + + test "slice/2 with steps" do + assert String.slice("elixir", 0..-2//2) == "eii" + assert String.slice("elixir", 1..3//2) == "lx" + assert String.slice("elixir", -5..-3//2) == "lx" + assert String.slice("elixir", -5..3//2) == "lx" + assert String.slice("あいうえお", 2..3//2) == "う" + assert String.slice("ειξήριολ", 2..4//2) == "ξρ" + assert String.slice("elixir", 3..6//2) == "xr" + assert String.slice("あいうえお", 3..7//2) == "え" + assert String.slice("ειξήριολ", 5..8//2) == "ιλ" + assert String.slice("elixir", -3..-2//2) == "x" + assert String.slice("あいうえお", -4..-2//2) == "いえ" + assert String.slice("ειξήριολ", -5..-3//2) == "ήι" + assert String.slice("elixir", 8..9//2) == "" + assert String.slice("", 0..0//2) == "" + assert String.slice("", 1..1//2) == "" + assert String.slice("あいうえお", -2..-4//2) == "" + assert String.slice("あいうえお", -10..-15//2) == "" + assert String.slice("hello あいうえお Unicode", 8..-1//2) == "うおUioe" + assert String.slice("abc", -1..14//2) == "c" + assert String.slice("a·̀ͯ‿.⁀:", 0..-2//2) == "a‿⁀" + end + + test "byte_slice/2" do + # ASCII + assert String.byte_slice("elixir", 0, 6) == "elixir" + assert String.byte_slice("elixir", 0, 5) == "elixi" + assert String.byte_slice("elixir", 1, 4) == "lixi" + assert String.byte_slice("elixir", 0, 10) == "elixir" + assert String.byte_slice("elixir", -3, 10) == "xir" + assert String.byte_slice("elixir", -10, 10) == "elixir" + assert String.byte_slice("elixir", 1, 0) == "" + assert String.byte_slice("elixir", 10, 10) == "" + + # 2 byte + assert String.byte_slice("héllò", 1, 4) == "éll" + assert String.byte_slice("héllò", 1, 5) == "éll" + assert String.byte_slice("héllò", 1, 6) == "éllò" + assert String.byte_slice("héllò", 2, 4) == "llò" + + # 3 byte + assert String.byte_slice("hかllか", 1, 4) == "かl" + assert String.byte_slice("hかllか", 1, 5) == "かll" + assert String.byte_slice("hかllか", 1, 6) == "かll" + assert String.byte_slice("hかllか", 1, 7) == "かll" + assert String.byte_slice("hかllか", 1, 8) == "かllか" + assert String.byte_slice("hかllか", 2, 4) == "ll" + assert String.byte_slice("hかllか", 2, 5) == "llか" + + # 4 byte + assert String.byte_slice("h😍ll😍", 1, 4) == "😍" + assert String.byte_slice("h😍ll😍", 1, 5) == "😍l" + assert String.byte_slice("h😍ll😍", 1, 6) == "😍ll" + assert String.byte_slice("h😍ll😍", 1, 7) == "😍ll" + assert String.byte_slice("h😍ll😍", 1, 8) == "😍ll" + assert String.byte_slice("h😍ll😍", 1, 9) == "😍ll" + assert String.byte_slice("h😍ll😍", 1, 10) == "😍ll😍" + assert String.byte_slice("h😍ll😍", 2, 5) == "ll" + assert String.byte_slice("h😍ll😍", 2, 6) == "ll😍" + + # Already truncated + assert String.byte_slice(<<178, "ll", 178>>, 0, 10) == "ll" + + # Already invalid + assert String.byte_slice(<<255, "ll", 255>>, 0, 10) == <<255, "ll", 255>> end - test :valid? do + test "valid?/1" do assert String.valid?("afds") assert String.valid?("øsdfh") assert String.valid?("dskfjあska") + assert String.valid?(<<0xEF, 0xB7, 0x90>>) - refute String.valid?(<<0xffff :: 16>>) - refute String.valid?("asd" <> <<0xffff :: 16>>) - end + refute String.valid?(<<0xFFFF::16>>) + refute String.valid?("asd" <> <<0xFFFF::16>>) - test :valid_character? do - assert String.valid_character?("a") - assert String.valid_character?("ø") - assert String.valid_character?("あ") + assert String.valid?("afdsafdsafds", :fast_ascii) + assert String.valid?("øsdfhøsdfh", :fast_ascii) + assert String.valid?("dskfjあskadskfjあska", :fast_ascii) + assert String.valid?(<<0xEF, 0xB7, 0x90, 0xEF, 0xB7, 0x90, 0xEF, 0xB7, 0x90>>, :fast_ascii) - refute String.valid_character?("\x{ffff}") - refute String.valid_character?("ab") + refute String.valid?(<<0xFFFF::16>>, :fast_ascii) + refute String.valid?("asdasdasd" <> <<0xFFFF::16>>, :fast_ascii) end - test :chunk_valid do - assert String.chunk("", :valid) == [] + test "replace_invalid" do + assert String.replace_invalid("") === "" + assert String.replace_invalid(<<0xFF>>) === "�" + assert String.replace_invalid(<<0xFF, 0xFF, 0xFF>>) === "���" - assert String.chunk("ødskfjあ\011ska", :valid) - == ["ødskfjあ\011ska"] - assert String.chunk("abc\x{0ffff}def", :valid) - == ["abc", <<0x0ffff::utf8>>, "def"] - assert String.chunk("\x{0fffe}\x{3ffff}привет\x{0ffff}мир", :valid) - == [<<0x0fffe::utf8, 0x3ffff::utf8>>, "привет", <<0x0ffff::utf8>>, "мир"] - assert String.chunk("日本\x{0ffff}\x{fdef}ござございます\x{fdd0}", :valid) - == ["日本", <<0x0ffff::utf8, 0xfdef::utf8>>, "ござございます", <<0xfdd0::utf8>>] - end + # Valid ASCII + assert String.replace_invalid("hello") === "hello" - test :chunk_printable do - assert String.chunk("", :printable) == [] + # Valid UTF-8 + assert String.replace_invalid("こんにちは") === "こんにちは" + + # 2/3 byte truncated "ề" + assert String.replace_invalid(<<225, 187>>) === "�" + assert String.replace_invalid("nem rán b" <> <<225, 187>> <> " bề") === "nem rán b� bề" + + # 2/4 byte truncated "😔" + assert String.replace_invalid(<<240, 159>>) === "�" + assert String.replace_invalid("It's so over " <> <<240, 159>>) === "It's so over �" + + # 3/4 byte truncated "😃" + assert String.replace_invalid(<<240, 159, 152>>) === "�" + assert String.replace_invalid("We're so back " <> <<240, 159, 152>>) === "We're so back �" - assert String.chunk("ødskfjあska", :printable) - == ["ødskfjあska"] - assert String.chunk("abc\x{0ffff}def", :printable) - == ["abc", <<0x0ffff::utf8>>, "def"] - assert String.chunk("\006ab\005cdef\003\000", :printable) - == [<<06>>, "ab", <<05>>, "cdef", <<03, 0>>] + # 3 byte overlong "e" + assert String.replace_invalid(<<0b11100000, 0b10000001, 0b10100101>>) === "���" end - test :starts_with? do - ## Normal cases ## - assert String.starts_with? "hello", "he" - assert String.starts_with? "hello", "hello" - assert String.starts_with? "hello", ["hellö", "hell"] - assert String.starts_with? "エリクシア", "エリ" - refute String.starts_with? "hello", "lo" - refute String.starts_with? "hello", "hellö" - refute String.starts_with? "hello", ["hellö", "goodbye"] - refute String.starts_with? "エリクシア", "仙丹" + test "chunk/2 with :valid trait" do + assert String.chunk("", :valid) == [] - ## Edge cases ## - assert String.starts_with? "", "" - assert String.starts_with? "", ["", "a"] - assert String.starts_with? "b", ["", "a"] + assert String.chunk("ødskfjあ\x11ska", :valid) == ["ødskfjあ\x11ska"] + end - assert String.starts_with? "abc", "" - assert String.starts_with? "abc", [""] + test "chunk/2 with :printable trait" do + assert String.chunk("", :printable) == [] - refute String.starts_with? "", "abc" - refute String.starts_with? "", [" "] + assert String.chunk("ødskfjあska", :printable) == ["ødskfjあska"] + assert String.chunk("abc\u{0FFFF}def", :printable) == ["abc", <<0x0FFFF::utf8>>, "def"] - ## Sanity checks ## - assert String.starts_with? "", ["", ""] - assert String.starts_with? "abc", ["", ""] + assert String.chunk("\x06ab\x05cdef\x03\0", :printable) == + [<<6>>, "ab", <<5>>, "cdef", <<3, 0>>] end - test :ends_with? do - ## Normal cases ## - assert String.ends_with? "hello", "lo" - assert String.ends_with? "hello", "hello" - assert String.ends_with? "hello", ["hell", "lo", "xx"] - assert String.ends_with? "hello", ["hellö", "lo"] - assert String.ends_with? "エリクシア", "シア" - refute String.ends_with? "hello", "he" - refute String.ends_with? "hello", "hellö" - refute String.ends_with? "hello", ["hel", "goodbye"] - refute String.ends_with? "エリクシア", "仙丹" + test "starts_with?/2" do + assert String.starts_with?("hello", "he") + assert String.starts_with?("hello", "hello") + refute String.starts_with?("hello", []) + assert String.starts_with?("hello", "") + assert String.starts_with?("hello", [""]) + assert String.starts_with?("hello", ["hellö", "hell"]) + assert String.starts_with?("エリクシア", "エリ") + refute String.starts_with?("hello", "lo") + refute String.starts_with?("hello", "hellö") + refute String.starts_with?("hello", ["hellö", "goodbye"]) + refute String.starts_with?("エリクシア", "仙丹") + end - ## Edge cases ## - assert String.ends_with? "", "" - assert String.ends_with? "", ["", "a"] - refute String.ends_with? "", ["a", "b"] + test "ends_with?/2" do + assert String.ends_with?("hello", "lo") + assert String.ends_with?("hello", "hello") + refute String.ends_with?("hello", []) + assert String.ends_with?("hello", ["hell", "lo", "xx"]) + assert String.ends_with?("hello", ["hellö", "lo"]) + assert String.ends_with?("エリクシア", "シア") + refute String.ends_with?("hello", "he") + refute String.ends_with?("hello", "hellö") + refute String.ends_with?("hello", ["hel", "goodbye"]) + refute String.ends_with?("エリクシア", "仙丹") + end - assert String.ends_with? "abc", "" - assert String.ends_with? "abc", ["", "x"] + test "contains?/2" do + assert String.contains?("elixir of life", "of") + assert String.contains?("エリクシア", "シ") + refute String.contains?("elixir of life", []) + assert String.contains?("elixir of life", "") + assert String.contains?("elixir of life", [""]) + assert String.contains?("elixir of life", ["mercury", "life"]) + refute String.contains?("elixir of life", "death") + refute String.contains?("エリクシア", "仙") + refute String.contains?("elixir of life", ["death", "mercury", "eternal life"]) + end + + test "to_charlist/1" do + assert String.to_charlist("æß") == [?æ, ?ß] + assert String.to_charlist("abc") == [?a, ?b, ?c] - refute String.ends_with? "", "abc" - refute String.ends_with? "", [" "] + assert_raise UnicodeConversionError, "invalid encoding starting at <<223, 255>>", fn -> + String.to_charlist(<<0xDF, 0xFF>>) + end - ## Sanity checks ## - assert String.ends_with? "", ["", ""] - assert String.ends_with? "abc", ["", ""] + assert_raise UnicodeConversionError, "incomplete encoding starting at <<195>>", fn -> + String.to_charlist(<<106, 111, 115, 195>>) + end end - test :contains? do - ## Normal cases ## - assert String.contains? "elixir of life", "of" - assert String.contains? "エリクシア", "シ" - assert String.contains? "elixir of life", ["mercury", "life"] - refute String.contains? "elixir of life", "death" - refute String.contains? "エリクシア", "仙" - refute String.contains? "elixir of life", ["death", "mercury", "eternal life"] + test "to_float/1" do + assert String.to_float("3.0") == 3.0 - ## Edge cases ## - assert String.contains? "", "" - assert String.contains? "abc", "" - assert String.contains? "abc", ["", "x"] + three = fn -> "3" end + assert_raise ArgumentError, fn -> String.to_float(three.()) end - refute String.contains? "", " " - refute String.contains? "", "a" + dot_three = fn -> ".3" end + assert_raise ArgumentError, fn -> String.to_float(dot_three.()) end + end - ## Sanity checks ## - assert String.contains? "", ["", ""] - assert String.contains? "abc", ["", ""] + test "jaro_distance/2" do + assert String.jaro_distance("same", "same") == 1.0 + assert String.jaro_distance("any", "") == 0.0 + assert String.jaro_distance("", "any") == 0.0 + assert String.jaro_distance("martha", "marhta") == 0.9444444444444445 + assert String.jaro_distance("martha", "marhha") == 0.888888888888889 + assert String.jaro_distance("marhha", "martha") == 0.888888888888889 + assert String.jaro_distance("dwayne", "duane") == 0.8222222222222223 + assert String.jaro_distance("dixon", "dicksonx") == 0.7666666666666666 + assert String.jaro_distance("xdicksonx", "dixon") == 0.7518518518518519 + assert String.jaro_distance("shackleford", "shackelford") == 0.9696969696969697 + assert String.jaro_distance("dunningham", "cunnigham") == 0.8962962962962964 + assert String.jaro_distance("nichleson", "nichulson") == 0.9259259259259259 + assert String.jaro_distance("jones", "johnson") == 0.7904761904761904 + assert String.jaro_distance("massey", "massie") == 0.888888888888889 + assert String.jaro_distance("abroms", "abrams") == 0.888888888888889 + assert String.jaro_distance("hardin", "martinez") == 0.7222222222222222 + assert String.jaro_distance("itman", "smith") == 0.4666666666666666 + assert String.jaro_distance("jeraldine", "geraldine") == 0.9259259259259259 + assert String.jaro_distance("michelle", "michael") == 0.8690476190476191 + assert String.jaro_distance("julies", "julius") == 0.888888888888889 + assert String.jaro_distance("tanya", "tonya") == 0.8666666666666667 + assert String.jaro_distance("sean", "susan") == 0.7833333333333333 + assert String.jaro_distance("jon", "john") == 0.9166666666666666 + assert String.jaro_distance("jon", "jan") == 0.7777777777777777 + assert String.jaro_distance("семена", "стремя") == 0.6666666666666666 + assert String.jaro_distance("Sunday", "Saturday") == 0.7194444444444444 end - test :to_char_list do - assert String.to_char_list("æß") == [?æ, ?ß] - assert String.to_char_list("abc") == [?a, ?b, ?c] + test "myers_difference/2" do + assert String.myers_difference("", "abc") == [ins: "abc"] + assert String.myers_difference("abc", "") == [del: "abc"] + assert String.myers_difference("", "") == [] + assert String.myers_difference("abc", "abc") == [eq: "abc"] + assert String.myers_difference("abc", "aйbc") == [eq: "a", ins: "й", eq: "bc"] + assert String.myers_difference("aйbc", "abc") == [eq: "a", del: "й", eq: "bc"] + end - assert_raise UnicodeConversionError, - "invalid encoding starting at <<223, 255>>", fn -> - String.to_char_list(<< 0xDF, 0xFF >>) - end + test "normalize/2" do + assert String.normalize("ŝ", :nfd) == "ŝ" + assert String.normalize("ḇravô", :nfd) == "ḇravô" + assert String.normalize("ṩierra", :nfd) == "ṩierra" + assert String.normalize("뢴", :nfd) == "뢴" + assert String.normalize("êchǭ", :nfc) == "êchǭ" + assert String.normalize("거̄", :nfc) == "거̄" + assert String.normalize("뢴", :nfc) == "뢴" + + ## Error cases + assert String.normalize(<<15, 216>>, :nfc) == <<15, 216>> + assert String.normalize(<<15, 216>>, :nfd) == <<15, 216>> + assert String.normalize(<<216, 15>>, :nfc) == <<216, 15>> + assert String.normalize(<<216, 15>>, :nfd) == <<216, 15>> + + assert String.normalize(<<15, 216>>, :nfkc) == <<15, 216>> + assert String.normalize(<<15, 216>>, :nfkd) == <<15, 216>> + assert String.normalize(<<216, 15>>, :nfkc) == <<216, 15>> + assert String.normalize(<<216, 15>>, :nfkd) == <<216, 15>> + + ## Cases from NormalizationTest.txt + + # 05B8 05B9 05B1 0591 05C3 05B0 05AC 059F + # 05B1 05B8 05B9 0591 05C3 05B0 05AC 059F + # HEBREW POINT QAMATS, HEBREW POINT HOLAM, HEBREW POINT HATAF SEGOL, + # HEBREW ACCENT ETNAHTA, HEBREW PUNCTUATION SOF PASUQ, HEBREW POINT SHEVA, + # HEBREW ACCENT ILUY, HEBREW ACCENT QARNEY PARA + assert String.normalize("ֱָֹ֑׃ְ֬֟", :nfc) == "ֱָֹ֑׃ְ֬֟" + + # 095D (exclusion list) + # 0922 093C + # DEVANAGARI LETTER RHA + assert String.normalize("ढ़", :nfc) == "ढ़" + + # 0061 0315 0300 05AE 0340 0062 + # 00E0 05AE 0300 0315 0062 + # LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, + # HEBREW ACCENT ZINOR, COMBINING GRAVE TONE MARK, LATIN SMALL LETTER B + assert String.normalize("à֮̀̕b", :nfc) == "à֮̀̕b" + + # 0344 + # 0308 0301 + # COMBINING GREEK DIALYTIKA TONOS + assert String.normalize("\u0344", :nfc) == "\u0308\u0301" + + # 115B9 0334 115AF + # 115B9 0334 115AF + # SIDDHAM VOWEL SIGN AI, COMBINING TILDE OVERLAY, SIDDHAM VOWEL SIGN AA + assert String.normalize("𑖹̴𑖯", :nfc) == "𑖹̴𑖯" + + # HEBREW ACCENT ETNAHTA, HEBREW PUNCTUATION SOF PASUQ, HEBREW POINT SHEVA, + # HEBREW ACCENT ILUY, HEBREW ACCENT QARNEY PARA + assert String.normalize("ֱָֹ֑׃ְ֬֟", :nfc) == "ֱָֹ֑׃ְ֬֟" + + # 095D (exclusion list) + # HEBREW ACCENT ETNAHTA, HEBREW PUNCTUATION SOF PASUQ, HEBREW POINT SHEVA, + # HEBREW ACCENT ILUY, HEBREW ACCENT QARNEY PARA + assert String.normalize("ֱָֹ֑׃ְ֬֟", :nfc) == "ֱָֹ֑׃ְ֬֟" + + # 095D (exclusion list) + # 0922 093C + # DEVANAGARI LETTER RHA + assert String.normalize("ढ़", :nfc) == "ढ़" + + # 0061 0315 0300 05AE 0340 0062 + # 00E0 05AE 0300 0315 0062 + # LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, + # HEBREW ACCENT ZINOR, COMBINING GRAVE TONE MARK, LATIN SMALL LETTER B + assert String.normalize("à֮̀̕b", :nfc) == "à֮̀̕b" + + # 0344 + # 0308 0301 + # COMBINING GREEK DIALYTIKA TONOS + assert String.normalize("\u0344", :nfc) == "\u0308\u0301" + + # 115B9 0334 115AF + # 115B9 0334 115AF + # SIDDHAM VOWEL SIGN AI, COMBINING TILDE OVERLAY, SIDDHAM VOWEL SIGN AA + assert String.normalize("𑖹̴𑖯", :nfc) == "𑖹̴𑖯" + + # (ff; ff; ff; ff; ff; ) LATIN SMALL LIGATURE FF + # FB00;FB00;FB00;0066 0066;0066 0066; + assert String.normalize("ff", :nfkd) == "\u0066\u0066" + + # (fl; fl; fl; fl; fl; ) LATIN SMALL LIGATURE FL + # FB02;FB02;FB02;0066 006C;0066 006C; + assert String.normalize("fl", :nfkd) == "\u0066\u006C" + + # (ſt; ſt; ſt; st; st; ) LATIN SMALL LIGATURE LONG S T + # FB05;FB05;FB05;0073 0074;0073 0074; + assert String.normalize("ſt", :nfkd) == "\u0073\u0074" + + # (st; st; st; st; st; ) LATIN SMALL LIGATURE ST + # FB06;FB06;FB06;0073 0074;0073 0074; + assert String.normalize("\u0073\u0074", :nfkc) == "\u0073\u0074" + + # (ﬓ; ﬓ; ﬓ; մն; մն; ) ARMENIAN SMALL LIGATURE MEN NOW + # FB13;FB13;FB13;0574 0576;0574 0576; + assert String.normalize("\u0574\u0576", :nfkc) == "\u0574\u0576" + end - assert_raise UnicodeConversionError, - "incomplete encoding starting at <<195>>", fn -> - String.to_char_list(<< 106, 111, 115, 195 >>) - end + # Carriage return can be a grapheme cluster if followed by + # newline so we test some corner cases here. + test "carriage return" do + assert String.at("\r\t\v", 0) == "\r" + assert String.at("\r\t\v", 1) == "\t" + assert String.at("\r\t\v", 2) == "\v" + assert String.at("\xFF\r\t\v", 1) == "\r" + assert String.at("\r\xFF\t\v", 2) == "\t" + assert String.at("\r\t\xFF\v", 3) == "\v" + + assert String.last("\r\t\v") == "\v" + assert String.last("\r\xFF\t\xFF\v") == "\v" + + assert String.next_grapheme("\r\t\v") == {"\r", "\t\v"} + assert String.next_grapheme("\t\v") == {"\t", "\v"} + assert String.next_grapheme("\v") == {"\v", ""} + + assert String.length("\r\t\v") == 3 + assert String.length("\r\xFF\t\v") == 4 + assert String.length("\r\t\xFF\v") == 4 + + assert String.bag_distance("\r\t\xFF\v", "\xFF\r\n\xFF") == 0.25 + assert String.split("\r\t\v", "") == ["", "\r", "\t", "\v", ""] end end diff --git a/lib/elixir/test/elixir/supervisor/spec_test.exs b/lib/elixir/test/elixir/supervisor/spec_test.exs deleted file mode 100644 index a15234bcef3..00000000000 --- a/lib/elixir/test/elixir/supervisor/spec_test.exs +++ /dev/null @@ -1,85 +0,0 @@ -Code.require_file "../test_helper.exs", __DIR__ - -defmodule Supervisor.SpecTest do - use ExUnit.Case, async: true - - import Supervisor.Spec - - test "worker/3" do - assert worker(Foo, [1, 2, 3]) == { - Foo, - {Foo, :start_link, [1, 2, 3]}, - :permanent, - 5000, - :worker, - [Foo] - } - - opts = [id: :sample, function: :start, modules: :dynamic, - restart: :temporary, shutdown: :brutal_kill] - - assert worker(Foo, [1, 2, 3], opts) == { - :sample, - {Foo, :start, [1, 2, 3]}, - :temporary, - :brutal_kill, - :worker, - :dynamic - } - end - - test "worker/3 with GenEvent" do - assert worker(GenEvent, [[name: :hello]]) == { - GenEvent, - {GenEvent, :start_link, [[name: :hello]]}, - :permanent, - 5000, - :worker, - :dynamic - } - end - - test "supervisor/3" do - assert supervisor(Foo, [1, 2, 3]) == { - Foo, - {Foo, :start_link, [1, 2, 3]}, - :permanent, - :infinity, - :supervisor, - [Foo] - } - - opts = [id: :sample, function: :start, modules: :dynamic, - restart: :temporary, shutdown: :brutal_kill] - - assert supervisor(Foo, [1, 2, 3], opts) == { - :sample, - {Foo, :start, [1, 2, 3]}, - :temporary, - :brutal_kill, - :supervisor, - :dynamic - } - end - - test "supervise/2" do - assert supervise([], strategy: :one_for_one) == { - :ok, {{:one_for_one, 5, 5}, []} - } - - children = [worker(GenEvent, [])] - options = [strategy: :one_for_all, max_restarts: 1, max_seconds: 1] - - assert supervise(children, options) == { - :ok, {{:one_for_all, 1, 1}, children} - } - end - - test "supervise/2 with duplicated ids" do - children = [worker(GenEvent, []), worker(GenEvent, [])] - - assert_raise ArgumentError, fn -> - supervise(children, strategy: :one_for_one) - end - end -end diff --git a/lib/elixir/test/elixir/supervisor_test.exs b/lib/elixir/test/elixir/supervisor_test.exs index eed8887e107..96d452e4420 100644 --- a/lib/elixir/test/elixir/supervisor_test.exs +++ b/lib/elixir/test/elixir/supervisor_test.exs @@ -1,4 +1,8 @@ -Code.require_file "test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("test_helper.exs", __DIR__) defmodule SupervisorTest do use ExUnit.Case, async: true @@ -6,43 +10,154 @@ defmodule SupervisorTest do defmodule Stack do use GenServer - def start_link(state, opts) do + def start_link({state, opts}) do GenServer.start_link(__MODULE__, state, opts) end - def handle_call(:pop, _from, [h|t]) do + def init(args) do + {:ok, args} + end + + def handle_call(:pop, _from, [h | t]) do {:reply, h, t} end def handle_call(:stop, _from, stack) do - # There is a race condition in between genserver terminations. + # There is a race condition between genserver terminations. # So we will explicitly unregister it here. try do - self |> Process.info(:registered_name) |> elem(1) |> Process.unregister + self() |> Process.info(:registered_name) |> elem(1) |> Process.unregister() rescue _ -> :ok end + {:stop, :normal, :ok, stack} end - def handle_cast({:push, h}, _from, t) do - {:noreply, [h|t]} + def handle_cast({:push, h}, t) do + {:noreply, [h | t]} end end defmodule Stack.Sup do use Supervisor - def init({arg, opts}) do - children = [worker(Stack, [arg, opts])] - supervise(children, strategy: :one_for_one) + def init(pair) do + Supervisor.init([{Stack, pair}], strategy: :one_for_one) + end + end + + test "generates child_spec/1" do + assert Stack.Sup.child_spec([:hello]) == %{ + id: Stack.Sup, + start: {Stack.Sup, :start_link, [[:hello]]}, + type: :supervisor + } + + defmodule CustomSup do + use Supervisor, + id: :id, + restart: :temporary, + start: {:foo, :bar, []} + + def init(arg) do + arg + end + end + + assert CustomSup.child_spec([:hello]) == %{ + id: :id, + restart: :temporary, + start: {:foo, :bar, []}, + type: :supervisor + } + end + + test "child_spec/2" do + assert Supervisor.child_spec(Task, []) == + %{id: Task, restart: :temporary, start: {Task, :start_link, [[]]}} + + assert Supervisor.child_spec({Task, :foo}, []) == + %{id: Task, restart: :temporary, start: {Task, :start_link, [:foo]}} + + assert Supervisor.child_spec(%{id: Task}, []) == %{id: Task} + + assert Supervisor.child_spec( + Task, + id: :foo, + start: {:foo, :bar, []}, + restart: :permanent, + shutdown: :infinity + ) == %{id: :foo, start: {:foo, :bar, []}, restart: :permanent, shutdown: :infinity} + + message = ~r"The module SupervisorTest was given as a child.*\nbut it does not implement"m + + assert_raise ArgumentError, message, fn -> + Supervisor.child_spec(SupervisorTest, []) + end + + message = ~r"The module Unknown was given as a child.*but it does not exist"m + + assert_raise ArgumentError, message, fn -> + Supervisor.child_spec(Unknown, []) + end + + message = ~r"supervisors expect each child to be one of" + + assert_raise ArgumentError, message, fn -> + Supervisor.child_spec("other", []) + end + end + + test "init/2" do + flags = %{intensity: 3, period: 5, strategy: :one_for_one, auto_shutdown: :never} + children = [%{id: Task, restart: :temporary, start: {Task, :start_link, [[]]}}] + assert Supervisor.init([Task], strategy: :one_for_one) == {:ok, {flags, children}} + + flags = %{intensity: 1, period: 2, strategy: :one_for_all, auto_shutdown: :never} + children = [%{id: Task, restart: :temporary, start: {Task, :start_link, [:foo]}}] + + assert Supervisor.init( + [{Task, :foo}], + strategy: :one_for_all, + max_restarts: 1, + max_seconds: 2 + ) == {:ok, {flags, children}} + + assert_raise ArgumentError, "expected :strategy option to be given", fn -> + Supervisor.init([], []) end end - import Supervisor.Spec + test "init/2 with old and new child specs" do + flags = %{intensity: 3, period: 5, strategy: :one_for_one, auto_shutdown: :never} + + children = [ + %{id: Task, restart: :temporary, start: {Task, :start_link, [[]]}}, + old_spec = {Task, {Task, :start_link, []}, :permanent, 5000, :worker, [Task]} + ] + + assert Supervisor.init([Task, old_spec], strategy: :one_for_one) == + {:ok, {flags, children}} + end + + test "start_link/2 with via" do + Supervisor.start_link([], strategy: :one_for_one, name: {:via, :global, :via_sup}) + assert Supervisor.which_children({:via, :global, :via_sup}) == [] + end + + test "start_link/3 with global" do + Supervisor.start_link([], strategy: :one_for_one, name: {:global, :global_sup}) + assert Supervisor.which_children({:global, :global_sup}) == [] + end + + test "start_link/3 with local" do + Supervisor.start_link([], strategy: :one_for_one, name: :my_sup) + assert Supervisor.which_children(:my_sup) == [] + end test "start_link/2" do - children = [worker(Stack, [[:hello], [name: :dyn_stack]])] + children = [{Stack, {[:hello], [name: :dyn_stack]}}] {:ok, pid} = Supervisor.start_link(children, strategy: :one_for_one) wait_until_registered(:dyn_stack) @@ -51,32 +166,108 @@ defmodule SupervisorTest do wait_until_registered(:dyn_stack) assert GenServer.call(:dyn_stack, :pop) == :hello + Supervisor.stop(pid) + + assert_raise ArgumentError, ~r"expected :name option to be one of the following:", fn -> + name = "my_gen_server_name" + Supervisor.start_link(children, name: name, strategy: :one_for_one) + end + + assert_raise ArgumentError, ~r"expected :name option to be one of the following:", fn -> + name = {:invalid_tuple, "my_gen_server_name"} + Supervisor.start_link(children, name: name, strategy: :one_for_one) + end + + assert_raise ArgumentError, ~r"expected :name option to be one of the following:", fn -> + name = {:via, "Via", "my_gen_server_name"} + Supervisor.start_link(children, name: name, strategy: :one_for_one) + end + end + + test "start_link/2 with old and new specs" do + children = [ + {Stack, {[:hello], []}}, + {:old_stack, {SupervisorTest.Stack, :start_link, [{[:hello], []}]}, :permanent, 5000, + :worker, [SupervisorTest.Stack]} + ] - Process.exit(pid, :normal) + {:ok, _} = Supervisor.start_link(children, strategy: :one_for_one) end test "start_link/3" do - {:ok, pid} = Supervisor.start_link(Stack.Sup, {[:hello], [name: :stat_stack]}, name: :stack_sup) - wait_until_registered(:stack_sup) - + {:ok, pid} = Supervisor.start_link(Stack.Sup, {[:hello], [name: :stat_stack]}) + wait_until_registered(:stat_stack) assert GenServer.call(:stat_stack, :pop) == :hello - Process.exit(pid, :normal) + Supervisor.stop(pid) end - test "*_child functions" do + describe "start_child/2" do + test "supports old child spec" do + {:ok, pid} = Supervisor.start_link([], strategy: :one_for_one) + child = {Task, {Task, :start_link, [fn -> :ok end]}, :temporary, 5000, :worker, [Task]} + assert {:ok, pid} = Supervisor.start_child(pid, child) + assert is_pid(pid) + end + + test "supports new child spec as tuple" do + {:ok, pid} = Supervisor.start_link([], strategy: :one_for_one) + child = %{id: Task, restart: :temporary, start: {Task, :start_link, [fn -> :ok end]}} + assert {:ok, pid} = Supervisor.start_child(pid, child) + assert is_pid(pid) + end + + test "supports new child spec" do + {:ok, pid} = Supervisor.start_link([], strategy: :one_for_one) + child = {Task, fn -> :timer.sleep(:infinity) end} + assert {:ok, pid} = Supervisor.start_child(pid, child) + assert is_pid(pid) + end + + test "with invalid child spec" do + {:ok, pid} = Supervisor.start_link([], strategy: :one_for_one) + + assert Supervisor.start_child(pid, %{}) == {:error, :missing_id} + assert Supervisor.start_child(pid, {1, 2, 3, 4, 5, 6}) == {:error, {:invalid_mfa, 2}} + + assert Supervisor.start_child(pid, %{id: 1, start: {Task, :foo, :bar}}) == + {:error, {:invalid_mfa, {Task, :foo, :bar}}} + + assert Supervisor.start_child(pid, %{id: 1, start: {Task, :foo, [:bar]}, shutdown: -1}) == + {:error, {:invalid_shutdown, -1}} + end + + test "with valid child spec" do + Process.flag(:trap_exit, true) + + {:ok, pid} = Supervisor.start_link([], strategy: :one_for_one) + + for n <- 0..1 do + assert {:ok, child_pid} = + Supervisor.start_child(pid, %{ + id: n, + start: {Task, :start_link, [fn -> Process.sleep(:infinity) end]}, + shutdown: n + }) + + assert_kill(child_pid, :shutdown) + end + end + end + + test "child life cycle" do {:ok, pid} = Supervisor.start_link([], strategy: :one_for_one) assert Supervisor.which_children(pid) == [] - assert Supervisor.count_children(pid) == - %{specs: 0, active: 0, supervisors: 0, workers: 0} + assert Supervisor.count_children(pid) == %{specs: 0, active: 0, supervisors: 0, workers: 0} - {:ok, stack} = Supervisor.start_child(pid, worker(Stack, [[:hello], []])) + child_spec = Supervisor.child_spec({Stack, {[:hello], []}}, []) + {:ok, stack} = Supervisor.start_child(pid, child_spec) assert GenServer.call(stack, :pop) == :hello assert Supervisor.which_children(pid) == - [{SupervisorTest.Stack, stack, :worker, [SupervisorTest.Stack]}] - assert Supervisor.count_children(pid) == - %{specs: 1, active: 1, supervisors: 0, workers: 1} + [{SupervisorTest.Stack, stack, :worker, [SupervisorTest.Stack]}] + + assert Supervisor.count_children(pid) == %{specs: 1, active: 1, supervisors: 0, workers: 1} assert Supervisor.delete_child(pid, Stack) == {:error, :running} assert Supervisor.terminate_child(pid, Stack) == :ok @@ -86,13 +277,18 @@ defmodule SupervisorTest do assert Supervisor.terminate_child(pid, Stack) == :ok assert Supervisor.delete_child(pid, Stack) == :ok - - Process.exit(pid, :normal) + Supervisor.stop(pid) end defp wait_until_registered(name) do - unless Process.whereis(name) do + if !Process.whereis(name) do wait_until_registered(name) end end + + defp assert_kill(pid, reason) do + ref = Process.monitor(pid) + Process.exit(pid, reason) + assert_receive {:DOWN, ^ref, _, _, _} + end end diff --git a/lib/elixir/test/elixir/system_test.exs b/lib/elixir/test/elixir/system_test.exs index 13949d346ec..b6d56dec778 100644 --- a/lib/elixir/test/elixir/system_test.exs +++ b/lib/elixir/test/elixir/system_test.exs @@ -1,78 +1,364 @@ -Code.require_file "test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("test_helper.exs", __DIR__) defmodule SystemTest do use ExUnit.Case import PathHelpers - test "build_info" do - assert is_map System.build_info - assert not nil?(System.build_info[:version]) - assert not nil?(System.build_info[:tag]) - assert not nil?(System.build_info[:date]) - end + test "build_info/0" do + build_info = System.build_info() + assert is_map(build_info) + assert is_binary(build_info[:build]) + assert is_binary(build_info[:date]) + assert is_binary(build_info[:revision]) + assert is_binary(build_info[:version]) + assert is_binary(build_info[:otp_release]) + + if build_info[:revision] != "" do + assert String.length(build_info[:revision]) >= 7 + end - test "cwd" do - assert is_binary System.cwd - assert is_binary System.cwd! + version_file = Path.join([__DIR__, "../../../..", "VERSION"]) |> Path.expand() + {:ok, version} = File.read(version_file) + assert build_info[:version] == String.trim(version) + assert build_info[:build] =~ "compiled with Erlang/OTP" end - if :file.native_name_encoding == :utf8 do - test "cwd_with_utf8" do - File.mkdir_p(tmp_path("héllò")) + test "user_home/0" do + assert is_binary(System.user_home()) + assert is_binary(System.user_home!()) + end - File.cd!(tmp_path("héllò"), fn -> - assert Path.basename(System.cwd!) == "héllò" - end) - after - File.rm_rf tmp_path("héllò") - end + test "tmp_dir/0" do + assert is_binary(System.tmp_dir()) + assert is_binary(System.tmp_dir!()) end - test "user_home" do - assert is_binary System.user_home - assert is_binary System.user_home! + test "endianness/0" do + assert System.endianness() in [:little, :big] + assert System.endianness() == System.compiled_endianness() end - test "tmp_dir" do - assert is_binary System.tmp_dir - assert is_binary System.tmp_dir! + test "pid/0" do + assert is_binary(System.pid()) end - test "argv" do - list = elixir('-e "IO.inspect System.argv" -- -o opt arg1 arg2 --long-opt 10') - {args, _} = Code.eval_string list, [] + test "argv/0" do + list = elixir(~c"-e \"IO.inspect System.argv()\" -- -o opt arg1 arg2 --long-opt 10") + {args, _} = Code.eval_string(list, []) assert args == ["-o", "opt", "arg1", "arg2", "--long-opt", "10"] end @test_var "SYSTEM_ELIXIR_ENV_TEST_VAR" - test "env" do + test "get_env/put_env/delete_env" do assert System.get_env(@test_var) == nil + assert System.get_env(@test_var, "SAMPLE") == "SAMPLE" + assert System.fetch_env(@test_var) == :error + + message = "could not fetch environment variable #{inspect(@test_var)} because it is not set" + assert_raise System.EnvError, message, fn -> System.fetch_env!(@test_var) end + System.put_env(@test_var, "SAMPLE") + assert System.get_env(@test_var) == "SAMPLE" assert System.get_env()[@test_var] == "SAMPLE" + assert System.fetch_env(@test_var) == {:ok, "SAMPLE"} + assert System.fetch_env!(@test_var) == "SAMPLE" System.delete_env(@test_var) assert System.get_env(@test_var) == nil - System.put_env(%{@test_var => "OTHER_SAMPLE"}) - assert System.get_env(@test_var) == "OTHER_SAMPLE" + assert_raise ArgumentError, ~r[cannot execute System.put_env/2 for key with \"=\"], fn -> + System.put_env("FOO=BAR", "BAZ") + end + end + + test "put_env/2" do + System.put_env(%{@test_var => "MAP_STRING"}) + assert System.get_env(@test_var) == "MAP_STRING" + + System.put_env([{String.to_atom(@test_var), "KW_ATOM"}]) + assert System.get_env(@test_var) == "KW_ATOM" + + System.put_env([{String.to_atom(@test_var), nil}]) + assert System.get_env(@test_var) == nil + end + + test "cmd/2 raises for null bytes" do + assert_raise ArgumentError, ~r"cannot execute System.cmd/3 for program with null byte", fn -> + System.cmd("null\0byte", []) + end + end + + test "cmd/3 raises with non-binary arguments" do + assert_raise ArgumentError, ~r"all arguments for System.cmd/3 must be binaries", fn -> + System.cmd("ls", [~c"/usr"]) + end + end + + describe "Windows" do + @describetag :windows + + test "cmd/2" do + assert {"hello\r\n", 0} = System.cmd("cmd", ~w[/c echo hello]) + end + + test "cmd/3 (with options)" do + opts = [ + into: [], + cd: File.cwd!(), + env: %{"foo" => "bar", "baz" => nil}, + arg0: "echo", + stderr_to_stdout: true, + parallelism: true, + use_stdio: true + ] + + assert {["hello\r\n"], 0} = System.cmd("cmd", ~w[/c echo hello], opts) + end + + @echo "echo-elixir-test" + @tag :tmp_dir + test "cmd/3 with absolute and relative paths", config do + echo = Path.join(config.tmp_dir, @echo) + File.mkdir_p!(Path.dirname(echo)) + File.ln_s!(System.find_executable("cmd"), echo) + + File.cd!(Path.dirname(echo), fn -> + # There is a bug in OTP where find_executable is finding + # entries on the current directory. If this is the case, + # we should avoid the assertion below. + if !System.find_executable(@echo) do + assert :enoent = catch_error(System.cmd(@echo, ~w[/c echo hello])) + end + + assert {"hello\r\n", 0} = + System.cmd(Path.join(File.cwd!(), @echo), ~w[/c echo hello], [{:arg0, "echo"}]) + end) + end + + test "shell/1" do + assert {"hello\r\n", 0} = System.shell("echo hello") + end + + test "shell/2 (with options)" do + opts = [ + into: [], + cd: File.cwd!(), + env: %{"foo" => "bar", "baz" => nil}, + stderr_to_stdout: true, + parallelism: true, + use_stdio: true + ] + + assert {["bar\r\n"], 0} = System.shell("echo %foo%", opts) + end + end + + describe "Unix" do + @describetag :unix + + test "cmd/2" do + assert {"hello\n", 0} = System.cmd("echo", ["hello"]) + end + + test "cmd/3 (with options)" do + opts = [ + into: [], + cd: File.cwd!(), + env: %{"foo" => "bar", "baz" => nil}, + arg0: "echo", + stderr_to_stdout: true, + parallelism: true, + use_stdio: true + ] + + assert {["hello\n"], 0} = System.cmd("echo", ["hello"], opts) + end + + test "cmd/3 (can't use `use_stdio: false, stderr_to_stdout: true`)" do + opts = [ + into: [], + cd: File.cwd!(), + env: %{"foo" => "bar", "baz" => nil}, + arg0: "echo", + stderr_to_stdout: true, + use_stdio: false + ] + + message = ~r"cannot use \"stderr_to_stdout: true\" and \"use_stdio: false\"" + + assert_raise ArgumentError, message, fn -> + System.cmd("echo", ["hello"], opts) + end + end + + test "cmd/3 by line" do + assert {["hello", "world"], 0} = + System.cmd("echo", ["hello\nworld"], into: [], lines: 1024) + + assert {["hello", "world"], 0} = + System.cmd("echo", ["-n", "hello\nworld"], into: [], lines: 3) + end + + @echo "echo-elixir-test" + @tag :tmp_dir + test "cmd/3 with absolute and relative paths", config do + echo = Path.join(config.tmp_dir, @echo) + File.mkdir_p!(Path.dirname(echo)) + File.ln_s!(System.find_executable("echo"), echo) + + File.cd!(Path.dirname(echo), fn -> + # There is a bug in OTP where find_executable is finding + # entries on the current directory. If this is the case, + # we should avoid the assertion below. + if !System.find_executable(@echo) do + assert :enoent = catch_error(System.cmd(@echo, ["hello"])) + end + + assert {"hello\n", 0} = + System.cmd(Path.join(File.cwd!(), @echo), ["hello"], [{:arg0, "echo"}]) + end) + end + + test "shell/1" do + assert {"hello\n", 0} = System.shell("echo hello") + end + + test "shell/1 with interpolation" do + assert {"1\n2\n", 0} = System.shell("x=1; echo $x; echo '2'") + end + + test "shell/1 with empty string" do + assert {"", 0} = System.shell("") + assert {"", 0} = System.shell(" ") + end + + @tag timeout: 1_000 + test "shell/1 returns when command awaits input" do + assert {"", 0} = System.shell("cat", close_stdin: true) + end + + test "shell/1 with comment" do + assert {"1\n", 0} = System.shell("echo '1' # comment") + end + + test "shell/2 (with options)" do + opts = [ + into: [], + cd: File.cwd!(), + env: %{"foo" => "bar", "baz" => nil}, + stderr_to_stdout: true, + use_stdio: true + ] + + assert {["bar\n"], 0} = System.shell("echo $foo", opts) + end end - test "cmd" do - assert is_binary(System.cmd "echo hello") - assert is_list(System.cmd 'echo hello') + @tag :unix + test "vm signals" do + assert System.trap_signal(:sigquit, :example, fn -> :ok end) == {:ok, :example} + assert System.trap_signal(:sigquit, :example, fn -> :ok end) == {:error, :already_registered} + assert {:ok, ref} = System.trap_signal(:sigquit, fn -> :ok end) + + assert System.untrap_signal(:sigquit, :example) == :ok + assert System.trap_signal(:sigquit, :example, fn -> :ok end) == {:ok, :example} + assert System.trap_signal(:sigquit, ref, fn -> :ok end) == {:error, :already_registered} + + assert System.untrap_signal(:sigusr1, :example) == {:error, :not_found} + assert System.untrap_signal(:sigquit, :example) == :ok + assert System.untrap_signal(:sigquit, :example) == {:error, :not_found} + assert System.untrap_signal(:sigquit, ref) == :ok + assert System.untrap_signal(:sigquit, ref) == {:error, :not_found} + end + + @tag :unix + test "os signals" do + parent = self() + + assert System.trap_signal(:sighup, :example, fn -> + send(parent, :sighup_called) + :ok + end) == {:ok, :example} + + {"", 0} = System.cmd("kill", ["-s", "hup", System.pid()]) + + assert_receive :sighup_called + after + System.untrap_signal(:sighup, :example) end - test "find_executable with binary" do + test "find_executable/1" do assert System.find_executable("erl") - assert is_binary System.find_executable("erl") + assert is_binary(System.find_executable("erl")) assert !System.find_executable("does-not-really-exist-from-elixir") + + message = ~r"cannot execute System.find_executable/1 for program with null byte" + + assert_raise ArgumentError, message, fn -> + System.find_executable("null\0byte") + end + end + + test "monotonic_time/0" do + assert is_integer(System.monotonic_time()) + end + + test "monotonic_time/1" do + assert is_integer(System.monotonic_time(:nanosecond)) + assert abs(System.monotonic_time(:microsecond)) < abs(System.monotonic_time(:nanosecond)) + end + + test "system_time/0" do + assert is_integer(System.system_time()) + end + + test "system_time/1" do + assert is_integer(System.system_time(:nanosecond)) + assert abs(System.system_time(:microsecond)) < abs(System.system_time(:nanosecond)) + end + + test "time_offset/0 and time_offset/1" do + assert is_integer(System.time_offset()) + assert is_integer(System.time_offset(:second)) + end + + test "os_time/0" do + assert is_integer(System.os_time()) + end + + test "os_time/1" do + assert is_integer(System.os_time(:nanosecond)) + assert abs(System.os_time(:microsecond)) < abs(System.os_time(:nanosecond)) + end + + test "unique_integer/0 and unique_integer/1" do + assert is_integer(System.unique_integer()) + assert System.unique_integer([:positive]) > 0 + + assert System.unique_integer([:positive, :monotonic]) < + System.unique_integer([:positive, :monotonic]) + end + + test "convert_time_unit/3" do + time = System.monotonic_time(:nanosecond) + assert abs(System.convert_time_unit(time, :nanosecond, :microsecond)) < abs(time) + end + + test "schedulers/0" do + assert System.schedulers() >= 1 + end + + test "schedulers_online/0" do + assert System.schedulers_online() >= 1 end - test "find_executable with list" do - assert System.find_executable('erl') - assert is_list System.find_executable('erl') - assert !System.find_executable('does-not-really-exist-from-elixir') + test "otp_release/0" do + assert is_binary(System.otp_release()) end end diff --git a/lib/elixir/test/elixir/task/supervisor_test.exs b/lib/elixir/test/elixir/task/supervisor_test.exs index 154773b9dbf..a16f1486b3e 100644 --- a/lib/elixir/test/elixir/task/supervisor_test.exs +++ b/lib/elixir/test/elixir/task/supervisor_test.exs @@ -1,65 +1,224 @@ -Code.require_file "../test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("../test_helper.exs", __DIR__) defmodule Task.SupervisorTest do - use ExUnit.Case, async: true + use ExUnit.Case + + @moduletag :capture_log setup do {:ok, pid} = Task.Supervisor.start_link() {:ok, supervisor: pid} end - setup do - :error_logger.tty(false) - on_exit fn -> :error_logger.tty(true) end - :ok - end - def wait_and_send(caller, atom) do - send caller, :ready + send(caller, :ready) receive do: (true -> true) - send caller, atom + send(caller, atom) end - test "async/1", config do - parent = self() - fun = fn -> wait_and_send(parent, :done) end - task = Task.Supervisor.async(config[:supervisor], fun) + def sleep(number) do + Process.sleep(number) + number + end + + test "can be supervised directly", config do + modules = [{Task.Supervisor, name: config.test}] + assert {:ok, _} = Supervisor.start_link(modules, strategy: :one_for_one) + assert Process.whereis(config.test) + end + + test "start with spawn_opt" do + {:ok, pid} = Task.Supervisor.start_link(spawn_opt: [priority: :high]) + assert Process.info(pid, :priority) == {:priority, :high} + end + + test "multiple supervisors can be supervised and identified with simple child spec" do + {:ok, _} = Registry.start_link(keys: :unique, name: TaskSup.Registry) + + children = [ + {Task.Supervisor, strategy: :one_for_one, name: :simple_name}, + {Task.Supervisor, strategy: :one_for_one, name: {:global, :global_name}}, + {Task.Supervisor, + strategy: :one_for_one, name: {:via, Registry, {TaskSup.Registry, "via_name"}}} + ] + + assert {:ok, supsup} = Supervisor.start_link(children, strategy: :one_for_one) + + assert {:ok, no_name_dynsup} = + Supervisor.start_child(supsup, {Task.Supervisor, strategy: :one_for_one}) + + assert Task.Supervisor.children(:simple_name) == [] + assert Task.Supervisor.children({:global, :global_name}) == [] + assert Task.Supervisor.children({:via, Registry, {TaskSup.Registry, "via_name"}}) == [] + assert Task.Supervisor.children(no_name_dynsup) == [] + + assert Supervisor.start_child(supsup, {Task.Supervisor, strategy: :one_for_one}) == + {:error, {:already_started, no_name_dynsup}} + end + + test "counts and returns children", config do + assert Task.Supervisor.children(config[:supervisor]) == [] + + assert Supervisor.count_children(config[:supervisor]) == + %{active: 0, specs: 0, supervisors: 0, workers: 0} + + assert DynamicSupervisor.count_children(config[:supervisor]) == + %{active: 0, specs: 0, supervisors: 0, workers: 0} + end + + describe "async/1" do + test "spawns tasks under the supervisor", config do + parent = self() + fun = fn -> wait_and_send(parent, :done) end + task = Task.Supervisor.async(config[:supervisor], fun) + assert Task.Supervisor.children(config[:supervisor]) == [task.pid] + + # Assert the struct + assert task.__struct__ == Task + assert is_pid(task.pid) + assert is_reference(task.ref) + + # Assert the link + {:links, links} = Process.info(self(), :links) + assert task.pid in links + + receive do: (:ready -> :ok) + + # Assert the initial call + {:name, fun_name} = Function.info(fun, :name) + assert {__MODULE__, fun_name, 0} === :proc_lib.translate_initial_call(task.pid) + + # Run the task + send(task.pid, true) + + # Assert response and monitoring messages + ref = task.ref + assert_receive {^ref, :done} + assert_receive {:DOWN, ^ref, _, _, :normal} + end + + test "with custom shutdown", config do + Process.flag(:trap_exit, true) + parent = self() + + fun = fn -> wait_and_send(parent, :done) end + %{pid: pid} = Task.Supervisor.async(config[:supervisor], fun, shutdown: :brutal_kill) + + Process.exit(config[:supervisor], :shutdown) + assert_receive {:DOWN, _, _, ^pid, :killed} + end + test "raises when :max_children is reached" do + {:ok, sup} = Task.Supervisor.start_link(max_children: 1) + Task.Supervisor.async(sup, fn -> Process.sleep(:infinity) end) + + assert_raise RuntimeError, ~r/reached the maximum number of tasks/, fn -> + Task.Supervisor.async(sup, fn -> :ok end) + end + end + + test "with $callers", config do + sup = config[:supervisor] + grandparent = self() + + Task.Supervisor.async(sup, fn -> + parent = self() + assert Process.get(:"$callers") == [grandparent] + assert Process.get(:"$ancestors") == [sup, grandparent] + + Task.Supervisor.async(sup, fn -> + assert Process.get(:"$callers") == [parent, grandparent] + assert Process.get(:"$ancestors") == [sup, grandparent] + end) + |> Task.await() + end) + |> Task.await() + end + end + + test "async/3", config do + args = [self(), :done] + task = Task.Supervisor.async(config[:supervisor], __MODULE__, :wait_and_send, args) assert Task.Supervisor.children(config[:supervisor]) == [task.pid] - # Assert the struct + receive do: (:ready -> :ok) + assert {__MODULE__, :wait_and_send, 2} === :proc_lib.translate_initial_call(task.pid) + + send(task.pid, true) assert task.__struct__ == Task - assert is_pid task.pid - assert is_reference task.ref + assert task.mfa == {__MODULE__, :wait_and_send, 2} + assert Task.await(task) == :done + end - # Assert the link - {:links, links} = Process.info(self, :links) - assert task.pid in links + describe "async_nolink/1" do + test "spawns a task under the supervisor without linking to the caller", config do + parent = self() + fun = fn -> wait_and_send(parent, :done) end + task = Task.Supervisor.async_nolink(config[:supervisor], fun) + assert Task.Supervisor.children(config[:supervisor]) == [task.pid] - receive do: (:ready -> :ok) + # Assert the struct + assert task.__struct__ == Task + assert is_pid(task.pid) + assert is_reference(task.ref) + assert task.mfa == {:erlang, :apply, 2} + + # Refute the link + {:links, links} = Process.info(self(), :links) + refute task.pid in links + + receive do: (:ready -> :ok) + + # Assert the initial call + {:name, fun_name} = Function.info(fun, :name) + assert {__MODULE__, fun_name, 0} === :proc_lib.translate_initial_call(task.pid) + + # Run the task + send(task.pid, true) - # Assert the initial call - {:name, fun_name} = :erlang.fun_info(fun, :name) - assert {__MODULE__, fun_name, 0} === :proc_lib.translate_initial_call(task.pid) + # Assert response and monitoring messages + ref = task.ref + assert_receive {^ref, :done} + assert_receive {:DOWN, ^ref, _, _, :normal} + end - # Run the task - send task.pid, true + test "with custom shutdown", config do + Process.flag(:trap_exit, true) + parent = self() - # Assert response and monitoring messages - ref = task.ref - assert_receive {^ref, :done} - assert_receive {:DOWN, ^ref, _, _, :normal} + fun = fn -> wait_and_send(parent, :done) end + %{pid: pid} = Task.Supervisor.async_nolink(config[:supervisor], fun, shutdown: :brutal_kill) + + Process.exit(config[:supervisor], :shutdown) + assert_receive {:DOWN, _, _, ^pid, :killed} + end + + test "raises when :max_children is reached" do + {:ok, sup} = Task.Supervisor.start_link(max_children: 1) + + Task.Supervisor.async_nolink(sup, fn -> Process.sleep(:infinity) end) + + assert_raise RuntimeError, ~r/reached the maximum number of tasks/, fn -> + Task.Supervisor.async_nolink(sup, fn -> :ok end) + end + end end - test "async/3", config do - task = Task.Supervisor.async(config[:supervisor], __MODULE__, :wait_and_send, [self(), :done]) + test "async_nolink/3", config do + args = [self(), :done] + task = Task.Supervisor.async_nolink(config[:supervisor], __MODULE__, :wait_and_send, args) assert Task.Supervisor.children(config[:supervisor]) == [task.pid] receive do: (:ready -> :ok) assert {__MODULE__, :wait_and_send, 2} === :proc_lib.translate_initial_call(task.pid) - send task.pid, true + send(task.pid, true) assert task.__struct__ == Task + assert task.mfa == {__MODULE__, :wait_and_send, 2} assert Task.await(task) == :done end @@ -69,57 +228,344 @@ defmodule Task.SupervisorTest do {:ok, pid} = Task.Supervisor.start_child(config[:supervisor], fun) assert Task.Supervisor.children(config[:supervisor]) == [pid] - {:links, links} = Process.info(self, :links) + {:links, links} = Process.info(self(), :links) refute pid in links receive do: (:ready -> :ok) - {:name, fun_name} = :erlang.fun_info(fun, :name) + {:name, fun_name} = Function.info(fun, :name) assert {__MODULE__, fun_name, 0} === :proc_lib.translate_initial_call(pid) - send pid, true + send(pid, true) assert_receive :done end test "start_child/3", config do - {:ok, pid} = Task.Supervisor.start_child(config[:supervisor], __MODULE__, :wait_and_send, [self(), :done]) + args = [self(), :done] + + {:ok, pid} = + Task.Supervisor.start_child(config[:supervisor], __MODULE__, :wait_and_send, args) + assert Task.Supervisor.children(config[:supervisor]) == [pid] - {:links, links} = Process.info(self, :links) + {:links, links} = Process.info(self(), :links) refute pid in links receive do: (:ready -> :ok) assert {__MODULE__, :wait_and_send, 2} === :proc_lib.translate_initial_call(pid) - send pid, true + send(pid, true) + assert_receive :done + + assert_raise FunctionClauseError, fn -> + Task.Supervisor.start_child(config[:supervisor], __MODULE__, :wait_and_send, :illegal_arg) + end + + assert_raise FunctionClauseError, fn -> + args = [self(), :done] + Task.Supervisor.start_child(config[:supervisor], __MODULE__, "wait_and_send", args) + end + end + + test "start_child/1 with custom shutdown", config do + Process.flag(:trap_exit, true) + parent = self() + + fun = fn -> wait_and_send(parent, :done) end + {:ok, pid} = Task.Supervisor.start_child(config[:supervisor], fun, shutdown: :brutal_kill) + + Process.monitor(pid) + Process.exit(config[:supervisor], :shutdown) + assert_receive {:DOWN, _, _, ^pid, :killed} + end + + test "start_child/1 with custom restart", config do + parent = self() + + fun = fn -> wait_and_send(parent, :done) end + {:ok, pid} = Task.Supervisor.start_child(config[:supervisor], fun, restart: :permanent) + + assert_receive :ready + Process.monitor(pid) + Process.exit(pid, :shutdown) + assert_receive {:DOWN, _, _, ^pid, :shutdown} + assert_receive :ready + end + + test "start_child/1 with $callers", config do + sup = config[:supervisor] + grandparent = self() + + Task.Supervisor.start_child(sup, fn -> + parent = self() + assert Process.get(:"$callers") == [grandparent] + assert Process.get(:"$ancestors") == [sup, grandparent] + + Task.Supervisor.start_child(sup, fn -> + assert Process.get(:"$callers") == [parent, grandparent] + assert Process.get(:"$ancestors") == [sup, grandparent] + send(grandparent, :done) + end) + end) + assert_receive :done end test "terminate_child/2", config do - {:ok, pid} = Task.Supervisor.start_child(config[:supervisor], __MODULE__, :wait_and_send, [self(), :done]) + args = [self(), :done] + + {:ok, pid} = + Task.Supervisor.start_child(config[:supervisor], __MODULE__, :wait_and_send, args) + assert Task.Supervisor.children(config[:supervisor]) == [pid] assert Task.Supervisor.terminate_child(config[:supervisor], pid) == :ok assert Task.Supervisor.children(config[:supervisor]) == [] - assert Task.Supervisor.terminate_child(config[:supervisor], pid) == :ok + assert Task.Supervisor.terminate_child(config[:supervisor], pid) == {:error, :not_found} end - test "await/1 exits on task throw", config do - Process.flag(:trap_exit, true) - task = Task.Supervisor.async(config[:supervisor], fn -> throw :unknown end) - assert {{{:nocatch, :unknown}, _}, {Task, :await, [^task, 5000]}} = - catch_exit(Task.await(task)) + describe "await/1" do + test "demonitors and unalias on timeout", config do + task = + Task.Supervisor.async(config[:supervisor], fn -> + assert_receive :go + :done + end) + + assert catch_exit(Task.await(task, 0)) == {:timeout, {Task, :await, [task, 0]}} + new_ref = Process.monitor(task.pid) + old_ref = task.ref + + send(task.pid, :go) + assert_receive {:DOWN, ^new_ref, _, _, _} + refute_received {^old_ref, :done} + refute_received {:DOWN, ^old_ref, _, _, _} + end + + test "exits on task throw", config do + Process.flag(:trap_exit, true) + task = Task.Supervisor.async(config[:supervisor], fn -> throw(:unknown) end) + + assert {{{:nocatch, :unknown}, _}, {Task, :await, [^task, 5000]}} = + catch_exit(Task.await(task)) + end + + test "exits on task error", config do + Process.flag(:trap_exit, true) + task = Task.Supervisor.async(config[:supervisor], fn -> raise "oops" end) + assert {{%RuntimeError{}, _}, {Task, :await, [^task, 5000]}} = catch_exit(Task.await(task)) + end + + test "exits on task exit", config do + Process.flag(:trap_exit, true) + task = Task.Supervisor.async(config[:supervisor], fn -> exit(:unknown) end) + assert {:unknown, {Task, :await, [^task, 5000]}} = catch_exit(Task.await(task)) + end end - test "await/1 exits on task error", config do - Process.flag(:trap_exit, true) - task = Task.Supervisor.async(config[:supervisor], fn -> raise "oops" end) - assert {{%RuntimeError{}, _}, {Task, :await, [^task, 5000]}} = - catch_exit(Task.await(task)) + describe "async_stream" do + @opts [] + test "streams an enumerable with fun", %{supervisor: supervisor} do + assert supervisor + |> Task.Supervisor.async_stream(1..4, &sleep/1, @opts) + |> Enum.to_list() == [ok: 1, ok: 2, ok: 3, ok: 4] + end + + test "streams an enumerable with mfa", %{supervisor: supervisor} do + assert supervisor + |> Task.Supervisor.async_stream(1..4, __MODULE__, :sleep, [], @opts) + |> Enum.to_list() == [ok: 1, ok: 2, ok: 3, ok: 4] + end + + test "streams an enumerable without leaking tasks", %{supervisor: supervisor} do + assert supervisor + |> Task.Supervisor.async_stream(1..4, &sleep/1, @opts) + |> Enum.to_list() == [ok: 1, ok: 2, ok: 3, ok: 4] + + refute_received _ + end + + test "streams an enumerable with slowest first", %{supervisor: supervisor} do + Process.flag(:trap_exit, true) + + assert supervisor + |> Task.Supervisor.async_stream(4..1//-1, &sleep/1, @opts) + |> Enum.to_list() == [ok: 4, ok: 3, ok: 2, ok: 1] + end + + test "streams an enumerable with exits", %{supervisor: supervisor} do + Process.flag(:trap_exit, true) + + assert supervisor + |> Task.Supervisor.async_stream(1..4, &yield_and_exit(Integer.to_string(&1)), @opts) + |> Enum.to_list() == [exit: "1", exit: "2", exit: "3", exit: "4"] + end + + test "shuts down unused tasks", %{supervisor: supervisor} do + collection = [0, :infinity, :infinity, :infinity] + + assert supervisor + |> Task.Supervisor.async_stream(collection, &sleep/1, @opts) + |> Enum.take(1) == [ok: 0] + + assert Process.info(self(), :links) == {:links, [supervisor]} + end + + test "shuts down unused tasks without leaking messages", %{supervisor: supervisor} do + collection = [0, :infinity, :infinity, :infinity] + + assert supervisor + |> Task.Supervisor.async_stream(collection, &sleep/1, @opts) + |> Enum.take(1) == [ok: 0] + + refute_received _ + end + + test "raises an error if :max_children is reached with clean stream shutdown", + %{supervisor: unused_supervisor} do + {:ok, supervisor} = Task.Supervisor.start_link(max_children: 1) + collection = [:infinity, :infinity, :infinity] + + assert_raise RuntimeError, ~r/reached the maximum number of tasks/, fn -> + supervisor + |> Task.Supervisor.async_stream(collection, &sleep/1, max_concurrency: 2) + |> Enum.to_list() + end + + {:links, links} = Process.info(self(), :links) + assert MapSet.new(links) == MapSet.new([unused_supervisor, supervisor]) + refute_received _ + end + + test "with $callers", config do + sup = config[:supervisor] + grandparent = self() + + Task.Supervisor.async_stream(sup, [1], fn 1 -> + parent = self() + assert Process.get(:"$callers") == [grandparent] + assert Process.get(:"$ancestors") == [sup, grandparent] + + Task.Supervisor.async_stream(sup, [1], fn 1 -> + assert Process.get(:"$callers") == [parent, grandparent] + assert Process.get(:"$ancestors") == [sup, grandparent] + send(grandparent, :done) + end) + |> Stream.run() + end) + |> Stream.run() + + assert_receive :done + end + + test "consuming from another process", config do + parent = self() + stream = Task.Supervisor.async_stream(config[:supervisor], [1, 2, 3], &send(parent, &1)) + Task.start(Stream, :run, [stream]) + assert_receive 1 + assert_receive 2 + assert_receive 3 + end + + test "with timeout and :zip_input_on_exit set to true", %{supervisor: supervisor} do + opts = Keyword.merge(@opts, zip_input_on_exit: true, on_timeout: :kill_task, timeout: 50) + + assert supervisor + |> Task.Supervisor.async_stream([1, 100], &sleep/1, opts) + |> Enum.to_list() == [ok: 1, exit: {100, :timeout}] + end + + test "with outer halt on failure and :zip_input_on_exit", %{supervisor: supervisor} do + Process.flag(:trap_exit, true) + opts = Keyword.merge(@opts, zip_input_on_exit: true) + + assert supervisor + |> Task.Supervisor.async_stream(1..8, &exit/1, opts) + |> Enum.take(4) == [exit: {1, 1}, exit: {2, 2}, exit: {3, 3}, exit: {4, 4}] + end + + test "does not allow streaming with invalid :shutdown", %{supervisor: supervisor} do + message = ":shutdown must be either a positive integer or :brutal_kill" + + assert_raise ArgumentError, message, fn -> + Task.Supervisor.async_stream(supervisor, [], fn _ -> :ok end, shutdown: :unknown) + end + end end - test "await/1 exits on task exit", config do - Process.flag(:trap_exit, true) - task = Task.Supervisor.async(config[:supervisor], fn -> exit :unknown end) - assert {:unknown, {Task, :await, [^task, 5000]}} = - catch_exit(Task.await(task)) + describe "async_stream_nolink" do + @opts [max_concurrency: 4] + + test "streams an enumerable with fun", %{supervisor: supervisor} do + assert supervisor + |> Task.Supervisor.async_stream_nolink(1..4, &sleep/1, @opts) + |> Enum.to_list() == [ok: 1, ok: 2, ok: 3, ok: 4] + end + + test "streams an enumerable with mfa", %{supervisor: supervisor} do + assert supervisor + |> Task.Supervisor.async_stream_nolink(1..4, __MODULE__, :sleep, [], @opts) + |> Enum.to_list() == [ok: 1, ok: 2, ok: 3, ok: 4] + end + + test "streams an enumerable without leaking tasks", %{supervisor: supervisor} do + assert supervisor + |> Task.Supervisor.async_stream_nolink(1..4, &sleep/1, @opts) + |> Enum.to_list() == [ok: 1, ok: 2, ok: 3, ok: 4] + + refute_received _ + end + + test "streams an enumerable with slowest first", %{supervisor: supervisor} do + assert supervisor + |> Task.Supervisor.async_stream_nolink(4..1//-1, &sleep/1, @opts) + |> Enum.to_list() == [ok: 4, ok: 3, ok: 2, ok: 1] + end + + test "streams an enumerable with exits", %{supervisor: supervisor} do + assert supervisor + |> Task.Supervisor.async_stream_nolink(1..4, &yield_and_exit/1, @opts) + |> Enum.to_list() == [exit: 1, exit: 2, exit: 3, exit: 4] + end + + test "shuts down unused tasks", %{supervisor: supervisor} do + collection = [0, :infinity, :infinity, :infinity] + + assert supervisor + |> Task.Supervisor.async_stream_nolink(collection, &sleep/1, @opts) + |> Enum.take(1) == [ok: 0] + + assert Process.info(self(), :links) == {:links, [supervisor]} + end + + test "shuts down unused tasks without leaking messages", %{supervisor: supervisor} do + collection = [0, :infinity, :infinity, :infinity] + + assert supervisor + |> Task.Supervisor.async_stream_nolink(collection, &sleep/1, @opts) + |> Enum.take(1) == [ok: 0] + + refute_received _ + end + + test "raises an error if :max_children is reached with clean stream shutdown", + %{supervisor: unused_supervisor} do + {:ok, supervisor} = Task.Supervisor.start_link(max_children: 1) + collection = [:infinity, :infinity, :infinity] + + assert_raise RuntimeError, ~r/reached the maximum number of tasks/, fn -> + supervisor + |> Task.Supervisor.async_stream_nolink(collection, &sleep/1, max_concurrency: 2) + |> Enum.to_list() + end + + {:links, links} = Process.info(self(), :links) + assert MapSet.new(links) == MapSet.new([unused_supervisor, supervisor]) + refute_received _ + end + end + + def yield_and_exit(value) do + # We call yield first so we give the parent a chance to monitor + :erlang.yield() + :erlang.exit(value) end end diff --git a/lib/elixir/test/elixir/task_test.exs b/lib/elixir/test/elixir/task_test.exs index a75d80228b7..1b5427543dc 100644 --- a/lib/elixir/test/elixir/task_test.exs +++ b/lib/elixir/test/elixir/task_test.exs @@ -1,18 +1,70 @@ -Code.require_file "test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec -defmodule TaskTest do - use ExUnit.Case, async: true +Code.require_file("test_helper.exs", __DIR__) - setup do - :error_logger.tty(false) - on_exit fn -> :error_logger.tty(true) end - :ok - end +defmodule TaskTest do + use ExUnit.Case + doctest Task + @moduletag :capture_log def wait_and_send(caller, atom) do - send caller, :ready + send(caller, :ready) receive do: (true -> true) - send caller, atom + send(caller, atom) + end + + defp create_task_in_other_process do + caller = self() + spawn(fn -> send(caller, Task.async(fn -> nil end)) end) + receive do: (task -> task) + end + + defp create_dummy_task(reason) do + {pid, ref} = spawn_monitor(Kernel, :exit, [reason]) + + receive do + {:DOWN, ^ref, _, _, _} -> + %Task{ref: ref, pid: pid, owner: self(), mfa: {__MODULE__, :create_dummy_task, 1}} + end + end + + def sleep(number) do + Process.sleep(number) + number + end + + def wait_until_down(task) do + ref = Process.monitor(task.pid) + assert_receive {:DOWN, ^ref, _, _, _} + end + + test "can be supervised directly" do + assert {:ok, _} = Supervisor.start_link([{Task, fn -> :ok end}], strategy: :one_for_one) + end + + test "generates child_spec/1" do + defmodule MyTask do + use Task + end + + assert MyTask.child_spec([:hello]) == %{ + id: MyTask, + restart: :temporary, + start: {MyTask, :start_link, [[:hello]]} + } + + defmodule CustomTask do + use Task, id: :id, restart: :permanent, shutdown: :infinity, start: {:foo, :bar, []} + end + + assert CustomTask.child_spec([:hello]) == %{ + id: :id, + restart: :permanent, + shutdown: :infinity, + start: {:foo, :bar, []} + } end test "async/1" do @@ -22,21 +74,22 @@ defmodule TaskTest do # Assert the struct assert task.__struct__ == Task - assert is_pid task.pid - assert is_reference task.ref + assert is_pid(task.pid) + assert is_reference(task.ref) + assert task.mfa == {:erlang, :apply, 2} # Assert the link - {:links, links} = Process.info(self, :links) + {:links, links} = Process.info(self(), :links) assert task.pid in links receive do: (:ready -> :ok) # Assert the initial call - {:name, fun_name} = :erlang.fun_info(fun, :name) + {:name, fun_name} = Function.info(fun, :name) assert {__MODULE__, fun_name, 0} === :proc_lib.translate_initial_call(task.pid) # Run the task - send task.pid, true + send(task.pid, true) # Assert response and monitoring messages ref = task.ref @@ -47,98 +100,1081 @@ defmodule TaskTest do test "async/3" do task = Task.async(__MODULE__, :wait_and_send, [self(), :done]) assert task.__struct__ == Task + assert task.mfa == {__MODULE__, :wait_and_send, 2} - {:links, links} = Process.info(self, :links) + {:links, links} = Process.info(self(), :links) assert task.pid in links receive do: (:ready -> :ok) - assert {__MODULE__, :wait_and_send, 2} === :proc_lib.translate_initial_call(task.pid) send(task.pid, true) - assert Task.await(task) === :done assert_receive :done end + test "async with $callers" do + grandparent = self() + + Task.async(fn -> + parent = self() + assert Process.get(:"$callers") == [grandparent] + + Task.async(fn -> + assert Process.get(:"$callers") == [parent, grandparent] + end) + |> Task.await() + end) + |> Task.await() + end + + test "start/1" do + parent = self() + fun = fn -> wait_and_send(parent, :done) end + {:ok, pid} = Task.start(fun) + + {:links, links} = Process.info(self(), :links) + refute pid in links + + receive do: (:ready -> :ok) + + {:name, fun_name} = Function.info(fun, :name) + assert {__MODULE__, fun_name, 0} === :proc_lib.translate_initial_call(pid) + + send(pid, true) + assert_receive :done + end + + test "start/3" do + {:ok, pid} = Task.start(__MODULE__, :wait_and_send, [self(), :done]) + + {:links, links} = Process.info(self(), :links) + refute pid in links + + receive do: (:ready -> :ok) + + assert {__MODULE__, :wait_and_send, 2} === :proc_lib.translate_initial_call(pid) + + send(pid, true) + assert_receive :done + end + + test "completed/1" do + task = Task.completed(:done) + assert task.__struct__ == Task + + refute task.pid + + assert Task.await(task) == :done + end + test "start_link/1" do parent = self() fun = fn -> wait_and_send(parent, :done) end {:ok, pid} = Task.start_link(fun) - {:links, links} = Process.info(self, :links) + {:links, links} = Process.info(self(), :links) assert pid in links receive do: (:ready -> :ok) - {:name, fun_name} = :erlang.fun_info(fun, :name) + {:name, fun_name} = Function.info(fun, :name) assert {__MODULE__, fun_name, 0} === :proc_lib.translate_initial_call(pid) - send pid, true + send(pid, true) assert_receive :done end test "start_link/3" do {:ok, pid} = Task.start_link(__MODULE__, :wait_and_send, [self(), :done]) - {:links, links} = Process.info(self, :links) + {:links, links} = Process.info(self(), :links) assert pid in links receive do: (:ready -> :ok) assert {__MODULE__, :wait_and_send, 2} === :proc_lib.translate_initial_call(pid) - send pid, true + send(pid, true) assert_receive :done end - test "await/1 exits on timeout" do - task = %Task{ref: make_ref()} - assert catch_exit(Task.await(task, 0)) == {:timeout, {Task, :await, [task, 0]}} + test "start_link with $callers" do + grandparent = self() + + Task.start_link(fn -> + parent = self() + assert Process.get(:"$callers") == [grandparent] + + Task.start_link(fn -> + assert Process.get(:"$callers") == [parent, grandparent] + send(grandparent, :done) + end) + end) + + assert_receive :done + end + + describe "ignore/1" do + test "discards on time replies" do + task = Task.async(fn -> :ok end) + wait_until_down(task) + assert Task.ignore(task) == {:ok, :ok} + assert catch_exit(Task.await(task, 0)) == {:timeout, {Task, :await, [task, 0]}} + end + + test "discards late replies" do + task = Task.async(fn -> assert_receive(:go) && :ok end) + assert Task.ignore(task) == nil + send(task.pid, :go) + wait_until_down(task) + assert catch_exit(Task.await(task, 0)) == {:timeout, {Task, :await, [task, 0]}} + end + + test "discards on-time failures" do + Process.flag(:trap_exit, true) + task = Task.async(fn -> exit(:oops) end) + wait_until_down(task) + assert Task.ignore(task) == {:exit, :oops} + assert catch_exit(Task.await(task, 0)) == {:timeout, {Task, :await, [task, 0]}} + end + + test "discards late failures" do + task = Task.async(fn -> assert_receive(:go) && exit(:oops) end) + assert Task.ignore(task) == nil + send(task.pid, :go) + wait_until_down(task) + assert catch_exit(Task.await(task, 0)) == {:timeout, {Task, :await, [task, 0]}} + end + + test "exits on :noconnection" do + ref = make_ref() + task = %Task{ref: ref, pid: self(), owner: self(), mfa: {__MODULE__, :test, 1}} + send(self(), {:DOWN, ref, self(), self(), :noconnection}) + assert catch_exit(Task.ignore(task)) |> elem(0) == {:nodedown, node()} + end + + test "can ignore completed tasks" do + assert Task.ignore(Task.completed(:done)) == {:ok, :done} + end end - test "await/1 exits on normal exit" do - task = Task.async(fn -> exit :normal end) - assert catch_exit(Task.await(task)) == {:normal, {Task, :await, [task, 5000]}} + describe "await/2" do + test "demonitors and unalias on timeout" do + task = + Task.async(fn -> + assert_receive :go + :done + end) + + assert catch_exit(Task.await(task, 0)) == {:timeout, {Task, :await, [task, 0]}} + send(task.pid, :go) + ref = task.ref + + wait_until_down(task) + refute_received {^ref, :done} + refute_received {:DOWN, ^ref, _, _, _} + end + + test "exits on timeout" do + task = %Task{ref: make_ref(), owner: self(), pid: nil, mfa: {__MODULE__, :test, 1}} + assert catch_exit(Task.await(task, 0)) == {:timeout, {Task, :await, [task, 0]}} + end + + test "exits on normal exit" do + task = Task.async(fn -> exit(:normal) end) + assert catch_exit(Task.await(task)) == {:normal, {Task, :await, [task, 5000]}} + end + + test "exits on task throw" do + Process.flag(:trap_exit, true) + task = Task.async(fn -> throw(:unknown) end) + + assert {{{:nocatch, :unknown}, _}, {Task, :await, [^task, 5000]}} = + catch_exit(Task.await(task)) + end + + test "exits on task error" do + Process.flag(:trap_exit, true) + task = Task.async(fn -> raise "oops" end) + assert {{%RuntimeError{}, _}, {Task, :await, [^task, 5000]}} = catch_exit(Task.await(task)) + end + + @compile {:no_warn_undefined, :module_does_not_exist} + + test "exits on task undef module error" do + Process.flag(:trap_exit, true) + task = Task.async(&:module_does_not_exist.undef/0) + + assert {exit_status, mfa} = catch_exit(Task.await(task)) + assert {:undef, [{:module_does_not_exist, :undef, _, _} | _]} = exit_status + assert {Task, :await, [^task, 5000]} = mfa + end + + @compile {:no_warn_undefined, {TaskTest, :undef, 0}} + + test "exits on task undef function error" do + Process.flag(:trap_exit, true) + task = Task.async(&TaskTest.undef/0) + + assert {{:undef, [{TaskTest, :undef, _, _} | _]}, {Task, :await, [^task, 5000]}} = + catch_exit(Task.await(task)) + end + + test "exits on task exit" do + Process.flag(:trap_exit, true) + task = Task.async(fn -> exit(:unknown) end) + assert {:unknown, {Task, :await, [^task, 5000]}} = catch_exit(Task.await(task)) + end + + test "exits on :noconnection" do + ref = make_ref() + task = %Task{ref: ref, pid: self(), owner: self(), mfa: {__MODULE__, :test, 1}} + send(self(), {:DOWN, ref, :process, self(), :noconnection}) + assert catch_exit(Task.await(task)) |> elem(0) == {:nodedown, node()} + end + + test "exits on :noconnection from named monitor" do + ref = make_ref() + task = %Task{ref: ref, owner: self(), pid: nil, mfa: {__MODULE__, :test, 1}} + send(self(), {:DOWN, ref, :process, {:name, :node}, :noconnection}) + assert catch_exit(Task.await(task)) |> elem(0) == {:nodedown, :node} + end + + test "raises when invoked from a non-owner process" do + task = create_task_in_other_process() + + message = + "task #{inspect(task)} must be queried from the owner " <> + "but was queried from #{inspect(self())}" + + assert_raise ArgumentError, message, fn -> Task.await(task, 1) end + end + end + + describe "await_many/2" do + test "returns list of replies" do + tasks = for val <- [1, 3, 9], do: Task.async(fn -> val end) + assert Task.await_many(tasks) == [1, 3, 9] + end + + test "returns replies in input order ignoring response order" do + refs = [ref_1 = make_ref(), ref_2 = make_ref(), ref_3 = make_ref()] + + tasks = + Enum.map(refs, fn ref -> + %Task{ref: ref, owner: self(), pid: nil, mfa: {__MODULE__, :test, 1}} + end) + + send(self(), {ref_2, 3}) + send(self(), {ref_3, 9}) + send(self(), {ref_1, 1}) + assert Task.await_many(tasks) == [1, 3, 9] + end + + test "returns an empty list immediately" do + assert Task.await_many([]) == [] + end + + test "ignores messages from other processes" do + other_ref = make_ref() + tasks = for val <- [:a, :b], do: Task.async(fn -> val end) + send(self(), other_ref) + send(self(), {other_ref, :z}) + send(self(), {:DOWN, other_ref, :process, 1, :goodbye}) + assert Task.await_many(tasks) == [:a, :b] + assert_received ^other_ref + assert_received {^other_ref, :z} + assert_received {:DOWN, ^other_ref, :process, 1, :goodbye} + end + + test "ignores additional messages after reply" do + refs = [ref_1 = make_ref(), ref_2 = make_ref()] + + tasks = + Enum.map(refs, fn ref -> + %Task{ref: ref, owner: self(), pid: nil, mfa: {__MODULE__, :test, 1}} + end) + + send(self(), {ref_2, :b}) + send(self(), {ref_2, :other}) + send(self(), {ref_1, :a}) + assert Task.await_many(tasks) == [:a, :b] + assert_received {^ref_2, :other} + end + + test "exits on timeout" do + tasks = [Task.async(fn -> Process.sleep(:infinity) end)] + assert catch_exit(Task.await_many(tasks, 0)) == {:timeout, {Task, :await_many, [tasks, 0]}} + end + + test "exits with same reason when task exits" do + tasks = [Task.async(fn -> exit(:normal) end)] + assert catch_exit(Task.await_many(tasks)) == {:normal, {Task, :await_many, [tasks, 5000]}} + end + + test "exits immediately when any task exits" do + tasks = [ + Task.async(fn -> Process.sleep(:infinity) end), + Task.async(fn -> exit(:normal) end) + ] + + assert catch_exit(Task.await_many(tasks)) == {:normal, {Task, :await_many, [tasks, 5000]}} + end + + test "exits immediately when any task crashes" do + Process.flag(:trap_exit, true) + + tasks = [ + Task.async(fn -> Process.sleep(:infinity) end), + Task.async(fn -> exit(:unknown) end) + ] + + assert catch_exit(Task.await_many(tasks)) == {:unknown, {Task, :await_many, [tasks, 5000]}} + + # Make sure all monitors are cleared up afterwards too + Enum.each(tasks, &Process.exit(&1.pid, :kill)) + refute_received {:DOWN, _, _, _, _} + end + + test "exits immediately when any task throws" do + Process.flag(:trap_exit, true) + + tasks = [ + Task.async(fn -> Process.sleep(:infinity) end), + Task.async(fn -> throw(:unknown) end) + ] + + assert {{{:nocatch, :unknown}, _}, {Task, :await_many, [^tasks, 5000]}} = + catch_exit(Task.await_many(tasks)) + end + + test "exits immediately on any task error" do + Process.flag(:trap_exit, true) + + tasks = [ + Task.async(fn -> Process.sleep(:infinity) end), + Task.async(fn -> raise "oops" end) + ] + + assert {{%RuntimeError{}, _}, {Task, :await_many, [^tasks, 5000]}} = + catch_exit(Task.await_many(tasks)) + end + + test "exits immediately on :noconnection" do + tasks = [ + Task.async(fn -> Process.sleep(:infinity) end), + %Task{ref: ref = make_ref(), owner: self(), pid: self(), mfa: {__MODULE__, :test, 1}} + ] + + send(self(), {:DOWN, ref, :process, self(), :noconnection}) + assert catch_exit(Task.await_many(tasks)) |> elem(0) == {:nodedown, node()} + end + + test "exits immediately on :noconnection from named monitor" do + tasks = [ + Task.async(fn -> Process.sleep(:infinity) end), + %Task{ref: ref = make_ref(), owner: self(), pid: nil, mfa: {__MODULE__, :test, 1}} + ] + + send(self(), {:DOWN, ref, :process, {:name, :node}, :noconnection}) + assert catch_exit(Task.await_many(tasks)) |> elem(0) == {:nodedown, :node} + end + + test "raises when invoked from a non-owner process" do + tasks = [ + Task.async(fn -> Process.sleep(:infinity) end), + bad_task = create_task_in_other_process() + ] + + message = + "task #{inspect(bad_task)} must be queried from the owner " <> + "but was queried from #{inspect(self())}" + + assert_raise ArgumentError, message, fn -> Task.await_many(tasks, 1) end + end + end + + describe "yield/2" do + test "returns {:ok, result} when reply and :DOWN in message queue" do + task = %Task{ref: make_ref(), owner: self(), pid: nil, mfa: {__MODULE__, :test, 1}} + send(self(), {task.ref, :result}) + send(self(), {:DOWN, task.ref, :process, self(), :abnormal}) + assert Task.yield(task, 0) == {:ok, :result} + refute_received {:DOWN, _, _, _, _} + end + + test "returns nil on timeout" do + task = %Task{ref: make_ref(), pid: nil, owner: self(), mfa: {__MODULE__, :test, 1}} + assert Task.yield(task, 0) == nil + end + + test "return exit on normal exit" do + task = Task.async(fn -> exit(:normal) end) + assert Task.yield(task) == {:exit, :normal} + end + + test "exits on :noconnection" do + ref = make_ref() + task = %Task{ref: ref, pid: self(), owner: self(), mfa: {__MODULE__, :test, 1}} + send(self(), {:DOWN, ref, self(), self(), :noconnection}) + assert catch_exit(Task.yield(task)) |> elem(0) == {:nodedown, node()} + end + + test "raises when invoked from a non-owner process" do + task = create_task_in_other_process() + + message = + "task #{inspect(task)} must be queried from the owner " <> + "but was queried from #{inspect(self())}" + + assert_raise ArgumentError, message, fn -> Task.yield(task, 1) end + end + end + + describe "yield_many/2" do + test "returns {:ok, result} when reply and :DOWN in message queue" do + task = %Task{ref: make_ref(), owner: self(), pid: nil, mfa: {__MODULE__, :test, 1}} + send(self(), {task.ref, :result}) + send(self(), {:DOWN, task.ref, :process, self(), :abnormal}) + assert Task.yield_many([task], 0) == [{task, {:ok, :result}}] + refute_received {:DOWN, _, _, _, _} + end + + test "returns nil on timeout by default" do + task = %Task{ref: make_ref(), owner: self(), pid: nil, mfa: {__MODULE__, :test, 1}} + assert Task.yield_many([task], 0) == [{task, nil}] + end + + test "shuts down on timeout when configured" do + Process.flag(:trap_exit, true) + task = Task.async(fn -> Process.sleep(:infinity) end) + assert Task.yield_many([task], timeout: 0, on_timeout: :kill_task) == [{task, nil}] + refute Process.alive?(task.pid) + end + + test "ignores on timeout when configured" do + task = + Task.async(fn -> + receive do + :done -> :ok + end + end) + + assert Task.yield_many([task], timeout: 0, on_timeout: :ignore) == [{task, nil}] + assert Process.alive?(task.pid) + + ref = Process.monitor(task.pid) + send(task.pid, :done) + assert_receive {:DOWN, ^ref, _, _, _} + + assert Task.yield(task, 0) == nil + end + + test "return exit on normal exit" do + task = Task.async(fn -> exit(:normal) end) + assert Task.yield_many([task]) == [{task, {:exit, :normal}}] + end + + test "exits on :noconnection" do + ref = make_ref() + task = %Task{ref: ref, pid: self(), owner: self(), mfa: {__MODULE__, :test, 1}} + send(self(), {:DOWN, ref, :process, self(), :noconnection}) + assert catch_exit(Task.yield_many([task])) |> elem(0) == {:nodedown, node()} + end + + test "raises when invoked from a non-owner process" do + task = create_task_in_other_process() + + message = + "task #{inspect(task)} must be queried from the owner " <> + "but was queried from #{inspect(self())}" + + assert_raise ArgumentError, message, fn -> Task.yield_many([task], 1) end + end + + test "returns results from multiple tasks" do + task1 = %Task{ref: make_ref(), owner: self(), pid: nil, mfa: {__MODULE__, :test, 1}} + task2 = %Task{ref: make_ref(), owner: self(), pid: nil, mfa: {__MODULE__, :test, 1}} + task3 = %Task{ref: make_ref(), owner: self(), pid: nil, mfa: {__MODULE__, :test, 1}} + + send(self(), {task1.ref, :result}) + send(self(), {:DOWN, task3.ref, :process, self(), :normal}) + + assert Task.yield_many([task1, task2, task3], 0) == + [{task1, {:ok, :result}}, {task2, nil}, {task3, {:exit, :normal}}] + end + + test "returns results from multiple tasks with limit" do + task1 = %Task{ref: make_ref(), owner: self(), pid: nil, mfa: {__MODULE__, :test, 1}} + task2 = %Task{ref: make_ref(), owner: self(), pid: nil, mfa: {__MODULE__, :test, 1}} + task3 = %Task{ref: make_ref(), owner: self(), pid: nil, mfa: {__MODULE__, :test, 1}} + + send(self(), {task1.ref, :result}) + send(self(), {:DOWN, task3.ref, :process, self(), :normal}) + + assert Task.yield_many([task1, task2, task3], limit: 1, timeout: :infinity) == + [{task1, {:ok, :result}}, {task2, nil}, {task3, nil}] + + assert Task.yield_many([task2, task3], limit: 1, timeout: :infinity) == + [{task2, nil}, {task3, {:exit, :normal}}] + end + + test "returns results from multiple tasks with limit and on timeout" do + Process.flag(:trap_exit, true) + task1 = Task.async(fn -> Process.sleep(:infinity) end) + task2 = Task.async(fn -> :done end) + + assert Task.yield_many([task1, task2], timeout: :infinity, on_timeout: :kill_task, limit: 1) == + [{task1, nil}, {task2, {:ok, :done}}] + + assert Process.alive?(task1.pid) + + assert Task.yield_many([task1], timeout: 0, on_timeout: :kill_task, limit: 1) == + [{task1, nil}] + + refute Process.alive?(task1.pid) + end + + test "returns results on infinity timeout" do + task1 = %Task{ref: make_ref(), owner: self(), pid: nil, mfa: {__MODULE__, :test, 1}} + task2 = %Task{ref: make_ref(), owner: self(), pid: nil, mfa: {__MODULE__, :test, 1}} + task3 = %Task{ref: make_ref(), owner: self(), pid: nil, mfa: {__MODULE__, :test, 1}} + + send(self(), {task1.ref, :result}) + send(self(), {task2.ref, :result}) + send(self(), {:DOWN, task3.ref, :process, self(), :normal}) + + assert Task.yield_many([task1, task2, task3], :infinity) == + [{task1, {:ok, :result}}, {task2, {:ok, :result}}, {task3, {:exit, :normal}}] + end end - test "await/1 exits on task throw" do - Process.flag(:trap_exit, true) - task = Task.async(fn -> throw :unknown end) - assert {{{:nocatch, :unknown}, _}, {Task, :await, [^task, 5000]}} = - catch_exit(Task.await(task)) + describe "shutdown/2" do + test "returns {:ok, result} when reply and abnormal :DOWN in message queue" do + task = create_dummy_task(:abnormal) + send(self(), {task.ref, :result}) + send(self(), {:DOWN, task.ref, :process, task.pid, :abnormal}) + assert Task.shutdown(task) == {:ok, :result} + refute_received {:DOWN, _, _, _, _} + end + + test "returns {:ok, result} when reply and normal :DOWN in message queue" do + task = create_dummy_task(:normal) + send(self(), {task.ref, :result}) + send(self(), {:DOWN, task.ref, :process, task.pid, :normal}) + assert Task.shutdown(task) == {:ok, :result} + refute_received {:DOWN, _, _, _, _} + end + + test "returns {:ok, result} when reply and shut down :DOWN in message queue" do + task = create_dummy_task(:shutdown) + send(self(), {task.ref, :result}) + send(self(), {:DOWN, task.ref, :process, task.pid, :shutdown}) + assert Task.shutdown(task) == {:ok, :result} + refute_received {:DOWN, _, _, _, _} + end + + test "returns nil on shutting down task" do + task = Task.async(:timer, :sleep, [:infinity]) + assert Task.shutdown(task) == nil + end + + test "returns exit on abnormal :DOWN in message queue" do + task = create_dummy_task(:abnormal) + send(self(), {:DOWN, task.ref, :process, task.pid, :abnormal}) + assert Task.shutdown(task) == {:exit, :abnormal} + end + + test "returns exit on normal :DOWN in message queue" do + task = create_dummy_task(:normal) + send(self(), {:DOWN, task.ref, :process, task.pid, :normal}) + assert Task.shutdown(task) == {:exit, :normal} + end + + test "returns nil on shutdown :DOWN in message queue" do + task = create_dummy_task(:shutdown) + send(self(), {:DOWN, task.ref, :process, task.pid, :shutdown}) + assert Task.shutdown(task) == nil + end + + test "returns exit on killed :DOWN in message queue" do + task = create_dummy_task(:killed) + send(self(), {:DOWN, task.ref, :process, task.pid, :killed}) + assert Task.shutdown(task) == {:exit, :killed} + end + + test "exits on noconnection :DOWN in message queue" do + task = create_dummy_task(:noconnection) + send(self(), {:DOWN, task.ref, :process, task.pid, :noconnection}) + + assert catch_exit(Task.shutdown(task)) == + {{:nodedown, node()}, {Task, :shutdown, [task, 5000]}} + end + + test "ignores if task PID is nil" do + ref = make_ref() + send(self(), {ref, :done}) + + assert Task.shutdown(%Task{ref: ref, owner: self(), pid: nil, mfa: {__MODULE__, :test, 1}}) == + {:ok, :done} + + ref = make_ref() + send(self(), {:DOWN, ref, :process, self(), :done}) + + assert Task.shutdown(%Task{ref: ref, owner: self(), pid: nil, mfa: {__MODULE__, :test, 1}}) == + {:exit, :done} + end + + test "raises when invoked from a non-owner process" do + task = create_task_in_other_process() + + message = + "task #{inspect(task)} must be queried from the owner " <> + "but was queried from #{inspect(self())}" + + assert_raise ArgumentError, message, fn -> Task.shutdown(task) end + end + + test "returns nil on killing task" do + caller = self() + + task = + Task.async(fn -> + Process.flag(:trap_exit, true) + wait_and_send(caller, :ready) + Process.sleep(:infinity) + end) + + receive do: (:ready -> :ok) + + assert Task.shutdown(task, :brutal_kill) == nil + refute_received {:DOWN, _, _, _, _} + end + + test "returns {:exit, :noproc} if task handled" do + task = create_dummy_task(:noproc) + assert Task.shutdown(task) == {:exit, :noproc} + end end - test "await/1 exits on task error" do - Process.flag(:trap_exit, true) - task = Task.async(fn -> raise "oops" end) - assert {{%RuntimeError{}, _}, {Task, :await, [^task, 5000]}} = - catch_exit(Task.await(task)) + describe "shutdown/2 with :brutal_kill" do + test "returns {:ok, result} when reply and abnormal :DOWN in message queue" do + task = create_dummy_task(:abnormal) + send(self(), {task.ref, :result}) + send(self(), {:DOWN, task.ref, :process, task.pid, :abnormal}) + assert Task.shutdown(task, :brutal_kill) == {:ok, :result} + refute_received {:DOWN, _, _, _, _} + end + + test "returns {:ok, result} when reply and normal :DOWN in message queue" do + task = create_dummy_task(:normal) + send(self(), {task.ref, :result}) + send(self(), {:DOWN, task.ref, :process, task.pid, :normal}) + assert Task.shutdown(task, :brutal_kill) == {:ok, :result} + refute_received {:DOWN, _, _, _, _} + end + + test "returns {:ok, result} when reply and shut down :DOWN in message queue" do + task = create_dummy_task(:shutdown) + send(self(), {task.ref, :result}) + send(self(), {:DOWN, task.ref, :process, task.pid, :shutdown}) + assert Task.shutdown(task, :brutal_kill) == {:ok, :result} + refute_received {:DOWN, _, _, _, _} + end + + test "returns nil on killed :DOWN in message queue" do + task = create_dummy_task(:killed) + send(self(), {:DOWN, task.ref, :process, task.pid, :killed}) + assert Task.shutdown(task, :brutal_kill) == nil + end + + test "returns exit on abnormal :DOWN in message queue" do + task = create_dummy_task(:abnormal) + send(self(), {:DOWN, task.ref, :process, task.pid, :abnormal}) + assert Task.shutdown(task, :brutal_kill) == {:exit, :abnormal} + end + + test "returns exit on normal :DOWN in message queue" do + task = create_dummy_task(:normal) + send(self(), {:DOWN, task.ref, :process, task.pid, :normal}) + assert Task.shutdown(task, :brutal_kill) == {:exit, :normal} + end + + test "returns exit on shutdown :DOWN in message queue" do + task = create_dummy_task(:shutdown) + send(self(), {:DOWN, task.ref, :process, task.pid, :shutdown}) + assert Task.shutdown(task, :brutal_kill) == {:exit, :shutdown} + end + + test "exits on noconnection :DOWN in message queue" do + task = create_dummy_task(:noconnection) + send(self(), {:DOWN, task.ref, :process, task.pid, :noconnection}) + + assert catch_exit(Task.shutdown(task, :brutal_kill)) == + {{:nodedown, node()}, {Task, :shutdown, [task, :brutal_kill]}} + end + + test "returns exit on killing task after shutdown timeout" do + caller = self() + + task = + Task.async(fn -> + Process.flag(:trap_exit, true) + wait_and_send(caller, :ready) + Process.sleep(:infinity) + end) + + receive do: (:ready -> :ok) + assert Task.shutdown(task, 1) == {:exit, :killed} + end + + test "returns {:exit, :noproc} if task handled" do + task = create_dummy_task(:noproc) + assert Task.shutdown(task, :brutal_kill) == {:exit, :noproc} + end end - test "await/1 exits on task exit" do - Process.flag(:trap_exit, true) - task = Task.async(fn -> exit :unknown end) - assert {:unknown, {Task, :await, [^task, 5000]}} = - catch_exit(Task.await(task)) + describe "async_stream/2" do + test "timeout" do + assert catch_exit([:infinity] |> Task.async_stream(&sleep/1, timeout: 0) |> Enum.to_list()) == + {:timeout, {Task.Supervised, :stream, [0]}} + + refute_received _ + end + + test "streams an enumerable with ordered: false" do + opts = [max_concurrency: 1, ordered: false] + + assert 4..1//-1 + |> Task.async_stream(&sleep(&1 * 100), opts) + |> Enum.to_list() == [ok: 400, ok: 300, ok: 200, ok: 100] + + opts = [max_concurrency: 4, ordered: false] + + assert 4..1//-1 + |> Task.async_stream(&sleep(&1 * 100), opts) + |> Enum.to_list() == [ok: 100, ok: 200, ok: 300, ok: 400] + end + + test "streams an enumerable with ordered: false, on_timeout: :kill_task" do + opts = [max_concurrency: 4, ordered: false, on_timeout: :kill_task, timeout: 50] + + assert [100, 1, 100, 1] + |> Task.async_stream(&sleep/1, opts) + |> Enum.to_list() == [ok: 1, ok: 1, exit: :timeout, exit: :timeout] + + refute_received _ + end + + test "streams an enumerable with infinite timeout" do + [ok: :ok] = Task.async_stream([1], fn _ -> :ok end, timeout: :infinity) |> Enum.to_list() + end + + test "does not allow streaming with max_concurrency = 0" do + assert_raise ArgumentError, ":max_concurrency must be an integer greater than zero", fn -> + Task.async_stream([1], fn _ -> :ok end, max_concurrency: 0) + end + end + + test "does not allow streaming with invalid :on_timeout" do + assert_raise ArgumentError, ":on_timeout must be either :exit or :kill_task", fn -> + Task.async_stream([1], fn _ -> :ok end, on_timeout: :unknown) + end + end + + test "does not allow streaming with invalid :timeout" do + assert_raise ArgumentError, ":timeout must be either a positive integer or :infinity", fn -> + Task.async_stream([1], fn _ -> :ok end, timeout: :unknown) + end + end + + test "streams with fake down messages on the inbox" do + parent = self() + + assert Task.async_stream([:ok], fn :ok -> + {:links, links} = Process.info(self(), :links) + + for link <- links do + send(link, {:DOWN, make_ref(), :process, parent, :oops}) + end + + :ok + end) + |> Enum.to_list() == [ok: :ok] + end + + test "with $callers" do + grandparent = self() + + Task.async_stream([1], fn 1 -> + parent = self() + assert Process.get(:"$callers") == [grandparent] + + Task.async_stream([1], fn 1 -> + assert Process.get(:"$callers") == [parent, grandparent] + send(grandparent, :done) + end) + |> Stream.run() + end) + |> Stream.run() + + assert_receive :done + end + + test "consuming from another process" do + parent = self() + stream = Task.async_stream([1, 2, 3], &send(parent, &1)) + Task.start(Stream, :run, [stream]) + assert_receive 1 + assert_receive 2 + assert_receive 3 + end + + test "wrapping a flat_map/concat with a haltable stream" do + result = + Stream.take([:foo, :bar], 1) + |> Stream.concat([1, 2]) + |> Task.async_stream(& &1) + |> Enum.to_list() + + assert result == [ok: :foo, ok: 1, ok: 2] + end end - test "await/1 exits on :noconnection" do - ref = make_ref() - task = %Task{ref: ref, pid: self()} - send self(), {:DOWN, ref, self(), self(), :noconnection} - assert catch_exit(Task.await(task)) |> elem(0) == {:nodedown, :nonode@nohost} + for {desc, concurrency} <- [==: 4, <: 2, >: 8] do + describe "async_stream with max_concurrency #{desc} tasks" do + @opts [max_concurrency: concurrency] + + test "streams an enumerable with fun" do + assert 1..4 + |> Task.async_stream(&sleep/1, @opts) + |> Enum.to_list() == [ok: 1, ok: 2, ok: 3, ok: 4] + end + + test "streams an enumerable with mfa" do + assert 1..4 + |> Task.async_stream(__MODULE__, :sleep, [], @opts) + |> Enum.to_list() == [ok: 1, ok: 2, ok: 3, ok: 4] + end + + test "streams an enumerable without leaking tasks" do + assert 1..4 + |> Task.async_stream(&sleep/1, @opts) + |> Enum.to_list() == [ok: 1, ok: 2, ok: 3, ok: 4] + + refute_received _ + end + + test "streams an enumerable with slowest first" do + Process.flag(:trap_exit, true) + + assert 4..1//-1 + |> Task.async_stream(&sleep/1, @opts) + |> Enum.to_list() == [ok: 4, ok: 3, ok: 2, ok: 1] + end + + test "streams an enumerable with exits" do + Process.flag(:trap_exit, true) + + assert 1..4 + |> Task.async_stream(&exit/1, @opts) + |> Enum.to_list() == [exit: 1, exit: 2, exit: 3, exit: 4] + + refute_received {:EXIT, _, _} + end + + test "shuts down unused tasks" do + assert [0, :infinity, :infinity, :infinity] + |> Task.async_stream(&sleep/1, @opts) + |> Enum.take(1) == [ok: 0] + + assert Process.info(self(), :links) == {:links, []} + end + + test "shuts down unused tasks without leaking messages" do + assert [0, :infinity, :infinity, :infinity] + |> Task.async_stream(&sleep/1, @opts) + |> Enum.take(1) == [ok: 0] + + refute_received _ + end + + test "is zippable on success" do + task = 1..4 |> Task.async_stream(&sleep/1, @opts) |> Stream.map(&elem(&1, 1)) + assert Enum.zip(task, task) == [{1, 1}, {2, 2}, {3, 3}, {4, 4}] + end + + test "is zippable on failure" do + Process.flag(:trap_exit, true) + task = 1..4 |> Task.async_stream(&exit/1, @opts) |> Stream.map(&elem(&1, 1)) + assert Enum.zip(task, task) == [{1, 1}, {2, 2}, {3, 3}, {4, 4}] + end + + test "is zippable with slowest first" do + task = 4..1//-1 |> Task.async_stream(&sleep/1, @opts) |> Stream.map(&elem(&1, 1)) + assert Enum.zip(task, task) == [{4, 4}, {3, 3}, {2, 2}, {1, 1}] + end + + test "with inner halt on success" do + assert 1..8 + |> Stream.take(4) + |> Task.async_stream(&sleep/1, @opts) + |> Enum.to_list() == [ok: 1, ok: 2, ok: 3, ok: 4] + end + + test "with inner halt on failure" do + Process.flag(:trap_exit, true) + + assert 1..8 + |> Stream.take(4) + |> Task.async_stream(&exit/1, @opts) + |> Enum.to_list() == [exit: 1, exit: 2, exit: 3, exit: 4] + end + + test "with inner halt and slowest first" do + assert 8..1//-1 + |> Stream.take(4) + |> Task.async_stream(&sleep/1, @opts) + |> Enum.to_list() == [ok: 8, ok: 7, ok: 6, ok: 5] + end + + test "with outer halt on success" do + assert 1..8 + |> Task.async_stream(&sleep/1, @opts) + |> Enum.take(4) == [ok: 1, ok: 2, ok: 3, ok: 4] + end + + test "with outer halt on failure" do + Process.flag(:trap_exit, true) + + assert 1..8 + |> Task.async_stream(&exit/1, @opts) + |> Enum.take(4) == [exit: 1, exit: 2, exit: 3, exit: 4] + end + + test "with outer halt and slowest first" do + assert 8..1//-1 + |> Task.async_stream(&sleep/1, @opts) + |> Enum.take(4) == [ok: 8, ok: 7, ok: 6, ok: 5] + end + + test "terminates inner effect" do + stream = + 1..4 + |> Task.async_stream(&sleep/1, @opts) + |> Stream.transform(fn -> :ok end, fn x, acc -> {[x], acc} end, fn _ -> + Process.put(:stream_transform, true) + end) + + Process.put(:stream_transform, false) + assert Enum.to_list(stream) == [ok: 1, ok: 2, ok: 3, ok: 4] + assert Process.get(:stream_transform) + end + + test "terminates outer effect" do + stream = + 1..4 + |> Stream.transform(fn -> :ok end, fn x, acc -> {[x], acc} end, fn _ -> + Process.put(:stream_transform, true) + end) + |> Task.async_stream(&sleep/1, @opts) + + Process.put(:stream_transform, false) + assert Enum.to_list(stream) == [ok: 1, ok: 2, ok: 3, ok: 4] + assert Process.get(:stream_transform) + end + + test "with :on_timeout set to :kill_task" do + opts = Keyword.merge(@opts, on_timeout: :kill_task, timeout: 50) + + assert [100, 1, 100, 1] + |> Task.async_stream(&sleep/1, opts) + |> Enum.to_list() == [exit: :timeout, ok: 1, exit: :timeout, ok: 1] + + refute_received _ + end + + test "with timeout and :zip_input_on_exit set to true" do + opts = Keyword.merge(@opts, zip_input_on_exit: true, on_timeout: :kill_task, timeout: 50) + + assert [1, 100] + |> Task.async_stream(&sleep/1, opts) + |> Enum.to_list() == [ok: 1, exit: {100, :timeout}] + end + + test "with outer halt on failure and :zip_input_on_exit" do + Process.flag(:trap_exit, true) + opts = Keyword.merge(@opts, zip_input_on_exit: true) + + assert 1..8 + |> Task.async_stream(&exit/1, opts) + |> Enum.take(4) == [exit: {1, 1}, exit: {2, 2}, exit: {3, 3}, exit: {4, 4}] + end + end end - test "find/2" do - task = %Task{ref: make_ref} - assert Task.find([task], {make_ref, :ok}) == nil - assert Task.find([task], {task.ref, :ok}) == {:ok, task} + describe "default :logger reporter" do + setup do + translator = :logger.get_primary_config().filters[:logger_translator] + assert :ok = :logger.remove_primary_filter(:logger_translator) + on_exit(fn -> :logger.add_primary_filter(:logger_translator, translator) end) + end - assert Task.find([task], {:DOWN, make_ref, :process, self, :kill}) == nil - msg = {:DOWN, task.ref, :process, self, :kill} - assert catch_exit(Task.find([task], msg)) == - {:kill, {Task, :find, [[task], msg]}} + test "logs a terminated task" do + parent = self() + {:ok, pid} = Task.start_link(__MODULE__, :task, [parent]) + + assert ExUnit.CaptureLog.capture_log(fn -> + ref = Process.monitor(pid) + send(pid, :go) + receive do: ({:DOWN, ^ref, _, _, _} -> :ok) + end) =~ ~r""" + \[error\] \*\* Task #{inspect(pid)} terminating + \*\* Started from #{inspect(parent)} + \*\* When function == &TaskTest.task/1 + \*\* arguments == \[#PID<\d+\.\d+\.\d+>\] + \*\* Reason for termination ==\s + \*\* {%RuntimeError{message: "oops"}, + """ + end + + test "logs a terminated task with a process label" do + fun = fn -> + Process.set_label({:any, "term"}) + raise "oops" + end + + parent = self() + {:ok, pid} = Task.start_link(__MODULE__, :task, [parent, fun]) + + assert ExUnit.CaptureLog.capture_log(fn -> + ref = Process.monitor(pid) + send(pid, :go) + receive do: ({:DOWN, ^ref, _, _, _} -> :ok) + end) =~ ~r""" + \[error\] \*\* Task #PID<\d+\.\d+\.\d+> terminating + \*\* Process Label == {:any, "term"} + \*\* Started from #PID<\d+\.\d+\.\d+> + \*\* When function == &TaskTest.task/2 + \*\* arguments == \[#PID<\d+\.\d+\.\d+>, + #Function<.+>\] + \*\* Reason for termination ==\s + \*\* {%RuntimeError{message: "oops"}, + """ + end end + def task(parent, fun \\ fn -> raise "oops" end) do + mon = Process.monitor(parent) + Process.unlink(parent) + + receive do + :go -> + fun.() + + {:DOWN, ^mon, _, _, _} -> + exit(:shutdown) + end + end end diff --git a/lib/elixir/test/elixir/test_helper.exs b/lib/elixir/test/elixir/test_helper.exs index 6edf43dd100..2ce3cd85236 100644 --- a/lib/elixir/test/elixir/test_helper.exs +++ b/lib/elixir/test/elixir/test_helper.exs @@ -1,4 +1,6 @@ -ExUnit.start [trace: "--trace" in System.argv] +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec # Beam files compiled on demand path = Path.expand("../../tmp/beams", __DIR__) @@ -6,7 +8,8 @@ File.rm_rf!(path) File.mkdir_p!(path) Code.prepend_path(path) -Code.compiler_options debug_info: true +Application.put_env(:elixir, :ansi_enabled, true) +Code.compiler_options(debug_info: true, infer_signatures: [:elixir]) defmodule PathHelpers do def fixture_path() do @@ -18,88 +21,131 @@ defmodule PathHelpers do end def fixture_path(extra) do - Path.join(fixture_path, extra) + Path.join(fixture_path(), extra) end def tmp_path(extra) do - Path.join(tmp_path, extra) + Path.join(tmp_path(), extra) end - def elixir(args) do - runcmd(elixir_executable, args) + def elixir(args, executable_extension \\ "") do + run_cmd(elixir_executable(executable_extension), args) end - def elixir_executable do - executable_path("elixir") + def elixir_executable(extension \\ "") do + executable_path("elixir", extension) end - def elixirc(args) do - runcmd(elixirc_executable, args) + def elixirc(args, executable_extension \\ "") do + run_cmd(elixirc_executable(executable_extension), args) end - def elixirc_executable do - executable_path("elixirc") + def elixirc_executable(extension \\ "") do + executable_path("elixirc", extension) + end + + def iex(args, executable_extension \\ "") do + run_cmd(iex_executable(executable_extension), args) + end + + def iex_executable(extension \\ "") do + executable_path("iex", extension) end def write_beam({:module, name, bin, _} = res) do File.mkdir_p!(unquote(path)) beam_path = Path.join(unquote(path), Atom.to_string(name) <> ".beam") File.write!(beam_path, bin) + + :code.purge(name) + :code.delete(name) + res end - defp runcmd(executable,args) do - :os.cmd :binary.bin_to_list("#{executable} #{IO.chardata_to_string(args)}#{redirect_std_err_on_win}") + defp run_cmd(executable, args) do + ~c"#{executable} #{IO.chardata_to_string(args)}#{redirect_std_err_on_win()}" + |> :os.cmd() + |> :unicode.characters_to_binary() end - defp executable_path(name) do - Path.expand("../../../../bin/#{name}#{executable_extension}", __DIR__) + defp executable_path(name, extension) do + Path.expand("../../../../bin/#{name}#{extension}", __DIR__) end - if match? {:win32, _}, :os.type do - def is_win?, do: true + if match?({:win32, _}, :os.type()) do + def windows?, do: true def executable_extension, do: ".bat" def redirect_std_err_on_win, do: " 2>&1" else - def is_win?, do: false + def windows?, do: false def executable_extension, do: "" def redirect_std_err_on_win, do: "" end end -defmodule CompileAssertion do - import ExUnit.Assertions - - def assert_compile_fail(exception, string) do - case format_rescue(string) do - {^exception, _} -> :ok - error -> - raise ExUnit.AssertionError, - left: inspect(elem(error, 0)), - right: inspect(exception), - message: "Expected match" +defmodule CodeFormatterHelpers do + defmacro assert_same(good, opts \\ []) do + quote bind_quoted: [good: good, opts: opts] do + assert IO.iodata_to_binary(Code.format_string!(good, opts)) == String.trim(good) end end - def assert_compile_fail(exception, message, string) do - case format_rescue(string) do - {^exception, ^message} -> :ok - error -> - raise ExUnit.AssertionError, - left: "#{inspect elem(error, 0)}[message: #{inspect elem(error, 1)}]", - right: "#{inspect exception}[message: #{inspect message}]", - message: "Expected match" + defmacro assert_format(bad, good, opts \\ []) do + quote bind_quoted: [bad: bad, good: good, opts: opts] do + result = String.trim(good) + assert IO.iodata_to_binary(Code.format_string!(bad, opts)) == result + assert IO.iodata_to_binary(Code.format_string!(good, opts)) == result end end +end - defp format_rescue(expr) do - result = try do - :elixir.eval(to_char_list(expr), []) - nil - rescue - error -> {error.__struct__, Exception.message(error)} - end +epmd_exclude = if match?({:win32, _}, :os.type()), do: [epmd: true], else: [] +os_exclude = if PathHelpers.windows?(), do: [unix: true], else: [windows: true] + +{line_exclude, line_include} = + if line = System.get_env("LINE"), do: {[:test], [line: line]}, else: {[], []} - result || flunk(message: "Expected expression to fail") +distributed_exclude = + if Code.ensure_loaded?(:peer) and Node.alive?() do + {:ok, _pid, node} = :peer.start(%{name: :secondary}) + true = :erpc.call(node, :code, :set_path, [:code.get_path()]) + {:ok, _} = :erpc.call(node, :application, :ensure_all_started, [:elixir]) + [] + else + [distributed: true] end -end + +source_exclude = + if :deterministic in :compile.env_compiler_options() do + [:requires_source] + else + [] + end + +Code.require_file("../../scripts/cover_record.exs", __DIR__) + +cover_exclude = + if CoverageRecorder.maybe_record("elixir") do + [:require_ast] + else + [] + end + +# OTP 28.1+ +re_import_exclude = + if Code.ensure_loaded?(:re) and function_exported?(:re, :import, 1) do + [] + else + [:re_import] + end + +ExUnit.start( + trace: !!System.get_env("TRACE"), + exclude: + epmd_exclude ++ + os_exclude ++ + line_exclude ++ distributed_exclude ++ source_exclude ++ cover_exclude ++ re_import_exclude, + include: line_include, + assert_receive_timeout: String.to_integer(System.get_env("ELIXIR_ASSERT_TIMEOUT", "300")) +) diff --git a/lib/elixir/test/elixir/tuple_test.exs b/lib/elixir/test/elixir/tuple_test.exs index 54af5e603b6..dea4d965f6f 100644 --- a/lib/elixir/test/elixir/tuple_test.exs +++ b/lib/elixir/test/elixir/tuple_test.exs @@ -1,35 +1,43 @@ -Code.require_file "test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("test_helper.exs", __DIR__) defmodule TupleTest do use ExUnit.Case, async: true - test :elem do + doctest Tuple + + # Tuple-related functions in the Kernel module. + + test "Kernel.elem/2" do assert elem({:a, :b, :c}, 1) == :b end - test :put_elem do + test "Kernel.put_elem/3" do assert put_elem({:a, :b, :c}, 1, :d) == {:a, :d, :c} end - test :keywords do + test "keyword syntax is supported in tuple literals" do assert {1, 2, three: :four} == {1, 2, [three: :four]} end - test :optional_comma do - assert {1} == {1,} - assert {1, 2, 3} == {1, 2, 3,} + test "optional comma is supported in tuple literals" do + assert Code.eval_string("{1,}") == {{1}, []} + assert Code.eval_string("{1, 2, 3,}") == {{1, 2, 3}, []} end - test :partial_application do + test "partial application" do assert (&{&1, 2}).(1) == {1, 2} assert (&{&1, &2}).(1, 2) == {1, 2} assert (&{&2, &1}).(2, 1) == {1, 2} end # Tuple module - # We check two variants due to inlining. + # We check two variants of each function due to inlining. - test :duplicate do + test "duplicate/2" do assert Tuple.duplicate(:foo, 0) == {} assert Tuple.duplicate(:foo, 3) == {:foo, :foo, :foo} @@ -38,17 +46,17 @@ defmodule TupleTest do assert mod.duplicate(:foo, 3) == {:foo, :foo, :foo} end - test :insert_at do + test "insert_at/3" do assert Tuple.insert_at({:bar, :baz}, 0, :foo) == {:foo, :bar, :baz} mod = Tuple assert mod.insert_at({:bar, :baz}, 0, :foo) == {:foo, :bar, :baz} end - test :delete_at do + test "delete_at/2" do assert Tuple.delete_at({:foo, :bar, :baz}, 0) == {:bar, :baz} mod = Tuple assert mod.delete_at({:foo, :bar, :baz}, 0) == {:bar, :baz} end -end \ No newline at end of file +end diff --git a/lib/elixir/test/elixir/typespec_test.exs b/lib/elixir/test/elixir/typespec_test.exs new file mode 100644 index 00000000000..ab24ce5e115 --- /dev/null +++ b/lib/elixir/test/elixir/typespec_test.exs @@ -0,0 +1,1664 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("test_helper.exs", __DIR__) + +# Holds tests for both Kernel.Typespec and Code.Typespec +defmodule TypespecTest do + use ExUnit.Case, async: true + alias TypespecTest.TypespecSample + + defstruct [:hello] + + defmacrop test_module(do: block) do + quote do + {:module, _, bytecode, _} = + defmodule TypespecSample do + unquote(block) + end + + :code.purge(TypespecSample) + :code.delete(TypespecSample) + bytecode + end + end + + defp types(bytecode) do + bytecode + |> Code.Typespec.fetch_types() + |> elem(1) + |> Enum.sort() + end + + @skip_specs [__info__: 1] + + defp specs(bytecode) do + bytecode + |> Code.Typespec.fetch_specs() + |> elem(1) + |> Enum.reject(fn {sign, _} -> sign in @skip_specs end) + |> Enum.sort() + end + + defp callbacks(bytecode) do + bytecode + |> Code.Typespec.fetch_callbacks() + |> elem(1) + |> Enum.sort() + end + + describe "Kernel.Typespec errors" do + test "invalid type specification" do + assert_raise Kernel.TypespecError, ~r"invalid type specification: my_type = 1", fn -> + test_module do + @type my_type = 1 + end + end + end + + test "unexpected expression in typespec" do + assert_raise Kernel.TypespecError, ~r"unexpected expression in typespec: \"foobar\"", fn -> + test_module do + @type my_type :: "foobar" + end + end + + assert_raise Kernel.TypespecError, + ~r"unexpected expression in typespec: integer\(\)\(\)", + fn -> + test_module do + @type my_type :: integer()() + end + end + + assert_raise Kernel.TypespecError, + ~r"unexpected expression in typespec: %URI\.t\(\)\{\}", + fn -> + test_module do + @type my_type :: %URI.t(){} + end + end + + assert_raise Kernel.TypespecError, + ~r"unexpected expression in typespec: t\.Foo", + fn -> + test_module do + @type my_type :: t.Foo + end + end + end + + test "invalid function specification" do + assert_raise Kernel.TypespecError, ~r"invalid type specification: \"not a spec\"", fn -> + test_module do + @spec "not a spec" + end + end + + assert_raise Kernel.TypespecError, ~r"invalid type specification: 1 :: 2", fn -> + test_module do + @spec 1 :: 2 + end + end + end + + test "undefined type" do + assert_raise Kernel.TypespecError, ~r"type foo/0 undefined", fn -> + test_module do + @type omg :: foo + end + end + + assert_raise Kernel.TypespecError, ~r"type foo/2 undefined", fn -> + test_module do + @type omg :: foo(atom, integer) + end + end + + assert_raise Kernel.TypespecError, ~r"type bar/0 undefined", fn -> + test_module do + @spec foo(bar, integer) :: {atom, integer} + def foo(var1, var2), do: {var1, var2} + end + end + + assert_raise Kernel.TypespecError, ~r"type foo/0 undefined", fn -> + test_module do + @type omg :: __MODULE__.foo() + end + end + end + + test "redefined type" do + assert_raise Kernel.TypespecError, + ~r"type foo/0 is already defined in .*test/elixir/typespec_test.exs:138", + fn -> + test_module do + @type foo :: atom + @type foo :: integer + end + end + + assert_raise Kernel.TypespecError, + ~r"type foo/2 is already defined in .*test/elixir/typespec_test.exs:148", + fn -> + test_module do + @type foo :: atom + @type foo(var1, var2) :: {var1, var2} + @type foo(x, y) :: {x, y} + end + end + + assert_raise Kernel.TypespecError, + ~r"type foo/0 is already defined in .*test/elixir/typespec_test.exs:157", + fn -> + test_module do + @type foo :: atom + @typep foo :: integer + end + end + end + + test "type variable unused (singleton type variable)" do + assert_raise Kernel.TypespecError, ~r"type variable x is used only once", fn -> + test_module do + @type foo(x) :: integer + end + end + end + + test "type variable starting with underscore" do + test_module do + assert @type(foo(_hello) :: integer) == :ok + end + end + + test "type variable named _" do + assert_raise Kernel.TypespecError, ~r"type variable '_' is invalid", fn -> + test_module do + @type foo(_) :: integer + end + end + + assert_raise Kernel.TypespecError, ~r"type variable '_' is invalid", fn -> + test_module do + @type foo(_, _) :: integer + end + end + end + + test "spec for undefined function" do + assert_compile_error(~r"spec for undefined function omg/0", fn -> + test_module do + @spec omg :: atom + end + end) + end + + test "spec variable used only once (singleton type variable)" do + assert_raise Kernel.TypespecError, ~r"type variable x is used only once", fn -> + test_module do + @spec foo(x, integer) :: integer when x: var + def foo(x, y), do: x + y + end + end + end + + test "spec with ... outside of fn and lists" do + assert_raise Kernel.TypespecError, ~r"... in typespecs is only allowed inside lists", fn -> + test_module do + @spec foo(...) :: :ok + def foo(x), do: x + end + end + end + + test "invalid optional callback" do + assert_compile_error(~r"invalid optional callback :foo", fn -> + test_module do + @optional_callbacks :foo + end + end) + end + + test "unknown optional callback" do + assert_compile_error(~r"unknown callback foo/1 given as optional callback", fn -> + test_module do + @optional_callbacks foo: 1 + end + end) + end + + test "repeated optional callback" do + message = ~r"foo/1 has been specified as optional callback more than once" + + assert_compile_error(message, fn -> + test_module do + @callback foo(:ok) :: :ok + @optional_callbacks foo: 1, foo: 1 + end + end) + end + + test "behaviour_info/1 explicitly defined alongside @callback/@macrocallback" do + message = ~r"cannot define @callback attribute for foo/1 when behaviour_info/1" + + assert_compile_error(message, fn -> + test_module do + @callback foo(:ok) :: :ok + def behaviour_info(_), do: [] + end + end) + + message = ~r"cannot define @macrocallback attribute for foo/1 when behaviour_info/1" + + assert_compile_error(message, fn -> + test_module do + @macrocallback foo(:ok) :: :ok + def behaviour_info(_), do: [] + end + end) + end + + test "default is not supported" do + assert_raise ArgumentError, fn -> + test_module do + @callback hello(num \\ 0 :: integer) :: integer + end + end + + assert_raise ArgumentError, fn -> + test_module do + @callback hello(num :: integer \\ 0) :: integer + end + end + + assert_raise ArgumentError, fn -> + test_module do + @macrocallback hello(num \\ 0 :: integer) :: Macro.t() + end + end + + assert_raise ArgumentError, fn -> + test_module do + @macrocallback hello(num :: integer \\ 0) :: Macro.t() + end + end + + assert_raise ArgumentError, fn -> + test_module do + @spec hello(num \\ 0 :: integer) :: integer + end + end + + assert_raise ArgumentError, fn -> + test_module do + @spec hello(num :: integer \\ 0) :: integer + end + end + end + + test "@spec shows readable error message when return type is missing" do + message = ~r"type specification missing return type: my_fun\(integer\)" + + assert_raise Kernel.TypespecError, message, fn -> + test_module do + @spec my_fun(integer) + end + end + end + end + + describe "Kernel.Typespec definitions" do + test "typespec declarations return :ok" do + test_module do + def foo(), do: nil + + assert @type(foo :: any()) == :ok + assert @typep(foop :: any()) == :ok + assert @spec(foo() :: nil) == :ok + assert @opaque(my_type :: atom) == :ok + assert @callback(foo(foop) :: integer) == :ok + assert @macrocallback(foo(integer) :: integer) == :ok + end + end + + test "@type with a single type" do + bytecode = + test_module do + @type my_type :: term + end + + assert [type: {:my_type, {:type, _, :term, []}, []}] = types(bytecode) + end + + test "@type with an atom/alias" do + bytecode = + test_module do + @type foo :: :foo + @type bar :: Bar + end + + assert [ + type: {:bar, {:atom, _, Bar}, []}, + type: {:foo, {:atom, _, :foo}, []} + ] = types(bytecode) + end + + test "@type with an integer" do + bytecode = + test_module do + @type pos :: 10 + @type neg :: -10 + end + + assert [ + type: {:neg, {:op, _, :-, {:integer, _, 10}}, []}, + type: {:pos, {:integer, _, 10}, []} + ] = types(bytecode) + end + + test "@type with a tuple" do + bytecode = + test_module do + @type tup :: tuple() + @type one :: {123} + end + + assert [ + type: {:one, {:type, _, :tuple, [{:integer, _, 123}]}, []}, + type: {:tup, {:type, _, :tuple, :any}, []} + ] = types(bytecode) + end + + test "@type with a remote type" do + bytecode = + test_module do + @type my_type :: Remote.Some.type() + @type my_type_arg :: Remote.type(integer) + end + + assert [type: my_type, type: my_type_arg] = types(bytecode) + + assert {:my_type, type, []} = my_type + assert {:remote_type, _, [{:atom, _, Remote.Some}, {:atom, _, :type}, []]} = type + + assert {:my_type_arg, type, []} = my_type_arg + assert {:remote_type, _, args} = type + assert [{:atom, _, Remote}, {:atom, _, :type}, [{:type, _, :integer, []}]] = args + end + + test "@type with a binary" do + bytecode = + test_module do + @type bin :: binary + @type empty :: <<>> + @type size :: <<_::3>> + @type unit :: <<_::_*8>> + @type size_and_unit :: <<_::3, _::_*8>> + @type size_prod_unit :: <<_::3*8>> + end + + assert [ + type: {:bin, {:type, _, :binary, []}, []}, + type: {:empty, {:type, _, :binary, [{:integer, _, 0}, {:integer, _, 0}]}, []}, + type: {:size, {:type, _, :binary, [{:integer, _, 3}, {:integer, _, 0}]}, []}, + type: + {:size_and_unit, {:type, _, :binary, [{:integer, _, 3}, {:integer, _, 8}]}, []}, + type: + {:size_prod_unit, + {:type, _, :binary, + [{:op, _, :*, {:integer, _, 3}, {:integer, _, 8}}, {:integer, _, 0}]}, []}, + type: {:unit, {:type, _, :binary, [{:integer, _, 0}, {:integer, _, 8}]}, []} + ] = types(bytecode) + end + + test "@type with invalid binary spec" do + assert_raise Kernel.TypespecError, ~r"invalid binary specification", fn -> + test_module do + @type my_type :: <<_::atom()>> + end + end + + assert_raise Kernel.TypespecError, ~r"invalid binary specification", fn -> + test_module do + @type my_type :: <<_::integer>> + end + end + + assert_raise Kernel.TypespecError, ~r"invalid binary specification", fn -> + test_module do + @type my_type :: <<_::(-4)>> + end + end + + assert_raise Kernel.TypespecError, ~r"invalid binary specification", fn -> + test_module do + @type my_type :: <<_::3, _::_*atom>> + end + end + + assert_raise Kernel.TypespecError, ~r"invalid binary specification", fn -> + test_module do + @type my_type :: <<_::3, _::_*(-8)>> + end + end + + assert_raise Kernel.TypespecError, ~r"invalid binary specification", fn -> + test_module do + @type my_type :: <<_::3, _::_*257>> + end + end + end + + test "@type with a range op" do + bytecode = + test_module do + @type range1 :: 1..10 + @type range2 :: -1..1 + end + + assert [ + {:type, {:range1, {:type, _, :range, range1_args}, []}}, + {:type, {:range2, {:type, _, :range, range2_args}, []}} + ] = types(bytecode) + + assert [{:integer, _, 1}, {:integer, _, 10}] = range1_args + assert [{:op, _, :-, {:integer, _, 1}}, {:integer, _, 1}] = range2_args + end + + test "@type with invalid range" do + assert_raise Kernel.TypespecError, ~r"invalid range specification", fn -> + test_module do + @type my_type :: atom..10 + end + end + end + + test "@type with a keyword map" do + bytecode = + test_module do + @type my_type :: %{hello: :world} + end + + assert [type: {:my_type, type, []}] = types(bytecode) + assert {:type, _, :map, [arg]} = type + assert {:type, _, :map_field_exact, [{:atom, _, :hello}, {:atom, _, :world}]} = arg + end + + test "@type with a map" do + bytecode = + test_module do + @type my_type :: %{required(:a) => :b, optional(:c) => :d} + end + + assert [type: {:my_type, type, []}] = types(bytecode) + assert {:type, _, :map, [arg1, arg2]} = type + assert {:type, _, :map_field_exact, [{:atom, _, :a}, {:atom, _, :b}]} = arg1 + assert {:type, _, :map_field_assoc, [{:atom, _, :c}, {:atom, _, :d}]} = arg2 + end + + test "@type with a struct" do + bytecode = + test_module do + defstruct hello: nil, other: nil + @type my_type :: %TypespecSample{hello: :world} + end + + assert [type: {:my_type, type, []}] = types(bytecode) + assert {:type, _, :map, [struct, arg1, arg2]} = type + assert {:type, _, :map_field_exact, struct_args} = struct + assert [{:atom, _, :__struct__}, {:atom, _, TypespecSample}] = struct_args + assert {:type, _, :map_field_exact, [{:atom, _, :hello}, {:atom, _, :world}]} = arg1 + assert {:type, _, :map_field_exact, [{:atom, _, :other}, {:type, _, :term, []}]} = arg2 + end + + test "@type with an exception struct" do + bytecode = + test_module do + defexception [:message] + @type my_type :: %TypespecSample{} + end + + assert [type: {:my_type, type, []}] = types(bytecode) + assert {:type, _, :map, [struct, arg1, arg2]} = type + assert {:type, _, :map_field_exact, struct_args} = struct + assert [{:atom, _, :__struct__}, {:atom, _, TypespecSample}] = struct_args + assert {:type, _, :map_field_exact, [{:atom, _, :__exception__}, {:atom, _, true}]} = arg1 + assert {:type, _, :map_field_exact, [{:atom, _, :message}, {:type, _, :term, []}]} = arg2 + end + + @fields Enum.map(10..42, &{:"f#{&1}", :ok}) + + test "@type with a large struct" do + bytecode = + test_module do + defstruct unquote(@fields) + @type my_type :: %TypespecSample{unquote_splicing(@fields)} + end + + assert [type: {:my_type, type, []}] = types(bytecode) + assert {:type, _, :map, [struct, arg1, arg2 | _]} = type + assert {:type, _, :map_field_exact, struct_args} = struct + assert [{:atom, _, :__struct__}, {:atom, _, TypespecSample}] = struct_args + assert {:type, _, :map_field_exact, [{:atom, _, :f10}, {:atom, _, :ok}]} = arg1 + assert {:type, _, :map_field_exact, [{:atom, _, :f11}, {:atom, _, :ok}]} = arg2 + end + + test "@type with struct does not @enforce_keys" do + bytecode = + test_module do + @enforce_keys [:other] + defstruct hello: nil, other: nil + @type my_type :: %TypespecSample{hello: :world} + end + + assert [type: {:my_type, _type, []}] = types(bytecode) + end + + test "@type with undefined struct" do + assert_raise ArgumentError, ~r"ThisModuleDoesNotExist.__struct__/1 is undefined", fn -> + test_module do + @type my_type :: %ThisModuleDoesNotExist{} + end + end + + assert_raise ArgumentError, ~r"cannot access struct TypespecTest.TypespecSample", fn -> + test_module do + @type my_type :: %TypespecSample{} + end + end + end + + test "@type with a struct with undefined field" do + assert_raise Kernel.TypespecError, + ~r"undefined field :no_field on struct TypespecTest.TypespecSample", + fn -> + test_module do + defstruct [:hello, :eric] + @type my_type :: %TypespecSample{no_field: :world} + end + end + + assert_raise Kernel.TypespecError, + ~r"undefined field :no_field on struct TypespecTest.TypespecSample", + fn -> + test_module do + defstruct [:hello, :eric] + @type my_type :: %__MODULE__{no_field: :world} + end + end + end + + test "@type when overriding Elixir built-in" do + assert_raise Kernel.TypespecError, ~r"type struct/0 is a built-in type", fn -> + test_module do + @type struct :: :oops + end + end + end + + test "@type when overriding Erlang built-in" do + assert_raise Kernel.TypespecError, ~r"type list/0 is a built-in type", fn -> + test_module do + @type list :: :oops + end + end + end + + test "@type with public record" do + bytecode = + test_module do + require Record + Record.defrecord(:timestamp, date: 1, time: 2) + @type my_type :: record(:timestamp, time: :foo) + end + + assert [type: {:my_type, type, []}] = types(bytecode) + assert {:type, _, :tuple, [timestamp, term, foo]} = type + assert {:atom, 0, :timestamp} = timestamp + assert {:ann_type, 0, [{:var, 0, :date}, {:type, 0, :term, []}]} = term + assert {:ann_type, 0, [{:var, 0, :time}, {:atom, 0, :foo}]} = foo + end + + test "@type with private record" do + bytecode = + test_module do + require Record + Record.defrecordp(:timestamp, date: 1, time: 2) + @type my_type :: record(:timestamp, time: :foo) + end + + assert [type: {:my_type, type, []}] = types(bytecode) + assert {:type, _, :tuple, args} = type + + assert [ + {:atom, 0, :timestamp}, + {:ann_type, 0, [{:var, 0, :date}, {:type, 0, :term, []}]}, + {:ann_type, 0, [{:var, 0, :time}, {:atom, 0, :foo}]} + ] = args + end + + test "@type with named record" do + bytecode = + test_module do + require Record + Record.defrecord(:timestamp, :my_timestamp, date: 1, time: 2) + @type my_type :: record(:timestamp, time: :foo) + end + + assert [type: {:my_type, type, []}] = types(bytecode) + assert {:type, _, :tuple, [my_timestamp, term, _foo]} = type + assert {:atom, 0, :my_timestamp} = my_timestamp + assert {:ann_type, 0, [{:var, 0, :date}, {:type, 0, :term, []}]} = term + assert {:ann_type, 0, [{:var, 0, :time}, {:atom, 0, :foo}]} + end + + test "@type with undefined record" do + assert_raise Kernel.TypespecError, ~r"unknown record :this_record_does_not_exist", fn -> + test_module do + @type my_type :: record(:this_record_does_not_exist, []) + end + end + end + + test "@type with a record with undefined field" do + assert_raise Kernel.TypespecError, ~r"undefined field no_field on record :timestamp", fn -> + test_module do + require Record + Record.defrecord(:timestamp, date: 1, time: 2) + @type my_type :: record(:timestamp, no_field: :foo) + end + end + end + + test "@type with a record which declares the name as the type `atom` rather than an atom literal" do + assert_raise Kernel.TypespecError, ~r"expected the record name to be an atom literal", fn -> + test_module do + @type my_type :: record(atom, field: :foo) + end + end + end + + test "@type can be named record" do + bytecode = + test_module do + @type record :: binary + @spec foo?(record) :: boolean + def foo?(_), do: true + end + + assert [type: {:record, {:type, _, :binary, []}, []}] = types(bytecode) + end + + test "@type with an invalid map notation" do + assert_raise Kernel.TypespecError, ~r"invalid map specification", fn -> + test_module do + @type content :: %{atom | String.t() => term} + end + end + end + + test "@type with list shortcuts" do + bytecode = + test_module do + @type my_type :: [] + @type my_type1 :: [integer] + @type my_type2 :: [integer, ...] + end + + assert [ + type: {:my_type, {:type, _, nil, []}, []}, + type: {:my_type1, {:type, _, :list, [{:type, _, :integer, []}]}, []}, + type: {:my_type2, {:type, _, :nonempty_list, [{:type, _, :integer, []}]}, []} + ] = types(bytecode) + end + + test "@type with a fun" do + bytecode = + test_module do + @type my_type :: (... -> any) + end + + assert [type: {:my_type, {:type, _, :fun, [{:type, _, :any}, {:type, _, :any, []}]}, []}] = + types(bytecode) + end + + test "@type with a fun with multiple arguments and return type" do + bytecode = + test_module do + @type my_type :: (integer, integer -> integer) + end + + assert [type: {:my_type, type, []}] = types(bytecode) + assert {:type, _, :fun, [args, return_type]} = type + assert {:type, _, :product, [{:type, _, :integer, []}, {:type, _, :integer, []}]} = args + assert {:type, _, :integer, []} = return_type + end + + test "@type with a fun with no arguments and return type" do + bytecode = + test_module do + @type my_type :: (-> integer) + end + + assert [type: {:my_type, type, []}] = types(bytecode) + assert {:type, _, :fun, [{:type, _, :product, []}, {:type, _, :integer, []}]} = type + end + + test "@type with a fun with any arity and return type" do + bytecode = + test_module do + @type my_type :: (... -> integer) + end + + assert [type: {:my_type, type, []}] = types(bytecode) + assert {:type, _, :fun, [{:type, _, :any}, {:type, _, :integer, []}]} = type + end + + test "@type with a union" do + bytecode = + test_module do + @type my_type :: integer | charlist | atom + end + + assert [type: {:my_type, type, []}] = types(bytecode) + assert {:type, _, :union, [integer, charlist, atom]} = type + assert {:type, _, :integer, []} = integer + assert {:remote_type, _, [{:atom, _, :elixir}, {:atom, _, :charlist}, []]} = charlist + assert {:type, _, :atom, []} = atom + end + + test "@type with keywords" do + bytecode = + test_module do + @type my_type :: [first: integer, step: integer, last: integer] + end + + assert [type: {:my_type, type, []}] = types(bytecode) + assert {:type, _, :list, [{:type, _, :union, union_types}]} = type + + assert [ + {:type, _, :tuple, [{:atom, _, :first}, {:type, _, :integer, []}]}, + {:type, _, :tuple, [{:atom, _, :step}, {:type, _, :integer, []}]}, + {:type, _, :tuple, [{:atom, _, :last}, {:type, _, :integer, []}]} + ] = union_types + end + + test "@type with parameters" do + bytecode = + test_module do + @type my_type(x) :: x + @type my_type1(x) :: list(x) + @type my_type2(x, y) :: {x, y} + end + + assert [ + type: {:my_type, {:var, _, :x}, [{:var, _, :x}]}, + type: {:my_type1, {:type, _, :list, [{:var, _, :x}]}, [{:var, _, :x}]}, + type: {:my_type2, my_type2, [{:var, _, :x}, {:var, _, :y}]} + ] = types(bytecode) + + assert {:type, _, :tuple, [{:var, _, :x}, {:var, _, :y}]} = my_type2 + end + + test "@type with annotations" do + bytecode = + test_module do + @type my_type :: named :: integer + @type my_type1 :: (a :: integer -> integer) + end + + assert [type: {:my_type, my_type, []}, type: {:my_type1, my_type1, []}] = types(bytecode) + + assert {:ann_type, _, [{:var, _, :named}, {:type, _, :integer, []}]} = my_type + + assert {:type, _, :fun, [fun_args, fun_return]} = my_type1 + assert {:type, _, :product, [{:ann_type, _, [a, {:type, _, :integer, []}]}]} = fun_args + assert {:var, _, :a} = a + assert {:type, _, :integer, []} = fun_return + end + + test "@type unquote fragment" do + quoted = + quote unquote: false do + name = :my_type + type = :foo + @type unquote(name)() :: unquote(type) + end + + bytecode = + test_module do + Code.eval_quoted(quoted, [], module: __MODULE__) + end + + assert [type: {:my_type, {:atom, _, :foo}, []}] = types(bytecode) + end + + test "@type with module attributes" do + bytecode = + test_module do + @keyword Keyword + @type kw :: @keyword.t + @type kw(value) :: @keyword.t(value) + end + + assert [type: {:kw, kw, _}, type: {:kw, kw_with_value, [{:var, _, :value}]}] = + types(bytecode) + + assert {:remote_type, _, [{:atom, _, Keyword}, {:atom, _, :t}, []]} = kw + assert {:remote_type, _, kw_with_value_args} = kw_with_value + assert [{:atom, _, Keyword}, {:atom, _, :t}, [{:var, _, :value}]] = kw_with_value_args + end + + test "@type with macro in alias" do + bytecode = + test_module do + defmacro module() do + quote do: __MODULE__ + end + + @type my_type :: module().Foo + end + + assert [type: {:my_type, {:atom, _, TypespecTest.TypespecSample.Foo}, []}] = types(bytecode) + end + + test "@type with a reserved signature" do + assert_raise Kernel.TypespecError, + ~r"type required\/1 is a reserved type and it cannot be defined", + fn -> + test_module do + @type required(arg) :: any() + end + end + + assert_raise Kernel.TypespecError, + ~r"type optional\/1 is a reserved type and it cannot be defined", + fn -> + test_module do + @type optional(arg) :: any() + end + end + + assert_raise Kernel.TypespecError, + ~r"type required\/1 is a reserved type and it cannot be defined", + fn -> + test_module do + @typep required(arg) :: any() + end + end + + assert_raise Kernel.TypespecError, + ~r"type optional\/1 is a reserved type and it cannot be defined", + fn -> + test_module do + @typep optional(arg) :: any() + end + end + + assert_raise Kernel.TypespecError, + ~r"type required\/1 is a reserved type and it cannot be defined", + fn -> + test_module do + @opaque required(arg) :: any() + end + end + + assert_raise Kernel.TypespecError, + ~r"type optional\/1 is a reserved type and it cannot be defined", + fn -> + test_module do + @opaque optional(arg) :: any() + end + end + end + + test "invalid remote @type with module attribute that does not evaluate to a module" do + assert_raise Kernel.TypespecError, ~r/\(@foo is "bar"\)/, fn -> + test_module do + @foo "bar" + @type t :: @foo.t + end + end + end + + test "defines_type?" do + test_module do + @type my_type :: tuple + @type my_type(a) :: [a] + assert Kernel.Typespec.defines_type?(__MODULE__, {:my_type, 0}) + assert Kernel.Typespec.defines_type?(__MODULE__, {:my_type, 1}) + refute Kernel.Typespec.defines_type?(__MODULE__, {:my_type, 2}) + end + end + + test "spec_to_callback/2" do + bytecode = + test_module do + @spec foo() :: term() + def foo(), do: :ok + Kernel.Typespec.spec_to_callback(__MODULE__, {:foo, 0}) + end + + assert specs(bytecode) == callbacks(bytecode) + end + + test "@opaque" do + bytecode = + test_module do + @opaque my_type(x) :: x + end + + assert [opaque: {:my_type, {:var, _, :x}, [{:var, _, :x}]}] = types(bytecode) + end + + test "@spec" do + bytecode = + test_module do + def my_fun1(x), do: x + def my_fun2(), do: :ok + def my_fun3(x, y), do: {x, y} + def my_fun4(x), do: x + @spec my_fun1(integer) :: integer + @spec my_fun2() :: integer + @spec my_fun3(integer, integer) :: {integer, integer} + @spec my_fun4(x :: integer) :: integer + end + + assert [my_fun1, my_fun2, my_fun3, my_fun4] = specs(bytecode) + + assert {{:my_fun1, 1}, [{:type, _, :fun, args}]} = my_fun1 + assert [{:type, _, :product, [{:type, _, :integer, []}]}, {:type, _, :integer, []}] = args + + assert {{:my_fun2, 0}, [{:type, _, :fun, args}]} = my_fun2 + assert [{:type, _, :product, []}, {:type, _, :integer, []}] = args + + assert {{:my_fun3, 2}, [{:type, _, :fun, [arg1, arg2]}]} = my_fun3 + assert {:type, _, :product, [{:type, _, :integer, []}, {:type, _, :integer, []}]} = arg1 + assert {:type, _, :tuple, [{:type, _, :integer, []}, {:type, _, :integer, []}]} = arg2 + + assert {{:my_fun4, 1}, [{:type, _, :fun, args}]} = my_fun4 + assert [x, {:type, _, :integer, []}] = args + assert {:type, _, :product, [{:ann_type, _, [{:var, _, :x}, {:type, _, :integer, []}]}]} = x + end + + test "@spec with vars matching built-ins" do + bytecode = + test_module do + def my_fun1(x), do: x + def my_fun2(x), do: x + @spec my_fun1(tuple) :: tuple + @spec my_fun2(tuple) :: tuple when tuple: {integer, integer} + end + + assert [my_fun1, my_fun2] = specs(bytecode) + + assert {{:my_fun1, 1}, [{:type, _, :fun, args}]} = my_fun1 + assert [{:type, _, :product, [{:type, _, :tuple, :any}]}, {:type, _, :tuple, :any}] = args + + assert {{:my_fun2, 1}, [{:type, _, :bounded_fun, args}]} = my_fun2 + + assert [type, _] = args + + assert {:type, _, :fun, [{:type, _, :product, [{:var, _, :tuple}]}, {:var, _, :tuple}]} = + type + end + + test "@spec with guards" do + bytecode = + test_module do + def my_fun1(x), do: x + @spec my_fun1(x) :: boolean when x: integer + + def my_fun2(x), do: x + @spec my_fun2(x) :: x when x: var + + def my_fun3(_x, y), do: y + @spec my_fun3(x, y) :: y when y: x, x: var + end + + assert [my_fun1, my_fun2, my_fun3] = specs(bytecode) + + assert {{:my_fun1, 1}, [{:type, _, :bounded_fun, args}]} = my_fun1 + assert [{:type, _, :fun, [product, {:type, _, :boolean, []}]}, constraints] = args + assert {:type, _, :product, [{:var, _, :x}]} = product + assert [{:type, _, :constraint, subtype}] = constraints + assert [{:atom, _, :is_subtype}, [{:var, _, :x}, {:type, _, :integer, []}]] = subtype + + assert {{:my_fun2, 1}, [{:type, _, :fun, args}]} = my_fun2 + assert [{:type, _, :product, [{:var, _, :x}]}, {:var, _, :x}] = args + + assert {{:my_fun3, 2}, [{:type, _, :bounded_fun, args}]} = my_fun3 + assert [{:type, _, :fun, fun_type}, [{:type, _, :constraint, constraint_type}]] = args + assert [{:type, _, :product, [{:var, _, :x}, {:var, _, :y}]}, {:var, _, :y}] = fun_type + assert [{:atom, _, :is_subtype}, [{:var, _, :y}, {:var, _, :x}]] = constraint_type + end + + test "@type, @opaque, and @typep as module attributes" do + defmodule TypeModuleAttributes do + @type type1 :: boolean + @opaque opaque1 :: boolean + @typep typep1 :: boolean + + def type1, do: @type + def opaque1, do: @opaque + def typep1, do: @typep + + @type type2 :: atom + @type type3 :: pid + @opaque opaque2 :: atom + @opaque opaque3 :: pid + @typep typep2 :: atom + + def type2, do: @type + def opaque2, do: @opaque + def typep2, do: @typep + + # Avoid unused warnings + @spec foo(typep1) :: typep2 + def foo(_x), do: :ok + end + + assert [ + {:type, {:"::", _, [{:type1, _, _}, {:boolean, _, _}]}, {TypeModuleAttributes, _}} + ] = TypeModuleAttributes.type1() + + assert [ + {:type, {:"::", _, [{:type3, _, _}, {:pid, _, _}]}, {TypeModuleAttributes, _}}, + {:type, {:"::", _, [{:type2, _, _}, {:atom, _, _}]}, {TypeModuleAttributes, _}}, + {:type, {:"::", _, [{:type1, _, _}, {:boolean, _, _}]}, {TypeModuleAttributes, _}} + ] = TypeModuleAttributes.type2() + + assert [ + {:opaque, {:"::", _, [{:opaque1, _, _}, {:boolean, _, _}]}, + {TypeModuleAttributes, _}} + ] = TypeModuleAttributes.opaque1() + + assert [ + {:opaque, {:"::", _, [{:opaque3, _, _}, {:pid, _, _}]}, {TypeModuleAttributes, _}}, + {:opaque, {:"::", _, [{:opaque2, _, _}, {:atom, _, _}]}, + {TypeModuleAttributes, _}}, + {:opaque, {:"::", _, [{:opaque1, _, _}, {:boolean, _, _}]}, + {TypeModuleAttributes, _}} + ] = TypeModuleAttributes.opaque2() + + assert [ + {:typep, {:"::", _, [{:typep1, _, _}, {:boolean, _, _}]}, + {TypeModuleAttributes, _}} + ] = TypeModuleAttributes.typep1() + + assert [ + {:typep, {:"::", _, [{:typep2, _, _}, {:atom, _, _}]}, {TypeModuleAttributes, _}}, + {:typep, {:"::", _, [{:typep1, _, _}, {:boolean, _, _}]}, + {TypeModuleAttributes, _}} + ] = TypeModuleAttributes.typep2() + after + :code.purge(TypeModuleAttributes) + :code.delete(TypeModuleAttributes) + end + + test "@spec, @callback, and @macrocallback as module attributes" do + defmodule SpecModuleAttributes do + @callback callback1 :: integer + @macrocallback macrocallback1 :: integer + + @spec spec1 :: boolean + def spec1, do: @spec + + @callback callback2 :: var when var: boolean + @macrocallback macrocallback2 :: var when var: boolean + + @spec spec2 :: atom + def spec2, do: @spec + + @spec spec3 :: pid + def spec3, do: :ok + def spec4, do: @spec + + def callback, do: @callback + def macrocallback, do: @macrocallback + end + + assert [ + {:spec, {:"::", _, [{:spec1, _, _}, {:boolean, _, _}]}, {SpecModuleAttributes, _}} + ] = SpecModuleAttributes.spec1() + + assert [ + {:spec, {:"::", _, [{:spec2, _, _}, {:atom, _, _}]}, {SpecModuleAttributes, _}}, + {:spec, {:"::", _, [{:spec1, _, _}, {:boolean, _, _}]}, {SpecModuleAttributes, _}} + ] = SpecModuleAttributes.spec2() + + assert [ + {:spec, {:"::", _, [{:spec3, _, _}, {:pid, _, _}]}, {SpecModuleAttributes, _}}, + {:spec, {:"::", _, [{:spec2, _, _}, {:atom, _, _}]}, {SpecModuleAttributes, _}}, + {:spec, {:"::", _, [{:spec1, _, _}, {:boolean, _, _}]}, {SpecModuleAttributes, _}} + ] = SpecModuleAttributes.spec4() + + assert [ + {:callback, + {:when, _, + [{:"::", _, [{:callback2, _, _}, {:var, _, _}]}, [var: {:boolean, _, _}]]}, + {SpecModuleAttributes, _}}, + {:callback, {:"::", _, [{:callback1, _, _}, {:integer, _, _}]}, + {SpecModuleAttributes, _}} + ] = SpecModuleAttributes.callback() + + assert [ + {:macrocallback, + {:when, _, + [{:"::", _, [{:macrocallback2, _, _}, {:var, _, _}]}, [var: {:boolean, _, _}]]}, + {SpecModuleAttributes, _}}, + {:macrocallback, {:"::", _, [{:macrocallback1, _, _}, {:integer, _, _}]}, + {SpecModuleAttributes, _}} + ] = SpecModuleAttributes.macrocallback() + after + :code.purge(SpecModuleAttributes) + :code.delete(SpecModuleAttributes) + end + + test "@callback" do + bytecode = + test_module do + @callback my_fun(integer) :: integer + @callback my_fun(list) :: list + @callback my_fun() :: integer + @callback my_fun(integer, integer) :: {integer, integer} + end + + assert [my_fun_0, my_fun_1, my_fun_2] = callbacks(bytecode) + + assert {{:my_fun, 0}, [{:type, _, :fun, args}]} = my_fun_0 + assert [{:type, _, :product, []}, {:type, _, :integer, []}] = args + + assert {{:my_fun, 1}, [clause1, clause2]} = my_fun_1 + assert {:type, _, :fun, args1} = clause1 + assert [{:type, _, :product, [{:type, _, :integer, []}]}, {:type, _, :integer, []}] = args1 + assert {:type, _, :fun, args2} = clause2 + assert [{:type, _, :product, [{:type, _, :list, []}]}, {:type, _, :list, []}] = args2 + + assert {{:my_fun, 2}, [{:type, _, :fun, [args_type, return_type]}]} = my_fun_2 + + assert {:type, _, :product, [{:type, _, :integer, []}, {:type, _, :integer, []}]} = + args_type + + assert {:type, _, :tuple, [{:type, _, :integer, []}, {:type, _, :integer, []}]} = + return_type + end + + test "block handling" do + bytecode = + test_module do + @spec foo((-> [integer])) :: integer + def foo(_), do: 1 + end + + assert [{{:foo, 1}, [{:type, _, :fun, [args, return]}]}] = specs(bytecode) + assert {:type, _, :product, [{:type, _, :fun, fun_args}]} = args + assert [{:type, _, :product, []}, {:type, _, :list, [{:type, _, :integer, []}]}] = fun_args + assert {:type, _, :integer, []} = return + end + end + + describe "Code.Typespec" do + test "type_to_quoted" do + quoted = + Enum.sort([ + quote(do: @type(tuple(arg) :: {:tuple, arg})), + quote(do: @type(with_ann() :: t :: atom())), + quote(do: @type(a_tuple() :: tuple())), + quote(do: @type(empty_tuple() :: {})), + quote(do: @type(one_tuple() :: {:foo})), + quote(do: @type(two_tuple() :: {:foo, :bar})), + quote(do: @type(custom_tuple() :: tuple(:foo))), + quote(do: @type(imm_type_1() :: 1)), + quote(do: @type(imm_type_2() :: :foo)), + quote(do: @type(simple_type() :: integer())), + quote(do: @type(param_type(p) :: [p])), + quote(do: @type(union_type() :: integer() | binary() | boolean())), + quote(do: @type(binary_type1() :: <<_::_*8>>)), + quote(do: @type(binary_type2() :: <<_::3>>)), + quote(do: @type(binary_type3() :: <<_::3, _::_*8>>)), + quote(do: @type(binary_type4() :: <<_::3*8>>)), + quote(do: @type(tuple_type() :: {integer()})), + quote(do: @type(ftype() :: (-> any()) | (-> integer()) | (integer() -> integer()))), + quote(do: @type(cl() :: charlist())), + quote(do: @type(st() :: struct())), + quote(do: @type(ab() :: as_boolean(term()))), + quote(do: @type(kw() :: keyword())), + quote(do: @type(kwt() :: keyword(term()))), + quote(do: @type(vaf() :: (... -> any()))), + quote(do: @type(rng() :: 1..10)), + quote(do: @type(opts() :: [first: integer(), step: integer(), last: integer()])), + quote(do: @type(ops() :: {+1, -1})), + quote(do: @type(map(arg) :: {:map, arg})), + quote(do: @type(a_map() :: map())), + quote(do: @type(empty_map() :: %{})), + quote(do: @type(my_map() :: %{hello: :world})), + quote(do: @type(my_req_map() :: %{required(0) => :foo})), + quote(do: @type(my_opt_map() :: %{optional(0) => :foo})), + quote(do: @type(my_struct() :: %TypespecTest{hello: :world})), + quote(do: @type(custom_map() :: map(:foo))), + quote(do: @type(list1() :: list())), + quote(do: @type(list2() :: [0])), + quote(do: @type(list3() :: [...])), + quote(do: @type(list4() :: [0, ...])), + quote(do: @type(nil_list() :: [])) + ]) + + bytecode = + test_module do + Code.eval_quoted(quoted, [], module: __MODULE__) + end + + types = types(bytecode) + + Enum.each(Enum.zip(types, quoted), fn {{:type, type}, definition} -> + ast = Code.Typespec.type_to_quoted(type) + assert Macro.to_string(quote(do: @type(unquote(ast)))) == Macro.to_string(definition) + end) + end + + test "type_to_quoted for paren_type" do + type = {:my_type, {:paren_type, 0, [{:type, 0, :integer, []}]}, []} + + assert Code.Typespec.type_to_quoted(type) == + {:"::", [], [{:my_type, [], []}, {:integer, [line: 0], []}]} + end + + test "spec_to_quoted" do + quoted = + Enum.sort([ + quote(do: @spec(foo() :: integer())), + quote(do: @spec(foo() :: union())), + quote(do: @spec(foo() :: union(integer()))), + quote(do: @spec(foo() :: truly_union())), + quote(do: @spec(foo(union()) :: union())), + quote(do: @spec(foo(union(integer())) :: union(integer()))), + quote(do: @spec(foo(truly_union()) :: truly_union())), + quote(do: @spec(foo(atom()) :: integer() | [{}])), + quote(do: @spec(foo(arg) :: integer() when [arg: integer()])), + quote(do: @spec(foo(arg) :: arg when [arg: var])), + quote(do: @spec(foo(arg :: atom()) :: atom())) + ]) + + bytecode = + test_module do + @type union :: any() + @type union(t) :: t + @type truly_union :: list | map | union + + def foo(), do: 1 + def foo(arg), do: arg + Code.eval_quoted(quote(do: (unquote_splicing(quoted))), [], module: __MODULE__) + end + + specs = + Enum.flat_map(specs(bytecode), fn {{_, _}, specs} -> + Enum.map(specs, fn spec -> + quote(do: @spec(unquote(Code.Typespec.spec_to_quoted(:foo, spec)))) + end) + end) + + specs_with_quoted = specs |> Enum.sort() |> Enum.zip(quoted) + + Enum.each(specs_with_quoted, fn {spec, definition} -> + assert Macro.to_string(spec) == Macro.to_string(definition) + end) + end + + test "spec_to_quoted with maps with __struct__ key" do + defmodule StructA do + defstruct [:key] + end + + defmodule StructB do + defstruct [:key] + end + + bytecode = + test_module do + @spec single_struct(%StructA{}) :: :ok + def single_struct(arg), do: {:ok, arg} + + @spec single_struct_key(%{__struct__: StructA}) :: :ok + def single_struct_key(arg), do: {:ok, arg} + + @spec single_struct_key_type(%{__struct__: atom()}) :: :ok + def single_struct_key_type(arg), do: {:ok, arg} + + @spec union_struct(%StructA{} | %StructB{}) :: :ok + def union_struct(arg), do: {:ok, arg} + + @spec union_struct_key(%{__struct__: StructA | StructB}) :: :ok + def union_struct_key(arg), do: {:ok, arg} + + @spec union_struct_key_type(%{__struct__: atom() | StructA | binary()}) :: :ok + def union_struct_key_type(arg), do: {:ok, arg} + end + + [ + {{:single_struct, 1}, [ast_single_struct]}, + {{:single_struct_key, 1}, [ast_single_struct_key]}, + {{:single_struct_key_type, 1}, [ast_single_struct_key_type]}, + {{:union_struct, 1}, [ast_union_struct]}, + {{:union_struct_key, 1}, [ast_union_struct_key]}, + {{:union_struct_key_type, 1}, [ast_union_struct_key_type]} + ] = specs(bytecode) + + assert Code.Typespec.spec_to_quoted(:single_struct, ast_single_struct) + |> Macro.to_string() == + "single_struct(%TypespecTest.StructA{key: term()}) :: :ok" + + assert Code.Typespec.spec_to_quoted(:single_struct_key, ast_single_struct_key) + |> Macro.to_string() == + "single_struct_key(%TypespecTest.StructA{}) :: :ok" + + assert Code.Typespec.spec_to_quoted(:single_struct_key_type, ast_single_struct_key_type) + |> Macro.to_string() == + "single_struct_key_type(%{__struct__: atom()}) :: :ok" + + assert Code.Typespec.spec_to_quoted(:union_struct, ast_union_struct) |> Macro.to_string() == + "union_struct(%TypespecTest.StructA{key: term()} | %TypespecTest.StructB{key: term()}) :: :ok" + + assert Code.Typespec.spec_to_quoted(:union_struct_key, ast_union_struct_key) + |> Macro.to_string() == + "union_struct_key(%{__struct__: TypespecTest.StructA | TypespecTest.StructB}) :: :ok" + + assert Code.Typespec.spec_to_quoted(:union_struct_key_type, ast_union_struct_key_type) + |> Macro.to_string() == + "union_struct_key_type(%{__struct__: atom() | TypespecTest.StructA | binary()}) :: :ok" + end + + test "non-variables are given as arguments" do + msg = ~r/The type one_bad_variable\/1 has an invalid argument\(s\): String.t\(\)/ + + assert_raise Kernel.TypespecError, msg, fn -> + test_module do + @type one_bad_variable(String.t()) :: String.t() + end + end + + msg = ~r/The type two_bad_variables\/2 has an invalid argument\(s\): :ok, Enumerable.t\(\)/ + + assert_raise Kernel.TypespecError, msg, fn -> + test_module do + @type two_bad_variables(:ok, Enumerable.t()) :: {:ok, []} + end + end + + msg = ~r/The type one_bad_one_good\/2 has an invalid argument\(s\): \"\"/ + + assert_raise Kernel.TypespecError, msg, fn -> + test_module do + @type one_bad_one_good(input1, "") :: {:ok, input1} + end + end + end + + test "retrieval invalid data" do + assert Code.Typespec.fetch_types(Unknown) == :error + assert Code.Typespec.fetch_specs(Unknown) == :error + end + + # This is a test that implements all types specified in lib/elixir/pages/typespecs.md + test "documented types and their AST" do + defmodule SomeStruct do + defstruct [:key] + end + + quoted = + Enum.sort([ + ## Basic types + quote(do: @type(basic_any() :: any())), + quote(do: @type(basic_none() :: none())), + quote(do: @type(basic_atom() :: atom())), + quote(do: @type(basic_map() :: map())), + quote(do: @type(basic_pid() :: pid())), + quote(do: @type(basic_port() :: port())), + quote(do: @type(basic_reference() :: reference())), + quote(do: @type(basic_struct() :: struct())), + quote(do: @type(basic_tuple() :: tuple())), + + # Numbers + quote(do: @type(basic_float() :: float())), + quote(do: @type(basic_integer() :: integer())), + quote(do: @type(basic_neg_integer() :: neg_integer())), + quote(do: @type(basic_non_neg_integer() :: non_neg_integer())), + quote(do: @type(basic_pos_integer() :: pos_integer())), + + # Lists + quote(do: @type(basic_list_type() :: list(integer()))), + quote(do: @type(basic_nonempty_list_type() :: nonempty_list(integer()))), + quote do + @type basic_maybe_improper_list_type() :: maybe_improper_list(integer(), atom()) + end, + quote do + @type basic_nonempty_improper_list_type() :: nonempty_improper_list(integer(), atom()) + end, + quote do + @type basic_nonempty_maybe_improper_list_type() :: + nonempty_maybe_improper_list(integer(), atom()) + end, + + ## Literals + quote(do: @type(literal_atom() :: :atom)), + quote(do: @type(literal_integer() :: 1)), + quote(do: @type(literal_integers() :: 1..10)), + quote(do: @type(literal_empty_bitstring() :: <<>>)), + quote(do: @type(literal_size_0() :: <<_::0>>)), + quote(do: @type(literal_unit_1() :: <<_::_*1>>)), + quote(do: @type(literal_size_1_unit_8() :: <<_::100, _::_*256>>)), + quote(do: @type(literal_function_arity_any() :: (... -> integer()))), + quote(do: @type(literal_function_arity_0() :: (-> integer()))), + quote(do: @type(literal_function_arity_2() :: (integer(), atom() -> integer()))), + quote(do: @type(literal_list_type() :: [integer()])), + quote(do: @type(literal_empty_list() :: [])), + quote(do: @type(literal_list_nonempty() :: [...])), + quote(do: @type(literal_nonempty_list_type() :: [atom(), ...])), + quote(do: @type(literal_keyword_list_fixed_key() :: [key: integer()])), + quote(do: @type(literal_keyword_list_fixed_key2() :: [{:key, integer()}])), + quote(do: @type(literal_keyword_list_type_key() :: [{binary(), integer()}])), + quote(do: @type(literal_empty_map() :: %{})), + quote(do: @type(literal_map_with_key() :: %{key: integer()})), + quote( + do: @type(literal_map_with_required_key() :: %{required(bitstring()) => integer()}) + ), + quote( + do: @type(literal_map_with_optional_key() :: %{optional(bitstring()) => integer()}) + ), + quote(do: @type(literal_struct_all_fields_any_type() :: %SomeStruct{})), + quote(do: @type(literal_struct_all_fields_key_type() :: %SomeStruct{key: integer()})), + quote(do: @type(literal_empty_tuple() :: {})), + quote(do: @type(literal_2_element_tuple() :: {1, 2})), + + ## Built-in types + quote(do: @type(built_in_term() :: term())), + quote(do: @type(built_in_arity() :: arity())), + quote(do: @type(built_in_as_boolean() :: as_boolean(:t))), + quote(do: @type(built_in_binary() :: binary())), + quote(do: @type(built_in_bitstring() :: bitstring())), + quote(do: @type(built_in_boolean() :: boolean())), + quote(do: @type(built_in_byte() :: byte())), + quote(do: @type(built_in_char() :: char())), + quote(do: @type(built_in_charlist() :: charlist())), + quote(do: @type(built_in_nonempty_charlist() :: nonempty_charlist())), + quote(do: @type(built_in_fun() :: fun())), + quote(do: @type(built_in_function() :: function())), + quote(do: @type(built_in_identifier() :: identifier())), + quote(do: @type(built_in_iodata() :: iodata())), + quote(do: @type(built_in_iolist() :: iolist())), + quote(do: @type(built_in_keyword() :: keyword())), + quote(do: @type(built_in_keyword_value_type() :: keyword(:t))), + quote(do: @type(built_in_list() :: list())), + quote(do: @type(built_in_nonempty_list() :: nonempty_list())), + quote(do: @type(built_in_maybe_improper_list() :: maybe_improper_list())), + quote( + do: @type(built_in_nonempty_maybe_improper_list() :: nonempty_maybe_improper_list()) + ), + quote(do: @type(built_in_mfa() :: mfa())), + quote(do: @type(built_in_module() :: module())), + quote(do: @type(built_in_no_return() :: no_return())), + quote(do: @type(built_in_node() :: node())), + quote(do: @type(built_in_number() :: number())), + quote(do: @type(built_in_struct() :: struct())), + quote(do: @type(built_in_timeout() :: timeout())), + + ## Remote types + quote(do: @type(remote_enum_t0() :: Enum.t())), + quote(do: @type(remote_keyword_t1() :: Keyword.t(integer()))) + ]) + + bytecode = + test_module do + Code.eval_quoted(quoted, [], module: __MODULE__) + end + + types = types(bytecode) + + Enum.each(Enum.zip(types, quoted), fn {{:type, type}, definition} -> + ast = Code.Typespec.type_to_quoted(type) + ast_string = Macro.to_string(quote(do: @type(unquote(ast)))) + + case type do + # These cases do not translate directly to their own string version. + {:basic_list_type, _, _} -> + assert ast_string == "@type basic_list_type() :: [integer()]" + + {:basic_nonempty_list_type, _, _} -> + assert ast_string == "@type basic_nonempty_list_type() :: [integer(), ...]" + + {:literal_empty_bitstring, _, _} -> + assert ast_string == "@type literal_empty_bitstring() :: <<_::0>>" + + {:literal_keyword_list_fixed_key, _, _} -> + assert ast_string == "@type literal_keyword_list_fixed_key() :: [{:key, integer()}]" + + {:literal_keyword_list_fixed_key2, _, _} -> + assert ast_string == "@type literal_keyword_list_fixed_key2() :: [{:key, integer()}]" + + {:literal_struct_all_fields_any_type, _, _} -> + assert ast_string == + "@type literal_struct_all_fields_any_type() :: %TypespecTest.SomeStruct{key: term()}" + + {:literal_struct_all_fields_key_type, _, _} -> + assert ast_string == + "@type literal_struct_all_fields_key_type() :: %TypespecTest.SomeStruct{key: integer()}" + + {:built_in_nonempty_list, _, _} -> + assert ast_string == "@type built_in_nonempty_list() :: [...]" + + _ -> + assert ast_string == Macro.to_string(definition) + end + end) + end + end + + describe "behaviour_info" do + defmodule SampleCallbacks do + @callback first(integer) :: integer + @callback foo(atom(), binary) :: binary + @callback bar(External.hello(), my_var :: binary) :: binary + @callback guarded(my_var) :: my_var when my_var: binary + @callback orr(atom | integer) :: atom + @callback literal(123, {atom}, :foo, [integer], true) :: atom + @macrocallback last(integer) :: Macro.t() + @macrocallback last() :: atom + @optional_callbacks bar: 2, last: 0 + @optional_callbacks first: 1 + end + + test "defines callbacks" do + expected_callbacks = [ + "MACRO-last": 1, + "MACRO-last": 2, + bar: 2, + first: 1, + foo: 2, + guarded: 1, + literal: 5, + orr: 1 + ] + + assert Enum.sort(SampleCallbacks.behaviour_info(:callbacks)) == expected_callbacks + end + + test "defines optional callbacks" do + assert Enum.sort(SampleCallbacks.behaviour_info(:optional_callbacks)) == + ["MACRO-last": 1, bar: 2, first: 1] + end + end + + @tag tmp_dir: true + test "erlang module", c do + erlc(c, :typespec_test_mod, """ + -module(typespec_test_mod). + -export([f/1]). + -export_type([t/1]). + + -type t(X) :: list(X). + + -spec f(X) -> X. + f(X) -> X. + """) + + [type: type] = types(:typespec_test_mod) + + assert Code.Typespec.type_to_quoted(type) == + {:"::", [], + [ + {:t, [], [{:x, meta(5, 9), nil}]}, + [{:x, meta(5, 20), nil}] + ]} + + [{{:f, 1}, [spec]}] = specs(:typespec_test_mod) + + assert Code.Typespec.spec_to_quoted(:f, spec) == + {:when, meta(7, 8), + [ + {:"::", meta(7, 8), + [{:f, meta(7, 8), [{:x, meta(7, 9), nil}]}, {:x, meta(7, 15), nil}]}, + [x: {:var, meta(7, 8), nil}] + ]} + end + + defp meta(line, column) do + [line: line, column: column] + end + + defp erlc(context, module, code) do + dir = context.tmp_dir + + src_path = Path.join([dir, "#{module}.erl"]) + src_path |> Path.dirname() |> File.mkdir_p!() + File.write!(src_path, code) + + ebin_dir = Path.join(dir, "ebin") + File.mkdir_p!(ebin_dir) + + {:ok, module} = + :compile.file(String.to_charlist(src_path), [ + :debug_info, + outdir: String.to_charlist(ebin_dir) + ]) + + true = Code.prepend_path(ebin_dir) + {:module, ^module} = :code.load_file(module) + + ExUnit.Callbacks.on_exit(fn -> + :code.purge(module) + :code.delete(module) + File.rm_rf!(dir) + end) + + :ok + end + + defp assert_compile_error(message, fun) do + assert ExUnit.CaptureIO.capture_io(:stderr, fn -> + assert_raise CompileError, fun + end) =~ message + end +end diff --git a/lib/elixir/test/elixir/uri_test.exs b/lib/elixir/test/elixir/uri_test.exs index 76126d74766..598d0af870c 100644 --- a/lib/elixir/test/elixir/uri_test.exs +++ b/lib/elixir/test/elixir/uri_test.exs @@ -1,202 +1,1048 @@ -Code.require_file "test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("test_helper.exs", __DIR__) defmodule URITest do use ExUnit.Case, async: true - test :encode do + doctest URI + + test "encode/1,2" do assert URI.encode("4_test.is-s~") == "4_test.is-s~" + assert URI.encode("\r\n&<%>\" ゆ", &URI.char_unreserved?/1) == - "%0D%0A%26%3C%25%3E%22%20%E3%82%86" + "%0D%0A%26%3C%25%3E%22%20%E3%82%86" end - test :encode_www_form do + test "encode_www_form/1" do assert URI.encode_www_form("4test ~1.x") == "4test+~1.x" assert URI.encode_www_form("poll:146%") == "poll%3A146%25" assert URI.encode_www_form("/\n+/ゆ") == "%2F%0A%2B%2F%E3%82%86" end - test :encode_query do + test "encode_query/1,2" do assert URI.encode_query([{:foo, :bar}, {:baz, :quux}]) == "foo=bar&baz=quux" assert URI.encode_query([{"foo", "bar"}, {"baz", "quux"}]) == "foo=bar&baz=quux" + assert URI.encode_query([{"foo z", :bar}]) == "foo+z=bar" + assert URI.encode_query([{"foo z", :bar}], :rfc3986) == "foo%20z=bar" + assert URI.encode_query([{"foo z", :bar}], :www_form) == "foo+z=bar" + + assert URI.encode_query([{"foo[]", "+=/?&# Ñ"}]) == + "foo%5B%5D=%2B%3D%2F%3F%26%23+%C3%91" + + assert URI.encode_query([{"foo[]", "+=/?&# Ñ"}], :rfc3986) == + "foo%5B%5D=%2B%3D%2F%3F%26%23%20%C3%91" + + assert URI.encode_query([{"foo[]", "+=/?&# Ñ"}], :www_form) == + "foo%5B%5D=%2B%3D%2F%3F%26%23+%C3%91" assert_raise ArgumentError, fn -> - URI.encode_query([{"foo", 'bar'}]) + URI.encode_query([{"foo", ~c"bar"}]) + end + + assert_raise ArgumentError, fn -> + URI.encode_query([{~c"foo", "bar"}]) end end - test :decode_query do - assert URI.decode_query("", []) == [] + test "decode_query/1,2,3" do assert URI.decode_query("", %{}) == %{} + assert URI.decode_query("safe=off", %{"cookie" => "foo"}) == + %{"safe" => "off", "cookie" => "foo"} + assert URI.decode_query("q=search%20query&cookie=ab%26cd&block+buster=") == - %{"block buster" => "", "cookie" => "ab&cd", "q" => "search query"} + %{"block buster" => "", "cookie" => "ab&cd", "q" => "search query"} - assert URI.decode_query("something=weird%3Dhappening") == - %{"something" => "weird=happening"} + assert URI.decode_query("q=search%20query&cookie=ab%26cd&block+buster=", %{}, :rfc3986) == + %{"block+buster" => "", "cookie" => "ab&cd", "q" => "search query"} - assert URI.decode_query("garbage") == - %{"garbage" => nil} - assert URI.decode_query("=value") == - %{"" => "value"} - assert URI.decode_query("something=weird=happening") == - %{"something" => "weird=happening"} + assert URI.decode_query("something=weird%3Dhappening") == %{"something" => "weird=happening"} + + assert URI.decode_query("=") == %{"" => ""} + assert URI.decode_query("key") == %{"key" => ""} + assert URI.decode_query("key=") == %{"key" => ""} + assert URI.decode_query("=value") == %{"" => "value"} + assert URI.decode_query("something=weird=happening") == %{"something" => "weird=happening"} end - test :decoder do - decoder = URI.query_decoder("q=search%20query&cookie=ab%26cd&block%20buster=") + test "query_decoder/1,2" do + decoder = URI.query_decoder("q=search%20query&cookie=ab%26cd&block+buster=") expected = [{"q", "search query"}, {"cookie", "ab&cd"}, {"block buster", ""}] - assert Enum.map(decoder, &(&1)) == expected + assert Enum.map(decoder, & &1) == expected + + decoder = URI.query_decoder("q=search%20query&cookie=ab%26cd&block+buster=", :rfc3986) + expected = [{"q", "search query"}, {"cookie", "ab&cd"}, {"block+buster", ""}] + assert Enum.map(decoder, & &1) == expected end - test :decode do + test "decode/1" do assert URI.decode("%0D%0A%26%3C%25%3E%22%20%E3%82%86") == "\r\n&<%>\" ゆ" assert URI.decode("%2f%41%4a%55") == "/AJU" assert URI.decode("4_t+st.is-s~") == "4_t+st.is-s~" - - assert_raise ArgumentError, ~R/malformed URI/, fn -> - URI.decode("% invalid") - end - assert_raise ArgumentError, ~R/malformed URI/, fn -> - URI.decode("invalid%") - end + assert URI.decode("% invalid") == "% invalid" + assert URI.decode("invalid %") == "invalid %" + assert URI.decode("%%") == "%%" end - test :decode_www_form do + test "decode_www_form/1" do assert URI.decode_www_form("%3Eval+ue%2B") == ">val ue+" assert URI.decode_www_form("%E3%82%86+") == "ゆ " + assert URI.decode_www_form("% invalid") == "% invalid" + assert URI.decode_www_form("invalid %") == "invalid %" + assert URI.decode_www_form("%%") == "%%" end - test :parse_uri do - assert URI.parse(uri = %URI{scheme: "http", host: "foo.com"}) == uri - end + describe "new/1" do + test "empty" do + assert URI.new("") == {:ok, %URI{}} + end - test :parse_http do - assert %URI{scheme: "http", host: "foo.com", path: "/path/to/something", - query: "foo=bar&bar=foo", fragment: "fragment", port: 80, - authority: "foo.com", userinfo: nil} == - URI.parse("http://foo.com/path/to/something?foo=bar&bar=foo#fragment") + test "errors on bad URIs" do + assert URI.new("/>") == {:error, ">"} + assert URI.new(":https") == {:error, ":"} + assert URI.new("ht\0tps://foo.com") == {:error, "\0"} + end end - test :parse_https do - assert %URI{scheme: "https", host: "foo.com", authority: "foo.com", - query: nil, fragment: nil, port: 443, path: nil, userinfo: nil} == - URI.parse("https://foo.com") - end + describe "new!/1" do + test "returns the given URI if a %URI{} struct is given" do + assert URI.new!(uri = %URI{scheme: "http", host: "foo.com"}) == uri + end - test :parse_file do - assert %URI{scheme: "file", host: nil, path: "/foo/bar/baz", userinfo: nil, - query: nil, fragment: nil, port: nil, authority: nil} == - URI.parse("file:///foo/bar/baz") - end + test "works with HTTP scheme" do + expected_uri = %URI{ + scheme: "http", + host: "foo.com", + path: "/path/to/something", + query: "foo=bar&bar=foo", + fragment: "fragment", + port: 80, + userinfo: nil + } - test :parse_ftp do - assert %URI{scheme: "ftp", host: "private.ftp-servers.example.com", - userinfo: "user001:secretpassword", authority: "user001:secretpassword@private.ftp-servers.example.com", - path: "/mydirectory/myfile.txt", query: nil, fragment: nil, - port: 21} == - URI.parse("ftp://user001:secretpassword@private.ftp-servers.example.com/mydirectory/myfile.txt") - end + assert URI.new!("http://foo.com/path/to/something?foo=bar&bar=foo#fragment") == + expected_uri + end - test :parse_sftp do - assert %URI{scheme: "sftp", host: "private.ftp-servers.example.com", - userinfo: "user001:secretpassword", authority: "user001:secretpassword@private.ftp-servers.example.com", - path: "/mydirectory/myfile.txt", query: nil, fragment: nil, port: 22} == - URI.parse("sftp://user001:secretpassword@private.ftp-servers.example.com/mydirectory/myfile.txt") - end + test "works with HTTPS scheme" do + expected_uri = %URI{ + scheme: "https", + host: "foo.com", + query: nil, + fragment: nil, + port: 443, + path: nil, + userinfo: nil + } - test :parse_tftp do - assert %URI{scheme: "tftp", host: "private.ftp-servers.example.com", - userinfo: "user001:secretpassword", authority: "user001:secretpassword@private.ftp-servers.example.com", - path: "/mydirectory/myfile.txt", query: nil, fragment: nil, port: 69} == - URI.parse("tftp://user001:secretpassword@private.ftp-servers.example.com/mydirectory/myfile.txt") - end + assert URI.new!("https://foo.com") == expected_uri + end + + test "works with file scheme" do + expected_uri = %URI{ + scheme: "file", + host: "", + path: "/foo/bar/baz", + userinfo: nil, + query: nil, + fragment: nil, + port: nil + } + + assert URI.new!("file:///foo/bar/baz") == expected_uri + end + + test "works with FTP scheme" do + expected_uri = %URI{ + scheme: "ftp", + host: "private.ftp-server.example.com", + userinfo: "user001:password", + path: "/my_directory/my_file.txt", + query: nil, + fragment: nil, + port: 21 + } + + ftp = "ftp://user001:password@private.ftp-server.example.com/my_directory/my_file.txt" + assert URI.new!(ftp) == expected_uri + end + + test "works with SFTP scheme" do + expected_uri = %URI{ + scheme: "sftp", + host: "private.ftp-server.example.com", + userinfo: "user001:password", + path: "/my_directory/my_file.txt", + query: nil, + fragment: nil, + port: 22 + } + + sftp = "sftp://user001:password@private.ftp-server.example.com/my_directory/my_file.txt" + assert URI.new!(sftp) == expected_uri + end + + test "works with TFTP scheme" do + expected_uri = %URI{ + scheme: "tftp", + host: "private.ftp-server.example.com", + userinfo: "user001:password", + path: "/my_directory/my_file.txt", + query: nil, + fragment: nil, + port: 69 + } + + tftp = "tftp://user001:password@private.ftp-server.example.com/my_directory/my_file.txt" + assert URI.new!(tftp) == expected_uri + end + + test "works with LDAP scheme" do + expected_uri = %URI{ + scheme: "ldap", + host: "", + userinfo: nil, + path: "/dc=example,dc=com", + query: "?sub?(givenName=John)", + fragment: nil, + port: 389 + } + + assert URI.new!("ldap:///dc=example,dc=com??sub?(givenName=John)") == expected_uri + + expected_uri = %URI{ + scheme: "ldap", + host: "ldap.example.com", + userinfo: nil, + path: "/cn=John%20Doe,dc=foo,dc=com", + fragment: nil, + port: 389, + query: nil + } + + assert URI.new!("ldap://ldap.example.com/cn=John%20Doe,dc=foo,dc=com") == expected_uri + end + test "can parse IPv6 addresses" do + addresses = [ + # undefined + "::", + # loopback + "::1", + # unicast + "1080::8:800:200C:417A", + # multicast + "FF01::101", + # link-local + "fe80::", + # abbreviated + "2607:f3f0:2:0:216:3cff:fef0:174a", + # mixed hex case + "2607:f3F0:2:0:216:3cFf:Fef0:174A", + # complete + "2051:0db8:2d5a:3521:8313:ffad:1242:8e2e", + # embedded IPv4 + "::00:192.168.10.184" + ] - test :parse_ldap do - assert %URI{scheme: "ldap", host: nil, authority: nil, userinfo: nil, - path: "/dc=example,dc=com", query: "?sub?(givenName=John)", - fragment: nil, port: 389} == - URI.parse("ldap:///dc=example,dc=com??sub?(givenName=John)") - assert %URI{scheme: "ldap", host: "ldap.example.com", authority: "ldap.example.com", - userinfo: nil, path: "/cn=John%20Doe,dc=example,dc=com", fragment: nil, - port: 389, query: nil} == - URI.parse("ldap://ldap.example.com/cn=John%20Doe,dc=example,dc=com") + Enum.each(addresses, fn addr -> + simple_uri = URI.new!("http://[#{addr}]/") + assert simple_uri.host == addr + + userinfo_uri = URI.new!("http://user:pass@[#{addr}]/") + assert userinfo_uri.host == addr + assert userinfo_uri.userinfo == "user:pass" + + port_uri = URI.new!("http://[#{addr}]:2222/") + assert port_uri.host == addr + assert port_uri.port == 2222 + + userinfo_port_uri = URI.new!("http://user:pass@[#{addr}]:2222/") + assert userinfo_port_uri.host == addr + assert userinfo_port_uri.userinfo == "user:pass" + assert userinfo_port_uri.port == 2222 + end) + end + + test "downcases the scheme" do + assert URI.new!("hTtP://google.com").scheme == "http" + end + + test "preserves empty fragments" do + assert URI.new!("http://example.com#").fragment == "" + assert URI.new!("http://example.com/#").fragment == "" + assert URI.new!("http://example.com/test#").fragment == "" + end + + test "preserves an empty query" do + assert URI.new!("http://foo.com/?").query == "" + end + + test "without scheme, undefined port after host translates to nil" do + assert URI.new!("//https://www.example.com") == + %URI{ + scheme: nil, + userinfo: nil, + host: "https", + port: nil, + path: "//www.example.com", + query: nil, + fragment: nil + } + end + + test "with scheme, undefined port after host translates to nil" do + assert URI.new!("myscheme://myhost:/path/info") == + %URI{ + scheme: "myscheme", + userinfo: nil, + host: "myhost", + port: nil, + path: "/path/info", + query: nil, + fragment: nil + } + end end - test :parse_splits_authority do - assert %URI{scheme: "http", host: "foo.com", path: nil, - query: nil, fragment: nil, port: 4444, - authority: "foo:bar@foo.com:4444", - userinfo: "foo:bar"} == - URI.parse("http://foo:bar@foo.com:4444") - assert %URI{scheme: "https", host: "foo.com", path: nil, - query: nil, fragment: nil, port: 443, - authority: "foo:bar@foo.com", userinfo: "foo:bar"} == - URI.parse("https://foo:bar@foo.com") - assert %URI{scheme: "http", host: "foo.com", path: nil, - query: nil, fragment: nil, port: 4444, - authority: "foo.com:4444", userinfo: nil} == - URI.parse("http://foo.com:4444") + test "http://http://http://@http://http://?http://#http://" do + assert URI.parse("http://http://http://@http://http://?http://#http://") == + %URI{ + scheme: "http", + authority: "http:", + userinfo: nil, + host: "http", + port: 80, + path: "//http://@http://http://", + query: "http://", + fragment: "http://" + } + + assert URI.new!("http://http://http://@http://http://?http://#http://") == + %URI{ + scheme: "http", + userinfo: nil, + host: "http", + port: 80, + path: "//http://@http://http://", + query: "http://", + fragment: "http://" + } end - test :default_port do + test "default_port/1,2" do assert URI.default_port("http") == 80 - assert URI.default_port("unknown") == nil + try do + URI.default_port("http", 8000) + assert URI.default_port("http") == 8000 + after + URI.default_port("http", 80) + end + + assert URI.default_port("unknown") == nil URI.default_port("unknown", 13) assert URI.default_port("unknown") == 13 end - test :parse_bad_uris do - assert URI.parse("https:??@?F?@#>F//23/") - assert URI.parse("") - assert URI.parse(":https") - assert URI.parse("https") + test "to_string/1 and Kernel.to_string/1" do + assert to_string(URI.new!("http://google.com")) == "http://google.com" + assert to_string(URI.new!("http://google.com:443")) == "http://google.com:443" + assert to_string(URI.new!("https://google.com:443")) == "https://google.com" + assert to_string(URI.new!("file:/path")) == "file:/path" + assert to_string(URI.new!("file:///path")) == "file:///path" + assert to_string(URI.new!("file://///path")) == "file://///path" + assert to_string(URI.new!("http://lol:wut@google.com")) == "http://lol:wut@google.com" + assert to_string(URI.new!("http://google.com/elixir")) == "http://google.com/elixir" + assert to_string(URI.new!("http://google.com?q=lol")) == "http://google.com?q=lol" + assert to_string(URI.new!("http://google.com?q=lol#omg")) == "http://google.com?q=lol#omg" + assert to_string(URI.new!("//google.com/elixir")) == "//google.com/elixir" + assert to_string(URI.new!("//google.com:8080/elixir")) == "//google.com:8080/elixir" + assert to_string(URI.new!("//user:password@google.com/")) == "//user:password@google.com/" + assert to_string(URI.new!("http://[2001:db8::]:8080")) == "http://[2001:db8::]:8080" + assert to_string(URI.new!("http://[2001:db8::]")) == "http://[2001:db8::]" + + assert URI.to_string(URI.new!("http://google.com")) == "http://google.com" + assert URI.to_string(URI.new!("gid:hello/123")) == "gid:hello/123" + + assert URI.to_string(URI.new!("//user:password@google.com/")) == + "//user:password@google.com/" + + assert_raise ArgumentError, + ~r":path in URI must be empty or an absolute path if URL has a :host", + fn -> %URI{host: "foo.com", path: "hello/123"} |> URI.to_string() end + end + + describe "merge/2" do + test "with valid paths" do + assert URI.merge("http://google.com/foo", "http://example.com/baz") + |> to_string() == "http://example.com/baz" + + assert URI.merge("http://google.com/foo", "http://example.com/.././bar/../../baz") + |> to_string() == "http://example.com/baz" + + assert URI.merge("http://google.com/foo", "//example.com/baz") + |> to_string() == "http://example.com/baz" + + assert URI.merge("http://google.com/foo", URI.new!("//example.com/baz")) + |> to_string() == "http://example.com/baz" + + assert URI.merge("http://google.com/foo", "//example.com/.././bar/../../../baz") + |> to_string() == "http://example.com/baz" + + assert URI.merge("http://example.com", URI.new!("/foo")) + |> to_string() == "http://example.com/foo" + + assert URI.merge("http://example.com", URI.new!("/.././bar/../../../baz")) + |> to_string() == "http://example.com/baz" + + base = URI.new!("http://example.com/foo/bar") + assert URI.merge(base, "") |> to_string() == "http://example.com/foo/bar" + assert URI.merge(base, "#fragment") |> to_string() == "http://example.com/foo/bar#fragment" + assert URI.merge(base, "?query") |> to_string() == "http://example.com/foo/bar?query" + assert URI.merge(base, %URI{}) |> to_string() == "http://example.com/foo/bar" + + assert URI.merge(base, %URI{fragment: "fragment"}) + |> to_string() == "http://example.com/foo/bar#fragment" + + base = URI.new!("http://example.com") + assert URI.merge(base, "/foo") |> to_string() == "http://example.com/foo" + assert URI.merge(base, "foo") |> to_string() == "http://example.com/foo" + + base = URI.new!("http://example.com/foo/bar") + assert URI.merge(base, "/baz") |> to_string() == "http://example.com/baz" + assert URI.merge(base, "baz") |> to_string() == "http://example.com/foo/baz" + assert URI.merge(base, "../baz") |> to_string() == "http://example.com/baz" + assert URI.merge(base, ".././baz") |> to_string() == "http://example.com/baz" + assert URI.merge(base, "./baz") |> to_string() == "http://example.com/foo/baz" + assert URI.merge(base, "bar/./baz") |> to_string() == "http://example.com/foo/bar/baz" + + base = URI.new!("http://example.com/foo/bar/") + assert URI.merge(base, "/baz") |> to_string() == "http://example.com/baz" + assert URI.merge(base, "baz") |> to_string() == "http://example.com/foo/bar/baz" + assert URI.merge(base, "../baz") |> to_string() == "http://example.com/foo/baz" + assert URI.merge(base, ".././baz") |> to_string() == "http://example.com/foo/baz" + assert URI.merge(base, "./baz") |> to_string() == "http://example.com/foo/bar/baz" + assert URI.merge(base, "bar/./baz") |> to_string() == "http://example.com/foo/bar/bar/baz" + + base = URI.new!("http://example.com/foo/bar/baz") + assert URI.merge(base, "../../foobar") |> to_string() == "http://example.com/foobar" + assert URI.merge(base, "../../../foobar") |> to_string() == "http://example.com/foobar" + + assert URI.merge(base, "../../../../../../foobar") |> to_string() == + "http://example.com/foobar" + + base = URI.new!("http://example.com/foo/../bar") + assert URI.merge(base, "baz") |> to_string() == "http://example.com/baz" + + base = URI.new!("http://example.com/foo/./bar") + assert URI.merge(base, "baz") |> to_string() == "http://example.com/foo/baz" + + base = URI.new!("http://example.com/foo?query1") + assert URI.merge(base, "?query2") |> to_string() == "http://example.com/foo?query2" + assert URI.merge(base, "") |> to_string() == "http://example.com/foo?query1" + + base = URI.new!("http://example.com/foo#fragment1") + assert URI.merge(base, "#fragment2") |> to_string() == "http://example.com/foo#fragment2" + assert URI.merge(base, "") |> to_string() == "http://example.com/foo" + + page_url = "https://example.com/guide/" + image_url = "https://images.example.com/t/1600x/https://images.example.com/foo.jpg" + + assert URI.merge(URI.new!(page_url), URI.new!(image_url)) |> to_string() == + "https://images.example.com/t/1600x/https://images.example.com/foo.jpg" + end + + test "error on relative base" do + assert_raise ArgumentError, "you must merge onto an absolute URI", fn -> + URI.merge("/relative", "") + end + end + + test "base without host" do + assert URI.merge("tag:example", "foo") |> to_string == "tag:foo" + assert URI.merge("tag:example", "#fragment") |> to_string == "tag:example#fragment" + end + + test "base without host and path" do + assert URI.merge("ex:", "test") |> to_string() == "ex:test" + assert URI.merge("ex:", "a/b/c") |> to_string() == "ex:a/b/c" + assert URI.merge("ex:", "a/b/./../c") |> to_string() == "ex:a/c" + assert URI.merge("ex:", "test?query=value") |> to_string() == "ex:test?query=value" + assert URI.merge("mailto:", "user@example.com") |> to_string() == "mailto:user@example.com" + assert URI.merge("urn:isbn", "0451450523") |> to_string() == "urn:0451450523" + end + + test "with RFC examples" do + # These are examples from: + # + # https://www.rfc-editor.org/rfc/rfc3986#section-5.4.1 + # https://www.rfc-editor.org/rfc/rfc3986#section-5.4.2 + # + # They are taken verbatim from the above document for easy comparison + + base = "http://a/b/c/d;p?q" + + rel_and_result = %{ + "g:h" => "g:h", + "g" => "http://a/b/c/g", + "./g" => "http://a/b/c/g", + "g/" => "http://a/b/c/g/", + "/g" => "http://a/g", + "//g" => "http://g", + "?y" => "http://a/b/c/d;p?y", + "g?y" => "http://a/b/c/g?y", + "#s" => "http://a/b/c/d;p?q#s", + "g#s" => "http://a/b/c/g#s", + "g?y#s" => "http://a/b/c/g?y#s", + ";x" => "http://a/b/c/;x", + "g;x" => "http://a/b/c/g;x", + "g;x?y#s" => "http://a/b/c/g;x?y#s", + "" => "http://a/b/c/d;p?q", + "." => "http://a/b/c/", + "./" => "http://a/b/c/", + ".." => "http://a/b/", + "../" => "http://a/b/", + "../g" => "http://a/b/g", + "../.." => "http://a/", + "../../" => "http://a/", + "../../g" => "http://a/g", + "../../../g" => "http://a/g", + "../../../../g" => "http://a/g", + "/./g" => "http://a/g", + "/../g" => "http://a/g", + "g." => "http://a/b/c/g.", + ".g" => "http://a/b/c/.g", + "g.." => "http://a/b/c/g..", + "..g" => "http://a/b/c/..g", + "./../g" => "http://a/b/g", + "./g/." => "http://a/b/c/g/", + "g/./h" => "http://a/b/c/g/h", + "g/../h" => "http://a/b/c/h", + "g;x=1/./y" => "http://a/b/c/g;x=1/y", + "g;x=1/../y" => "http://a/b/c/y", + "g?y/./x" => "http://a/b/c/g?y/./x", + "g?y/../x" => "http://a/b/c/g?y/../x", + "g#s/./x" => "http://a/b/c/g#s/./x", + "g#s/../x" => "http://a/b/c/g#s/../x", + "http:g" => "http:g" + } + + for {rel, result} <- rel_and_result do + assert URI.merge(base, rel) |> URI.to_string() == result + end + end + + test "with W3C examples" do + # These examples are from the W3C JSON-LD test suite: + # + # https://w3c.github.io/json-ld-api/tests/toRdf-manifest#t0124 + # https://w3c.github.io/json-ld-api/tests/toRdf-manifest#t0125 + # https://w3c.github.io/json-ld-api/tests/toRdf-manifest#t0123 + + base1 = "http://a/bb/ccc/." + + rel_and_result1 = %{ + "g:h" => "g:h", + "g" => "http://a/bb/ccc/g", + "./g" => "http://a/bb/ccc/g", + "g/" => "http://a/bb/ccc/g/", + "/g" => "http://a/g", + "//g" => "http://g", + "?y" => "http://a/bb/ccc/.?y", + "g?y" => "http://a/bb/ccc/g?y", + "#s" => "http://a/bb/ccc/.#s", + "g#s" => "http://a/bb/ccc/g#s", + "g?y#s" => "http://a/bb/ccc/g?y#s", + ";x" => "http://a/bb/ccc/;x", + "g;x" => "http://a/bb/ccc/g;x", + "g;x?y#s" => "http://a/bb/ccc/g;x?y#s", + "" => "http://a/bb/ccc/.", + "." => "http://a/bb/ccc/", + "./" => "http://a/bb/ccc/", + ".." => "http://a/bb/", + "../" => "http://a/bb/", + "../g" => "http://a/bb/g", + "../.." => "http://a/", + "../../" => "http://a/", + "../../g" => "http://a/g", + "../../../g" => "http://a/g", + "../../../../g" => "http://a/g", + "/./g" => "http://a/g", + "/../g" => "http://a/g", + "g." => "http://a/bb/ccc/g.", + ".g" => "http://a/bb/ccc/.g", + "g.." => "http://a/bb/ccc/g..", + "..g" => "http://a/bb/ccc/..g", + "./../g" => "http://a/bb/g", + "./g/." => "http://a/bb/ccc/g/", + "g/./h" => "http://a/bb/ccc/g/h", + "g/../h" => "http://a/bb/ccc/h", + "g;x=1/./y" => "http://a/bb/ccc/g;x=1/y", + "g;x=1/../y" => "http://a/bb/ccc/y", + "g?y/./x" => "http://a/bb/ccc/g?y/./x", + "g?y/../x" => "http://a/bb/ccc/g?y/../x", + "g#s/./x" => "http://a/bb/ccc/g#s/./x", + "g#s/../x" => "http://a/bb/ccc/g#s/../x", + "http:g" => "http:g" + } + + for {rel, result} <- rel_and_result1 do + assert URI.merge(base1, rel) |> URI.to_string() == result + end + + base2 = "http://a/bb/ccc/.." + + rel_and_result2 = %{ + "g:h" => "g:h", + "g" => "http://a/bb/ccc/g", + "./g" => "http://a/bb/ccc/g", + "g/" => "http://a/bb/ccc/g/", + "/g" => "http://a/g", + "//g" => "http://g", + "?y" => "http://a/bb/ccc/..?y", + "g?y" => "http://a/bb/ccc/g?y", + "#s" => "http://a/bb/ccc/..#s", + "g#s" => "http://a/bb/ccc/g#s", + "g?y#s" => "http://a/bb/ccc/g?y#s", + ";x" => "http://a/bb/ccc/;x", + "g;x" => "http://a/bb/ccc/g;x", + "g;x?y#s" => "http://a/bb/ccc/g;x?y#s", + "" => "http://a/bb/ccc/..", + "." => "http://a/bb/ccc/", + "./" => "http://a/bb/ccc/", + ".." => "http://a/bb/", + "../" => "http://a/bb/", + "../g" => "http://a/bb/g", + "../.." => "http://a/", + "../../" => "http://a/", + "../../g" => "http://a/g", + "../../../g" => "http://a/g", + "../../../../g" => "http://a/g", + "/./g" => "http://a/g", + "/../g" => "http://a/g", + "g." => "http://a/bb/ccc/g.", + ".g" => "http://a/bb/ccc/.g", + "g.." => "http://a/bb/ccc/g..", + "..g" => "http://a/bb/ccc/..g", + "./../g" => "http://a/bb/g", + "./g/." => "http://a/bb/ccc/g/", + "g/./h" => "http://a/bb/ccc/g/h", + "g/../h" => "http://a/bb/ccc/h", + "g;x=1/./y" => "http://a/bb/ccc/g;x=1/y", + "g;x=1/../y" => "http://a/bb/ccc/y", + "g?y/./x" => "http://a/bb/ccc/g?y/./x", + "g?y/../x" => "http://a/bb/ccc/g?y/../x", + "g#s/./x" => "http://a/bb/ccc/g#s/./x", + "g#s/../x" => "http://a/bb/ccc/g#s/../x", + "http:g" => "http:g" + } + + for {rel, result} <- rel_and_result2 do + assert URI.merge(base2, rel) |> URI.to_string() == result + end + + base3 = "http://a/bb/ccc/../d;p?q" + + rel_and_result = %{ + "g:h" => "g:h", + "g" => "http://a/bb/g", + "./g" => "http://a/bb/g", + "g/" => "http://a/bb/g/", + "/g" => "http://a/g", + "//g" => "http://g", + "?y" => "http://a/bb/ccc/../d;p?y", + "g?y" => "http://a/bb/g?y", + "#s" => "http://a/bb/ccc/../d;p?q#s", + "g#s" => "http://a/bb/g#s", + "g?y#s" => "http://a/bb/g?y#s", + ";x" => "http://a/bb/;x", + "g;x" => "http://a/bb/g;x", + "g;x?y#s" => "http://a/bb/g;x?y#s", + "" => "http://a/bb/ccc/../d;p?q", + "." => "http://a/bb/", + "./" => "http://a/bb/", + ".." => "http://a/", + "../" => "http://a/", + "../g" => "http://a/g", + "../.." => "http://a/", + "../../" => "http://a/", + "../../g" => "http://a/g", + "../../../g" => "http://a/g", + "../../../../g" => "http://a/g", + "/./g" => "http://a/g", + "/../g" => "http://a/g", + "g." => "http://a/bb/g.", + ".g" => "http://a/bb/.g", + "g.." => "http://a/bb/g..", + "..g" => "http://a/bb/..g", + "./../g" => "http://a/g", + "./g/." => "http://a/bb/g/", + "g/./h" => "http://a/bb/g/h", + "g/../h" => "http://a/bb/h", + "g;x=1/./y" => "http://a/bb/g;x=1/y", + "g;x=1/../y" => "http://a/bb/y", + "g?y/./x" => "http://a/bb/g?y/./x", + "g?y/../x" => "http://a/bb/g?y/../x", + "g#s/./x" => "http://a/bb/g#s/./x", + "g#s/../x" => "http://a/bb/g#s/../x", + "http:g" => "http:g" + } + + for {rel, result} <- rel_and_result do + assert URI.merge(base3, rel) |> URI.to_string() == result + end + end + end + + test "append_query/2" do + assert URI.append_query(URI.parse("http://example.com/?x=1"), "x=2").query == "x=1&x=2" + assert URI.append_query(URI.parse("http://example.com/?x=1&"), "x=2").query == "x=1&x=2" end - test :ipv6_addresses do - addrs = [ - "::", # undefined - "::1", # loopback - "1080::8:800:200C:417A", # unicast - "FF01::101", # multicast - "2607:f3f0:2:0:216:3cff:fef0:174a", # abbreviated - "2607:f3F0:2:0:216:3cFf:Fef0:174A", # mixed hex case - "2051:0db8:2d5a:3521:8313:ffad:1242:8e2e", # complete - "::00:192.168.10.184" # embedded IPv4 - ] + describe "append_path/2" do + test "with valid paths" do + examples = [ + {"http://example.com", "/", "http://example.com/"}, + {"http://example.com/", "/foo", "http://example.com/foo"}, + {"http://example.com/foo", "/bar", "http://example.com/foo/bar"}, + {"http://example.com/foo", "/bar/", "http://example.com/foo/bar/"}, + {"http://example.com/foo", "/bar/baz", "http://example.com/foo/bar/baz"}, + {"http://example.com/foo?var=1", "/bar/", "http://example.com/foo/bar/?var=1"}, + {"https://example.com/page/", "/urn:example:page", + "https://example.com/page/urn:example:page"} + ] - Enum.each addrs, fn(addr) -> - simple_uri = URI.parse("http://[#{addr}]/") - assert simple_uri.host == addr + for {base_url, path, expected_result} <- examples do + result = + base_url + |> URI.parse() + |> URI.append_path(path) + |> URI.to_string() - userinfo_uri = URI.parse("http://user:pass@[#{addr}]/") - assert userinfo_uri.host == addr - assert userinfo_uri.userinfo == "user:pass" + assert result == expected_result, """ + Path did not append as expected - port_uri = URI.parse("http://[#{addr}]:2222/") - assert port_uri.host == addr - assert port_uri.port == 2222 + base_url: #{inspect(base_url)} + path: #{inspect(path)} - userinfo_port_uri = URI.parse("http://user:pass@[#{addr}]:2222/") - assert userinfo_port_uri.host == addr - assert userinfo_port_uri.userinfo == "user:pass" - assert userinfo_port_uri.port == 2222 + result: #{inspect(result)} + expected_result: #{inspect(expected_result)} + """ + end + end + + test "errors on invalid paths" do + base_uri = URI.parse("http://example.com") + + assert_raise ArgumentError, + ~S|path must start with "/", got: "foo"|, + fn -> + URI.append_path(base_uri, "foo") + end + + assert_raise ArgumentError, + ~S|path cannot start with "//", got: "//foo"|, + fn -> + URI.append_path(base_uri, "//foo") + end end end - test :downcase_scheme do - assert URI.parse("hTtP://google.com").scheme == "http" + ## Deprecate API + + describe "authority" do + test "to_string" do + assert URI.to_string(%URI{authority: "foo@example.com:80"}) == + "//foo@example.com:80" + + assert URI.to_string(%URI{userinfo: "bar", host: "example.org", port: 81}) == + "//bar@example.org:81" + + assert URI.to_string(%URI{ + authority: "foo@example.com:80", + userinfo: "bar", + host: "example.org", + port: 81 + }) == + "//bar@example.org:81" + end end - test :to_string do - assert to_string(URI.parse("http://google.com")) == "http://google.com" - assert to_string(URI.parse("http://google.com:443")) == "http://google.com:443" - assert to_string(URI.parse("https://google.com:443")) == "https://google.com" - assert to_string(URI.parse("http://lol:wut@google.com")) == "http://lol:wut@google.com" - assert to_string(URI.parse("http://google.com/elixir")) == "http://google.com/elixir" - assert to_string(URI.parse("http://google.com?q=lol")) == "http://google.com?q=lol" - assert to_string(URI.parse("http://google.com?q=lol#omg")) == "http://google.com?q=lol#omg" + describe "parse/1" do + test "returns the given URI if a %URI{} struct is given" do + assert URI.parse(uri = %URI{scheme: "http", host: "foo.com"}) == uri + end + + test "works with HTTP scheme" do + expected_uri = %URI{ + scheme: "http", + host: "foo.com", + path: "/path/to/something", + query: "foo=bar&bar=foo", + fragment: "fragment", + port: 80, + authority: "foo.com", + userinfo: nil + } + + assert URI.parse("http://foo.com/path/to/something?foo=bar&bar=foo#fragment") == + expected_uri + end + + test "works with HTTPS scheme" do + expected_uri = %URI{ + scheme: "https", + host: "foo.com", + authority: "foo.com", + query: nil, + fragment: nil, + port: 443, + path: nil, + userinfo: nil + } + + assert URI.parse("https://foo.com") == expected_uri + end + + test "works with \"file\" scheme" do + expected_uri = %URI{ + scheme: "file", + host: "", + path: "/foo/bar/baz", + userinfo: nil, + query: nil, + fragment: nil, + port: nil, + authority: "" + } + + assert URI.parse("file:///foo/bar/baz") == expected_uri + end + + test "works with FTP scheme" do + expected_uri = %URI{ + scheme: "ftp", + host: "private.ftp-server.example.com", + userinfo: "user001:password", + authority: "user001:password@private.ftp-server.example.com", + path: "/my_directory/my_file.txt", + query: nil, + fragment: nil, + port: 21 + } + + ftp = "ftp://user001:password@private.ftp-server.example.com/my_directory/my_file.txt" + assert URI.parse(ftp) == expected_uri + end + + test "works with SFTP scheme" do + expected_uri = %URI{ + scheme: "sftp", + host: "private.ftp-server.example.com", + userinfo: "user001:password", + authority: "user001:password@private.ftp-server.example.com", + path: "/my_directory/my_file.txt", + query: nil, + fragment: nil, + port: 22 + } + + sftp = "sftp://user001:password@private.ftp-server.example.com/my_directory/my_file.txt" + assert URI.parse(sftp) == expected_uri + end + + test "works with TFTP scheme" do + expected_uri = %URI{ + scheme: "tftp", + host: "private.ftp-server.example.com", + userinfo: "user001:password", + authority: "user001:password@private.ftp-server.example.com", + path: "/my_directory/my_file.txt", + query: nil, + fragment: nil, + port: 69 + } + + tftp = "tftp://user001:password@private.ftp-server.example.com/my_directory/my_file.txt" + assert URI.parse(tftp) == expected_uri + end + + test "works with LDAP scheme" do + expected_uri = %URI{ + scheme: "ldap", + host: "", + authority: "", + userinfo: nil, + path: "/dc=example,dc=com", + query: "?sub?(givenName=John)", + fragment: nil, + port: 389 + } + + assert URI.parse("ldap:///dc=example,dc=com??sub?(givenName=John)") == expected_uri + + expected_uri = %URI{ + scheme: "ldap", + host: "ldap.example.com", + authority: "ldap.example.com", + userinfo: nil, + path: "/cn=John%20Doe,dc=foo,dc=com", + fragment: nil, + port: 389, + query: nil + } + + assert URI.parse("ldap://ldap.example.com/cn=John%20Doe,dc=foo,dc=com") == expected_uri + end + + test "works with WebSocket scheme" do + expected_uri = %URI{ + authority: "ws.example.com", + fragment: "content", + host: "ws.example.com", + path: "/path/to", + port: 80, + query: "here", + scheme: "ws", + userinfo: nil + } + + assert URI.parse("ws://ws.example.com/path/to?here#content") == expected_uri + end + + test "works with WebSocket Secure scheme" do + expected_uri = %URI{ + authority: "ws.example.com", + fragment: "content", + host: "ws.example.com", + path: "/path/to", + port: 443, + query: "here", + scheme: "wss", + userinfo: nil + } + + assert URI.parse("wss://ws.example.com/path/to?here#content") == expected_uri + end + + test "splits authority" do + expected_uri = %URI{ + scheme: "http", + host: "foo.com", + path: nil, + query: nil, + fragment: nil, + port: 4444, + authority: "foo:bar@foo.com:4444", + userinfo: "foo:bar" + } + + assert URI.parse("http://foo:bar@foo.com:4444") == expected_uri + + expected_uri = %URI{ + scheme: "https", + host: "foo.com", + path: nil, + query: nil, + fragment: nil, + port: 443, + authority: "foo:bar@foo.com", + userinfo: "foo:bar" + } + + assert URI.parse("https://foo:bar@foo.com") == expected_uri + + expected_uri = %URI{ + scheme: "http", + host: "foo.com", + path: nil, + query: nil, + fragment: nil, + port: 4444, + authority: "foo.com:4444", + userinfo: nil + } + + assert URI.parse("http://foo.com:4444") == expected_uri + end + + test "can parse bad URIs" do + assert URI.parse("") + assert URI.parse("https:??@?F?@#>F//23/") + + assert URI.parse(":https").path == ":https" + assert URI.parse("https").path == "https" + assert URI.parse("ht\0tps://foo.com").path == "ht\0tps://foo.com" + end + + test "can parse IPv6 addresses" do + addresses = [ + # undefined + "::", + # loopback + "::1", + # unicast + "1080::8:800:200C:417A", + # multicast + "FF01::101", + # link-local + "fe80::", + # abbreviated + "2607:f3f0:2:0:216:3cff:fef0:174a", + # mixed hex case + "2607:f3F0:2:0:216:3cFf:Fef0:174A", + # complete + "2051:0db8:2d5a:3521:8313:ffad:1242:8e2e", + # embedded IPv4 + "::00:192.168.10.184" + ] + + Enum.each(addresses, fn addr -> + simple_uri = URI.parse("http://[#{addr}]/") + assert simple_uri.authority == "[#{addr}]" + assert simple_uri.host == addr + + userinfo_uri = URI.parse("http://user:pass@[#{addr}]/") + assert userinfo_uri.authority == "user:pass@[#{addr}]" + assert userinfo_uri.host == addr + assert userinfo_uri.userinfo == "user:pass" + + port_uri = URI.parse("http://[#{addr}]:2222/") + assert port_uri.authority == "[#{addr}]:2222" + assert port_uri.host == addr + assert port_uri.port == 2222 + + userinfo_port_uri = URI.parse("http://user:pass@[#{addr}]:2222/") + assert userinfo_port_uri.authority == "user:pass@[#{addr}]:2222" + assert userinfo_port_uri.host == addr + assert userinfo_port_uri.userinfo == "user:pass" + assert userinfo_port_uri.port == 2222 + end) + end + + test "downcases the scheme" do + assert URI.parse("hTtP://google.com").scheme == "http" + end + + test "preserves empty fragments" do + assert URI.parse("http://example.com#").fragment == "" + assert URI.parse("http://example.com/#").fragment == "" + assert URI.parse("http://example.com/test#").fragment == "" + end + + test "preserves an empty query" do + assert URI.parse("http://foo.com/?").query == "" + end + + test "merges empty path" do + base = URI.parse("http://example.com") + assert URI.merge(base, "/foo") |> to_string() == "http://example.com/foo" + assert URI.merge(base, "foo") |> to_string() == "http://example.com/foo" + end end end diff --git a/lib/elixir/test/elixir/version_test.exs b/lib/elixir/test/elixir/version_test.exs index 04c96b5d08f..208f625f226 100644 --- a/lib/elixir/test/elixir/version_test.exs +++ b/lib/elixir/test/elixir/version_test.exs @@ -1,199 +1,348 @@ -Code.require_file "test_helper.exs", __DIR__ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team +# SPDX-FileCopyrightText: 2012 Plataformatec + +Code.require_file("test_helper.exs", __DIR__) defmodule VersionTest do - use ExUnit.Case, async: true - alias Version.Parser, as: P - alias Version, as: V - - test "compare" do - assert :gt == V.compare("1.0.1", "1.0.0") - assert :gt == V.compare("1.1.0", "1.0.1") - assert :gt == V.compare("2.1.1", "1.2.2") - assert :gt == V.compare("1.0.0", "1.0.0-dev") - assert :gt == V.compare("1.2.3-dev", "0.1.2") - assert :gt == V.compare("1.0.0-a.b", "1.0.0-a") - assert :gt == V.compare("1.0.0-b", "1.0.0-a.b") - assert :gt == V.compare("1.0.0-a", "1.0.0-0") - assert :gt == V.compare("1.0.0-a.b", "1.0.0-a.a") - - assert :lt == V.compare("1.0.0", "1.0.1") - assert :lt == V.compare("1.0.1", "1.1.0") - assert :lt == V.compare("1.2.2", "2.1.1") - assert :lt == V.compare("1.0.0-dev", "1.0.0") - assert :lt == V.compare("0.1.2", "1.2.3-dev") - assert :lt == V.compare("1.0.0-a", "1.0.0-a.b") - assert :lt == V.compare("1.0.0-a.b", "1.0.0-b") - assert :lt == V.compare("1.0.0-0", "1.0.0-a") - assert :lt == V.compare("1.0.0-a.a", "1.0.0-a.b") - - assert :eq == V.compare("1.0.0", "1.0.0") - assert :eq == V.compare("1.0.0-dev", "1.0.0-dev") - assert :eq == V.compare("1.0.0-a", "1.0.0-a") - end - - test "invalid compare" do - assert_raise V.InvalidVersionError, fn -> - V.compare("1.0", "1.0.0") + use ExUnit.Case, async: true + + doctest Version + + alias Version.Parser + + test "compare/2 with valid versions" do + assert Version.compare("1.0.1", "1.0.0") == :gt + assert Version.compare("1.1.0", "1.0.1") == :gt + assert Version.compare("2.1.1", "1.2.2") == :gt + assert Version.compare("1.0.0", "1.0.0-dev") == :gt + assert Version.compare("1.2.3-dev", "0.1.2") == :gt + assert Version.compare("1.0.0-a.b", "1.0.0-a") == :gt + assert Version.compare("1.0.0-b", "1.0.0-a.b") == :gt + assert Version.compare("1.0.0-a", "1.0.0-0") == :gt + assert Version.compare("1.0.0-a.b", "1.0.0-a.a") == :gt + + assert Version.compare("1.0.0", "1.0.1") == :lt + assert Version.compare("1.0.1", "1.1.0") == :lt + assert Version.compare("1.2.2", "2.1.1") == :lt + assert Version.compare("1.0.0-dev", "1.0.0") == :lt + assert Version.compare("0.1.2", "1.2.3-dev") == :lt + assert Version.compare("1.0.0-a", "1.0.0-a.b") == :lt + assert Version.compare("1.0.0-a.b", "1.0.0-b") == :lt + assert Version.compare("1.0.0-0", "1.0.0-a") == :lt + assert Version.compare("1.0.0-a.a", "1.0.0-a.b") == :lt + + assert Version.compare("1.0.0", "1.0.0") == :eq + assert Version.compare("1.0.0-dev", "1.0.0-dev") == :eq + assert Version.compare("1.0.0-a", "1.0.0-a") == :eq + assert Version.compare("1.5.0-rc.0", "1.5.0-rc0") == :lt + end + + test "compare/2 with invalid versions" do + assert_raise Version.InvalidVersionError, fn -> + Version.compare("1.0", "1.0.0") end - assert_raise V.InvalidVersionError, fn -> - V.compare("1.0.0-dev", "1.0") + assert_raise Version.InvalidVersionError, fn -> + Version.compare("1.0.0-dev", "1.0") end - assert_raise V.InvalidVersionError, fn -> - V.compare("foo", "1.0.0-a") + assert_raise Version.InvalidVersionError, fn -> + Version.compare("foo", "1.0.0-a") end end test "lexes specifications properly" do - assert P.lexer("== != > >= < <= ~>", []) == [:'==', :'!=', :'>', :'>=', :'<', :'<=', :'~>'] - assert P.lexer("2.3.0", []) == [:'==', "2.3.0"] - assert P.lexer("!2.3.0", []) == [:'!=', "2.3.0"] - assert P.lexer(">>=", []) == [:'>', :'>='] - assert P.lexer(">2.4.0", []) == [:'>', "2.4.0"] - assert P.lexer(" > 2.4.0", []) == [:'>', "2.4.0"] + assert Parser.lexer("== > >= < <= ~>") |> Enum.reverse() == [:==, :>, :>=, :<, :<=, :~>] + assert Parser.lexer("2.3.0") |> Enum.reverse() == [:==, "2.3.0"] + assert Parser.lexer(">>=") |> Enum.reverse() == [:>, :>=] + assert Parser.lexer(">2.4.0") |> Enum.reverse() == [:>, "2.4.0"] + assert Parser.lexer("> 2.4.0") |> Enum.reverse() == [:>, "2.4.0"] + assert Parser.lexer(" > 2.4.0") |> Enum.reverse() == [:>, "2.4.0"] + assert Parser.lexer(" or 2.1.0") |> Enum.reverse() == [:or, :==, "2.1.0"] + assert Parser.lexer(" and 2.1.0") |> Enum.reverse() == [:and, :==, "2.1.0"] + + assert Parser.lexer(">= 2.0.0 and < 2.1.0") |> Enum.reverse() == + [:>=, "2.0.0", :and, :<, "2.1.0"] + + assert Parser.lexer(">= 2.0.0 or < 2.1.0") |> Enum.reverse() == + [:>=, "2.0.0", :or, :<, "2.1.0"] end - test "parse" do - assert {:ok, %V{major: 1, minor: 2, patch: 3}} = V.parse("1.2.3") - assert {:ok, %V{major: 1, minor: 4, patch: 5}} = V.parse("1.4.5+ignore") - assert {:ok, %V{major: 1, minor: 4, patch: 5, pre: ["6-g3318bd5"]}} = V.parse("1.4.5-6-g3318bd5") - assert {:ok, %V{major: 1, minor: 4, patch: 5, pre: [6, 7, "eight"]}} = V.parse("1.4.5-6.7.eight") - assert {:ok, %V{major: 1, minor: 4, patch: 5, pre: ["6-g3318bd5"]}} = V.parse("1.4.5-6-g3318bd5+ignore") + test "parse/1" do + assert {:ok, %Version{major: 1, minor: 2, patch: 3}} = Version.parse("1.2.3") + + assert {:ok, %Version{major: 1, minor: 4, patch: 5, build: "ignore"}} = + Version.parse("1.4.5+ignore") + + assert {:ok, %Version{major: 0, minor: 0, patch: 1, build: "sha.0702245"}} = + Version.parse("0.0.1+sha.0702245") + + assert {:ok, %Version{major: 1, minor: 4, patch: 5, pre: ["6-g3318bd5"]}} = + Version.parse("1.4.5-6-g3318bd5") + + assert {:ok, %Version{major: 1, minor: 4, patch: 5, pre: [6, 7, "eight"]}} = + Version.parse("1.4.5-6.7.eight") + + assert {:ok, %Version{major: 1, minor: 4, patch: 5, pre: ["6-g3318bd5"]}} = + Version.parse("1.4.5-6-g3318bd5+ignore") + + assert Version.parse("foobar") == :error + assert Version.parse("2") == :error + assert Version.parse("2.") == :error + assert Version.parse("2.3") == :error + assert Version.parse("2.3.") == :error + assert Version.parse("2.3.0-") == :error + assert Version.parse("2.3.0+") == :error + assert Version.parse("2.3.0.") == :error + assert Version.parse("2.3.0.4") == :error + assert Version.parse("2.3.-rc.1") == :error + assert Version.parse("2.3.+rc.1") == :error + assert Version.parse("2.3.0-01") == :error + assert Version.parse("2.3.00-1") == :error + assert Version.parse("2.3.00") == :error + assert Version.parse("2.03.0") == :error + assert Version.parse("02.3.0") == :error + assert Version.parse("0. 0.0") == :error + assert Version.parse("0.1.0-&&pre") == :error + end - assert :error = V.parse("foobar") - assert :error = V.parse("2.3") - assert :error = V.parse("2") - assert :error = V.parse("2.3.0-01") + test "to_string/1" do + assert Version.parse!("1.0.0") |> Version.to_string() == "1.0.0" + assert Version.parse!("1.0.0-dev") |> Version.to_string() == "1.0.0-dev" + assert Version.parse!("1.0.0+lol") |> Version.to_string() == "1.0.0+lol" + assert Version.parse!("1.0.0-dev+lol") |> Version.to_string() == "1.0.0-dev+lol" + assert Version.parse!("1.0.0-dev+lol.4") |> Version.to_string() == "1.0.0-dev+lol.4" + assert Version.parse!("1.0.0-0") |> Version.to_string() == "1.0.0-0" + assert Version.parse!("1.0.0-rc.0") |> Version.to_string() == "1.0.0-rc.0" + assert %Version{major: 1, minor: 0, patch: 0} |> Version.to_string() == "1.0.0" end - test "to_string" do - assert V.parse("1.0.0") |> elem(1) |> to_string == "1.0.0" - assert V.parse("1.0.0-dev") |> elem(1) |> to_string == "1.0.0-dev" - assert V.parse("1.0.0+lol") |> elem(1) |> to_string == "1.0.0+lol" - assert V.parse("1.0.0-dev+lol") |> elem(1) |> to_string == "1.0.0-dev+lol" + test "to_string/1 via protocol" do + assert Version.parse!("1.0.0") |> to_string() == "1.0.0" end - test "invalid match" do - assert_raise V.InvalidVersionError, fn -> - V.match?("foo", "2.3.0") + test "inspect/1" do + assert Version.parse!("1.0.0") |> inspect() == "%Version{major: 1, minor: 0, patch: 0}" + end + + test "match?/2 with invalid versions" do + assert_raise Version.InvalidVersionError, fn -> + Version.match?("foo", "2.3.0") end - assert_raise V.InvalidVersionError, fn -> - V.match?("2.3", "2.3.0") + assert_raise Version.InvalidVersionError, fn -> + Version.match?("2.3", "2.3.0") end - assert_raise V.InvalidRequirementError, fn -> - V.match?("2.3.0", "foo") + assert_raise Version.InvalidRequirementError, fn -> + Version.match?("2.3.0", "foo") end - assert_raise V.InvalidRequirementError, fn -> - V.match?("2.3.0", "2.3") + assert_raise Version.InvalidRequirementError, fn -> + Version.match?("2.3.0", "2.3") end end test "==" do - assert V.match?("2.3.0", "2.3.0") - refute V.match?("2.4.0", "2.3.0") + assert Version.match?("2.3.0", "2.3.0") + refute Version.match?("2.4.0", "2.3.0") + + assert Version.match?("2.3.0", "== 2.3.0") + refute Version.match?("2.4.0", "== 2.3.0") - assert V.match?("2.3.0", "== 2.3.0") - refute V.match?("2.4.0", "== 2.3.0") + assert Version.match?("1.0.0", "1.0.0") + assert Version.match?("1.0.0", "1.0.0") - assert V.match?("1.0.0", "1.0.0") - assert V.match?("1.0.0", "1.0.0") + assert Version.match?("1.2.3-alpha", "1.2.3-alpha") - assert V.match?("1.2.3-alpha", "1.2.3-alpha") + assert Version.match?("0.9.3", "== 0.9.3+dev") - assert V.match?("0.9.3", "== 0.9.3+dev") + {:ok, vsn} = Version.parse("2.3.0") + assert Version.match?(vsn, "2.3.0") end test "!=" do - assert V.match?("2.4.0", "!2.3.0") - refute V.match?("2.3.0", "!2.3.0") + ExUnit.CaptureIO.capture_io(:stderr, fn -> + assert Version.match?("2.4.0", "!2.3.0") + refute Version.match?("2.3.0", "!2.3.0") - assert V.match?("2.4.0", "!= 2.3.0") - refute V.match?("2.3.0", "!= 2.3.0") + assert Version.match?("2.4.0", "!= 2.3.0") + refute Version.match?("2.3.0", "!= 2.3.0") + end) end test ">" do - assert V.match?("2.4.0", "> 2.3.0") - refute V.match?("2.2.0", "> 2.3.0") - refute V.match?("2.3.0", "> 2.3.0") - - assert V.match?("1.2.3", "> 1.2.3-alpha") - assert V.match?("1.2.3-alpha.1", "> 1.2.3-alpha") - assert V.match?("1.2.3-alpha.beta.sigma", "> 1.2.3-alpha.beta") - refute V.match?("1.2.3-alpha.10", "< 1.2.3-alpha.1") - refute V.match?("0.10.2-dev", "> 0.10.2") + assert Version.match?("2.4.0", "> 2.3.0") + refute Version.match?("2.2.0", "> 2.3.0") + refute Version.match?("2.3.0", "> 2.3.0") + + assert Version.match?("1.2.3", "> 1.2.3-alpha") + assert Version.match?("1.2.3-alpha.1", "> 1.2.3-alpha") + assert Version.match?("1.2.3-alpha.beta.sigma", "> 1.2.3-alpha.beta") + refute Version.match?("1.2.3-alpha.10", "< 1.2.3-alpha.1") + refute Version.match?("0.10.2-dev", "> 0.10.2") + + refute Version.match?("1.5.0-rc.0", "> 1.5.0-rc0") + assert Version.match?("1.5.0-rc0", "> 1.5.0-rc.0") end test ">=" do - assert V.match?("2.4.0", ">= 2.3.0") - refute V.match?("2.2.0", ">= 2.3.0") - assert V.match?("2.3.0", ">= 2.3.0") + assert Version.match?("2.4.0", ">= 2.3.0") + refute Version.match?("2.2.0", ">= 2.3.0") + assert Version.match?("2.3.0", ">= 2.3.0") + + assert Version.match?("2.0.0", ">= 1.0.0") + assert Version.match?("1.0.0", ">= 1.0.0") - assert V.match?("2.0.0", ">= 1.0.0") - assert V.match?("1.0.0", ">= 1.0.0") + refute Version.match?("1.5.0-rc.0", ">= 1.5.0-rc0") + assert Version.match?("1.5.0-rc0", ">= 1.5.0-rc.0") end test "<" do - assert V.match?("2.2.0", "< 2.3.0") - refute V.match?("2.4.0", "< 2.3.0") - refute V.match?("2.3.0", "< 2.3.0") + assert Version.match?("2.2.0", "< 2.3.0") + refute Version.match?("2.4.0", "< 2.3.0") + refute Version.match?("2.3.0", "< 2.3.0") - assert V.match?("0.10.2-dev", "< 0.10.2") + assert Version.match?("0.10.2-dev", "< 0.10.2") - refute V.match?("1.0.0", "< 1.0.0-dev") - refute V.match?("1.2.3-dev", "< 0.1.2") + refute Version.match?("1.0.0", "< 1.0.0-dev") + refute Version.match?("1.2.3-dev", "< 0.1.2") end test "<=" do - assert V.match?("2.2.0", "<= 2.3.0") - refute V.match?("2.4.0", "<= 2.3.0") - assert V.match?("2.3.0", "<= 2.3.0") + assert Version.match?("2.2.0", "<= 2.3.0") + refute Version.match?("2.4.0", "<= 2.3.0") + assert Version.match?("2.3.0", "<= 2.3.0") end - test "~>" do - assert V.match?("3.0.0", "~> 3.0") - assert V.match?("3.2.0", "~> 3.0") - refute V.match?("4.0.0", "~> 3.0") - refute V.match?("4.4.0", "~> 3.0") - - assert V.match?("3.0.2", "~> 3.0.0") - assert V.match?("3.0.0", "~> 3.0.0") - refute V.match?("3.1.0", "~> 3.0.0") - refute V.match?("3.4.0", "~> 3.0.0") - - assert V.match?("3.6.0", "~> 3.5") - assert V.match?("3.5.0", "~> 3.5") - refute V.match?("4.0.0", "~> 3.5") - refute V.match?("5.0.0", "~> 3.5") - - assert V.match?("3.5.2", "~> 3.5.0") - assert V.match?("3.5.4", "~> 3.5.0") - refute V.match?("3.6.0", "~> 3.5.0") - refute V.match?("3.6.3", "~> 3.5.0") - - assert V.match?("0.9.3", "~> 0.9.3-dev") - refute V.match?("0.10.0", "~> 0.9.3-dev") - - refute V.match?("0.3.0-dev", "~> 0.2.0") + describe "~>" do + test "regular cases" do + assert Version.match?("3.0.0", "~> 3.0") + assert Version.match?("3.2.0", "~> 3.0") + refute Version.match?("4.0.0", "~> 3.0") + refute Version.match?("4.4.0", "~> 3.0") + + assert Version.match?("3.0.2", "~> 3.0.0") + assert Version.match?("3.0.0", "~> 3.0.0") + refute Version.match?("3.1.0", "~> 3.0.0") + refute Version.match?("3.4.0", "~> 3.0.0") + + assert Version.match?("3.6.0", "~> 3.5") + assert Version.match?("3.5.0", "~> 3.5") + refute Version.match?("4.0.0", "~> 3.5") + refute Version.match?("5.0.0", "~> 3.5") + + assert Version.match?("3.5.2", "~> 3.5.0") + assert Version.match?("3.5.4", "~> 3.5.0") + refute Version.match?("3.6.0", "~> 3.5.0") + refute Version.match?("3.6.3", "~> 3.5.0") + + assert Version.match?("0.9.3", "~> 0.9.3-dev") + refute Version.match?("0.10.0", "~> 0.9.3-dev") + + refute Version.match?("0.3.0-dev", "~> 0.2.0") + + assert Version.match?("1.11.0-dev", "~> 1.11-dev") + assert Version.match?("1.11.0", "~> 1.11-dev") + assert Version.match?("1.12.0", "~> 1.11-dev") + refute Version.match?("1.10.0", "~> 1.11-dev") + refute Version.match?("2.0.0", "~> 1.11-dev") + + refute Version.match?("1.5.0-rc.0", "~> 1.5.0-rc0") + assert Version.match?("1.5.0-rc0", "~> 1.5.0-rc.0") + + assert_raise Version.InvalidRequirementError, fn -> + Version.match?("3.0.0", "~> 3") + end + end - assert_raise V.InvalidRequirementError, fn -> - V.match?("3.0.0", "~> 3") + test "~> will never include pre-release versions of its upper bound" do + refute Version.match?("2.2.0-dev", "~> 2.1.0") + refute Version.match?("2.2.0-dev", "~> 2.1.0", allow_pre: false) + refute Version.match?("2.2.0-dev", "~> 2.1.0-dev") + refute Version.match?("2.2.0-dev", "~> 2.1.0-dev", allow_pre: false) end end + test "allow_pre" do + assert Version.match?("1.1.0", "~> 1.0", allow_pre: true) + assert Version.match?("1.1.0", "~> 1.0", allow_pre: false) + assert Version.match?("1.1.0-beta", "~> 1.0", allow_pre: true) + refute Version.match?("1.1.0-beta", "~> 1.0", allow_pre: false) + assert Version.match?("1.0.1-beta", "~> 1.0.0-beta", allow_pre: false) + + assert Version.match?("1.1.0", ">= 1.0.0", allow_pre: true) + assert Version.match?("1.1.0", ">= 1.0.0", allow_pre: false) + assert Version.match?("1.1.0-beta", ">= 1.0.0", allow_pre: true) + refute Version.match?("1.1.0-beta", ">= 1.0.0", allow_pre: false) + assert Version.match?("1.1.0-beta", ">= 1.0.0-beta", allow_pre: false) + end + test "and" do - assert V.match?("0.9.3", "> 0.9.0 and < 0.10.0") - refute V.match?("0.10.2", "> 0.9.0 and < 0.10.0") + assert Version.match?("0.9.3", "> 0.9.0 and < 0.10.0") + refute Version.match?("0.10.2", "> 0.9.0 and < 0.10.0") end test "or" do - assert V.match?("0.9.1", "0.9.1 or 0.9.3 or 0.9.5") - assert V.match?("0.9.3", "0.9.1 or 0.9.3 or 0.9.5") - assert V.match?("0.9.5", "0.9.1 or 0.9.3 or 0.9.5") + assert Version.match?("0.9.1", "0.9.1 or 0.9.3 or 0.9.5") + assert Version.match?("0.9.3", "0.9.1 or 0.9.3 or 0.9.5") + assert Version.match?("0.9.5", "0.9.1 or 0.9.3 or 0.9.5") + refute Version.match?("0.9.6", "0.9.1 or 0.9.3 or 0.9.5") + end + + test "and/or" do + req = "< 0.2.0 and >= 0.1.0 or >= 0.7.0" + assert Version.match?("0.1.0", req) + assert Version.match?("0.1.5", req) + refute Version.match?("0.3.0", req) + refute Version.match?("0.6.0", req) + assert Version.match?("0.7.0", req) + assert Version.match?("0.7.5", req) + + req = ">= 0.7.0 or < 0.2.0 and >= 0.1.0" + assert Version.match?("0.1.0", req) + assert Version.match?("0.1.5", req) + refute Version.match?("0.3.0", req) + refute Version.match?("0.6.0", req) + assert Version.match?("0.7.0", req) + assert Version.match?("0.7.5", req) + + req = "< 0.2.0 and >= 0.1.0 or < 0.8.0 and >= 0.7.0" + assert Version.match?("0.1.0", req) + assert Version.match?("0.1.5", req) + refute Version.match?("0.3.0", req) + refute Version.match?("0.6.0", req) + assert Version.match?("0.7.0", req) + assert Version.match?("0.7.5", req) + + req = "== 0.2.0 or >= 0.3.0 and < 0.4.0 or == 0.7.0" + assert Version.match?("0.2.0", req) + refute Version.match?("0.2.5", req) + assert Version.match?("0.3.0", req) + assert Version.match?("0.3.5", req) + refute Version.match?("0.4.0", req) + assert Version.match?("0.7.0", req) + end + + describe "requirement" do + test "compile_requirement/1" do + {:ok, req} = Version.parse_requirement("1.2.3") + assert req == Version.compile_requirement(req) + + assert Version.match?("1.2.3", req) + refute Version.match?("1.2.4", req) - refute V.match?("0.9.6", "0.9.1 or 0.9.3 or 0.9.5") + assert Version.parse_requirement("1 . 2 . 3") == :error + assert Version.parse_requirement("== >= 1.2.3") == :error + assert Version.parse_requirement("1.2.3 and or 4.5.6") == :error + assert Version.parse_requirement(">= 1") == :error + assert Version.parse_requirement("1.2.3 >=") == :error + end + + test "inspect/1" do + assert Version.parse_requirement!("1.0.0") |> inspect() == + "Version.parse_requirement!(\"1.0.0\")" + end end end diff --git a/lib/elixir/test/erlang/atom_test.erl b/lib/elixir/test/erlang/atom_test.erl index 3b84b90fb44..064e4331703 100644 --- a/lib/elixir/test/erlang/atom_test.erl +++ b/lib/elixir/test/erlang/atom_test.erl @@ -1,42 +1,46 @@ +%% SPDX-License-Identifier: Apache-2.0 +%% SPDX-FileCopyrightText: 2021 The Elixir Team +%% SPDX-FileCopyrightText: 2012 Plataformatec + -module(atom_test). -export([kv/1]). --include("elixir.hrl"). -include_lib("eunit/include/eunit.hrl"). eval(Content) -> - {Value, Binding, _, _} = elixir:eval(Content, []), + Quoted = elixir:'string_to_quoted!'(Content, 1, 1, <<"nofile">>, []), + {Value, Binding, _} = elixir:eval_forms(Quoted, [], elixir:env_for_eval([])), {Value, Binding}. -kv([{Key,nil}]) -> Key. +kv([{Key, nil}]) -> Key. atom_with_punctuation_test() -> - {foo@bar,[]} = eval(":foo@bar"), - {'a?',[]} = eval(":a?"), - {'a!',[]} = eval(":a!"), - {'||',[]} = eval(":||"), - {'...',[]} = eval(":..."). + {foo@bar, []} = eval(":foo@bar"), + {'a?', []} = eval(":a?"), + {'a!', []} = eval(":a!"), + {'||', []} = eval(":||"), + {'...', []} = eval(":..."). atom_quoted_call_test() -> - {3,[]} = eval("Kernel.'+'(1, 2)"). + {3, []} = eval("Kernel.\"+\"(1, 2)"). kv_with_quotes_test() -> - {'foo bar',[]} = eval(":atom_test.kv(\"foo bar\": nil)"). + {'foo bar', []} = eval(":atom_test.kv(\"foo bar\": nil)"). kv_with_interpolation_test() -> - {'foo',[]} = eval(":atom_test.kv(\"#{\"foo\"}\": nil)"), - {'foo',[]} = eval(":atom_test.kv(\"#{\"fo\"}o\": nil)"), - {'foo',_} = eval("a = \"f\"; :atom_test.kv(\"#{a}#{\"o\"}o\": nil)"). + {'foo', []} = eval(":atom_test.kv(\"#{\"foo\"}\": nil)"), + {'foo', []} = eval(":atom_test.kv(\"#{\"fo\"}o\": nil)"), + {'foo', _} = eval("a = \"f\"; :atom_test.kv(\"#{a}#{\"o\"}o\": nil)"). quoted_atom_test() -> - {foo,[]} = eval(":\"foo\""), - {foo,[]} = eval(":'foo'"). + {'+', []} = eval(":\"+\""), + {'foo bar', []} = eval(":\"foo bar\""). atom_with_interpolation_test() -> - {foo,[]} = eval(":\"f#{\"o\"}o\""), - {foo,_} = eval("a=\"foo\"; :\"#{a}\""), - {foo,_} = eval("a=\"oo\"; :\"f#{a}\""), - {foo,_} = eval("a=\"fo\"; :\"#{a}o\""), - {fof,_} = eval("a=\"f\"; :\"#{a}o#{a}\""). + {foo, []} = eval(":\"f#{\"o\"}o\""), + {foo, _} = eval("a=\"foo\"; :\"#{a}\""), + {foo, _} = eval("a=\"oo\"; :\"f#{a}\""), + {foo, _} = eval("a=\"fo\"; :\"#{a}o\""), + {fof, _} = eval("a=\"f\"; :\"#{a}o#{a}\""). quoted_atom_chars_are_escaped_test() -> - {'"',[]} = eval(":\"\\\"\""). + {'"', []} = eval(":\"\\\"\""). diff --git a/lib/elixir/test/erlang/control_test.erl b/lib/elixir/test/erlang/control_test.erl index de7fbac7fac..d4ced5df0a5 100644 --- a/lib/elixir/test/erlang/control_test.erl +++ b/lib/elixir/test/erlang/control_test.erl @@ -1,283 +1,265 @@ +%% SPDX-License-Identifier: Apache-2.0 +%% SPDX-FileCopyrightText: 2021 The Elixir Team +%% SPDX-FileCopyrightText: 2012 Plataformatec + -module(control_test). --include("elixir.hrl"). -include_lib("eunit/include/eunit.hrl"). -eval(Content) -> - {Value, Binding, _, _} = elixir:eval(Content, []), - {Value, Binding}. - to_erl(String) -> - Forms = elixir:'string_to_quoted!'(String, 1, <<"nofile">>, []), - {Expr, _, _} = elixir:quoted_to_erl(Forms, elixir:env_for_eval([])), + Forms = elixir:'string_to_quoted!'(String, 1, 1, <<"nofile">>, []), + {Expr, _, _, _} = elixir:quoted_to_erl(Forms, elixir:env_for_eval([])), Expr. -% Booleans - -booleans_test() -> - {nil, _} = eval("nil"), - {true, _} = eval("true"), - {false, _} = eval("false"). - -% If - -if_else_kv_args_test() -> - {1, _} = eval("if(true, do: 1)"), - {nil, _} = eval("if(false, do: 1)"), - {2, _} = eval("if(false, do: 1, else: 2)"). - -if_else_kv_blocks_test() -> - {2, _} = eval("if(false) do\n1\nelse\n2\nend"), - {2, _} = eval("if(false) do\n1\n3\nelse\n2\nend"), - {2, _} = eval("if(false) do 1 else 2 end"), - {2, _} = eval("if(false) do 1;else 2; end"), - {3, _} = eval("if(false) do 1;else 2; 3; end"). - -vars_if_test() -> - F = fun() -> - {1, [{foo,1}]} = eval("if foo = 1 do; true; else false; end; foo"), - eval("defmodule Bar do\ndef foo, do: 1\ndef bar(x) do\nif x do; foo = 2; else foo = foo; end; foo; end\nend"), - {1, _} = eval("Bar.bar(false)"), - {2, _} = eval("Bar.bar(true)") - end, - test_helper:run_and_remove(F, ['Elixir.Bar']). - -multi_assigned_if_test() -> - {3, _} = eval("x = 1\nif true do\nx = 2\nx = 3\nelse true\nend\nx"), - {3, _} = eval("x = 1\nif true do\n^x = 1\nx = 2\nx = 3\nelse true\nend\nx"), - {1, _} = eval("if true do\nx = 1\nelse true\nend\nx"), - {nil, _} = eval("if false do\nx = 1\nelse true\nend\nx"). - -multi_line_if_test() -> - {1, _} = eval("if true\ndo\n1\nelse\n2\nend"). - -% Try - -try_test() -> - {2, _} = eval("try do\n:foo.bar\ncatch\n:error, :undef -> 2\nend"). - -try_else_test() -> - {true, _} = eval("try do\n1\nelse 2 -> false\n1 -> true\nrescue\nErlangError -> nil\nend"), - {true, _} = eval("try do\n1\nelse {x,y} -> false\nx -> true\nrescue\nErlangError -> nil\nend"), - {true, _} = eval("try do\n{1,2}\nelse {3,4} -> false\n_ -> true\nrescue\nErlangError -> nil\nend"). - -% Receive - -receive_test() -> - {10, _} = eval("send self(), :foo\nreceive do\n:foo -> 10\nend"), - {20, _} = eval("send self(), :bar\nreceive do\n:foo -> 10\n_ -> 20\nend"), - {30, _} = eval("receive do\nafter 1 -> 30\nend"). - -vars_receive_test() -> - {10, _} = eval("send self(), :foo\nreceive do\n:foo ->\na = 10\n:bar -> nil\nend\na"), - {nil, _} = eval("send self(), :bar\nreceive do\n:foo ->\nb = 10\n_ -> 20\nend\nb"), - {30, _} = eval("receive do\n:foo -> nil\nafter\n1 -> c = 30\nend\nc"), - {30, _} = eval("x = 1\nreceive do\n:foo -> nil\nafter\nx -> c = 30\nend\nc"). - -% Case - -case_test() -> - {true, []} = eval("case 1 do\n2 -> false\n1 -> true\nend"), - {true, []} = eval("case 1 do\n{x,y} -> false\nx -> true\nend"), - {true, []} = eval("case {1,2} do;{3,4} -> false\n_ -> true\nend"). - -case_with_do_ambiguity_test() -> - {true,_} = eval("case Atom.to_char_list(true) do\n_ -> true\nend"). +cond_line_test() -> + {'case', 1, _, + [{clause, 2, _, _, _}, + {clause, 3, _, _, _}] + } = to_erl("cond do\n 1 -> :ok\n 2 -> :ok\nend"). -case_with_match_do_ambiguity_test() -> - {true,_} = eval("case x = Atom.to_char_list(true) do\n_ -> true\nend"). - -case_with_unary_do_ambiguity_test() -> - {false,_} = eval("! case Atom.to_char_list(true) do\n_ -> true\nend"). - -multi_assigned_case_test() -> - {3, _} = eval("x = 1\ncase true do\n true ->\nx = 2\nx = 3\n_ -> true\nend\nx"), - {3, _} = eval("x = 1\ncase 1 do\n ^x -> x = 2\nx = 3\n_ -> true\nend\nx"), - {1, _} = eval("case true do\ntrue -> x = 1\n_ -> true\nend\nx"), - {nil, _} = eval("case true do\nfalse -> x = 1\n_ -> true\nend\nx"). - -vars_case_test() -> - F = fun() -> - eval("defmodule Bar do\ndef foo, do: 1\ndef bar(x) do\ncase x do\ntrue -> foo = 2\nfalse -> foo = foo\nend\nfoo\nend\nend"), - {1, _} = eval("Bar.bar(false)"), - {2, _} = eval("Bar.bar(true)") - end, - test_helper:run_and_remove(F, ['Elixir.Bar']). - -% Comparison - -equal_test() -> - {true,_} = eval(":a == :a"), - {true,_} = eval("1 == 1"), - {true,_} = eval("{1,2} == {1,2}"), - {false,_} = eval("1 == 2"), - {false,_} = eval("{1,2} == {1,3}"). - -not_equal_test() -> - {false,_} = eval(":a != :a"), - {false,_} = eval("1 != 1"), - {false,_} = eval("{1,2} != {1,2}"), - {true,_} = eval("1 != 2"), - {true,_} = eval("{1,2} != {1,3}"). - -not_exclamation_mark_test() -> - {false,_} = eval("! :a"), - {false,_} = eval("!true"), - {false,_} = eval("!1"), - {false,_} = eval("![]"), - {true,_} = eval("!nil"), - {true,_} = eval("!false"). - -notnot_exclamation_mark_test() -> - {true,_} = eval("!! :a"), - {true,_} = eval("!!true"), - {true,_} = eval("!!1"), - {true,_} = eval("!![]"), - {false,_} = eval("!!nil"), - {false,_} = eval("!!false"). - -less_greater_test() -> - {true,_} = eval("1 < 2"), - {true,_} = eval("1 < :a"), - {false,_} = eval("1 < 1.0"), - {false,_} = eval("1 < 1"), - {true,_} = eval("1 <= 1.0"), - {true,_} = eval("1 <= 1"), - {true,_} = eval("1 <= :a"), - {false,_} = eval("1 > 2"), - {false,_} = eval("1 > :a"), - {false,_} = eval("1 > 1.0"), - {false,_} = eval("1 > 1"), - {true,_} = eval("1 >= 1.0"), - {true,_} = eval("1 >= 1"), - {false,_} = eval("1 >= :a"). - -integer_and_float_test() -> - {true,_} = eval("1 == 1"), - {false,_} = eval("1 != 1"), - {true,_} = eval("1 == 1.0"), - {false,_} = eval("1 != 1.0"), - {true,_} = eval("1 === 1"), - {false,_} = eval("1 !== 1"), - {false,_} = eval("1 === 1.0"), - {true,_} = eval("1 !== 1.0"). - -and_test() -> - F = fun() -> - eval("defmodule Bar do\ndef foo, do: true\ndef bar, do: false\n def baz(x), do: x == 1\nend"), - {true, _} = eval("true and true"), - {false, _} = eval("true and false"), - {false, _} = eval("false and true"), - {false, _} = eval("false and false"), - {true, _} = eval("Bar.foo and Bar.foo"), - {false, _} = eval("Bar.foo and Bar.bar"), - {true, _} = eval("Bar.foo and Bar.baz 1"), - {false, _} = eval("Bar.foo and Bar.baz 2"), - {true, _} = eval("false and false or true"), - {3, _} = eval("Bar.foo and 1 + 2"), - {false, _} = eval("Bar.bar and :erlang.error(:bad)"), - ?assertError({badarg, 1}, eval("1 and 2")) - end, - test_helper:run_and_remove(F, ['Elixir.Bar']). - -or_test() -> - F = fun() -> - eval("defmodule Bar do\ndef foo, do: true\ndef bar, do: false\n def baz(x), do: x == 1\nend"), - {true, _} = eval("true or true"), - {true, _} = eval("true or false"), - {true, _} = eval("false or true"), - {false, _} = eval("false or false"), - {true, _} = eval("Bar.foo or Bar.foo"), - {true, _} = eval("Bar.foo or Bar.bar"), - {false, _} = eval("Bar.bar or Bar.bar"), - {true, _} = eval("Bar.bar or Bar.baz 1"), - {false, _} = eval("Bar.bar or Bar.baz 2"), - {3, _} = eval("Bar.bar or 1 + 2"), - {true, _} = eval("Bar.foo or :erlang.error(:bad)"), - ?assertError({badarg, 1}, eval("1 or 2")) - end, - test_helper:run_and_remove(F, ['Elixir.Bar']). - -not_test() -> - {false, _} = eval("not true"), - {true, _} = eval("not false"), - ?assertError(badarg, eval("not 1")). - -andand_test() -> - F = fun() -> - eval("defmodule Bar do\ndef foo, do: true\ndef bar, do: false\n def baz(x), do: x == 1\nend"), - {true, _} = eval("Kernel.&&(true, true)"), - {true, _} = eval("true && true"), - {false, _} = eval("true && false"), - {false, _} = eval("false && true"), - {false, _} = eval("false && false"), - {nil, _} = eval("true && nil"), - {nil, _} = eval("nil && true"), - {false, _} = eval("false && nil"), - {true, _} = eval("Bar.foo && Bar.foo"), - {false, _} = eval("Bar.foo && Bar.bar"), - {true, _} = eval("Bar.foo && Bar.baz 1"), - {false, _} = eval("Bar.foo && Bar.baz 2"), - {true, _} = eval("1 == 1 && 2 < 3"), - {3, _} = eval("Bar.foo && 1 + 2"), - {false, _} = eval("Bar.bar && :erlang.error(:bad)"), - {2, _} = eval("1 && 2"), - {nil, _} = eval("nil && 2") - end, - test_helper:run_and_remove(F, ['Elixir.Bar']). - -andand_with_literal_test() -> - {[nil, nil, nil], _} = eval("[nil && 2, nil && 3, nil && 4]"). - -oror_test() -> - F = fun() -> - eval("defmodule Bar do\ndef foo, do: true\ndef bar, do: false\n def baz(x), do: x == 1\nend"), - {true, _} = eval("Kernel.||(false, true)"), - {true, _} = eval("true || true"), - {true, _} = eval("true || false"), - {true, _} = eval("false || true"), - {false, _} = eval("false || false"), - {false, _} = eval("nil || false"), - {nil, _} = eval("false || nil"), - {true, _} = eval("false || nil || true"), - {true, _} = eval("Bar.foo || Bar.foo"), - {true, _} = eval("Bar.foo || Bar.bar"), - {false, _} = eval("Bar.bar || Bar.bar"), - {true, _} = eval("Bar.bar || Bar.baz 1"), - {false, _} = eval("Bar.bar || Bar.baz 2"), - {false, _} = eval("1 == 2 || 2 > 3"), - {3, _} = eval("Bar.bar || 1 + 2"), - {true, _} = eval("Bar.foo || :erlang.error(:bad)"), - {1, _} = eval("1 || 2"), - {2, _} = eval("nil || 2"), - {true, _} = eval("false && false || true") - end, - test_helper:run_and_remove(F, ['Elixir.Bar']). +float_match_test() -> + {'case', _, _, + [{clause, _, [{op, _, '+', {float, _, +0.0}}], [], [{atom, _, pos}]}, + {clause, _, [{op, _, '-', {float, _, +0.0}}], [], [{atom, _, neg}]}] + } = to_erl("case X do\n +0.0 -> :pos\n -0.0 -> :neg\nend"). % Optimized optimized_if_test() -> {'case', _, _, - [{clause,_,[{atom,_,false}],[],[{atom,_,else}]}, - {clause,_,[{atom,_,true}],[],[{atom,_,do}]}] - } = to_erl("if is_list([]), do: :do, else: :else"). + [{clause, _, [{atom, _, false}], [], [{atom, _, 'else'}]}, + {clause, _, [{atom, _, true}], [], [{atom, _, do}]}] + } = to_erl("if is_list([]), do: :do, else: :else"). optimized_andand_test() -> {'case', _, _, - [{clause,_, - [{var,_,Var}], - [[{op,_,'orelse',_,_}]], - [{var,_,Var}]}, - {clause,_,[{var,_,'_'}],[],[{atom,0,done}]}] - } = to_erl("is_list([]) && :done"). + [{clause, _, + [{var, _, Var}], + [[{op, _, 'orelse', _, _}]], + [{var, _, Var}]}, + {clause, _, [{var, _, '_'}], [], [{atom, 1, done}]}] + } = to_erl("is_list([]) && :done"). optimized_oror_test() -> {'case', _, _, - [{clause,1, - [{var,1,_}], - [[{op,1,'orelse',_,_}]], - [{atom,0,done}]}, - {clause,1,[{var,1,Var}],[],[{var,1,Var}]}] - } = to_erl("is_list([]) || :done"). + [{clause, 1, + [{var, 1, _}], + [[{op, 1, 'orelse', _, _}]], + [{atom, 1, done}]}, + {clause, 1, [{var, 1, Var}], [], [{var, 1, Var}]}] + } = to_erl("is_list([]) || :done"). + +optimized_and_test() -> + {'case',_, _, + [{clause, _, [{atom, _, false}], [], [{atom, _, false}]}, + {clause, _, [{atom, _, true}], [], [{atom, _, done}]}] + } = to_erl("is_list([]) and :done"). + +optimized_or_test() -> + {'case', _, _, + [{clause, _, [{atom, _, false}], [], [{atom, _, done}]}, + {clause, _, [{atom, _, true}], [], [{atom, _, true}]}] + } = to_erl("is_list([]) or :done"). no_after_in_try_test() -> - {'try', _, [_], [_], _, []} = to_erl("try do :foo.bar() else _ -> :ok end"). \ No newline at end of file + {'try', _, [_], [], [_], []} = to_erl("try do :foo.bar() catch _ -> :ok end"). + +optimized_inspect_interpolation_test() -> + {bin, _, + [{bin_element, _, + {call, _, {remote, _,{atom, _, 'Elixir.Kernel'}, {atom, _, inspect}}, [_]}, + default, [binary]}]} = to_erl("\"#{inspect(1)}\""). + +optimized_map_put_test() -> + {map, _, + [{map_field_assoc, _, {atom, _, a}, {integer, _, 1}}, + {map_field_assoc, _, {atom, _, b}, {integer, _, 2}}] + } = to_erl("Map.put(%{a: 1}, :b, 2)"). + +optimized_map_put_variable_test() -> + {block, _, + [_, + {map, _, {var, _, _}, + [{map_field_assoc, _, {atom, _, a}, {integer, _, 1}}] + }] + } = to_erl("x = %{}; Map.put(x, :a, 1)"). + +optimized_nested_map_put_variable_test() -> + {block, _, + [_, + {map, _, {var, _, _}, + [{map_field_assoc, _, {atom, _, a}, {integer, _, 1}}, + {map_field_assoc, _, {atom, _, b}, {integer, _, 2}}] + }] + } = to_erl("x = %{}; Map.put(Map.put(x, :a, 1), :b, 2)"). + +optimized_map_merge_test() -> + {map, _, + [{map_field_assoc, _, {atom, _, a}, {integer, _, 1}}, + {map_field_assoc, _, {atom, _, b}, {integer, _, 2}}, + {map_field_assoc, _, {atom, _, c}, {integer, _, 3}}] + } = to_erl("Map.merge(%{a: 1, b: 2}, %{c: 3})"). + +optimized_map_merge_variable_test() -> + {block, _, + [_, + {map, _, {var, _, _}, + [{map_field_assoc, _, {atom, _, a}, {integer, _, 1}}] + }] + } = to_erl("x = %{}; Map.merge(x, %{a: 1})"). + +optimized_map_update_and_merge_test() -> + {block, _, + [_, + {map, _, {var, _, _}, + [{map_field_exact, _, {atom, _, a}, {integer, _, 2}}, + {map_field_assoc, _, {atom, _, b}, {integer, _, 3}}] + }] + } = to_erl("x = %{a: 1}; Map.merge(%{x | a: 2}, %{b: 3})"), + {block, _, + [_, + {call, _, {remote, _, {atom, _, maps}, {atom, _, merge}}, + [{map, _, + [{map_field_assoc, _, {atom, _, a}, {integer, _, 2}}]}, + {map, _, {var, _, _}, + [{map_field_exact, _, {atom, _, b}, {integer, _, 3}}]}] + }] + } = to_erl("x = %{a: 1}; Map.merge(%{a: 2}, %{x | b: 3})"). + +optimized_nested_map_merge_variable_test() -> + {block, _, + [_, + {map, _, {var, _, _}, + [{map_field_assoc, _, {atom, _, a}, {integer, _, 1}}, + {map_field_assoc, _, {atom, _, b}, {integer, _, 2}}] + }] + } = to_erl("x = %{}; Map.merge(Map.merge(x, %{a: 1}), %{b: 2})"). + +optimized_map_set_new_test() -> + {map, _, + [ + {map_field_assoc, _, {atom, _, '__struct__'}, {atom, _, 'Elixir.MapSet'}}, + {map_field_assoc, _, + {atom, _, map}, + {map, _, [ + {map_field_assoc, _, {integer, _, 1}, {nil, _}}, + {map_field_assoc, _, {integer, _, 2}, {nil, _}}, + {map_field_assoc, _, {integer, _, 3}, {nil, _}} + ]} + } + ] + } = to_erl("MapSet.new([1, 2, 3])"). + +not_optimized_map_set_new_with_range_test() -> + {call, _, + {remote, _, {atom, _, 'Elixir.MapSet'}, {atom, _, new}}, [ + {map, _, [ + {map_field_assoc, _, {atom, _, '__struct__'}, {atom, _, 'Elixir.Range'}}, + {map_field_assoc, _, {atom, _, first}, {integer, _, 1}}, + {map_field_assoc, _, {atom, _, last}, {integer, _, 3}}, + {map_field_assoc, _, {atom, _, step}, {integer, _, 1}} + ]} + ] + } = to_erl("MapSet.new(1..3)"). + +map_set_new_with_failing_args_test() -> + {call, _, + {remote, _, {atom, _, 'Elixir.MapSet'}, {atom, _, new}}, [ + {atom, _, not_an_enumerable} + ] + } = to_erl("MapSet.new(:not_an_enumerable)"). + +optimized_date_shift_duration_test() -> + {call, _, + {remote, _, {atom, _, 'Elixir.Date'}, {atom, _, shift}}, [ + {atom, _, non_important}, + {map, _, [ + {map_field_assoc, _, {atom, _, '__struct__'}, {atom, _, 'Elixir.Duration'}}, + {map_field_assoc, _, {atom, _, day}, {integer, _, 0}}, + {map_field_assoc, _, {atom, _, hour}, {integer, _, 0}}, + {map_field_assoc, _, {atom, _, microsecond}, {tuple, _, [{integer, _, 0}, {integer, _, 0}]}}, + {map_field_assoc, _, {atom, _, minute}, {integer, _, 0}}, + {map_field_assoc, _, {atom, _, month}, {integer, _, 0}}, + {map_field_assoc, _, {atom, _, second}, {integer, _, 0}}, + {map_field_assoc, _, {atom, _, week}, {integer, _, 1}}, + {map_field_assoc, _, {atom, _, year}, {integer, _, 0}} + ]} + ] + } = to_erl("Date.shift(:non_important, week: 1)"). + +not_optimized_date_shift_duration_unsupported_unit_test() -> + {call, _, + {remote, _, {atom, _, 'Elixir.Date'}, {atom, _, shift}}, [ + {atom, _, non_important}, + {cons, _, {tuple, _, [{atom, _, hour}, {integer, _, 1}]}, {nil, _}} + ] + } = to_erl("Date.shift(:non_important, hour: 1)"). + +optimized_time_shift_duration_test() -> + {call, _, + {remote, _, {atom, _, 'Elixir.Time'}, {atom, _, shift}}, [ + {atom, _, non_important}, + {map, _, [ + {map_field_assoc, _, {atom, _, '__struct__'}, {atom, _, 'Elixir.Duration'}}, + {map_field_assoc, _, {atom, _, day}, {integer, _, 0}}, + {map_field_assoc, _, {atom, _, hour}, {integer, _, 0}}, + {map_field_assoc, _, {atom, _, microsecond}, {tuple, _, [{integer, _, 0}, {integer, _, 0}]}}, + {map_field_assoc, _, {atom, _, minute}, {integer, _, 0}}, + {map_field_assoc, _, {atom, _, month}, {integer, _, 0}}, + {map_field_assoc, _, {atom, _, second}, {integer, _, 2}}, + {map_field_assoc, _, {atom, _, week}, {integer, _, 0}}, + {map_field_assoc, _, {atom, _, year}, {integer, _, 0}} + ]} + ] + } = to_erl("Time.shift(:non_important, second: 2)"). + +not_optimized_time_shift_duration_unsupported_unit_test() -> + {call, _, + {remote, _, {atom, _, 'Elixir.Time'}, {atom, _, shift}}, [ + {atom, _, non_important}, + {cons, _, {tuple, _, [{atom, _, day}, {integer, _, 2}]}, {nil, _}} + ] + } = to_erl("Time.shift(:non_important, day: 2)"). + +optimized_date_time_shift_duration_test() -> + {call, _, + {remote, _, {atom, _, 'Elixir.DateTime'}, {atom, _, shift}}, [ + {atom, _, non_important}, + {map, _, [ + {map_field_assoc, _, {atom, _, '__struct__'}, {atom, _, 'Elixir.Duration'}}, + {map_field_assoc, _, {atom, _, day}, {integer, _, 0}}, + {map_field_assoc, _, {atom, _, hour}, {integer, _, 0}}, + {map_field_assoc, _, {atom, _, microsecond}, {tuple, _, [{integer, _, 0}, {integer, _, 0}]}}, + {map_field_assoc, _, {atom, _, minute}, {integer, _, 3}}, + {map_field_assoc, _, {atom, _, month}, {integer, _, 0}}, + {map_field_assoc, _, {atom, _, second}, {integer, _, 0}}, + {map_field_assoc, _, {atom, _, week}, {integer, _, 0}}, + {map_field_assoc, _, {atom, _, year}, {integer, _, 0}} + ]} + ] + } = to_erl("DateTime.shift(:non_important, minute: 3)"). + +non_optimized_date_time_shift_duration_unknown_unit_test() -> + {call, _, + {remote, _, {atom, _, 'Elixir.DateTime'}, {atom, _, shift}}, [ + {atom, _, non_important}, + {cons, _, {tuple, _, [{atom, _, unknown}, {integer, _, 3}]}, {nil, _}} + ] + } = to_erl("DateTime.shift(:non_important, unknown: 3)"). + +optimized_naive_date_time_shift_duration_test() -> + {call, _, + {remote, _, {atom, _, 'Elixir.NaiveDateTime'}, {atom, _, shift}}, [ + {atom, _, non_important}, + {map, _, [ + {map_field_assoc, _, {atom, _, '__struct__'}, {atom, _, 'Elixir.Duration'}}, + {map_field_assoc, _, {atom, _, day}, {integer, _, 0}}, + {map_field_assoc, _, {atom, _, hour}, {integer, _, 0}}, + {map_field_assoc, _, {atom, _, microsecond}, {tuple, _, [{integer, _, 0}, {integer, _, 0}]}}, + {map_field_assoc, _, {atom, _, minute}, {integer, _, 0}}, + {map_field_assoc, _, {atom, _, month}, {integer, _, 0}}, + {map_field_assoc, _, {atom, _, second}, {integer, _, 0}}, + {map_field_assoc, _, {atom, _, week}, {integer, _, 0}}, + {map_field_assoc, _, {atom, _, year}, {integer, _, 4}} + ]} + ] + } = to_erl("NaiveDateTime.shift(:non_important, year: 4)"). \ No newline at end of file diff --git a/lib/elixir/test/erlang/function_test.erl b/lib/elixir/test/erlang/function_test.erl index 8f875a5817b..2d7f8156aa2 100644 --- a/lib/elixir/test/erlang/function_test.erl +++ b/lib/elixir/test/erlang/function_test.erl @@ -1,22 +1,27 @@ +%% SPDX-License-Identifier: Apache-2.0 +%% SPDX-FileCopyrightText: 2021 The Elixir Team +%% SPDX-FileCopyrightText: 2012 Plataformatec + -module(function_test). -include_lib("eunit/include/eunit.hrl"). eval(Content) -> - {Value, Binding, _, _} = elixir:eval(Content, []), - {Value, Binding}. + Quoted = elixir:'string_to_quoted!'(Content, 1, 1, <<"nofile">>, []), + {Value, Binding, _} = elixir:eval_forms(Quoted, [], elixir:env_for_eval([])), + {Value, lists:sort(Binding)}. function_arg_do_end_test() -> {3, _} = eval("if true do\n1 + 2\nend"), {nil, _} = eval("if true do end"). function_stab_end_test() -> - {_, [{a, Fun1}]} = eval("a = fn -> end"), - nil = Fun1(), - {_, [{a, Fun2}]} = eval("a = fn() -> end"), - nil = Fun2(), {_, [{a, Fun3}]} = eval("a = fn -> 1 + 2 end"), 3 = Fun3(). +function_stab_newlines_test() -> + {_, [{a, Fun3}]} = eval("a = fn\n->\n1 + 2\nend"), + 3 = Fun3(). + function_stab_many_test() -> {_, [{a, Fun}]} = eval("a = fn\n{:foo, x} -> x\n{:bar, x} -> x\nend"), 1 = Fun({foo, 1}), @@ -29,47 +34,47 @@ function_stab_inline_test() -> function_with_args_test() -> {Fun, _} = eval("fn(a, b) -> a + b end"), - 3 = Fun(1,2). + 3 = Fun(1, 2). function_with_kv_args_test() -> {Fun, _} = eval("fn(a, [other: b, another: c]) -> a + b + c end"), - 6 = Fun(1,[{other,2}, {another,3}]). + 6 = Fun(1, [{other, 2}, {another, 3}]). function_as_closure_test() -> - {_, [{a, Res1}|_]} = eval("b = 1; a = fn -> b + 2 end"), + {_, [{a, Res1} | _]} = eval("b = 1; a = fn -> b + 2 end"), 3 = Res1(). function_apply_test() -> - {3,_} = eval("a = fn -> 3 end; apply a, []"). + {3, _} = eval("a = fn -> 3 end; apply a, []"). function_apply_with_args_test() -> - {3,_} = eval("a = fn b -> b + 2 end; apply a, [1]"). + {3, _} = eval("a = fn b -> b + 2 end; apply a, [1]"). function_apply_and_clojure_test() -> - {3,_} = eval("b = 1; a = fn -> b + 2 end; apply a, []"). + {3, _} = eval("b = 1; a = fn -> b + 2 end; apply a, []"). function_parens_test() -> - {0,_} = eval("(fn() -> 0 end).()"), - {1,_} = eval("(fn(1) -> 1 end).(1)"), - {3,_} = eval("(fn(1, 2) -> 3 end).(1, 2)"), + {0, _} = eval("(fn() -> 0 end).()"), + {1, _} = eval("(fn(1) -> 1 end).(1)"), + {3, _} = eval("(fn(1, 2) -> 3 end).(1, 2)"), - {0,_} = eval("(fn () -> 0 end).()"), - {1,_} = eval("(fn (1) -> 1 end).(1)"), - {3,_} = eval("(fn (1, 2) -> 3 end).(1, 2)"). + {0, _} = eval("(fn() -> 0 end).()"), + {1, _} = eval("(fn(1) -> 1 end).(1)"), + {3, _} = eval("(fn(1, 2) -> 3 end).(1, 2)"). %% Function calls function_call_test() -> - {3, _} = eval("x = fn a, b -> a + b end\nx.(1,2)"). + {3, _} = eval("x = fn a, b -> a + b end\nx.(1, 2)"). function_call_without_arg_test() -> {3, _} = eval("x = fn -> 2 + 1 end\nx.()"). function_call_do_end_test() -> - {[1,[{do,2},{else,3}]], _} = eval("x = fn a, b -> [a,b] end\nx.(1) do\n2\nelse 3\nend"). + {[1, [{do, 2}, {'else', 3}]], _} = eval("x = fn a, b -> [a, b] end\nx.(1) do\n2\nelse 3\nend"). function_call_with_assignment_test() -> - {3, [{a,_},{c, 3}]} = eval("a = fn x -> x + 2 end; c = a.(1)"). + {3, [{a, _}, {c, 3}]} = eval("a = fn x -> x + 2 end; c = a.(1)"). function_calls_with_multiple_expressions_test() -> {26, _} = eval("a = fn a, b -> a + b end; a.((3 + 4 - 1), (2 * 10))"). @@ -78,29 +83,29 @@ function_calls_with_multiple_args_with_line_breaks_test() -> {5, _} = eval("a = fn a, b -> a + b end; a.(\n3,\n2\n)"). function_calls_with_parenthesis_test() -> - {3, [{a,_},{b,1}]} = eval("a = (fn x -> x + 2 end).(b = 1)"). + {3, [{a, _}, {b, 1}]} = eval("a = (fn x -> x + 2 end).(b = 1)"). function_call_with_a_single_space_test() -> - {3, _} = eval("a = fn a, b -> a + b end; a. (1,2)"), - {3, _} = eval("a = fn a, b -> a + b end; a .(1,2)"). + {3, _} = eval("a = fn a, b -> a + b end; a. (1, 2)"), + {3, _} = eval("a = fn a, b -> a + b end; a .(1, 2)"). function_call_with_spaces_test() -> - {3, _} = eval("a = fn a, b -> a + b end; a . (1,2)"). + {3, _} = eval("a = fn a, b -> a + b end; a . (1, 2)"). function_call_without_assigning_with_spaces_test() -> - {3, _} = eval("(fn a, b -> a + b end) . (1,2)"). + {3, _} = eval("(fn a, b -> a + b end) . (1, 2)"). function_call_with_assignment_and_spaces_test() -> - {3, [{a,_},{c,3}]} = eval("a = fn x -> x + 2 end; c = a . (1)"). + {3, [{a, _}, {c, 3}]} = eval("a = fn x -> x + 2 end; c = a . (1)"). function_call_with_multiple_spaces_test() -> - {3, _} = eval("a = fn a, b -> a + b end; a . (1,2)"). + {3, _} = eval("a = fn a, b -> a + b end; a . (1, 2)"). function_call_with_multiline_test() -> - {3, _} = eval("a = fn a, b -> a + b end; a . \n (1,2)"). + {3, _} = eval("a = fn a, b -> a + b end; a . \n (1, 2)"). function_call_with_tabs_test() -> - {3, _} = eval("a = fn a, b -> a + b end; a .\n\t(1,2)"). + {3, _} = eval("a = fn a, b -> a + b end; a .\n\t(1, 2)"). function_call_with_args_and_nested_when_test() -> {Fun, _} = eval("fn a, b when a == 1 when b == 2 -> a + b end"), diff --git a/lib/elixir/test/erlang/match_test.erl b/lib/elixir/test/erlang/match_test.erl deleted file mode 100644 index 317b066dadf..00000000000 --- a/lib/elixir/test/erlang/match_test.erl +++ /dev/null @@ -1,118 +0,0 @@ --module(match_test). --include_lib("eunit/include/eunit.hrl"). - -eval(Content) -> eval(Content, []). - -eval(Content, Initial) -> - {Value, Binding, _, _} = elixir:eval(Content, Initial), - {Value, Binding}. - -no_assignment_test() -> - {nil, []} = eval(""). - -% Var/assignment test -arithmetic_test() -> - ?assertError({badmatch, _}, eval("-1 = 1")). - -assignment_test() -> - {1, [{a, 1}]} = eval("a = 1"). - -not_single_assignment_test() -> - {2, [{a, 2}]} = eval("a = 1\na = 2\na"), - {1, [{a, 1}]} = eval("{a,a} = {1,1}\na"), - {2, [{a, 2}]} = eval("a = 1\n{^a,a} = {1,2}\na"), - ?assertError({badmatch, _}, eval("{a,a} = {1,2}")), - ?assertError({badmatch, _}, eval("{1 = a,a} = {1,2}")), - ?assertError({badmatch, _}, eval("{a = 1,a} = {1,2}")), - ?assertError({badmatch, _}, eval("a = 0;{a,a} = {1,2}")), - ?assertError({badmatch, _}, eval("a = 0;{1 = a,a} = {1,2}")), - ?assertError({badmatch, _}, eval("a = 1\n^a = 2")). - -duplicated_assignment_on_module_with_tuple_test() -> - F = fun() -> - eval("defmodule Foo do\ndef v({a, _left}, {a, _right}), do: a\nend"), - {1,_} = eval("Foo.v({1, :foo}, {1, :bar})"), - ?assertError(function_clause, eval("Foo.v({1, :foo}, {2, :bar})")) - end, - test_helper:run_and_remove(F, ['Elixir.Foo']). - -duplicated_assignment_on_module_with_list_test() -> - F = fun() -> - eval("defmodule Foo do\ndef v([ a, _left ], [ a, _right ]), do: a\nend"), - {1,_} = eval("Foo.v([ 1, :foo ], [ 1, :bar ])"), - ?assertError(function_clause, eval("Foo.v([ 1, :foo ], [ 2, :bar ])")) - end, - test_helper:run_and_remove(F, ['Elixir.Foo']). - -multiline_assignment_test() -> - {1, [{a, 1}]} = eval("a =\n1"), - {1, [{a, 1}, {b, 1}]} = eval("a = 1\nb = 1"). - -multiple_assignment_test() -> - {1, [{a, 1}, {b, 1}]} = eval("a = b = 1"). - -multiple_assignment_with_parens_test() -> - {1, [{a, 1}, {b, 1}]} = eval("a = (b = 1)"). - -multiple_assignment_with_left_parens_test() -> - {1, [{a, 1}, {b, 1}]} = eval("(a) = (b = 1)"). - -multiple_assignment_with_expression_test() -> - {-4, [{a, -4}, {b, -4}]} = eval("a = (b = -(2 * 2))"). - -multiple_assignment_with_binding_expression_test() -> - {3, [{a, 3}, {b, 1}]} = eval("a = (2 + b)", [{b, 1}]). - -underscore_assignment_test() -> - {1, []} = eval("_ = 1"). - -assignment_precedence_test() -> - {_, [{x,{'__block__', _, [1,2,3]}}]} = eval("x = quote do\n1\n2\n3\nend"). - -% Tuples match -simple_tuple_test() -> - {{}, _} = eval("a = {}"), - {{1,2,3}, _} = eval("a = {1, 2, 3}"), - {{1,2,3}, _} = eval("a = {1, 1 + 1, 3}"), - {{1,{2},3}, _} = eval("a = {1, {2}, 3}"). - -tuple_match_test() -> - {_, _} = eval("{1,2,3} = {1, 2, 3}"), - ?assertError({badmatch, _}, eval("{1, 3, 2} = {1, 2, 3}")). - -% Lists match -simple_list_test() -> - {[], _} = eval("a = []"), - {[1,2,3], _} = eval("a = [1, 2, 3]"), - {[1,2,3], _} = eval("a = [1, 1 + 1, 3]"), - {[1,[2],3], _} = eval("a = [1, [2], 3]"), - {[1,{2},3], _} = eval("a = [1, {2}, 3]"). - -list_match_test() -> - {_, _} = eval("[1, 2, 3] = [1, 2, 3]"), - ?assertError({badmatch, _}, eval("[1, 3, 2] = [1, 2, 3]")). - -list_vars_test() -> - {[3,1], [{x,3}]} = eval("x = 1\n[x = x + 2, x]"). - -head_and_tail_test() -> - {_,[{h,1},{t,[2,3]}]} = eval("[h|t] = [1,2,3]"), - {_,[{h,2},{t,[3]}]} = eval("[1,h|t] = [1,2,3]"), - {_,[{t,[3]}]} = eval("[1,2|t] = [1,2,3]"), - {_,[{h,1}]} = eval("[h|[2,3]] = [1,2,3]"), - {_,[{t,[2,3]}]} = eval("[+1|t] = [1,2,3]"), - ?assertError({badmatch, _}, eval("[2,h|t] = [1,2,3]")). - -% Keyword match - -orrdict_match_test() -> - {[{a,1},{b,2}], _} = eval("a = [a: 1, b: 2]"). - -% Function match - -function_clause_test() -> - F = fun() -> - eval("defmodule Foo do\ndef a([{_k,_}=e|_]), do: e\nend"), - {{foo,bar},_} = eval("Foo.a([{:foo,:bar}])") - end, - test_helper:run_and_remove(F, ['Elixir.Foo']). \ No newline at end of file diff --git a/lib/elixir/test/erlang/module_test.erl b/lib/elixir/test/erlang/module_test.erl deleted file mode 100644 index 1b9b835f2e7..00000000000 --- a/lib/elixir/test/erlang/module_test.erl +++ /dev/null @@ -1,112 +0,0 @@ --module(module_test). --include_lib("eunit/include/eunit.hrl"). - -eval(Content) -> - {Value, Binding, _, _} = elixir:eval(Content, []), - {Value, Binding}. - -definition_test() -> - F = fun() -> - eval("defmodule Foo.Bar.Baz, do: nil") - end, - test_helper:run_and_remove(F, ['Elixir.Foo.Bar.Baz']). - -module_vars_test() -> - F = fun() -> - eval("a = 1; b = 2; c = 3; defmodule Foo do\n1 = a; 2 = b; 3 = c\nend") - end, - test_helper:run_and_remove(F, ['Elixir.Foo']). - -function_test() -> - F = fun() -> - eval("defmodule Foo.Bar.Baz do\ndef sum(a, b) do\na + b\nend\nend"), - 3 = 'Elixir.Foo.Bar.Baz':sum(1, 2) - end, - test_helper:run_and_remove(F, ['Elixir.Foo.Bar.Baz']). - -quote_unquote_splicing_test() -> - {{'{}', [], [1,2,3,4,5]}, _} = eval("x = [2,3,4]\nquote do: {1, unquote_splicing(x), 5}"). - -def_shortcut_test() -> - F = fun() -> - {1,[]} = eval("defmodule Foo do\ndef version, do: 1\nend\nFoo.version") - end, - test_helper:run_and_remove(F, ['Elixir.Foo']). - -macro_test() -> - F = fun() -> - {'Elixir.Foo',[]} = eval("defmodule Foo do\ndef version, do: __MODULE__\nend\nFoo.version"), - {nil,[]} = eval("__MODULE__") - end, - test_helper:run_and_remove(F, ['Elixir.Foo']). - -macro_line_test() -> - F = fun() -> - ?assertMatch({2, []}, eval("defmodule Foo do\ndef line, do: __ENV__.line\nend\nFoo.line")), - ?assertMatch({1, []}, eval("__ENV__.line")) - end, - test_helper:run_and_remove(F, ['Elixir.Foo']). - -def_default_test() -> - F = fun() -> - eval("defmodule Foo do\ndef version(x \\\\ 1), do: x\nend"), - ?assertEqual({1, []}, eval("Foo.version")), - ?assertEqual({2, []}, eval("Foo.version(2)")) - end, - test_helper:run_and_remove(F, ['Elixir.Foo']). - -def_left_default_test() -> - F = fun() -> - eval("defmodule Foo do\ndef version(x \\\\ 1, y), do: x + y\nend"), - ?assertEqual({4, []}, eval("Foo.version(3)")), - ?assertEqual({5, []}, eval("Foo.version(2, 3)")) - end, - test_helper:run_and_remove(F, ['Elixir.Foo']). - -def_with_guard_test() -> - F = fun() -> - eval("defmodule Foo do\ndef v(x) when x < 10, do: true\ndef v(x) when x >= 10, do: false\nend"), - {true,_} = eval("Foo.v(0)"), - {false,_} = eval("Foo.v(20)") - end, - test_helper:run_and_remove(F, ['Elixir.Foo']). - -do_end_test() -> - F = fun() -> - eval("defmodule Foo do\ndef a, do: 1\ndefmodule Bar do\ndef b, do: 2\nend\ndef c, do: 3\nend"), - {1,_} = eval("Foo.a"), - {2,_} = eval("Foo.Bar.b"), - {3,_} = eval("Foo.c") - end, - test_helper:run_and_remove(F, ['Elixir.Foo', 'Elixir.Foo.Bar']). - -nesting_test() -> - F = fun() -> - eval("defmodule Foo do\ndefmodule Elixir.Bar do\ndef b, do: 2\nend\nend"), - {2,_} = eval("Bar.b") - end, - test_helper:run_and_remove(F, ['Elixir.Foo', 'Elixir.Bar']). - -dot_alias_test() -> - {'Elixir.Foo.Bar.Baz', _} = eval("Foo.Bar.Baz"). - -dot_dyn_alias_test() -> - {'Elixir.Foo.Bar.Baz', _} = eval("a = Foo.Bar; a.Baz"). - -single_ref_test() -> - {'Elixir.Foo', _} = eval("Foo"), - {'Elixir.Foo', _} = eval("Elixir.Foo"). - -nested_ref_test() -> - {'Elixir.Foo.Bar.Baz', _} = eval("Foo.Bar.Baz"). - -module_with_elixir_as_a_name_test() -> - ?assertError(#{'__struct__' := 'Elixir.CompileError'}, eval("defmodule Elixir do\nend")). - -dynamic_defmodule_test() -> - F = fun() -> - eval("defmodule Foo do\ndef a(name) do\ndefmodule name, do: (def x, do: 1)\nend\nend"), - {_,_} = eval("Foo.a(Bar)"), - {1,_} = eval("Bar.x") - end, - test_helper:run_and_remove(F, ['Elixir.Foo', 'Elixir.Bar']). \ No newline at end of file diff --git a/lib/elixir/test/erlang/operators_test.erl b/lib/elixir/test/erlang/operators_test.erl deleted file mode 100644 index 6ef07a89f14..00000000000 --- a/lib/elixir/test/erlang/operators_test.erl +++ /dev/null @@ -1,92 +0,0 @@ --module(operators_test). --include_lib("eunit/include/eunit.hrl"). - -eval(Content) -> - {Value, Binding, _, _} = elixir:eval(Content, []), - {Value, Binding}. - -separator_test() -> - {334,[]} = eval("3_34"), - {600,[]} = eval("2_00+45_5-5_5"). - -integer_sum_test() -> - {3,[]} = eval("1+2"), - {6,[]} = eval("1+2+3"), - {6,[]} = eval("1+2 +3"), - {6,[]} = eval("1 + 2 + 3"). - -integer_sum_minus_test() -> - {-4,[]} = eval("1-2-3"), - {0,[]} = eval("1+2-3"), - {0,[]} = eval("1 + 2 - 3"). - -integer_mult_test() -> - {6,[]} = eval("1*2*3"), - {6,[]} = eval("1 * 2 * 3"). - -integer_div_test() -> - {0.5,[]} = eval("1 / 2"), - {2.0,[]} = eval("4 / 2"). - -integer_div_rem_test() -> - {2,[]} = eval("div 5, 2"), - {1,[]} = eval("rem 5, 2"). - -integer_mult_div_test() -> - {1.0,[]} = eval("2*1/2"), - {6.0,[]} = eval("3 * 4 / 2"). - -integer_without_parens_test() -> - {17,[]} = eval("3 * 5 + 2"), - {17,[]} = eval("2 + 3 * 5"), - {6.0,[]} = eval("4 / 4 + 5"). - -integer_with_parens_test() -> - {21,[]} = eval("3 * (5 + 2)"), - {21,[]} = eval("3 * (((5 + (2))))"), - {25,[]} = eval("(2 + 3) * 5"), - {0.25,[]} = eval("4 / (11 + 5)"). - -integer_with_unary_test() -> - {2,[]} = eval("- 1 * - 2"). - -integer_eol_test() -> - {3,[]} = eval("1 +\n2"), - {2,[]} = eval("1 *\n2"), - {8,[]} = eval("1 + 2\n3 + 5"), - {8,[]} = eval("1 + 2\n\n\n3 + 5"), - {8,[]} = eval("1 + 2;\n\n3 + 5"), - {8,[]} = eval("1 + (\n2\n) + 3 + 2"), - {8,[]} = eval("1 + (\n\n 2\n\n) + 3 + 2"), - {3,[]} = eval(";1 + 2"), - ?assertError(#{'__struct__' := 'Elixir.SyntaxError'}, eval("1 + 2;\n;\n3 + 5")). - -float_with_parens_and_unary_test() -> - {-21.0,[]} = eval("-3.0 * (5 + 2)"), - {25.0,[]} = eval("(2 + 3.0) * 5"), - {0.25,[]} = eval("4 / (11.0 + 5)"). - -operators_precedence_test() -> - {2, _} = eval("max -1, 2"), - {5, []} = eval("abs -10 + 5"), - {15, []} = eval("abs(-10) + 5"). - -operators_variables_precedence_test() -> - {30, _} = eval("a = 10\nb= 20\na+b"), - {30, _} = eval("a = 10\nb= 20\na + b"). - -operators_variables_precedence_on_namespaces_test() -> - F = fun() -> - eval("defmodule Foo do; def l, do: 1; end; defmodule Bar do; def l(_x), do: 1; end"), - {3,[]} = eval("1 + Foo.l + 1"), - {3,[]} = eval("1 + Foo.l+1"), - {2,[]} = eval("1 + Bar.l +1") - end, - test_helper:run_and_remove(F, ['Elixir.Foo', 'Elixir.Bar']). - -add_add_op_test() -> - {[1,2,3,4],[]} = eval("[1,2] ++ [3,4]"). - -minus_minus_op_test() -> - {[1,2],[]} = eval("[1,2,3] -- [3]"), - {[1,2,3],[]} = eval("[1,2,3] -- [3] -- [3]"). \ No newline at end of file diff --git a/lib/elixir/test/erlang/string_test.erl b/lib/elixir/test/erlang/string_test.erl index aa787110fbb..f74a0fae2d7 100644 --- a/lib/elixir/test/erlang/string_test.erl +++ b/lib/elixir/test/erlang/string_test.erl @@ -1,115 +1,129 @@ +%% SPDX-License-Identifier: Apache-2.0 +%% SPDX-FileCopyrightText: 2021 The Elixir Team +%% SPDX-FileCopyrightText: 2012 Plataformatec + -module(string_test). --include("elixir.hrl"). +-include("../../src/elixir.hrl"). -include_lib("eunit/include/eunit.hrl"). eval(Content) -> - {Value, Binding, _, _} = elixir:eval(Content, []), + Quoted = elixir:'string_to_quoted!'(Content, 1, 1, <<"nofile">>, []), + {Value, Binding, _} = elixir:eval_forms(Quoted, [], elixir:env_for_eval([])), {Value, Binding}. extract_interpolations(String) -> - element(2, elixir_interpolation:extract(1, - #elixir_tokenizer{file = <<"nofile">>}, true, String ++ [$"], $")). + case elixir_interpolation:extract(1, 1, #elixir_tokenizer{}, true, String ++ [$"], $") of + {error, Error} -> + Error; + {_, _, Parts, _, _} -> + Parts + end. % Interpolations extract_interpolations_without_interpolation_test() -> - [<<"foo">>] = extract_interpolations("foo"). + ["foo"] = extract_interpolations("foo"). extract_interpolations_with_escaped_interpolation_test() -> - [<<"f#{o}o">>] = extract_interpolations("f\\#{o}o"). + ["f\\#{o}o"] = extract_interpolations("f\\#{o}o"), + {1, 10, ["f\\#{o}o"], [], _} = + elixir_interpolation:extract(1, 2, #elixir_tokenizer{}, true, "f\\#{o}o\"", $"). extract_interpolations_with_interpolation_test() -> - [<<"f">>, - {1,[{atom,1,o}]}, - <<"o">>] = extract_interpolations("f#{:o}o"). + ["f", + {{1, 2, nil}, {1, 6, nil}, [{atom, {1, 4, _}, o}]}, + "o"] = extract_interpolations("f#{:o}o"). extract_interpolations_with_two_interpolations_test() -> - [<<"f">>, - {1,[{atom,1,o}]},{1,[{atom,1,o}]}, - <<"o">>] = extract_interpolations("f#{:o}#{:o}o"). + ["f", + {{1, 2, nil}, {1, 6, nil}, [{atom, {1, 4, _}, o}]}, + {{1, 7, nil}, {1, 11, nil}, [{atom, {1, 9, _}, o}]}, + "o"] = extract_interpolations("f#{:o}#{:o}o"). extract_interpolations_with_only_two_interpolations_test() -> - [{1,[{atom,1,o}]}, - {1,[{atom,1,o}]}] = extract_interpolations("#{:o}#{:o}"). + [{{1, 1, nil}, {1, 5, nil}, [{atom, {1, 3, _}, o}]}, + {{1, 6, nil}, {1, 10, nil}, [{atom, {1, 8, _}, o}]}] = extract_interpolations("#{:o}#{:o}"). extract_interpolations_with_tuple_inside_interpolation_test() -> - [<<"f">>, - {1,[{'{',1},{number,1,1},{'}',1}]}, - <<"o">>] = extract_interpolations("f#{{1}}o"). + ["f", + {{1, 2, nil}, {1, 7, nil}, [{'{', {1, 4, nil}}, {int, {1, 5, 1}, "1"}, {'}', {1, 6, nil}}]}, + "o"] = extract_interpolations("f#{{1}}o"). extract_interpolations_with_many_expressions_inside_interpolation_test() -> - [<<"f">>, - {1,[{number,1,1},{eol,1,newline},{number,2,2}]}, - <<"o">>] = extract_interpolations("f#{1\n2}o"). + ["f", + {{1, 2, nil}, {2, 2, nil}, [{int, {1, 4, 1}, "1"}, {eol, {1, 5, 1}}, {int, {2, 1, 2}, "2"}]}, + "o"] = extract_interpolations("f#{1\n2}o"). extract_interpolations_with_right_curly_inside_string_inside_interpolation_test() -> - [<<"f">>, - {1,[{bin_string,1,[<<"f}o">>]}]}, - <<"o">>] = extract_interpolations("f#{\"f}o\"}o"). + ["f", + {{1, 2, nil}, {1, 9, nil}, [{bin_string, {1, 4, nil}, [<<"f}o">>]}]}, + "o"] = extract_interpolations("f#{\"f}o\"}o"). extract_interpolations_with_left_curly_inside_string_inside_interpolation_test() -> - [<<"f">>, - {1,[{bin_string,1,[<<"f{o">>]}]}, - <<"o">>] = extract_interpolations("f#{\"f{o\"}o"). + ["f", + {{1, 2, nil}, {1, 9, nil}, [{bin_string, {1, 4, nil}, [<<"f{o">>]}]}, + "o"] = extract_interpolations("f#{\"f{o\"}o"). extract_interpolations_with_escaped_quote_inside_string_inside_interpolation_test() -> - [<<"f">>, - {1,[{bin_string,1,[<<"f\"o">>]}]}, - <<"o">>] = extract_interpolations("f#{\"f\\\"o\"}o"). + ["f", + {{1, 2, nil}, {1, 10, nil}, [{bin_string, {1, 4, nil}, [<<"f\"o">>]}]}, + "o"] = extract_interpolations("f#{\"f\\\"o\"}o"). extract_interpolations_with_less_than_operation_inside_interpolation_test() -> - [<<"f">>, - {1,[{number,1,1},{rel_op,1,'<'},{number,1,2}]}, - <<"o">>] = extract_interpolations("f#{1<2}o"). + ["f", + {{1, 2, nil}, {1, 7, nil}, [{int, {1, 4, 1}, "1"}, {rel_op, {1, 5, nil}, '<'}, {int, {1, 6, 2}, "2"}]}, + "o"] = extract_interpolations("f#{1<2}o"). + +extract_interpolations_with_an_escaped_character_test() -> + ["f", + {{1, 2, nil}, {1, 16, nil}, [{char, {1, 4, "?\\a"}, 7}, {rel_op, {1, 8, nil}, '>'}, {char, {1, 10, "?\\a"}, 7}]} + ] = extract_interpolations("f#{?\\a > ?\\a }"). extract_interpolations_with_invalid_expression_inside_interpolation_test() -> - {1,"invalid token: ",":1}o\""} = extract_interpolations("f#{:1}o"). + {[{line, 1}, {column, 4}], "unexpected token: ", _} = extract_interpolations("f#{:1}o"). %% Bin strings -empty_bin_string_test() -> +empty_test() -> {<<"">>, _} = eval("\"\""). -simple_bin_string_test() -> - {<<"foo">>, _} = eval("\"foo\""). - -bin_string_with_double_quotes_test() -> +string_with_double_quotes_test() -> {<<"f\"o\"o">>, _} = eval("\"f\\\"o\\\"o\""). -bin_string_with_newline_test() -> +string_with_newline_test() -> {<<"f\no">>, _} = eval("\"f\no\""). -bin_string_with_slash_test() -> +string_with_slash_test() -> {<<"f\\o">>, _} = eval("\"f\\\\o\""). -bin_string_with_bell_character_test() -> +string_with_bell_character_test() -> {<<"f\ao">>, _} = eval("\"f\ao\""). -bin_string_with_interpolation_test() -> +string_with_interpolation_test() -> {<<"foo">>, _} = eval("\"f#{\"o\"}o\""). -bin_string_with_another_string_inside_string_inside_interpolation_test() -> +string_with_another_string_inside_string_inside_interpolation_test() -> {<<"fbaro">>, _} = eval("\"f#{\"b#{\"a\"}r\"}o\""). -bin_string_with_another_string_with_curly_inside_interpolation_test() -> +string_with_another_string_with_curly_inside_interpolation_test() -> {<<"fb}ro">>, _} = eval("\"f#{\"b}r\"}o\""). -bin_string_with_atom_with_separator_inside_interpolation_test() -> +string_with_atom_with_separator_inside_interpolation_test() -> {<<"f}o">>, _} = eval("\"f#{\"}\"}o\""). -bin_string_with_lower_case_hex_interpolation_test() -> +string_with_lower_case_hex_interpolation_test() -> {<<"jklmno">>, _} = eval("\"\\x6a\\x6b\\x6c\\x6d\\x6e\\x6f\""). -bin_string_with_upper_case_hex_interpolation_test() -> +string_with_upper_case_hex_interpolation_test() -> {<<"jklmno">>, _} = eval("\"\\x6A\\x6B\\x6C\\x6D\\x6E\\x6F\""). -bin_string_without_interpolation_and_escaped_test() -> +string_without_interpolation_and_escaped_test() -> {<<"f#o">>, _} = eval("\"f\\#o\""). -bin_string_with_escaped_interpolation_test() -> +string_with_escaped_interpolation_test() -> {<<"f#{'o}o">>, _} = eval("\"f\\#{'o}o\""). -bin_string_with_the_end_of_line_slash_test() -> +string_with_the_end_of_line_slash_test() -> {<<"fo">>, _} = eval("\"f\\\no\""), {<<"fo">>, _} = eval("\"f\\\r\no\""). @@ -120,57 +134,7 @@ invalid_string_interpolation_test() -> unterminated_string_interpolation_test() -> ?assertError(#{'__struct__' := 'Elixir.TokenMissingError'}, eval("\"foo")). -%% List strings - -empty_list_string_test() -> - {[], _} = eval("\'\'"). - -simple_list_string_test() -> - {"foo", _} = eval("'foo'"). - -list_string_with_double_quotes_test() -> - {"f'o'o", _} = eval("'f\\'o\\'o'"). - -list_string_with_newline_test() -> - {"f\no", _} = eval("'f\no'"). - -list_string_with_slash_test() -> - {"f\\o", _} = eval("'f\\\\o'"). - -list_string_with_bell_character_test() -> - {"f\ao", _} = eval("'f\ao'"). - -list_string_with_interpolation_test() -> - {"foo", _} = eval("'f#{\"o\"}o'"). - -list_string_with_another_string_with_curly_inside_interpolation_test() -> - {"fb}ro", _} = eval("'f#{\"b}r\"}o'"). - -list_string_with_atom_with_separator_inside_interpolation_test() -> - {"f}o", _} = eval("'f#{\"}\"}o'"). - -list_string_with_lower_case_hex_interpolation_test() -> - {"JKLMNO", _} = eval("'\\x4a\\x4b\\x4c\\x4d\\x4e\\x4f'"). - -list_string_with_upper_case_hex_interpolation_test() -> - {"JKLMNO", _} = eval("'\\x4A\\x4B\\x4C\\x4D\\x4E\\x4F'"). - -list_string_without_interpolation_and_escaped_test() -> - {"f#o", _} = eval("'f\\#o'"). - -list_string_with_escaped_interpolation_test() -> - {"f#{\"o}o", _} = eval("'f\\#{\"o}o'"). - -list_string_with_the_end_of_line_slash_test() -> - {"fo", _} = eval("'f\\\no'"), - {"fo", _} = eval("'f\\\r\no'"). - char_test() -> - {99,[]} = eval("?1 + ?2"), - {10,[]} = eval("?\\n"), - {40,[]} = eval("?\\("). - -%% Binaries - -bitstr_with_integer_test() -> - {<<"fdo">>, _} = eval("<< \"f\", 50+50, \"o\" >>"). + {99, []} = eval("?1 + ?2"), + {10, []} = eval("?\\n"), + {40, []} = eval("?("). diff --git a/lib/elixir/test/erlang/test_helper.erl b/lib/elixir/test/erlang/test_helper.erl index e0890855250..668ae0558e6 100644 --- a/lib/elixir/test/erlang/test_helper.erl +++ b/lib/elixir/test/erlang/test_helper.erl @@ -1,19 +1,19 @@ +%% SPDX-License-Identifier: Apache-2.0 +%% SPDX-FileCopyrightText: 2021 The Elixir Team +%% SPDX-FileCopyrightText: 2012 Plataformatec + -module(test_helper). --include("elixir.hrl"). -export([test/0, run_and_remove/2, throw_elixir/1, throw_erlang/1]). -define(TESTS, [ atom_test, control_test, function_test, - match_test, - module_test, - operators_test, string_test, tokenizer_test ]). test() -> - application:start(elixir), + application:ensure_all_started(elixir), case eunit:test(?TESTS) of error -> erlang:halt(1); _Res -> erlang:halt(0) @@ -30,8 +30,8 @@ run_and_remove(Fun, Modules) -> % Throws an error with the Erlang Abstract Form from the Elixir string throw_elixir(String) -> - Forms = elixir:'string_to_quoted!'(String, 1, <<"nofile">>, []), - {Expr, _, _} = elixir:quoted_to_erl(Forms, elixir:env_for_eval([])), + Forms = elixir:'string_to_quoted!'(String, 1, 1, <<"nofile">>, []), + {Expr, _, _, _} = elixir:quoted_to_erl(Forms, elixir:env_for_eval([])), erlang:error(io:format("~p~n", [Expr])). % Throws an error with the Erlang Abstract Form from the Erlang string diff --git a/lib/elixir/test/erlang/tokenizer_test.erl b/lib/elixir/test/erlang/tokenizer_test.erl index 4cd713860e8..5fecbf6aac5 100644 --- a/lib/elixir/test/erlang/tokenizer_test.erl +++ b/lib/elixir/test/erlang/tokenizer_test.erl @@ -1,146 +1,295 @@ +%% SPDX-License-Identifier: Apache-2.0 +%% SPDX-FileCopyrightText: 2021 The Elixir Team +%% SPDX-FileCopyrightText: 2012 Plataformatec + -module(tokenizer_test). --include("elixir.hrl"). -include_lib("eunit/include/eunit.hrl"). tokenize(String) -> - {ok, _Line, Result} = elixir_tokenizer:tokenize(String, 1, []), - Result. + tokenize(String, []). + +tokenize(String, Opts) -> + {ok, _Line, _Column, _Warnings, Result, []} = elixir_tokenizer:tokenize(String, 1, Opts), + lists:reverse(Result). tokenize_error(String) -> - {error, Error, _, _} = elixir_tokenizer:tokenize(String, 1, []), + {error, Error, _, _, _} = elixir_tokenizer:tokenize(String, 1, []), Error. +tokenize_warnings(String) -> + {ok, _Line, _Column, Warnings, Result, []} = elixir_tokenizer:tokenize(String, 1, []), + {lists:reverse(Result), Warnings}. + type_test() -> - [{number,1,1},{type_op,1,'::'},{number,1,3}] = tokenize("1 :: 3"), - [{identifier,1,foo}, - {'.',1}, - {paren_identifier,1,'::'}, - {'(',1}, - {number,1,3}, - {')',1}] = tokenize("foo.::(3)"). + [{int, {1, 1, 1}, "1"}, + {type_op, {1, 3, nil}, '::'}, + {int, {1, 6, 3}, "3"}] = tokenize("1 :: 3"), + [{'true', {1, 1, nil}}, + {type_op, {1, 5, nil}, '::'}, + {int, {1, 7, 3}, "3"}] = tokenize("true::3"), + [{identifier, {1, 1, _}, name}, + {'.', {1, 5, nil}}, + {paren_identifier, {1, 6, _}, '::'}, + {'(', {1, 8, nil}}, + {int, {1, 9, 3}, "3"}, + {')', {1, 10, nil}}] = tokenize("name.::(3)"). arithmetic_test() -> - [{number,1,1},{dual_op,1,'+'},{number,1,2},{dual_op,1,'+'},{number,1,3}] = tokenize("1 + 2 + 3"). + [{int, {1, 1, 1}, "1"}, + {dual_op, {1, 3, nil}, '+'}, + {int, {1, 5, 2}, "2"}, + {dual_op, {1, 7, nil}, '+'}, + {int, {1, 9, 3}, "3"}] = tokenize("1 + 2 + 3"). op_kw_test() -> - [{atom,1,foo},{dual_op,1,'+'},{atom,1,bar}] = tokenize(":foo+:bar"). + [{atom, {1, 1, _}, foo}, + {dual_op, {1, 5, nil}, '+'}, + {atom, {1, 6, _}, bar}] = tokenize(":foo+:bar"). scientific_test() -> - [{number, 1, 0.1}] = tokenize("1.0e-1"). + [{flt, {1, 1, 0.1}, "1.0e-1"}] = tokenize("1.0e-1"), + [{flt, {1, 1, 0.1}, "1.0E-1"}] = tokenize("1.0E-1"), + [{flt, {1, 1, 1.2345678e-7}, "1_234.567_8e-10"}] = tokenize("1_234.567_8e-10"), + {[{line, 1}, {column, 1}], "invalid float number ", "1.0e309"} = tokenize_error("1.0e309"). hex_bin_octal_test() -> - [{number,1,255}] = tokenize("0xFF"), - [{number,1,255}] = tokenize("0Xff"), - [{number,1,63}] = tokenize("077"), - [{number,1,63}] = tokenize("077"), - [{number,1,3}] = tokenize("0b11"), - [{number,1,3}] = tokenize("0B11"). + [{int, {1, 1, 255}, "0xFF"}] = tokenize("0xFF"), + [{int, {1, 1, 255}, "0xF_F"}] = tokenize("0xF_F"), + [{int, {1, 1, 63}, "0o77"}] = tokenize("0o77"), + [{int, {1, 1, 63}, "0o7_7"}] = tokenize("0o7_7"), + [{int, {1, 1, 3}, "0b11"}] = tokenize("0b11"), + [{int, {1, 1, 3}, "0b1_1"}] = tokenize("0b1_1"). unquoted_atom_test() -> - [{atom, 1, '+'}] = tokenize(":+"), - [{atom, 1, '-'}] = tokenize(":-"), - [{atom, 1, '*'}] = tokenize(":*"), - [{atom, 1, '/'}] = tokenize(":/"), - [{atom, 1, '='}] = tokenize(":="), - [{atom, 1, '&&'}] = tokenize(":&&"). + [{atom, {1, 1, _}, '+'}] = tokenize(":+"), + [{atom, {1, 1, _}, '-'}] = tokenize(":-"), + [{atom, {1, 1, _}, '*'}] = tokenize(":*"), + [{atom, {1, 1, _}, '/'}] = tokenize(":/"), + [{atom, {1, 1, _}, '='}] = tokenize(":="), + [{atom, {1, 1, _}, '&&'}] = tokenize(":&&"). quoted_atom_test() -> - [{atom_unsafe, 1, [<<"foo bar">>]}] = tokenize(":\"foo bar\""). + [{atom_quoted, {1, 1, $"}, 'foo bar'}] = tokenize(":\"foo bar\""). oversized_atom_test() -> - OversizedAtom = [$:|string:copies("a", 256)], - {1, "atom length must be less than system limit", ":"} = tokenize_error(OversizedAtom). + OversizedAtom = string:copies("a", 256), + {[{line, 1}, {column, 1}], "atom length must be less than system limit: ", OversizedAtom} = + tokenize_error([$: | OversizedAtom]). op_atom_test() -> - [{atom,1,f0_1}] = tokenize(":f0_1"). + [{atom, {1, 1, _}, f0_1}] = tokenize(":f0_1"). kw_test() -> - [{kw_identifier, 1, do}] = tokenize("do: "), - [{kw_identifier_unsafe, 1, [<<"foo bar">>]}] = tokenize("\"foo bar\": "). + [{kw_identifier, {1, 1, _}, do}] = tokenize("do: "), + [{kw_identifier, {1, 1, _}, a@}] = tokenize("a@: "), + [{kw_identifier, {1, 1, _}, 'A@'}] = tokenize("A@: "), + [{kw_identifier, {1, 1, _}, a@b}] = tokenize("a@b: "), + [{kw_identifier, {1, 1, _}, 'A@!'}] = tokenize("A@!: "), + [{kw_identifier, {1, 1, _}, 'a@!'}] = tokenize("a@!: "), + [{kw_identifier, {1, 1, _}, foo}, {bin_string, {1, 6, nil}, [<<"bar">>]}] = tokenize("foo: \"bar\""), + [{kw_identifier, {1, 1, _}, '+'}, {bin_string, {1, 6, nil}, [<<"bar">>]}] = tokenize("\"+\": \"bar\""). -integer_test() -> - [{number, 1, 123}] = tokenize("123"), - [{number, 1, 123},{eol, 1, ';'}] = tokenize("123;"), - [{eol, 1, newline}, {number, 3, 123}] = tokenize("\n\n123"), - [{number, 1, 123}, {number, 1, 234}] = tokenize(" 123 234 "). +int_test() -> + [{int, {1, 1, 123}, "123"}] = tokenize("123"), + [{int, {1, 1, 123}, "123"}, {';', {1, 4, 0}}] = tokenize("123;"), + [{eol, {1, 1, 2}}, {int, {3, 1, 123}, "123"}] = tokenize("\n\n123"), + [{int, {1, 3, 123}, "123"}, {int, {1, 8, 234}, "234"}] = tokenize(" 123 234 "), + [{int, {1, 1, 7}, "007"}] = tokenize("007"), + [{int, {1, 1, 100000}, "0100000"}] = tokenize("0100000"). float_test() -> - [{number, 1, 12.3}] = tokenize("12.3"), - [{number, 1, 12.3},{eol, 1, ';'}] = tokenize("12.3;"), - [{eol, 1, newline}, {number, 3, 12.3}] = tokenize("\n\n12.3"), - [{number, 1, 12.3}, {number, 1, 23.4}] = tokenize(" 12.3 23.4 "). - -comments_test() -> - [{number, 1, 1},{eol, 1, newline},{number,2,2}] = tokenize("1 # Comment\n2"). + [{flt, {1, 1, 12.3}, "12.3"}] = tokenize("12.3"), + [{flt, {1, 1, 12.3}, "12.3"}, {';', {1, 5, 0}}] = tokenize("12.3;"), + [{eol, {1, 1, 2}}, {flt, {3, 1, 12.3}, "12.3"}] = tokenize("\n\n12.3"), + [{flt, {1, 3, 12.3}, "12.3"}, {flt, {1, 9, 23.4}, "23.4"}] = tokenize(" 12.3 23.4 "), + [{flt, {1, 1, 12.3}, "00_12.3_00"}] = tokenize("00_12.3_00"), + OversizedFloat = string:copies("9", 310) ++ ".0", + {[{line, 1}, {column, 1}], "invalid float number ", OversizedFloat} = tokenize_error(OversizedFloat). identifier_test() -> - [{identifier,1,abc}] = tokenize("abc "), - [{identifier,1,'abc?'}] = tokenize("abc?"), - [{identifier,1,'abc!'}] = tokenize("abc!"), - [{identifier,1,'a0c!'}] = tokenize("a0c!"), - [{paren_identifier,1,'a0c'},{'(',1},{')',1}] = tokenize("a0c()"), - [{paren_identifier,1,'a0c!'},{'(',1},{')',1}] = tokenize("a0c!()"). + [{identifier, {1, 1, _}, abc}] = tokenize("abc "), + [{identifier, {1, 1, _}, 'abc?'}] = tokenize("abc?"), + [{identifier, {1, 1, _}, 'abc!'}] = tokenize("abc!"), + [{identifier, {1, 1, _}, 'a0c!'}] = tokenize("a0c!"), + [{paren_identifier, {1, 1, _}, 'a0c'}, {'(', {1, 4, nil}}, {')', {1, 5, nil}}] = tokenize("a0c()"), + [{paren_identifier, {1, 1, _}, 'a0c!'}, {'(', {1, 5, nil}}, {')', {1, 6, nil}}] = tokenize("a0c!()"). module_macro_test() -> - [{identifier,1,'__MODULE__'}] = tokenize("__MODULE__"). - -triple_dot_test() -> - [{identifier,1,'...'}] = tokenize("..."), - [{'.',1},{identifier,1,'..'}] = tokenize(". .."). + [{identifier, {1, 1, _}, '__MODULE__'}] = tokenize("__MODULE__"). dot_test() -> - [{identifier,1,foo}, - {'.',1}, - {identifier,1,bar}, - {'.',1}, - {identifier,1,baz}] = tokenize("foo.bar.baz"). + [{identifier, {1, 1, _}, foo}, + {'.', {1, 4, nil}}, + {identifier, {1, 5, _}, bar}, + {'.', {1, 8, nil}}, + {identifier, {1, 9, _}, baz}] = tokenize("foo.bar.baz"). dot_keyword_test() -> - [{identifier,1,foo}, - {'.',1}, - {identifier,1,do}] = tokenize("foo.do"). + [{identifier, {1, 1, _}, foo}, + {'.', {1, 4, nil}}, + {identifier, {1, 5, _}, do}] = tokenize("foo.do"). newline_test() -> - [{identifier,1,foo}, - {'.',2}, - {identifier,2,bar}] = tokenize("foo\n.bar"), - [{number,1,1}, - {two_op,2,'++'}, - {number,2,2}] = tokenize("1\n++2"). + [{identifier, {1, 1, _}, foo}, + {'.', {2, 1, nil}}, + {identifier, {2, 2, _}, bar}] = tokenize("foo\n.bar"), + [{int, {1, 1, 1}, "1"}, + {concat_op, {2, 1, 1}, '++'}, + {int, {2, 3, 2}, "2"}] = tokenize("1\n++2"). + +dot_newline_operator_test() -> + [{identifier, {1, 1, _}, foo}, + {'.', {1, 4, nil}}, + {identifier, {2, 1, _}, '+'}, + {int, {2, 2, 1}, "1"}] = tokenize("foo.\n+1"), + [{identifier, {1, 1, _}, foo}, + {'.', {1, 4, nil}}, + {identifier, {2, 1, _}, '+'}, + {int, {2, 2, 1}, "1"}] = tokenize("foo.#bar\n+1"). + +dot_call_operator_test() -> + [{identifier, {1, 1, _}, f}, + {dot_call_op, {1, 2, nil}, '.'}, + {'(', {1, 3, nil}}, + {')', {1, 4, nil}}] = tokenize("f.()"). aliases_test() -> - [{'aliases',1,['Foo']}] = tokenize("Foo"), - [{'aliases',1,['Foo']}, - {'.',1}, - {'aliases',1,['Bar']}, - {'.',1}, - {'aliases',1,['Baz']}] = tokenize("Foo.Bar.Baz"). + [{'alias', {1, 1, _}, 'Foo'}] = tokenize("Foo"), + [{'alias', {1, 1, _}, 'Foo'}, + {'.', {1, 4, nil}}, + {'alias', {1, 5, _}, 'Bar'}, + {'.', {1, 8, nil}}, + {'alias', {1, 9, _}, 'Baz'}] = tokenize("Foo.Bar.Baz"). string_test() -> - [{bin_string,1,[<<"foo">>]}] = tokenize("\"foo\""), - [{list_string,1,[<<"foo">>]}] = tokenize("'foo'"). + [{bin_string, {1, 1, nil}, [<<"foo">>]}] = tokenize("\"foo\""), + [{bin_string, {1, 1, nil}, [<<"f\"">>]}] = tokenize("\"f\\\"\""). + +heredoc_test() -> + [{bin_heredoc, {1, 1, nil}, 0, [<<"heredoc\n">>]}] = tokenize("\"\"\"\nheredoc\n\"\"\""), + [{bin_heredoc, {1, 1, nil}, 1, [<<"heredoc\n">>]}, {';', {3, 5, 0}}] = tokenize("\"\"\"\n heredoc\n \"\"\";"). empty_string_test() -> - [{bin_string,1,[<<>>]}] = tokenize("\"\""), - [{list_string,1,[<<>>]}] = tokenize("''"). + [{bin_string, {1, 1, nil}, [<<>>]}] = tokenize("\"\""). -addadd_test() -> - [{identifier,1,x},{two_op,1,'++'},{identifier,1,y}] = tokenize("x ++ y"). +concat_test() -> + [{identifier, {1, 1, _}, x}, + {concat_op, {1, 3, nil}, '++'}, + {identifier, {1, 6, _}, y}] = tokenize("x ++ y"), + [{identifier, {1, 1, _}, x}, + {concat_op, {1, 3, nil}, '+++'}, + {identifier, {1, 7, _}, y}] = tokenize("x +++ y"). + +space_test() -> + [{op_identifier, {1, 1, _}, foo}, + {dual_op, {1, 5, nil}, '-'}, + {int, {1, 6, 2}, "2"}] = tokenize("foo -2"), + [{op_identifier, {1, 1, _}, foo}, + {dual_op, {1, 6, nil}, '-'}, + {int, {1, 7, 2}, "2"}] = tokenize("foo -2"). chars_test() -> - [{number,1,97}] = tokenize("?a"), - [{number,1,99}] = tokenize("?c"), - [{number,1,7}] = tokenize("?\\a"), - [{number,1,10}] = tokenize("?\\n"), - [{number,1,92}] = tokenize("?\\\\"), - [{number,1,10}] = tokenize("?\\xa"), - [{number,1,26}] = tokenize("?\\X1a"), - [{number,1,6}] = tokenize("?\\6"), - [{number,1,49}] = tokenize("?\\61"), - [{number,1,255}] = tokenize("?\\377"), - [{number,1,10}] = tokenize("?\\x{a}"), - [{number,1,171}] = tokenize("?\\x{ab}"), - [{number,1,2748}] = tokenize("?\\x{abc}"), - [{number,1,43981}] = tokenize("?\\x{abcd}"), - [{number,1,703710}] = tokenize("?\\x{abcde}"), - [{number,1,1092557}] = tokenize("?\\x{10abcd}"). + [{char, {1, 1, "?a"}, 97}] = tokenize("?a"), + [{char, {1, 1, "?c"}, 99}] = tokenize("?c"), + [{char, {1, 1, "?\\0"}, 0}] = tokenize("?\\0"), + [{char, {1, 1, "?\\a"}, 7}] = tokenize("?\\a"), + [{char, {1, 1, "?\\n"}, 10}] = tokenize("?\\n"), + [{char, {1, 1, "?\\\\"}, 92}] = tokenize("?\\\\"). + +interpolation_test() -> + [{bin_string, {1, 1, nil}, [<<"f">>, {{1, 3, nil},{1, 7, nil}, [{identifier, {1, 5, _}, oo}]}]}, + {concat_op, {1, 10, nil}, '<>'}, + {bin_string, {1, 13, nil}, [<<>>]}] = tokenize("\"f#{oo}\" <> \"\""). + +escaped_interpolation_test() -> + [{bin_string, {1, 1, nil}, [<<"f#{oo}">>]}, + {concat_op, {1, 11, nil}, '<>'}, + {bin_string, {1, 14, nil}, [<<>>]}] = tokenize("\"f\\#{oo}\" <> \"\""). + +capture_test() -> + % Parens precedence + [{capture_op, {1, 1, nil}, '&'}, + {unary_op, {1, 2, nil}, 'not'}, + {int, {1, 6, 1}, "1"}, + {',', {1, 7, 0}}, + {int, {1, 9, 2}, "2"}] = tokenize("¬ 1, 2"), + + % Operators + [{capture_op, {1, 1, nil}, '&'}, + {identifier, {1, 2, _}, '||'}, + {mult_op, {1, 4, nil}, '/'}, + {int, {1, 5, 2}, "2"}] = tokenize("&||/2"), + [{capture_op, {1, 1, nil}, '&'}, + {identifier, {1, 2, _}, 'or'}, + {mult_op, {1, 4, nil}, '/'}, + {int, {1, 5, 2}, "2"}] = tokenize("&or/2"), + [{capture_op,{1,1,nil},'&'}, + {identifier,{1,3,_},'+'}, + {mult_op,{1,4,nil},'/'}, + {int,{1,5,1},"1"}] = tokenize("& +/1"), + [{capture_op,{1,1,nil},'&'}, + {identifier,{1,3,_},'&'}, + {mult_op,{1,4,nil},'/'}, + {int,{1,5,1},"1"}] = tokenize("& &/1"), + [{capture_op,{1,1,nil},'&'}, + {identifier,{1,3,_},'..//'}, + {mult_op,{1,7,nil},'/'}, + {int,{1,8,3},"3"}] = tokenize("& ..///3"), + [{capture_op, {1,1,nil}, '&'}, + {identifier, {1,3,_}, '/'}, + {mult_op, {1,5,nil}, '/'}, + {int, {1,6,2}, "2"}] = tokenize("& / /2"), + [{capture_op, {1,1,nil}, '&'}, + {identifier, {1,2,_}, '/'}, + {mult_op, {1,4,nil}, '/'}, + {int, {1,5,2}, "2"}] = tokenize("&/ /2"), + + % Only operators + [{identifier,{1,1,_},'&'}, + {mult_op,{1,2,nil},'/'}, + {int,{1,3,1},"1"}] = tokenize("&/1"), + [{identifier,{1,1,_},'+'}, + {mult_op,{1,2,nil},'/'}, + {int,{1,3,1},"1"}] = tokenize("+/1"), + [{identifier, {1,1,_}, '/'}, + {mult_op, {1,3,nil}, '/'}, + {int, {1,4,2}, "2"}] = tokenize("/ /2"), + [{identifier, {1,1,_}, '..//'}, + {mult_op, {1,5,nil}, '/'}, + {int, {1,6,3}, "3"}] = tokenize("..///3"). + +vc_merge_conflict_test() -> + {[{line, 1}, {column, 1}], "found an unexpected version control marker, please resolve the conflicts: ", "<<<<<<< HEAD"} = + tokenize_error("<<<<<<< HEAD\n[1, 2, 3]"). + +sigil_terminator_test() -> + [{sigil, {1, 1, nil}, sigil_r, [<<"foo">>], "", nil, <<"/">>}] = tokenize("~r/foo/"), + [{sigil, {1, 1, nil}, sigil_r, [<<"foo">>], "", nil, <<"[">>}] = tokenize("~r[foo]"), + [{sigil, {1, 1, nil}, sigil_r, [<<"foo">>], "", nil, <<"\"">>}] = tokenize("~r\"foo\""), + [{sigil, {1, 1, nil}, sigil_r, [<<"foo">>], "", nil, <<"/">>}, + {comp_op, {1, 9, nil}, '=='}, + {identifier, {1, 12, _}, bar}] = tokenize("~r/foo/ == bar"), + [{sigil, {1, 1, nil}, sigil_r, [<<"foo">>], "iu", nil, <<"/">>}, + {comp_op, {1, 11, nil}, '=='}, + {identifier, {1, 14, _}, bar}] = tokenize("~r/foo/iu == bar"), + [{sigil, {1, 1, nil}, sigil_M, [<<"1 2 3">>], "u8", nil, <<"[">>}] = tokenize("~M[1 2 3]u8"). + +sigil_heredoc_test() -> + [{sigil, {1, 1, nil}, sigil_S, [<<"sigil heredoc\n">>], "", 0, <<"\"\"\"">>}] = tokenize("~S\"\"\"\nsigil heredoc\n\"\"\""), + [{sigil, {1, 1, nil}, sigil_S, [<<"sigil heredoc\n">>], "", 0, <<"'''">>}] = tokenize("~S'''\nsigil heredoc\n'''"), + [{sigil, {1, 1, nil}, sigil_S, [<<"sigil heredoc\n">>], "", 2, <<"\"\"\"">>}] = tokenize("~S\"\"\"\n sigil heredoc\n \"\"\""), + [{sigil, {1, 1, nil}, sigil_s, [<<"sigil heredoc\n">>], "", 2, <<"\"\"\"">>}] = tokenize("~s\"\"\"\n sigil heredoc\n \"\"\""). + +invalid_sigil_delimiter_test() -> + {[{line, 1}, {column, 1}], "invalid sigil delimiter: ", Message} = tokenize_error("~s\\"), + true = lists:prefix("\"\\\" (column 3, code point U+005C)", lists:flatten(Message)). + +deprecated_operators_test() -> + { + [{xor_op, {1, 1, nil}, '^^^'}, {int, {1, 4, 1}, "1"}], + [{{1, 1}, "^^^ is deprecated. It is typically used as xor but it has the wrong precedence, use Bitwise.bxor/2 instead"}] + } = tokenize_warnings("^^^1"), + { + [{unary_op, {1, 1, nil}, '~~~'}, {int, {1, 4, 1}, "1"}], + [{{1, 1}, "~~~ is deprecated. Use Bitwise.bnot/1 instead for clarity"}] + } = tokenize_warnings("~~~1"). diff --git a/lib/elixir/unicode/GraphemeBreakProperty.txt b/lib/elixir/unicode/GraphemeBreakProperty.txt deleted file mode 100644 index f13970a2567..00000000000 --- a/lib/elixir/unicode/GraphemeBreakProperty.txt +++ /dev/null @@ -1,1252 +0,0 @@ -000D ; CR # Cc -000A ; LF # Cc -0000..0009 ; Control # Cc [10] .. -000B..000C ; Control # Cc [2] .. -000E..001F ; Control # Cc [18] .. -007F..009F ; Control # Cc [33] .. -00AD ; Control # Cf SOFT HYPHEN -0600..0605 ; Control # Cf [6] ARABIC NUMBER SIGN..ARABIC NUMBER MARK ABOVE -061C ; Control # Cf ARABIC LETTER MARK -06DD ; Control # Cf ARABIC END OF AYAH -070F ; Control # Cf SYRIAC ABBREVIATION MARK -180E ; Control # Cf MONGOLIAN VOWEL SEPARATOR -200B ; Control # Cf ZERO WIDTH SPACE -200E..200F ; Control # Cf [2] LEFT-TO-RIGHT MARK..RIGHT-TO-LEFT MARK -2028 ; Control # Zl LINE SEPARATOR -2029 ; Control # Zp PARAGRAPH SEPARATOR -202A..202E ; Control # Cf [5] LEFT-TO-RIGHT EMBEDDING..RIGHT-TO-LEFT OVERRIDE -2060..2064 ; Control # Cf [5] WORD JOINER..INVISIBLE PLUS -2065 ; Control # Cn -2066..206F ; Control # Cf [10] LEFT-TO-RIGHT ISOLATE..NOMINAL DIGIT SHAPES -D800..DFFF ; Control # Cs [2048] .. -FEFF ; Control # Cf ZERO WIDTH NO-BREAK SPACE -FFF0..FFF8 ; Control # Cn [9] .. -FFF9..FFFB ; Control # Cf [3] INTERLINEAR ANNOTATION ANCHOR..INTERLINEAR ANNOTATION TERMINATOR -110BD ; Control # Cf KAITHI NUMBER SIGN -1BCA0..1BCA3 ; Control # Cf [4] SHORTHAND FORMAT LETTER OVERLAP..SHORTHAND FORMAT UP STEP -1D173..1D17A ; Control # Cf [8] MUSICAL SYMBOL BEGIN BEAM..MUSICAL SYMBOL END PHRASE -E0000 ; Control # Cn -E0001 ; Control # Cf LANGUAGE TAG -E0002..E001F ; Control # Cn [30] .. -E0020..E007F ; Control # Cf [96] TAG SPACE..CANCEL TAG -E0080..E00FF ; Control # Cn [128] .. -E01F0..E0FFF ; Control # Cn [3600] .. -0300..036F ; Extend # Mn [112] COMBINING GRAVE ACCENT..COMBINING LATIN SMALL LETTER X -0483..0487 ; Extend # Mn [5] COMBINING CYRILLIC TITLO..COMBINING CYRILLIC POKRYTIE -0488..0489 ; Extend # Me [2] COMBINING CYRILLIC HUNDRED THOUSANDS SIGN..COMBINING CYRILLIC MILLIONS SIGN -0591..05BD ; Extend # Mn [45] HEBREW ACCENT ETNAHTA..HEBREW POINT METEG -05BF ; Extend # Mn HEBREW POINT RAFE -05C1..05C2 ; Extend # Mn [2] HEBREW POINT SHIN DOT..HEBREW POINT SIN DOT -05C4..05C5 ; Extend # Mn [2] HEBREW MARK UPPER DOT..HEBREW MARK LOWER DOT -05C7 ; Extend # Mn HEBREW POINT QAMATS QATAN -0610..061A ; Extend # Mn [11] ARABIC SIGN SALLALLAHOU ALAYHE WASSALLAM..ARABIC SMALL KASRA -064B..065F ; Extend # Mn [21] ARABIC FATHATAN..ARABIC WAVY HAMZA BELOW -0670 ; Extend # Mn ARABIC LETTER SUPERSCRIPT ALEF -06D6..06DC ; Extend # Mn [7] ARABIC SMALL HIGH LIGATURE SAD WITH LAM WITH ALEF MAKSURA..ARABIC SMALL HIGH SEEN -06DF..06E4 ; Extend # Mn [6] ARABIC SMALL HIGH ROUNDED ZERO..ARABIC SMALL HIGH MADDA -06E7..06E8 ; Extend # Mn [2] ARABIC SMALL HIGH YEH..ARABIC SMALL HIGH NOON -06EA..06ED ; Extend # Mn [4] ARABIC EMPTY CENTRE LOW STOP..ARABIC SMALL LOW MEEM -0711 ; Extend # Mn SYRIAC LETTER SUPERSCRIPT ALAPH -0730..074A ; Extend # Mn [27] SYRIAC PTHAHA ABOVE..SYRIAC BARREKH -07A6..07B0 ; Extend # Mn [11] THAANA ABAFILI..THAANA SUKUN -07EB..07F3 ; Extend # Mn [9] NKO COMBINING SHORT HIGH TONE..NKO COMBINING DOUBLE DOT ABOVE -0816..0819 ; Extend # Mn [4] SAMARITAN MARK IN..SAMARITAN MARK DAGESH -081B..0823 ; Extend # Mn [9] SAMARITAN MARK EPENTHETIC YUT..SAMARITAN VOWEL SIGN A -0825..0827 ; Extend # Mn [3] SAMARITAN VOWEL SIGN SHORT A..SAMARITAN VOWEL SIGN U -0829..082D ; Extend # Mn [5] SAMARITAN VOWEL SIGN LONG I..SAMARITAN MARK NEQUDAA -0859..085B ; Extend # Mn [3] MANDAIC AFFRICATION MARK..MANDAIC GEMINATION MARK -08E4..0902 ; Extend # Mn [31] ARABIC CURLY FATHA..DEVANAGARI SIGN ANUSVARA -093A ; Extend # Mn DEVANAGARI VOWEL SIGN OE -093C ; Extend # Mn DEVANAGARI SIGN NUKTA -0941..0948 ; Extend # Mn [8] DEVANAGARI VOWEL SIGN U..DEVANAGARI VOWEL SIGN AI -094D ; Extend # Mn DEVANAGARI SIGN VIRAMA -0951..0957 ; Extend # Mn [7] DEVANAGARI STRESS SIGN UDATTA..DEVANAGARI VOWEL SIGN UUE -0962..0963 ; Extend # Mn [2] DEVANAGARI VOWEL SIGN VOCALIC L..DEVANAGARI VOWEL SIGN VOCALIC LL -0981 ; Extend # Mn BENGALI SIGN CANDRABINDU -09BC ; Extend # Mn BENGALI SIGN NUKTA -09BE ; Extend # Mc BENGALI VOWEL SIGN AA -09C1..09C4 ; Extend # Mn [4] BENGALI VOWEL SIGN U..BENGALI VOWEL SIGN VOCALIC RR -09CD ; Extend # Mn BENGALI SIGN VIRAMA -09D7 ; Extend # Mc BENGALI AU LENGTH MARK -09E2..09E3 ; Extend # Mn [2] BENGALI VOWEL SIGN VOCALIC L..BENGALI VOWEL SIGN VOCALIC LL -0A01..0A02 ; Extend # Mn [2] GURMUKHI SIGN ADAK BINDI..GURMUKHI SIGN BINDI -0A3C ; Extend # Mn GURMUKHI SIGN NUKTA -0A41..0A42 ; Extend # Mn [2] GURMUKHI VOWEL SIGN U..GURMUKHI VOWEL SIGN UU -0A47..0A48 ; Extend # Mn [2] GURMUKHI VOWEL SIGN EE..GURMUKHI VOWEL SIGN AI -0A4B..0A4D ; Extend # Mn [3] GURMUKHI VOWEL SIGN OO..GURMUKHI SIGN VIRAMA -0A51 ; Extend # Mn GURMUKHI SIGN UDAAT -0A70..0A71 ; Extend # Mn [2] GURMUKHI TIPPI..GURMUKHI ADDAK -0A75 ; Extend # Mn GURMUKHI SIGN YAKASH -0A81..0A82 ; Extend # Mn [2] GUJARATI SIGN CANDRABINDU..GUJARATI SIGN ANUSVARA -0ABC ; Extend # Mn GUJARATI SIGN NUKTA -0AC1..0AC5 ; Extend # Mn [5] GUJARATI VOWEL SIGN U..GUJARATI VOWEL SIGN CANDRA E -0AC7..0AC8 ; Extend # Mn [2] GUJARATI VOWEL SIGN E..GUJARATI VOWEL SIGN AI -0ACD ; Extend # Mn GUJARATI SIGN VIRAMA -0AE2..0AE3 ; Extend # Mn [2] GUJARATI VOWEL SIGN VOCALIC L..GUJARATI VOWEL SIGN VOCALIC LL -0B01 ; Extend # Mn ORIYA SIGN CANDRABINDU -0B3C ; Extend # Mn ORIYA SIGN NUKTA -0B3E ; Extend # Mc ORIYA VOWEL SIGN AA -0B3F ; Extend # Mn ORIYA VOWEL SIGN I -0B41..0B44 ; Extend # Mn [4] ORIYA VOWEL SIGN U..ORIYA VOWEL SIGN VOCALIC RR -0B4D ; Extend # Mn ORIYA SIGN VIRAMA -0B56 ; Extend # Mn ORIYA AI LENGTH MARK -0B57 ; Extend # Mc ORIYA AU LENGTH MARK -0B62..0B63 ; Extend # Mn [2] ORIYA VOWEL SIGN VOCALIC L..ORIYA VOWEL SIGN VOCALIC LL -0B82 ; Extend # Mn TAMIL SIGN ANUSVARA -0BBE ; Extend # Mc TAMIL VOWEL SIGN AA -0BC0 ; Extend # Mn TAMIL VOWEL SIGN II -0BCD ; Extend # Mn TAMIL SIGN VIRAMA -0BD7 ; Extend # Mc TAMIL AU LENGTH MARK -0C00 ; Extend # Mn TELUGU SIGN COMBINING CANDRABINDU ABOVE -0C3E..0C40 ; Extend # Mn [3] TELUGU VOWEL SIGN AA..TELUGU VOWEL SIGN II -0C46..0C48 ; Extend # Mn [3] TELUGU VOWEL SIGN E..TELUGU VOWEL SIGN AI -0C4A..0C4D ; Extend # Mn [4] TELUGU VOWEL SIGN O..TELUGU SIGN VIRAMA -0C55..0C56 ; Extend # Mn [2] TELUGU LENGTH MARK..TELUGU AI LENGTH MARK -0C62..0C63 ; Extend # Mn [2] TELUGU VOWEL SIGN VOCALIC L..TELUGU VOWEL SIGN VOCALIC LL -0C81 ; Extend # Mn KANNADA SIGN CANDRABINDU -0CBC ; Extend # Mn KANNADA SIGN NUKTA -0CBF ; Extend # Mn KANNADA VOWEL SIGN I -0CC2 ; Extend # Mc KANNADA VOWEL SIGN UU -0CC6 ; Extend # Mn KANNADA VOWEL SIGN E -0CCC..0CCD ; Extend # Mn [2] KANNADA VOWEL SIGN AU..KANNADA SIGN VIRAMA -0CD5..0CD6 ; Extend # Mc [2] KANNADA LENGTH MARK..KANNADA AI LENGTH MARK -0CE2..0CE3 ; Extend # Mn [2] KANNADA VOWEL SIGN VOCALIC L..KANNADA VOWEL SIGN VOCALIC LL -0D01 ; Extend # Mn MALAYALAM SIGN CANDRABINDU -0D3E ; Extend # Mc MALAYALAM VOWEL SIGN AA -0D41..0D44 ; Extend # Mn [4] MALAYALAM VOWEL SIGN U..MALAYALAM VOWEL SIGN VOCALIC RR -0D4D ; Extend # Mn MALAYALAM SIGN VIRAMA -0D57 ; Extend # Mc MALAYALAM AU LENGTH MARK -0D62..0D63 ; Extend # Mn [2] MALAYALAM VOWEL SIGN VOCALIC L..MALAYALAM VOWEL SIGN VOCALIC LL -0DCA ; Extend # Mn SINHALA SIGN AL-LAKUNA -0DCF ; Extend # Mc SINHALA VOWEL SIGN AELA-PILLA -0DD2..0DD4 ; Extend # Mn [3] SINHALA VOWEL SIGN KETTI IS-PILLA..SINHALA VOWEL SIGN KETTI PAA-PILLA -0DD6 ; Extend # Mn SINHALA VOWEL SIGN DIGA PAA-PILLA -0DDF ; Extend # Mc SINHALA VOWEL SIGN GAYANUKITTA -0E31 ; Extend # Mn THAI CHARACTER MAI HAN-AKAT -0E34..0E3A ; Extend # Mn [7] THAI CHARACTER SARA I..THAI CHARACTER PHINTHU -0E47..0E4E ; Extend # Mn [8] THAI CHARACTER MAITAIKHU..THAI CHARACTER YAMAKKAN -0EB1 ; Extend # Mn LAO VOWEL SIGN MAI KAN -0EB4..0EB9 ; Extend # Mn [6] LAO VOWEL SIGN I..LAO VOWEL SIGN UU -0EBB..0EBC ; Extend # Mn [2] LAO VOWEL SIGN MAI KON..LAO SEMIVOWEL SIGN LO -0EC8..0ECD ; Extend # Mn [6] LAO TONE MAI EK..LAO NIGGAHITA -0F18..0F19 ; Extend # Mn [2] TIBETAN ASTROLOGICAL SIGN -KHYUD PA..TIBETAN ASTROLOGICAL SIGN SDONG TSHUGS -0F35 ; Extend # Mn TIBETAN MARK NGAS BZUNG NYI ZLA -0F37 ; Extend # Mn TIBETAN MARK NGAS BZUNG SGOR RTAGS -0F39 ; Extend # Mn TIBETAN MARK TSA -PHRU -0F71..0F7E ; Extend # Mn [14] TIBETAN VOWEL SIGN AA..TIBETAN SIGN RJES SU NGA RO -0F80..0F84 ; Extend # Mn [5] TIBETAN VOWEL SIGN REVERSED I..TIBETAN MARK HALANTA -0F86..0F87 ; Extend # Mn [2] TIBETAN SIGN LCI RTAGS..TIBETAN SIGN YANG RTAGS -0F8D..0F97 ; Extend # Mn [11] TIBETAN SUBJOINED SIGN LCE TSA CAN..TIBETAN SUBJOINED LETTER JA -0F99..0FBC ; Extend # Mn [36] TIBETAN SUBJOINED LETTER NYA..TIBETAN SUBJOINED LETTER FIXED-FORM RA -0FC6 ; Extend # Mn TIBETAN SYMBOL PADMA GDAN -102D..1030 ; Extend # Mn [4] MYANMAR VOWEL SIGN I..MYANMAR VOWEL SIGN UU -1032..1037 ; Extend # Mn [6] MYANMAR VOWEL SIGN AI..MYANMAR SIGN DOT BELOW -1039..103A ; Extend # Mn [2] MYANMAR SIGN VIRAMA..MYANMAR SIGN ASAT -103D..103E ; Extend # Mn [2] MYANMAR CONSONANT SIGN MEDIAL WA..MYANMAR CONSONANT SIGN MEDIAL HA -1058..1059 ; Extend # Mn [2] MYANMAR VOWEL SIGN VOCALIC L..MYANMAR VOWEL SIGN VOCALIC LL -105E..1060 ; Extend # Mn [3] MYANMAR CONSONANT SIGN MON MEDIAL NA..MYANMAR CONSONANT SIGN MON MEDIAL LA -1071..1074 ; Extend # Mn [4] MYANMAR VOWEL SIGN GEBA KAREN I..MYANMAR VOWEL SIGN KAYAH EE -1082 ; Extend # Mn MYANMAR CONSONANT SIGN SHAN MEDIAL WA -1085..1086 ; Extend # Mn [2] MYANMAR VOWEL SIGN SHAN E ABOVE..MYANMAR VOWEL SIGN SHAN FINAL Y -108D ; Extend # Mn MYANMAR SIGN SHAN COUNCIL EMPHATIC TONE -109D ; Extend # Mn MYANMAR VOWEL SIGN AITON AI -135D..135F ; Extend # Mn [3] ETHIOPIC COMBINING GEMINATION AND VOWEL LENGTH MARK..ETHIOPIC COMBINING GEMINATION MARK -1712..1714 ; Extend # Mn [3] TAGALOG VOWEL SIGN I..TAGALOG SIGN VIRAMA -1732..1734 ; Extend # Mn [3] HANUNOO VOWEL SIGN I..HANUNOO SIGN PAMUDPOD -1752..1753 ; Extend # Mn [2] BUHID VOWEL SIGN I..BUHID VOWEL SIGN U -1772..1773 ; Extend # Mn [2] TAGBANWA VOWEL SIGN I..TAGBANWA VOWEL SIGN U -17B4..17B5 ; Extend # Mn [2] KHMER VOWEL INHERENT AQ..KHMER VOWEL INHERENT AA -17B7..17BD ; Extend # Mn [7] KHMER VOWEL SIGN I..KHMER VOWEL SIGN UA -17C6 ; Extend # Mn KHMER SIGN NIKAHIT -17C9..17D3 ; Extend # Mn [11] KHMER SIGN MUUSIKATOAN..KHMER SIGN BATHAMASAT -17DD ; Extend # Mn KHMER SIGN ATTHACAN -180B..180D ; Extend # Mn [3] MONGOLIAN FREE VARIATION SELECTOR ONE..MONGOLIAN FREE VARIATION SELECTOR THREE -18A9 ; Extend # Mn MONGOLIAN LETTER ALI GALI DAGALGA -1920..1922 ; Extend # Mn [3] LIMBU VOWEL SIGN A..LIMBU VOWEL SIGN U -1927..1928 ; Extend # Mn [2] LIMBU VOWEL SIGN E..LIMBU VOWEL SIGN O -1932 ; Extend # Mn LIMBU SMALL LETTER ANUSVARA -1939..193B ; Extend # Mn [3] LIMBU SIGN MUKPHRENG..LIMBU SIGN SA-I -1A17..1A18 ; Extend # Mn [2] BUGINESE VOWEL SIGN I..BUGINESE VOWEL SIGN U -1A1B ; Extend # Mn BUGINESE VOWEL SIGN AE -1A56 ; Extend # Mn TAI THAM CONSONANT SIGN MEDIAL LA -1A58..1A5E ; Extend # Mn [7] TAI THAM SIGN MAI KANG LAI..TAI THAM CONSONANT SIGN SA -1A60 ; Extend # Mn TAI THAM SIGN SAKOT -1A62 ; Extend # Mn TAI THAM VOWEL SIGN MAI SAT -1A65..1A6C ; Extend # Mn [8] TAI THAM VOWEL SIGN I..TAI THAM VOWEL SIGN OA BELOW -1A73..1A7C ; Extend # Mn [10] TAI THAM VOWEL SIGN OA ABOVE..TAI THAM SIGN KHUEN-LUE KARAN -1A7F ; Extend # Mn TAI THAM COMBINING CRYPTOGRAMMIC DOT -1AB0..1ABD ; Extend # Mn [14] COMBINING DOUBLED CIRCUMFLEX ACCENT..COMBINING PARENTHESES BELOW -1ABE ; Extend # Me COMBINING PARENTHESES OVERLAY -1B00..1B03 ; Extend # Mn [4] BALINESE SIGN ULU RICEM..BALINESE SIGN SURANG -1B34 ; Extend # Mn BALINESE SIGN REREKAN -1B36..1B3A ; Extend # Mn [5] BALINESE VOWEL SIGN ULU..BALINESE VOWEL SIGN RA REPA -1B3C ; Extend # Mn BALINESE VOWEL SIGN LA LENGA -1B42 ; Extend # Mn BALINESE VOWEL SIGN PEPET -1B6B..1B73 ; Extend # Mn [9] BALINESE MUSICAL SYMBOL COMBINING TEGEH..BALINESE MUSICAL SYMBOL COMBINING GONG -1B80..1B81 ; Extend # Mn [2] SUNDANESE SIGN PANYECEK..SUNDANESE SIGN PANGLAYAR -1BA2..1BA5 ; Extend # Mn [4] SUNDANESE CONSONANT SIGN PANYAKRA..SUNDANESE VOWEL SIGN PANYUKU -1BA8..1BA9 ; Extend # Mn [2] SUNDANESE VOWEL SIGN PAMEPET..SUNDANESE VOWEL SIGN PANEULEUNG -1BAB..1BAD ; Extend # Mn [3] SUNDANESE SIGN VIRAMA..SUNDANESE CONSONANT SIGN PASANGAN WA -1BE6 ; Extend # Mn BATAK SIGN TOMPI -1BE8..1BE9 ; Extend # Mn [2] BATAK VOWEL SIGN PAKPAK E..BATAK VOWEL SIGN EE -1BED ; Extend # Mn BATAK VOWEL SIGN KARO O -1BEF..1BF1 ; Extend # Mn [3] BATAK VOWEL SIGN U FOR SIMALUNGUN SA..BATAK CONSONANT SIGN H -1C2C..1C33 ; Extend # Mn [8] LEPCHA VOWEL SIGN E..LEPCHA CONSONANT SIGN T -1C36..1C37 ; Extend # Mn [2] LEPCHA SIGN RAN..LEPCHA SIGN NUKTA -1CD0..1CD2 ; Extend # Mn [3] VEDIC TONE KARSHANA..VEDIC TONE PRENKHA -1CD4..1CE0 ; Extend # Mn [13] VEDIC SIGN YAJURVEDIC MIDLINE SVARITA..VEDIC TONE RIGVEDIC KASHMIRI INDEPENDENT SVARITA -1CE2..1CE8 ; Extend # Mn [7] VEDIC SIGN VISARGA SVARITA..VEDIC SIGN VISARGA ANUDATTA WITH TAIL -1CED ; Extend # Mn VEDIC SIGN TIRYAK -1CF4 ; Extend # Mn VEDIC TONE CANDRA ABOVE -1CF8..1CF9 ; Extend # Mn [2] VEDIC TONE RING ABOVE..VEDIC TONE DOUBLE RING ABOVE -1DC0..1DF5 ; Extend # Mn [54] COMBINING DOTTED GRAVE ACCENT..COMBINING UP TACK ABOVE -1DFC..1DFF ; Extend # Mn [4] COMBINING DOUBLE INVERTED BREVE BELOW..COMBINING RIGHT ARROWHEAD AND DOWN ARROWHEAD BELOW -200C..200D ; Extend # Cf [2] ZERO WIDTH NON-JOINER..ZERO WIDTH JOINER -20D0..20DC ; Extend # Mn [13] COMBINING LEFT HARPOON ABOVE..COMBINING FOUR DOTS ABOVE -20DD..20E0 ; Extend # Me [4] COMBINING ENCLOSING CIRCLE..COMBINING ENCLOSING CIRCLE BACKSLASH -20E1 ; Extend # Mn COMBINING LEFT RIGHT ARROW ABOVE -20E2..20E4 ; Extend # Me [3] COMBINING ENCLOSING SCREEN..COMBINING ENCLOSING UPWARD POINTING TRIANGLE -20E5..20F0 ; Extend # Mn [12] COMBINING REVERSE SOLIDUS OVERLAY..COMBINING ASTERISK ABOVE -2CEF..2CF1 ; Extend # Mn [3] COPTIC COMBINING NI ABOVE..COPTIC COMBINING SPIRITUS LENIS -2D7F ; Extend # Mn TIFINAGH CONSONANT JOINER -2DE0..2DFF ; Extend # Mn [32] COMBINING CYRILLIC LETTER BE..COMBINING CYRILLIC LETTER IOTIFIED BIG YUS -302A..302D ; Extend # Mn [4] IDEOGRAPHIC LEVEL TONE MARK..IDEOGRAPHIC ENTERING TONE MARK -302E..302F ; Extend # Mc [2] HANGUL SINGLE DOT TONE MARK..HANGUL DOUBLE DOT TONE MARK -3099..309A ; Extend # Mn [2] COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK..COMBINING KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK -A66F ; Extend # Mn COMBINING CYRILLIC VZMET -A670..A672 ; Extend # Me [3] COMBINING CYRILLIC TEN MILLIONS SIGN..COMBINING CYRILLIC THOUSAND MILLIONS SIGN -A674..A67D ; Extend # Mn [10] COMBINING CYRILLIC LETTER UKRAINIAN IE..COMBINING CYRILLIC PAYEROK -A69F ; Extend # Mn COMBINING CYRILLIC LETTER IOTIFIED E -A6F0..A6F1 ; Extend # Mn [2] BAMUM COMBINING MARK KOQNDON..BAMUM COMBINING MARK TUKWENTIS -A802 ; Extend # Mn SYLOTI NAGRI SIGN DVISVARA -A806 ; Extend # Mn SYLOTI NAGRI SIGN HASANTA -A80B ; Extend # Mn SYLOTI NAGRI SIGN ANUSVARA -A825..A826 ; Extend # Mn [2] SYLOTI NAGRI VOWEL SIGN U..SYLOTI NAGRI VOWEL SIGN E -A8C4 ; Extend # Mn SAURASHTRA SIGN VIRAMA -A8E0..A8F1 ; Extend # Mn [18] COMBINING DEVANAGARI DIGIT ZERO..COMBINING DEVANAGARI SIGN AVAGRAHA -A926..A92D ; Extend # Mn [8] KAYAH LI VOWEL UE..KAYAH LI TONE CALYA PLOPHU -A947..A951 ; Extend # Mn [11] REJANG VOWEL SIGN I..REJANG CONSONANT SIGN R -A980..A982 ; Extend # Mn [3] JAVANESE SIGN PANYANGGA..JAVANESE SIGN LAYAR -A9B3 ; Extend # Mn JAVANESE SIGN CECAK TELU -A9B6..A9B9 ; Extend # Mn [4] JAVANESE VOWEL SIGN WULU..JAVANESE VOWEL SIGN SUKU MENDUT -A9BC ; Extend # Mn JAVANESE VOWEL SIGN PEPET -A9E5 ; Extend # Mn MYANMAR SIGN SHAN SAW -AA29..AA2E ; Extend # Mn [6] CHAM VOWEL SIGN AA..CHAM VOWEL SIGN OE -AA31..AA32 ; Extend # Mn [2] CHAM VOWEL SIGN AU..CHAM VOWEL SIGN UE -AA35..AA36 ; Extend # Mn [2] CHAM CONSONANT SIGN LA..CHAM CONSONANT SIGN WA -AA43 ; Extend # Mn CHAM CONSONANT SIGN FINAL NG -AA4C ; Extend # Mn CHAM CONSONANT SIGN FINAL M -AA7C ; Extend # Mn MYANMAR SIGN TAI LAING TONE-2 -AAB0 ; Extend # Mn TAI VIET MAI KANG -AAB2..AAB4 ; Extend # Mn [3] TAI VIET VOWEL I..TAI VIET VOWEL U -AAB7..AAB8 ; Extend # Mn [2] TAI VIET MAI KHIT..TAI VIET VOWEL IA -AABE..AABF ; Extend # Mn [2] TAI VIET VOWEL AM..TAI VIET TONE MAI EK -AAC1 ; Extend # Mn TAI VIET TONE MAI THO -AAEC..AAED ; Extend # Mn [2] MEETEI MAYEK VOWEL SIGN UU..MEETEI MAYEK VOWEL SIGN AAI -AAF6 ; Extend # Mn MEETEI MAYEK VIRAMA -ABE5 ; Extend # Mn MEETEI MAYEK VOWEL SIGN ANAP -ABE8 ; Extend # Mn MEETEI MAYEK VOWEL SIGN UNAP -ABED ; Extend # Mn MEETEI MAYEK APUN IYEK -FB1E ; Extend # Mn HEBREW POINT JUDEO-SPANISH VARIKA -FE00..FE0F ; Extend # Mn [16] VARIATION SELECTOR-1..VARIATION SELECTOR-16 -FE20..FE2D ; Extend # Mn [14] COMBINING LIGATURE LEFT HALF..COMBINING CONJOINING MACRON BELOW -FF9E..FF9F ; Extend # Lm [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK -101FD ; Extend # Mn PHAISTOS DISC SIGN COMBINING OBLIQUE STROKE -102E0 ; Extend # Mn COPTIC EPACT THOUSANDS MARK -10376..1037A ; Extend # Mn [5] COMBINING OLD PERMIC LETTER AN..COMBINING OLD PERMIC LETTER SII -10A01..10A03 ; Extend # Mn [3] KHAROSHTHI VOWEL SIGN I..KHAROSHTHI VOWEL SIGN VOCALIC R -10A05..10A06 ; Extend # Mn [2] KHAROSHTHI VOWEL SIGN E..KHAROSHTHI VOWEL SIGN O -10A0C..10A0F ; Extend # Mn [4] KHAROSHTHI VOWEL LENGTH MARK..KHAROSHTHI SIGN VISARGA -10A38..10A3A ; Extend # Mn [3] KHAROSHTHI SIGN BAR ABOVE..KHAROSHTHI SIGN DOT BELOW -10A3F ; Extend # Mn KHAROSHTHI VIRAMA -10AE5..10AE6 ; Extend # Mn [2] MANICHAEAN ABBREVIATION MARK ABOVE..MANICHAEAN ABBREVIATION MARK BELOW -11001 ; Extend # Mn BRAHMI SIGN ANUSVARA -11038..11046 ; Extend # Mn [15] BRAHMI VOWEL SIGN AA..BRAHMI VIRAMA -1107F..11081 ; Extend # Mn [3] BRAHMI NUMBER JOINER..KAITHI SIGN ANUSVARA -110B3..110B6 ; Extend # Mn [4] KAITHI VOWEL SIGN U..KAITHI VOWEL SIGN AI -110B9..110BA ; Extend # Mn [2] KAITHI SIGN VIRAMA..KAITHI SIGN NUKTA -11100..11102 ; Extend # Mn [3] CHAKMA SIGN CANDRABINDU..CHAKMA SIGN VISARGA -11127..1112B ; Extend # Mn [5] CHAKMA VOWEL SIGN A..CHAKMA VOWEL SIGN UU -1112D..11134 ; Extend # Mn [8] CHAKMA VOWEL SIGN AI..CHAKMA MAAYYAA -11173 ; Extend # Mn MAHAJANI SIGN NUKTA -11180..11181 ; Extend # Mn [2] SHARADA SIGN CANDRABINDU..SHARADA SIGN ANUSVARA -111B6..111BE ; Extend # Mn [9] SHARADA VOWEL SIGN U..SHARADA VOWEL SIGN O -1122F..11231 ; Extend # Mn [3] KHOJKI VOWEL SIGN U..KHOJKI VOWEL SIGN AI -11234 ; Extend # Mn KHOJKI SIGN ANUSVARA -11236..11237 ; Extend # Mn [2] KHOJKI SIGN NUKTA..KHOJKI SIGN SHADDA -112DF ; Extend # Mn KHUDAWADI SIGN ANUSVARA -112E3..112EA ; Extend # Mn [8] KHUDAWADI VOWEL SIGN U..KHUDAWADI SIGN VIRAMA -11301 ; Extend # Mn GRANTHA SIGN CANDRABINDU -1133C ; Extend # Mn GRANTHA SIGN NUKTA -1133E ; Extend # Mc GRANTHA VOWEL SIGN AA -11340 ; Extend # Mn GRANTHA VOWEL SIGN II -11357 ; Extend # Mc GRANTHA AU LENGTH MARK -11366..1136C ; Extend # Mn [7] COMBINING GRANTHA DIGIT ZERO..COMBINING GRANTHA DIGIT SIX -11370..11374 ; Extend # Mn [5] COMBINING GRANTHA LETTER A..COMBINING GRANTHA LETTER PA -114B0 ; Extend # Mc TIRHUTA VOWEL SIGN AA -114B3..114B8 ; Extend # Mn [6] TIRHUTA VOWEL SIGN U..TIRHUTA VOWEL SIGN VOCALIC LL -114BA ; Extend # Mn TIRHUTA VOWEL SIGN SHORT E -114BD ; Extend # Mc TIRHUTA VOWEL SIGN SHORT O -114BF..114C0 ; Extend # Mn [2] TIRHUTA SIGN CANDRABINDU..TIRHUTA SIGN ANUSVARA -114C2..114C3 ; Extend # Mn [2] TIRHUTA SIGN VIRAMA..TIRHUTA SIGN NUKTA -115AF ; Extend # Mc SIDDHAM VOWEL SIGN AA -115B2..115B5 ; Extend # Mn [4] SIDDHAM VOWEL SIGN U..SIDDHAM VOWEL SIGN VOCALIC RR -115BC..115BD ; Extend # Mn [2] SIDDHAM SIGN CANDRABINDU..SIDDHAM SIGN ANUSVARA -115BF..115C0 ; Extend # Mn [2] SIDDHAM SIGN VIRAMA..SIDDHAM SIGN NUKTA -11633..1163A ; Extend # Mn [8] MODI VOWEL SIGN U..MODI VOWEL SIGN AI -1163D ; Extend # Mn MODI SIGN ANUSVARA -1163F..11640 ; Extend # Mn [2] MODI SIGN VIRAMA..MODI SIGN ARDHACANDRA -116AB ; Extend # Mn TAKRI SIGN ANUSVARA -116AD ; Extend # Mn TAKRI VOWEL SIGN AA -116B0..116B5 ; Extend # Mn [6] TAKRI VOWEL SIGN U..TAKRI VOWEL SIGN AU -116B7 ; Extend # Mn TAKRI SIGN NUKTA -16AF0..16AF4 ; Extend # Mn [5] BASSA VAH COMBINING HIGH TONE..BASSA VAH COMBINING HIGH-LOW TONE -16B30..16B36 ; Extend # Mn [7] PAHAWH HMONG MARK CIM TUB..PAHAWH HMONG MARK CIM TAUM -16F8F..16F92 ; Extend # Mn [4] MIAO TONE RIGHT..MIAO TONE BELOW -1BC9D..1BC9E ; Extend # Mn [2] DUPLOYAN THICK LETTER SELECTOR..DUPLOYAN DOUBLE MARK -1D165 ; Extend # Mc MUSICAL SYMBOL COMBINING STEM -1D167..1D169 ; Extend # Mn [3] MUSICAL SYMBOL COMBINING TREMOLO-1..MUSICAL SYMBOL COMBINING TREMOLO-3 -1D16E..1D172 ; Extend # Mc [5] MUSICAL SYMBOL COMBINING FLAG-1..MUSICAL SYMBOL COMBINING FLAG-5 -1D17B..1D182 ; Extend # Mn [8] MUSICAL SYMBOL COMBINING ACCENT..MUSICAL SYMBOL COMBINING LOURE -1D185..1D18B ; Extend # Mn [7] MUSICAL SYMBOL COMBINING DOIT..MUSICAL SYMBOL COMBINING TRIPLE TONGUE -1D1AA..1D1AD ; Extend # Mn [4] MUSICAL SYMBOL COMBINING DOWN BOW..MUSICAL SYMBOL COMBINING SNAP PIZZICATO -1D242..1D244 ; Extend # Mn [3] COMBINING GREEK MUSICAL TRISEME..COMBINING GREEK MUSICAL PENTASEME -1E8D0..1E8D6 ; Extend # Mn [7] MENDE KIKAKUI COMBINING NUMBER TEENS..MENDE KIKAKUI COMBINING NUMBER MILLIONS -E0100..E01EF ; Extend # Mn [240] VARIATION SELECTOR-17..VARIATION SELECTOR-256 -1F1E6..1F1FF ; Regional_Indicator # So [26] REGIONAL INDICATOR SYMBOL LETTER A..REGIONAL INDICATOR SYMBOL LETTER Z -0903 ; SpacingMark # Mc DEVANAGARI SIGN VISARGA -093B ; SpacingMark # Mc DEVANAGARI VOWEL SIGN OOE -093E..0940 ; SpacingMark # Mc [3] DEVANAGARI VOWEL SIGN AA..DEVANAGARI VOWEL SIGN II -0949..094C ; SpacingMark # Mc [4] DEVANAGARI VOWEL SIGN CANDRA O..DEVANAGARI VOWEL SIGN AU -094E..094F ; SpacingMark # Mc [2] DEVANAGARI VOWEL SIGN PRISHTHAMATRA E..DEVANAGARI VOWEL SIGN AW -0982..0983 ; SpacingMark # Mc [2] BENGALI SIGN ANUSVARA..BENGALI SIGN VISARGA -09BF..09C0 ; SpacingMark # Mc [2] BENGALI VOWEL SIGN I..BENGALI VOWEL SIGN II -09C7..09C8 ; SpacingMark # Mc [2] BENGALI VOWEL SIGN E..BENGALI VOWEL SIGN AI -09CB..09CC ; SpacingMark # Mc [2] BENGALI VOWEL SIGN O..BENGALI VOWEL SIGN AU -0A03 ; SpacingMark # Mc GURMUKHI SIGN VISARGA -0A3E..0A40 ; SpacingMark # Mc [3] GURMUKHI VOWEL SIGN AA..GURMUKHI VOWEL SIGN II -0A83 ; SpacingMark # Mc GUJARATI SIGN VISARGA -0ABE..0AC0 ; SpacingMark # Mc [3] GUJARATI VOWEL SIGN AA..GUJARATI VOWEL SIGN II -0AC9 ; SpacingMark # Mc GUJARATI VOWEL SIGN CANDRA O -0ACB..0ACC ; SpacingMark # Mc [2] GUJARATI VOWEL SIGN O..GUJARATI VOWEL SIGN AU -0B02..0B03 ; SpacingMark # Mc [2] ORIYA SIGN ANUSVARA..ORIYA SIGN VISARGA -0B40 ; SpacingMark # Mc ORIYA VOWEL SIGN II -0B47..0B48 ; SpacingMark # Mc [2] ORIYA VOWEL SIGN E..ORIYA VOWEL SIGN AI -0B4B..0B4C ; SpacingMark # Mc [2] ORIYA VOWEL SIGN O..ORIYA VOWEL SIGN AU -0BBF ; SpacingMark # Mc TAMIL VOWEL SIGN I -0BC1..0BC2 ; SpacingMark # Mc [2] TAMIL VOWEL SIGN U..TAMIL VOWEL SIGN UU -0BC6..0BC8 ; SpacingMark # Mc [3] TAMIL VOWEL SIGN E..TAMIL VOWEL SIGN AI -0BCA..0BCC ; SpacingMark # Mc [3] TAMIL VOWEL SIGN O..TAMIL VOWEL SIGN AU -0C01..0C03 ; SpacingMark # Mc [3] TELUGU SIGN CANDRABINDU..TELUGU SIGN VISARGA -0C41..0C44 ; SpacingMark # Mc [4] TELUGU VOWEL SIGN U..TELUGU VOWEL SIGN VOCALIC RR -0C82..0C83 ; SpacingMark # Mc [2] KANNADA SIGN ANUSVARA..KANNADA SIGN VISARGA -0CBE ; SpacingMark # Mc KANNADA VOWEL SIGN AA -0CC0..0CC1 ; SpacingMark # Mc [2] KANNADA VOWEL SIGN II..KANNADA VOWEL SIGN U -0CC3..0CC4 ; SpacingMark # Mc [2] KANNADA VOWEL SIGN VOCALIC R..KANNADA VOWEL SIGN VOCALIC RR -0CC7..0CC8 ; SpacingMark # Mc [2] KANNADA VOWEL SIGN EE..KANNADA VOWEL SIGN AI -0CCA..0CCB ; SpacingMark # Mc [2] KANNADA VOWEL SIGN O..KANNADA VOWEL SIGN OO -0D02..0D03 ; SpacingMark # Mc [2] MALAYALAM SIGN ANUSVARA..MALAYALAM SIGN VISARGA -0D3F..0D40 ; SpacingMark # Mc [2] MALAYALAM VOWEL SIGN I..MALAYALAM VOWEL SIGN II -0D46..0D48 ; SpacingMark # Mc [3] MALAYALAM VOWEL SIGN E..MALAYALAM VOWEL SIGN AI -0D4A..0D4C ; SpacingMark # Mc [3] MALAYALAM VOWEL SIGN O..MALAYALAM VOWEL SIGN AU -0D82..0D83 ; SpacingMark # Mc [2] SINHALA SIGN ANUSVARAYA..SINHALA SIGN VISARGAYA -0DD0..0DD1 ; SpacingMark # Mc [2] SINHALA VOWEL SIGN KETTI AEDA-PILLA..SINHALA VOWEL SIGN DIGA AEDA-PILLA -0DD8..0DDE ; SpacingMark # Mc [7] SINHALA VOWEL SIGN GAETTA-PILLA..SINHALA VOWEL SIGN KOMBUVA HAA GAYANUKITTA -0DF2..0DF3 ; SpacingMark # Mc [2] SINHALA VOWEL SIGN DIGA GAETTA-PILLA..SINHALA VOWEL SIGN DIGA GAYANUKITTA -0E33 ; SpacingMark # Lo THAI CHARACTER SARA AM -0EB3 ; SpacingMark # Lo LAO VOWEL SIGN AM -0F3E..0F3F ; SpacingMark # Mc [2] TIBETAN SIGN YAR TSHES..TIBETAN SIGN MAR TSHES -0F7F ; SpacingMark # Mc TIBETAN SIGN RNAM BCAD -1031 ; SpacingMark # Mc MYANMAR VOWEL SIGN E -103B..103C ; SpacingMark # Mc [2] MYANMAR CONSONANT SIGN MEDIAL YA..MYANMAR CONSONANT SIGN MEDIAL RA -1056..1057 ; SpacingMark # Mc [2] MYANMAR VOWEL SIGN VOCALIC R..MYANMAR VOWEL SIGN VOCALIC RR -1084 ; SpacingMark # Mc MYANMAR VOWEL SIGN SHAN E -17B6 ; SpacingMark # Mc KHMER VOWEL SIGN AA -17BE..17C5 ; SpacingMark # Mc [8] KHMER VOWEL SIGN OE..KHMER VOWEL SIGN AU -17C7..17C8 ; SpacingMark # Mc [2] KHMER SIGN REAHMUK..KHMER SIGN YUUKALEAPINTU -1923..1926 ; SpacingMark # Mc [4] LIMBU VOWEL SIGN EE..LIMBU VOWEL SIGN AU -1929..192B ; SpacingMark # Mc [3] LIMBU SUBJOINED LETTER YA..LIMBU SUBJOINED LETTER WA -1930..1931 ; SpacingMark # Mc [2] LIMBU SMALL LETTER KA..LIMBU SMALL LETTER NGA -1933..1938 ; SpacingMark # Mc [6] LIMBU SMALL LETTER TA..LIMBU SMALL LETTER LA -19B5..19B7 ; SpacingMark # Mc [3] NEW TAI LUE VOWEL SIGN E..NEW TAI LUE VOWEL SIGN O -19BA ; SpacingMark # Mc NEW TAI LUE VOWEL SIGN AY -1A19..1A1A ; SpacingMark # Mc [2] BUGINESE VOWEL SIGN E..BUGINESE VOWEL SIGN O -1A55 ; SpacingMark # Mc TAI THAM CONSONANT SIGN MEDIAL RA -1A57 ; SpacingMark # Mc TAI THAM CONSONANT SIGN LA TANG LAI -1A6D..1A72 ; SpacingMark # Mc [6] TAI THAM VOWEL SIGN OY..TAI THAM VOWEL SIGN THAM AI -1B04 ; SpacingMark # Mc BALINESE SIGN BISAH -1B35 ; SpacingMark # Mc BALINESE VOWEL SIGN TEDUNG -1B3B ; SpacingMark # Mc BALINESE VOWEL SIGN RA REPA TEDUNG -1B3D..1B41 ; SpacingMark # Mc [5] BALINESE VOWEL SIGN LA LENGA TEDUNG..BALINESE VOWEL SIGN TALING REPA TEDUNG -1B43..1B44 ; SpacingMark # Mc [2] BALINESE VOWEL SIGN PEPET TEDUNG..BALINESE ADEG ADEG -1B82 ; SpacingMark # Mc SUNDANESE SIGN PANGWISAD -1BA1 ; SpacingMark # Mc SUNDANESE CONSONANT SIGN PAMINGKAL -1BA6..1BA7 ; SpacingMark # Mc [2] SUNDANESE VOWEL SIGN PANAELAENG..SUNDANESE VOWEL SIGN PANOLONG -1BAA ; SpacingMark # Mc SUNDANESE SIGN PAMAAEH -1BE7 ; SpacingMark # Mc BATAK VOWEL SIGN E -1BEA..1BEC ; SpacingMark # Mc [3] BATAK VOWEL SIGN I..BATAK VOWEL SIGN O -1BEE ; SpacingMark # Mc BATAK VOWEL SIGN U -1BF2..1BF3 ; SpacingMark # Mc [2] BATAK PANGOLAT..BATAK PANONGONAN -1C24..1C2B ; SpacingMark # Mc [8] LEPCHA SUBJOINED LETTER YA..LEPCHA VOWEL SIGN UU -1C34..1C35 ; SpacingMark # Mc [2] LEPCHA CONSONANT SIGN NYIN-DO..LEPCHA CONSONANT SIGN KANG -1CE1 ; SpacingMark # Mc VEDIC TONE ATHARVAVEDIC INDEPENDENT SVARITA -1CF2..1CF3 ; SpacingMark # Mc [2] VEDIC SIGN ARDHAVISARGA..VEDIC SIGN ROTATED ARDHAVISARGA -A823..A824 ; SpacingMark # Mc [2] SYLOTI NAGRI VOWEL SIGN A..SYLOTI NAGRI VOWEL SIGN I -A827 ; SpacingMark # Mc SYLOTI NAGRI VOWEL SIGN OO -A880..A881 ; SpacingMark # Mc [2] SAURASHTRA SIGN ANUSVARA..SAURASHTRA SIGN VISARGA -A8B4..A8C3 ; SpacingMark # Mc [16] SAURASHTRA CONSONANT SIGN HAARU..SAURASHTRA VOWEL SIGN AU -A952..A953 ; SpacingMark # Mc [2] REJANG CONSONANT SIGN H..REJANG VIRAMA -A983 ; SpacingMark # Mc JAVANESE SIGN WIGNYAN -A9B4..A9B5 ; SpacingMark # Mc [2] JAVANESE VOWEL SIGN TARUNG..JAVANESE VOWEL SIGN TOLONG -A9BA..A9BB ; SpacingMark # Mc [2] JAVANESE VOWEL SIGN TALING..JAVANESE VOWEL SIGN DIRGA MURE -A9BD..A9C0 ; SpacingMark # Mc [4] JAVANESE CONSONANT SIGN KERET..JAVANESE PANGKON -AA2F..AA30 ; SpacingMark # Mc [2] CHAM VOWEL SIGN O..CHAM VOWEL SIGN AI -AA33..AA34 ; SpacingMark # Mc [2] CHAM CONSONANT SIGN YA..CHAM CONSONANT SIGN RA -AA4D ; SpacingMark # Mc CHAM CONSONANT SIGN FINAL H -AAEB ; SpacingMark # Mc MEETEI MAYEK VOWEL SIGN II -AAEE..AAEF ; SpacingMark # Mc [2] MEETEI MAYEK VOWEL SIGN AU..MEETEI MAYEK VOWEL SIGN AAU -AAF5 ; SpacingMark # Mc MEETEI MAYEK VOWEL SIGN VISARGA -ABE3..ABE4 ; SpacingMark # Mc [2] MEETEI MAYEK VOWEL SIGN ONAP..MEETEI MAYEK VOWEL SIGN INAP -ABE6..ABE7 ; SpacingMark # Mc [2] MEETEI MAYEK VOWEL SIGN YENAP..MEETEI MAYEK VOWEL SIGN SOUNAP -ABE9..ABEA ; SpacingMark # Mc [2] MEETEI MAYEK VOWEL SIGN CHEINAP..MEETEI MAYEK VOWEL SIGN NUNG -ABEC ; SpacingMark # Mc MEETEI MAYEK LUM IYEK -11000 ; SpacingMark # Mc BRAHMI SIGN CANDRABINDU -11002 ; SpacingMark # Mc BRAHMI SIGN VISARGA -11082 ; SpacingMark # Mc KAITHI SIGN VISARGA -110B0..110B2 ; SpacingMark # Mc [3] KAITHI VOWEL SIGN AA..KAITHI VOWEL SIGN II -110B7..110B8 ; SpacingMark # Mc [2] KAITHI VOWEL SIGN O..KAITHI VOWEL SIGN AU -1112C ; SpacingMark # Mc CHAKMA VOWEL SIGN E -11182 ; SpacingMark # Mc SHARADA SIGN VISARGA -111B3..111B5 ; SpacingMark # Mc [3] SHARADA VOWEL SIGN AA..SHARADA VOWEL SIGN II -111BF..111C0 ; SpacingMark # Mc [2] SHARADA VOWEL SIGN AU..SHARADA SIGN VIRAMA -1122C..1122E ; SpacingMark # Mc [3] KHOJKI VOWEL SIGN AA..KHOJKI VOWEL SIGN II -11232..11233 ; SpacingMark # Mc [2] KHOJKI VOWEL SIGN O..KHOJKI VOWEL SIGN AU -11235 ; SpacingMark # Mc KHOJKI SIGN VIRAMA -112E0..112E2 ; SpacingMark # Mc [3] KHUDAWADI VOWEL SIGN AA..KHUDAWADI VOWEL SIGN II -11302..11303 ; SpacingMark # Mc [2] GRANTHA SIGN ANUSVARA..GRANTHA SIGN VISARGA -1133F ; SpacingMark # Mc GRANTHA VOWEL SIGN I -11341..11344 ; SpacingMark # Mc [4] GRANTHA VOWEL SIGN U..GRANTHA VOWEL SIGN VOCALIC RR -11347..11348 ; SpacingMark # Mc [2] GRANTHA VOWEL SIGN EE..GRANTHA VOWEL SIGN AI -1134B..1134D ; SpacingMark # Mc [3] GRANTHA VOWEL SIGN OO..GRANTHA SIGN VIRAMA -11362..11363 ; SpacingMark # Mc [2] GRANTHA VOWEL SIGN VOCALIC L..GRANTHA VOWEL SIGN VOCALIC LL -114B1..114B2 ; SpacingMark # Mc [2] TIRHUTA VOWEL SIGN I..TIRHUTA VOWEL SIGN II -114B9 ; SpacingMark # Mc TIRHUTA VOWEL SIGN E -114BB..114BC ; SpacingMark # Mc [2] TIRHUTA VOWEL SIGN AI..TIRHUTA VOWEL SIGN O -114BE ; SpacingMark # Mc TIRHUTA VOWEL SIGN AU -114C1 ; SpacingMark # Mc TIRHUTA SIGN VISARGA -115B0..115B1 ; SpacingMark # Mc [2] SIDDHAM VOWEL SIGN I..SIDDHAM VOWEL SIGN II -115B8..115BB ; SpacingMark # Mc [4] SIDDHAM VOWEL SIGN E..SIDDHAM VOWEL SIGN AU -115BE ; SpacingMark # Mc SIDDHAM SIGN VISARGA -11630..11632 ; SpacingMark # Mc [3] MODI VOWEL SIGN AA..MODI VOWEL SIGN II -1163B..1163C ; SpacingMark # Mc [2] MODI VOWEL SIGN O..MODI VOWEL SIGN AU -1163E ; SpacingMark # Mc MODI SIGN VISARGA -116AC ; SpacingMark # Mc TAKRI SIGN VISARGA -116AE..116AF ; SpacingMark # Mc [2] TAKRI VOWEL SIGN I..TAKRI VOWEL SIGN II -116B6 ; SpacingMark # Mc TAKRI SIGN VIRAMA -16F51..16F7E ; SpacingMark # Mc [46] MIAO SIGN ASPIRATION..MIAO VOWEL SIGN NG -1D166 ; SpacingMark # Mc MUSICAL SYMBOL COMBINING SPRECHGESANG STEM -1D16D ; SpacingMark # Mc MUSICAL SYMBOL COMBINING AUGMENTATION DOT -1100..115F ; L # Lo [96] HANGUL CHOSEONG KIYEOK..HANGUL CHOSEONG FILLER -A960..A97C ; L # Lo [29] HANGUL CHOSEONG TIKEUT-MIEUM..HANGUL CHOSEONG SSANGYEORINHIEUH -1160..11A7 ; V # Lo [72] HANGUL JUNGSEONG FILLER..HANGUL JUNGSEONG O-YAE -D7B0..D7C6 ; V # Lo [23] HANGUL JUNGSEONG O-YEO..HANGUL JUNGSEONG ARAEA-E -11A8..11FF ; T # Lo [88] HANGUL JONGSEONG KIYEOK..HANGUL JONGSEONG SSANGNIEUN -D7CB..D7FB ; T # Lo [49] HANGUL JONGSEONG NIEUN-RIEUL..HANGUL JONGSEONG PHIEUPH-THIEUTH -AC00 ; LV # Lo HANGUL SYLLABLE GA -AC1C ; LV # Lo HANGUL SYLLABLE GAE -AC38 ; LV # Lo HANGUL SYLLABLE GYA -AC54 ; LV # Lo HANGUL SYLLABLE GYAE -AC70 ; LV # Lo HANGUL SYLLABLE GEO -AC8C ; LV # Lo HANGUL SYLLABLE GE -ACA8 ; LV # Lo HANGUL SYLLABLE GYEO -ACC4 ; LV # Lo HANGUL SYLLABLE GYE -ACE0 ; LV # Lo HANGUL SYLLABLE GO -ACFC ; LV # Lo HANGUL SYLLABLE GWA -AD18 ; LV # Lo HANGUL SYLLABLE GWAE -AD34 ; LV # Lo HANGUL SYLLABLE GOE -AD50 ; LV # Lo HANGUL SYLLABLE GYO -AD6C ; LV # Lo HANGUL SYLLABLE GU -AD88 ; LV # Lo HANGUL SYLLABLE GWEO -ADA4 ; LV # Lo HANGUL SYLLABLE GWE -ADC0 ; LV # Lo HANGUL SYLLABLE GWI -ADDC ; LV # Lo HANGUL SYLLABLE GYU -ADF8 ; LV # Lo HANGUL SYLLABLE GEU -AE14 ; LV # Lo HANGUL SYLLABLE GYI -AE30 ; LV # Lo HANGUL SYLLABLE GI -AE4C ; LV # Lo HANGUL SYLLABLE GGA -AE68 ; LV # Lo HANGUL SYLLABLE GGAE -AE84 ; LV # Lo HANGUL SYLLABLE GGYA -AEA0 ; LV # Lo HANGUL SYLLABLE GGYAE -AEBC ; LV # Lo HANGUL SYLLABLE GGEO -AED8 ; LV # Lo HANGUL SYLLABLE GGE -AEF4 ; LV # Lo HANGUL SYLLABLE GGYEO -AF10 ; LV # Lo HANGUL SYLLABLE GGYE -AF2C ; LV # Lo HANGUL SYLLABLE GGO -AF48 ; LV # Lo HANGUL SYLLABLE GGWA -AF64 ; LV # Lo HANGUL SYLLABLE GGWAE -AF80 ; LV # Lo HANGUL SYLLABLE GGOE -AF9C ; LV # Lo HANGUL SYLLABLE GGYO -AFB8 ; LV # Lo HANGUL SYLLABLE GGU -AFD4 ; LV # Lo HANGUL SYLLABLE GGWEO -AFF0 ; LV # Lo HANGUL SYLLABLE GGWE -B00C ; LV # Lo HANGUL SYLLABLE GGWI -B028 ; LV # Lo HANGUL SYLLABLE GGYU -B044 ; LV # Lo HANGUL SYLLABLE GGEU -B060 ; LV # Lo HANGUL SYLLABLE GGYI -B07C ; LV # Lo HANGUL SYLLABLE GGI -B098 ; LV # Lo HANGUL SYLLABLE NA -B0B4 ; LV # Lo HANGUL SYLLABLE NAE -B0D0 ; LV # Lo HANGUL SYLLABLE NYA -B0EC ; LV # Lo HANGUL SYLLABLE NYAE -B108 ; LV # Lo HANGUL SYLLABLE NEO -B124 ; LV # Lo HANGUL SYLLABLE NE -B140 ; LV # Lo HANGUL SYLLABLE NYEO -B15C ; LV # Lo HANGUL SYLLABLE NYE -B178 ; LV # Lo HANGUL SYLLABLE NO -B194 ; LV # Lo HANGUL SYLLABLE NWA -B1B0 ; LV # Lo HANGUL SYLLABLE NWAE -B1CC ; LV # Lo HANGUL SYLLABLE NOE -B1E8 ; LV # Lo HANGUL SYLLABLE NYO -B204 ; LV # Lo HANGUL SYLLABLE NU -B220 ; LV # Lo HANGUL SYLLABLE NWEO -B23C ; LV # Lo HANGUL SYLLABLE NWE -B258 ; LV # Lo HANGUL SYLLABLE NWI -B274 ; LV # Lo HANGUL SYLLABLE NYU -B290 ; LV # Lo HANGUL SYLLABLE NEU -B2AC ; LV # Lo HANGUL SYLLABLE NYI -B2C8 ; LV # Lo HANGUL SYLLABLE NI -B2E4 ; LV # Lo HANGUL SYLLABLE DA -B300 ; LV # Lo HANGUL SYLLABLE DAE -B31C ; LV # Lo HANGUL SYLLABLE DYA -B338 ; LV # Lo HANGUL SYLLABLE DYAE -B354 ; LV # Lo HANGUL SYLLABLE DEO -B370 ; LV # Lo HANGUL SYLLABLE DE -B38C ; LV # Lo HANGUL SYLLABLE DYEO -B3A8 ; LV # Lo HANGUL SYLLABLE DYE -B3C4 ; LV # Lo HANGUL SYLLABLE DO -B3E0 ; LV # Lo HANGUL SYLLABLE DWA -B3FC ; LV # Lo HANGUL SYLLABLE DWAE -B418 ; LV # Lo HANGUL SYLLABLE DOE -B434 ; LV # Lo HANGUL SYLLABLE DYO -B450 ; LV # Lo HANGUL SYLLABLE DU -B46C ; LV # Lo HANGUL SYLLABLE DWEO -B488 ; LV # Lo HANGUL SYLLABLE DWE -B4A4 ; LV # Lo HANGUL SYLLABLE DWI -B4C0 ; LV # Lo HANGUL SYLLABLE DYU -B4DC ; LV # Lo HANGUL SYLLABLE DEU -B4F8 ; LV # Lo HANGUL SYLLABLE DYI -B514 ; LV # Lo HANGUL SYLLABLE DI -B530 ; LV # Lo HANGUL SYLLABLE DDA -B54C ; LV # Lo HANGUL SYLLABLE DDAE -B568 ; LV # Lo HANGUL SYLLABLE DDYA -B584 ; LV # Lo HANGUL SYLLABLE DDYAE -B5A0 ; LV # Lo HANGUL SYLLABLE DDEO -B5BC ; LV # Lo HANGUL SYLLABLE DDE -B5D8 ; LV # Lo HANGUL SYLLABLE DDYEO -B5F4 ; LV # Lo HANGUL SYLLABLE DDYE -B610 ; LV # Lo HANGUL SYLLABLE DDO -B62C ; LV # Lo HANGUL SYLLABLE DDWA -B648 ; LV # Lo HANGUL SYLLABLE DDWAE -B664 ; LV # Lo HANGUL SYLLABLE DDOE -B680 ; LV # Lo HANGUL SYLLABLE DDYO -B69C ; LV # Lo HANGUL SYLLABLE DDU -B6B8 ; LV # Lo HANGUL SYLLABLE DDWEO -B6D4 ; LV # Lo HANGUL SYLLABLE DDWE -B6F0 ; LV # Lo HANGUL SYLLABLE DDWI -B70C ; LV # Lo HANGUL SYLLABLE DDYU -B728 ; LV # Lo HANGUL SYLLABLE DDEU -B744 ; LV # Lo HANGUL SYLLABLE DDYI -B760 ; LV # Lo HANGUL SYLLABLE DDI -B77C ; LV # Lo HANGUL SYLLABLE RA -B798 ; LV # Lo HANGUL SYLLABLE RAE -B7B4 ; LV # Lo HANGUL SYLLABLE RYA -B7D0 ; LV # Lo HANGUL SYLLABLE RYAE -B7EC ; LV # Lo HANGUL SYLLABLE REO -B808 ; LV # Lo HANGUL SYLLABLE RE -B824 ; LV # Lo HANGUL SYLLABLE RYEO -B840 ; LV # Lo HANGUL SYLLABLE RYE -B85C ; LV # Lo HANGUL SYLLABLE RO -B878 ; LV # Lo HANGUL SYLLABLE RWA -B894 ; LV # Lo HANGUL SYLLABLE RWAE -B8B0 ; LV # Lo HANGUL SYLLABLE ROE -B8CC ; LV # Lo HANGUL SYLLABLE RYO -B8E8 ; LV # Lo HANGUL SYLLABLE RU -B904 ; LV # Lo HANGUL SYLLABLE RWEO -B920 ; LV # Lo HANGUL SYLLABLE RWE -B93C ; LV # Lo HANGUL SYLLABLE RWI -B958 ; LV # Lo HANGUL SYLLABLE RYU -B974 ; LV # Lo HANGUL SYLLABLE REU -B990 ; LV # Lo HANGUL SYLLABLE RYI -B9AC ; LV # Lo HANGUL SYLLABLE RI -B9C8 ; LV # Lo HANGUL SYLLABLE MA -B9E4 ; LV # Lo HANGUL SYLLABLE MAE -BA00 ; LV # Lo HANGUL SYLLABLE MYA -BA1C ; LV # Lo HANGUL SYLLABLE MYAE -BA38 ; LV # Lo HANGUL SYLLABLE MEO -BA54 ; LV # Lo HANGUL SYLLABLE ME -BA70 ; LV # Lo HANGUL SYLLABLE MYEO -BA8C ; LV # Lo HANGUL SYLLABLE MYE -BAA8 ; LV # Lo HANGUL SYLLABLE MO -BAC4 ; LV # Lo HANGUL SYLLABLE MWA -BAE0 ; LV # Lo HANGUL SYLLABLE MWAE -BAFC ; LV # Lo HANGUL SYLLABLE MOE -BB18 ; LV # Lo HANGUL SYLLABLE MYO -BB34 ; LV # Lo HANGUL SYLLABLE MU -BB50 ; LV # Lo HANGUL SYLLABLE MWEO -BB6C ; LV # Lo HANGUL SYLLABLE MWE -BB88 ; LV # Lo HANGUL SYLLABLE MWI -BBA4 ; LV # Lo HANGUL SYLLABLE MYU -BBC0 ; LV # Lo HANGUL SYLLABLE MEU -BBDC ; LV # Lo HANGUL SYLLABLE MYI -BBF8 ; LV # Lo HANGUL SYLLABLE MI -BC14 ; LV # Lo HANGUL SYLLABLE BA -BC30 ; LV # Lo HANGUL SYLLABLE BAE -BC4C ; LV # Lo HANGUL SYLLABLE BYA -BC68 ; LV # Lo HANGUL SYLLABLE BYAE -BC84 ; LV # Lo HANGUL SYLLABLE BEO -BCA0 ; LV # Lo HANGUL SYLLABLE BE -BCBC ; LV # Lo HANGUL SYLLABLE BYEO -BCD8 ; LV # Lo HANGUL SYLLABLE BYE -BCF4 ; LV # Lo HANGUL SYLLABLE BO -BD10 ; LV # Lo HANGUL SYLLABLE BWA -BD2C ; LV # Lo HANGUL SYLLABLE BWAE -BD48 ; LV # Lo HANGUL SYLLABLE BOE -BD64 ; LV # Lo HANGUL SYLLABLE BYO -BD80 ; LV # Lo HANGUL SYLLABLE BU -BD9C ; LV # Lo HANGUL SYLLABLE BWEO -BDB8 ; LV # Lo HANGUL SYLLABLE BWE -BDD4 ; LV # Lo HANGUL SYLLABLE BWI -BDF0 ; LV # Lo HANGUL SYLLABLE BYU -BE0C ; LV # Lo HANGUL SYLLABLE BEU -BE28 ; LV # Lo HANGUL SYLLABLE BYI -BE44 ; LV # Lo HANGUL SYLLABLE BI -BE60 ; LV # Lo HANGUL SYLLABLE BBA -BE7C ; LV # Lo HANGUL SYLLABLE BBAE -BE98 ; LV # Lo HANGUL SYLLABLE BBYA -BEB4 ; LV # Lo HANGUL SYLLABLE BBYAE -BED0 ; LV # Lo HANGUL SYLLABLE BBEO -BEEC ; LV # Lo HANGUL SYLLABLE BBE -BF08 ; LV # Lo HANGUL SYLLABLE BBYEO -BF24 ; LV # Lo HANGUL SYLLABLE BBYE -BF40 ; LV # Lo HANGUL SYLLABLE BBO -BF5C ; LV # Lo HANGUL SYLLABLE BBWA -BF78 ; LV # Lo HANGUL SYLLABLE BBWAE -BF94 ; LV # Lo HANGUL SYLLABLE BBOE -BFB0 ; LV # Lo HANGUL SYLLABLE BBYO -BFCC ; LV # Lo HANGUL SYLLABLE BBU -BFE8 ; LV # Lo HANGUL SYLLABLE BBWEO -C004 ; LV # Lo HANGUL SYLLABLE BBWE -C020 ; LV # Lo HANGUL SYLLABLE BBWI -C03C ; LV # Lo HANGUL SYLLABLE BBYU -C058 ; LV # Lo HANGUL SYLLABLE BBEU -C074 ; LV # Lo HANGUL SYLLABLE BBYI -C090 ; LV # Lo HANGUL SYLLABLE BBI -C0AC ; LV # Lo HANGUL SYLLABLE SA -C0C8 ; LV # Lo HANGUL SYLLABLE SAE -C0E4 ; LV # Lo HANGUL SYLLABLE SYA -C100 ; LV # Lo HANGUL SYLLABLE SYAE -C11C ; LV # Lo HANGUL SYLLABLE SEO -C138 ; LV # Lo HANGUL SYLLABLE SE -C154 ; LV # Lo HANGUL SYLLABLE SYEO -C170 ; LV # Lo HANGUL SYLLABLE SYE -C18C ; LV # Lo HANGUL SYLLABLE SO -C1A8 ; LV # Lo HANGUL SYLLABLE SWA -C1C4 ; LV # Lo HANGUL SYLLABLE SWAE -C1E0 ; LV # Lo HANGUL SYLLABLE SOE -C1FC ; LV # Lo HANGUL SYLLABLE SYO -C218 ; LV # Lo HANGUL SYLLABLE SU -C234 ; LV # Lo HANGUL SYLLABLE SWEO -C250 ; LV # Lo HANGUL SYLLABLE SWE -C26C ; LV # Lo HANGUL SYLLABLE SWI -C288 ; LV # Lo HANGUL SYLLABLE SYU -C2A4 ; LV # Lo HANGUL SYLLABLE SEU -C2C0 ; LV # Lo HANGUL SYLLABLE SYI -C2DC ; LV # Lo HANGUL SYLLABLE SI -C2F8 ; LV # Lo HANGUL SYLLABLE SSA -C314 ; LV # Lo HANGUL SYLLABLE SSAE -C330 ; LV # Lo HANGUL SYLLABLE SSYA -C34C ; LV # Lo HANGUL SYLLABLE SSYAE -C368 ; LV # Lo HANGUL SYLLABLE SSEO -C384 ; LV # Lo HANGUL SYLLABLE SSE -C3A0 ; LV # Lo HANGUL SYLLABLE SSYEO -C3BC ; LV # Lo HANGUL SYLLABLE SSYE -C3D8 ; LV # Lo HANGUL SYLLABLE SSO -C3F4 ; LV # Lo HANGUL SYLLABLE SSWA -C410 ; LV # Lo HANGUL SYLLABLE SSWAE -C42C ; LV # Lo HANGUL SYLLABLE SSOE -C448 ; LV # Lo HANGUL SYLLABLE SSYO -C464 ; LV # Lo HANGUL SYLLABLE SSU -C480 ; LV # Lo HANGUL SYLLABLE SSWEO -C49C ; LV # Lo HANGUL SYLLABLE SSWE -C4B8 ; LV # Lo HANGUL SYLLABLE SSWI -C4D4 ; LV # Lo HANGUL SYLLABLE SSYU -C4F0 ; LV # Lo HANGUL SYLLABLE SSEU -C50C ; LV # Lo HANGUL SYLLABLE SSYI -C528 ; LV # Lo HANGUL SYLLABLE SSI -C544 ; LV # Lo HANGUL SYLLABLE A -C560 ; LV # Lo HANGUL SYLLABLE AE -C57C ; LV # Lo HANGUL SYLLABLE YA -C598 ; LV # Lo HANGUL SYLLABLE YAE -C5B4 ; LV # Lo HANGUL SYLLABLE EO -C5D0 ; LV # Lo HANGUL SYLLABLE E -C5EC ; LV # Lo HANGUL SYLLABLE YEO -C608 ; LV # Lo HANGUL SYLLABLE YE -C624 ; LV # Lo HANGUL SYLLABLE O -C640 ; LV # Lo HANGUL SYLLABLE WA -C65C ; LV # Lo HANGUL SYLLABLE WAE -C678 ; LV # Lo HANGUL SYLLABLE OE -C694 ; LV # Lo HANGUL SYLLABLE YO -C6B0 ; LV # Lo HANGUL SYLLABLE U -C6CC ; LV # Lo HANGUL SYLLABLE WEO -C6E8 ; LV # Lo HANGUL SYLLABLE WE -C704 ; LV # Lo HANGUL SYLLABLE WI -C720 ; LV # Lo HANGUL SYLLABLE YU -C73C ; LV # Lo HANGUL SYLLABLE EU -C758 ; LV # Lo HANGUL SYLLABLE YI -C774 ; LV # Lo HANGUL SYLLABLE I -C790 ; LV # Lo HANGUL SYLLABLE JA -C7AC ; LV # Lo HANGUL SYLLABLE JAE -C7C8 ; LV # Lo HANGUL SYLLABLE JYA -C7E4 ; LV # Lo HANGUL SYLLABLE JYAE -C800 ; LV # Lo HANGUL SYLLABLE JEO -C81C ; LV # Lo HANGUL SYLLABLE JE -C838 ; LV # Lo HANGUL SYLLABLE JYEO -C854 ; LV # Lo HANGUL SYLLABLE JYE -C870 ; LV # Lo HANGUL SYLLABLE JO -C88C ; LV # Lo HANGUL SYLLABLE JWA -C8A8 ; LV # Lo HANGUL SYLLABLE JWAE -C8C4 ; LV # Lo HANGUL SYLLABLE JOE -C8E0 ; LV # Lo HANGUL SYLLABLE JYO -C8FC ; LV # Lo HANGUL SYLLABLE JU -C918 ; LV # Lo HANGUL SYLLABLE JWEO -C934 ; LV # Lo HANGUL SYLLABLE JWE -C950 ; LV # Lo HANGUL SYLLABLE JWI -C96C ; LV # Lo HANGUL SYLLABLE JYU -C988 ; LV # Lo HANGUL SYLLABLE JEU -C9A4 ; LV # Lo HANGUL SYLLABLE JYI -C9C0 ; LV # Lo HANGUL SYLLABLE JI -C9DC ; LV # Lo HANGUL SYLLABLE JJA -C9F8 ; LV # Lo HANGUL SYLLABLE JJAE -CA14 ; LV # Lo HANGUL SYLLABLE JJYA -CA30 ; LV # Lo HANGUL SYLLABLE JJYAE -CA4C ; LV # Lo HANGUL SYLLABLE JJEO -CA68 ; LV # Lo HANGUL SYLLABLE JJE -CA84 ; LV # Lo HANGUL SYLLABLE JJYEO -CAA0 ; LV # Lo HANGUL SYLLABLE JJYE -CABC ; LV # Lo HANGUL SYLLABLE JJO -CAD8 ; LV # Lo HANGUL SYLLABLE JJWA -CAF4 ; LV # Lo HANGUL SYLLABLE JJWAE -CB10 ; LV # Lo HANGUL SYLLABLE JJOE -CB2C ; LV # Lo HANGUL SYLLABLE JJYO -CB48 ; LV # Lo HANGUL SYLLABLE JJU -CB64 ; LV # Lo HANGUL SYLLABLE JJWEO -CB80 ; LV # Lo HANGUL SYLLABLE JJWE -CB9C ; LV # Lo HANGUL SYLLABLE JJWI -CBB8 ; LV # Lo HANGUL SYLLABLE JJYU -CBD4 ; LV # Lo HANGUL SYLLABLE JJEU -CBF0 ; LV # Lo HANGUL SYLLABLE JJYI -CC0C ; LV # Lo HANGUL SYLLABLE JJI -CC28 ; LV # Lo HANGUL SYLLABLE CA -CC44 ; LV # Lo HANGUL SYLLABLE CAE -CC60 ; LV # Lo HANGUL SYLLABLE CYA -CC7C ; LV # Lo HANGUL SYLLABLE CYAE -CC98 ; LV # Lo HANGUL SYLLABLE CEO -CCB4 ; LV # Lo HANGUL SYLLABLE CE -CCD0 ; LV # Lo HANGUL SYLLABLE CYEO -CCEC ; LV # Lo HANGUL SYLLABLE CYE -CD08 ; LV # Lo HANGUL SYLLABLE CO -CD24 ; LV # Lo HANGUL SYLLABLE CWA -CD40 ; LV # Lo HANGUL SYLLABLE CWAE -CD5C ; LV # Lo HANGUL SYLLABLE COE -CD78 ; LV # Lo HANGUL SYLLABLE CYO -CD94 ; LV # Lo HANGUL SYLLABLE CU -CDB0 ; LV # Lo HANGUL SYLLABLE CWEO -CDCC ; LV # Lo HANGUL SYLLABLE CWE -CDE8 ; LV # Lo HANGUL SYLLABLE CWI -CE04 ; LV # Lo HANGUL SYLLABLE CYU -CE20 ; LV # Lo HANGUL SYLLABLE CEU -CE3C ; LV # Lo HANGUL SYLLABLE CYI -CE58 ; LV # Lo HANGUL SYLLABLE CI -CE74 ; LV # Lo HANGUL SYLLABLE KA -CE90 ; LV # Lo HANGUL SYLLABLE KAE -CEAC ; LV # Lo HANGUL SYLLABLE KYA -CEC8 ; LV # Lo HANGUL SYLLABLE KYAE -CEE4 ; LV # Lo HANGUL SYLLABLE KEO -CF00 ; LV # Lo HANGUL SYLLABLE KE -CF1C ; LV # Lo HANGUL SYLLABLE KYEO -CF38 ; LV # Lo HANGUL SYLLABLE KYE -CF54 ; LV # Lo HANGUL SYLLABLE KO -CF70 ; LV # Lo HANGUL SYLLABLE KWA -CF8C ; LV # Lo HANGUL SYLLABLE KWAE -CFA8 ; LV # Lo HANGUL SYLLABLE KOE -CFC4 ; LV # Lo HANGUL SYLLABLE KYO -CFE0 ; LV # Lo HANGUL SYLLABLE KU -CFFC ; LV # Lo HANGUL SYLLABLE KWEO -D018 ; LV # Lo HANGUL SYLLABLE KWE -D034 ; LV # Lo HANGUL SYLLABLE KWI -D050 ; LV # Lo HANGUL SYLLABLE KYU -D06C ; LV # Lo HANGUL SYLLABLE KEU -D088 ; LV # Lo HANGUL SYLLABLE KYI -D0A4 ; LV # Lo HANGUL SYLLABLE KI -D0C0 ; LV # Lo HANGUL SYLLABLE TA -D0DC ; LV # Lo HANGUL SYLLABLE TAE -D0F8 ; LV # Lo HANGUL SYLLABLE TYA -D114 ; LV # Lo HANGUL SYLLABLE TYAE -D130 ; LV # Lo HANGUL SYLLABLE TEO -D14C ; LV # Lo HANGUL SYLLABLE TE -D168 ; LV # Lo HANGUL SYLLABLE TYEO -D184 ; LV # Lo HANGUL SYLLABLE TYE -D1A0 ; LV # Lo HANGUL SYLLABLE TO -D1BC ; LV # Lo HANGUL SYLLABLE TWA -D1D8 ; LV # Lo HANGUL SYLLABLE TWAE -D1F4 ; LV # Lo HANGUL SYLLABLE TOE -D210 ; LV # Lo HANGUL SYLLABLE TYO -D22C ; LV # Lo HANGUL SYLLABLE TU -D248 ; LV # Lo HANGUL SYLLABLE TWEO -D264 ; LV # Lo HANGUL SYLLABLE TWE -D280 ; LV # Lo HANGUL SYLLABLE TWI -D29C ; LV # Lo HANGUL SYLLABLE TYU -D2B8 ; LV # Lo HANGUL SYLLABLE TEU -D2D4 ; LV # Lo HANGUL SYLLABLE TYI -D2F0 ; LV # Lo HANGUL SYLLABLE TI -D30C ; LV # Lo HANGUL SYLLABLE PA -D328 ; LV # Lo HANGUL SYLLABLE PAE -D344 ; LV # Lo HANGUL SYLLABLE PYA -D360 ; LV # Lo HANGUL SYLLABLE PYAE -D37C ; LV # Lo HANGUL SYLLABLE PEO -D398 ; LV # Lo HANGUL SYLLABLE PE -D3B4 ; LV # Lo HANGUL SYLLABLE PYEO -D3D0 ; LV # Lo HANGUL SYLLABLE PYE -D3EC ; LV # Lo HANGUL SYLLABLE PO -D408 ; LV # Lo HANGUL SYLLABLE PWA -D424 ; LV # Lo HANGUL SYLLABLE PWAE -D440 ; LV # Lo HANGUL SYLLABLE POE -D45C ; LV # Lo HANGUL SYLLABLE PYO -D478 ; LV # Lo HANGUL SYLLABLE PU -D494 ; LV # Lo HANGUL SYLLABLE PWEO -D4B0 ; LV # Lo HANGUL SYLLABLE PWE -D4CC ; LV # Lo HANGUL SYLLABLE PWI -D4E8 ; LV # Lo HANGUL SYLLABLE PYU -D504 ; LV # Lo HANGUL SYLLABLE PEU -D520 ; LV # Lo HANGUL SYLLABLE PYI -D53C ; LV # Lo HANGUL SYLLABLE PI -D558 ; LV # Lo HANGUL SYLLABLE HA -D574 ; LV # Lo HANGUL SYLLABLE HAE -D590 ; LV # Lo HANGUL SYLLABLE HYA -D5AC ; LV # Lo HANGUL SYLLABLE HYAE -D5C8 ; LV # Lo HANGUL SYLLABLE HEO -D5E4 ; LV # Lo HANGUL SYLLABLE HE -D600 ; LV # Lo HANGUL SYLLABLE HYEO -D61C ; LV # Lo HANGUL SYLLABLE HYE -D638 ; LV # Lo HANGUL SYLLABLE HO -D654 ; LV # Lo HANGUL SYLLABLE HWA -D670 ; LV # Lo HANGUL SYLLABLE HWAE -D68C ; LV # Lo HANGUL SYLLABLE HOE -D6A8 ; LV # Lo HANGUL SYLLABLE HYO -D6C4 ; LV # Lo HANGUL SYLLABLE HU -D6E0 ; LV # Lo HANGUL SYLLABLE HWEO -D6FC ; LV # Lo HANGUL SYLLABLE HWE -D718 ; LV # Lo HANGUL SYLLABLE HWI -D734 ; LV # Lo HANGUL SYLLABLE HYU -D750 ; LV # Lo HANGUL SYLLABLE HEU -D76C ; LV # Lo HANGUL SYLLABLE HYI -D788 ; LV # Lo HANGUL SYLLABLE HI -AC01..AC1B ; LVT # Lo [27] HANGUL SYLLABLE GAG..HANGUL SYLLABLE GAH -AC1D..AC37 ; LVT # Lo [27] HANGUL SYLLABLE GAEG..HANGUL SYLLABLE GAEH -AC39..AC53 ; LVT # Lo [27] HANGUL SYLLABLE GYAG..HANGUL SYLLABLE GYAH -AC55..AC6F ; LVT # Lo [27] HANGUL SYLLABLE GYAEG..HANGUL SYLLABLE GYAEH -AC71..AC8B ; LVT # Lo [27] HANGUL SYLLABLE GEOG..HANGUL SYLLABLE GEOH -AC8D..ACA7 ; LVT # Lo [27] HANGUL SYLLABLE GEG..HANGUL SYLLABLE GEH -ACA9..ACC3 ; LVT # Lo [27] HANGUL SYLLABLE GYEOG..HANGUL SYLLABLE GYEOH -ACC5..ACDF ; LVT # Lo [27] HANGUL SYLLABLE GYEG..HANGUL SYLLABLE GYEH -ACE1..ACFB ; LVT # Lo [27] HANGUL SYLLABLE GOG..HANGUL SYLLABLE GOH -ACFD..AD17 ; LVT # Lo [27] HANGUL SYLLABLE GWAG..HANGUL SYLLABLE GWAH -AD19..AD33 ; LVT # Lo [27] HANGUL SYLLABLE GWAEG..HANGUL SYLLABLE GWAEH -AD35..AD4F ; LVT # Lo [27] HANGUL SYLLABLE GOEG..HANGUL SYLLABLE GOEH -AD51..AD6B ; LVT # Lo [27] HANGUL SYLLABLE GYOG..HANGUL SYLLABLE GYOH -AD6D..AD87 ; LVT # Lo [27] HANGUL SYLLABLE GUG..HANGUL SYLLABLE GUH -AD89..ADA3 ; LVT # Lo [27] HANGUL SYLLABLE GWEOG..HANGUL SYLLABLE GWEOH -ADA5..ADBF ; LVT # Lo [27] HANGUL SYLLABLE GWEG..HANGUL SYLLABLE GWEH -ADC1..ADDB ; LVT # Lo [27] HANGUL SYLLABLE GWIG..HANGUL SYLLABLE GWIH -ADDD..ADF7 ; LVT # Lo [27] HANGUL SYLLABLE GYUG..HANGUL SYLLABLE GYUH -ADF9..AE13 ; LVT # Lo [27] HANGUL SYLLABLE GEUG..HANGUL SYLLABLE GEUH -AE15..AE2F ; LVT # Lo [27] HANGUL SYLLABLE GYIG..HANGUL SYLLABLE GYIH -AE31..AE4B ; LVT # Lo [27] HANGUL SYLLABLE GIG..HANGUL SYLLABLE GIH -AE4D..AE67 ; LVT # Lo [27] HANGUL SYLLABLE GGAG..HANGUL SYLLABLE GGAH -AE69..AE83 ; LVT # Lo [27] HANGUL SYLLABLE GGAEG..HANGUL SYLLABLE GGAEH -AE85..AE9F ; LVT # Lo [27] HANGUL SYLLABLE GGYAG..HANGUL SYLLABLE GGYAH -AEA1..AEBB ; LVT # Lo [27] HANGUL SYLLABLE GGYAEG..HANGUL SYLLABLE GGYAEH -AEBD..AED7 ; LVT # Lo [27] HANGUL SYLLABLE GGEOG..HANGUL SYLLABLE GGEOH -AED9..AEF3 ; LVT # Lo [27] HANGUL SYLLABLE GGEG..HANGUL SYLLABLE GGEH -AEF5..AF0F ; LVT # Lo [27] HANGUL SYLLABLE GGYEOG..HANGUL SYLLABLE GGYEOH -AF11..AF2B ; LVT # Lo [27] HANGUL SYLLABLE GGYEG..HANGUL SYLLABLE GGYEH -AF2D..AF47 ; LVT # Lo [27] HANGUL SYLLABLE GGOG..HANGUL SYLLABLE GGOH -AF49..AF63 ; LVT # Lo [27] HANGUL SYLLABLE GGWAG..HANGUL SYLLABLE GGWAH -AF65..AF7F ; LVT # Lo [27] HANGUL SYLLABLE GGWAEG..HANGUL SYLLABLE GGWAEH -AF81..AF9B ; LVT # Lo [27] HANGUL SYLLABLE GGOEG..HANGUL SYLLABLE GGOEH -AF9D..AFB7 ; LVT # Lo [27] HANGUL SYLLABLE GGYOG..HANGUL SYLLABLE GGYOH -AFB9..AFD3 ; LVT # Lo [27] HANGUL SYLLABLE GGUG..HANGUL SYLLABLE GGUH -AFD5..AFEF ; LVT # Lo [27] HANGUL SYLLABLE GGWEOG..HANGUL SYLLABLE GGWEOH -AFF1..B00B ; LVT # Lo [27] HANGUL SYLLABLE GGWEG..HANGUL SYLLABLE GGWEH -B00D..B027 ; LVT # Lo [27] HANGUL SYLLABLE GGWIG..HANGUL SYLLABLE GGWIH -B029..B043 ; LVT # Lo [27] HANGUL SYLLABLE GGYUG..HANGUL SYLLABLE GGYUH -B045..B05F ; LVT # Lo [27] HANGUL SYLLABLE GGEUG..HANGUL SYLLABLE GGEUH -B061..B07B ; LVT # Lo [27] HANGUL SYLLABLE GGYIG..HANGUL SYLLABLE GGYIH -B07D..B097 ; LVT # Lo [27] HANGUL SYLLABLE GGIG..HANGUL SYLLABLE GGIH -B099..B0B3 ; LVT # Lo [27] HANGUL SYLLABLE NAG..HANGUL SYLLABLE NAH -B0B5..B0CF ; LVT # Lo [27] HANGUL SYLLABLE NAEG..HANGUL SYLLABLE NAEH -B0D1..B0EB ; LVT # Lo [27] HANGUL SYLLABLE NYAG..HANGUL SYLLABLE NYAH -B0ED..B107 ; LVT # Lo [27] HANGUL SYLLABLE NYAEG..HANGUL SYLLABLE NYAEH -B109..B123 ; LVT # Lo [27] HANGUL SYLLABLE NEOG..HANGUL SYLLABLE NEOH -B125..B13F ; LVT # Lo [27] HANGUL SYLLABLE NEG..HANGUL SYLLABLE NEH -B141..B15B ; LVT # Lo [27] HANGUL SYLLABLE NYEOG..HANGUL SYLLABLE NYEOH -B15D..B177 ; LVT # Lo [27] HANGUL SYLLABLE NYEG..HANGUL SYLLABLE NYEH -B179..B193 ; LVT # Lo [27] HANGUL SYLLABLE NOG..HANGUL SYLLABLE NOH -B195..B1AF ; LVT # Lo [27] HANGUL SYLLABLE NWAG..HANGUL SYLLABLE NWAH -B1B1..B1CB ; LVT # Lo [27] HANGUL SYLLABLE NWAEG..HANGUL SYLLABLE NWAEH -B1CD..B1E7 ; LVT # Lo [27] HANGUL SYLLABLE NOEG..HANGUL SYLLABLE NOEH -B1E9..B203 ; LVT # Lo [27] HANGUL SYLLABLE NYOG..HANGUL SYLLABLE NYOH -B205..B21F ; LVT # Lo [27] HANGUL SYLLABLE NUG..HANGUL SYLLABLE NUH -B221..B23B ; LVT # Lo [27] HANGUL SYLLABLE NWEOG..HANGUL SYLLABLE NWEOH -B23D..B257 ; LVT # Lo [27] HANGUL SYLLABLE NWEG..HANGUL SYLLABLE NWEH -B259..B273 ; LVT # Lo [27] HANGUL SYLLABLE NWIG..HANGUL SYLLABLE NWIH -B275..B28F ; LVT # Lo [27] HANGUL SYLLABLE NYUG..HANGUL SYLLABLE NYUH -B291..B2AB ; LVT # Lo [27] HANGUL SYLLABLE NEUG..HANGUL SYLLABLE NEUH -B2AD..B2C7 ; LVT # Lo [27] HANGUL SYLLABLE NYIG..HANGUL SYLLABLE NYIH -B2C9..B2E3 ; LVT # Lo [27] HANGUL SYLLABLE NIG..HANGUL SYLLABLE NIH -B2E5..B2FF ; LVT # Lo [27] HANGUL SYLLABLE DAG..HANGUL SYLLABLE DAH -B301..B31B ; LVT # Lo [27] HANGUL SYLLABLE DAEG..HANGUL SYLLABLE DAEH -B31D..B337 ; LVT # Lo [27] HANGUL SYLLABLE DYAG..HANGUL SYLLABLE DYAH -B339..B353 ; LVT # Lo [27] HANGUL SYLLABLE DYAEG..HANGUL SYLLABLE DYAEH -B355..B36F ; LVT # Lo [27] HANGUL SYLLABLE DEOG..HANGUL SYLLABLE DEOH -B371..B38B ; LVT # Lo [27] HANGUL SYLLABLE DEG..HANGUL SYLLABLE DEH -B38D..B3A7 ; LVT # Lo [27] HANGUL SYLLABLE DYEOG..HANGUL SYLLABLE DYEOH -B3A9..B3C3 ; LVT # Lo [27] HANGUL SYLLABLE DYEG..HANGUL SYLLABLE DYEH -B3C5..B3DF ; LVT # Lo [27] HANGUL SYLLABLE DOG..HANGUL SYLLABLE DOH -B3E1..B3FB ; LVT # Lo [27] HANGUL SYLLABLE DWAG..HANGUL SYLLABLE DWAH -B3FD..B417 ; LVT # Lo [27] HANGUL SYLLABLE DWAEG..HANGUL SYLLABLE DWAEH -B419..B433 ; LVT # Lo [27] HANGUL SYLLABLE DOEG..HANGUL SYLLABLE DOEH -B435..B44F ; LVT # Lo [27] HANGUL SYLLABLE DYOG..HANGUL SYLLABLE DYOH -B451..B46B ; LVT # Lo [27] HANGUL SYLLABLE DUG..HANGUL SYLLABLE DUH -B46D..B487 ; LVT # Lo [27] HANGUL SYLLABLE DWEOG..HANGUL SYLLABLE DWEOH -B489..B4A3 ; LVT # Lo [27] HANGUL SYLLABLE DWEG..HANGUL SYLLABLE DWEH -B4A5..B4BF ; LVT # Lo [27] HANGUL SYLLABLE DWIG..HANGUL SYLLABLE DWIH -B4C1..B4DB ; LVT # Lo [27] HANGUL SYLLABLE DYUG..HANGUL SYLLABLE DYUH -B4DD..B4F7 ; LVT # Lo [27] HANGUL SYLLABLE DEUG..HANGUL SYLLABLE DEUH -B4F9..B513 ; LVT # Lo [27] HANGUL SYLLABLE DYIG..HANGUL SYLLABLE DYIH -B515..B52F ; LVT # Lo [27] HANGUL SYLLABLE DIG..HANGUL SYLLABLE DIH -B531..B54B ; LVT # Lo [27] HANGUL SYLLABLE DDAG..HANGUL SYLLABLE DDAH -B54D..B567 ; LVT # Lo [27] HANGUL SYLLABLE DDAEG..HANGUL SYLLABLE DDAEH -B569..B583 ; LVT # Lo [27] HANGUL SYLLABLE DDYAG..HANGUL SYLLABLE DDYAH -B585..B59F ; LVT # Lo [27] HANGUL SYLLABLE DDYAEG..HANGUL SYLLABLE DDYAEH -B5A1..B5BB ; LVT # Lo [27] HANGUL SYLLABLE DDEOG..HANGUL SYLLABLE DDEOH -B5BD..B5D7 ; LVT # Lo [27] HANGUL SYLLABLE DDEG..HANGUL SYLLABLE DDEH -B5D9..B5F3 ; LVT # Lo [27] HANGUL SYLLABLE DDYEOG..HANGUL SYLLABLE DDYEOH -B5F5..B60F ; LVT # Lo [27] HANGUL SYLLABLE DDYEG..HANGUL SYLLABLE DDYEH -B611..B62B ; LVT # Lo [27] HANGUL SYLLABLE DDOG..HANGUL SYLLABLE DDOH -B62D..B647 ; LVT # Lo [27] HANGUL SYLLABLE DDWAG..HANGUL SYLLABLE DDWAH -B649..B663 ; LVT # Lo [27] HANGUL SYLLABLE DDWAEG..HANGUL SYLLABLE DDWAEH -B665..B67F ; LVT # Lo [27] HANGUL SYLLABLE DDOEG..HANGUL SYLLABLE DDOEH -B681..B69B ; LVT # Lo [27] HANGUL SYLLABLE DDYOG..HANGUL SYLLABLE DDYOH -B69D..B6B7 ; LVT # Lo [27] HANGUL SYLLABLE DDUG..HANGUL SYLLABLE DDUH -B6B9..B6D3 ; LVT # Lo [27] HANGUL SYLLABLE DDWEOG..HANGUL SYLLABLE DDWEOH -B6D5..B6EF ; LVT # Lo [27] HANGUL SYLLABLE DDWEG..HANGUL SYLLABLE DDWEH -B6F1..B70B ; LVT # Lo [27] HANGUL SYLLABLE DDWIG..HANGUL SYLLABLE DDWIH -B70D..B727 ; LVT # Lo [27] HANGUL SYLLABLE DDYUG..HANGUL SYLLABLE DDYUH -B729..B743 ; LVT # Lo [27] HANGUL SYLLABLE DDEUG..HANGUL SYLLABLE DDEUH -B745..B75F ; LVT # Lo [27] HANGUL SYLLABLE DDYIG..HANGUL SYLLABLE DDYIH -B761..B77B ; LVT # Lo [27] HANGUL SYLLABLE DDIG..HANGUL SYLLABLE DDIH -B77D..B797 ; LVT # Lo [27] HANGUL SYLLABLE RAG..HANGUL SYLLABLE RAH -B799..B7B3 ; LVT # Lo [27] HANGUL SYLLABLE RAEG..HANGUL SYLLABLE RAEH -B7B5..B7CF ; LVT # Lo [27] HANGUL SYLLABLE RYAG..HANGUL SYLLABLE RYAH -B7D1..B7EB ; LVT # Lo [27] HANGUL SYLLABLE RYAEG..HANGUL SYLLABLE RYAEH -B7ED..B807 ; LVT # Lo [27] HANGUL SYLLABLE REOG..HANGUL SYLLABLE REOH -B809..B823 ; LVT # Lo [27] HANGUL SYLLABLE REG..HANGUL SYLLABLE REH -B825..B83F ; LVT # Lo [27] HANGUL SYLLABLE RYEOG..HANGUL SYLLABLE RYEOH -B841..B85B ; LVT # Lo [27] HANGUL SYLLABLE RYEG..HANGUL SYLLABLE RYEH -B85D..B877 ; LVT # Lo [27] HANGUL SYLLABLE ROG..HANGUL SYLLABLE ROH -B879..B893 ; LVT # Lo [27] HANGUL SYLLABLE RWAG..HANGUL SYLLABLE RWAH -B895..B8AF ; LVT # Lo [27] HANGUL SYLLABLE RWAEG..HANGUL SYLLABLE RWAEH -B8B1..B8CB ; LVT # Lo [27] HANGUL SYLLABLE ROEG..HANGUL SYLLABLE ROEH -B8CD..B8E7 ; LVT # Lo [27] HANGUL SYLLABLE RYOG..HANGUL SYLLABLE RYOH -B8E9..B903 ; LVT # Lo [27] HANGUL SYLLABLE RUG..HANGUL SYLLABLE RUH -B905..B91F ; LVT # Lo [27] HANGUL SYLLABLE RWEOG..HANGUL SYLLABLE RWEOH -B921..B93B ; LVT # Lo [27] HANGUL SYLLABLE RWEG..HANGUL SYLLABLE RWEH -B93D..B957 ; LVT # Lo [27] HANGUL SYLLABLE RWIG..HANGUL SYLLABLE RWIH -B959..B973 ; LVT # Lo [27] HANGUL SYLLABLE RYUG..HANGUL SYLLABLE RYUH -B975..B98F ; LVT # Lo [27] HANGUL SYLLABLE REUG..HANGUL SYLLABLE REUH -B991..B9AB ; LVT # Lo [27] HANGUL SYLLABLE RYIG..HANGUL SYLLABLE RYIH -B9AD..B9C7 ; LVT # Lo [27] HANGUL SYLLABLE RIG..HANGUL SYLLABLE RIH -B9C9..B9E3 ; LVT # Lo [27] HANGUL SYLLABLE MAG..HANGUL SYLLABLE MAH -B9E5..B9FF ; LVT # Lo [27] HANGUL SYLLABLE MAEG..HANGUL SYLLABLE MAEH -BA01..BA1B ; LVT # Lo [27] HANGUL SYLLABLE MYAG..HANGUL SYLLABLE MYAH -BA1D..BA37 ; LVT # Lo [27] HANGUL SYLLABLE MYAEG..HANGUL SYLLABLE MYAEH -BA39..BA53 ; LVT # Lo [27] HANGUL SYLLABLE MEOG..HANGUL SYLLABLE MEOH -BA55..BA6F ; LVT # Lo [27] HANGUL SYLLABLE MEG..HANGUL SYLLABLE MEH -BA71..BA8B ; LVT # Lo [27] HANGUL SYLLABLE MYEOG..HANGUL SYLLABLE MYEOH -BA8D..BAA7 ; LVT # Lo [27] HANGUL SYLLABLE MYEG..HANGUL SYLLABLE MYEH -BAA9..BAC3 ; LVT # Lo [27] HANGUL SYLLABLE MOG..HANGUL SYLLABLE MOH -BAC5..BADF ; LVT # Lo [27] HANGUL SYLLABLE MWAG..HANGUL SYLLABLE MWAH -BAE1..BAFB ; LVT # Lo [27] HANGUL SYLLABLE MWAEG..HANGUL SYLLABLE MWAEH -BAFD..BB17 ; LVT # Lo [27] HANGUL SYLLABLE MOEG..HANGUL SYLLABLE MOEH -BB19..BB33 ; LVT # Lo [27] HANGUL SYLLABLE MYOG..HANGUL SYLLABLE MYOH -BB35..BB4F ; LVT # Lo [27] HANGUL SYLLABLE MUG..HANGUL SYLLABLE MUH -BB51..BB6B ; LVT # Lo [27] HANGUL SYLLABLE MWEOG..HANGUL SYLLABLE MWEOH -BB6D..BB87 ; LVT # Lo [27] HANGUL SYLLABLE MWEG..HANGUL SYLLABLE MWEH -BB89..BBA3 ; LVT # Lo [27] HANGUL SYLLABLE MWIG..HANGUL SYLLABLE MWIH -BBA5..BBBF ; LVT # Lo [27] HANGUL SYLLABLE MYUG..HANGUL SYLLABLE MYUH -BBC1..BBDB ; LVT # Lo [27] HANGUL SYLLABLE MEUG..HANGUL SYLLABLE MEUH -BBDD..BBF7 ; LVT # Lo [27] HANGUL SYLLABLE MYIG..HANGUL SYLLABLE MYIH -BBF9..BC13 ; LVT # Lo [27] HANGUL SYLLABLE MIG..HANGUL SYLLABLE MIH -BC15..BC2F ; LVT # Lo [27] HANGUL SYLLABLE BAG..HANGUL SYLLABLE BAH -BC31..BC4B ; LVT # Lo [27] HANGUL SYLLABLE BAEG..HANGUL SYLLABLE BAEH -BC4D..BC67 ; LVT # Lo [27] HANGUL SYLLABLE BYAG..HANGUL SYLLABLE BYAH -BC69..BC83 ; LVT # Lo [27] HANGUL SYLLABLE BYAEG..HANGUL SYLLABLE BYAEH -BC85..BC9F ; LVT # Lo [27] HANGUL SYLLABLE BEOG..HANGUL SYLLABLE BEOH -BCA1..BCBB ; LVT # Lo [27] HANGUL SYLLABLE BEG..HANGUL SYLLABLE BEH -BCBD..BCD7 ; LVT # Lo [27] HANGUL SYLLABLE BYEOG..HANGUL SYLLABLE BYEOH -BCD9..BCF3 ; LVT # Lo [27] HANGUL SYLLABLE BYEG..HANGUL SYLLABLE BYEH -BCF5..BD0F ; LVT # Lo [27] HANGUL SYLLABLE BOG..HANGUL SYLLABLE BOH -BD11..BD2B ; LVT # Lo [27] HANGUL SYLLABLE BWAG..HANGUL SYLLABLE BWAH -BD2D..BD47 ; LVT # Lo [27] HANGUL SYLLABLE BWAEG..HANGUL SYLLABLE BWAEH -BD49..BD63 ; LVT # Lo [27] HANGUL SYLLABLE BOEG..HANGUL SYLLABLE BOEH -BD65..BD7F ; LVT # Lo [27] HANGUL SYLLABLE BYOG..HANGUL SYLLABLE BYOH -BD81..BD9B ; LVT # Lo [27] HANGUL SYLLABLE BUG..HANGUL SYLLABLE BUH -BD9D..BDB7 ; LVT # Lo [27] HANGUL SYLLABLE BWEOG..HANGUL SYLLABLE BWEOH -BDB9..BDD3 ; LVT # Lo [27] HANGUL SYLLABLE BWEG..HANGUL SYLLABLE BWEH -BDD5..BDEF ; LVT # Lo [27] HANGUL SYLLABLE BWIG..HANGUL SYLLABLE BWIH -BDF1..BE0B ; LVT # Lo [27] HANGUL SYLLABLE BYUG..HANGUL SYLLABLE BYUH -BE0D..BE27 ; LVT # Lo [27] HANGUL SYLLABLE BEUG..HANGUL SYLLABLE BEUH -BE29..BE43 ; LVT # Lo [27] HANGUL SYLLABLE BYIG..HANGUL SYLLABLE BYIH -BE45..BE5F ; LVT # Lo [27] HANGUL SYLLABLE BIG..HANGUL SYLLABLE BIH -BE61..BE7B ; LVT # Lo [27] HANGUL SYLLABLE BBAG..HANGUL SYLLABLE BBAH -BE7D..BE97 ; LVT # Lo [27] HANGUL SYLLABLE BBAEG..HANGUL SYLLABLE BBAEH -BE99..BEB3 ; LVT # Lo [27] HANGUL SYLLABLE BBYAG..HANGUL SYLLABLE BBYAH -BEB5..BECF ; LVT # Lo [27] HANGUL SYLLABLE BBYAEG..HANGUL SYLLABLE BBYAEH -BED1..BEEB ; LVT # Lo [27] HANGUL SYLLABLE BBEOG..HANGUL SYLLABLE BBEOH -BEED..BF07 ; LVT # Lo [27] HANGUL SYLLABLE BBEG..HANGUL SYLLABLE BBEH -BF09..BF23 ; LVT # Lo [27] HANGUL SYLLABLE BBYEOG..HANGUL SYLLABLE BBYEOH -BF25..BF3F ; LVT # Lo [27] HANGUL SYLLABLE BBYEG..HANGUL SYLLABLE BBYEH -BF41..BF5B ; LVT # Lo [27] HANGUL SYLLABLE BBOG..HANGUL SYLLABLE BBOH -BF5D..BF77 ; LVT # Lo [27] HANGUL SYLLABLE BBWAG..HANGUL SYLLABLE BBWAH -BF79..BF93 ; LVT # Lo [27] HANGUL SYLLABLE BBWAEG..HANGUL SYLLABLE BBWAEH -BF95..BFAF ; LVT # Lo [27] HANGUL SYLLABLE BBOEG..HANGUL SYLLABLE BBOEH -BFB1..BFCB ; LVT # Lo [27] HANGUL SYLLABLE BBYOG..HANGUL SYLLABLE BBYOH -BFCD..BFE7 ; LVT # Lo [27] HANGUL SYLLABLE BBUG..HANGUL SYLLABLE BBUH -BFE9..C003 ; LVT # Lo [27] HANGUL SYLLABLE BBWEOG..HANGUL SYLLABLE BBWEOH -C005..C01F ; LVT # Lo [27] HANGUL SYLLABLE BBWEG..HANGUL SYLLABLE BBWEH -C021..C03B ; LVT # Lo [27] HANGUL SYLLABLE BBWIG..HANGUL SYLLABLE BBWIH -C03D..C057 ; LVT # Lo [27] HANGUL SYLLABLE BBYUG..HANGUL SYLLABLE BBYUH -C059..C073 ; LVT # Lo [27] HANGUL SYLLABLE BBEUG..HANGUL SYLLABLE BBEUH -C075..C08F ; LVT # Lo [27] HANGUL SYLLABLE BBYIG..HANGUL SYLLABLE BBYIH -C091..C0AB ; LVT # Lo [27] HANGUL SYLLABLE BBIG..HANGUL SYLLABLE BBIH -C0AD..C0C7 ; LVT # Lo [27] HANGUL SYLLABLE SAG..HANGUL SYLLABLE SAH -C0C9..C0E3 ; LVT # Lo [27] HANGUL SYLLABLE SAEG..HANGUL SYLLABLE SAEH -C0E5..C0FF ; LVT # Lo [27] HANGUL SYLLABLE SYAG..HANGUL SYLLABLE SYAH -C101..C11B ; LVT # Lo [27] HANGUL SYLLABLE SYAEG..HANGUL SYLLABLE SYAEH -C11D..C137 ; LVT # Lo [27] HANGUL SYLLABLE SEOG..HANGUL SYLLABLE SEOH -C139..C153 ; LVT # Lo [27] HANGUL SYLLABLE SEG..HANGUL SYLLABLE SEH -C155..C16F ; LVT # Lo [27] HANGUL SYLLABLE SYEOG..HANGUL SYLLABLE SYEOH -C171..C18B ; LVT # Lo [27] HANGUL SYLLABLE SYEG..HANGUL SYLLABLE SYEH -C18D..C1A7 ; LVT # Lo [27] HANGUL SYLLABLE SOG..HANGUL SYLLABLE SOH -C1A9..C1C3 ; LVT # Lo [27] HANGUL SYLLABLE SWAG..HANGUL SYLLABLE SWAH -C1C5..C1DF ; LVT # Lo [27] HANGUL SYLLABLE SWAEG..HANGUL SYLLABLE SWAEH -C1E1..C1FB ; LVT # Lo [27] HANGUL SYLLABLE SOEG..HANGUL SYLLABLE SOEH -C1FD..C217 ; LVT # Lo [27] HANGUL SYLLABLE SYOG..HANGUL SYLLABLE SYOH -C219..C233 ; LVT # Lo [27] HANGUL SYLLABLE SUG..HANGUL SYLLABLE SUH -C235..C24F ; LVT # Lo [27] HANGUL SYLLABLE SWEOG..HANGUL SYLLABLE SWEOH -C251..C26B ; LVT # Lo [27] HANGUL SYLLABLE SWEG..HANGUL SYLLABLE SWEH -C26D..C287 ; LVT # Lo [27] HANGUL SYLLABLE SWIG..HANGUL SYLLABLE SWIH -C289..C2A3 ; LVT # Lo [27] HANGUL SYLLABLE SYUG..HANGUL SYLLABLE SYUH -C2A5..C2BF ; LVT # Lo [27] HANGUL SYLLABLE SEUG..HANGUL SYLLABLE SEUH -C2C1..C2DB ; LVT # Lo [27] HANGUL SYLLABLE SYIG..HANGUL SYLLABLE SYIH -C2DD..C2F7 ; LVT # Lo [27] HANGUL SYLLABLE SIG..HANGUL SYLLABLE SIH -C2F9..C313 ; LVT # Lo [27] HANGUL SYLLABLE SSAG..HANGUL SYLLABLE SSAH -C315..C32F ; LVT # Lo [27] HANGUL SYLLABLE SSAEG..HANGUL SYLLABLE SSAEH -C331..C34B ; LVT # Lo [27] HANGUL SYLLABLE SSYAG..HANGUL SYLLABLE SSYAH -C34D..C367 ; LVT # Lo [27] HANGUL SYLLABLE SSYAEG..HANGUL SYLLABLE SSYAEH -C369..C383 ; LVT # Lo [27] HANGUL SYLLABLE SSEOG..HANGUL SYLLABLE SSEOH -C385..C39F ; LVT # Lo [27] HANGUL SYLLABLE SSEG..HANGUL SYLLABLE SSEH -C3A1..C3BB ; LVT # Lo [27] HANGUL SYLLABLE SSYEOG..HANGUL SYLLABLE SSYEOH -C3BD..C3D7 ; LVT # Lo [27] HANGUL SYLLABLE SSYEG..HANGUL SYLLABLE SSYEH -C3D9..C3F3 ; LVT # Lo [27] HANGUL SYLLABLE SSOG..HANGUL SYLLABLE SSOH -C3F5..C40F ; LVT # Lo [27] HANGUL SYLLABLE SSWAG..HANGUL SYLLABLE SSWAH -C411..C42B ; LVT # Lo [27] HANGUL SYLLABLE SSWAEG..HANGUL SYLLABLE SSWAEH -C42D..C447 ; LVT # Lo [27] HANGUL SYLLABLE SSOEG..HANGUL SYLLABLE SSOEH -C449..C463 ; LVT # Lo [27] HANGUL SYLLABLE SSYOG..HANGUL SYLLABLE SSYOH -C465..C47F ; LVT # Lo [27] HANGUL SYLLABLE SSUG..HANGUL SYLLABLE SSUH -C481..C49B ; LVT # Lo [27] HANGUL SYLLABLE SSWEOG..HANGUL SYLLABLE SSWEOH -C49D..C4B7 ; LVT # Lo [27] HANGUL SYLLABLE SSWEG..HANGUL SYLLABLE SSWEH -C4B9..C4D3 ; LVT # Lo [27] HANGUL SYLLABLE SSWIG..HANGUL SYLLABLE SSWIH -C4D5..C4EF ; LVT # Lo [27] HANGUL SYLLABLE SSYUG..HANGUL SYLLABLE SSYUH -C4F1..C50B ; LVT # Lo [27] HANGUL SYLLABLE SSEUG..HANGUL SYLLABLE SSEUH -C50D..C527 ; LVT # Lo [27] HANGUL SYLLABLE SSYIG..HANGUL SYLLABLE SSYIH -C529..C543 ; LVT # Lo [27] HANGUL SYLLABLE SSIG..HANGUL SYLLABLE SSIH -C545..C55F ; LVT # Lo [27] HANGUL SYLLABLE AG..HANGUL SYLLABLE AH -C561..C57B ; LVT # Lo [27] HANGUL SYLLABLE AEG..HANGUL SYLLABLE AEH -C57D..C597 ; LVT # Lo [27] HANGUL SYLLABLE YAG..HANGUL SYLLABLE YAH -C599..C5B3 ; LVT # Lo [27] HANGUL SYLLABLE YAEG..HANGUL SYLLABLE YAEH -C5B5..C5CF ; LVT # Lo [27] HANGUL SYLLABLE EOG..HANGUL SYLLABLE EOH -C5D1..C5EB ; LVT # Lo [27] HANGUL SYLLABLE EG..HANGUL SYLLABLE EH -C5ED..C607 ; LVT # Lo [27] HANGUL SYLLABLE YEOG..HANGUL SYLLABLE YEOH -C609..C623 ; LVT # Lo [27] HANGUL SYLLABLE YEG..HANGUL SYLLABLE YEH -C625..C63F ; LVT # Lo [27] HANGUL SYLLABLE OG..HANGUL SYLLABLE OH -C641..C65B ; LVT # Lo [27] HANGUL SYLLABLE WAG..HANGUL SYLLABLE WAH -C65D..C677 ; LVT # Lo [27] HANGUL SYLLABLE WAEG..HANGUL SYLLABLE WAEH -C679..C693 ; LVT # Lo [27] HANGUL SYLLABLE OEG..HANGUL SYLLABLE OEH -C695..C6AF ; LVT # Lo [27] HANGUL SYLLABLE YOG..HANGUL SYLLABLE YOH -C6B1..C6CB ; LVT # Lo [27] HANGUL SYLLABLE UG..HANGUL SYLLABLE UH -C6CD..C6E7 ; LVT # Lo [27] HANGUL SYLLABLE WEOG..HANGUL SYLLABLE WEOH -C6E9..C703 ; LVT # Lo [27] HANGUL SYLLABLE WEG..HANGUL SYLLABLE WEH -C705..C71F ; LVT # Lo [27] HANGUL SYLLABLE WIG..HANGUL SYLLABLE WIH -C721..C73B ; LVT # Lo [27] HANGUL SYLLABLE YUG..HANGUL SYLLABLE YUH -C73D..C757 ; LVT # Lo [27] HANGUL SYLLABLE EUG..HANGUL SYLLABLE EUH -C759..C773 ; LVT # Lo [27] HANGUL SYLLABLE YIG..HANGUL SYLLABLE YIH -C775..C78F ; LVT # Lo [27] HANGUL SYLLABLE IG..HANGUL SYLLABLE IH -C791..C7AB ; LVT # Lo [27] HANGUL SYLLABLE JAG..HANGUL SYLLABLE JAH -C7AD..C7C7 ; LVT # Lo [27] HANGUL SYLLABLE JAEG..HANGUL SYLLABLE JAEH -C7C9..C7E3 ; LVT # Lo [27] HANGUL SYLLABLE JYAG..HANGUL SYLLABLE JYAH -C7E5..C7FF ; LVT # Lo [27] HANGUL SYLLABLE JYAEG..HANGUL SYLLABLE JYAEH -C801..C81B ; LVT # Lo [27] HANGUL SYLLABLE JEOG..HANGUL SYLLABLE JEOH -C81D..C837 ; LVT # Lo [27] HANGUL SYLLABLE JEG..HANGUL SYLLABLE JEH -C839..C853 ; LVT # Lo [27] HANGUL SYLLABLE JYEOG..HANGUL SYLLABLE JYEOH -C855..C86F ; LVT # Lo [27] HANGUL SYLLABLE JYEG..HANGUL SYLLABLE JYEH -C871..C88B ; LVT # Lo [27] HANGUL SYLLABLE JOG..HANGUL SYLLABLE JOH -C88D..C8A7 ; LVT # Lo [27] HANGUL SYLLABLE JWAG..HANGUL SYLLABLE JWAH -C8A9..C8C3 ; LVT # Lo [27] HANGUL SYLLABLE JWAEG..HANGUL SYLLABLE JWAEH -C8C5..C8DF ; LVT # Lo [27] HANGUL SYLLABLE JOEG..HANGUL SYLLABLE JOEH -C8E1..C8FB ; LVT # Lo [27] HANGUL SYLLABLE JYOG..HANGUL SYLLABLE JYOH -C8FD..C917 ; LVT # Lo [27] HANGUL SYLLABLE JUG..HANGUL SYLLABLE JUH -C919..C933 ; LVT # Lo [27] HANGUL SYLLABLE JWEOG..HANGUL SYLLABLE JWEOH -C935..C94F ; LVT # Lo [27] HANGUL SYLLABLE JWEG..HANGUL SYLLABLE JWEH -C951..C96B ; LVT # Lo [27] HANGUL SYLLABLE JWIG..HANGUL SYLLABLE JWIH -C96D..C987 ; LVT # Lo [27] HANGUL SYLLABLE JYUG..HANGUL SYLLABLE JYUH -C989..C9A3 ; LVT # Lo [27] HANGUL SYLLABLE JEUG..HANGUL SYLLABLE JEUH -C9A5..C9BF ; LVT # Lo [27] HANGUL SYLLABLE JYIG..HANGUL SYLLABLE JYIH -C9C1..C9DB ; LVT # Lo [27] HANGUL SYLLABLE JIG..HANGUL SYLLABLE JIH -C9DD..C9F7 ; LVT # Lo [27] HANGUL SYLLABLE JJAG..HANGUL SYLLABLE JJAH -C9F9..CA13 ; LVT # Lo [27] HANGUL SYLLABLE JJAEG..HANGUL SYLLABLE JJAEH -CA15..CA2F ; LVT # Lo [27] HANGUL SYLLABLE JJYAG..HANGUL SYLLABLE JJYAH -CA31..CA4B ; LVT # Lo [27] HANGUL SYLLABLE JJYAEG..HANGUL SYLLABLE JJYAEH -CA4D..CA67 ; LVT # Lo [27] HANGUL SYLLABLE JJEOG..HANGUL SYLLABLE JJEOH -CA69..CA83 ; LVT # Lo [27] HANGUL SYLLABLE JJEG..HANGUL SYLLABLE JJEH -CA85..CA9F ; LVT # Lo [27] HANGUL SYLLABLE JJYEOG..HANGUL SYLLABLE JJYEOH -CAA1..CABB ; LVT # Lo [27] HANGUL SYLLABLE JJYEG..HANGUL SYLLABLE JJYEH -CABD..CAD7 ; LVT # Lo [27] HANGUL SYLLABLE JJOG..HANGUL SYLLABLE JJOH -CAD9..CAF3 ; LVT # Lo [27] HANGUL SYLLABLE JJWAG..HANGUL SYLLABLE JJWAH -CAF5..CB0F ; LVT # Lo [27] HANGUL SYLLABLE JJWAEG..HANGUL SYLLABLE JJWAEH -CB11..CB2B ; LVT # Lo [27] HANGUL SYLLABLE JJOEG..HANGUL SYLLABLE JJOEH -CB2D..CB47 ; LVT # Lo [27] HANGUL SYLLABLE JJYOG..HANGUL SYLLABLE JJYOH -CB49..CB63 ; LVT # Lo [27] HANGUL SYLLABLE JJUG..HANGUL SYLLABLE JJUH -CB65..CB7F ; LVT # Lo [27] HANGUL SYLLABLE JJWEOG..HANGUL SYLLABLE JJWEOH -CB81..CB9B ; LVT # Lo [27] HANGUL SYLLABLE JJWEG..HANGUL SYLLABLE JJWEH -CB9D..CBB7 ; LVT # Lo [27] HANGUL SYLLABLE JJWIG..HANGUL SYLLABLE JJWIH -CBB9..CBD3 ; LVT # Lo [27] HANGUL SYLLABLE JJYUG..HANGUL SYLLABLE JJYUH -CBD5..CBEF ; LVT # Lo [27] HANGUL SYLLABLE JJEUG..HANGUL SYLLABLE JJEUH -CBF1..CC0B ; LVT # Lo [27] HANGUL SYLLABLE JJYIG..HANGUL SYLLABLE JJYIH -CC0D..CC27 ; LVT # Lo [27] HANGUL SYLLABLE JJIG..HANGUL SYLLABLE JJIH -CC29..CC43 ; LVT # Lo [27] HANGUL SYLLABLE CAG..HANGUL SYLLABLE CAH -CC45..CC5F ; LVT # Lo [27] HANGUL SYLLABLE CAEG..HANGUL SYLLABLE CAEH -CC61..CC7B ; LVT # Lo [27] HANGUL SYLLABLE CYAG..HANGUL SYLLABLE CYAH -CC7D..CC97 ; LVT # Lo [27] HANGUL SYLLABLE CYAEG..HANGUL SYLLABLE CYAEH -CC99..CCB3 ; LVT # Lo [27] HANGUL SYLLABLE CEOG..HANGUL SYLLABLE CEOH -CCB5..CCCF ; LVT # Lo [27] HANGUL SYLLABLE CEG..HANGUL SYLLABLE CEH -CCD1..CCEB ; LVT # Lo [27] HANGUL SYLLABLE CYEOG..HANGUL SYLLABLE CYEOH -CCED..CD07 ; LVT # Lo [27] HANGUL SYLLABLE CYEG..HANGUL SYLLABLE CYEH -CD09..CD23 ; LVT # Lo [27] HANGUL SYLLABLE COG..HANGUL SYLLABLE COH -CD25..CD3F ; LVT # Lo [27] HANGUL SYLLABLE CWAG..HANGUL SYLLABLE CWAH -CD41..CD5B ; LVT # Lo [27] HANGUL SYLLABLE CWAEG..HANGUL SYLLABLE CWAEH -CD5D..CD77 ; LVT # Lo [27] HANGUL SYLLABLE COEG..HANGUL SYLLABLE COEH -CD79..CD93 ; LVT # Lo [27] HANGUL SYLLABLE CYOG..HANGUL SYLLABLE CYOH -CD95..CDAF ; LVT # Lo [27] HANGUL SYLLABLE CUG..HANGUL SYLLABLE CUH -CDB1..CDCB ; LVT # Lo [27] HANGUL SYLLABLE CWEOG..HANGUL SYLLABLE CWEOH -CDCD..CDE7 ; LVT # Lo [27] HANGUL SYLLABLE CWEG..HANGUL SYLLABLE CWEH -CDE9..CE03 ; LVT # Lo [27] HANGUL SYLLABLE CWIG..HANGUL SYLLABLE CWIH -CE05..CE1F ; LVT # Lo [27] HANGUL SYLLABLE CYUG..HANGUL SYLLABLE CYUH -CE21..CE3B ; LVT # Lo [27] HANGUL SYLLABLE CEUG..HANGUL SYLLABLE CEUH -CE3D..CE57 ; LVT # Lo [27] HANGUL SYLLABLE CYIG..HANGUL SYLLABLE CYIH -CE59..CE73 ; LVT # Lo [27] HANGUL SYLLABLE CIG..HANGUL SYLLABLE CIH -CE75..CE8F ; LVT # Lo [27] HANGUL SYLLABLE KAG..HANGUL SYLLABLE KAH -CE91..CEAB ; LVT # Lo [27] HANGUL SYLLABLE KAEG..HANGUL SYLLABLE KAEH -CEAD..CEC7 ; LVT # Lo [27] HANGUL SYLLABLE KYAG..HANGUL SYLLABLE KYAH -CEC9..CEE3 ; LVT # Lo [27] HANGUL SYLLABLE KYAEG..HANGUL SYLLABLE KYAEH -CEE5..CEFF ; LVT # Lo [27] HANGUL SYLLABLE KEOG..HANGUL SYLLABLE KEOH -CF01..CF1B ; LVT # Lo [27] HANGUL SYLLABLE KEG..HANGUL SYLLABLE KEH -CF1D..CF37 ; LVT # Lo [27] HANGUL SYLLABLE KYEOG..HANGUL SYLLABLE KYEOH -CF39..CF53 ; LVT # Lo [27] HANGUL SYLLABLE KYEG..HANGUL SYLLABLE KYEH -CF55..CF6F ; LVT # Lo [27] HANGUL SYLLABLE KOG..HANGUL SYLLABLE KOH -CF71..CF8B ; LVT # Lo [27] HANGUL SYLLABLE KWAG..HANGUL SYLLABLE KWAH -CF8D..CFA7 ; LVT # Lo [27] HANGUL SYLLABLE KWAEG..HANGUL SYLLABLE KWAEH -CFA9..CFC3 ; LVT # Lo [27] HANGUL SYLLABLE KOEG..HANGUL SYLLABLE KOEH -CFC5..CFDF ; LVT # Lo [27] HANGUL SYLLABLE KYOG..HANGUL SYLLABLE KYOH -CFE1..CFFB ; LVT # Lo [27] HANGUL SYLLABLE KUG..HANGUL SYLLABLE KUH -CFFD..D017 ; LVT # Lo [27] HANGUL SYLLABLE KWEOG..HANGUL SYLLABLE KWEOH -D019..D033 ; LVT # Lo [27] HANGUL SYLLABLE KWEG..HANGUL SYLLABLE KWEH -D035..D04F ; LVT # Lo [27] HANGUL SYLLABLE KWIG..HANGUL SYLLABLE KWIH -D051..D06B ; LVT # Lo [27] HANGUL SYLLABLE KYUG..HANGUL SYLLABLE KYUH -D06D..D087 ; LVT # Lo [27] HANGUL SYLLABLE KEUG..HANGUL SYLLABLE KEUH -D089..D0A3 ; LVT # Lo [27] HANGUL SYLLABLE KYIG..HANGUL SYLLABLE KYIH -D0A5..D0BF ; LVT # Lo [27] HANGUL SYLLABLE KIG..HANGUL SYLLABLE KIH -D0C1..D0DB ; LVT # Lo [27] HANGUL SYLLABLE TAG..HANGUL SYLLABLE TAH -D0DD..D0F7 ; LVT # Lo [27] HANGUL SYLLABLE TAEG..HANGUL SYLLABLE TAEH -D0F9..D113 ; LVT # Lo [27] HANGUL SYLLABLE TYAG..HANGUL SYLLABLE TYAH -D115..D12F ; LVT # Lo [27] HANGUL SYLLABLE TYAEG..HANGUL SYLLABLE TYAEH -D131..D14B ; LVT # Lo [27] HANGUL SYLLABLE TEOG..HANGUL SYLLABLE TEOH -D14D..D167 ; LVT # Lo [27] HANGUL SYLLABLE TEG..HANGUL SYLLABLE TEH -D169..D183 ; LVT # Lo [27] HANGUL SYLLABLE TYEOG..HANGUL SYLLABLE TYEOH -D185..D19F ; LVT # Lo [27] HANGUL SYLLABLE TYEG..HANGUL SYLLABLE TYEH -D1A1..D1BB ; LVT # Lo [27] HANGUL SYLLABLE TOG..HANGUL SYLLABLE TOH -D1BD..D1D7 ; LVT # Lo [27] HANGUL SYLLABLE TWAG..HANGUL SYLLABLE TWAH -D1D9..D1F3 ; LVT # Lo [27] HANGUL SYLLABLE TWAEG..HANGUL SYLLABLE TWAEH -D1F5..D20F ; LVT # Lo [27] HANGUL SYLLABLE TOEG..HANGUL SYLLABLE TOEH -D211..D22B ; LVT # Lo [27] HANGUL SYLLABLE TYOG..HANGUL SYLLABLE TYOH -D22D..D247 ; LVT # Lo [27] HANGUL SYLLABLE TUG..HANGUL SYLLABLE TUH -D249..D263 ; LVT # Lo [27] HANGUL SYLLABLE TWEOG..HANGUL SYLLABLE TWEOH -D265..D27F ; LVT # Lo [27] HANGUL SYLLABLE TWEG..HANGUL SYLLABLE TWEH -D281..D29B ; LVT # Lo [27] HANGUL SYLLABLE TWIG..HANGUL SYLLABLE TWIH -D29D..D2B7 ; LVT # Lo [27] HANGUL SYLLABLE TYUG..HANGUL SYLLABLE TYUH -D2B9..D2D3 ; LVT # Lo [27] HANGUL SYLLABLE TEUG..HANGUL SYLLABLE TEUH -D2D5..D2EF ; LVT # Lo [27] HANGUL SYLLABLE TYIG..HANGUL SYLLABLE TYIH -D2F1..D30B ; LVT # Lo [27] HANGUL SYLLABLE TIG..HANGUL SYLLABLE TIH -D30D..D327 ; LVT # Lo [27] HANGUL SYLLABLE PAG..HANGUL SYLLABLE PAH -D329..D343 ; LVT # Lo [27] HANGUL SYLLABLE PAEG..HANGUL SYLLABLE PAEH -D345..D35F ; LVT # Lo [27] HANGUL SYLLABLE PYAG..HANGUL SYLLABLE PYAH -D361..D37B ; LVT # Lo [27] HANGUL SYLLABLE PYAEG..HANGUL SYLLABLE PYAEH -D37D..D397 ; LVT # Lo [27] HANGUL SYLLABLE PEOG..HANGUL SYLLABLE PEOH -D399..D3B3 ; LVT # Lo [27] HANGUL SYLLABLE PEG..HANGUL SYLLABLE PEH -D3B5..D3CF ; LVT # Lo [27] HANGUL SYLLABLE PYEOG..HANGUL SYLLABLE PYEOH -D3D1..D3EB ; LVT # Lo [27] HANGUL SYLLABLE PYEG..HANGUL SYLLABLE PYEH -D3ED..D407 ; LVT # Lo [27] HANGUL SYLLABLE POG..HANGUL SYLLABLE POH -D409..D423 ; LVT # Lo [27] HANGUL SYLLABLE PWAG..HANGUL SYLLABLE PWAH -D425..D43F ; LVT # Lo [27] HANGUL SYLLABLE PWAEG..HANGUL SYLLABLE PWAEH -D441..D45B ; LVT # Lo [27] HANGUL SYLLABLE POEG..HANGUL SYLLABLE POEH -D45D..D477 ; LVT # Lo [27] HANGUL SYLLABLE PYOG..HANGUL SYLLABLE PYOH -D479..D493 ; LVT # Lo [27] HANGUL SYLLABLE PUG..HANGUL SYLLABLE PUH -D495..D4AF ; LVT # Lo [27] HANGUL SYLLABLE PWEOG..HANGUL SYLLABLE PWEOH -D4B1..D4CB ; LVT # Lo [27] HANGUL SYLLABLE PWEG..HANGUL SYLLABLE PWEH -D4CD..D4E7 ; LVT # Lo [27] HANGUL SYLLABLE PWIG..HANGUL SYLLABLE PWIH -D4E9..D503 ; LVT # Lo [27] HANGUL SYLLABLE PYUG..HANGUL SYLLABLE PYUH -D505..D51F ; LVT # Lo [27] HANGUL SYLLABLE PEUG..HANGUL SYLLABLE PEUH -D521..D53B ; LVT # Lo [27] HANGUL SYLLABLE PYIG..HANGUL SYLLABLE PYIH -D53D..D557 ; LVT # Lo [27] HANGUL SYLLABLE PIG..HANGUL SYLLABLE PIH -D559..D573 ; LVT # Lo [27] HANGUL SYLLABLE HAG..HANGUL SYLLABLE HAH -D575..D58F ; LVT # Lo [27] HANGUL SYLLABLE HAEG..HANGUL SYLLABLE HAEH -D591..D5AB ; LVT # Lo [27] HANGUL SYLLABLE HYAG..HANGUL SYLLABLE HYAH -D5AD..D5C7 ; LVT # Lo [27] HANGUL SYLLABLE HYAEG..HANGUL SYLLABLE HYAEH -D5C9..D5E3 ; LVT # Lo [27] HANGUL SYLLABLE HEOG..HANGUL SYLLABLE HEOH -D5E5..D5FF ; LVT # Lo [27] HANGUL SYLLABLE HEG..HANGUL SYLLABLE HEH -D601..D61B ; LVT # Lo [27] HANGUL SYLLABLE HYEOG..HANGUL SYLLABLE HYEOH -D61D..D637 ; LVT # Lo [27] HANGUL SYLLABLE HYEG..HANGUL SYLLABLE HYEH -D639..D653 ; LVT # Lo [27] HANGUL SYLLABLE HOG..HANGUL SYLLABLE HOH -D655..D66F ; LVT # Lo [27] HANGUL SYLLABLE HWAG..HANGUL SYLLABLE HWAH -D671..D68B ; LVT # Lo [27] HANGUL SYLLABLE HWAEG..HANGUL SYLLABLE HWAEH -D68D..D6A7 ; LVT # Lo [27] HANGUL SYLLABLE HOEG..HANGUL SYLLABLE HOEH -D6A9..D6C3 ; LVT # Lo [27] HANGUL SYLLABLE HYOG..HANGUL SYLLABLE HYOH -D6C5..D6DF ; LVT # Lo [27] HANGUL SYLLABLE HUG..HANGUL SYLLABLE HUH -D6E1..D6FB ; LVT # Lo [27] HANGUL SYLLABLE HWEOG..HANGUL SYLLABLE HWEOH -D6FD..D717 ; LVT # Lo [27] HANGUL SYLLABLE HWEG..HANGUL SYLLABLE HWEH -D719..D733 ; LVT # Lo [27] HANGUL SYLLABLE HWIG..HANGUL SYLLABLE HWIH -D735..D74F ; LVT # Lo [27] HANGUL SYLLABLE HYUG..HANGUL SYLLABLE HYUH -D751..D76B ; LVT # Lo [27] HANGUL SYLLABLE HEUG..HANGUL SYLLABLE HEUH -D76D..D787 ; LVT # Lo [27] HANGUL SYLLABLE HYIG..HANGUL SYLLABLE HYIH -D789..D7A3 ; LVT # Lo [27] HANGUL SYLLABLE HIG..HANGUL SYLLABLE HIH diff --git a/lib/elixir/unicode/IdentifierType.txt b/lib/elixir/unicode/IdentifierType.txt new file mode 100644 index 00000000000..90b6f116f4b --- /dev/null +++ b/lib/elixir/unicode/IdentifierType.txt @@ -0,0 +1,5273 @@ +# IdentifierType.txt +# Date: 2025-08-04, 21:58:43 GMT +# © 2025 Unicode®, Inc. +# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. +# For terms of use and license, see https://www.unicode.org/terms_of_use.html +# +# Unicode Security Mechanisms for UTS #39 +# Version: 17.0.0 +# +# For documentation and usage, see https://www.unicode.org/reports/tr39 +# +# Format +# +# Field 0: code point +# Field 1: set of Identifier_Type values +# See the "Identifier_Status and Identifier_Type" table of UTS #39: +# https://www.unicode.org/reports/tr39/#Identifier_Status_and_Type + +# +# For the purpose of regular expressions, the property Identifier_Type is defined as +# mapping each code point to a set of enumerated values. +# The short name of Identifier_Type is ID_Type. +# The possible values are: +# Not_Character, Deprecated, Default_Ignorable, Not_NFKC, Not_XID, +# Exclusion, Obsolete, Technical, Uncommon_Use, Limited_Use, Inclusion, Recommended +# The short name of each value is the same as its long name. + +# All code points not explicitly listed for Identifier_Type +# have the value Not_Character. + +# @missing: 0000..10FFFF; Not_Character + +# As usual, sets are unordered, with no duplicate values. + + +# Identifier_Type: Recommended + +0030..0039 ; Recommended # 1.1 [10] DIGIT ZERO..DIGIT NINE +0041..005A ; Recommended # 1.1 [26] LATIN CAPITAL LETTER A..LATIN CAPITAL LETTER Z +005F ; Recommended # 1.1 LOW LINE +0061..007A ; Recommended # 1.1 [26] LATIN SMALL LETTER A..LATIN SMALL LETTER Z +00C0..00D6 ; Recommended # 1.1 [23] LATIN CAPITAL LETTER A WITH GRAVE..LATIN CAPITAL LETTER O WITH DIAERESIS +00D8..00F6 ; Recommended # 1.1 [31] LATIN CAPITAL LETTER O WITH STROKE..LATIN SMALL LETTER O WITH DIAERESIS +00F8..0113 ; Recommended # 1.1 [28] LATIN SMALL LETTER O WITH STROKE..LATIN SMALL LETTER E WITH MACRON +0116..012B ; Recommended # 1.1 [22] LATIN CAPITAL LETTER E WITH DOT ABOVE..LATIN SMALL LETTER I WITH MACRON +012E..0131 ; Recommended # 1.1 [4] LATIN CAPITAL LETTER I WITH OGONEK..LATIN SMALL LETTER DOTLESS I +0134..0137 ; Recommended # 1.1 [4] LATIN CAPITAL LETTER J WITH CIRCUMFLEX..LATIN SMALL LETTER K WITH CEDILLA +0139..013E ; Recommended # 1.1 [6] LATIN CAPITAL LETTER L WITH ACUTE..LATIN SMALL LETTER L WITH CARON +0141..0148 ; Recommended # 1.1 [8] LATIN CAPITAL LETTER L WITH STROKE..LATIN SMALL LETTER N WITH CARON +014A..014D ; Recommended # 1.1 [4] LATIN CAPITAL LETTER ENG..LATIN SMALL LETTER O WITH MACRON +0150..0155 ; Recommended # 1.1 [6] LATIN CAPITAL LETTER O WITH DOUBLE ACUTE..LATIN SMALL LETTER R WITH ACUTE +0158..0161 ; Recommended # 1.1 [10] LATIN CAPITAL LETTER R WITH CARON..LATIN SMALL LETTER S WITH CARON +0164..017E ; Recommended # 1.1 [27] LATIN CAPITAL LETTER T WITH CARON..LATIN SMALL LETTER Z WITH CARON +0181 ; Recommended # 1.1 LATIN CAPITAL LETTER B WITH HOOK +0186 ; Recommended # 1.1 LATIN CAPITAL LETTER OPEN O +0189..018A ; Recommended # 1.1 [2] LATIN CAPITAL LETTER AFRICAN D..LATIN CAPITAL LETTER D WITH HOOK +018E..0192 ; Recommended # 1.1 [5] LATIN CAPITAL LETTER REVERSED E..LATIN SMALL LETTER F WITH HOOK +0194 ; Recommended # 1.1 LATIN CAPITAL LETTER GAMMA +0196..0199 ; Recommended # 1.1 [4] LATIN CAPITAL LETTER IOTA..LATIN SMALL LETTER K WITH HOOK +019D ; Recommended # 1.1 LATIN CAPITAL LETTER N WITH LEFT HOOK +01A0..01A1 ; Recommended # 1.1 [2] LATIN CAPITAL LETTER O WITH HORN..LATIN SMALL LETTER O WITH HORN +01AF..01B0 ; Recommended # 1.1 [2] LATIN CAPITAL LETTER U WITH HORN..LATIN SMALL LETTER U WITH HORN +01B2..01B4 ; Recommended # 1.1 [3] LATIN CAPITAL LETTER V WITH HOOK..LATIN SMALL LETTER Y WITH HOOK +01B7 ; Recommended # 1.1 LATIN CAPITAL LETTER EZH +01CD..01D4 ; Recommended # 1.1 [8] LATIN CAPITAL LETTER A WITH CARON..LATIN SMALL LETTER U WITH CARON +01DD ; Recommended # 1.1 LATIN SMALL LETTER TURNED E +01E6..01E9 ; Recommended # 1.1 [4] LATIN CAPITAL LETTER G WITH CARON..LATIN SMALL LETTER K WITH CARON +01EE..01EF ; Recommended # 1.1 [2] LATIN CAPITAL LETTER EZH WITH CARON..LATIN SMALL LETTER EZH WITH CARON +01F8..01F9 ; Recommended # 3.0 [2] LATIN CAPITAL LETTER N WITH GRAVE..LATIN SMALL LETTER N WITH GRAVE +0218..021B ; Recommended # 3.0 [4] LATIN CAPITAL LETTER S WITH COMMA BELOW..LATIN SMALL LETTER T WITH COMMA BELOW +0244 ; Recommended # 5.0 LATIN CAPITAL LETTER U BAR +024C..024D ; Recommended # 5.0 [2] LATIN CAPITAL LETTER R WITH STROKE..LATIN SMALL LETTER R WITH STROKE +0253..0254 ; Recommended # 1.1 [2] LATIN SMALL LETTER B WITH HOOK..LATIN SMALL LETTER OPEN O +0256..0257 ; Recommended # 1.1 [2] LATIN SMALL LETTER D WITH TAIL..LATIN SMALL LETTER D WITH HOOK +0259 ; Recommended # 1.1 LATIN SMALL LETTER SCHWA +025B ; Recommended # 1.1 LATIN SMALL LETTER OPEN E +0263 ; Recommended # 1.1 LATIN SMALL LETTER GAMMA +0268..0269 ; Recommended # 1.1 [2] LATIN SMALL LETTER I WITH STROKE..LATIN SMALL LETTER IOTA +0272 ; Recommended # 1.1 LATIN SMALL LETTER N WITH LEFT HOOK +0289 ; Recommended # 1.1 LATIN SMALL LETTER U BAR +028B ; Recommended # 1.1 LATIN SMALL LETTER V WITH HOOK +0292 ; Recommended # 1.1 LATIN SMALL LETTER EZH +0300..0304 ; Recommended # 1.1 [5] COMBINING GRAVE ACCENT..COMBINING MACRON +0306..030C ; Recommended # 1.1 [7] COMBINING BREVE..COMBINING CARON +031B ; Recommended # 1.1 COMBINING HORN +0323 ; Recommended # 1.1 COMBINING DOT BELOW +0326..0328 ; Recommended # 1.1 [3] COMBINING COMMA BELOW..COMBINING OGONEK +0331 ; Recommended # 1.1 COMBINING MACRON BELOW +0386 ; Recommended # 1.1 GREEK CAPITAL LETTER ALPHA WITH TONOS +0388..038A ; Recommended # 1.1 [3] GREEK CAPITAL LETTER EPSILON WITH TONOS..GREEK CAPITAL LETTER IOTA WITH TONOS +038C ; Recommended # 1.1 GREEK CAPITAL LETTER OMICRON WITH TONOS +038E..03A1 ; Recommended # 1.1 [20] GREEK CAPITAL LETTER UPSILON WITH TONOS..GREEK CAPITAL LETTER RHO +03A3..03CE ; Recommended # 1.1 [44] GREEK CAPITAL LETTER SIGMA..GREEK SMALL LETTER OMEGA WITH TONOS +0401..040C ; Recommended # 1.1 [12] CYRILLIC CAPITAL LETTER IO..CYRILLIC CAPITAL LETTER KJE +040E..044F ; Recommended # 1.1 [66] CYRILLIC CAPITAL LETTER SHORT U..CYRILLIC SMALL LETTER YA +0451..045C ; Recommended # 1.1 [12] CYRILLIC SMALL LETTER IO..CYRILLIC SMALL LETTER KJE +045E..045F ; Recommended # 1.1 [2] CYRILLIC SMALL LETTER SHORT U..CYRILLIC SMALL LETTER DZHE +0490..049B ; Recommended # 1.1 [12] CYRILLIC CAPITAL LETTER GHE WITH UPTURN..CYRILLIC SMALL LETTER KA WITH DESCENDER +049E..04A5 ; Recommended # 1.1 [8] CYRILLIC CAPITAL LETTER KA WITH STROKE..CYRILLIC SMALL LIGATURE EN GHE +04A8..04B7 ; Recommended # 1.1 [16] CYRILLIC CAPITAL LETTER ABKHASIAN HA..CYRILLIC SMALL LETTER CHE WITH DESCENDER +04BA..04C0 ; Recommended # 1.1 [7] CYRILLIC CAPITAL LETTER SHHA..CYRILLIC LETTER PALOCHKA +04CF ; Recommended # 5.0 CYRILLIC SMALL LETTER PALOCHKA +04D0..04D9 ; Recommended # 1.1 [10] CYRILLIC CAPITAL LETTER A WITH BREVE..CYRILLIC SMALL LETTER SCHWA +04DC..04E9 ; Recommended # 1.1 [14] CYRILLIC CAPITAL LETTER ZHE WITH DIAERESIS..CYRILLIC SMALL LETTER BARRED O +04EE..04F5 ; Recommended # 1.1 [8] CYRILLIC CAPITAL LETTER U WITH MACRON..CYRILLIC SMALL LETTER CHE WITH DIAERESIS +04F8..04F9 ; Recommended # 1.1 [2] CYRILLIC CAPITAL LETTER YERU WITH DIAERESIS..CYRILLIC SMALL LETTER YERU WITH DIAERESIS +0524..0525 ; Recommended # 5.2 [2] CYRILLIC CAPITAL LETTER PE WITH DESCENDER..CYRILLIC SMALL LETTER PE WITH DESCENDER +0531..0556 ; Recommended # 1.1 [38] ARMENIAN CAPITAL LETTER AYB..ARMENIAN CAPITAL LETTER FEH +0561..0586 ; Recommended # 1.1 [38] ARMENIAN SMALL LETTER AYB..ARMENIAN SMALL LETTER FEH +05D0..05EA ; Recommended # 1.1 [27] HEBREW LETTER ALEF..HEBREW LETTER TAV +0620 ; Recommended # 6.0 ARABIC LETTER KASHMIRI YEH +0621..063A ; Recommended # 1.1 [26] ARABIC LETTER HAMZA..ARABIC LETTER GHAIN +063D ; Recommended # 5.1 ARABIC LETTER FARSI YEH WITH INVERTED V +0641..0652 ; Recommended # 1.1 [18] ARABIC LETTER FEH..ARABIC SUKUN +0654..0655 ; Recommended # 3.0 [2] ARABIC HAMZA ABOVE..ARABIC HAMZA BELOW +0660..0669 ; Recommended # 1.1 [10] ARABIC-INDIC DIGIT ZERO..ARABIC-INDIC DIGIT NINE +0670 ; Recommended # 1.1 ARABIC LETTER SUPERSCRIPT ALEF +0672 ; Recommended # 1.1 ARABIC LETTER ALEF WITH WAVY HAMZA ABOVE +0674 ; Recommended # 1.1 ARABIC LETTER HIGH HAMZA +0679..068F ; Recommended # 1.1 [23] ARABIC LETTER TTEH..ARABIC LETTER DAL WITH THREE DOTS ABOVE DOWNWARDS +0691..069A ; Recommended # 1.1 [10] ARABIC LETTER RREH..ARABIC LETTER SEEN WITH DOT BELOW AND DOT ABOVE +069F..06A0 ; Recommended # 1.1 [2] ARABIC LETTER TAH WITH THREE DOTS ABOVE..ARABIC LETTER AIN WITH THREE DOTS ABOVE +06A2 ; Recommended # 1.1 ARABIC LETTER FEH WITH DOT MOVED BELOW +06A4..06AB ; Recommended # 1.1 [8] ARABIC LETTER VEH..ARABIC LETTER KAF WITH RING +06AD..06B1 ; Recommended # 1.1 [5] ARABIC LETTER NG..ARABIC LETTER NGOEH +06B3 ; Recommended # 1.1 ARABIC LETTER GUEH +06B5..06B7 ; Recommended # 1.1 [3] ARABIC LETTER LAM WITH SMALL V..ARABIC LETTER LAM WITH THREE DOTS ABOVE +06BA..06BE ; Recommended # 1.1 [5] ARABIC LETTER NOON GHUNNA..ARABIC LETTER HEH DOACHASHMEE +06C0..06CE ; Recommended # 1.1 [15] ARABIC LETTER HEH WITH YEH ABOVE..ARABIC LETTER YEH WITH SMALL V +06CF ; Recommended # 3.0 ARABIC LETTER WAW WITH DOT ABOVE +06D0..06D3 ; Recommended # 1.1 [4] ARABIC LETTER E..ARABIC LETTER YEH BARREE WITH HAMZA ABOVE +06D5 ; Recommended # 1.1 ARABIC LETTER AE +06EE..06EF ; Recommended # 4.0 [2] ARABIC LETTER DAL WITH INVERTED V..ARABIC LETTER REH WITH INVERTED V +06F0..06F9 ; Recommended # 1.1 [10] EXTENDED ARABIC-INDIC DIGIT ZERO..EXTENDED ARABIC-INDIC DIGIT NINE +06FF ; Recommended # 4.0 ARABIC LETTER HEH WITH INVERTED V +0751..0752 ; Recommended # 4.1 [2] ARABIC LETTER BEH WITH DOT BELOW AND THREE DOTS ABOVE..ARABIC LETTER BEH WITH THREE DOTS POINTING UPWARDS BELOW +0756 ; Recommended # 4.1 ARABIC LETTER BEH WITH SMALL V +0760 ; Recommended # 4.1 ARABIC LETTER FEH WITH TWO DOTS BELOW +0762..0763 ; Recommended # 4.1 [2] ARABIC LETTER KEHEH WITH DOT ABOVE..ARABIC LETTER KEHEH WITH THREE DOTS ABOVE +0766..0768 ; Recommended # 4.1 [3] ARABIC LETTER MEEM WITH DOT BELOW..ARABIC LETTER NOON WITH SMALL TAH +076A ; Recommended # 4.1 ARABIC LETTER LAM WITH BAR +076E..0771 ; Recommended # 5.1 [4] ARABIC LETTER HAH WITH SMALL ARABIC LETTER TAH BELOW..ARABIC LETTER REH WITH SMALL ARABIC LETTER TAH AND TWO DOTS +0780..07B0 ; Recommended # 3.0 [49] THAANA LETTER HAA..THAANA SUKUN +07B1 ; Recommended # 3.2 THAANA LETTER NAA +088F ; Recommended # 17.0 ARABIC LETTER NOON WITH RING ABOVE +08A0 ; Recommended # 6.1 ARABIC LETTER BEH WITH SMALL V BELOW +08A2..08A9 ; Recommended # 6.1 [8] ARABIC LETTER JEEM WITH TWO DOTS ABOVE..ARABIC LETTER YEH WITH TWO DOTS BELOW AND DOT ABOVE +08BB..08BD ; Recommended # 9.0 [3] ARABIC LETTER AFRICAN FEH..ARABIC LETTER AFRICAN NOON +08BE..08C2 ; Recommended # 13.0 [5] ARABIC LETTER PEH WITH SMALL V..ARABIC LETTER KEHEH WITH SMALL V +08C7 ; Recommended # 13.0 ARABIC LETTER LAM WITH SMALL ARABIC LETTER TAH ABOVE +0901..0903 ; Recommended # 1.1 [3] DEVANAGARI SIGN CANDRABINDU..DEVANAGARI SIGN VISARGA +0905..090B ; Recommended # 1.1 [7] DEVANAGARI LETTER A..DEVANAGARI LETTER VOCALIC R +090D..0928 ; Recommended # 1.1 [28] DEVANAGARI LETTER CANDRA E..DEVANAGARI LETTER NA +092A..0933 ; Recommended # 1.1 [10] DEVANAGARI LETTER PA..DEVANAGARI LETTER LLA +0935..0939 ; Recommended # 1.1 [5] DEVANAGARI LETTER VA..DEVANAGARI LETTER HA +093A..093B ; Recommended # 6.0 [2] DEVANAGARI VOWEL SIGN OE..DEVANAGARI VOWEL SIGN OOE +093C ; Recommended # 1.1 DEVANAGARI SIGN NUKTA +093E..0943 ; Recommended # 1.1 [6] DEVANAGARI VOWEL SIGN AA..DEVANAGARI VOWEL SIGN VOCALIC R +0945..094D ; Recommended # 1.1 [9] DEVANAGARI VOWEL SIGN CANDRA E..DEVANAGARI SIGN VIRAMA +094F ; Recommended # 6.0 DEVANAGARI VOWEL SIGN AW +0956..0957 ; Recommended # 6.0 [2] DEVANAGARI VOWEL SIGN UE..DEVANAGARI VOWEL SIGN UUE +0966..096F ; Recommended # 1.1 [10] DEVANAGARI DIGIT ZERO..DEVANAGARI DIGIT NINE +0972 ; Recommended # 5.1 DEVANAGARI LETTER CANDRA A +0973..0977 ; Recommended # 6.0 [5] DEVANAGARI LETTER OE..DEVANAGARI LETTER UUE +097B..097C ; Recommended # 5.0 [2] DEVANAGARI LETTER GGA..DEVANAGARI LETTER JJA +097E..097F ; Recommended # 5.0 [2] DEVANAGARI LETTER DDDA..DEVANAGARI LETTER BBA +0981..0983 ; Recommended # 1.1 [3] BENGALI SIGN CANDRABINDU..BENGALI SIGN VISARGA +0985..098B ; Recommended # 1.1 [7] BENGALI LETTER A..BENGALI LETTER VOCALIC R +098F..0990 ; Recommended # 1.1 [2] BENGALI LETTER E..BENGALI LETTER AI +0993..09A8 ; Recommended # 1.1 [22] BENGALI LETTER O..BENGALI LETTER NA +09AA..09B0 ; Recommended # 1.1 [7] BENGALI LETTER PA..BENGALI LETTER RA +09B2 ; Recommended # 1.1 BENGALI LETTER LA +09B6..09B9 ; Recommended # 1.1 [4] BENGALI LETTER SHA..BENGALI LETTER HA +09BC ; Recommended # 1.1 BENGALI SIGN NUKTA +09BE..09C4 ; Recommended # 1.1 [7] BENGALI VOWEL SIGN AA..BENGALI VOWEL SIGN VOCALIC RR +09C7..09C8 ; Recommended # 1.1 [2] BENGALI VOWEL SIGN E..BENGALI VOWEL SIGN AI +09CB..09CD ; Recommended # 1.1 [3] BENGALI VOWEL SIGN O..BENGALI SIGN VIRAMA +09CE ; Recommended # 4.1 BENGALI LETTER KHANDA TA +09E6..09F1 ; Recommended # 1.1 [12] BENGALI DIGIT ZERO..BENGALI LETTER RA WITH LOWER DIAGONAL +0A02 ; Recommended # 1.1 GURMUKHI SIGN BINDI +0A05..0A0A ; Recommended # 1.1 [6] GURMUKHI LETTER A..GURMUKHI LETTER UU +0A0F..0A10 ; Recommended # 1.1 [2] GURMUKHI LETTER EE..GURMUKHI LETTER AI +0A13..0A28 ; Recommended # 1.1 [22] GURMUKHI LETTER OO..GURMUKHI LETTER NA +0A2A..0A30 ; Recommended # 1.1 [7] GURMUKHI LETTER PA..GURMUKHI LETTER RA +0A32 ; Recommended # 1.1 GURMUKHI LETTER LA +0A35 ; Recommended # 1.1 GURMUKHI LETTER VA +0A38..0A39 ; Recommended # 1.1 [2] GURMUKHI LETTER SA..GURMUKHI LETTER HA +0A3C ; Recommended # 1.1 GURMUKHI SIGN NUKTA +0A3E..0A42 ; Recommended # 1.1 [5] GURMUKHI VOWEL SIGN AA..GURMUKHI VOWEL SIGN UU +0A47..0A48 ; Recommended # 1.1 [2] GURMUKHI VOWEL SIGN EE..GURMUKHI VOWEL SIGN AI +0A4B..0A4D ; Recommended # 1.1 [3] GURMUKHI VOWEL SIGN OO..GURMUKHI SIGN VIRAMA +0A5C ; Recommended # 1.1 GURMUKHI LETTER RRA +0A70..0A71 ; Recommended # 1.1 [2] GURMUKHI TIPPI..GURMUKHI ADDAK +0A82..0A83 ; Recommended # 1.1 [2] GUJARATI SIGN ANUSVARA..GUJARATI SIGN VISARGA +0A85..0A8B ; Recommended # 1.1 [7] GUJARATI LETTER A..GUJARATI LETTER VOCALIC R +0A8C ; Recommended # 4.0 GUJARATI LETTER VOCALIC L +0A8D ; Recommended # 1.1 GUJARATI VOWEL CANDRA E +0A8F..0A91 ; Recommended # 1.1 [3] GUJARATI LETTER E..GUJARATI VOWEL CANDRA O +0A93..0AA8 ; Recommended # 1.1 [22] GUJARATI LETTER O..GUJARATI LETTER NA +0AAA..0AB0 ; Recommended # 1.1 [7] GUJARATI LETTER PA..GUJARATI LETTER RA +0AB2..0AB3 ; Recommended # 1.1 [2] GUJARATI LETTER LA..GUJARATI LETTER LLA +0AB5..0AB9 ; Recommended # 1.1 [5] GUJARATI LETTER VA..GUJARATI LETTER HA +0ABC ; Recommended # 1.1 GUJARATI SIGN NUKTA +0ABE..0AC5 ; Recommended # 1.1 [8] GUJARATI VOWEL SIGN AA..GUJARATI VOWEL SIGN CANDRA E +0AC7..0AC9 ; Recommended # 1.1 [3] GUJARATI VOWEL SIGN E..GUJARATI VOWEL SIGN CANDRA O +0ACB..0ACD ; Recommended # 1.1 [3] GUJARATI VOWEL SIGN O..GUJARATI SIGN VIRAMA +0AE6..0AEF ; Recommended # 1.1 [10] GUJARATI DIGIT ZERO..GUJARATI DIGIT NINE +0B01..0B03 ; Recommended # 1.1 [3] ORIYA SIGN CANDRABINDU..ORIYA SIGN VISARGA +0B05..0B0B ; Recommended # 1.1 [7] ORIYA LETTER A..ORIYA LETTER VOCALIC R +0B0F..0B10 ; Recommended # 1.1 [2] ORIYA LETTER E..ORIYA LETTER AI +0B13..0B28 ; Recommended # 1.1 [22] ORIYA LETTER O..ORIYA LETTER NA +0B2A..0B30 ; Recommended # 1.1 [7] ORIYA LETTER PA..ORIYA LETTER RA +0B32..0B33 ; Recommended # 1.1 [2] ORIYA LETTER LA..ORIYA LETTER LLA +0B36..0B39 ; Recommended # 1.1 [4] ORIYA LETTER SHA..ORIYA LETTER HA +0B3C ; Recommended # 1.1 ORIYA SIGN NUKTA +0B3E..0B43 ; Recommended # 1.1 [6] ORIYA VOWEL SIGN AA..ORIYA VOWEL SIGN VOCALIC R +0B47..0B48 ; Recommended # 1.1 [2] ORIYA VOWEL SIGN E..ORIYA VOWEL SIGN AI +0B4B..0B4D ; Recommended # 1.1 [3] ORIYA VOWEL SIGN O..ORIYA SIGN VIRAMA +0B56 ; Recommended # 1.1 ORIYA AI LENGTH MARK +0B5F ; Recommended # 1.1 ORIYA LETTER YYA +0B71 ; Recommended # 4.0 ORIYA LETTER WA +0B83 ; Recommended # 1.1 TAMIL SIGN VISARGA +0B85..0B8A ; Recommended # 1.1 [6] TAMIL LETTER A..TAMIL LETTER UU +0B8E..0B90 ; Recommended # 1.1 [3] TAMIL LETTER E..TAMIL LETTER AI +0B92..0B95 ; Recommended # 1.1 [4] TAMIL LETTER O..TAMIL LETTER KA +0B99..0B9A ; Recommended # 1.1 [2] TAMIL LETTER NGA..TAMIL LETTER CA +0B9C ; Recommended # 1.1 TAMIL LETTER JA +0B9E..0B9F ; Recommended # 1.1 [2] TAMIL LETTER NYA..TAMIL LETTER TTA +0BA3..0BA4 ; Recommended # 1.1 [2] TAMIL LETTER NNA..TAMIL LETTER TA +0BA8..0BAA ; Recommended # 1.1 [3] TAMIL LETTER NA..TAMIL LETTER PA +0BAE..0BB5 ; Recommended # 1.1 [8] TAMIL LETTER MA..TAMIL LETTER VA +0BB6 ; Recommended # 4.1 TAMIL LETTER SHA +0BB7..0BB9 ; Recommended # 1.1 [3] TAMIL LETTER SSA..TAMIL LETTER HA +0BBE..0BC2 ; Recommended # 1.1 [5] TAMIL VOWEL SIGN AA..TAMIL VOWEL SIGN UU +0BC6..0BC8 ; Recommended # 1.1 [3] TAMIL VOWEL SIGN E..TAMIL VOWEL SIGN AI +0BCA..0BCD ; Recommended # 1.1 [4] TAMIL VOWEL SIGN O..TAMIL SIGN VIRAMA +0C02..0C03 ; Recommended # 1.1 [2] TELUGU SIGN ANUSVARA..TELUGU SIGN VISARGA +0C05..0C0B ; Recommended # 1.1 [7] TELUGU LETTER A..TELUGU LETTER VOCALIC R +0C0E..0C10 ; Recommended # 1.1 [3] TELUGU LETTER E..TELUGU LETTER AI +0C12..0C28 ; Recommended # 1.1 [23] TELUGU LETTER O..TELUGU LETTER NA +0C2A..0C30 ; Recommended # 1.1 [7] TELUGU LETTER PA..TELUGU LETTER RA +0C32..0C33 ; Recommended # 1.1 [2] TELUGU LETTER LA..TELUGU LETTER LLA +0C35..0C39 ; Recommended # 1.1 [5] TELUGU LETTER VA..TELUGU LETTER HA +0C3E..0C44 ; Recommended # 1.1 [7] TELUGU VOWEL SIGN AA..TELUGU VOWEL SIGN VOCALIC RR +0C46..0C48 ; Recommended # 1.1 [3] TELUGU VOWEL SIGN E..TELUGU VOWEL SIGN AI +0C4A..0C4D ; Recommended # 1.1 [4] TELUGU VOWEL SIGN O..TELUGU SIGN VIRAMA +0C82..0C83 ; Recommended # 1.1 [2] KANNADA SIGN ANUSVARA..KANNADA SIGN VISARGA +0C85..0C8B ; Recommended # 1.1 [7] KANNADA LETTER A..KANNADA LETTER VOCALIC R +0C8E..0C90 ; Recommended # 1.1 [3] KANNADA LETTER E..KANNADA LETTER AI +0C92..0CA8 ; Recommended # 1.1 [23] KANNADA LETTER O..KANNADA LETTER NA +0CAA..0CB0 ; Recommended # 1.1 [7] KANNADA LETTER PA..KANNADA LETTER RA +0CB2..0CB3 ; Recommended # 1.1 [2] KANNADA LETTER LA..KANNADA LETTER LLA +0CB5..0CB9 ; Recommended # 1.1 [5] KANNADA LETTER VA..KANNADA LETTER HA +0CBE..0CC3 ; Recommended # 1.1 [6] KANNADA VOWEL SIGN AA..KANNADA VOWEL SIGN VOCALIC R +0CC6..0CC8 ; Recommended # 1.1 [3] KANNADA VOWEL SIGN E..KANNADA VOWEL SIGN AI +0CCA..0CCD ; Recommended # 1.1 [4] KANNADA VOWEL SIGN O..KANNADA SIGN VIRAMA +0CE6..0CEF ; Recommended # 1.1 [10] KANNADA DIGIT ZERO..KANNADA DIGIT NINE +0D02..0D03 ; Recommended # 1.1 [2] MALAYALAM SIGN ANUSVARA..MALAYALAM SIGN VISARGA +0D05..0D0B ; Recommended # 1.1 [7] MALAYALAM LETTER A..MALAYALAM LETTER VOCALIC R +0D0E..0D10 ; Recommended # 1.1 [3] MALAYALAM LETTER E..MALAYALAM LETTER AI +0D12..0D28 ; Recommended # 1.1 [23] MALAYALAM LETTER O..MALAYALAM LETTER NA +0D2A..0D39 ; Recommended # 1.1 [16] MALAYALAM LETTER PA..MALAYALAM LETTER HA +0D3E..0D43 ; Recommended # 1.1 [6] MALAYALAM VOWEL SIGN AA..MALAYALAM VOWEL SIGN VOCALIC R +0D46..0D48 ; Recommended # 1.1 [3] MALAYALAM VOWEL SIGN E..MALAYALAM VOWEL SIGN AI +0D4A..0D4B ; Recommended # 1.1 [2] MALAYALAM VOWEL SIGN O..MALAYALAM VOWEL SIGN OO +0D4D ; Recommended # 1.1 MALAYALAM SIGN VIRAMA +0D57 ; Recommended # 1.1 MALAYALAM AU LENGTH MARK +0D7A..0D7F ; Recommended # 5.1 [6] MALAYALAM LETTER CHILLU NN..MALAYALAM LETTER CHILLU K +0D82..0D83 ; Recommended # 3.0 [2] SINHALA SIGN ANUSVARAYA..SINHALA SIGN VISARGAYA +0D85..0D8D ; Recommended # 3.0 [9] SINHALA LETTER AYANNA..SINHALA LETTER IRUYANNA +0D91..0D96 ; Recommended # 3.0 [6] SINHALA LETTER EYANNA..SINHALA LETTER AUYANNA +0D9A..0D9D ; Recommended # 3.0 [4] SINHALA LETTER ALPAPRAANA KAYANNA..SINHALA LETTER MAHAAPRAANA GAYANNA +0D9F..0DB1 ; Recommended # 3.0 [19] SINHALA LETTER SANYAKA GAYANNA..SINHALA LETTER DANTAJA NAYANNA +0DB3..0DBB ; Recommended # 3.0 [9] SINHALA LETTER SANYAKA DAYANNA..SINHALA LETTER RAYANNA +0DBD ; Recommended # 3.0 SINHALA LETTER DANTAJA LAYANNA +0DC0..0DC6 ; Recommended # 3.0 [7] SINHALA LETTER VAYANNA..SINHALA LETTER FAYANNA +0DCA ; Recommended # 3.0 SINHALA SIGN AL-LAKUNA +0DCF..0DD4 ; Recommended # 3.0 [6] SINHALA VOWEL SIGN AELA-PILLA..SINHALA VOWEL SIGN KETTI PAA-PILLA +0DD6 ; Recommended # 3.0 SINHALA VOWEL SIGN DIGA PAA-PILLA +0DD8..0DDE ; Recommended # 3.0 [7] SINHALA VOWEL SIGN GAETTA-PILLA..SINHALA VOWEL SIGN KOMBUVA HAA GAYANUKITTA +0DF2 ; Recommended # 3.0 SINHALA VOWEL SIGN DIGA GAETTA-PILLA +0E01..0E32 ; Recommended # 1.1 [50] THAI CHARACTER KO KAI..THAI CHARACTER SARA AA +0E34..0E3A ; Recommended # 1.1 [7] THAI CHARACTER SARA I..THAI CHARACTER PHINTHU +0E40..0E4D ; Recommended # 1.1 [14] THAI CHARACTER SARA E..THAI CHARACTER NIKHAHIT +0E50..0E59 ; Recommended # 1.1 [10] THAI DIGIT ZERO..THAI DIGIT NINE +0E81..0E82 ; Recommended # 1.1 [2] LAO LETTER KO..LAO LETTER KHO SUNG +0E84 ; Recommended # 1.1 LAO LETTER KHO TAM +0E87..0E88 ; Recommended # 1.1 [2] LAO LETTER NGO..LAO LETTER CO +0E8A ; Recommended # 1.1 LAO LETTER SO TAM +0E8D ; Recommended # 1.1 LAO LETTER NYO +0E94..0E97 ; Recommended # 1.1 [4] LAO LETTER DO..LAO LETTER THO TAM +0E99..0E9F ; Recommended # 1.1 [7] LAO LETTER NO..LAO LETTER FO SUNG +0EA1..0EA3 ; Recommended # 1.1 [3] LAO LETTER MO..LAO LETTER LO LING +0EA5 ; Recommended # 1.1 LAO LETTER LO LOOT +0EA7 ; Recommended # 1.1 LAO LETTER WO +0EAA..0EAB ; Recommended # 1.1 [2] LAO LETTER SO SUNG..LAO LETTER HO SUNG +0EAD..0EAE ; Recommended # 1.1 [2] LAO LETTER O..LAO LETTER HO TAM +0EB0..0EB2 ; Recommended # 1.1 [3] LAO VOWEL SIGN A..LAO VOWEL SIGN AA +0EB4..0EB9 ; Recommended # 1.1 [6] LAO VOWEL SIGN I..LAO VOWEL SIGN UU +0EBB..0EBD ; Recommended # 1.1 [3] LAO VOWEL SIGN MAI KON..LAO SEMIVOWEL SIGN NYO +0EC0..0EC4 ; Recommended # 1.1 [5] LAO VOWEL SIGN E..LAO VOWEL SIGN AI +0EC6 ; Recommended # 1.1 LAO KO LA +0EC8..0ECD ; Recommended # 1.1 [6] LAO TONE MAI EK..LAO NIGGAHITA +0ED0..0ED9 ; Recommended # 1.1 [10] LAO DIGIT ZERO..LAO DIGIT NINE +0F20..0F29 ; Recommended # 2.0 [10] TIBETAN DIGIT ZERO..TIBETAN DIGIT NINE +0F40..0F42 ; Recommended # 2.0 [3] TIBETAN LETTER KA..TIBETAN LETTER GA +0F44..0F47 ; Recommended # 2.0 [4] TIBETAN LETTER NGA..TIBETAN LETTER JA +0F49..0F4C ; Recommended # 2.0 [4] TIBETAN LETTER NYA..TIBETAN LETTER DDA +0F4E..0F51 ; Recommended # 2.0 [4] TIBETAN LETTER NNA..TIBETAN LETTER DA +0F53..0F56 ; Recommended # 2.0 [4] TIBETAN LETTER NA..TIBETAN LETTER BA +0F58..0F5B ; Recommended # 2.0 [4] TIBETAN LETTER MA..TIBETAN LETTER DZA +0F5D..0F68 ; Recommended # 2.0 [12] TIBETAN LETTER WA..TIBETAN LETTER A +0F71..0F72 ; Recommended # 2.0 [2] TIBETAN VOWEL SIGN AA..TIBETAN VOWEL SIGN I +0F74 ; Recommended # 2.0 TIBETAN VOWEL SIGN U +0F7A..0F80 ; Recommended # 2.0 [7] TIBETAN VOWEL SIGN E..TIBETAN VOWEL SIGN REVERSED I +0F84 ; Recommended # 2.0 TIBETAN MARK HALANTA +0F90..0F92 ; Recommended # 2.0 [3] TIBETAN SUBJOINED LETTER KA..TIBETAN SUBJOINED LETTER GA +0F94..0F95 ; Recommended # 2.0 [2] TIBETAN SUBJOINED LETTER NGA..TIBETAN SUBJOINED LETTER CA +0F96 ; Recommended # 3.0 TIBETAN SUBJOINED LETTER CHA +0F97 ; Recommended # 2.0 TIBETAN SUBJOINED LETTER JA +0F99..0F9C ; Recommended # 2.0 [4] TIBETAN SUBJOINED LETTER NYA..TIBETAN SUBJOINED LETTER DDA +0F9E..0FA1 ; Recommended # 2.0 [4] TIBETAN SUBJOINED LETTER NNA..TIBETAN SUBJOINED LETTER DA +0FA3..0FA6 ; Recommended # 2.0 [4] TIBETAN SUBJOINED LETTER NA..TIBETAN SUBJOINED LETTER BA +0FA8..0FAB ; Recommended # 2.0 [4] TIBETAN SUBJOINED LETTER MA..TIBETAN SUBJOINED LETTER DZA +0FAD ; Recommended # 2.0 TIBETAN SUBJOINED LETTER WA +0FB1..0FB7 ; Recommended # 2.0 [7] TIBETAN SUBJOINED LETTER YA..TIBETAN SUBJOINED LETTER HA +0FB8 ; Recommended # 3.0 TIBETAN SUBJOINED LETTER A +0FBA..0FBC ; Recommended # 3.0 [3] TIBETAN SUBJOINED LETTER FIXED-FORM WA..TIBETAN SUBJOINED LETTER FIXED-FORM RA +1000..1021 ; Recommended # 3.0 [34] MYANMAR LETTER KA..MYANMAR LETTER A +1022 ; Recommended # 5.1 MYANMAR LETTER SHAN A +1023..1027 ; Recommended # 3.0 [5] MYANMAR LETTER I..MYANMAR LETTER E +1028 ; Recommended # 5.1 MYANMAR LETTER MON E +1029..102A ; Recommended # 3.0 [2] MYANMAR LETTER O..MYANMAR LETTER AU +102B ; Recommended # 5.1 MYANMAR VOWEL SIGN TALL AA +102C..1032 ; Recommended # 3.0 [7] MYANMAR VOWEL SIGN AA..MYANMAR VOWEL SIGN AI +1033..1035 ; Recommended # 5.1 [3] MYANMAR VOWEL SIGN MON II..MYANMAR VOWEL SIGN E ABOVE +1036..1039 ; Recommended # 3.0 [4] MYANMAR SIGN ANUSVARA..MYANMAR SIGN VIRAMA +103A..103F ; Recommended # 5.1 [6] MYANMAR SIGN ASAT..MYANMAR LETTER GREAT SA +1040..1049 ; Recommended # 3.0 [10] MYANMAR DIGIT ZERO..MYANMAR DIGIT NINE +105A..1064 ; Recommended # 5.1 [11] MYANMAR LETTER MON NGA..MYANMAR TONE MARK SGAW KAREN KE PHO +1075..108A ; Recommended # 5.1 [22] MYANMAR LETTER SHAN KA..MYANMAR SIGN SHAN TONE-6 +108F ; Recommended # 5.1 MYANMAR SIGN RUMAI PALAUNG TONE-5 +10C7 ; Recommended # 6.1 GEORGIAN CAPITAL LETTER YN +10CD ; Recommended # 6.1 GEORGIAN CAPITAL LETTER AEN +10D0..10F0 ; Recommended # 1.1 [33] GEORGIAN LETTER AN..GEORGIAN LETTER HAE +1200..1206 ; Recommended # 3.0 [7] ETHIOPIC SYLLABLE HA..ETHIOPIC SYLLABLE HO +1208..1246 ; Recommended # 3.0 [63] ETHIOPIC SYLLABLE LA..ETHIOPIC SYLLABLE QO +1247 ; Recommended # 4.1 ETHIOPIC SYLLABLE QOA +1248 ; Recommended # 3.0 ETHIOPIC SYLLABLE QWA +124A..124D ; Recommended # 3.0 [4] ETHIOPIC SYLLABLE QWI..ETHIOPIC SYLLABLE QWE +1250..1256 ; Recommended # 3.0 [7] ETHIOPIC SYLLABLE QHA..ETHIOPIC SYLLABLE QHO +1258 ; Recommended # 3.0 ETHIOPIC SYLLABLE QHWA +125A..125D ; Recommended # 3.0 [4] ETHIOPIC SYLLABLE QHWI..ETHIOPIC SYLLABLE QHWE +1260..1286 ; Recommended # 3.0 [39] ETHIOPIC SYLLABLE BA..ETHIOPIC SYLLABLE XO +1288 ; Recommended # 3.0 ETHIOPIC SYLLABLE XWA +128A..128D ; Recommended # 3.0 [4] ETHIOPIC SYLLABLE XWI..ETHIOPIC SYLLABLE XWE +1290..12AE ; Recommended # 3.0 [31] ETHIOPIC SYLLABLE NA..ETHIOPIC SYLLABLE KO +12B0 ; Recommended # 3.0 ETHIOPIC SYLLABLE KWA +12B2..12B5 ; Recommended # 3.0 [4] ETHIOPIC SYLLABLE KWI..ETHIOPIC SYLLABLE KWE +12B8..12BE ; Recommended # 3.0 [7] ETHIOPIC SYLLABLE KXA..ETHIOPIC SYLLABLE KXO +12C0 ; Recommended # 3.0 ETHIOPIC SYLLABLE KXWA +12C2..12C5 ; Recommended # 3.0 [4] ETHIOPIC SYLLABLE KXWI..ETHIOPIC SYLLABLE KXWE +12C8..12CE ; Recommended # 3.0 [7] ETHIOPIC SYLLABLE WA..ETHIOPIC SYLLABLE WO +12CF ; Recommended # 4.1 ETHIOPIC SYLLABLE WOA +12D0..12D6 ; Recommended # 3.0 [7] ETHIOPIC SYLLABLE PHARYNGEAL A..ETHIOPIC SYLLABLE PHARYNGEAL O +12D8..12EE ; Recommended # 3.0 [23] ETHIOPIC SYLLABLE ZA..ETHIOPIC SYLLABLE YO +12EF ; Recommended # 4.1 ETHIOPIC SYLLABLE YOA +12F0..12F7 ; Recommended # 3.0 [8] ETHIOPIC SYLLABLE DA..ETHIOPIC SYLLABLE DWA +1300..130E ; Recommended # 3.0 [15] ETHIOPIC SYLLABLE JA..ETHIOPIC SYLLABLE GO +1310 ; Recommended # 3.0 ETHIOPIC SYLLABLE GWA +1312..1315 ; Recommended # 3.0 [4] ETHIOPIC SYLLABLE GWI..ETHIOPIC SYLLABLE GWE +1318..131E ; Recommended # 3.0 [7] ETHIOPIC SYLLABLE GGA..ETHIOPIC SYLLABLE GGO +1320..1346 ; Recommended # 3.0 [39] ETHIOPIC SYLLABLE THA..ETHIOPIC SYLLABLE TZO +1348..1359 ; Recommended # 3.0 [18] ETHIOPIC SYLLABLE FA..ETHIOPIC SYLLABLE MYA +1780..179C ; Recommended # 3.0 [29] KHMER LETTER KA..KHMER LETTER VO +179F..17A2 ; Recommended # 3.0 [4] KHMER LETTER SA..KHMER LETTER QA +17A5..17A7 ; Recommended # 3.0 [3] KHMER INDEPENDENT VOWEL QI..KHMER INDEPENDENT VOWEL QU +17AA..17B3 ; Recommended # 3.0 [10] KHMER INDEPENDENT VOWEL QUUV..KHMER INDEPENDENT VOWEL QAU +17B6..17CD ; Recommended # 3.0 [24] KHMER VOWEL SIGN AA..KHMER SIGN TOANDAKHIAT +17D0 ; Recommended # 3.0 KHMER SIGN SAMYOK SANNYA +17D2 ; Recommended # 3.0 KHMER SIGN COENG +17E0..17E9 ; Recommended # 3.0 [10] KHMER DIGIT ZERO..KHMER DIGIT NINE +1C90..1CBA ; Recommended # 11.0 [43] GEORGIAN MTAVRULI CAPITAL LETTER AN..GEORGIAN MTAVRULI CAPITAL LETTER AIN +1CBD..1CBF ; Recommended # 11.0 [3] GEORGIAN MTAVRULI CAPITAL LETTER AEN..GEORGIAN MTAVRULI CAPITAL LETTER LABIAL SIGN +1E0C..1E0D ; Recommended # 1.1 [2] LATIN CAPITAL LETTER D WITH DOT BELOW..LATIN SMALL LETTER D WITH DOT BELOW +1E12..1E13 ; Recommended # 1.1 [2] LATIN CAPITAL LETTER D WITH CIRCUMFLEX BELOW..LATIN SMALL LETTER D WITH CIRCUMFLEX BELOW +1E20..1E21 ; Recommended # 1.1 [2] LATIN CAPITAL LETTER G WITH MACRON..LATIN SMALL LETTER G WITH MACRON +1E24..1E25 ; Recommended # 1.1 [2] LATIN CAPITAL LETTER H WITH DOT BELOW..LATIN SMALL LETTER H WITH DOT BELOW +1E36..1E37 ; Recommended # 1.1 [2] LATIN CAPITAL LETTER L WITH DOT BELOW..LATIN SMALL LETTER L WITH DOT BELOW +1E3C..1E3F ; Recommended # 1.1 [4] LATIN CAPITAL LETTER L WITH CIRCUMFLEX BELOW..LATIN SMALL LETTER M WITH ACUTE +1E42..1E4B ; Recommended # 1.1 [10] LATIN CAPITAL LETTER M WITH DOT BELOW..LATIN SMALL LETTER N WITH CIRCUMFLEX BELOW +1E5A..1E5B ; Recommended # 1.1 [2] LATIN CAPITAL LETTER R WITH DOT BELOW..LATIN SMALL LETTER R WITH DOT BELOW +1E62..1E63 ; Recommended # 1.1 [2] LATIN CAPITAL LETTER S WITH DOT BELOW..LATIN SMALL LETTER S WITH DOT BELOW +1E6C..1E6D ; Recommended # 1.1 [2] LATIN CAPITAL LETTER T WITH DOT BELOW..LATIN SMALL LETTER T WITH DOT BELOW +1E70..1E71 ; Recommended # 1.1 [2] LATIN CAPITAL LETTER T WITH CIRCUMFLEX BELOW..LATIN SMALL LETTER T WITH CIRCUMFLEX BELOW +1E8C..1E8D ; Recommended # 1.1 [2] LATIN CAPITAL LETTER X WITH DIAERESIS..LATIN SMALL LETTER X WITH DIAERESIS +1E92..1E93 ; Recommended # 1.1 [2] LATIN CAPITAL LETTER Z WITH DOT BELOW..LATIN SMALL LETTER Z WITH DOT BELOW +1E9E ; Recommended # 5.1 LATIN CAPITAL LETTER SHARP S +1EA0..1EF9 ; Recommended # 1.1 [90] LATIN CAPITAL LETTER A WITH DOT BELOW..LATIN SMALL LETTER Y WITH TILDE +1FA0..1FAF ; Recommended # 1.1 [16] GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI..GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1FB2..1FB4 ; Recommended # 1.1 [3] GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI +1FEC ; Recommended # 1.1 GREEK CAPITAL LETTER RHO WITH DASIA +3005..3007 ; Recommended # 1.1 [3] IDEOGRAPHIC ITERATION MARK..IDEOGRAPHIC NUMBER ZERO +3041..3094 ; Recommended # 1.1 [84] HIRAGANA LETTER SMALL A..HIRAGANA LETTER VU +3095..3096 ; Recommended # 3.2 [2] HIRAGANA LETTER SMALL KA..HIRAGANA LETTER SMALL KE +309D..309E ; Recommended # 1.1 [2] HIRAGANA ITERATION MARK..HIRAGANA VOICED ITERATION MARK +30A1..30FA ; Recommended # 1.1 [90] KATAKANA LETTER SMALL A..KATAKANA LETTER VO +30FC..30FE ; Recommended # 1.1 [3] KATAKANA-HIRAGANA PROLONGED SOUND MARK..KATAKANA VOICED ITERATION MARK +3447 ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-3447 +3473 ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-3473 +34E4 ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-34E4 +3577 ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-3577 +359E ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-359E +35A1 ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-35A1 +35AD ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-35AD +35BF ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-35BF +35CE ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-35CE +35F3 ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-35F3 +35FE ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-35FE +360E ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-360E +361A ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-361A +3918 ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-3918 +3960 ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-3960 +396E ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-396E +39CF..39D0 ; Recommended # 3.0 [2] CJK UNIFIED IDEOGRAPH-39CF..CJK UNIFIED IDEOGRAPH-39D0 +39DB ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-39DB +39DF ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-39DF +39F8 ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-39F8 +39FE ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-39FE +3A18 ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-3A18 +3A52 ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-3A52 +3A5C ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-3A5C +3A67 ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-3A67 +3A73 ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-3A73 +3B39 ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-3B39 +3B4E ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-3B4E +3BA3 ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-3BA3 +3C6E ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-3C6E +3CE0 ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-3CE0 +3DE7 ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-3DE7 +3DEB ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-3DEB +3E74 ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-3E74 +3ED0 ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-3ED0 +4056 ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-4056 +4065 ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-4065 +406A ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-406A +40BB ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-40BB +40DF ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-40DF +4137 ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-4137 +415F ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-415F +4337 ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-4337 +43AC ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-43AC +43B1 ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-43B1 +43D3 ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-43D3 +43DD ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-43DD +4443 ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-4443 +44D6 ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-44D6 +44EA ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-44EA +4606 ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-4606 +464C ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-464C +4661 ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-4661 +4723 ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-4723 +4729 ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-4729 +477C ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-477C +478D ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-478D +47F4 ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-47F4 +4882 ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-4882 +4947 ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-4947 +497A ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-497A +497D ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-497D +4982..4983 ; Recommended # 3.0 [2] CJK UNIFIED IDEOGRAPH-4982..CJK UNIFIED IDEOGRAPH-4983 +4985..4986 ; Recommended # 3.0 [2] CJK UNIFIED IDEOGRAPH-4985..CJK UNIFIED IDEOGRAPH-4986 +499B ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-499B +499F ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-499F +49B6..49B7 ; Recommended # 3.0 [2] CJK UNIFIED IDEOGRAPH-49B6..CJK UNIFIED IDEOGRAPH-49B7 +4A12 ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-4A12 +4AB8 ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-4AB8 +4C77 ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-4C77 +4C7D ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-4C7D +4C81 ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-4C81 +4C85 ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-4C85 +4C9D..4CA3 ; Recommended # 3.0 [7] CJK UNIFIED IDEOGRAPH-4C9D..CJK UNIFIED IDEOGRAPH-4CA3 +4D13..4D19 ; Recommended # 3.0 [7] CJK UNIFIED IDEOGRAPH-4D13..CJK UNIFIED IDEOGRAPH-4D19 +4DAE ; Recommended # 3.0 CJK UNIFIED IDEOGRAPH-4DAE +4E00..4E11 ; Recommended # 1.1 [18] CJK UNIFIED IDEOGRAPH-4E00..CJK UNIFIED IDEOGRAPH-4E11 +4E13..4E28 ; Recommended # 1.1 [22] CJK UNIFIED IDEOGRAPH-4E13..CJK UNIFIED IDEOGRAPH-4E28 +4E2A..4E67 ; Recommended # 1.1 [62] CJK UNIFIED IDEOGRAPH-4E2A..CJK UNIFIED IDEOGRAPH-4E67 +4E69..4E78 ; Recommended # 1.1 [16] CJK UNIFIED IDEOGRAPH-4E69..CJK UNIFIED IDEOGRAPH-4E78 +4E7A..4E95 ; Recommended # 1.1 [28] CJK UNIFIED IDEOGRAPH-4E7A..CJK UNIFIED IDEOGRAPH-4E95 +4E97..4EA2 ; Recommended # 1.1 [12] CJK UNIFIED IDEOGRAPH-4E97..CJK UNIFIED IDEOGRAPH-4EA2 +4EA4..4EBB ; Recommended # 1.1 [24] CJK UNIFIED IDEOGRAPH-4EA4..CJK UNIFIED IDEOGRAPH-4EBB +4EBD..4ECB ; Recommended # 1.1 [15] CJK UNIFIED IDEOGRAPH-4EBD..CJK UNIFIED IDEOGRAPH-4ECB +4ECD..4EE6 ; Recommended # 1.1 [26] CJK UNIFIED IDEOGRAPH-4ECD..CJK UNIFIED IDEOGRAPH-4EE6 +4EE8..4EF7 ; Recommended # 1.1 [16] CJK UNIFIED IDEOGRAPH-4EE8..CJK UNIFIED IDEOGRAPH-4EF7 +4EFB ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-4EFB +4EFD ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-4EFD +4EFF..4F06 ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-4EFF..CJK UNIFIED IDEOGRAPH-4F06 +4F08..4F15 ; Recommended # 1.1 [14] CJK UNIFIED IDEOGRAPH-4F08..CJK UNIFIED IDEOGRAPH-4F15 +4F17..4F27 ; Recommended # 1.1 [17] CJK UNIFIED IDEOGRAPH-4F17..CJK UNIFIED IDEOGRAPH-4F27 +4F29..4F30 ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-4F29..CJK UNIFIED IDEOGRAPH-4F30 +4F32..4F34 ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-4F32..CJK UNIFIED IDEOGRAPH-4F34 +4F36 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-4F36 +4F38..4F3F ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-4F38..CJK UNIFIED IDEOGRAPH-4F3F +4F41..4F43 ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-4F41..CJK UNIFIED IDEOGRAPH-4F43 +4F45..4F70 ; Recommended # 1.1 [44] CJK UNIFIED IDEOGRAPH-4F45..CJK UNIFIED IDEOGRAPH-4F70 +4F72..4F8B ; Recommended # 1.1 [26] CJK UNIFIED IDEOGRAPH-4F72..CJK UNIFIED IDEOGRAPH-4F8B +4F8D ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-4F8D +4F8F..4FA1 ; Recommended # 1.1 [19] CJK UNIFIED IDEOGRAPH-4F8F..CJK UNIFIED IDEOGRAPH-4FA1 +4FA3..4FBC ; Recommended # 1.1 [26] CJK UNIFIED IDEOGRAPH-4FA3..CJK UNIFIED IDEOGRAPH-4FBC +4FBE..4FC5 ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-4FBE..CJK UNIFIED IDEOGRAPH-4FC5 +4FC7 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-4FC7 +4FC9..4FCB ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-4FC9..CJK UNIFIED IDEOGRAPH-4FCB +4FCD..4FE1 ; Recommended # 1.1 [21] CJK UNIFIED IDEOGRAPH-4FCD..CJK UNIFIED IDEOGRAPH-4FE1 +4FE3..4FFB ; Recommended # 1.1 [25] CJK UNIFIED IDEOGRAPH-4FE3..CJK UNIFIED IDEOGRAPH-4FFB +4FFE..500F ; Recommended # 1.1 [18] CJK UNIFIED IDEOGRAPH-4FFE..CJK UNIFIED IDEOGRAPH-500F +5011..5033 ; Recommended # 1.1 [35] CJK UNIFIED IDEOGRAPH-5011..CJK UNIFIED IDEOGRAPH-5033 +5035..5037 ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-5035..CJK UNIFIED IDEOGRAPH-5037 +5039..503C ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-5039..CJK UNIFIED IDEOGRAPH-503C +503E..5041 ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-503E..CJK UNIFIED IDEOGRAPH-5041 +5043..5051 ; Recommended # 1.1 [15] CJK UNIFIED IDEOGRAPH-5043..CJK UNIFIED IDEOGRAPH-5051 +5053..5057 ; Recommended # 1.1 [5] CJK UNIFIED IDEOGRAPH-5053..CJK UNIFIED IDEOGRAPH-5057 +5059..507B ; Recommended # 1.1 [35] CJK UNIFIED IDEOGRAPH-5059..CJK UNIFIED IDEOGRAPH-507B +507D..5080 ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-507D..CJK UNIFIED IDEOGRAPH-5080 +5082..5092 ; Recommended # 1.1 [17] CJK UNIFIED IDEOGRAPH-5082..CJK UNIFIED IDEOGRAPH-5092 +5094..5096 ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-5094..CJK UNIFIED IDEOGRAPH-5096 +5098..509E ; Recommended # 1.1 [7] CJK UNIFIED IDEOGRAPH-5098..CJK UNIFIED IDEOGRAPH-509E +50A2..50B8 ; Recommended # 1.1 [23] CJK UNIFIED IDEOGRAPH-50A2..CJK UNIFIED IDEOGRAPH-50B8 +50BA..50C2 ; Recommended # 1.1 [9] CJK UNIFIED IDEOGRAPH-50BA..CJK UNIFIED IDEOGRAPH-50C2 +50C4..50D7 ; Recommended # 1.1 [20] CJK UNIFIED IDEOGRAPH-50C4..CJK UNIFIED IDEOGRAPH-50D7 +50D9..50DE ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-50D9..CJK UNIFIED IDEOGRAPH-50DE +50E0 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-50E0 +50E3..50EA ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-50E3..CJK UNIFIED IDEOGRAPH-50EA +50EC..50F3 ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-50EC..CJK UNIFIED IDEOGRAPH-50F3 +50F5..50F6 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-50F5..CJK UNIFIED IDEOGRAPH-50F6 +50F8..511A ; Recommended # 1.1 [35] CJK UNIFIED IDEOGRAPH-50F8..CJK UNIFIED IDEOGRAPH-511A +511C..5127 ; Recommended # 1.1 [12] CJK UNIFIED IDEOGRAPH-511C..CJK UNIFIED IDEOGRAPH-5127 +5129..512A ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-5129..CJK UNIFIED IDEOGRAPH-512A +512C..5141 ; Recommended # 1.1 [22] CJK UNIFIED IDEOGRAPH-512C..CJK UNIFIED IDEOGRAPH-5141 +5143..5149 ; Recommended # 1.1 [7] CJK UNIFIED IDEOGRAPH-5143..CJK UNIFIED IDEOGRAPH-5149 +514B..514E ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-514B..CJK UNIFIED IDEOGRAPH-514E +5150..5152 ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-5150..CJK UNIFIED IDEOGRAPH-5152 +5154..5157 ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-5154..CJK UNIFIED IDEOGRAPH-5157 +5159..515F ; Recommended # 1.1 [7] CJK UNIFIED IDEOGRAPH-5159..CJK UNIFIED IDEOGRAPH-515F +5161..5163 ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-5161..CJK UNIFIED IDEOGRAPH-5163 +5165..5171 ; Recommended # 1.1 [13] CJK UNIFIED IDEOGRAPH-5165..CJK UNIFIED IDEOGRAPH-5171 +5173..517D ; Recommended # 1.1 [11] CJK UNIFIED IDEOGRAPH-5173..CJK UNIFIED IDEOGRAPH-517D +517F..5182 ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-517F..CJK UNIFIED IDEOGRAPH-5182 +5185..518D ; Recommended # 1.1 [9] CJK UNIFIED IDEOGRAPH-5185..CJK UNIFIED IDEOGRAPH-518D +518F..51A0 ; Recommended # 1.1 [18] CJK UNIFIED IDEOGRAPH-518F..CJK UNIFIED IDEOGRAPH-51A0 +51A2 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-51A2 +51A4..51AC ; Recommended # 1.1 [9] CJK UNIFIED IDEOGRAPH-51A4..CJK UNIFIED IDEOGRAPH-51AC +51AE..51B7 ; Recommended # 1.1 [10] CJK UNIFIED IDEOGRAPH-51AE..CJK UNIFIED IDEOGRAPH-51B7 +51B9 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-51B9 +51BB..51C1 ; Recommended # 1.1 [7] CJK UNIFIED IDEOGRAPH-51BB..CJK UNIFIED IDEOGRAPH-51C1 +51C3..51D1 ; Recommended # 1.1 [15] CJK UNIFIED IDEOGRAPH-51C3..CJK UNIFIED IDEOGRAPH-51D1 +51D4..51DE ; Recommended # 1.1 [11] CJK UNIFIED IDEOGRAPH-51D4..CJK UNIFIED IDEOGRAPH-51DE +51E0..51EB ; Recommended # 1.1 [12] CJK UNIFIED IDEOGRAPH-51E0..CJK UNIFIED IDEOGRAPH-51EB +51ED ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-51ED +51EF..51F1 ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-51EF..CJK UNIFIED IDEOGRAPH-51F1 +51F3..5252 ; Recommended # 1.1 [96] CJK UNIFIED IDEOGRAPH-51F3..CJK UNIFIED IDEOGRAPH-5252 +5254..5265 ; Recommended # 1.1 [18] CJK UNIFIED IDEOGRAPH-5254..CJK UNIFIED IDEOGRAPH-5265 +5267..5278 ; Recommended # 1.1 [18] CJK UNIFIED IDEOGRAPH-5267..CJK UNIFIED IDEOGRAPH-5278 +527A..5284 ; Recommended # 1.1 [11] CJK UNIFIED IDEOGRAPH-527A..CJK UNIFIED IDEOGRAPH-5284 +5286..528D ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-5286..CJK UNIFIED IDEOGRAPH-528D +528F..52C3 ; Recommended # 1.1 [53] CJK UNIFIED IDEOGRAPH-528F..CJK UNIFIED IDEOGRAPH-52C3 +52C5..52C7 ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-52C5..CJK UNIFIED IDEOGRAPH-52C7 +52C9..52CB ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-52C9..CJK UNIFIED IDEOGRAPH-52CB +52CD ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-52CD +52CF..52D0 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-52CF..CJK UNIFIED IDEOGRAPH-52D0 +52D2..52D3 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-52D2..CJK UNIFIED IDEOGRAPH-52D3 +52D5..52E0 ; Recommended # 1.1 [12] CJK UNIFIED IDEOGRAPH-52D5..CJK UNIFIED IDEOGRAPH-52E0 +52E2..52E4 ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-52E2..CJK UNIFIED IDEOGRAPH-52E4 +52E6..52ED ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-52E6..CJK UNIFIED IDEOGRAPH-52ED +52EF..5302 ; Recommended # 1.1 [20] CJK UNIFIED IDEOGRAPH-52EF..CJK UNIFIED IDEOGRAPH-5302 +5305..5317 ; Recommended # 1.1 [19] CJK UNIFIED IDEOGRAPH-5305..CJK UNIFIED IDEOGRAPH-5317 +5319..531A ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-5319..CJK UNIFIED IDEOGRAPH-531A +531C..531D ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-531C..CJK UNIFIED IDEOGRAPH-531D +531F..5326 ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-531F..CJK UNIFIED IDEOGRAPH-5326 +5328 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-5328 +532A..5331 ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-532A..CJK UNIFIED IDEOGRAPH-5331 +5333..5334 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-5333..CJK UNIFIED IDEOGRAPH-5334 +5337..5341 ; Recommended # 1.1 [11] CJK UNIFIED IDEOGRAPH-5337..CJK UNIFIED IDEOGRAPH-5341 +5343..535A ; Recommended # 1.1 [24] CJK UNIFIED IDEOGRAPH-5343..CJK UNIFIED IDEOGRAPH-535A +535C ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-535C +535E..5369 ; Recommended # 1.1 [12] CJK UNIFIED IDEOGRAPH-535E..CJK UNIFIED IDEOGRAPH-5369 +536B..536C ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-536B..CJK UNIFIED IDEOGRAPH-536C +536E..537F ; Recommended # 1.1 [18] CJK UNIFIED IDEOGRAPH-536E..CJK UNIFIED IDEOGRAPH-537F +5381..53A0 ; Recommended # 1.1 [32] CJK UNIFIED IDEOGRAPH-5381..CJK UNIFIED IDEOGRAPH-53A0 +53A2..53A9 ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-53A2..CJK UNIFIED IDEOGRAPH-53A9 +53AC..53AE ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-53AC..CJK UNIFIED IDEOGRAPH-53AE +53B0..53B9 ; Recommended # 1.1 [10] CJK UNIFIED IDEOGRAPH-53B0..CJK UNIFIED IDEOGRAPH-53B9 +53BB..53C4 ; Recommended # 1.1 [10] CJK UNIFIED IDEOGRAPH-53BB..CJK UNIFIED IDEOGRAPH-53C4 +53C6..53CE ; Recommended # 1.1 [9] CJK UNIFIED IDEOGRAPH-53C6..CJK UNIFIED IDEOGRAPH-53CE +53D0..53DC ; Recommended # 1.1 [13] CJK UNIFIED IDEOGRAPH-53D0..CJK UNIFIED IDEOGRAPH-53DC +53DF..53E6 ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-53DF..CJK UNIFIED IDEOGRAPH-53E6 +53E8..53FE ; Recommended # 1.1 [23] CJK UNIFIED IDEOGRAPH-53E8..CJK UNIFIED IDEOGRAPH-53FE +5401..5419 ; Recommended # 1.1 [25] CJK UNIFIED IDEOGRAPH-5401..CJK UNIFIED IDEOGRAPH-5419 +541B..5421 ; Recommended # 1.1 [7] CJK UNIFIED IDEOGRAPH-541B..CJK UNIFIED IDEOGRAPH-5421 +5423..544B ; Recommended # 1.1 [41] CJK UNIFIED IDEOGRAPH-5423..CJK UNIFIED IDEOGRAPH-544B +544D..545C ; Recommended # 1.1 [16] CJK UNIFIED IDEOGRAPH-544D..CJK UNIFIED IDEOGRAPH-545C +545E..5468 ; Recommended # 1.1 [11] CJK UNIFIED IDEOGRAPH-545E..CJK UNIFIED IDEOGRAPH-5468 +546A..5489 ; Recommended # 1.1 [32] CJK UNIFIED IDEOGRAPH-546A..CJK UNIFIED IDEOGRAPH-5489 +548B..54B4 ; Recommended # 1.1 [42] CJK UNIFIED IDEOGRAPH-548B..CJK UNIFIED IDEOGRAPH-54B4 +54B6..54F5 ; Recommended # 1.1 [64] CJK UNIFIED IDEOGRAPH-54B6..CJK UNIFIED IDEOGRAPH-54F5 +54F7..5514 ; Recommended # 1.1 [30] CJK UNIFIED IDEOGRAPH-54F7..CJK UNIFIED IDEOGRAPH-5514 +5516..5517 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-5516..CJK UNIFIED IDEOGRAPH-5517 +551A..5546 ; Recommended # 1.1 [45] CJK UNIFIED IDEOGRAPH-551A..CJK UNIFIED IDEOGRAPH-5546 +5548..555F ; Recommended # 1.1 [24] CJK UNIFIED IDEOGRAPH-5548..CJK UNIFIED IDEOGRAPH-555F +5561..5579 ; Recommended # 1.1 [25] CJK UNIFIED IDEOGRAPH-5561..CJK UNIFIED IDEOGRAPH-5579 +557B..55DF ; Recommended # 1.1 [101] CJK UNIFIED IDEOGRAPH-557B..CJK UNIFIED IDEOGRAPH-55DF +55E1..55F7 ; Recommended # 1.1 [23] CJK UNIFIED IDEOGRAPH-55E1..CJK UNIFIED IDEOGRAPH-55F7 +55F9..5609 ; Recommended # 1.1 [17] CJK UNIFIED IDEOGRAPH-55F9..CJK UNIFIED IDEOGRAPH-5609 +560C..561F ; Recommended # 1.1 [20] CJK UNIFIED IDEOGRAPH-560C..CJK UNIFIED IDEOGRAPH-561F +5621..562A ; Recommended # 1.1 [10] CJK UNIFIED IDEOGRAPH-5621..CJK UNIFIED IDEOGRAPH-562A +562C..5636 ; Recommended # 1.1 [11] CJK UNIFIED IDEOGRAPH-562C..CJK UNIFIED IDEOGRAPH-5636 +5638..563B ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-5638..CJK UNIFIED IDEOGRAPH-563B +563D..5643 ; Recommended # 1.1 [7] CJK UNIFIED IDEOGRAPH-563D..CJK UNIFIED IDEOGRAPH-5643 +5645..564A ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-5645..CJK UNIFIED IDEOGRAPH-564A +564C..5650 ; Recommended # 1.1 [5] CJK UNIFIED IDEOGRAPH-564C..CJK UNIFIED IDEOGRAPH-5650 +5652..5655 ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-5652..CJK UNIFIED IDEOGRAPH-5655 +5657..565E ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-5657..CJK UNIFIED IDEOGRAPH-565E +5660 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-5660 +5662..5674 ; Recommended # 1.1 [19] CJK UNIFIED IDEOGRAPH-5662..CJK UNIFIED IDEOGRAPH-5674 +5676..567C ; Recommended # 1.1 [7] CJK UNIFIED IDEOGRAPH-5676..CJK UNIFIED IDEOGRAPH-567C +567E..5687 ; Recommended # 1.1 [10] CJK UNIFIED IDEOGRAPH-567E..CJK UNIFIED IDEOGRAPH-5687 +5689..568A ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-5689..CJK UNIFIED IDEOGRAPH-568A +568C..5695 ; Recommended # 1.1 [10] CJK UNIFIED IDEOGRAPH-568C..CJK UNIFIED IDEOGRAPH-5695 +5697..569D ; Recommended # 1.1 [7] CJK UNIFIED IDEOGRAPH-5697..CJK UNIFIED IDEOGRAPH-569D +569F..56B9 ; Recommended # 1.1 [27] CJK UNIFIED IDEOGRAPH-569F..CJK UNIFIED IDEOGRAPH-56B9 +56BB..56CE ; Recommended # 1.1 [20] CJK UNIFIED IDEOGRAPH-56BB..CJK UNIFIED IDEOGRAPH-56CE +56D0..56D8 ; Recommended # 1.1 [9] CJK UNIFIED IDEOGRAPH-56D0..CJK UNIFIED IDEOGRAPH-56D8 +56DA..56E5 ; Recommended # 1.1 [12] CJK UNIFIED IDEOGRAPH-56DA..CJK UNIFIED IDEOGRAPH-56E5 +56E7..56F5 ; Recommended # 1.1 [15] CJK UNIFIED IDEOGRAPH-56E7..CJK UNIFIED IDEOGRAPH-56F5 +56F7 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-56F7 +56F9..56FA ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-56F9..CJK UNIFIED IDEOGRAPH-56FA +56FD..5704 ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-56FD..CJK UNIFIED IDEOGRAPH-5704 +5706..5710 ; Recommended # 1.1 [11] CJK UNIFIED IDEOGRAPH-5706..CJK UNIFIED IDEOGRAPH-5710 +5712..5716 ; Recommended # 1.1 [5] CJK UNIFIED IDEOGRAPH-5712..CJK UNIFIED IDEOGRAPH-5716 +5718..5720 ; Recommended # 1.1 [9] CJK UNIFIED IDEOGRAPH-5718..CJK UNIFIED IDEOGRAPH-5720 +5722..5723 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-5722..CJK UNIFIED IDEOGRAPH-5723 +5725..573C ; Recommended # 1.1 [24] CJK UNIFIED IDEOGRAPH-5725..CJK UNIFIED IDEOGRAPH-573C +573E..5742 ; Recommended # 1.1 [5] CJK UNIFIED IDEOGRAPH-573E..CJK UNIFIED IDEOGRAPH-5742 +5744..5747 ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-5744..CJK UNIFIED IDEOGRAPH-5747 +5749..5754 ; Recommended # 1.1 [12] CJK UNIFIED IDEOGRAPH-5749..CJK UNIFIED IDEOGRAPH-5754 +5757 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-5757 +5759..5762 ; Recommended # 1.1 [10] CJK UNIFIED IDEOGRAPH-5759..CJK UNIFIED IDEOGRAPH-5762 +5764..5777 ; Recommended # 1.1 [20] CJK UNIFIED IDEOGRAPH-5764..CJK UNIFIED IDEOGRAPH-5777 +5779..5780 ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-5779..CJK UNIFIED IDEOGRAPH-5780 +5782..5786 ; Recommended # 1.1 [5] CJK UNIFIED IDEOGRAPH-5782..CJK UNIFIED IDEOGRAPH-5786 +5788..5795 ; Recommended # 1.1 [14] CJK UNIFIED IDEOGRAPH-5788..CJK UNIFIED IDEOGRAPH-5795 +5797..57A7 ; Recommended # 1.1 [17] CJK UNIFIED IDEOGRAPH-5797..CJK UNIFIED IDEOGRAPH-57A7 +57A9..57C9 ; Recommended # 1.1 [33] CJK UNIFIED IDEOGRAPH-57A9..CJK UNIFIED IDEOGRAPH-57C9 +57CB..57D0 ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-57CB..CJK UNIFIED IDEOGRAPH-57D0 +57D2..57DA ; Recommended # 1.1 [9] CJK UNIFIED IDEOGRAPH-57D2..CJK UNIFIED IDEOGRAPH-57DA +57DC..5816 ; Recommended # 1.1 [59] CJK UNIFIED IDEOGRAPH-57DC..CJK UNIFIED IDEOGRAPH-5816 +5819..584F ; Recommended # 1.1 [55] CJK UNIFIED IDEOGRAPH-5819..CJK UNIFIED IDEOGRAPH-584F +5851..5855 ; Recommended # 1.1 [5] CJK UNIFIED IDEOGRAPH-5851..CJK UNIFIED IDEOGRAPH-5855 +5857..585F ; Recommended # 1.1 [9] CJK UNIFIED IDEOGRAPH-5857..CJK UNIFIED IDEOGRAPH-585F +5861..5865 ; Recommended # 1.1 [5] CJK UNIFIED IDEOGRAPH-5861..CJK UNIFIED IDEOGRAPH-5865 +5868..5876 ; Recommended # 1.1 [15] CJK UNIFIED IDEOGRAPH-5868..CJK UNIFIED IDEOGRAPH-5876 +5878..5894 ; Recommended # 1.1 [29] CJK UNIFIED IDEOGRAPH-5878..CJK UNIFIED IDEOGRAPH-5894 +5896..58A9 ; Recommended # 1.1 [20] CJK UNIFIED IDEOGRAPH-5896..CJK UNIFIED IDEOGRAPH-58A9 +58AB..58B5 ; Recommended # 1.1 [11] CJK UNIFIED IDEOGRAPH-58AB..CJK UNIFIED IDEOGRAPH-58B5 +58B7..58BF ; Recommended # 1.1 [9] CJK UNIFIED IDEOGRAPH-58B7..CJK UNIFIED IDEOGRAPH-58BF +58C1..58C2 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-58C1..CJK UNIFIED IDEOGRAPH-58C2 +58C5..58CC ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-58C5..CJK UNIFIED IDEOGRAPH-58CC +58CE..58CF ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-58CE..CJK UNIFIED IDEOGRAPH-58CF +58D1..58E0 ; Recommended # 1.1 [16] CJK UNIFIED IDEOGRAPH-58D1..CJK UNIFIED IDEOGRAPH-58E0 +58E2..58E5 ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-58E2..CJK UNIFIED IDEOGRAPH-58E5 +58E7..58F4 ; Recommended # 1.1 [14] CJK UNIFIED IDEOGRAPH-58E7..CJK UNIFIED IDEOGRAPH-58F4 +58F6..5900 ; Recommended # 1.1 [11] CJK UNIFIED IDEOGRAPH-58F6..CJK UNIFIED IDEOGRAPH-5900 +5902..5904 ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-5902..CJK UNIFIED IDEOGRAPH-5904 +5906..5907 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-5906..CJK UNIFIED IDEOGRAPH-5907 +5909..5910 ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-5909..CJK UNIFIED IDEOGRAPH-5910 +5912 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-5912 +5914..5922 ; Recommended # 1.1 [15] CJK UNIFIED IDEOGRAPH-5914..CJK UNIFIED IDEOGRAPH-5922 +5924..5932 ; Recommended # 1.1 [15] CJK UNIFIED IDEOGRAPH-5924..CJK UNIFIED IDEOGRAPH-5932 +5934..5935 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-5934..CJK UNIFIED IDEOGRAPH-5935 +5937..5958 ; Recommended # 1.1 [34] CJK UNIFIED IDEOGRAPH-5937..CJK UNIFIED IDEOGRAPH-5958 +595A ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-595A +595C..59B6 ; Recommended # 1.1 [91] CJK UNIFIED IDEOGRAPH-595C..CJK UNIFIED IDEOGRAPH-59B6 +59B8..59E6 ; Recommended # 1.1 [47] CJK UNIFIED IDEOGRAPH-59B8..CJK UNIFIED IDEOGRAPH-59E6 +59E8..5A23 ; Recommended # 1.1 [60] CJK UNIFIED IDEOGRAPH-59E8..CJK UNIFIED IDEOGRAPH-5A23 +5A25 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-5A25 +5A27..5A2B ; Recommended # 1.1 [5] CJK UNIFIED IDEOGRAPH-5A27..CJK UNIFIED IDEOGRAPH-5A2B +5A2D..5A2F ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-5A2D..CJK UNIFIED IDEOGRAPH-5A2F +5A31..5A53 ; Recommended # 1.1 [35] CJK UNIFIED IDEOGRAPH-5A31..CJK UNIFIED IDEOGRAPH-5A53 +5A55..5A58 ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-5A55..CJK UNIFIED IDEOGRAPH-5A58 +5A5A..5A6E ; Recommended # 1.1 [21] CJK UNIFIED IDEOGRAPH-5A5A..CJK UNIFIED IDEOGRAPH-5A6E +5A70 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-5A70 +5A72..5A86 ; Recommended # 1.1 [21] CJK UNIFIED IDEOGRAPH-5A72..CJK UNIFIED IDEOGRAPH-5A86 +5A88..5A8C ; Recommended # 1.1 [5] CJK UNIFIED IDEOGRAPH-5A88..CJK UNIFIED IDEOGRAPH-5A8C +5A8E..5AAA ; Recommended # 1.1 [29] CJK UNIFIED IDEOGRAPH-5A8E..CJK UNIFIED IDEOGRAPH-5AAA +5AAC..5AD2 ; Recommended # 1.1 [39] CJK UNIFIED IDEOGRAPH-5AAC..CJK UNIFIED IDEOGRAPH-5AD2 +5AD4..5AEE ; Recommended # 1.1 [27] CJK UNIFIED IDEOGRAPH-5AD4..CJK UNIFIED IDEOGRAPH-5AEE +5AF1..5B09 ; Recommended # 1.1 [25] CJK UNIFIED IDEOGRAPH-5AF1..CJK UNIFIED IDEOGRAPH-5B09 +5B0B..5B0C ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-5B0B..CJK UNIFIED IDEOGRAPH-5B0C +5B0E..5B38 ; Recommended # 1.1 [43] CJK UNIFIED IDEOGRAPH-5B0E..CJK UNIFIED IDEOGRAPH-5B38 +5B3A..5B45 ; Recommended # 1.1 [12] CJK UNIFIED IDEOGRAPH-5B3A..CJK UNIFIED IDEOGRAPH-5B45 +5B47..5B4E ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-5B47..CJK UNIFIED IDEOGRAPH-5B4E +5B50..5B51 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-5B50..CJK UNIFIED IDEOGRAPH-5B51 +5B53..5B5F ; Recommended # 1.1 [13] CJK UNIFIED IDEOGRAPH-5B53..CJK UNIFIED IDEOGRAPH-5B5F +5B62..5B6E ; Recommended # 1.1 [13] CJK UNIFIED IDEOGRAPH-5B62..CJK UNIFIED IDEOGRAPH-5B6E +5B70..5B78 ; Recommended # 1.1 [9] CJK UNIFIED IDEOGRAPH-5B70..CJK UNIFIED IDEOGRAPH-5B78 +5B7A..5B7D ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-5B7A..CJK UNIFIED IDEOGRAPH-5B7D +5B7F..5B85 ; Recommended # 1.1 [7] CJK UNIFIED IDEOGRAPH-5B7F..CJK UNIFIED IDEOGRAPH-5B85 +5B87..5B8F ; Recommended # 1.1 [9] CJK UNIFIED IDEOGRAPH-5B87..CJK UNIFIED IDEOGRAPH-5B8F +5B91..5BA8 ; Recommended # 1.1 [24] CJK UNIFIED IDEOGRAPH-5B91..CJK UNIFIED IDEOGRAPH-5BA8 +5BAA..5BB1 ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-5BAA..CJK UNIFIED IDEOGRAPH-5BB1 +5BB3..5BB6 ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-5BB3..CJK UNIFIED IDEOGRAPH-5BB6 +5BB8..5BBB ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-5BB8..CJK UNIFIED IDEOGRAPH-5BBB +5BBD..5BC7 ; Recommended # 1.1 [11] CJK UNIFIED IDEOGRAPH-5BBD..CJK UNIFIED IDEOGRAPH-5BC7 +5BC9..5BD9 ; Recommended # 1.1 [17] CJK UNIFIED IDEOGRAPH-5BC9..CJK UNIFIED IDEOGRAPH-5BD9 +5BDB..5BFF ; Recommended # 1.1 [37] CJK UNIFIED IDEOGRAPH-5BDB..CJK UNIFIED IDEOGRAPH-5BFF +5C01..5C1A ; Recommended # 1.1 [26] CJK UNIFIED IDEOGRAPH-5C01..CJK UNIFIED IDEOGRAPH-5C1A +5C1C..5C22 ; Recommended # 1.1 [7] CJK UNIFIED IDEOGRAPH-5C1C..CJK UNIFIED IDEOGRAPH-5C22 +5C24..5C25 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-5C24..CJK UNIFIED IDEOGRAPH-5C25 +5C27..5C28 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-5C27..CJK UNIFIED IDEOGRAPH-5C28 +5C2A..5C35 ; Recommended # 1.1 [12] CJK UNIFIED IDEOGRAPH-5C2A..CJK UNIFIED IDEOGRAPH-5C35 +5C37..5C59 ; Recommended # 1.1 [35] CJK UNIFIED IDEOGRAPH-5C37..CJK UNIFIED IDEOGRAPH-5C59 +5C5B..5C84 ; Recommended # 1.1 [42] CJK UNIFIED IDEOGRAPH-5C5B..CJK UNIFIED IDEOGRAPH-5C84 +5C86..5CB3 ; Recommended # 1.1 [46] CJK UNIFIED IDEOGRAPH-5C86..CJK UNIFIED IDEOGRAPH-5CB3 +5CB5..5CB8 ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-5CB5..CJK UNIFIED IDEOGRAPH-5CB8 +5CBA..5CD4 ; Recommended # 1.1 [27] CJK UNIFIED IDEOGRAPH-5CBA..CJK UNIFIED IDEOGRAPH-5CD4 +5CD6..5CDC ; Recommended # 1.1 [7] CJK UNIFIED IDEOGRAPH-5CD6..CJK UNIFIED IDEOGRAPH-5CDC +5CDE..5CF4 ; Recommended # 1.1 [23] CJK UNIFIED IDEOGRAPH-5CDE..CJK UNIFIED IDEOGRAPH-5CF4 +5CF6..5D2A ; Recommended # 1.1 [53] CJK UNIFIED IDEOGRAPH-5CF6..CJK UNIFIED IDEOGRAPH-5D2A +5D2C..5D2E ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-5D2C..CJK UNIFIED IDEOGRAPH-5D2E +5D30..5D3A ; Recommended # 1.1 [11] CJK UNIFIED IDEOGRAPH-5D30..CJK UNIFIED IDEOGRAPH-5D3A +5D3C..5D52 ; Recommended # 1.1 [23] CJK UNIFIED IDEOGRAPH-5D3C..CJK UNIFIED IDEOGRAPH-5D52 +5D54..5D56 ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-5D54..CJK UNIFIED IDEOGRAPH-5D56 +5D58..5D5F ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-5D58..CJK UNIFIED IDEOGRAPH-5D5F +5D61..5D82 ; Recommended # 1.1 [34] CJK UNIFIED IDEOGRAPH-5D61..CJK UNIFIED IDEOGRAPH-5D82 +5D84..5D95 ; Recommended # 1.1 [18] CJK UNIFIED IDEOGRAPH-5D84..CJK UNIFIED IDEOGRAPH-5D95 +5D97..5DA2 ; Recommended # 1.1 [12] CJK UNIFIED IDEOGRAPH-5D97..CJK UNIFIED IDEOGRAPH-5DA2 +5DA5..5DAA ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-5DA5..CJK UNIFIED IDEOGRAPH-5DAA +5DAC..5DB2 ; Recommended # 1.1 [7] CJK UNIFIED IDEOGRAPH-5DAC..CJK UNIFIED IDEOGRAPH-5DB2 +5DB4..5DB8 ; Recommended # 1.1 [5] CJK UNIFIED IDEOGRAPH-5DB4..CJK UNIFIED IDEOGRAPH-5DB8 +5DBA..5DC3 ; Recommended # 1.1 [10] CJK UNIFIED IDEOGRAPH-5DBA..CJK UNIFIED IDEOGRAPH-5DC3 +5DC5..5DD6 ; Recommended # 1.1 [18] CJK UNIFIED IDEOGRAPH-5DC5..CJK UNIFIED IDEOGRAPH-5DD6 +5DD8..5DD9 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-5DD8..CJK UNIFIED IDEOGRAPH-5DD9 +5DDB ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-5DDB +5DDD..5DF5 ; Recommended # 1.1 [25] CJK UNIFIED IDEOGRAPH-5DDD..CJK UNIFIED IDEOGRAPH-5DF5 +5DF7..5E11 ; Recommended # 1.1 [27] CJK UNIFIED IDEOGRAPH-5DF7..CJK UNIFIED IDEOGRAPH-5E11 +5E13..5E47 ; Recommended # 1.1 [53] CJK UNIFIED IDEOGRAPH-5E13..CJK UNIFIED IDEOGRAPH-5E47 +5E49..5E50 ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-5E49..CJK UNIFIED IDEOGRAPH-5E50 +5E52..5E91 ; Recommended # 1.1 [64] CJK UNIFIED IDEOGRAPH-5E52..CJK UNIFIED IDEOGRAPH-5E91 +5E93..5EB9 ; Recommended # 1.1 [39] CJK UNIFIED IDEOGRAPH-5E93..CJK UNIFIED IDEOGRAPH-5EB9 +5EBB..5EBF ; Recommended # 1.1 [5] CJK UNIFIED IDEOGRAPH-5EBB..CJK UNIFIED IDEOGRAPH-5EBF +5EC1..5EEA ; Recommended # 1.1 [42] CJK UNIFIED IDEOGRAPH-5EC1..CJK UNIFIED IDEOGRAPH-5EEA +5EEC..5EF8 ; Recommended # 1.1 [13] CJK UNIFIED IDEOGRAPH-5EEC..CJK UNIFIED IDEOGRAPH-5EF8 +5EFA..5F0D ; Recommended # 1.1 [20] CJK UNIFIED IDEOGRAPH-5EFA..CJK UNIFIED IDEOGRAPH-5F0D +5F0F..5F3A ; Recommended # 1.1 [44] CJK UNIFIED IDEOGRAPH-5F0F..CJK UNIFIED IDEOGRAPH-5F3A +5F3C ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-5F3C +5F3E..5F8E ; Recommended # 1.1 [81] CJK UNIFIED IDEOGRAPH-5F3E..CJK UNIFIED IDEOGRAPH-5F8E +5F90..5F99 ; Recommended # 1.1 [10] CJK UNIFIED IDEOGRAPH-5F90..CJK UNIFIED IDEOGRAPH-5F99 +5F9B..5FA2 ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-5F9B..CJK UNIFIED IDEOGRAPH-5FA2 +5FA5..5FAF ; Recommended # 1.1 [11] CJK UNIFIED IDEOGRAPH-5FA5..CJK UNIFIED IDEOGRAPH-5FAF +5FB1..5FC1 ; Recommended # 1.1 [17] CJK UNIFIED IDEOGRAPH-5FB1..CJK UNIFIED IDEOGRAPH-5FC1 +5FC3..5FCD ; Recommended # 1.1 [11] CJK UNIFIED IDEOGRAPH-5FC3..CJK UNIFIED IDEOGRAPH-5FCD +5FCF..5FDA ; Recommended # 1.1 [12] CJK UNIFIED IDEOGRAPH-5FCF..CJK UNIFIED IDEOGRAPH-5FDA +5FDC..5FE1 ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-5FDC..CJK UNIFIED IDEOGRAPH-5FE1 +5FE3..5FEB ; Recommended # 1.1 [9] CJK UNIFIED IDEOGRAPH-5FE3..CJK UNIFIED IDEOGRAPH-5FEB +5FED..5FFB ; Recommended # 1.1 [15] CJK UNIFIED IDEOGRAPH-5FED..CJK UNIFIED IDEOGRAPH-5FFB +5FFD..6022 ; Recommended # 1.1 [38] CJK UNIFIED IDEOGRAPH-5FFD..CJK UNIFIED IDEOGRAPH-6022 +6024..6055 ; Recommended # 1.1 [50] CJK UNIFIED IDEOGRAPH-6024..CJK UNIFIED IDEOGRAPH-6055 +6057..6060 ; Recommended # 1.1 [10] CJK UNIFIED IDEOGRAPH-6057..CJK UNIFIED IDEOGRAPH-6060 +6062..6070 ; Recommended # 1.1 [15] CJK UNIFIED IDEOGRAPH-6062..CJK UNIFIED IDEOGRAPH-6070 +6072..6073 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-6072..CJK UNIFIED IDEOGRAPH-6073 +6075..6090 ; Recommended # 1.1 [28] CJK UNIFIED IDEOGRAPH-6075..CJK UNIFIED IDEOGRAPH-6090 +6092 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-6092 +6094..60A4 ; Recommended # 1.1 [17] CJK UNIFIED IDEOGRAPH-6094..CJK UNIFIED IDEOGRAPH-60A4 +60A6..60D1 ; Recommended # 1.1 [44] CJK UNIFIED IDEOGRAPH-60A6..CJK UNIFIED IDEOGRAPH-60D1 +60D3..60D5 ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-60D3..CJK UNIFIED IDEOGRAPH-60D5 +60D7..60DD ; Recommended # 1.1 [7] CJK UNIFIED IDEOGRAPH-60D7..CJK UNIFIED IDEOGRAPH-60DD +60DF..60E4 ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-60DF..CJK UNIFIED IDEOGRAPH-60E4 +60E6..60FC ; Recommended # 1.1 [23] CJK UNIFIED IDEOGRAPH-60E6..CJK UNIFIED IDEOGRAPH-60FC +60FE..6101 ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-60FE..CJK UNIFIED IDEOGRAPH-6101 +6103..6106 ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-6103..CJK UNIFIED IDEOGRAPH-6106 +6108..6110 ; Recommended # 1.1 [9] CJK UNIFIED IDEOGRAPH-6108..CJK UNIFIED IDEOGRAPH-6110 +6112..611D ; Recommended # 1.1 [12] CJK UNIFIED IDEOGRAPH-6112..CJK UNIFIED IDEOGRAPH-611D +611F..6130 ; Recommended # 1.1 [18] CJK UNIFIED IDEOGRAPH-611F..CJK UNIFIED IDEOGRAPH-6130 +6132 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-6132 +6134 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-6134 +6136..6137 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-6136..CJK UNIFIED IDEOGRAPH-6137 +613A..615F ; Recommended # 1.1 [38] CJK UNIFIED IDEOGRAPH-613A..CJK UNIFIED IDEOGRAPH-615F +6161..617A ; Recommended # 1.1 [26] CJK UNIFIED IDEOGRAPH-6161..CJK UNIFIED IDEOGRAPH-617A +617C..617E ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-617C..CJK UNIFIED IDEOGRAPH-617E +6180..6185 ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-6180..CJK UNIFIED IDEOGRAPH-6185 +6187..6196 ; Recommended # 1.1 [16] CJK UNIFIED IDEOGRAPH-6187..CJK UNIFIED IDEOGRAPH-6196 +6198..619B ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-6198..CJK UNIFIED IDEOGRAPH-619B +619D..61B8 ; Recommended # 1.1 [28] CJK UNIFIED IDEOGRAPH-619D..CJK UNIFIED IDEOGRAPH-61B8 +61BA ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-61BA +61BC..61D2 ; Recommended # 1.1 [23] CJK UNIFIED IDEOGRAPH-61BC..CJK UNIFIED IDEOGRAPH-61D2 +61D4 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-61D4 +61D6..61EB ; Recommended # 1.1 [22] CJK UNIFIED IDEOGRAPH-61D6..CJK UNIFIED IDEOGRAPH-61EB +61ED..61EE ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-61ED..CJK UNIFIED IDEOGRAPH-61EE +61F0..6204 ; Recommended # 1.1 [21] CJK UNIFIED IDEOGRAPH-61F0..CJK UNIFIED IDEOGRAPH-6204 +6206..6234 ; Recommended # 1.1 [47] CJK UNIFIED IDEOGRAPH-6206..CJK UNIFIED IDEOGRAPH-6234 +6236..6238 ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-6236..CJK UNIFIED IDEOGRAPH-6238 +623A..6256 ; Recommended # 1.1 [29] CJK UNIFIED IDEOGRAPH-623A..CJK UNIFIED IDEOGRAPH-6256 +6258..628C ; Recommended # 1.1 [53] CJK UNIFIED IDEOGRAPH-6258..CJK UNIFIED IDEOGRAPH-628C +628E..629C ; Recommended # 1.1 [15] CJK UNIFIED IDEOGRAPH-628E..CJK UNIFIED IDEOGRAPH-629C +629E..62DD ; Recommended # 1.1 [64] CJK UNIFIED IDEOGRAPH-629E..CJK UNIFIED IDEOGRAPH-62DD +62DF..62E9 ; Recommended # 1.1 [11] CJK UNIFIED IDEOGRAPH-62DF..CJK UNIFIED IDEOGRAPH-62E9 +62EB..6309 ; Recommended # 1.1 [31] CJK UNIFIED IDEOGRAPH-62EB..CJK UNIFIED IDEOGRAPH-6309 +630B..6316 ; Recommended # 1.1 [12] CJK UNIFIED IDEOGRAPH-630B..CJK UNIFIED IDEOGRAPH-6316 +6318..6330 ; Recommended # 1.1 [25] CJK UNIFIED IDEOGRAPH-6318..CJK UNIFIED IDEOGRAPH-6330 +6332..6336 ; Recommended # 1.1 [5] CJK UNIFIED IDEOGRAPH-6332..CJK UNIFIED IDEOGRAPH-6336 +6338..635A ; Recommended # 1.1 [35] CJK UNIFIED IDEOGRAPH-6338..CJK UNIFIED IDEOGRAPH-635A +635C..638A ; Recommended # 1.1 [47] CJK UNIFIED IDEOGRAPH-635C..CJK UNIFIED IDEOGRAPH-638A +638C..6392 ; Recommended # 1.1 [7] CJK UNIFIED IDEOGRAPH-638C..CJK UNIFIED IDEOGRAPH-6392 +6394..63D0 ; Recommended # 1.1 [61] CJK UNIFIED IDEOGRAPH-6394..CJK UNIFIED IDEOGRAPH-63D0 +63D2..643A ; Recommended # 1.1 [105] CJK UNIFIED IDEOGRAPH-63D2..CJK UNIFIED IDEOGRAPH-643A +643D..6448 ; Recommended # 1.1 [12] CJK UNIFIED IDEOGRAPH-643D..CJK UNIFIED IDEOGRAPH-6448 +644A..6459 ; Recommended # 1.1 [16] CJK UNIFIED IDEOGRAPH-644A..CJK UNIFIED IDEOGRAPH-6459 +645B..647D ; Recommended # 1.1 [35] CJK UNIFIED IDEOGRAPH-645B..CJK UNIFIED IDEOGRAPH-647D +647F..6485 ; Recommended # 1.1 [7] CJK UNIFIED IDEOGRAPH-647F..CJK UNIFIED IDEOGRAPH-6485 +6487..64A0 ; Recommended # 1.1 [26] CJK UNIFIED IDEOGRAPH-6487..CJK UNIFIED IDEOGRAPH-64A0 +64A2..64AE ; Recommended # 1.1 [13] CJK UNIFIED IDEOGRAPH-64A2..CJK UNIFIED IDEOGRAPH-64AE +64B0..64B5 ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-64B0..CJK UNIFIED IDEOGRAPH-64B5 +64B7..64C7 ; Recommended # 1.1 [17] CJK UNIFIED IDEOGRAPH-64B7..CJK UNIFIED IDEOGRAPH-64C7 +64C9..64D4 ; Recommended # 1.1 [12] CJK UNIFIED IDEOGRAPH-64C9..CJK UNIFIED IDEOGRAPH-64D4 +64D6..64ED ; Recommended # 1.1 [24] CJK UNIFIED IDEOGRAPH-64D6..CJK UNIFIED IDEOGRAPH-64ED +64EF..64F4 ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-64EF..CJK UNIFIED IDEOGRAPH-64F4 +64F6..64F8 ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-64F6..CJK UNIFIED IDEOGRAPH-64F8 +64FA..6501 ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-64FA..CJK UNIFIED IDEOGRAPH-6501 +6503..6509 ; Recommended # 1.1 [7] CJK UNIFIED IDEOGRAPH-6503..CJK UNIFIED IDEOGRAPH-6509 +650B..651E ; Recommended # 1.1 [20] CJK UNIFIED IDEOGRAPH-650B..CJK UNIFIED IDEOGRAPH-651E +6520..6527 ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-6520..CJK UNIFIED IDEOGRAPH-6527 +6529..653F ; Recommended # 1.1 [23] CJK UNIFIED IDEOGRAPH-6529..CJK UNIFIED IDEOGRAPH-653F +6541 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-6541 +6543..6559 ; Recommended # 1.1 [23] CJK UNIFIED IDEOGRAPH-6543..CJK UNIFIED IDEOGRAPH-6559 +655B..655E ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-655B..CJK UNIFIED IDEOGRAPH-655E +6560..657C ; Recommended # 1.1 [29] CJK UNIFIED IDEOGRAPH-6560..CJK UNIFIED IDEOGRAPH-657C +657E..6589 ; Recommended # 1.1 [12] CJK UNIFIED IDEOGRAPH-657E..CJK UNIFIED IDEOGRAPH-6589 +658B..6599 ; Recommended # 1.1 [15] CJK UNIFIED IDEOGRAPH-658B..CJK UNIFIED IDEOGRAPH-6599 +659B..65B4 ; Recommended # 1.1 [26] CJK UNIFIED IDEOGRAPH-659B..CJK UNIFIED IDEOGRAPH-65B4 +65B6..65BD ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-65B6..CJK UNIFIED IDEOGRAPH-65BD +65BF..65C7 ; Recommended # 1.1 [9] CJK UNIFIED IDEOGRAPH-65BF..CJK UNIFIED IDEOGRAPH-65C7 +65CA..65D0 ; Recommended # 1.1 [7] CJK UNIFIED IDEOGRAPH-65CA..CJK UNIFIED IDEOGRAPH-65D0 +65D2..65D7 ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-65D2..CJK UNIFIED IDEOGRAPH-65D7 +65D9..65DB ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-65D9..CJK UNIFIED IDEOGRAPH-65DB +65DD..65E3 ; Recommended # 1.1 [7] CJK UNIFIED IDEOGRAPH-65DD..CJK UNIFIED IDEOGRAPH-65E3 +65E5..65E9 ; Recommended # 1.1 [5] CJK UNIFIED IDEOGRAPH-65E5..CJK UNIFIED IDEOGRAPH-65E9 +65EB..65F8 ; Recommended # 1.1 [14] CJK UNIFIED IDEOGRAPH-65EB..CJK UNIFIED IDEOGRAPH-65F8 +65FA..65FD ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-65FA..CJK UNIFIED IDEOGRAPH-65FD +65FF..6616 ; Recommended # 1.1 [24] CJK UNIFIED IDEOGRAPH-65FF..CJK UNIFIED IDEOGRAPH-6616 +6618..662B ; Recommended # 1.1 [20] CJK UNIFIED IDEOGRAPH-6618..CJK UNIFIED IDEOGRAPH-662B +662D..6636 ; Recommended # 1.1 [10] CJK UNIFIED IDEOGRAPH-662D..CJK UNIFIED IDEOGRAPH-6636 +6639..6647 ; Recommended # 1.1 [15] CJK UNIFIED IDEOGRAPH-6639..CJK UNIFIED IDEOGRAPH-6647 +6649..664C ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-6649..CJK UNIFIED IDEOGRAPH-664C +664E..665F ; Recommended # 1.1 [18] CJK UNIFIED IDEOGRAPH-664E..CJK UNIFIED IDEOGRAPH-665F +6661..6662 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-6661..CJK UNIFIED IDEOGRAPH-6662 +6664..6691 ; Recommended # 1.1 [46] CJK UNIFIED IDEOGRAPH-6664..CJK UNIFIED IDEOGRAPH-6691 +6693..669B ; Recommended # 1.1 [9] CJK UNIFIED IDEOGRAPH-6693..CJK UNIFIED IDEOGRAPH-669B +669D ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-669D +669F..66AB ; Recommended # 1.1 [13] CJK UNIFIED IDEOGRAPH-669F..CJK UNIFIED IDEOGRAPH-66AB +66AE..66CF ; Recommended # 1.1 [34] CJK UNIFIED IDEOGRAPH-66AE..CJK UNIFIED IDEOGRAPH-66CF +66D1..66D2 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-66D1..CJK UNIFIED IDEOGRAPH-66D2 +66D4..66D6 ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-66D4..CJK UNIFIED IDEOGRAPH-66D6 +66D8..66DE ; Recommended # 1.1 [7] CJK UNIFIED IDEOGRAPH-66D8..CJK UNIFIED IDEOGRAPH-66DE +66E0..66EE ; Recommended # 1.1 [15] CJK UNIFIED IDEOGRAPH-66E0..CJK UNIFIED IDEOGRAPH-66EE +66F0..6701 ; Recommended # 1.1 [18] CJK UNIFIED IDEOGRAPH-66F0..CJK UNIFIED IDEOGRAPH-6701 +6703..6706 ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-6703..CJK UNIFIED IDEOGRAPH-6706 +6708..6718 ; Recommended # 1.1 [17] CJK UNIFIED IDEOGRAPH-6708..CJK UNIFIED IDEOGRAPH-6718 +671A..6723 ; Recommended # 1.1 [10] CJK UNIFIED IDEOGRAPH-671A..CJK UNIFIED IDEOGRAPH-6723 +6725..6728 ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-6725..CJK UNIFIED IDEOGRAPH-6728 +672A..6766 ; Recommended # 1.1 [61] CJK UNIFIED IDEOGRAPH-672A..CJK UNIFIED IDEOGRAPH-6766 +6768..6787 ; Recommended # 1.1 [32] CJK UNIFIED IDEOGRAPH-6768..CJK UNIFIED IDEOGRAPH-6787 +6789..6795 ; Recommended # 1.1 [13] CJK UNIFIED IDEOGRAPH-6789..CJK UNIFIED IDEOGRAPH-6795 +6797..67BC ; Recommended # 1.1 [38] CJK UNIFIED IDEOGRAPH-6797..CJK UNIFIED IDEOGRAPH-67BC +67BE ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-67BE +67C0..67D4 ; Recommended # 1.1 [21] CJK UNIFIED IDEOGRAPH-67C0..CJK UNIFIED IDEOGRAPH-67D4 +67D6 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-67D6 +67D8..67F8 ; Recommended # 1.1 [33] CJK UNIFIED IDEOGRAPH-67D8..CJK UNIFIED IDEOGRAPH-67F8 +67FA..6800 ; Recommended # 1.1 [7] CJK UNIFIED IDEOGRAPH-67FA..CJK UNIFIED IDEOGRAPH-6800 +6802..6814 ; Recommended # 1.1 [19] CJK UNIFIED IDEOGRAPH-6802..CJK UNIFIED IDEOGRAPH-6814 +6816..6826 ; Recommended # 1.1 [17] CJK UNIFIED IDEOGRAPH-6816..CJK UNIFIED IDEOGRAPH-6826 +6828..682F ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-6828..CJK UNIFIED IDEOGRAPH-682F +6831..6857 ; Recommended # 1.1 [39] CJK UNIFIED IDEOGRAPH-6831..CJK UNIFIED IDEOGRAPH-6857 +6859 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-6859 +685B..685D ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-685B..CJK UNIFIED IDEOGRAPH-685D +685F..6879 ; Recommended # 1.1 [27] CJK UNIFIED IDEOGRAPH-685F..CJK UNIFIED IDEOGRAPH-6879 +687B..6894 ; Recommended # 1.1 [26] CJK UNIFIED IDEOGRAPH-687B..CJK UNIFIED IDEOGRAPH-6894 +6896..6898 ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-6896..CJK UNIFIED IDEOGRAPH-6898 +689A..68A4 ; Recommended # 1.1 [11] CJK UNIFIED IDEOGRAPH-689A..CJK UNIFIED IDEOGRAPH-68A4 +68A6..68B7 ; Recommended # 1.1 [18] CJK UNIFIED IDEOGRAPH-68A6..CJK UNIFIED IDEOGRAPH-68B7 +68B9..68C2 ; Recommended # 1.1 [10] CJK UNIFIED IDEOGRAPH-68B9..CJK UNIFIED IDEOGRAPH-68C2 +68C4..68D8 ; Recommended # 1.1 [21] CJK UNIFIED IDEOGRAPH-68C4..CJK UNIFIED IDEOGRAPH-68D8 +68DA..68E1 ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-68DA..CJK UNIFIED IDEOGRAPH-68E1 +68E3..68E4 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-68E3..CJK UNIFIED IDEOGRAPH-68E4 +68E6..6908 ; Recommended # 1.1 [35] CJK UNIFIED IDEOGRAPH-68E6..CJK UNIFIED IDEOGRAPH-6908 +690A..693D ; Recommended # 1.1 [52] CJK UNIFIED IDEOGRAPH-690A..CJK UNIFIED IDEOGRAPH-693D +693F..694C ; Recommended # 1.1 [14] CJK UNIFIED IDEOGRAPH-693F..CJK UNIFIED IDEOGRAPH-694C +694E..699E ; Recommended # 1.1 [81] CJK UNIFIED IDEOGRAPH-694E..CJK UNIFIED IDEOGRAPH-699E +69A0..69A1 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-69A0..CJK UNIFIED IDEOGRAPH-69A1 +69A3..69BF ; Recommended # 1.1 [29] CJK UNIFIED IDEOGRAPH-69A3..CJK UNIFIED IDEOGRAPH-69BF +69C1..69D0 ; Recommended # 1.1 [16] CJK UNIFIED IDEOGRAPH-69C1..CJK UNIFIED IDEOGRAPH-69D0 +69D3..69D4 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-69D3..CJK UNIFIED IDEOGRAPH-69D4 +69D8..6A02 ; Recommended # 1.1 [43] CJK UNIFIED IDEOGRAPH-69D8..CJK UNIFIED IDEOGRAPH-6A02 +6A04..6A1B ; Recommended # 1.1 [24] CJK UNIFIED IDEOGRAPH-6A04..CJK UNIFIED IDEOGRAPH-6A1B +6A1D..6A23 ; Recommended # 1.1 [7] CJK UNIFIED IDEOGRAPH-6A1D..CJK UNIFIED IDEOGRAPH-6A23 +6A25..6A36 ; Recommended # 1.1 [18] CJK UNIFIED IDEOGRAPH-6A25..CJK UNIFIED IDEOGRAPH-6A36 +6A38..6A49 ; Recommended # 1.1 [18] CJK UNIFIED IDEOGRAPH-6A38..CJK UNIFIED IDEOGRAPH-6A49 +6A4B..6A5B ; Recommended # 1.1 [17] CJK UNIFIED IDEOGRAPH-6A4B..CJK UNIFIED IDEOGRAPH-6A5B +6A5D..6A6D ; Recommended # 1.1 [17] CJK UNIFIED IDEOGRAPH-6A5D..CJK UNIFIED IDEOGRAPH-6A6D +6A6F ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-6A6F +6A71..6A85 ; Recommended # 1.1 [21] CJK UNIFIED IDEOGRAPH-6A71..CJK UNIFIED IDEOGRAPH-6A85 +6A87..6A89 ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-6A87..CJK UNIFIED IDEOGRAPH-6A89 +6A8B..6A8E ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-6A8B..CJK UNIFIED IDEOGRAPH-6A8E +6A90..6A98 ; Recommended # 1.1 [9] CJK UNIFIED IDEOGRAPH-6A90..CJK UNIFIED IDEOGRAPH-6A98 +6A9A..6A9C ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-6A9A..CJK UNIFIED IDEOGRAPH-6A9C +6A9E..6AB0 ; Recommended # 1.1 [19] CJK UNIFIED IDEOGRAPH-6A9E..CJK UNIFIED IDEOGRAPH-6AB0 +6AB2..6ABD ; Recommended # 1.1 [12] CJK UNIFIED IDEOGRAPH-6AB2..CJK UNIFIED IDEOGRAPH-6ABD +6ABF ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-6ABF +6AC1..6AC3 ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-6AC1..CJK UNIFIED IDEOGRAPH-6AC3 +6AC5..6AC8 ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-6AC5..CJK UNIFIED IDEOGRAPH-6AC8 +6ACA..6AD7 ; Recommended # 1.1 [14] CJK UNIFIED IDEOGRAPH-6ACA..CJK UNIFIED IDEOGRAPH-6AD7 +6AD9..6AE8 ; Recommended # 1.1 [16] CJK UNIFIED IDEOGRAPH-6AD9..CJK UNIFIED IDEOGRAPH-6AE8 +6AEA..6B0D ; Recommended # 1.1 [36] CJK UNIFIED IDEOGRAPH-6AEA..CJK UNIFIED IDEOGRAPH-6B0D +6B0F..6B1A ; Recommended # 1.1 [12] CJK UNIFIED IDEOGRAPH-6B0F..CJK UNIFIED IDEOGRAPH-6B1A +6B1C..6B2D ; Recommended # 1.1 [18] CJK UNIFIED IDEOGRAPH-6B1C..CJK UNIFIED IDEOGRAPH-6B2D +6B2F..6B34 ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-6B2F..CJK UNIFIED IDEOGRAPH-6B34 +6B36..6B3F ; Recommended # 1.1 [10] CJK UNIFIED IDEOGRAPH-6B36..CJK UNIFIED IDEOGRAPH-6B3F +6B41..6B56 ; Recommended # 1.1 [22] CJK UNIFIED IDEOGRAPH-6B41..CJK UNIFIED IDEOGRAPH-6B56 +6B59..6B5C ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-6B59..CJK UNIFIED IDEOGRAPH-6B5C +6B5E..6B67 ; Recommended # 1.1 [10] CJK UNIFIED IDEOGRAPH-6B5E..CJK UNIFIED IDEOGRAPH-6B67 +6B69..6B6B ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-6B69..CJK UNIFIED IDEOGRAPH-6B6B +6B6D ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-6B6D +6B6F..6B70 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-6B6F..CJK UNIFIED IDEOGRAPH-6B70 +6B72..6B74 ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-6B72..CJK UNIFIED IDEOGRAPH-6B74 +6B76..6B7C ; Recommended # 1.1 [7] CJK UNIFIED IDEOGRAPH-6B76..CJK UNIFIED IDEOGRAPH-6B7C +6B7E..6BB7 ; Recommended # 1.1 [58] CJK UNIFIED IDEOGRAPH-6B7E..CJK UNIFIED IDEOGRAPH-6BB7 +6BB9..6BE8 ; Recommended # 1.1 [48] CJK UNIFIED IDEOGRAPH-6BB9..CJK UNIFIED IDEOGRAPH-6BE8 +6BEA..6BF0 ; Recommended # 1.1 [7] CJK UNIFIED IDEOGRAPH-6BEA..CJK UNIFIED IDEOGRAPH-6BF0 +6BF2..6BF3 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-6BF2..CJK UNIFIED IDEOGRAPH-6BF3 +6BF5..6BF9 ; Recommended # 1.1 [5] CJK UNIFIED IDEOGRAPH-6BF5..CJK UNIFIED IDEOGRAPH-6BF9 +6BFB..6C09 ; Recommended # 1.1 [15] CJK UNIFIED IDEOGRAPH-6BFB..CJK UNIFIED IDEOGRAPH-6C09 +6C0B..6C1B ; Recommended # 1.1 [17] CJK UNIFIED IDEOGRAPH-6C0B..CJK UNIFIED IDEOGRAPH-6C1B +6C1D..6C2C ; Recommended # 1.1 [16] CJK UNIFIED IDEOGRAPH-6C1D..CJK UNIFIED IDEOGRAPH-6C2C +6C2E..6C3B ; Recommended # 1.1 [14] CJK UNIFIED IDEOGRAPH-6C2E..CJK UNIFIED IDEOGRAPH-6C3B +6C3D..6C44 ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-6C3D..CJK UNIFIED IDEOGRAPH-6C44 +6C46..6C6B ; Recommended # 1.1 [38] CJK UNIFIED IDEOGRAPH-6C46..CJK UNIFIED IDEOGRAPH-6C6B +6C6D ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-6C6D +6C6F..6C9F ; Recommended # 1.1 [49] CJK UNIFIED IDEOGRAPH-6C6F..CJK UNIFIED IDEOGRAPH-6C9F +6CA1..6CD7 ; Recommended # 1.1 [55] CJK UNIFIED IDEOGRAPH-6CA1..CJK UNIFIED IDEOGRAPH-6CD7 +6CD9..6CF3 ; Recommended # 1.1 [27] CJK UNIFIED IDEOGRAPH-6CD9..CJK UNIFIED IDEOGRAPH-6CF3 +6CF5..6D01 ; Recommended # 1.1 [13] CJK UNIFIED IDEOGRAPH-6CF5..CJK UNIFIED IDEOGRAPH-6D01 +6D03..6D1B ; Recommended # 1.1 [25] CJK UNIFIED IDEOGRAPH-6D03..CJK UNIFIED IDEOGRAPH-6D1B +6D1D..6D23 ; Recommended # 1.1 [7] CJK UNIFIED IDEOGRAPH-6D1D..CJK UNIFIED IDEOGRAPH-6D23 +6D25..6D70 ; Recommended # 1.1 [76] CJK UNIFIED IDEOGRAPH-6D25..CJK UNIFIED IDEOGRAPH-6D70 +6D72..6D80 ; Recommended # 1.1 [15] CJK UNIFIED IDEOGRAPH-6D72..CJK UNIFIED IDEOGRAPH-6D80 +6D82..6D95 ; Recommended # 1.1 [20] CJK UNIFIED IDEOGRAPH-6D82..CJK UNIFIED IDEOGRAPH-6D95 +6D97..6DAF ; Recommended # 1.1 [25] CJK UNIFIED IDEOGRAPH-6D97..CJK UNIFIED IDEOGRAPH-6DAF +6DB2..6DB5 ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-6DB2..CJK UNIFIED IDEOGRAPH-6DB5 +6DB7..6DFD ; Recommended # 1.1 [71] CJK UNIFIED IDEOGRAPH-6DB7..CJK UNIFIED IDEOGRAPH-6DFD +6E00 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-6E00 +6E03..6E05 ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-6E03..CJK UNIFIED IDEOGRAPH-6E05 +6E07..6E11 ; Recommended # 1.1 [11] CJK UNIFIED IDEOGRAPH-6E07..CJK UNIFIED IDEOGRAPH-6E11 +6E13..6E17 ; Recommended # 1.1 [5] CJK UNIFIED IDEOGRAPH-6E13..CJK UNIFIED IDEOGRAPH-6E17 +6E19..6E29 ; Recommended # 1.1 [17] CJK UNIFIED IDEOGRAPH-6E19..CJK UNIFIED IDEOGRAPH-6E29 +6E2B..6E4B ; Recommended # 1.1 [33] CJK UNIFIED IDEOGRAPH-6E2B..CJK UNIFIED IDEOGRAPH-6E4B +6E4D..6E6B ; Recommended # 1.1 [31] CJK UNIFIED IDEOGRAPH-6E4D..CJK UNIFIED IDEOGRAPH-6E6B +6E6D..6E7A ; Recommended # 1.1 [14] CJK UNIFIED IDEOGRAPH-6E6D..CJK UNIFIED IDEOGRAPH-6E7A +6E7E..6E8A ; Recommended # 1.1 [13] CJK UNIFIED IDEOGRAPH-6E7E..CJK UNIFIED IDEOGRAPH-6E8A +6E8C..6E94 ; Recommended # 1.1 [9] CJK UNIFIED IDEOGRAPH-6E8C..CJK UNIFIED IDEOGRAPH-6E94 +6E96..6EDA ; Recommended # 1.1 [69] CJK UNIFIED IDEOGRAPH-6E96..CJK UNIFIED IDEOGRAPH-6EDA +6EDC..6EE2 ; Recommended # 1.1 [7] CJK UNIFIED IDEOGRAPH-6EDC..CJK UNIFIED IDEOGRAPH-6EE2 +6EE4..6F03 ; Recommended # 1.1 [32] CJK UNIFIED IDEOGRAPH-6EE4..CJK UNIFIED IDEOGRAPH-6F03 +6F05..6F0A ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-6F05..CJK UNIFIED IDEOGRAPH-6F0A +6F0C..6F41 ; Recommended # 1.1 [54] CJK UNIFIED IDEOGRAPH-6F0C..CJK UNIFIED IDEOGRAPH-6F41 +6F43..6F47 ; Recommended # 1.1 [5] CJK UNIFIED IDEOGRAPH-6F43..CJK UNIFIED IDEOGRAPH-6F47 +6F49 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-6F49 +6F4B..6F78 ; Recommended # 1.1 [46] CJK UNIFIED IDEOGRAPH-6F4B..CJK UNIFIED IDEOGRAPH-6F78 +6F7A..6F97 ; Recommended # 1.1 [30] CJK UNIFIED IDEOGRAPH-6F7A..CJK UNIFIED IDEOGRAPH-6F97 +6F99 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-6F99 +6F9B..6F9E ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-6F9B..CJK UNIFIED IDEOGRAPH-6F9E +6FA0..6FB6 ; Recommended # 1.1 [23] CJK UNIFIED IDEOGRAPH-6FA0..CJK UNIFIED IDEOGRAPH-6FB6 +6FB8..6FC4 ; Recommended # 1.1 [13] CJK UNIFIED IDEOGRAPH-6FB8..CJK UNIFIED IDEOGRAPH-6FC4 +6FC6..6FCF ; Recommended # 1.1 [10] CJK UNIFIED IDEOGRAPH-6FC6..CJK UNIFIED IDEOGRAPH-6FCF +6FD1..6FD2 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-6FD1..CJK UNIFIED IDEOGRAPH-6FD2 +6FD4..6FF4 ; Recommended # 1.1 [33] CJK UNIFIED IDEOGRAPH-6FD4..CJK UNIFIED IDEOGRAPH-6FF4 +6FF6..6FFC ; Recommended # 1.1 [7] CJK UNIFIED IDEOGRAPH-6FF6..CJK UNIFIED IDEOGRAPH-6FFC +6FFE..700F ; Recommended # 1.1 [18] CJK UNIFIED IDEOGRAPH-6FFE..CJK UNIFIED IDEOGRAPH-700F +7011..7012 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-7011..CJK UNIFIED IDEOGRAPH-7012 +7014..7046 ; Recommended # 1.1 [51] CJK UNIFIED IDEOGRAPH-7014..CJK UNIFIED IDEOGRAPH-7046 +7048..704A ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-7048..CJK UNIFIED IDEOGRAPH-704A +704C..704D ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-704C..CJK UNIFIED IDEOGRAPH-704D +704F..7071 ; Recommended # 1.1 [35] CJK UNIFIED IDEOGRAPH-704F..CJK UNIFIED IDEOGRAPH-7071 +7074..707A ; Recommended # 1.1 [7] CJK UNIFIED IDEOGRAPH-7074..CJK UNIFIED IDEOGRAPH-707A +707C..7080 ; Recommended # 1.1 [5] CJK UNIFIED IDEOGRAPH-707C..CJK UNIFIED IDEOGRAPH-7080 +7082..708C ; Recommended # 1.1 [11] CJK UNIFIED IDEOGRAPH-7082..CJK UNIFIED IDEOGRAPH-708C +708E..7096 ; Recommended # 1.1 [9] CJK UNIFIED IDEOGRAPH-708E..CJK UNIFIED IDEOGRAPH-7096 +7098..709A ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-7098..CJK UNIFIED IDEOGRAPH-709A +709C..70A9 ; Recommended # 1.1 [14] CJK UNIFIED IDEOGRAPH-709C..CJK UNIFIED IDEOGRAPH-70A9 +70AB..70B1 ; Recommended # 1.1 [7] CJK UNIFIED IDEOGRAPH-70AB..CJK UNIFIED IDEOGRAPH-70B1 +70B3..70B5 ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-70B3..CJK UNIFIED IDEOGRAPH-70B5 +70B7..70D4 ; Recommended # 1.1 [30] CJK UNIFIED IDEOGRAPH-70B7..CJK UNIFIED IDEOGRAPH-70D4 +70D6..70FD ; Recommended # 1.1 [40] CJK UNIFIED IDEOGRAPH-70D6..CJK UNIFIED IDEOGRAPH-70FD +70FF..7107 ; Recommended # 1.1 [9] CJK UNIFIED IDEOGRAPH-70FF..CJK UNIFIED IDEOGRAPH-7107 +7109..7123 ; Recommended # 1.1 [27] CJK UNIFIED IDEOGRAPH-7109..CJK UNIFIED IDEOGRAPH-7123 +7125..7132 ; Recommended # 1.1 [14] CJK UNIFIED IDEOGRAPH-7125..CJK UNIFIED IDEOGRAPH-7132 +7135..7156 ; Recommended # 1.1 [34] CJK UNIFIED IDEOGRAPH-7135..CJK UNIFIED IDEOGRAPH-7156 +7158..716A ; Recommended # 1.1 [19] CJK UNIFIED IDEOGRAPH-7158..CJK UNIFIED IDEOGRAPH-716A +716C ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-716C +716E..718C ; Recommended # 1.1 [31] CJK UNIFIED IDEOGRAPH-716E..CJK UNIFIED IDEOGRAPH-718C +718E..7195 ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-718E..CJK UNIFIED IDEOGRAPH-7195 +7197..71A5 ; Recommended # 1.1 [15] CJK UNIFIED IDEOGRAPH-7197..CJK UNIFIED IDEOGRAPH-71A5 +71A7..71AA ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-71A7..CJK UNIFIED IDEOGRAPH-71AA +71AC..71B5 ; Recommended # 1.1 [10] CJK UNIFIED IDEOGRAPH-71AC..CJK UNIFIED IDEOGRAPH-71B5 +71B7..71CB ; Recommended # 1.1 [21] CJK UNIFIED IDEOGRAPH-71B7..CJK UNIFIED IDEOGRAPH-71CB +71CD..71D2 ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-71CD..CJK UNIFIED IDEOGRAPH-71D2 +71D4..71F2 ; Recommended # 1.1 [31] CJK UNIFIED IDEOGRAPH-71D4..CJK UNIFIED IDEOGRAPH-71F2 +71F4..71F9 ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-71F4..CJK UNIFIED IDEOGRAPH-71F9 +71FB..720A ; Recommended # 1.1 [16] CJK UNIFIED IDEOGRAPH-71FB..CJK UNIFIED IDEOGRAPH-720A +720C..7210 ; Recommended # 1.1 [5] CJK UNIFIED IDEOGRAPH-720C..CJK UNIFIED IDEOGRAPH-7210 +7212..7214 ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-7212..CJK UNIFIED IDEOGRAPH-7214 +7216 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-7216 +7218..721F ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-7218..CJK UNIFIED IDEOGRAPH-721F +7221..7223 ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-7221..CJK UNIFIED IDEOGRAPH-7223 +7226..722E ; Recommended # 1.1 [9] CJK UNIFIED IDEOGRAPH-7226..CJK UNIFIED IDEOGRAPH-722E +7230..7233 ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-7230..CJK UNIFIED IDEOGRAPH-7233 +7235..7244 ; Recommended # 1.1 [16] CJK UNIFIED IDEOGRAPH-7235..CJK UNIFIED IDEOGRAPH-7244 +7246..724D ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-7246..CJK UNIFIED IDEOGRAPH-724D +724F ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-724F +7251..7254 ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-7251..CJK UNIFIED IDEOGRAPH-7254 +7256..72AA ; Recommended # 1.1 [85] CJK UNIFIED IDEOGRAPH-7256..CJK UNIFIED IDEOGRAPH-72AA +72AC..72BD ; Recommended # 1.1 [18] CJK UNIFIED IDEOGRAPH-72AC..CJK UNIFIED IDEOGRAPH-72BD +72BF..7301 ; Recommended # 1.1 [67] CJK UNIFIED IDEOGRAPH-72BF..CJK UNIFIED IDEOGRAPH-7301 +7303..730F ; Recommended # 1.1 [13] CJK UNIFIED IDEOGRAPH-7303..CJK UNIFIED IDEOGRAPH-730F +7311..7327 ; Recommended # 1.1 [23] CJK UNIFIED IDEOGRAPH-7311..CJK UNIFIED IDEOGRAPH-7327 +7329..7352 ; Recommended # 1.1 [42] CJK UNIFIED IDEOGRAPH-7329..CJK UNIFIED IDEOGRAPH-7352 +7354..739B ; Recommended # 1.1 [72] CJK UNIFIED IDEOGRAPH-7354..CJK UNIFIED IDEOGRAPH-739B +739D..73C0 ; Recommended # 1.1 [36] CJK UNIFIED IDEOGRAPH-739D..CJK UNIFIED IDEOGRAPH-73C0 +73C2..73F2 ; Recommended # 1.1 [49] CJK UNIFIED IDEOGRAPH-73C2..CJK UNIFIED IDEOGRAPH-73F2 +73F4..73FA ; Recommended # 1.1 [7] CJK UNIFIED IDEOGRAPH-73F4..CJK UNIFIED IDEOGRAPH-73FA +73FC..7417 ; Recommended # 1.1 [28] CJK UNIFIED IDEOGRAPH-73FC..CJK UNIFIED IDEOGRAPH-7417 +7419..7438 ; Recommended # 1.1 [32] CJK UNIFIED IDEOGRAPH-7419..CJK UNIFIED IDEOGRAPH-7438 +743A..743D ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-743A..CJK UNIFIED IDEOGRAPH-743D +743F..7446 ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-743F..CJK UNIFIED IDEOGRAPH-7446 +7448 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-7448 +744A..7457 ; Recommended # 1.1 [14] CJK UNIFIED IDEOGRAPH-744A..CJK UNIFIED IDEOGRAPH-7457 +7459..747A ; Recommended # 1.1 [34] CJK UNIFIED IDEOGRAPH-7459..CJK UNIFIED IDEOGRAPH-747A +747C..7483 ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-747C..CJK UNIFIED IDEOGRAPH-7483 +7485..7495 ; Recommended # 1.1 [17] CJK UNIFIED IDEOGRAPH-7485..CJK UNIFIED IDEOGRAPH-7495 +7497..749C ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-7497..CJK UNIFIED IDEOGRAPH-749C +749E..74C6 ; Recommended # 1.1 [41] CJK UNIFIED IDEOGRAPH-749E..CJK UNIFIED IDEOGRAPH-74C6 +74C8 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-74C8 +74CA..74CB ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-74CA..CJK UNIFIED IDEOGRAPH-74CB +74CD..74EA ; Recommended # 1.1 [30] CJK UNIFIED IDEOGRAPH-74CD..CJK UNIFIED IDEOGRAPH-74EA +74EC..751F ; Recommended # 1.1 [52] CJK UNIFIED IDEOGRAPH-74EC..CJK UNIFIED IDEOGRAPH-751F +7521..7540 ; Recommended # 1.1 [32] CJK UNIFIED IDEOGRAPH-7521..CJK UNIFIED IDEOGRAPH-7540 +7542..7551 ; Recommended # 1.1 [16] CJK UNIFIED IDEOGRAPH-7542..CJK UNIFIED IDEOGRAPH-7551 +7553..7554 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-7553..CJK UNIFIED IDEOGRAPH-7554 +7556..755D ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-7556..CJK UNIFIED IDEOGRAPH-755D +755F..7560 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-755F..CJK UNIFIED IDEOGRAPH-7560 +7562..7570 ; Recommended # 1.1 [15] CJK UNIFIED IDEOGRAPH-7562..CJK UNIFIED IDEOGRAPH-7570 +7572..757A ; Recommended # 1.1 [9] CJK UNIFIED IDEOGRAPH-7572..CJK UNIFIED IDEOGRAPH-757A +757C..7584 ; Recommended # 1.1 [9] CJK UNIFIED IDEOGRAPH-757C..CJK UNIFIED IDEOGRAPH-7584 +7586..75A8 ; Recommended # 1.1 [35] CJK UNIFIED IDEOGRAPH-7586..CJK UNIFIED IDEOGRAPH-75A8 +75AA..75B6 ; Recommended # 1.1 [13] CJK UNIFIED IDEOGRAPH-75AA..CJK UNIFIED IDEOGRAPH-75B6 +75B8..75DB ; Recommended # 1.1 [36] CJK UNIFIED IDEOGRAPH-75B8..CJK UNIFIED IDEOGRAPH-75DB +75DD..75ED ; Recommended # 1.1 [17] CJK UNIFIED IDEOGRAPH-75DD..CJK UNIFIED IDEOGRAPH-75ED +75EF..762B ; Recommended # 1.1 [61] CJK UNIFIED IDEOGRAPH-75EF..CJK UNIFIED IDEOGRAPH-762B +762D..7643 ; Recommended # 1.1 [23] CJK UNIFIED IDEOGRAPH-762D..CJK UNIFIED IDEOGRAPH-7643 +7646..7650 ; Recommended # 1.1 [11] CJK UNIFIED IDEOGRAPH-7646..CJK UNIFIED IDEOGRAPH-7650 +7652..7654 ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-7652..CJK UNIFIED IDEOGRAPH-7654 +7656..7672 ; Recommended # 1.1 [29] CJK UNIFIED IDEOGRAPH-7656..CJK UNIFIED IDEOGRAPH-7672 +7674..768C ; Recommended # 1.1 [25] CJK UNIFIED IDEOGRAPH-7674..CJK UNIFIED IDEOGRAPH-768C +768E..76A0 ; Recommended # 1.1 [19] CJK UNIFIED IDEOGRAPH-768E..CJK UNIFIED IDEOGRAPH-76A0 +76A3..76A4 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-76A3..CJK UNIFIED IDEOGRAPH-76A4 +76A6..76A7 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-76A6..CJK UNIFIED IDEOGRAPH-76A7 +76A9..76B2 ; Recommended # 1.1 [10] CJK UNIFIED IDEOGRAPH-76A9..CJK UNIFIED IDEOGRAPH-76B2 +76B4..76B5 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-76B4..CJK UNIFIED IDEOGRAPH-76B5 +76B7..76C0 ; Recommended # 1.1 [10] CJK UNIFIED IDEOGRAPH-76B7..CJK UNIFIED IDEOGRAPH-76C0 +76C2..76CA ; Recommended # 1.1 [9] CJK UNIFIED IDEOGRAPH-76C2..CJK UNIFIED IDEOGRAPH-76CA +76CC..76D8 ; Recommended # 1.1 [13] CJK UNIFIED IDEOGRAPH-76CC..CJK UNIFIED IDEOGRAPH-76D8 +76DA..76EA ; Recommended # 1.1 [17] CJK UNIFIED IDEOGRAPH-76DA..CJK UNIFIED IDEOGRAPH-76EA +76EC..76FF ; Recommended # 1.1 [20] CJK UNIFIED IDEOGRAPH-76EC..CJK UNIFIED IDEOGRAPH-76FF +7701 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-7701 +7703..770D ; Recommended # 1.1 [11] CJK UNIFIED IDEOGRAPH-7703..CJK UNIFIED IDEOGRAPH-770D +770F..7720 ; Recommended # 1.1 [18] CJK UNIFIED IDEOGRAPH-770F..CJK UNIFIED IDEOGRAPH-7720 +7722..772A ; Recommended # 1.1 [9] CJK UNIFIED IDEOGRAPH-7722..CJK UNIFIED IDEOGRAPH-772A +772C..773E ; Recommended # 1.1 [19] CJK UNIFIED IDEOGRAPH-772C..CJK UNIFIED IDEOGRAPH-773E +7740..7741 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-7740..CJK UNIFIED IDEOGRAPH-7741 +7743..7763 ; Recommended # 1.1 [33] CJK UNIFIED IDEOGRAPH-7743..CJK UNIFIED IDEOGRAPH-7763 +7765..7795 ; Recommended # 1.1 [49] CJK UNIFIED IDEOGRAPH-7765..CJK UNIFIED IDEOGRAPH-7795 +7797..77A3 ; Recommended # 1.1 [13] CJK UNIFIED IDEOGRAPH-7797..CJK UNIFIED IDEOGRAPH-77A3 +77A5..77BD ; Recommended # 1.1 [25] CJK UNIFIED IDEOGRAPH-77A5..CJK UNIFIED IDEOGRAPH-77BD +77BF..77C0 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-77BF..CJK UNIFIED IDEOGRAPH-77C0 +77C2..77D1 ; Recommended # 1.1 [16] CJK UNIFIED IDEOGRAPH-77C2..CJK UNIFIED IDEOGRAPH-77D1 +77D3..77DC ; Recommended # 1.1 [10] CJK UNIFIED IDEOGRAPH-77D3..CJK UNIFIED IDEOGRAPH-77DC +77DE..77E3 ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-77DE..CJK UNIFIED IDEOGRAPH-77E3 +77E5 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-77E5 +77E7..77F3 ; Recommended # 1.1 [13] CJK UNIFIED IDEOGRAPH-77E7..CJK UNIFIED IDEOGRAPH-77F3 +77F6..7823 ; Recommended # 1.1 [46] CJK UNIFIED IDEOGRAPH-77F6..CJK UNIFIED IDEOGRAPH-7823 +7825..7835 ; Recommended # 1.1 [17] CJK UNIFIED IDEOGRAPH-7825..CJK UNIFIED IDEOGRAPH-7835 +7837..7841 ; Recommended # 1.1 [11] CJK UNIFIED IDEOGRAPH-7837..CJK UNIFIED IDEOGRAPH-7841 +7843..7845 ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-7843..CJK UNIFIED IDEOGRAPH-7845 +7847..784A ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-7847..CJK UNIFIED IDEOGRAPH-784A +784C..7875 ; Recommended # 1.1 [42] CJK UNIFIED IDEOGRAPH-784C..CJK UNIFIED IDEOGRAPH-7875 +7877..7887 ; Recommended # 1.1 [17] CJK UNIFIED IDEOGRAPH-7877..CJK UNIFIED IDEOGRAPH-7887 +7889..78C1 ; Recommended # 1.1 [57] CJK UNIFIED IDEOGRAPH-7889..CJK UNIFIED IDEOGRAPH-78C1 +78C3..78C6 ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-78C3..CJK UNIFIED IDEOGRAPH-78C6 +78C8..78D1 ; Recommended # 1.1 [10] CJK UNIFIED IDEOGRAPH-78C8..CJK UNIFIED IDEOGRAPH-78D1 +78D3..78EF ; Recommended # 1.1 [29] CJK UNIFIED IDEOGRAPH-78D3..CJK UNIFIED IDEOGRAPH-78EF +78F1..78F7 ; Recommended # 1.1 [7] CJK UNIFIED IDEOGRAPH-78F1..CJK UNIFIED IDEOGRAPH-78F7 +78F9..78FF ; Recommended # 1.1 [7] CJK UNIFIED IDEOGRAPH-78F9..CJK UNIFIED IDEOGRAPH-78FF +7901..7907 ; Recommended # 1.1 [7] CJK UNIFIED IDEOGRAPH-7901..CJK UNIFIED IDEOGRAPH-7907 +7909..790C ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-7909..CJK UNIFIED IDEOGRAPH-790C +790E..7914 ; Recommended # 1.1 [7] CJK UNIFIED IDEOGRAPH-790E..CJK UNIFIED IDEOGRAPH-7914 +7916..791E ; Recommended # 1.1 [9] CJK UNIFIED IDEOGRAPH-7916..CJK UNIFIED IDEOGRAPH-791E +7921..7931 ; Recommended # 1.1 [17] CJK UNIFIED IDEOGRAPH-7921..CJK UNIFIED IDEOGRAPH-7931 +7933..7935 ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-7933..CJK UNIFIED IDEOGRAPH-7935 +7937..7958 ; Recommended # 1.1 [34] CJK UNIFIED IDEOGRAPH-7937..CJK UNIFIED IDEOGRAPH-7958 +795A..796B ; Recommended # 1.1 [18] CJK UNIFIED IDEOGRAPH-795A..CJK UNIFIED IDEOGRAPH-796B +796D ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-796D +796F..7974 ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-796F..CJK UNIFIED IDEOGRAPH-7974 +7977..7985 ; Recommended # 1.1 [15] CJK UNIFIED IDEOGRAPH-7977..CJK UNIFIED IDEOGRAPH-7985 +7988..799D ; Recommended # 1.1 [22] CJK UNIFIED IDEOGRAPH-7988..CJK UNIFIED IDEOGRAPH-799D +799F..79A8 ; Recommended # 1.1 [10] CJK UNIFIED IDEOGRAPH-799F..CJK UNIFIED IDEOGRAPH-79A8 +79AA..79BB ; Recommended # 1.1 [18] CJK UNIFIED IDEOGRAPH-79AA..CJK UNIFIED IDEOGRAPH-79BB +79BD..79C3 ; Recommended # 1.1 [7] CJK UNIFIED IDEOGRAPH-79BD..CJK UNIFIED IDEOGRAPH-79C3 +79C5..79C6 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-79C5..CJK UNIFIED IDEOGRAPH-79C6 +79C8..79CB ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-79C8..CJK UNIFIED IDEOGRAPH-79CB +79CD..79D3 ; Recommended # 1.1 [7] CJK UNIFIED IDEOGRAPH-79CD..CJK UNIFIED IDEOGRAPH-79D3 +79D5..79D6 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-79D5..CJK UNIFIED IDEOGRAPH-79D6 +79D8..7A00 ; Recommended # 1.1 [41] CJK UNIFIED IDEOGRAPH-79D8..CJK UNIFIED IDEOGRAPH-7A00 +7A02..7A06 ; Recommended # 1.1 [5] CJK UNIFIED IDEOGRAPH-7A02..CJK UNIFIED IDEOGRAPH-7A06 +7A08 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-7A08 +7A0A..7A2B ; Recommended # 1.1 [34] CJK UNIFIED IDEOGRAPH-7A0A..CJK UNIFIED IDEOGRAPH-7A2B +7A2D..7A37 ; Recommended # 1.1 [11] CJK UNIFIED IDEOGRAPH-7A2D..CJK UNIFIED IDEOGRAPH-7A37 +7A39 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-7A39 +7A3B..7A63 ; Recommended # 1.1 [41] CJK UNIFIED IDEOGRAPH-7A3B..CJK UNIFIED IDEOGRAPH-7A63 +7A65..7A69 ; Recommended # 1.1 [5] CJK UNIFIED IDEOGRAPH-7A65..CJK UNIFIED IDEOGRAPH-7A69 +7A6B..7A6E ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-7A6B..CJK UNIFIED IDEOGRAPH-7A6E +7A70..7A81 ; Recommended # 1.1 [18] CJK UNIFIED IDEOGRAPH-7A70..CJK UNIFIED IDEOGRAPH-7A81 +7A83..7A99 ; Recommended # 1.1 [23] CJK UNIFIED IDEOGRAPH-7A83..CJK UNIFIED IDEOGRAPH-7A99 +7A9C..7AB8 ; Recommended # 1.1 [29] CJK UNIFIED IDEOGRAPH-7A9C..CJK UNIFIED IDEOGRAPH-7AB8 +7ABA ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-7ABA +7ABE..7AC1 ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-7ABE..CJK UNIFIED IDEOGRAPH-7AC1 +7AC3..7AC5 ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-7AC3..CJK UNIFIED IDEOGRAPH-7AC5 +7AC7..7AE8 ; Recommended # 1.1 [34] CJK UNIFIED IDEOGRAPH-7AC7..CJK UNIFIED IDEOGRAPH-7AE8 +7AEA..7AF4 ; Recommended # 1.1 [11] CJK UNIFIED IDEOGRAPH-7AEA..CJK UNIFIED IDEOGRAPH-7AF4 +7AF6..7AFB ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-7AF6..CJK UNIFIED IDEOGRAPH-7AFB +7AFD..7B06 ; Recommended # 1.1 [10] CJK UNIFIED IDEOGRAPH-7AFD..CJK UNIFIED IDEOGRAPH-7B06 +7B08..7B1E ; Recommended # 1.1 [23] CJK UNIFIED IDEOGRAPH-7B08..CJK UNIFIED IDEOGRAPH-7B1E +7B20..7B26 ; Recommended # 1.1 [7] CJK UNIFIED IDEOGRAPH-7B20..CJK UNIFIED IDEOGRAPH-7B26 +7B28 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-7B28 +7B2A..7B41 ; Recommended # 1.1 [24] CJK UNIFIED IDEOGRAPH-7B2A..CJK UNIFIED IDEOGRAPH-7B41 +7B43..7B52 ; Recommended # 1.1 [16] CJK UNIFIED IDEOGRAPH-7B43..CJK UNIFIED IDEOGRAPH-7B52 +7B54..7BA2 ; Recommended # 1.1 [79] CJK UNIFIED IDEOGRAPH-7B54..CJK UNIFIED IDEOGRAPH-7BA2 +7BA4 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-7BA4 +7BA6..7BAF ; Recommended # 1.1 [10] CJK UNIFIED IDEOGRAPH-7BA6..CJK UNIFIED IDEOGRAPH-7BAF +7BB1 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-7BB1 +7BB3..7BF9 ; Recommended # 1.1 [71] CJK UNIFIED IDEOGRAPH-7BB3..CJK UNIFIED IDEOGRAPH-7BF9 +7BFB..7C1A ; Recommended # 1.1 [32] CJK UNIFIED IDEOGRAPH-7BFB..CJK UNIFIED IDEOGRAPH-7C1A +7C1C..7C2D ; Recommended # 1.1 [18] CJK UNIFIED IDEOGRAPH-7C1C..CJK UNIFIED IDEOGRAPH-7C2D +7C30..7C51 ; Recommended # 1.1 [34] CJK UNIFIED IDEOGRAPH-7C30..CJK UNIFIED IDEOGRAPH-7C51 +7C53..7C54 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-7C53..CJK UNIFIED IDEOGRAPH-7C54 +7C56..7C5C ; Recommended # 1.1 [7] CJK UNIFIED IDEOGRAPH-7C56..CJK UNIFIED IDEOGRAPH-7C5C +7C5E..7C75 ; Recommended # 1.1 [24] CJK UNIFIED IDEOGRAPH-7C5E..CJK UNIFIED IDEOGRAPH-7C75 +7C77..7C86 ; Recommended # 1.1 [16] CJK UNIFIED IDEOGRAPH-7C77..CJK UNIFIED IDEOGRAPH-7C86 +7C88..7C92 ; Recommended # 1.1 [11] CJK UNIFIED IDEOGRAPH-7C88..CJK UNIFIED IDEOGRAPH-7C92 +7C94..7C99 ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-7C94..CJK UNIFIED IDEOGRAPH-7C99 +7C9B..7CAB ; Recommended # 1.1 [17] CJK UNIFIED IDEOGRAPH-7C9B..CJK UNIFIED IDEOGRAPH-7CAB +7CAD..7CD2 ; Recommended # 1.1 [38] CJK UNIFIED IDEOGRAPH-7CAD..CJK UNIFIED IDEOGRAPH-7CD2 +7CD4..7CD9 ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-7CD4..CJK UNIFIED IDEOGRAPH-7CD9 +7CDC..7CE0 ; Recommended # 1.1 [5] CJK UNIFIED IDEOGRAPH-7CDC..CJK UNIFIED IDEOGRAPH-7CE0 +7CE2 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-7CE2 +7CE4 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-7CE4 +7CE7..7CFB ; Recommended # 1.1 [21] CJK UNIFIED IDEOGRAPH-7CE7..CJK UNIFIED IDEOGRAPH-7CFB +7CFD..7CFE ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-7CFD..CJK UNIFIED IDEOGRAPH-7CFE +7D00..7D22 ; Recommended # 1.1 [35] CJK UNIFIED IDEOGRAPH-7D00..CJK UNIFIED IDEOGRAPH-7D22 +7D24..7D29 ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-7D24..CJK UNIFIED IDEOGRAPH-7D29 +7D2B..7D2C ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-7D2B..CJK UNIFIED IDEOGRAPH-7D2C +7D2E..7D47 ; Recommended # 1.1 [26] CJK UNIFIED IDEOGRAPH-7D2E..CJK UNIFIED IDEOGRAPH-7D47 +7D49..7D4C ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-7D49..CJK UNIFIED IDEOGRAPH-7D4C +7D4E..7D59 ; Recommended # 1.1 [12] CJK UNIFIED IDEOGRAPH-7D4E..CJK UNIFIED IDEOGRAPH-7D59 +7D5B..7D63 ; Recommended # 1.1 [9] CJK UNIFIED IDEOGRAPH-7D5B..CJK UNIFIED IDEOGRAPH-7D63 +7D65..7D77 ; Recommended # 1.1 [19] CJK UNIFIED IDEOGRAPH-7D65..CJK UNIFIED IDEOGRAPH-7D77 +7D79..7D81 ; Recommended # 1.1 [9] CJK UNIFIED IDEOGRAPH-7D79..CJK UNIFIED IDEOGRAPH-7D81 +7D83..7D94 ; Recommended # 1.1 [18] CJK UNIFIED IDEOGRAPH-7D83..CJK UNIFIED IDEOGRAPH-7D94 +7D96..7D97 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-7D96..CJK UNIFIED IDEOGRAPH-7D97 +7D99..7DA3 ; Recommended # 1.1 [11] CJK UNIFIED IDEOGRAPH-7D99..CJK UNIFIED IDEOGRAPH-7DA3 +7DA5..7DA7 ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-7DA5..CJK UNIFIED IDEOGRAPH-7DA7 +7DA9..7DCC ; Recommended # 1.1 [36] CJK UNIFIED IDEOGRAPH-7DA9..CJK UNIFIED IDEOGRAPH-7DCC +7DCE..7DD2 ; Recommended # 1.1 [5] CJK UNIFIED IDEOGRAPH-7DCE..CJK UNIFIED IDEOGRAPH-7DD2 +7DD4..7DE4 ; Recommended # 1.1 [17] CJK UNIFIED IDEOGRAPH-7DD4..CJK UNIFIED IDEOGRAPH-7DE4 +7DE6..7DEA ; Recommended # 1.1 [5] CJK UNIFIED IDEOGRAPH-7DE6..CJK UNIFIED IDEOGRAPH-7DEA +7DEC..7DFC ; Recommended # 1.1 [17] CJK UNIFIED IDEOGRAPH-7DEC..CJK UNIFIED IDEOGRAPH-7DFC +7E00..7E17 ; Recommended # 1.1 [24] CJK UNIFIED IDEOGRAPH-7E00..CJK UNIFIED IDEOGRAPH-7E17 +7E19..7E5A ; Recommended # 1.1 [66] CJK UNIFIED IDEOGRAPH-7E19..CJK UNIFIED IDEOGRAPH-7E5A +7E5C..7E63 ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-7E5C..CJK UNIFIED IDEOGRAPH-7E63 +7E65..7E9C ; Recommended # 1.1 [56] CJK UNIFIED IDEOGRAPH-7E65..CJK UNIFIED IDEOGRAPH-7E9C +7E9E..7F3A ; Recommended # 1.1 [157] CJK UNIFIED IDEOGRAPH-7E9E..CJK UNIFIED IDEOGRAPH-7F3A +7F3D..7F40 ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-7F3D..CJK UNIFIED IDEOGRAPH-7F40 +7F42..7F45 ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-7F42..CJK UNIFIED IDEOGRAPH-7F45 +7F47..7F58 ; Recommended # 1.1 [18] CJK UNIFIED IDEOGRAPH-7F47..CJK UNIFIED IDEOGRAPH-7F58 +7F5A..7F83 ; Recommended # 1.1 [42] CJK UNIFIED IDEOGRAPH-7F5A..CJK UNIFIED IDEOGRAPH-7F83 +7F85..7F8F ; Recommended # 1.1 [11] CJK UNIFIED IDEOGRAPH-7F85..CJK UNIFIED IDEOGRAPH-7F8F +7F91..7F96 ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-7F91..CJK UNIFIED IDEOGRAPH-7F96 +7F98 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-7F98 +7F9A..7FB3 ; Recommended # 1.1 [26] CJK UNIFIED IDEOGRAPH-7F9A..CJK UNIFIED IDEOGRAPH-7FB3 +7FB5..7FD5 ; Recommended # 1.1 [33] CJK UNIFIED IDEOGRAPH-7FB5..CJK UNIFIED IDEOGRAPH-7FD5 +7FD7..7FDC ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-7FD7..CJK UNIFIED IDEOGRAPH-7FDC +7FDE..7FE3 ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-7FDE..CJK UNIFIED IDEOGRAPH-7FE3 +7FE5..8009 ; Recommended # 1.1 [37] CJK UNIFIED IDEOGRAPH-7FE5..CJK UNIFIED IDEOGRAPH-8009 +800B..802E ; Recommended # 1.1 [36] CJK UNIFIED IDEOGRAPH-800B..CJK UNIFIED IDEOGRAPH-802E +8030..803B ; Recommended # 1.1 [12] CJK UNIFIED IDEOGRAPH-8030..CJK UNIFIED IDEOGRAPH-803B +803D..803F ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-803D..CJK UNIFIED IDEOGRAPH-803F +8041..8065 ; Recommended # 1.1 [37] CJK UNIFIED IDEOGRAPH-8041..CJK UNIFIED IDEOGRAPH-8065 +8067..8087 ; Recommended # 1.1 [33] CJK UNIFIED IDEOGRAPH-8067..CJK UNIFIED IDEOGRAPH-8087 +8089..808D ; Recommended # 1.1 [5] CJK UNIFIED IDEOGRAPH-8089..CJK UNIFIED IDEOGRAPH-808D +808F..8093 ; Recommended # 1.1 [5] CJK UNIFIED IDEOGRAPH-808F..CJK UNIFIED IDEOGRAPH-8093 +8095..80A5 ; Recommended # 1.1 [17] CJK UNIFIED IDEOGRAPH-8095..CJK UNIFIED IDEOGRAPH-80A5 +80A9..80B2 ; Recommended # 1.1 [10] CJK UNIFIED IDEOGRAPH-80A9..CJK UNIFIED IDEOGRAPH-80B2 +80B4..80B8 ; Recommended # 1.1 [5] CJK UNIFIED IDEOGRAPH-80B4..CJK UNIFIED IDEOGRAPH-80B8 +80BA..80DE ; Recommended # 1.1 [37] CJK UNIFIED IDEOGRAPH-80BA..CJK UNIFIED IDEOGRAPH-80DE +80E0..8102 ; Recommended # 1.1 [35] CJK UNIFIED IDEOGRAPH-80E0..CJK UNIFIED IDEOGRAPH-8102 +8105..8133 ; Recommended # 1.1 [47] CJK UNIFIED IDEOGRAPH-8105..CJK UNIFIED IDEOGRAPH-8133 +8136..8183 ; Recommended # 1.1 [78] CJK UNIFIED IDEOGRAPH-8136..CJK UNIFIED IDEOGRAPH-8183 +8185..818F ; Recommended # 1.1 [11] CJK UNIFIED IDEOGRAPH-8185..CJK UNIFIED IDEOGRAPH-818F +8191..8195 ; Recommended # 1.1 [5] CJK UNIFIED IDEOGRAPH-8191..CJK UNIFIED IDEOGRAPH-8195 +8197..81CA ; Recommended # 1.1 [52] CJK UNIFIED IDEOGRAPH-8197..CJK UNIFIED IDEOGRAPH-81CA +81CC..81E3 ; Recommended # 1.1 [24] CJK UNIFIED IDEOGRAPH-81CC..CJK UNIFIED IDEOGRAPH-81E3 +81E5..81EE ; Recommended # 1.1 [10] CJK UNIFIED IDEOGRAPH-81E5..CJK UNIFIED IDEOGRAPH-81EE +81F1..8212 ; Recommended # 1.1 [34] CJK UNIFIED IDEOGRAPH-81F1..CJK UNIFIED IDEOGRAPH-8212 +8214..8223 ; Recommended # 1.1 [16] CJK UNIFIED IDEOGRAPH-8214..CJK UNIFIED IDEOGRAPH-8223 +8225..8240 ; Recommended # 1.1 [28] CJK UNIFIED IDEOGRAPH-8225..CJK UNIFIED IDEOGRAPH-8240 +8242..8264 ; Recommended # 1.1 [35] CJK UNIFIED IDEOGRAPH-8242..CJK UNIFIED IDEOGRAPH-8264 +8266..828B ; Recommended # 1.1 [38] CJK UNIFIED IDEOGRAPH-8266..CJK UNIFIED IDEOGRAPH-828B +828D..82B1 ; Recommended # 1.1 [37] CJK UNIFIED IDEOGRAPH-828D..CJK UNIFIED IDEOGRAPH-82B1 +82B3..82E1 ; Recommended # 1.1 [47] CJK UNIFIED IDEOGRAPH-82B3..CJK UNIFIED IDEOGRAPH-82E1 +82E3..82FB ; Recommended # 1.1 [25] CJK UNIFIED IDEOGRAPH-82E3..CJK UNIFIED IDEOGRAPH-82FB +82FD..8309 ; Recommended # 1.1 [13] CJK UNIFIED IDEOGRAPH-82FD..CJK UNIFIED IDEOGRAPH-8309 +830B..830F ; Recommended # 1.1 [5] CJK UNIFIED IDEOGRAPH-830B..CJK UNIFIED IDEOGRAPH-830F +8311..832F ; Recommended # 1.1 [31] CJK UNIFIED IDEOGRAPH-8311..CJK UNIFIED IDEOGRAPH-832F +8331..8354 ; Recommended # 1.1 [36] CJK UNIFIED IDEOGRAPH-8331..CJK UNIFIED IDEOGRAPH-8354 +8356..83BD ; Recommended # 1.1 [104] CJK UNIFIED IDEOGRAPH-8356..CJK UNIFIED IDEOGRAPH-83BD +83BF..83E5 ; Recommended # 1.1 [39] CJK UNIFIED IDEOGRAPH-83BF..CJK UNIFIED IDEOGRAPH-83E5 +83E7..83EC ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-83E7..CJK UNIFIED IDEOGRAPH-83EC +83EE..8413 ; Recommended # 1.1 [38] CJK UNIFIED IDEOGRAPH-83EE..CJK UNIFIED IDEOGRAPH-8413 +8415 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-8415 +8418..841E ; Recommended # 1.1 [7] CJK UNIFIED IDEOGRAPH-8418..CJK UNIFIED IDEOGRAPH-841E +8420..8457 ; Recommended # 1.1 [56] CJK UNIFIED IDEOGRAPH-8420..CJK UNIFIED IDEOGRAPH-8457 +8459..8482 ; Recommended # 1.1 [42] CJK UNIFIED IDEOGRAPH-8459..CJK UNIFIED IDEOGRAPH-8482 +8484..8494 ; Recommended # 1.1 [17] CJK UNIFIED IDEOGRAPH-8484..CJK UNIFIED IDEOGRAPH-8494 +8496..84B6 ; Recommended # 1.1 [33] CJK UNIFIED IDEOGRAPH-8496..CJK UNIFIED IDEOGRAPH-84B6 +84B8..84C2 ; Recommended # 1.1 [11] CJK UNIFIED IDEOGRAPH-84B8..CJK UNIFIED IDEOGRAPH-84C2 +84C4..84EC ; Recommended # 1.1 [41] CJK UNIFIED IDEOGRAPH-84C4..CJK UNIFIED IDEOGRAPH-84EC +84EE..8504 ; Recommended # 1.1 [23] CJK UNIFIED IDEOGRAPH-84EE..CJK UNIFIED IDEOGRAPH-8504 +8506..850F ; Recommended # 1.1 [10] CJK UNIFIED IDEOGRAPH-8506..CJK UNIFIED IDEOGRAPH-850F +8511..8531 ; Recommended # 1.1 [33] CJK UNIFIED IDEOGRAPH-8511..CJK UNIFIED IDEOGRAPH-8531 +8534..854B ; Recommended # 1.1 [24] CJK UNIFIED IDEOGRAPH-8534..CJK UNIFIED IDEOGRAPH-854B +854D..854F ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-854D..CJK UNIFIED IDEOGRAPH-854F +8551..857E ; Recommended # 1.1 [46] CJK UNIFIED IDEOGRAPH-8551..CJK UNIFIED IDEOGRAPH-857E +8580..8592 ; Recommended # 1.1 [19] CJK UNIFIED IDEOGRAPH-8580..CJK UNIFIED IDEOGRAPH-8592 +8594..85B1 ; Recommended # 1.1 [30] CJK UNIFIED IDEOGRAPH-8594..CJK UNIFIED IDEOGRAPH-85B1 +85B3..85BA ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-85B3..CJK UNIFIED IDEOGRAPH-85BA +85BC..85CB ; Recommended # 1.1 [16] CJK UNIFIED IDEOGRAPH-85BC..CJK UNIFIED IDEOGRAPH-85CB +85CD..85ED ; Recommended # 1.1 [33] CJK UNIFIED IDEOGRAPH-85CD..CJK UNIFIED IDEOGRAPH-85ED +85EF..85F2 ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-85EF..CJK UNIFIED IDEOGRAPH-85F2 +85F4..85FB ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-85F4..CJK UNIFIED IDEOGRAPH-85FB +85FD..8602 ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-85FD..CJK UNIFIED IDEOGRAPH-8602 +8604..860C ; Recommended # 1.1 [9] CJK UNIFIED IDEOGRAPH-8604..CJK UNIFIED IDEOGRAPH-860C +860F ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-860F +8611..8614 ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-8611..CJK UNIFIED IDEOGRAPH-8614 +8616..861C ; Recommended # 1.1 [7] CJK UNIFIED IDEOGRAPH-8616..CJK UNIFIED IDEOGRAPH-861C +861E..8636 ; Recommended # 1.1 [25] CJK UNIFIED IDEOGRAPH-861E..CJK UNIFIED IDEOGRAPH-8636 +8638..8656 ; Recommended # 1.1 [31] CJK UNIFIED IDEOGRAPH-8638..CJK UNIFIED IDEOGRAPH-8656 +8658..8674 ; Recommended # 1.1 [29] CJK UNIFIED IDEOGRAPH-8658..CJK UNIFIED IDEOGRAPH-8674 +8676..8688 ; Recommended # 1.1 [19] CJK UNIFIED IDEOGRAPH-8676..CJK UNIFIED IDEOGRAPH-8688 +868A..8691 ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-868A..CJK UNIFIED IDEOGRAPH-8691 +8693..869F ; Recommended # 1.1 [13] CJK UNIFIED IDEOGRAPH-8693..CJK UNIFIED IDEOGRAPH-869F +86A1..86A5 ; Recommended # 1.1 [5] CJK UNIFIED IDEOGRAPH-86A1..CJK UNIFIED IDEOGRAPH-86A5 +86A7..86D4 ; Recommended # 1.1 [46] CJK UNIFIED IDEOGRAPH-86A7..CJK UNIFIED IDEOGRAPH-86D4 +86D6..86DF ; Recommended # 1.1 [10] CJK UNIFIED IDEOGRAPH-86D6..CJK UNIFIED IDEOGRAPH-86DF +86E1..86E6 ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-86E1..CJK UNIFIED IDEOGRAPH-86E6 +86E8..86FC ; Recommended # 1.1 [21] CJK UNIFIED IDEOGRAPH-86E8..CJK UNIFIED IDEOGRAPH-86FC +86FE..871C ; Recommended # 1.1 [31] CJK UNIFIED IDEOGRAPH-86FE..CJK UNIFIED IDEOGRAPH-871C +871E..872E ; Recommended # 1.1 [17] CJK UNIFIED IDEOGRAPH-871E..CJK UNIFIED IDEOGRAPH-872E +8730..873C ; Recommended # 1.1 [13] CJK UNIFIED IDEOGRAPH-8730..CJK UNIFIED IDEOGRAPH-873C +873E..8744 ; Recommended # 1.1 [7] CJK UNIFIED IDEOGRAPH-873E..CJK UNIFIED IDEOGRAPH-8744 +8746..8770 ; Recommended # 1.1 [43] CJK UNIFIED IDEOGRAPH-8746..CJK UNIFIED IDEOGRAPH-8770 +8772..878D ; Recommended # 1.1 [28] CJK UNIFIED IDEOGRAPH-8772..CJK UNIFIED IDEOGRAPH-878D +878F..8798 ; Recommended # 1.1 [10] CJK UNIFIED IDEOGRAPH-878F..CJK UNIFIED IDEOGRAPH-8798 +879A..87D9 ; Recommended # 1.1 [64] CJK UNIFIED IDEOGRAPH-879A..CJK UNIFIED IDEOGRAPH-87D9 +87DB..87EF ; Recommended # 1.1 [21] CJK UNIFIED IDEOGRAPH-87DB..CJK UNIFIED IDEOGRAPH-87EF +87F1..8806 ; Recommended # 1.1 [22] CJK UNIFIED IDEOGRAPH-87F1..CJK UNIFIED IDEOGRAPH-8806 +8808..8811 ; Recommended # 1.1 [10] CJK UNIFIED IDEOGRAPH-8808..CJK UNIFIED IDEOGRAPH-8811 +8813..882C ; Recommended # 1.1 [26] CJK UNIFIED IDEOGRAPH-8813..CJK UNIFIED IDEOGRAPH-882C +882E..8839 ; Recommended # 1.1 [12] CJK UNIFIED IDEOGRAPH-882E..CJK UNIFIED IDEOGRAPH-8839 +883B..8846 ; Recommended # 1.1 [12] CJK UNIFIED IDEOGRAPH-883B..CJK UNIFIED IDEOGRAPH-8846 +8848..8857 ; Recommended # 1.1 [16] CJK UNIFIED IDEOGRAPH-8848..CJK UNIFIED IDEOGRAPH-8857 +8859..885B ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-8859..CJK UNIFIED IDEOGRAPH-885B +885D..885E ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-885D..CJK UNIFIED IDEOGRAPH-885E +8860..8879 ; Recommended # 1.1 [26] CJK UNIFIED IDEOGRAPH-8860..CJK UNIFIED IDEOGRAPH-8879 +887B..88E5 ; Recommended # 1.1 [107] CJK UNIFIED IDEOGRAPH-887B..CJK UNIFIED IDEOGRAPH-88E5 +88E7..88E8 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-88E7..CJK UNIFIED IDEOGRAPH-88E8 +88EA..88EC ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-88EA..CJK UNIFIED IDEOGRAPH-88EC +88EE..8902 ; Recommended # 1.1 [21] CJK UNIFIED IDEOGRAPH-88EE..CJK UNIFIED IDEOGRAPH-8902 +8904..890E ; Recommended # 1.1 [11] CJK UNIFIED IDEOGRAPH-8904..CJK UNIFIED IDEOGRAPH-890E +8910..8923 ; Recommended # 1.1 [20] CJK UNIFIED IDEOGRAPH-8910..CJK UNIFIED IDEOGRAPH-8923 +8925..8964 ; Recommended # 1.1 [64] CJK UNIFIED IDEOGRAPH-8925..CJK UNIFIED IDEOGRAPH-8964 +8966..8974 ; Recommended # 1.1 [15] CJK UNIFIED IDEOGRAPH-8966..CJK UNIFIED IDEOGRAPH-8974 +8976..897C ; Recommended # 1.1 [7] CJK UNIFIED IDEOGRAPH-8976..CJK UNIFIED IDEOGRAPH-897C +897E..898C ; Recommended # 1.1 [15] CJK UNIFIED IDEOGRAPH-897E..CJK UNIFIED IDEOGRAPH-898C +898E..898F ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-898E..CJK UNIFIED IDEOGRAPH-898F +8991..8993 ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-8991..CJK UNIFIED IDEOGRAPH-8993 +8995..8998 ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-8995..CJK UNIFIED IDEOGRAPH-8998 +899A..89AF ; Recommended # 1.1 [22] CJK UNIFIED IDEOGRAPH-899A..CJK UNIFIED IDEOGRAPH-89AF +89B1..89B3 ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-89B1..CJK UNIFIED IDEOGRAPH-89B3 +89B5..89BA ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-89B5..CJK UNIFIED IDEOGRAPH-89BA +89BD..89ED ; Recommended # 1.1 [49] CJK UNIFIED IDEOGRAPH-89BD..CJK UNIFIED IDEOGRAPH-89ED +89EF..89F4 ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-89EF..CJK UNIFIED IDEOGRAPH-89F4 +89F6..89F8 ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-89F6..CJK UNIFIED IDEOGRAPH-89F8 +89FA..89FC ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-89FA..CJK UNIFIED IDEOGRAPH-89FC +89FE..8A04 ; Recommended # 1.1 [7] CJK UNIFIED IDEOGRAPH-89FE..CJK UNIFIED IDEOGRAPH-8A04 +8A07..8A13 ; Recommended # 1.1 [13] CJK UNIFIED IDEOGRAPH-8A07..CJK UNIFIED IDEOGRAPH-8A13 +8A15..8A18 ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-8A15..CJK UNIFIED IDEOGRAPH-8A18 +8A1A..8A1F ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-8A1A..CJK UNIFIED IDEOGRAPH-8A1F +8A22..8A2A ; Recommended # 1.1 [9] CJK UNIFIED IDEOGRAPH-8A22..CJK UNIFIED IDEOGRAPH-8A2A +8A2C..8A3C ; Recommended # 1.1 [17] CJK UNIFIED IDEOGRAPH-8A2C..CJK UNIFIED IDEOGRAPH-8A3C +8A3E..8A4A ; Recommended # 1.1 [13] CJK UNIFIED IDEOGRAPH-8A3E..CJK UNIFIED IDEOGRAPH-8A4A +8A4C..8A63 ; Recommended # 1.1 [24] CJK UNIFIED IDEOGRAPH-8A4C..CJK UNIFIED IDEOGRAPH-8A63 +8A65..8A77 ; Recommended # 1.1 [19] CJK UNIFIED IDEOGRAPH-8A65..CJK UNIFIED IDEOGRAPH-8A77 +8A79..8A7C ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-8A79..CJK UNIFIED IDEOGRAPH-8A7C +8A7E..8A87 ; Recommended # 1.1 [10] CJK UNIFIED IDEOGRAPH-8A7E..CJK UNIFIED IDEOGRAPH-8A87 +8A89..8A9E ; Recommended # 1.1 [22] CJK UNIFIED IDEOGRAPH-8A89..CJK UNIFIED IDEOGRAPH-8A9E +8AA0..8AAE ; Recommended # 1.1 [15] CJK UNIFIED IDEOGRAPH-8AA0..CJK UNIFIED IDEOGRAPH-8AAE +8AB0..8AB6 ; Recommended # 1.1 [7] CJK UNIFIED IDEOGRAPH-8AB0..CJK UNIFIED IDEOGRAPH-8AB6 +8AB8..8ACF ; Recommended # 1.1 [24] CJK UNIFIED IDEOGRAPH-8AB8..CJK UNIFIED IDEOGRAPH-8ACF +8AD1..8AEB ; Recommended # 1.1 [27] CJK UNIFIED IDEOGRAPH-8AD1..CJK UNIFIED IDEOGRAPH-8AEB +8AED..8B28 ; Recommended # 1.1 [60] CJK UNIFIED IDEOGRAPH-8AED..CJK UNIFIED IDEOGRAPH-8B28 +8B2A..8B31 ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-8B2A..CJK UNIFIED IDEOGRAPH-8B31 +8B33..8B37 ; Recommended # 1.1 [5] CJK UNIFIED IDEOGRAPH-8B33..CJK UNIFIED IDEOGRAPH-8B37 +8B39..8B3E ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-8B39..CJK UNIFIED IDEOGRAPH-8B3E +8B40..8B60 ; Recommended # 1.1 [33] CJK UNIFIED IDEOGRAPH-8B40..CJK UNIFIED IDEOGRAPH-8B60 +8B63..8B68 ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-8B63..CJK UNIFIED IDEOGRAPH-8B68 +8B6A..8B74 ; Recommended # 1.1 [11] CJK UNIFIED IDEOGRAPH-8B6A..CJK UNIFIED IDEOGRAPH-8B74 +8B76..8B7B ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-8B76..CJK UNIFIED IDEOGRAPH-8B7B +8B7D..8B80 ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-8B7D..CJK UNIFIED IDEOGRAPH-8B80 +8B82..8B86 ; Recommended # 1.1 [5] CJK UNIFIED IDEOGRAPH-8B82..CJK UNIFIED IDEOGRAPH-8B86 +8B88..8B8C ; Recommended # 1.1 [5] CJK UNIFIED IDEOGRAPH-8B88..CJK UNIFIED IDEOGRAPH-8B8C +8B8E ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-8B8E +8B90..8B9A ; Recommended # 1.1 [11] CJK UNIFIED IDEOGRAPH-8B90..CJK UNIFIED IDEOGRAPH-8B9A +8B9C..8C37 ; Recommended # 1.1 [156] CJK UNIFIED IDEOGRAPH-8B9C..CJK UNIFIED IDEOGRAPH-8C37 +8C39..8C3F ; Recommended # 1.1 [7] CJK UNIFIED IDEOGRAPH-8C39..CJK UNIFIED IDEOGRAPH-8C3F +8C41..8C43 ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-8C41..CJK UNIFIED IDEOGRAPH-8C43 +8C45..8C50 ; Recommended # 1.1 [12] CJK UNIFIED IDEOGRAPH-8C45..CJK UNIFIED IDEOGRAPH-8C50 +8C54..8C57 ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-8C54..CJK UNIFIED IDEOGRAPH-8C57 +8C59..8C73 ; Recommended # 1.1 [27] CJK UNIFIED IDEOGRAPH-8C59..CJK UNIFIED IDEOGRAPH-8C73 +8C75..8C7E ; Recommended # 1.1 [10] CJK UNIFIED IDEOGRAPH-8C75..CJK UNIFIED IDEOGRAPH-8C7E +8C80..8C82 ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-8C80..CJK UNIFIED IDEOGRAPH-8C82 +8C84..8C86 ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-8C84..CJK UNIFIED IDEOGRAPH-8C86 +8C88..8C8A ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-8C88..CJK UNIFIED IDEOGRAPH-8C8A +8C8C..8C9A ; Recommended # 1.1 [15] CJK UNIFIED IDEOGRAPH-8C8C..CJK UNIFIED IDEOGRAPH-8C9A +8C9C..8CA5 ; Recommended # 1.1 [10] CJK UNIFIED IDEOGRAPH-8C9C..CJK UNIFIED IDEOGRAPH-8CA5 +8CA7..8CCA ; Recommended # 1.1 [36] CJK UNIFIED IDEOGRAPH-8CA7..CJK UNIFIED IDEOGRAPH-8CCA +8CCC..8CD5 ; Recommended # 1.1 [10] CJK UNIFIED IDEOGRAPH-8CCC..CJK UNIFIED IDEOGRAPH-8CD5 +8CD7 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-8CD7 +8CD9..8CE8 ; Recommended # 1.1 [16] CJK UNIFIED IDEOGRAPH-8CD9..CJK UNIFIED IDEOGRAPH-8CE8 +8CEA..8CF6 ; Recommended # 1.1 [13] CJK UNIFIED IDEOGRAPH-8CEA..CJK UNIFIED IDEOGRAPH-8CF6 +8CF8..8D00 ; Recommended # 1.1 [9] CJK UNIFIED IDEOGRAPH-8CF8..CJK UNIFIED IDEOGRAPH-8D00 +8D02..8D10 ; Recommended # 1.1 [15] CJK UNIFIED IDEOGRAPH-8D02..CJK UNIFIED IDEOGRAPH-8D10 +8D13..8D7B ; Recommended # 1.1 [105] CJK UNIFIED IDEOGRAPH-8D13..CJK UNIFIED IDEOGRAPH-8D7B +8D7D..8DA5 ; Recommended # 1.1 [41] CJK UNIFIED IDEOGRAPH-8D7D..CJK UNIFIED IDEOGRAPH-8DA5 +8DA7..8DBF ; Recommended # 1.1 [25] CJK UNIFIED IDEOGRAPH-8DA7..CJK UNIFIED IDEOGRAPH-8DBF +8DC1..8DE4 ; Recommended # 1.1 [36] CJK UNIFIED IDEOGRAPH-8DC1..CJK UNIFIED IDEOGRAPH-8DE4 +8DE6..8E00 ; Recommended # 1.1 [27] CJK UNIFIED IDEOGRAPH-8DE6..CJK UNIFIED IDEOGRAPH-8E00 +8E02..8E0A ; Recommended # 1.1 [9] CJK UNIFIED IDEOGRAPH-8E02..CJK UNIFIED IDEOGRAPH-8E0A +8E0C..8E31 ; Recommended # 1.1 [38] CJK UNIFIED IDEOGRAPH-8E0C..CJK UNIFIED IDEOGRAPH-8E31 +8E33..8E45 ; Recommended # 1.1 [19] CJK UNIFIED IDEOGRAPH-8E33..CJK UNIFIED IDEOGRAPH-8E45 +8E47..8E4E ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-8E47..CJK UNIFIED IDEOGRAPH-8E4E +8E50..8E6D ; Recommended # 1.1 [30] CJK UNIFIED IDEOGRAPH-8E50..CJK UNIFIED IDEOGRAPH-8E6D +8E6F..8E74 ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-8E6F..CJK UNIFIED IDEOGRAPH-8E74 +8E76 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-8E76 +8E78 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-8E78 +8E7A..8E9A ; Recommended # 1.1 [33] CJK UNIFIED IDEOGRAPH-8E7A..CJK UNIFIED IDEOGRAPH-8E9A +8E9C..8EA1 ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-8E9C..CJK UNIFIED IDEOGRAPH-8EA1 +8EA3..8EB2 ; Recommended # 1.1 [16] CJK UNIFIED IDEOGRAPH-8EA3..CJK UNIFIED IDEOGRAPH-8EB2 +8EB4..8EB5 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-8EB4..CJK UNIFIED IDEOGRAPH-8EB5 +8EB8..8EC0 ; Recommended # 1.1 [9] CJK UNIFIED IDEOGRAPH-8EB8..CJK UNIFIED IDEOGRAPH-8EC0 +8EC2..8EC3 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-8EC2..CJK UNIFIED IDEOGRAPH-8EC3 +8EC5..8ED8 ; Recommended # 1.1 [20] CJK UNIFIED IDEOGRAPH-8EC5..CJK UNIFIED IDEOGRAPH-8ED8 +8EDA..8EEF ; Recommended # 1.1 [22] CJK UNIFIED IDEOGRAPH-8EDA..CJK UNIFIED IDEOGRAPH-8EEF +8EF1..8F0E ; Recommended # 1.1 [30] CJK UNIFIED IDEOGRAPH-8EF1..CJK UNIFIED IDEOGRAPH-8F0E +8F10..8F2C ; Recommended # 1.1 [29] CJK UNIFIED IDEOGRAPH-8F10..CJK UNIFIED IDEOGRAPH-8F2C +8F2E..8F39 ; Recommended # 1.1 [12] CJK UNIFIED IDEOGRAPH-8F2E..CJK UNIFIED IDEOGRAPH-8F39 +8F3B..8F40 ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-8F3B..CJK UNIFIED IDEOGRAPH-8F40 +8F42..8F9C ; Recommended # 1.1 [91] CJK UNIFIED IDEOGRAPH-8F42..CJK UNIFIED IDEOGRAPH-8F9C +8F9E..8FA3 ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-8F9E..CJK UNIFIED IDEOGRAPH-8FA3 +8FA5..8FB2 ; Recommended # 1.1 [14] CJK UNIFIED IDEOGRAPH-8FA5..CJK UNIFIED IDEOGRAPH-8FB2 +8FB4..8FC2 ; Recommended # 1.1 [15] CJK UNIFIED IDEOGRAPH-8FB4..CJK UNIFIED IDEOGRAPH-8FC2 +8FC4..8FC9 ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-8FC4..CJK UNIFIED IDEOGRAPH-8FC9 +8FCB..8FE6 ; Recommended # 1.1 [28] CJK UNIFIED IDEOGRAPH-8FCB..CJK UNIFIED IDEOGRAPH-8FE6 +8FE8..9029 ; Recommended # 1.1 [66] CJK UNIFIED IDEOGRAPH-8FE8..CJK UNIFIED IDEOGRAPH-9029 +902B ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-902B +902D..9036 ; Recommended # 1.1 [10] CJK UNIFIED IDEOGRAPH-902D..CJK UNIFIED IDEOGRAPH-9036 +9038..903F ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-9038..CJK UNIFIED IDEOGRAPH-903F +9041..9045 ; Recommended # 1.1 [5] CJK UNIFIED IDEOGRAPH-9041..CJK UNIFIED IDEOGRAPH-9045 +9047..90AA ; Recommended # 1.1 [100] CJK UNIFIED IDEOGRAPH-9047..CJK UNIFIED IDEOGRAPH-90AA +90AC..90CB ; Recommended # 1.1 [32] CJK UNIFIED IDEOGRAPH-90AC..CJK UNIFIED IDEOGRAPH-90CB +90CE..90D1 ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-90CE..CJK UNIFIED IDEOGRAPH-90D1 +90D3..90F5 ; Recommended # 1.1 [35] CJK UNIFIED IDEOGRAPH-90D3..CJK UNIFIED IDEOGRAPH-90F5 +90F7..9109 ; Recommended # 1.1 [19] CJK UNIFIED IDEOGRAPH-90F7..CJK UNIFIED IDEOGRAPH-9109 +910B..913B ; Recommended # 1.1 [49] CJK UNIFIED IDEOGRAPH-910B..CJK UNIFIED IDEOGRAPH-913B +913E..9158 ; Recommended # 1.1 [27] CJK UNIFIED IDEOGRAPH-913E..CJK UNIFIED IDEOGRAPH-9158 +915A..917A ; Recommended # 1.1 [33] CJK UNIFIED IDEOGRAPH-915A..CJK UNIFIED IDEOGRAPH-917A +917C..9194 ; Recommended # 1.1 [25] CJK UNIFIED IDEOGRAPH-917C..CJK UNIFIED IDEOGRAPH-9194 +9196..9197 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-9196..CJK UNIFIED IDEOGRAPH-9197 +9199..91A8 ; Recommended # 1.1 [16] CJK UNIFIED IDEOGRAPH-9199..CJK UNIFIED IDEOGRAPH-91A8 +91AA..91BE ; Recommended # 1.1 [21] CJK UNIFIED IDEOGRAPH-91AA..CJK UNIFIED IDEOGRAPH-91BE +91C0..91C3 ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-91C0..CJK UNIFIED IDEOGRAPH-91C3 +91C5..91DF ; Recommended # 1.1 [27] CJK UNIFIED IDEOGRAPH-91C5..CJK UNIFIED IDEOGRAPH-91DF +91E1..91EE ; Recommended # 1.1 [14] CJK UNIFIED IDEOGRAPH-91E1..CJK UNIFIED IDEOGRAPH-91EE +91F0..9212 ; Recommended # 1.1 [35] CJK UNIFIED IDEOGRAPH-91F0..CJK UNIFIED IDEOGRAPH-9212 +9214..921E ; Recommended # 1.1 [11] CJK UNIFIED IDEOGRAPH-9214..CJK UNIFIED IDEOGRAPH-921E +9220..9221 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-9220..CJK UNIFIED IDEOGRAPH-9221 +9223..9242 ; Recommended # 1.1 [32] CJK UNIFIED IDEOGRAPH-9223..CJK UNIFIED IDEOGRAPH-9242 +9244..9268 ; Recommended # 1.1 [37] CJK UNIFIED IDEOGRAPH-9244..CJK UNIFIED IDEOGRAPH-9268 +926B..9280 ; Recommended # 1.1 [22] CJK UNIFIED IDEOGRAPH-926B..CJK UNIFIED IDEOGRAPH-9280 +9282..9283 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-9282..CJK UNIFIED IDEOGRAPH-9283 +9285..929D ; Recommended # 1.1 [25] CJK UNIFIED IDEOGRAPH-9285..CJK UNIFIED IDEOGRAPH-929D +929F..92BC ; Recommended # 1.1 [30] CJK UNIFIED IDEOGRAPH-929F..CJK UNIFIED IDEOGRAPH-92BC +92BE..92D3 ; Recommended # 1.1 [22] CJK UNIFIED IDEOGRAPH-92BE..CJK UNIFIED IDEOGRAPH-92D3 +92D5..92DA ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-92D5..CJK UNIFIED IDEOGRAPH-92DA +92DC..92E1 ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-92DC..CJK UNIFIED IDEOGRAPH-92E1 +92E3..931B ; Recommended # 1.1 [57] CJK UNIFIED IDEOGRAPH-92E3..CJK UNIFIED IDEOGRAPH-931B +931D..932F ; Recommended # 1.1 [19] CJK UNIFIED IDEOGRAPH-931D..CJK UNIFIED IDEOGRAPH-932F +9332..9361 ; Recommended # 1.1 [48] CJK UNIFIED IDEOGRAPH-9332..CJK UNIFIED IDEOGRAPH-9361 +9363..9367 ; Recommended # 1.1 [5] CJK UNIFIED IDEOGRAPH-9363..CJK UNIFIED IDEOGRAPH-9367 +9369..936A ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-9369..CJK UNIFIED IDEOGRAPH-936A +936C..936E ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-936C..CJK UNIFIED IDEOGRAPH-936E +9370..9372 ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-9370..CJK UNIFIED IDEOGRAPH-9372 +9374..9377 ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-9374..CJK UNIFIED IDEOGRAPH-9377 +9379..937E ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-9379..CJK UNIFIED IDEOGRAPH-937E +9380 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-9380 +9382..938A ; Recommended # 1.1 [9] CJK UNIFIED IDEOGRAPH-9382..CJK UNIFIED IDEOGRAPH-938A +938C..939B ; Recommended # 1.1 [16] CJK UNIFIED IDEOGRAPH-938C..CJK UNIFIED IDEOGRAPH-939B +939D..939F ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-939D..CJK UNIFIED IDEOGRAPH-939F +93A1..93AA ; Recommended # 1.1 [10] CJK UNIFIED IDEOGRAPH-93A1..CJK UNIFIED IDEOGRAPH-93AA +93AC..93BA ; Recommended # 1.1 [15] CJK UNIFIED IDEOGRAPH-93AC..CJK UNIFIED IDEOGRAPH-93BA +93BC..93DF ; Recommended # 1.1 [36] CJK UNIFIED IDEOGRAPH-93BC..CJK UNIFIED IDEOGRAPH-93DF +93E1..93F2 ; Recommended # 1.1 [18] CJK UNIFIED IDEOGRAPH-93E1..CJK UNIFIED IDEOGRAPH-93F2 +93F4..9401 ; Recommended # 1.1 [14] CJK UNIFIED IDEOGRAPH-93F4..CJK UNIFIED IDEOGRAPH-9401 +9403..9416 ; Recommended # 1.1 [20] CJK UNIFIED IDEOGRAPH-9403..CJK UNIFIED IDEOGRAPH-9416 +9418..941B ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-9418..CJK UNIFIED IDEOGRAPH-941B +941D ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-941D +9420..9423 ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-9420..CJK UNIFIED IDEOGRAPH-9423 +9425..9442 ; Recommended # 1.1 [30] CJK UNIFIED IDEOGRAPH-9425..CJK UNIFIED IDEOGRAPH-9442 +9444..944D ; Recommended # 1.1 [10] CJK UNIFIED IDEOGRAPH-9444..CJK UNIFIED IDEOGRAPH-944D +944F..946B ; Recommended # 1.1 [29] CJK UNIFIED IDEOGRAPH-944F..CJK UNIFIED IDEOGRAPH-946B +946D..947A ; Recommended # 1.1 [14] CJK UNIFIED IDEOGRAPH-946D..CJK UNIFIED IDEOGRAPH-947A +947C..9577 ; Recommended # 1.1 [252] CJK UNIFIED IDEOGRAPH-947C..CJK UNIFIED IDEOGRAPH-9577 +957A..957D ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-957A..CJK UNIFIED IDEOGRAPH-957D +957F..9584 ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-957F..CJK UNIFIED IDEOGRAPH-9584 +9586..9596 ; Recommended # 1.1 [17] CJK UNIFIED IDEOGRAPH-9586..CJK UNIFIED IDEOGRAPH-9596 +9598..95B2 ; Recommended # 1.1 [27] CJK UNIFIED IDEOGRAPH-9598..CJK UNIFIED IDEOGRAPH-95B2 +95B5..95B7 ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-95B5..CJK UNIFIED IDEOGRAPH-95B7 +95B9..95C0 ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-95B9..CJK UNIFIED IDEOGRAPH-95C0 +95C2..95D8 ; Recommended # 1.1 [23] CJK UNIFIED IDEOGRAPH-95C2..CJK UNIFIED IDEOGRAPH-95D8 +95DA..95DC ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-95DA..CJK UNIFIED IDEOGRAPH-95DC +95DE..9624 ; Recommended # 1.1 [71] CJK UNIFIED IDEOGRAPH-95DE..CJK UNIFIED IDEOGRAPH-9624 +9627..9628 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-9627..CJK UNIFIED IDEOGRAPH-9628 +962A..963D ; Recommended # 1.1 [20] CJK UNIFIED IDEOGRAPH-962A..CJK UNIFIED IDEOGRAPH-963D +963F..9655 ; Recommended # 1.1 [23] CJK UNIFIED IDEOGRAPH-963F..CJK UNIFIED IDEOGRAPH-9655 +9658..9678 ; Recommended # 1.1 [33] CJK UNIFIED IDEOGRAPH-9658..CJK UNIFIED IDEOGRAPH-9678 +967A ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-967A +967C..967E ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-967C..CJK UNIFIED IDEOGRAPH-967E +9680 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-9680 +9683..968B ; Recommended # 1.1 [9] CJK UNIFIED IDEOGRAPH-9683..CJK UNIFIED IDEOGRAPH-968B +968D..9695 ; Recommended # 1.1 [9] CJK UNIFIED IDEOGRAPH-968D..CJK UNIFIED IDEOGRAPH-9695 +9697..9699 ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-9697..CJK UNIFIED IDEOGRAPH-9699 +969B..969C ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-969B..CJK UNIFIED IDEOGRAPH-969C +969E ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-969E +96A0..96AA ; Recommended # 1.1 [11] CJK UNIFIED IDEOGRAPH-96A0..CJK UNIFIED IDEOGRAPH-96AA +96AC..96AE ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-96AC..CJK UNIFIED IDEOGRAPH-96AE +96B0..96B4 ; Recommended # 1.1 [5] CJK UNIFIED IDEOGRAPH-96B0..CJK UNIFIED IDEOGRAPH-96B4 +96B6..96E3 ; Recommended # 1.1 [46] CJK UNIFIED IDEOGRAPH-96B6..CJK UNIFIED IDEOGRAPH-96E3 +96E5 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-96E5 +96E8..96FB ; Recommended # 1.1 [20] CJK UNIFIED IDEOGRAPH-96E8..CJK UNIFIED IDEOGRAPH-96FB +96FD..9713 ; Recommended # 1.1 [23] CJK UNIFIED IDEOGRAPH-96FD..CJK UNIFIED IDEOGRAPH-9713 +9715..9716 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-9715..CJK UNIFIED IDEOGRAPH-9716 +9718..9719 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-9718..CJK UNIFIED IDEOGRAPH-9719 +971C..9732 ; Recommended # 1.1 [23] CJK UNIFIED IDEOGRAPH-971C..CJK UNIFIED IDEOGRAPH-9732 +9735..9736 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-9735..CJK UNIFIED IDEOGRAPH-9736 +9738..973F ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-9738..CJK UNIFIED IDEOGRAPH-973F +9742..974C ; Recommended # 1.1 [11] CJK UNIFIED IDEOGRAPH-9742..CJK UNIFIED IDEOGRAPH-974C +974E..9756 ; Recommended # 1.1 [9] CJK UNIFIED IDEOGRAPH-974E..CJK UNIFIED IDEOGRAPH-9756 +9758..9762 ; Recommended # 1.1 [11] CJK UNIFIED IDEOGRAPH-9758..CJK UNIFIED IDEOGRAPH-9762 +9764..9774 ; Recommended # 1.1 [17] CJK UNIFIED IDEOGRAPH-9764..CJK UNIFIED IDEOGRAPH-9774 +9776..9786 ; Recommended # 1.1 [17] CJK UNIFIED IDEOGRAPH-9776..CJK UNIFIED IDEOGRAPH-9786 +9788 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-9788 +978A..979A ; Recommended # 1.1 [17] CJK UNIFIED IDEOGRAPH-978A..CJK UNIFIED IDEOGRAPH-979A +979C..97A8 ; Recommended # 1.1 [13] CJK UNIFIED IDEOGRAPH-979C..CJK UNIFIED IDEOGRAPH-97A8 +97AA..97AF ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-97AA..CJK UNIFIED IDEOGRAPH-97AF +97B2..97B4 ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-97B2..CJK UNIFIED IDEOGRAPH-97B4 +97B6..97BD ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-97B6..CJK UNIFIED IDEOGRAPH-97BD +97BF ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-97BF +97C1..97D1 ; Recommended # 1.1 [17] CJK UNIFIED IDEOGRAPH-97C1..CJK UNIFIED IDEOGRAPH-97D1 +97D3..97FB ; Recommended # 1.1 [41] CJK UNIFIED IDEOGRAPH-97D3..CJK UNIFIED IDEOGRAPH-97FB +97FD..981E ; Recommended # 1.1 [34] CJK UNIFIED IDEOGRAPH-97FD..CJK UNIFIED IDEOGRAPH-981E +9820..9824 ; Recommended # 1.1 [5] CJK UNIFIED IDEOGRAPH-9820..CJK UNIFIED IDEOGRAPH-9824 +9826..9829 ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-9826..CJK UNIFIED IDEOGRAPH-9829 +982B..9832 ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-982B..CJK UNIFIED IDEOGRAPH-9832 +9834..9839 ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-9834..CJK UNIFIED IDEOGRAPH-9839 +983B..983D ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-983B..CJK UNIFIED IDEOGRAPH-983D +983F..9841 ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-983F..CJK UNIFIED IDEOGRAPH-9841 +9843..9846 ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-9843..CJK UNIFIED IDEOGRAPH-9846 +9848..9855 ; Recommended # 1.1 [14] CJK UNIFIED IDEOGRAPH-9848..CJK UNIFIED IDEOGRAPH-9855 +9857..9865 ; Recommended # 1.1 [15] CJK UNIFIED IDEOGRAPH-9857..CJK UNIFIED IDEOGRAPH-9865 +9867 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-9867 +9869..98B6 ; Recommended # 1.1 [78] CJK UNIFIED IDEOGRAPH-9869..CJK UNIFIED IDEOGRAPH-98B6 +98B8..98C9 ; Recommended # 1.1 [18] CJK UNIFIED IDEOGRAPH-98B8..CJK UNIFIED IDEOGRAPH-98C9 +98CB..98E3 ; Recommended # 1.1 [25] CJK UNIFIED IDEOGRAPH-98CB..CJK UNIFIED IDEOGRAPH-98E3 +98E5..98EB ; Recommended # 1.1 [7] CJK UNIFIED IDEOGRAPH-98E5..CJK UNIFIED IDEOGRAPH-98EB +98ED..98F0 ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-98ED..CJK UNIFIED IDEOGRAPH-98F0 +98F2..98F7 ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-98F2..CJK UNIFIED IDEOGRAPH-98F7 +98F9..98FA ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-98F9..CJK UNIFIED IDEOGRAPH-98FA +98FC..9918 ; Recommended # 1.1 [29] CJK UNIFIED IDEOGRAPH-98FC..CJK UNIFIED IDEOGRAPH-9918 +991A..993A ; Recommended # 1.1 [33] CJK UNIFIED IDEOGRAPH-991A..CJK UNIFIED IDEOGRAPH-993A +993C..9943 ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-993C..CJK UNIFIED IDEOGRAPH-9943 +9945..9959 ; Recommended # 1.1 [21] CJK UNIFIED IDEOGRAPH-9945..CJK UNIFIED IDEOGRAPH-9959 +995B..995C ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-995B..CJK UNIFIED IDEOGRAPH-995C +995E..99BE ; Recommended # 1.1 [97] CJK UNIFIED IDEOGRAPH-995E..CJK UNIFIED IDEOGRAPH-99BE +99C0..99DF ; Recommended # 1.1 [32] CJK UNIFIED IDEOGRAPH-99C0..CJK UNIFIED IDEOGRAPH-99DF +99E1..99E5 ; Recommended # 1.1 [5] CJK UNIFIED IDEOGRAPH-99E1..CJK UNIFIED IDEOGRAPH-99E5 +99E7..99EA ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-99E7..CJK UNIFIED IDEOGRAPH-99EA +99EC..99F4 ; Recommended # 1.1 [9] CJK UNIFIED IDEOGRAPH-99EC..CJK UNIFIED IDEOGRAPH-99F4 +99F6..9A0F ; Recommended # 1.1 [26] CJK UNIFIED IDEOGRAPH-99F6..CJK UNIFIED IDEOGRAPH-9A0F +9A11..9A16 ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-9A11..CJK UNIFIED IDEOGRAPH-9A16 +9A19..9A3A ; Recommended # 1.1 [34] CJK UNIFIED IDEOGRAPH-9A19..CJK UNIFIED IDEOGRAPH-9A3A +9A3C..9A50 ; Recommended # 1.1 [21] CJK UNIFIED IDEOGRAPH-9A3C..CJK UNIFIED IDEOGRAPH-9A50 +9A52..9A57 ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-9A52..CJK UNIFIED IDEOGRAPH-9A57 +9A59..9A5C ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-9A59..CJK UNIFIED IDEOGRAPH-9A5C +9A5E..9A62 ; Recommended # 1.1 [5] CJK UNIFIED IDEOGRAPH-9A5E..CJK UNIFIED IDEOGRAPH-9A62 +9A64..9AA8 ; Recommended # 1.1 [69] CJK UNIFIED IDEOGRAPH-9A64..CJK UNIFIED IDEOGRAPH-9AA8 +9AAA..9ABC ; Recommended # 1.1 [19] CJK UNIFIED IDEOGRAPH-9AAA..CJK UNIFIED IDEOGRAPH-9ABC +9ABE..9AC7 ; Recommended # 1.1 [10] CJK UNIFIED IDEOGRAPH-9ABE..CJK UNIFIED IDEOGRAPH-9AC7 +9AC9..9AD6 ; Recommended # 1.1 [14] CJK UNIFIED IDEOGRAPH-9AC9..CJK UNIFIED IDEOGRAPH-9AD6 +9AD8..9ADF ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-9AD8..CJK UNIFIED IDEOGRAPH-9ADF +9AE1..9AE3 ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-9AE1..CJK UNIFIED IDEOGRAPH-9AE3 +9AE5..9AE7 ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-9AE5..CJK UNIFIED IDEOGRAPH-9AE7 +9AEA..9AEF ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-9AEA..CJK UNIFIED IDEOGRAPH-9AEF +9AF1..9AFF ; Recommended # 1.1 [15] CJK UNIFIED IDEOGRAPH-9AF1..CJK UNIFIED IDEOGRAPH-9AFF +9B01 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-9B01 +9B03..9B08 ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-9B03..CJK UNIFIED IDEOGRAPH-9B08 +9B0A..9B13 ; Recommended # 1.1 [10] CJK UNIFIED IDEOGRAPH-9B0A..CJK UNIFIED IDEOGRAPH-9B13 +9B15..9B1A ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-9B15..CJK UNIFIED IDEOGRAPH-9B1A +9B1C..9B33 ; Recommended # 1.1 [24] CJK UNIFIED IDEOGRAPH-9B1C..CJK UNIFIED IDEOGRAPH-9B33 +9B35..9B3C ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-9B35..CJK UNIFIED IDEOGRAPH-9B3C +9B3E..9B3F ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-9B3E..CJK UNIFIED IDEOGRAPH-9B3F +9B41..9B4F ; Recommended # 1.1 [15] CJK UNIFIED IDEOGRAPH-9B41..CJK UNIFIED IDEOGRAPH-9B4F +9B51..9B56 ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-9B51..CJK UNIFIED IDEOGRAPH-9B56 +9B58..9B61 ; Recommended # 1.1 [10] CJK UNIFIED IDEOGRAPH-9B58..CJK UNIFIED IDEOGRAPH-9B61 +9B63..9B71 ; Recommended # 1.1 [15] CJK UNIFIED IDEOGRAPH-9B63..CJK UNIFIED IDEOGRAPH-9B71 +9B73..9B88 ; Recommended # 1.1 [22] CJK UNIFIED IDEOGRAPH-9B73..CJK UNIFIED IDEOGRAPH-9B88 +9B8A..9B8B ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-9B8A..CJK UNIFIED IDEOGRAPH-9B8B +9B8D..9B98 ; Recommended # 1.1 [12] CJK UNIFIED IDEOGRAPH-9B8D..CJK UNIFIED IDEOGRAPH-9B98 +9B9A..9BC1 ; Recommended # 1.1 [40] CJK UNIFIED IDEOGRAPH-9B9A..CJK UNIFIED IDEOGRAPH-9BC1 +9BC3..9BF5 ; Recommended # 1.1 [51] CJK UNIFIED IDEOGRAPH-9BC3..CJK UNIFIED IDEOGRAPH-9BF5 +9BF7..9BFF ; Recommended # 1.1 [9] CJK UNIFIED IDEOGRAPH-9BF7..CJK UNIFIED IDEOGRAPH-9BFF +9C02 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-9C02 +9C04..9C41 ; Recommended # 1.1 [62] CJK UNIFIED IDEOGRAPH-9C04..CJK UNIFIED IDEOGRAPH-9C41 +9C43..9C4E ; Recommended # 1.1 [12] CJK UNIFIED IDEOGRAPH-9C43..CJK UNIFIED IDEOGRAPH-9C4E +9C50 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-9C50 +9C52..9C60 ; Recommended # 1.1 [15] CJK UNIFIED IDEOGRAPH-9C52..CJK UNIFIED IDEOGRAPH-9C60 +9C62..9C63 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-9C62..CJK UNIFIED IDEOGRAPH-9C63 +9C65..9C7A ; Recommended # 1.1 [22] CJK UNIFIED IDEOGRAPH-9C65..CJK UNIFIED IDEOGRAPH-9C7A +9C7C..9D0B ; Recommended # 1.1 [144] CJK UNIFIED IDEOGRAPH-9C7C..CJK UNIFIED IDEOGRAPH-9D0B +9D0E..9D10 ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-9D0E..CJK UNIFIED IDEOGRAPH-9D10 +9D12..9D26 ; Recommended # 1.1 [21] CJK UNIFIED IDEOGRAPH-9D12..CJK UNIFIED IDEOGRAPH-9D26 +9D28..9D34 ; Recommended # 1.1 [13] CJK UNIFIED IDEOGRAPH-9D28..CJK UNIFIED IDEOGRAPH-9D34 +9D36..9D3B ; Recommended # 1.1 [6] CJK UNIFIED IDEOGRAPH-9D36..CJK UNIFIED IDEOGRAPH-9D3B +9D3D..9D6C ; Recommended # 1.1 [48] CJK UNIFIED IDEOGRAPH-9D3D..CJK UNIFIED IDEOGRAPH-9D6C +9D6E..9D94 ; Recommended # 1.1 [39] CJK UNIFIED IDEOGRAPH-9D6E..CJK UNIFIED IDEOGRAPH-9D94 +9D96..9DAD ; Recommended # 1.1 [24] CJK UNIFIED IDEOGRAPH-9D96..CJK UNIFIED IDEOGRAPH-9DAD +9DAF..9DBC ; Recommended # 1.1 [14] CJK UNIFIED IDEOGRAPH-9DAF..CJK UNIFIED IDEOGRAPH-9DBC +9DBE..9DBF ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-9DBE..CJK UNIFIED IDEOGRAPH-9DBF +9DC1..9DE9 ; Recommended # 1.1 [41] CJK UNIFIED IDEOGRAPH-9DC1..CJK UNIFIED IDEOGRAPH-9DE9 +9DEB..9DFB ; Recommended # 1.1 [17] CJK UNIFIED IDEOGRAPH-9DEB..CJK UNIFIED IDEOGRAPH-9DFB +9DFD..9E0D ; Recommended # 1.1 [17] CJK UNIFIED IDEOGRAPH-9DFD..CJK UNIFIED IDEOGRAPH-9E0D +9E0F..9E15 ; Recommended # 1.1 [7] CJK UNIFIED IDEOGRAPH-9E0F..CJK UNIFIED IDEOGRAPH-9E15 +9E17..9E1B ; Recommended # 1.1 [5] CJK UNIFIED IDEOGRAPH-9E17..CJK UNIFIED IDEOGRAPH-9E1B +9E1D..9E7A ; Recommended # 1.1 [94] CJK UNIFIED IDEOGRAPH-9E1D..CJK UNIFIED IDEOGRAPH-9E7A +9E7C..9E8E ; Recommended # 1.1 [19] CJK UNIFIED IDEOGRAPH-9E7C..CJK UNIFIED IDEOGRAPH-9E8E +9E91..9E97 ; Recommended # 1.1 [7] CJK UNIFIED IDEOGRAPH-9E91..CJK UNIFIED IDEOGRAPH-9E97 +9E99..9E9D ; Recommended # 1.1 [5] CJK UNIFIED IDEOGRAPH-9E99..CJK UNIFIED IDEOGRAPH-9E9D +9E9F..9EA1 ; Recommended # 1.1 [3] CJK UNIFIED IDEOGRAPH-9E9F..CJK UNIFIED IDEOGRAPH-9EA1 +9EA3..9EAA ; Recommended # 1.1 [8] CJK UNIFIED IDEOGRAPH-9EA3..CJK UNIFIED IDEOGRAPH-9EAA +9EAD..9EB0 ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-9EAD..CJK UNIFIED IDEOGRAPH-9EB0 +9EB2..9EEB ; Recommended # 1.1 [58] CJK UNIFIED IDEOGRAPH-9EB2..CJK UNIFIED IDEOGRAPH-9EEB +9EED..9EF0 ; Recommended # 1.1 [4] CJK UNIFIED IDEOGRAPH-9EED..CJK UNIFIED IDEOGRAPH-9EF0 +9EF2..9F02 ; Recommended # 1.1 [17] CJK UNIFIED IDEOGRAPH-9EF2..CJK UNIFIED IDEOGRAPH-9F02 +9F04..9F10 ; Recommended # 1.1 [13] CJK UNIFIED IDEOGRAPH-9F04..CJK UNIFIED IDEOGRAPH-9F10 +9F12..9F13 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-9F12..CJK UNIFIED IDEOGRAPH-9F13 +9F15..9F25 ; Recommended # 1.1 [17] CJK UNIFIED IDEOGRAPH-9F15..CJK UNIFIED IDEOGRAPH-9F25 +9F27..9F44 ; Recommended # 1.1 [30] CJK UNIFIED IDEOGRAPH-9F27..CJK UNIFIED IDEOGRAPH-9F44 +9F46..9F52 ; Recommended # 1.1 [13] CJK UNIFIED IDEOGRAPH-9F46..CJK UNIFIED IDEOGRAPH-9F52 +9F54..9F6C ; Recommended # 1.1 [25] CJK UNIFIED IDEOGRAPH-9F54..CJK UNIFIED IDEOGRAPH-9F6C +9F6E..9FA0 ; Recommended # 1.1 [51] CJK UNIFIED IDEOGRAPH-9F6E..CJK UNIFIED IDEOGRAPH-9FA0 +9FA2 ; Recommended # 1.1 CJK UNIFIED IDEOGRAPH-9FA2 +9FA4..9FA5 ; Recommended # 1.1 [2] CJK UNIFIED IDEOGRAPH-9FA4..CJK UNIFIED IDEOGRAPH-9FA5 +A78D ; Recommended # 6.0 LATIN CAPITAL LETTER TURNED H +A7AA ; Recommended # 6.1 LATIN CAPITAL LETTER H WITH HOOK +AA7B ; Recommended # 5.2 MYANMAR SIGN PAO KAREN TONE +AC00..D7A3 ; Recommended # 2.0 [11172] HANGUL SYLLABLE GA..HANGUL SYLLABLE HIH +11301 ; Recommended # 7.0 GRANTHA SIGN CANDRABINDU +11303 ; Recommended # 7.0 GRANTHA SIGN VISARGA +1133C ; Recommended # 7.0 GRANTHA SIGN NUKTA +1E7E0..1E7E6 ; Recommended # 14.0 [7] ETHIOPIC SYLLABLE HHYA..ETHIOPIC SYLLABLE HHYO +1E7E8..1E7EB ; Recommended # 14.0 [4] ETHIOPIC SYLLABLE GURAGE HHWA..ETHIOPIC SYLLABLE HHWE +1E7ED..1E7EE ; Recommended # 14.0 [2] ETHIOPIC SYLLABLE GURAGE MWI..ETHIOPIC SYLLABLE GURAGE MWEE +1E7F0..1E7FE ; Recommended # 14.0 [15] ETHIOPIC SYLLABLE GURAGE QWI..ETHIOPIC SYLLABLE GURAGE PWEE +2070E ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-2070E +20731 ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-20731 +20779 ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-20779 +20C53 ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-20C53 +20C78 ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-20C78 +20C96 ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-20C96 +20CCF ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-20CCF +20CD5 ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-20CD5 +20D15 ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-20D15 +20D7C ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-20D7C +20D7F ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-20D7F +20E0E..20E0F ; Recommended # 3.1 [2] CJK UNIFIED IDEOGRAPH-20E0E..CJK UNIFIED IDEOGRAPH-20E0F +20E77 ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-20E77 +20E9D ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-20E9D +20EA2 ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-20EA2 +20ED7 ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-20ED7 +20EF9..20EFA ; Recommended # 3.1 [2] CJK UNIFIED IDEOGRAPH-20EF9..CJK UNIFIED IDEOGRAPH-20EFA +20F2D..20F2E ; Recommended # 3.1 [2] CJK UNIFIED IDEOGRAPH-20F2D..CJK UNIFIED IDEOGRAPH-20F2E +20F4C ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-20F4C +20FB4 ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-20FB4 +20FBC ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-20FBC +20FEA ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-20FEA +2105C ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-2105C +2106F ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-2106F +21075..21076 ; Recommended # 3.1 [2] CJK UNIFIED IDEOGRAPH-21075..CJK UNIFIED IDEOGRAPH-21076 +2107B ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-2107B +210C1 ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-210C1 +210C9 ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-210C9 +211D9 ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-211D9 +220C7 ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-220C7 +227B5 ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-227B5 +22AD5 ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-22AD5 +22B43 ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-22B43 +22BCA ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-22BCA +22C51 ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-22C51 +22C55 ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-22C55 +22CC2 ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-22CC2 +22D08 ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-22D08 +22D4C ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-22D4C +22D67 ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-22D67 +22EB3 ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-22EB3 +23CB7 ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-23CB7 +244D3 ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-244D3 +24DB8 ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-24DB8 +24DEA ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-24DEA +2512B ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-2512B +26258 ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-26258 +267CC ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-267CC +269F2 ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-269F2 +269FA ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-269FA +27A3E ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-27A3E +2815D ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-2815D +28207 ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-28207 +282E2 ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-282E2 +28CCA ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-28CCA +28CCD ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-28CCD +28CD2 ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-28CD2 +29D98 ; Recommended # 3.1 CJK UNIFIED IDEOGRAPH-29D98 + +# Total code points: 33773 + +# Identifier_Type: Inclusion + +0027 ; Inclusion # 1.1 APOSTROPHE +002D..002E ; Inclusion # 1.1 [2] HYPHEN-MINUS..FULL STOP +003A ; Inclusion # 1.1 COLON +00B7 ; Inclusion # 1.1 MIDDLE DOT +02BB..02BC ; Inclusion # 1.1 [2] MODIFIER LETTER TURNED COMMA..MODIFIER LETTER APOSTROPHE +058A ; Inclusion # 3.0 ARMENIAN HYPHEN +05F3..05F4 ; Inclusion # 1.1 [2] HEBREW PUNCTUATION GERESH..HEBREW PUNCTUATION GERSHAYIM +06FD..06FE ; Inclusion # 3.0 [2] ARABIC SIGN SINDHI AMPERSAND..ARABIC SIGN SINDHI POSTPOSITION MEN +0F0B ; Inclusion # 2.0 TIBETAN MARK INTERSYLLABIC TSHEG +2010 ; Inclusion # 1.1 HYPHEN +2019 ; Inclusion # 1.1 RIGHT SINGLE QUOTATION MARK +2027 ; Inclusion # 1.1 HYPHENATION POINT +30A0 ; Inclusion # 3.2 KATAKANA-HIRAGANA DOUBLE HYPHEN +30FB ; Inclusion # 1.1 KATAKANA MIDDLE DOT + +# Total code points: 18 + +# Identifier_Type: Limited_Use + +0710..072C ; Limited_Use # 3.0 [29] SYRIAC LETTER ALAPH..SYRIAC LETTER TAW +072D..072F ; Limited_Use # 4.0 [3] SYRIAC LETTER PERSIAN BHETH..SYRIAC LETTER PERSIAN DHALATH +0730..073F ; Limited_Use # 3.0 [16] SYRIAC PTHAHA ABOVE..SYRIAC RWAHA +074D..074F ; Limited_Use # 4.0 [3] SYRIAC LETTER SOGDIAN ZHAIN..SYRIAC LETTER SOGDIAN FE +07C0..07E7 ; Limited_Use # 5.0 [40] NKO DIGIT ZERO..NKO LETTER NYA WOLOSO +07EB..07F5 ; Limited_Use # 5.0 [11] NKO COMBINING SHORT HIGH TONE..NKO LOW TONE APOSTROPHE +07FD ; Limited_Use # 11.0 NKO DANTAYALAN +0840..085B ; Limited_Use # 6.0 [28] MANDAIC LETTER HALQA..MANDAIC GEMINATION MARK +0860..086A ; Limited_Use # 10.0 [11] SYRIAC LETTER MALAYALAM NGA..SYRIAC LETTER MALAYALAM SSA +13A0..13F4 ; Limited_Use # 3.0 [85] CHEROKEE LETTER A..CHEROKEE LETTER YV +13F5 ; Limited_Use # 8.0 CHEROKEE LETTER MV +13F8..13FD ; Limited_Use # 8.0 [6] CHEROKEE SMALL LETTER YE..CHEROKEE SMALL LETTER MV +1401..166C ; Limited_Use # 3.0 [620] CANADIAN SYLLABICS E..CANADIAN SYLLABICS CARRIER TTSA +166F..1676 ; Limited_Use # 3.0 [8] CANADIAN SYLLABICS QAI..CANADIAN SYLLABICS NNGAA +1677..167F ; Limited_Use # 5.2 [9] CANADIAN SYLLABICS WOODS-CREE THWEE..CANADIAN SYLLABICS BLACKFOOT W +18B0..18F5 ; Limited_Use # 5.2 [70] CANADIAN SYLLABICS OY..CANADIAN SYLLABICS CARRIER DENTAL S +1900..191C ; Limited_Use # 4.0 [29] LIMBU VOWEL-CARRIER LETTER..LIMBU LETTER HA +191D..191E ; Limited_Use # 7.0 [2] LIMBU LETTER GYAN..LIMBU LETTER TRA +1920..192B ; Limited_Use # 4.0 [12] LIMBU VOWEL SIGN A..LIMBU SUBJOINED LETTER WA +1930..193B ; Limited_Use # 4.0 [12] LIMBU SMALL LETTER KA..LIMBU SIGN SA-I +1946..196D ; Limited_Use # 4.0 [40] LIMBU DIGIT ZERO..TAI LE LETTER AI +1970..1974 ; Limited_Use # 4.0 [5] TAI LE LETTER TONE-2..TAI LE LETTER TONE-6 +1980..19A9 ; Limited_Use # 4.1 [42] NEW TAI LUE LETTER HIGH QA..NEW TAI LUE LETTER LOW XVA +19AA..19AB ; Limited_Use # 5.2 [2] NEW TAI LUE LETTER HIGH SUA..NEW TAI LUE LETTER LOW SUA +19B0..19C9 ; Limited_Use # 4.1 [26] NEW TAI LUE VOWEL SIGN VOWEL SHORTENER..NEW TAI LUE TONE MARK-2 +19D0..19D9 ; Limited_Use # 4.1 [10] NEW TAI LUE DIGIT ZERO..NEW TAI LUE DIGIT NINE +19DA ; Limited_Use # 5.2 NEW TAI LUE THAM DIGIT ONE +1A20..1A5E ; Limited_Use # 5.2 [63] TAI THAM LETTER HIGH KA..TAI THAM CONSONANT SIGN SA +1A60..1A7C ; Limited_Use # 5.2 [29] TAI THAM SIGN SAKOT..TAI THAM SIGN KHUEN-LUE KARAN +1A7F..1A89 ; Limited_Use # 5.2 [11] TAI THAM COMBINING CRYPTOGRAMMIC DOT..TAI THAM HORA DIGIT NINE +1A90..1A99 ; Limited_Use # 5.2 [10] TAI THAM THAM DIGIT ZERO..TAI THAM THAM DIGIT NINE +1AA7 ; Limited_Use # 5.2 TAI THAM SIGN MAI YAMOK +1B00..1B4B ; Limited_Use # 5.0 [76] BALINESE SIGN ULU RICEM..BALINESE LETTER ASYURA SASAK +1B4C ; Limited_Use # 14.0 BALINESE LETTER ARCHAIC JNYA +1B50..1B59 ; Limited_Use # 5.0 [10] BALINESE DIGIT ZERO..BALINESE DIGIT NINE +1B80..1BAA ; Limited_Use # 5.1 [43] SUNDANESE SIGN PANYECEK..SUNDANESE SIGN PAMAAEH +1BAB..1BAD ; Limited_Use # 6.1 [3] SUNDANESE SIGN VIRAMA..SUNDANESE CONSONANT SIGN PASANGAN WA +1BAE..1BB9 ; Limited_Use # 5.1 [12] SUNDANESE LETTER KHA..SUNDANESE DIGIT NINE +1BBA..1BBF ; Limited_Use # 6.1 [6] SUNDANESE AVAGRAHA..SUNDANESE LETTER FINAL M +1BC0..1BF3 ; Limited_Use # 6.0 [52] BATAK LETTER A..BATAK PANONGONAN +1C00..1C37 ; Limited_Use # 5.1 [56] LEPCHA LETTER KA..LEPCHA SIGN NUKTA +1C40..1C49 ; Limited_Use # 5.1 [10] LEPCHA DIGIT ZERO..LEPCHA DIGIT NINE +1C4D..1C7D ; Limited_Use # 5.1 [49] LEPCHA LETTER TTA..OL CHIKI AHAD +2D30..2D65 ; Limited_Use # 4.1 [54] TIFINAGH LETTER YA..TIFINAGH LETTER YAZZ +2D66..2D67 ; Limited_Use # 6.1 [2] TIFINAGH LETTER YE..TIFINAGH LETTER YO +2D7F ; Limited_Use # 6.0 TIFINAGH CONSONANT JOINER +3105..312C ; Limited_Use # 1.1 [40] BOPOMOFO LETTER B..BOPOMOFO LETTER GN +312D ; Limited_Use # 5.1 BOPOMOFO LETTER IH +312F ; Limited_Use # 11.0 BOPOMOFO LETTER NN +31A0..31B7 ; Limited_Use # 3.0 [24] BOPOMOFO LETTER BU..BOPOMOFO FINAL LETTER H +31B8..31BA ; Limited_Use # 6.0 [3] BOPOMOFO LETTER GH..BOPOMOFO LETTER ZY +31BB..31BF ; Limited_Use # 13.0 [5] BOPOMOFO FINAL LETTER G..BOPOMOFO LETTER AH +A000..A48C ; Limited_Use # 3.0 [1165] YI SYLLABLE IT..YI SYLLABLE YYR +A4D0..A4FD ; Limited_Use # 5.2 [46] LISU LETTER BA..LISU LETTER TONE MYA JEU +A500..A60C ; Limited_Use # 5.1 [269] VAI SYLLABLE EE..VAI SYLLABLE LENGTHENER +A613..A629 ; Limited_Use # 5.1 [23] VAI SYMBOL FEENG..VAI DIGIT NINE +A6A0..A6F1 ; Limited_Use # 5.2 [82] BAMUM LETTER A..BAMUM COMBINING MARK TUKWENTIS +A800..A827 ; Limited_Use # 4.1 [40] SYLOTI NAGRI LETTER A..SYLOTI NAGRI VOWEL SIGN OO +A82C ; Limited_Use # 13.0 SYLOTI NAGRI SIGN ALTERNATE HASANTA +A880..A8C4 ; Limited_Use # 5.1 [69] SAURASHTRA SIGN ANUSVARA..SAURASHTRA SIGN VIRAMA +A8C5 ; Limited_Use # 9.0 SAURASHTRA SIGN CANDRABINDU +A8D0..A8D9 ; Limited_Use # 5.1 [10] SAURASHTRA DIGIT ZERO..SAURASHTRA DIGIT NINE +A900..A92D ; Limited_Use # 5.1 [46] KAYAH LI DIGIT ZERO..KAYAH LI TONE CALYA PLOPHU +A980..A9C0 ; Limited_Use # 5.2 [65] JAVANESE SIGN PANYANGGA..JAVANESE PANGKON +A9D0..A9D9 ; Limited_Use # 5.2 [10] JAVANESE DIGIT ZERO..JAVANESE DIGIT NINE +AA00..AA36 ; Limited_Use # 5.1 [55] CHAM LETTER A..CHAM CONSONANT SIGN WA +AA40..AA4D ; Limited_Use # 5.1 [14] CHAM LETTER FINAL K..CHAM CONSONANT SIGN FINAL H +AA50..AA59 ; Limited_Use # 5.1 [10] CHAM DIGIT ZERO..CHAM DIGIT NINE +AA80..AAC2 ; Limited_Use # 5.2 [67] TAI VIET LETTER LOW KO..TAI VIET TONE MAI SONG +AADB..AADD ; Limited_Use # 5.2 [3] TAI VIET SYMBOL KON..TAI VIET SYMBOL SAM +AAE0..AAEF ; Limited_Use # 6.1 [16] MEETEI MAYEK LETTER E..MEETEI MAYEK VOWEL SIGN AAU +AAF2..AAF6 ; Limited_Use # 6.1 [5] MEETEI MAYEK ANJI..MEETEI MAYEK VIRAMA +AB70..ABBF ; Limited_Use # 8.0 [80] CHEROKEE SMALL LETTER A..CHEROKEE SMALL LETTER YA +ABC0..ABEA ; Limited_Use # 5.2 [43] MEETEI MAYEK LETTER KOK..MEETEI MAYEK VOWEL SIGN NUNG +ABEC..ABED ; Limited_Use # 5.2 [2] MEETEI MAYEK LUM IYEK..MEETEI MAYEK APUN IYEK +ABF0..ABF9 ; Limited_Use # 5.2 [10] MEETEI MAYEK DIGIT ZERO..MEETEI MAYEK DIGIT NINE +104B0..104D3 ; Limited_Use # 9.0 [36] OSAGE CAPITAL LETTER A..OSAGE CAPITAL LETTER ZHA +104D8..104FB ; Limited_Use # 9.0 [36] OSAGE SMALL LETTER A..OSAGE SMALL LETTER ZHA +10D00..10D27 ; Limited_Use # 11.0 [40] HANIFI ROHINGYA LETTER A..HANIFI ROHINGYA SIGN TASSI +10D30..10D39 ; Limited_Use # 11.0 [10] HANIFI ROHINGYA DIGIT ZERO..HANIFI ROHINGYA DIGIT NINE +11100..11134 ; Limited_Use # 6.1 [53] CHAKMA SIGN CANDRABINDU..CHAKMA MAAYYAA +11136..1113F ; Limited_Use # 6.1 [10] CHAKMA DIGIT ZERO..CHAKMA DIGIT NINE +11144..11146 ; Limited_Use # 11.0 [3] CHAKMA LETTER LHAA..CHAKMA VOWEL SIGN EI +11147 ; Limited_Use # 13.0 CHAKMA LETTER VAA +11400..1144A ; Limited_Use # 9.0 [75] NEWA LETTER A..NEWA SIDDHI +11450..11459 ; Limited_Use # 9.0 [10] NEWA DIGIT ZERO..NEWA DIGIT NINE +1145E ; Limited_Use # 11.0 NEWA SANDHI MARK +1145F ; Limited_Use # 12.0 NEWA LETTER VEDIC ANUSVARA +11460..11461 ; Limited_Use # 13.0 [2] NEWA SIGN JIHVAMULIYA..NEWA SIGN UPADHMANIYA +11AB0..11ABF ; Limited_Use # 14.0 [16] CANADIAN SYLLABICS NATTILIK HI..CANADIAN SYLLABICS SPA +11FB0 ; Limited_Use # 13.0 LISU LETTER YHA +16800..16A38 ; Limited_Use # 6.0 [569] BAMUM LETTER PHASE-A NGKUE MFON..BAMUM LETTER PHASE-F VUEQ +16F00..16F44 ; Limited_Use # 6.1 [69] MIAO LETTER PA..MIAO LETTER HHA +16F45..16F4A ; Limited_Use # 12.0 [6] MIAO LETTER BRI..MIAO LETTER RTE +16F4F ; Limited_Use # 12.0 MIAO SIGN CONSONANT MODIFIER BAR +16F50..16F7E ; Limited_Use # 6.1 [47] MIAO LETTER NASALIZATION..MIAO VOWEL SIGN NG +16F7F..16F87 ; Limited_Use # 12.0 [9] MIAO VOWEL SIGN UOG..MIAO VOWEL SIGN UI +16F8F..16F9F ; Limited_Use # 6.1 [17] MIAO TONE RIGHT..MIAO LETTER REFORMED TONE-8 +1E100..1E12C ; Limited_Use # 12.0 [45] NYIAKENG PUACHUE HMONG LETTER MA..NYIAKENG PUACHUE HMONG LETTER W +1E130..1E13D ; Limited_Use # 12.0 [14] NYIAKENG PUACHUE HMONG TONE-B..NYIAKENG PUACHUE HMONG SYLLABLE LENGTHENER +1E140..1E149 ; Limited_Use # 12.0 [10] NYIAKENG PUACHUE HMONG DIGIT ZERO..NYIAKENG PUACHUE HMONG DIGIT NINE +1E14E ; Limited_Use # 12.0 NYIAKENG PUACHUE HMONG LOGOGRAM NYAJ +1E2C0..1E2F9 ; Limited_Use # 12.0 [58] WANCHO LETTER AA..WANCHO DIGIT NINE +1E900..1E94A ; Limited_Use # 9.0 [75] ADLAM CAPITAL LETTER ALIF..ADLAM NUKTA +1E94B ; Limited_Use # 12.0 ADLAM NASALIZATION MARK +1E950..1E959 ; Limited_Use # 9.0 [10] ADLAM DIGIT ZERO..ADLAM DIGIT NINE + +# Total code points: 5044 + +# Identifier_Type: Limited_Use Uncommon_Use + +A9CF ; Limited_Use Uncommon_Use # 5.2 JAVANESE PANGRANGKEP + +# Total code points: 1 + +# Identifier_Type: Limited_Use Technical + +0740..074A ; Limited_Use Technical # 3.0 [11] SYRIAC FEMININE DOT..SYRIAC BARREKH +1B6B..1B73 ; Limited_Use Technical # 5.0 [9] BALINESE MUSICAL SYMBOL COMBINING TEGEH..BALINESE MUSICAL SYMBOL COMBINING GONG +1DFA ; Limited_Use Technical # 14.0 COMBINING DOT BELOW LEFT + +# Total code points: 21 + +# Identifier_Type: Limited_Use Obsolete + +07E8..07EA ; Limited_Use Obsolete # 5.0 [3] NKO LETTER JONA JA..NKO LETTER JONA RA +07FA ; Limited_Use Obsolete # 5.0 NKO LAJANYALAN +312E ; Limited_Use Obsolete # 10.0 BOPOMOFO LETTER O WITH DOT ABOVE +A610..A612 ; Limited_Use Obsolete # 5.1 [3] VAI SYLLABLE NDOLE FA..VAI SYLLABLE NDOLE SOO +A62A..A62B ; Limited_Use Obsolete # 5.1 [2] VAI SYLLABLE NDOLE MA..VAI SYLLABLE NDOLE DO + +# Total code points: 10 + +# Identifier_Type: Limited_Use Not_XID + +02EA..02EB ; Limited_Use Not_XID # 3.0 [2] MODIFIER LETTER YIN DEPARTING TONE MARK..MODIFIER LETTER YANG DEPARTING TONE MARK +0700..070D ; Limited_Use Not_XID # 3.0 [14] SYRIAC END OF PARAGRAPH..SYRIAC HARKLEAN ASTERISCUS +070F ; Limited_Use Not_XID # 3.0 SYRIAC ABBREVIATION MARK +07F6..07F9 ; Limited_Use Not_XID # 5.0 [4] NKO SYMBOL OO DENNEN..NKO EXCLAMATION MARK +07FE..07FF ; Limited_Use Not_XID # 11.0 [2] NKO DOROME SIGN..NKO TAMAN SIGN +085E ; Limited_Use Not_XID # 6.0 MANDAIC PUNCTUATION +1400 ; Limited_Use Not_XID # 5.2 CANADIAN SYLLABICS HYPHEN +166D..166E ; Limited_Use Not_XID # 3.0 [2] CANADIAN SYLLABICS CHI SIGN..CANADIAN SYLLABICS FULL STOP +1940 ; Limited_Use Not_XID # 4.0 LIMBU SIGN LOO +1944..1945 ; Limited_Use Not_XID # 4.0 [2] LIMBU EXCLAMATION MARK..LIMBU QUESTION MARK +19DE..19DF ; Limited_Use Not_XID # 4.1 [2] NEW TAI LUE SIGN LAE..NEW TAI LUE SIGN LAEV +1AA0..1AA6 ; Limited_Use Not_XID # 5.2 [7] TAI THAM SIGN WIANG..TAI THAM SIGN REVERSED ROTATED RANA +1AA8..1AAD ; Limited_Use Not_XID # 5.2 [6] TAI THAM SIGN KAAN..TAI THAM SIGN CAANG +1B4E..1B4F ; Limited_Use Not_XID # 16.0 [2] BALINESE INVERTED CARIK SIKI..BALINESE INVERTED CARIK PAREREN +1B5A..1B6A ; Limited_Use Not_XID # 5.0 [17] BALINESE PANTI..BALINESE MUSICAL SYMBOL DANG GEDE +1B74..1B7C ; Limited_Use Not_XID # 5.0 [9] BALINESE MUSICAL SYMBOL RIGHT-HAND OPEN DUG..BALINESE MUSICAL SYMBOL LEFT-HAND OPEN PING +1B7D..1B7E ; Limited_Use Not_XID # 14.0 [2] BALINESE PANTI LANTANG..BALINESE PAMADA LANTANG +1B7F ; Limited_Use Not_XID # 16.0 BALINESE PANTI BAWAK +1BFC..1BFF ; Limited_Use Not_XID # 6.0 [4] BATAK SYMBOL BINDU NA METEK..BATAK SYMBOL BINDU PANGOLAT +1C3B..1C3F ; Limited_Use Not_XID # 5.1 [5] LEPCHA PUNCTUATION TA-ROL..LEPCHA PUNCTUATION TSHOOK +1C7E..1C7F ; Limited_Use Not_XID # 5.1 [2] OL CHIKI PUNCTUATION MUCAAD..OL CHIKI PUNCTUATION DOUBLE MUCAAD +1CC0..1CC7 ; Limited_Use Not_XID # 6.1 [8] SUNDANESE PUNCTUATION BINDU SURYA..SUNDANESE PUNCTUATION BINDU BA SATANGA +2D70 ; Limited_Use Not_XID # 6.0 TIFINAGH SEPARATOR MARK +A490..A4A1 ; Limited_Use Not_XID # 3.0 [18] YI RADICAL QOT..YI RADICAL GA +A4A2..A4A3 ; Limited_Use Not_XID # 3.2 [2] YI RADICAL ZUP..YI RADICAL CYT +A4A4..A4B3 ; Limited_Use Not_XID # 3.0 [16] YI RADICAL DDUR..YI RADICAL JO +A4B4 ; Limited_Use Not_XID # 3.2 YI RADICAL NZUP +A4B5..A4C0 ; Limited_Use Not_XID # 3.0 [12] YI RADICAL JJY..YI RADICAL SHAT +A4C1 ; Limited_Use Not_XID # 3.2 YI RADICAL ZUR +A4C2..A4C4 ; Limited_Use Not_XID # 3.0 [3] YI RADICAL SHOP..YI RADICAL ZZIET +A4C5 ; Limited_Use Not_XID # 3.2 YI RADICAL NBIE +A4C6 ; Limited_Use Not_XID # 3.0 YI RADICAL KE +A4FE..A4FF ; Limited_Use Not_XID # 5.2 [2] LISU PUNCTUATION COMMA..LISU PUNCTUATION FULL STOP +A60D..A60F ; Limited_Use Not_XID # 5.1 [3] VAI COMMA..VAI QUESTION MARK +A6F2..A6F7 ; Limited_Use Not_XID # 5.2 [6] BAMUM NJAEMLI..BAMUM QUESTION MARK +A828..A82B ; Limited_Use Not_XID # 4.1 [4] SYLOTI NAGRI POETRY MARK-1..SYLOTI NAGRI POETRY MARK-4 +A8CE..A8CF ; Limited_Use Not_XID # 5.1 [2] SAURASHTRA DANDA..SAURASHTRA DOUBLE DANDA +A92F ; Limited_Use Not_XID # 5.1 KAYAH LI SIGN SHYA +A9C1..A9CD ; Limited_Use Not_XID # 5.2 [13] JAVANESE LEFT RERENGGAN..JAVANESE TURNED PADA PISELEH +A9DE..A9DF ; Limited_Use Not_XID # 5.2 [2] JAVANESE PADA TIRTA TUMETES..JAVANESE PADA ISEN-ISEN +AA5C..AA5F ; Limited_Use Not_XID # 5.1 [4] CHAM PUNCTUATION SPIRAL..CHAM PUNCTUATION TRIPLE DANDA +AADE..AADF ; Limited_Use Not_XID # 5.2 [2] TAI VIET SYMBOL HO HOI..TAI VIET SYMBOL KOI KOI +AAF0..AAF1 ; Limited_Use Not_XID # 6.1 [2] MEETEI MAYEK CHEIKHAN..MEETEI MAYEK AHANG KHUDAM +ABEB ; Limited_Use Not_XID # 5.2 MEETEI MAYEK CHEIKHEI +11140..11143 ; Limited_Use Not_XID # 6.1 [4] CHAKMA SECTION MARK..CHAKMA QUESTION MARK +1144B..1144F ; Limited_Use Not_XID # 9.0 [5] NEWA DANDA..NEWA ABBREVIATION SIGN +1145A ; Limited_Use Not_XID # 13.0 NEWA DOUBLE COMMA +1145B ; Limited_Use Not_XID # 9.0 NEWA PLACEHOLDER MARK +1145D ; Limited_Use Not_XID # 9.0 NEWA INSERTION SIGN +1E14F ; Limited_Use Not_XID # 12.0 NYIAKENG PUACHUE HMONG CIRCLED CA +1E2FF ; Limited_Use Not_XID # 12.0 WANCHO NGUN SIGN +1E95E..1E95F ; Limited_Use Not_XID # 9.0 [2] ADLAM INITIAL EXCLAMATION MARK..ADLAM INITIAL QUESTION MARK + +# Total code points: 209 + +# Identifier_Type: Uncommon_Use + +0114..0115 ; Uncommon_Use # 1.1 [2] LATIN CAPITAL LETTER E WITH BREVE..LATIN SMALL LETTER E WITH BREVE +012C..012D ; Uncommon_Use # 1.1 [2] LATIN CAPITAL LETTER I WITH BREVE..LATIN SMALL LETTER I WITH BREVE +014E..014F ; Uncommon_Use # 1.1 [2] LATIN CAPITAL LETTER O WITH BREVE..LATIN SMALL LETTER O WITH BREVE +0156..0157 ; Uncommon_Use # 1.1 [2] LATIN CAPITAL LETTER R WITH CEDILLA..LATIN SMALL LETTER R WITH CEDILLA +0162..0163 ; Uncommon_Use # 1.1 [2] LATIN CAPITAL LETTER T WITH CEDILLA..LATIN SMALL LETTER T WITH CEDILLA +0182..0185 ; Uncommon_Use # 1.1 [4] LATIN CAPITAL LETTER B WITH TOPBAR..LATIN SMALL LETTER TONE SIX +0187..0188 ; Uncommon_Use # 1.1 [2] LATIN CAPITAL LETTER C WITH HOOK..LATIN SMALL LETTER C WITH HOOK +018B..018C ; Uncommon_Use # 1.1 [2] LATIN CAPITAL LETTER D WITH TOPBAR..LATIN SMALL LETTER D WITH TOPBAR +0193 ; Uncommon_Use # 1.1 LATIN CAPITAL LETTER G WITH HOOK +0195 ; Uncommon_Use # 1.1 LATIN SMALL LETTER HV +019A..019C ; Uncommon_Use # 1.1 [3] LATIN SMALL LETTER L WITH BAR..LATIN CAPITAL LETTER TURNED M +019E..019F ; Uncommon_Use # 1.1 [2] LATIN SMALL LETTER N WITH LONG RIGHT LEG..LATIN CAPITAL LETTER O WITH MIDDLE TILDE +01A2..01A9 ; Uncommon_Use # 1.1 [8] LATIN CAPITAL LETTER OI..LATIN CAPITAL LETTER ESH +01AC..01AE ; Uncommon_Use # 1.1 [3] LATIN CAPITAL LETTER T WITH HOOK..LATIN CAPITAL LETTER T WITH RETROFLEX HOOK +01B1 ; Uncommon_Use # 1.1 LATIN CAPITAL LETTER UPSILON +01B5..01B6 ; Uncommon_Use # 1.1 [2] LATIN CAPITAL LETTER Z WITH STROKE..LATIN SMALL LETTER Z WITH STROKE +01B8 ; Uncommon_Use # 1.1 LATIN CAPITAL LETTER EZH REVERSED +01BC..01BD ; Uncommon_Use # 1.1 [2] LATIN CAPITAL LETTER TONE FIVE..LATIN SMALL LETTER TONE FIVE +01D5..01DC ; Uncommon_Use # 1.1 [8] LATIN CAPITAL LETTER U WITH DIAERESIS AND MACRON..LATIN SMALL LETTER U WITH DIAERESIS AND GRAVE +01DE..01E5 ; Uncommon_Use # 1.1 [8] LATIN CAPITAL LETTER A WITH DIAERESIS AND MACRON..LATIN SMALL LETTER G WITH STROKE +01EA..01ED ; Uncommon_Use # 1.1 [4] LATIN CAPITAL LETTER O WITH OGONEK..LATIN SMALL LETTER O WITH OGONEK AND MACRON +01F0 ; Uncommon_Use # 1.1 LATIN SMALL LETTER J WITH CARON +01F4..01F5 ; Uncommon_Use # 1.1 [2] LATIN CAPITAL LETTER G WITH ACUTE..LATIN SMALL LETTER G WITH ACUTE +01FA..01FF ; Uncommon_Use # 1.1 [6] LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE..LATIN SMALL LETTER O WITH STROKE AND ACUTE +021E..021F ; Uncommon_Use # 3.0 [2] LATIN CAPITAL LETTER H WITH CARON..LATIN SMALL LETTER H WITH CARON +0220 ; Uncommon_Use # 3.2 LATIN CAPITAL LETTER N WITH LONG RIGHT LEG +0221 ; Uncommon_Use # 4.0 LATIN SMALL LETTER D WITH CURL +0222..0233 ; Uncommon_Use # 3.0 [18] LATIN CAPITAL LETTER OU..LATIN SMALL LETTER Y WITH MACRON +0237..0241 ; Uncommon_Use # 4.1 [11] LATIN SMALL LETTER DOTLESS J..LATIN CAPITAL LETTER GLOTTAL STOP +0242..0243 ; Uncommon_Use # 5.0 [2] LATIN SMALL LETTER GLOTTAL STOP..LATIN CAPITAL LETTER B WITH STROKE +0245..024B ; Uncommon_Use # 5.0 [7] LATIN CAPITAL LETTER TURNED V..LATIN SMALL LETTER Q WITH HOOK TAIL +024E..024F ; Uncommon_Use # 5.0 [2] LATIN CAPITAL LETTER Y WITH STROKE..LATIN SMALL LETTER Y WITH STROKE +0305 ; Uncommon_Use # 1.1 COMBINING OVERLINE +030D ; Uncommon_Use # 1.1 COMBINING VERTICAL LINE ABOVE +0316 ; Uncommon_Use # 1.1 COMBINING GRAVE ACCENT BELOW +0321..0322 ; Uncommon_Use # 1.1 [2] COMBINING PALATALIZED HOOK BELOW..COMBINING RETROFLEX HOOK BELOW +0332 ; Uncommon_Use # 1.1 COMBINING LOW LINE +0334 ; Uncommon_Use # 1.1 COMBINING TILDE OVERLAY +0336 ; Uncommon_Use # 1.1 COMBINING LONG STROKE OVERLAY +0358 ; Uncommon_Use # 4.1 COMBINING DOT ABOVE RIGHT +0400 ; Uncommon_Use # 3.0 CYRILLIC CAPITAL LETTER IE WITH GRAVE +040D ; Uncommon_Use # 3.0 CYRILLIC CAPITAL LETTER I WITH GRAVE +0450 ; Uncommon_Use # 3.0 CYRILLIC SMALL LETTER IE WITH GRAVE +045D ; Uncommon_Use # 3.0 CYRILLIC SMALL LETTER I WITH GRAVE +048A..048B ; Uncommon_Use # 3.2 [2] CYRILLIC CAPITAL LETTER SHORT I WITH TAIL..CYRILLIC SMALL LETTER SHORT I WITH TAIL +048C..048F ; Uncommon_Use # 3.0 [4] CYRILLIC CAPITAL LETTER SEMISOFT SIGN..CYRILLIC SMALL LETTER ER WITH TICK +04C1..04C4 ; Uncommon_Use # 1.1 [4] CYRILLIC CAPITAL LETTER ZHE WITH BREVE..CYRILLIC SMALL LETTER KA WITH HOOK +04C5..04C6 ; Uncommon_Use # 3.2 [2] CYRILLIC CAPITAL LETTER EL WITH TAIL..CYRILLIC SMALL LETTER EL WITH TAIL +04C7..04C8 ; Uncommon_Use # 1.1 [2] CYRILLIC CAPITAL LETTER EN WITH HOOK..CYRILLIC SMALL LETTER EN WITH HOOK +04C9..04CA ; Uncommon_Use # 3.2 [2] CYRILLIC CAPITAL LETTER EN WITH TAIL..CYRILLIC SMALL LETTER EN WITH TAIL +04CB..04CC ; Uncommon_Use # 1.1 [2] CYRILLIC CAPITAL LETTER KHAKASSIAN CHE..CYRILLIC SMALL LETTER KHAKASSIAN CHE +04CD..04CE ; Uncommon_Use # 3.2 [2] CYRILLIC CAPITAL LETTER EM WITH TAIL..CYRILLIC SMALL LETTER EM WITH TAIL +04DA..04DB ; Uncommon_Use # 1.1 [2] CYRILLIC CAPITAL LETTER SCHWA WITH DIAERESIS..CYRILLIC SMALL LETTER SCHWA WITH DIAERESIS +04EA..04EB ; Uncommon_Use # 1.1 [2] CYRILLIC CAPITAL LETTER BARRED O WITH DIAERESIS..CYRILLIC SMALL LETTER BARRED O WITH DIAERESIS +04EC..04ED ; Uncommon_Use # 3.0 [2] CYRILLIC CAPITAL LETTER E WITH DIAERESIS..CYRILLIC SMALL LETTER E WITH DIAERESIS +04F6..04F7 ; Uncommon_Use # 4.1 [2] CYRILLIC CAPITAL LETTER GHE WITH DESCENDER..CYRILLIC SMALL LETTER GHE WITH DESCENDER +04FA..04FF ; Uncommon_Use # 5.0 [6] CYRILLIC CAPITAL LETTER GHE WITH STROKE AND HOOK..CYRILLIC SMALL LETTER HA WITH STROKE +0510..0513 ; Uncommon_Use # 5.0 [4] CYRILLIC CAPITAL LETTER REVERSED ZE..CYRILLIC SMALL LETTER EL WITH HOOK +0591..05A1 ; Uncommon_Use # 2.0 [17] HEBREW ACCENT ETNAHTA..HEBREW ACCENT PAZER +05A3..05AF ; Uncommon_Use # 2.0 [13] HEBREW ACCENT MUNAH..HEBREW MARK MASORA CIRCLE +05B0..05B9 ; Uncommon_Use # 1.1 [10] HEBREW POINT SHEVA..HEBREW POINT HOLAM +05BA ; Uncommon_Use # 5.0 HEBREW POINT HOLAM HASER FOR VAV +05BB..05BD ; Uncommon_Use # 1.1 [3] HEBREW POINT QUBUTS..HEBREW POINT METEG +05BF ; Uncommon_Use # 1.1 HEBREW POINT RAFE +05C1..05C2 ; Uncommon_Use # 1.1 [2] HEBREW POINT SHIN DOT..HEBREW POINT SIN DOT +05C4 ; Uncommon_Use # 2.0 HEBREW MARK UPPER DOT +05EF ; Uncommon_Use # 11.0 HEBREW YOD TRIANGLE +05F0..05F2 ; Uncommon_Use # 1.1 [3] HEBREW LIGATURE YIDDISH DOUBLE VAV..HEBREW LIGATURE YIDDISH DOUBLE YOD +0610..0615 ; Uncommon_Use # 4.0 [6] ARABIC SIGN SALLALLAHOU ALAYHE WASSALLAM..ARABIC SMALL HIGH TAH +0616..061A ; Uncommon_Use # 5.1 [5] ARABIC SMALL HIGH LIGATURE ALEF WITH LAM WITH YEH..ARABIC SMALL KASRA +0656..0658 ; Uncommon_Use # 4.0 [3] ARABIC SUBSCRIPT ALEF..ARABIC MARK NOON GHUNNA +0659..065E ; Uncommon_Use # 4.1 [6] ARABIC ZWARAKAY..ARABIC FATHA WITH TWO DOTS +065F ; Uncommon_Use # 6.0 ARABIC WAVY HAMZA BELOW +069B..069E ; Uncommon_Use # 1.1 [4] ARABIC LETTER SEEN WITH THREE DOTS BELOW..ARABIC LETTER SAD WITH THREE DOTS ABOVE +06A1 ; Uncommon_Use # 1.1 ARABIC LETTER DOTLESS FEH +06A3 ; Uncommon_Use # 1.1 ARABIC LETTER FEH WITH DOT BELOW +06B2 ; Uncommon_Use # 1.1 ARABIC LETTER GAF WITH TWO DOTS BELOW +06B4 ; Uncommon_Use # 1.1 ARABIC LETTER GAF WITH THREE DOTS ABOVE +06B8..06B9 ; Uncommon_Use # 3.0 [2] ARABIC LETTER LAM WITH THREE DOTS BELOW..ARABIC LETTER NOON WITH DOT BELOW +06BF ; Uncommon_Use # 3.0 ARABIC LETTER TCHEH WITH DOT ABOVE +06D6..06DC ; Uncommon_Use # 1.1 [7] ARABIC SMALL HIGH LIGATURE SAD WITH LAM WITH ALEF MAKSURA..ARABIC SMALL HIGH SEEN +06DF..06E4 ; Uncommon_Use # 1.1 [6] ARABIC SMALL HIGH ROUNDED ZERO..ARABIC SMALL HIGH MADDA +06E7..06E8 ; Uncommon_Use # 1.1 [2] ARABIC SMALL HIGH YEH..ARABIC SMALL HIGH NOON +06EA..06ED ; Uncommon_Use # 1.1 [4] ARABIC EMPTY CENTRE LOW STOP..ARABIC SMALL LOW MEEM +06FA..06FC ; Uncommon_Use # 3.0 [3] ARABIC LETTER SHEEN WITH DOT BELOW..ARABIC LETTER GHAIN WITH DOT BELOW +0750 ; Uncommon_Use # 4.1 ARABIC LETTER BEH WITH THREE DOTS HORIZONTALLY BELOW +0753..0755 ; Uncommon_Use # 4.1 [3] ARABIC LETTER BEH WITH THREE DOTS POINTING UPWARDS BELOW AND TWO DOTS ABOVE..ARABIC LETTER BEH WITH INVERTED SMALL V BELOW +0757..075F ; Uncommon_Use # 4.1 [9] ARABIC LETTER HAH WITH TWO DOTS ABOVE..ARABIC LETTER AIN WITH TWO DOTS VERTICALLY ABOVE +0761 ; Uncommon_Use # 4.1 ARABIC LETTER FEH WITH THREE DOTS POINTING UPWARDS BELOW +0764..0765 ; Uncommon_Use # 4.1 [2] ARABIC LETTER KEHEH WITH THREE DOTS POINTING UPWARDS BELOW..ARABIC LETTER MEEM WITH DOT ABOVE +0769 ; Uncommon_Use # 4.1 ARABIC LETTER NOON WITH SMALL V +076B..076D ; Uncommon_Use # 4.1 [3] ARABIC LETTER REH WITH TWO DOTS VERTICALLY ABOVE..ARABIC LETTER SEEN WITH TWO DOTS VERTICALLY ABOVE +0772..077D ; Uncommon_Use # 5.1 [12] ARABIC LETTER HAH WITH SMALL ARABIC LETTER TAH ABOVE..ARABIC LETTER SEEN WITH EXTENDED ARABIC-INDIC DIGIT FOUR ABOVE +0889..088D ; Uncommon_Use # 14.0 [5] ARABIC LETTER NOON WITH INVERTED SMALL V..ARABIC LETTER KEHEH WITH TWO DOTS VERTICALLY BELOW +0897 ; Uncommon_Use # 16.0 ARABIC PEPET +0898..089F ; Uncommon_Use # 14.0 [8] ARABIC SMALL HIGH WORD AL-JUZ..ARABIC HALF MADDA OVER MADDA +08A1 ; Uncommon_Use # 7.0 ARABIC LETTER BEH WITH HAMZA ABOVE +08AA..08AC ; Uncommon_Use # 6.1 [3] ARABIC LETTER REH WITH LOOP..ARABIC LETTER ROHINGYA YEH +08B2 ; Uncommon_Use # 7.0 ARABIC LETTER ZAIN WITH INVERTED V ABOVE +08B3..08B4 ; Uncommon_Use # 8.0 [2] ARABIC LETTER AIN WITH THREE DOTS BELOW..ARABIC LETTER KAF WITH DOT BELOW +08B6..08BA ; Uncommon_Use # 9.0 [5] ARABIC LETTER BEH WITH SMALL MEEM ABOVE..ARABIC LETTER YEH WITH TWO DOTS BELOW AND SMALL NOON ABOVE +08C3..08C6 ; Uncommon_Use # 13.0 [4] ARABIC LETTER GHAIN WITH THREE DOTS ABOVE..ARABIC LETTER JEEM WITH THREE DOTS BELOW +08C8 ; Uncommon_Use # 14.0 ARABIC LETTER GRAF +08CA..08D2 ; Uncommon_Use # 14.0 [9] ARABIC SMALL HIGH FARSI YEH..ARABIC LARGE ROUND DOT INSIDE CIRCLE BELOW +08D3 ; Uncommon_Use # 11.0 ARABIC SMALL LOW WAW +08D4..08E1 ; Uncommon_Use # 9.0 [14] ARABIC SMALL HIGH WORD AR-RUB..ARABIC SMALL HIGH SIGN SAFHA +08E3 ; Uncommon_Use # 8.0 ARABIC TURNED DAMMA BELOW +08E4..08FE ; Uncommon_Use # 6.1 [27] ARABIC CURLY FATHA..ARABIC DAMMA WITH DOT +08FF ; Uncommon_Use # 7.0 ARABIC MARK SIDEWAYS NOON GHUNNA +0900 ; Uncommon_Use # 5.2 DEVANAGARI SIGN INVERTED CANDRABINDU +0904 ; Uncommon_Use # 4.0 DEVANAGARI LETTER SHORT A +0929 ; Uncommon_Use # 1.1 DEVANAGARI LETTER NNNA +0934 ; Uncommon_Use # 1.1 DEVANAGARI LETTER LLLA +0944 ; Uncommon_Use # 1.1 DEVANAGARI VOWEL SIGN VOCALIC RR +0955 ; Uncommon_Use # 5.2 DEVANAGARI VOWEL SIGN CANDRA LONG E +0979..097A ; Uncommon_Use # 5.2 [2] DEVANAGARI LETTER ZHA..DEVANAGARI LETTER HEAVY YA +098C ; Uncommon_Use # 1.1 BENGALI LETTER VOCALIC L +09D7 ; Uncommon_Use # 1.1 BENGALI AU LENGTH MARK +09FE ; Uncommon_Use # 11.0 BENGALI SANDHI MARK +0A01 ; Uncommon_Use # 4.0 GURMUKHI SIGN ADAK BINDI +0A03 ; Uncommon_Use # 4.0 GURMUKHI SIGN VISARGA +0A51 ; Uncommon_Use # 5.1 GURMUKHI SIGN UDAAT +0A66..0A6F ; Uncommon_Use # 1.1 [10] GURMUKHI DIGIT ZERO..GURMUKHI DIGIT NINE +0A72..0A73 ; Uncommon_Use # 1.1 [2] GURMUKHI IRI..GURMUKHI URA +0A75 ; Uncommon_Use # 5.1 GURMUKHI SIGN YAKASH +0A81 ; Uncommon_Use # 1.1 GUJARATI SIGN CANDRABINDU +0AF9 ; Uncommon_Use # 8.0 GUJARATI LETTER ZHA +0AFA..0AFF ; Uncommon_Use # 10.0 [6] GUJARATI SIGN SUKUN..GUJARATI SIGN TWO-CIRCLE NUKTA ABOVE +0B0C ; Uncommon_Use # 1.1 ORIYA LETTER VOCALIC L +0B35 ; Uncommon_Use # 4.0 ORIYA LETTER VA +0B44 ; Uncommon_Use # 5.1 ORIYA VOWEL SIGN VOCALIC RR +0B55 ; Uncommon_Use # 13.0 ORIYA SIGN OVERLINE +0B57 ; Uncommon_Use # 1.1 ORIYA AU LENGTH MARK +0B62..0B63 ; Uncommon_Use # 5.1 [2] ORIYA VOWEL SIGN VOCALIC L..ORIYA VOWEL SIGN VOCALIC LL +0B66..0B6F ; Uncommon_Use # 1.1 [10] ORIYA DIGIT ZERO..ORIYA DIGIT NINE +0BD7 ; Uncommon_Use # 1.1 TAMIL AU LENGTH MARK +0BE6 ; Uncommon_Use # 4.1 TAMIL DIGIT ZERO +0BE7..0BEF ; Uncommon_Use # 1.1 [9] TAMIL DIGIT ONE..TAMIL DIGIT NINE +0C01 ; Uncommon_Use # 1.1 TELUGU SIGN CANDRABINDU +0C04 ; Uncommon_Use # 11.0 TELUGU SIGN COMBINING ANUSVARA ABOVE +0C0C ; Uncommon_Use # 1.1 TELUGU LETTER VOCALIC L +0C31 ; Uncommon_Use # 1.1 TELUGU LETTER RRA +0C3C ; Uncommon_Use # 14.0 TELUGU SIGN NUKTA +0C55..0C56 ; Uncommon_Use # 1.1 [2] TELUGU LENGTH MARK..TELUGU AI LENGTH MARK +0C5A ; Uncommon_Use # 8.0 TELUGU LETTER RRRA +0C5D ; Uncommon_Use # 14.0 TELUGU LETTER NAKAARA POLLU +0C62..0C63 ; Uncommon_Use # 5.1 [2] TELUGU VOWEL SIGN VOCALIC L..TELUGU VOWEL SIGN VOCALIC LL +0C66..0C6F ; Uncommon_Use # 1.1 [10] TELUGU DIGIT ZERO..TELUGU DIGIT NINE +0C80 ; Uncommon_Use # 9.0 KANNADA SIGN SPACING CANDRABINDU +0CBC ; Uncommon_Use # 4.0 KANNADA SIGN NUKTA +0CC4 ; Uncommon_Use # 1.1 KANNADA VOWEL SIGN VOCALIC RR +0CD5..0CD6 ; Uncommon_Use # 1.1 [2] KANNADA LENGTH MARK..KANNADA AI LENGTH MARK +0CDD ; Uncommon_Use # 14.0 KANNADA LETTER NAKAARA POLLU +0CF3 ; Uncommon_Use # 15.0 KANNADA SIGN COMBINING ANUSVARA ABOVE RIGHT +0D00 ; Uncommon_Use # 10.0 MALAYALAM SIGN COMBINING ANUSVARA ABOVE +0D0C ; Uncommon_Use # 1.1 MALAYALAM LETTER VOCALIC L +0D29 ; Uncommon_Use # 6.0 MALAYALAM LETTER NNNA +0D44 ; Uncommon_Use # 5.1 MALAYALAM VOWEL SIGN VOCALIC RR +0D54..0D56 ; Uncommon_Use # 9.0 [3] MALAYALAM LETTER CHILLU M..MALAYALAM LETTER CHILLU LLL +0D62..0D63 ; Uncommon_Use # 5.1 [2] MALAYALAM VOWEL SIGN VOCALIC L..MALAYALAM VOWEL SIGN VOCALIC LL +0D66..0D6F ; Uncommon_Use # 1.1 [10] MALAYALAM DIGIT ZERO..MALAYALAM DIGIT NINE +0D8E ; Uncommon_Use # 3.0 SINHALA LETTER IRUUYANNA +0DE6..0DEF ; Uncommon_Use # 7.0 [10] SINHALA LITH DIGIT ZERO..SINHALA LITH DIGIT NINE +0E4E ; Uncommon_Use # 1.1 THAI CHARACTER YAMAKKAN +0E86 ; Uncommon_Use # 12.0 LAO LETTER PALI GHA +0E89 ; Uncommon_Use # 12.0 LAO LETTER PALI CHA +0E8C ; Uncommon_Use # 12.0 LAO LETTER PALI JHA +0E8E..0E93 ; Uncommon_Use # 12.0 [6] LAO LETTER PALI NYA..LAO LETTER PALI NNA +0E98 ; Uncommon_Use # 12.0 LAO LETTER PALI DHA +0EA0 ; Uncommon_Use # 12.0 LAO LETTER PALI BHA +0EA8..0EA9 ; Uncommon_Use # 12.0 [2] LAO LETTER SANSKRIT SHA..LAO LETTER SANSKRIT SSA +0EAC ; Uncommon_Use # 12.0 LAO LETTER PALI LLA +0EBA ; Uncommon_Use # 12.0 LAO SIGN PALI VIRAMA +0ECE ; Uncommon_Use # 15.0 LAO YAMAKKAN +0EDE..0EDF ; Uncommon_Use # 6.1 [2] LAO LETTER KHMU GO..LAO LETTER KHMU NYO +0F39 ; Uncommon_Use # 2.0 TIBETAN MARK TSA -PHRU +0F6B..0F6C ; Uncommon_Use # 5.1 [2] TIBETAN LETTER KKA..TIBETAN LETTER RRA +0FAE..0FB0 ; Uncommon_Use # 3.0 [3] TIBETAN SUBJOINED LETTER ZHA..TIBETAN SUBJOINED LETTER -A +1065..1074 ; Uncommon_Use # 5.1 [16] MYANMAR LETTER WESTERN PWO KAREN THA..MYANMAR VOWEL SIGN KAYAH EE +108B..108E ; Uncommon_Use # 5.1 [4] MYANMAR SIGN SHAN COUNCIL TONE-2..MYANMAR LETTER RUMAI PALAUNG FA +1090..1099 ; Uncommon_Use # 5.1 [10] MYANMAR SHAN DIGIT ZERO..MYANMAR SHAN DIGIT NINE +109A..109D ; Uncommon_Use # 5.2 [4] MYANMAR SIGN KHAMTI TONE-1..MYANMAR VOWEL SIGN AITON AI +10F7..10F8 ; Uncommon_Use # 3.2 [2] GEORGIAN LETTER YN..GEORGIAN LETTER ELIFI +10FD..10FF ; Uncommon_Use # 6.1 [3] GEORGIAN LETTER AEN..GEORGIAN LETTER LABIAL SIGN +1207 ; Uncommon_Use # 4.1 ETHIOPIC SYLLABLE HOA +1287 ; Uncommon_Use # 4.1 ETHIOPIC SYLLABLE XOA +12AF ; Uncommon_Use # 4.1 ETHIOPIC SYLLABLE KOA +12F8..12FF ; Uncommon_Use # 3.0 [8] ETHIOPIC SYLLABLE DDA..ETHIOPIC SYLLABLE DDWA +130F ; Uncommon_Use # 4.1 ETHIOPIC SYLLABLE GOA +131F ; Uncommon_Use # 4.1 ETHIOPIC SYLLABLE GGWAA +1347 ; Uncommon_Use # 4.1 ETHIOPIC SYLLABLE TZOA +135A ; Uncommon_Use # 3.0 ETHIOPIC SYLLABLE FYA +135D..135E ; Uncommon_Use # 6.0 [2] ETHIOPIC COMBINING GEMINATION AND VOWEL LENGTH MARK..ETHIOPIC COMBINING VOWEL LENGTH MARK +135F ; Uncommon_Use # 4.1 ETHIOPIC COMBINING GEMINATION MARK +1380..138F ; Uncommon_Use # 4.1 [16] ETHIOPIC SYLLABLE SEBATBEIT MWA..ETHIOPIC SYLLABLE PWE +179D..179E ; Uncommon_Use # 3.0 [2] KHMER LETTER SHA..KHMER LETTER SSO +17A9 ; Uncommon_Use # 3.0 KHMER INDEPENDENT VOWEL QUU +17D7 ; Uncommon_Use # 3.0 KHMER SIGN LEK TOO +1AC1..1ACE ; Uncommon_Use # 14.0 [14] COMBINING LEFT PARENTHESIS ABOVE LEFT..COMBINING LATIN SMALL LETTER INSULAR T +1C89..1C8A ; Uncommon_Use # 16.0 [2] CYRILLIC CAPITAL LETTER TJE..CYRILLIC SMALL LETTER TJE +1E02..1E0B ; Uncommon_Use # 1.1 [10] LATIN CAPITAL LETTER B WITH DOT ABOVE..LATIN SMALL LETTER D WITH DOT ABOVE +1E0E..1E11 ; Uncommon_Use # 1.1 [4] LATIN CAPITAL LETTER D WITH LINE BELOW..LATIN SMALL LETTER D WITH CEDILLA +1E14..1E17 ; Uncommon_Use # 1.1 [4] LATIN CAPITAL LETTER E WITH MACRON AND GRAVE..LATIN SMALL LETTER E WITH MACRON AND ACUTE +1E1C..1E1F ; Uncommon_Use # 1.1 [4] LATIN CAPITAL LETTER E WITH CEDILLA AND BREVE..LATIN SMALL LETTER F WITH DOT ABOVE +1E22..1E23 ; Uncommon_Use # 1.1 [2] LATIN CAPITAL LETTER H WITH DOT ABOVE..LATIN SMALL LETTER H WITH DOT ABOVE +1E26..1E29 ; Uncommon_Use # 1.1 [4] LATIN CAPITAL LETTER H WITH DIAERESIS..LATIN SMALL LETTER H WITH CEDILLA +1E2E..1E35 ; Uncommon_Use # 1.1 [8] LATIN CAPITAL LETTER I WITH DIAERESIS AND ACUTE..LATIN SMALL LETTER K WITH LINE BELOW +1E38..1E3B ; Uncommon_Use # 1.1 [4] LATIN CAPITAL LETTER L WITH DOT BELOW AND MACRON..LATIN SMALL LETTER L WITH LINE BELOW +1E40..1E41 ; Uncommon_Use # 1.1 [2] LATIN CAPITAL LETTER M WITH DOT ABOVE..LATIN SMALL LETTER M WITH DOT ABOVE +1E4C..1E59 ; Uncommon_Use # 1.1 [14] LATIN CAPITAL LETTER O WITH TILDE AND ACUTE..LATIN SMALL LETTER R WITH DOT ABOVE +1E5C..1E61 ; Uncommon_Use # 1.1 [6] LATIN CAPITAL LETTER R WITH DOT BELOW AND MACRON..LATIN SMALL LETTER S WITH DOT ABOVE +1E64..1E6B ; Uncommon_Use # 1.1 [8] LATIN CAPITAL LETTER S WITH ACUTE AND DOT ABOVE..LATIN SMALL LETTER T WITH DOT ABOVE +1E6E..1E6F ; Uncommon_Use # 1.1 [2] LATIN CAPITAL LETTER T WITH LINE BELOW..LATIN SMALL LETTER T WITH LINE BELOW +1E78..1E8B ; Uncommon_Use # 1.1 [20] LATIN CAPITAL LETTER U WITH TILDE AND ACUTE..LATIN SMALL LETTER X WITH DOT ABOVE +1E8E..1E91 ; Uncommon_Use # 1.1 [4] LATIN CAPITAL LETTER Y WITH DOT ABOVE..LATIN SMALL LETTER Z WITH CIRCUMFLEX +1E94..1E99 ; Uncommon_Use # 1.1 [6] LATIN CAPITAL LETTER Z WITH LINE BELOW..LATIN SMALL LETTER Y WITH RING ABOVE +2054 ; Uncommon_Use # 4.0 INVERTED UNDERTIE +2C68..2C6C ; Uncommon_Use # 5.0 [5] LATIN SMALL LETTER H WITH DESCENDER..LATIN SMALL LETTER Z WITH DESCENDER +2D80..2D96 ; Uncommon_Use # 4.1 [23] ETHIOPIC SYLLABLE LOA..ETHIOPIC SYLLABLE GGWE +2DA0..2DA6 ; Uncommon_Use # 4.1 [7] ETHIOPIC SYLLABLE SSA..ETHIOPIC SYLLABLE SSO +2DA8..2DAE ; Uncommon_Use # 4.1 [7] ETHIOPIC SYLLABLE CCA..ETHIOPIC SYLLABLE CCO +2DB0..2DB6 ; Uncommon_Use # 4.1 [7] ETHIOPIC SYLLABLE ZZA..ETHIOPIC SYLLABLE ZZO +2DB8..2DBE ; Uncommon_Use # 4.1 [7] ETHIOPIC SYLLABLE CCHA..ETHIOPIC SYLLABLE CCHO +2DC0..2DC6 ; Uncommon_Use # 4.1 [7] ETHIOPIC SYLLABLE QYA..ETHIOPIC SYLLABLE QYO +2DC8..2DCE ; Uncommon_Use # 4.1 [7] ETHIOPIC SYLLABLE KYA..ETHIOPIC SYLLABLE KYO +2DD0..2DD6 ; Uncommon_Use # 4.1 [7] ETHIOPIC SYLLABLE XYA..ETHIOPIC SYLLABLE XYO +2DD8..2DDE ; Uncommon_Use # 4.1 [7] ETHIOPIC SYLLABLE GYA..ETHIOPIC SYLLABLE GYO +3099..309A ; Uncommon_Use # 1.1 [2] COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK..COMBINING KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK +3400..3446 ; Uncommon_Use # 3.0 [71] CJK UNIFIED IDEOGRAPH-3400..CJK UNIFIED IDEOGRAPH-3446 +3448..3472 ; Uncommon_Use # 3.0 [43] CJK UNIFIED IDEOGRAPH-3448..CJK UNIFIED IDEOGRAPH-3472 +3474..34E3 ; Uncommon_Use # 3.0 [112] CJK UNIFIED IDEOGRAPH-3474..CJK UNIFIED IDEOGRAPH-34E3 +34E5..3576 ; Uncommon_Use # 3.0 [146] CJK UNIFIED IDEOGRAPH-34E5..CJK UNIFIED IDEOGRAPH-3576 +3578..359D ; Uncommon_Use # 3.0 [38] CJK UNIFIED IDEOGRAPH-3578..CJK UNIFIED IDEOGRAPH-359D +359F..35A0 ; Uncommon_Use # 3.0 [2] CJK UNIFIED IDEOGRAPH-359F..CJK UNIFIED IDEOGRAPH-35A0 +35A2..35AC ; Uncommon_Use # 3.0 [11] CJK UNIFIED IDEOGRAPH-35A2..CJK UNIFIED IDEOGRAPH-35AC +35AE..35BE ; Uncommon_Use # 3.0 [17] CJK UNIFIED IDEOGRAPH-35AE..CJK UNIFIED IDEOGRAPH-35BE +35C0..35CD ; Uncommon_Use # 3.0 [14] CJK UNIFIED IDEOGRAPH-35C0..CJK UNIFIED IDEOGRAPH-35CD +35CF..35F2 ; Uncommon_Use # 3.0 [36] CJK UNIFIED IDEOGRAPH-35CF..CJK UNIFIED IDEOGRAPH-35F2 +35F4..35FD ; Uncommon_Use # 3.0 [10] CJK UNIFIED IDEOGRAPH-35F4..CJK UNIFIED IDEOGRAPH-35FD +35FF..360D ; Uncommon_Use # 3.0 [15] CJK UNIFIED IDEOGRAPH-35FF..CJK UNIFIED IDEOGRAPH-360D +360F..3619 ; Uncommon_Use # 3.0 [11] CJK UNIFIED IDEOGRAPH-360F..CJK UNIFIED IDEOGRAPH-3619 +361B..3917 ; Uncommon_Use # 3.0 [765] CJK UNIFIED IDEOGRAPH-361B..CJK UNIFIED IDEOGRAPH-3917 +3919..395F ; Uncommon_Use # 3.0 [71] CJK UNIFIED IDEOGRAPH-3919..CJK UNIFIED IDEOGRAPH-395F +3961..396D ; Uncommon_Use # 3.0 [13] CJK UNIFIED IDEOGRAPH-3961..CJK UNIFIED IDEOGRAPH-396D +396F..39CE ; Uncommon_Use # 3.0 [96] CJK UNIFIED IDEOGRAPH-396F..CJK UNIFIED IDEOGRAPH-39CE +39D1..39DA ; Uncommon_Use # 3.0 [10] CJK UNIFIED IDEOGRAPH-39D1..CJK UNIFIED IDEOGRAPH-39DA +39DC..39DE ; Uncommon_Use # 3.0 [3] CJK UNIFIED IDEOGRAPH-39DC..CJK UNIFIED IDEOGRAPH-39DE +39E0..39F7 ; Uncommon_Use # 3.0 [24] CJK UNIFIED IDEOGRAPH-39E0..CJK UNIFIED IDEOGRAPH-39F7 +39F9..39FD ; Uncommon_Use # 3.0 [5] CJK UNIFIED IDEOGRAPH-39F9..CJK UNIFIED IDEOGRAPH-39FD +39FF..3A17 ; Uncommon_Use # 3.0 [25] CJK UNIFIED IDEOGRAPH-39FF..CJK UNIFIED IDEOGRAPH-3A17 +3A19..3A51 ; Uncommon_Use # 3.0 [57] CJK UNIFIED IDEOGRAPH-3A19..CJK UNIFIED IDEOGRAPH-3A51 +3A53..3A5B ; Uncommon_Use # 3.0 [9] CJK UNIFIED IDEOGRAPH-3A53..CJK UNIFIED IDEOGRAPH-3A5B +3A5D..3A66 ; Uncommon_Use # 3.0 [10] CJK UNIFIED IDEOGRAPH-3A5D..CJK UNIFIED IDEOGRAPH-3A66 +3A68..3A72 ; Uncommon_Use # 3.0 [11] CJK UNIFIED IDEOGRAPH-3A68..CJK UNIFIED IDEOGRAPH-3A72 +3A74..3B38 ; Uncommon_Use # 3.0 [197] CJK UNIFIED IDEOGRAPH-3A74..CJK UNIFIED IDEOGRAPH-3B38 +3B3A..3B4D ; Uncommon_Use # 3.0 [20] CJK UNIFIED IDEOGRAPH-3B3A..CJK UNIFIED IDEOGRAPH-3B4D +3B4F..3BA2 ; Uncommon_Use # 3.0 [84] CJK UNIFIED IDEOGRAPH-3B4F..CJK UNIFIED IDEOGRAPH-3BA2 +3BA4..3C6D ; Uncommon_Use # 3.0 [202] CJK UNIFIED IDEOGRAPH-3BA4..CJK UNIFIED IDEOGRAPH-3C6D +3C6F..3CDF ; Uncommon_Use # 3.0 [113] CJK UNIFIED IDEOGRAPH-3C6F..CJK UNIFIED IDEOGRAPH-3CDF +3CE1..3DE6 ; Uncommon_Use # 3.0 [262] CJK UNIFIED IDEOGRAPH-3CE1..CJK UNIFIED IDEOGRAPH-3DE6 +3DE8..3DEA ; Uncommon_Use # 3.0 [3] CJK UNIFIED IDEOGRAPH-3DE8..CJK UNIFIED IDEOGRAPH-3DEA +3DEC..3E73 ; Uncommon_Use # 3.0 [136] CJK UNIFIED IDEOGRAPH-3DEC..CJK UNIFIED IDEOGRAPH-3E73 +3E75..3ECF ; Uncommon_Use # 3.0 [91] CJK UNIFIED IDEOGRAPH-3E75..CJK UNIFIED IDEOGRAPH-3ECF +3ED1..4055 ; Uncommon_Use # 3.0 [389] CJK UNIFIED IDEOGRAPH-3ED1..CJK UNIFIED IDEOGRAPH-4055 +4057..4064 ; Uncommon_Use # 3.0 [14] CJK UNIFIED IDEOGRAPH-4057..CJK UNIFIED IDEOGRAPH-4064 +4066..4069 ; Uncommon_Use # 3.0 [4] CJK UNIFIED IDEOGRAPH-4066..CJK UNIFIED IDEOGRAPH-4069 +406B..40BA ; Uncommon_Use # 3.0 [80] CJK UNIFIED IDEOGRAPH-406B..CJK UNIFIED IDEOGRAPH-40BA +40BC..40DE ; Uncommon_Use # 3.0 [35] CJK UNIFIED IDEOGRAPH-40BC..CJK UNIFIED IDEOGRAPH-40DE +40E0..4136 ; Uncommon_Use # 3.0 [87] CJK UNIFIED IDEOGRAPH-40E0..CJK UNIFIED IDEOGRAPH-4136 +4138..415E ; Uncommon_Use # 3.0 [39] CJK UNIFIED IDEOGRAPH-4138..CJK UNIFIED IDEOGRAPH-415E +4160..4336 ; Uncommon_Use # 3.0 [471] CJK UNIFIED IDEOGRAPH-4160..CJK UNIFIED IDEOGRAPH-4336 +4338..43AB ; Uncommon_Use # 3.0 [116] CJK UNIFIED IDEOGRAPH-4338..CJK UNIFIED IDEOGRAPH-43AB +43AD..43B0 ; Uncommon_Use # 3.0 [4] CJK UNIFIED IDEOGRAPH-43AD..CJK UNIFIED IDEOGRAPH-43B0 +43B2..43D2 ; Uncommon_Use # 3.0 [33] CJK UNIFIED IDEOGRAPH-43B2..CJK UNIFIED IDEOGRAPH-43D2 +43D4..43DC ; Uncommon_Use # 3.0 [9] CJK UNIFIED IDEOGRAPH-43D4..CJK UNIFIED IDEOGRAPH-43DC +43DE..4442 ; Uncommon_Use # 3.0 [101] CJK UNIFIED IDEOGRAPH-43DE..CJK UNIFIED IDEOGRAPH-4442 +4444..44D5 ; Uncommon_Use # 3.0 [146] CJK UNIFIED IDEOGRAPH-4444..CJK UNIFIED IDEOGRAPH-44D5 +44D7..44E9 ; Uncommon_Use # 3.0 [19] CJK UNIFIED IDEOGRAPH-44D7..CJK UNIFIED IDEOGRAPH-44E9 +44EB..4605 ; Uncommon_Use # 3.0 [283] CJK UNIFIED IDEOGRAPH-44EB..CJK UNIFIED IDEOGRAPH-4605 +4607..464B ; Uncommon_Use # 3.0 [69] CJK UNIFIED IDEOGRAPH-4607..CJK UNIFIED IDEOGRAPH-464B +464D..4660 ; Uncommon_Use # 3.0 [20] CJK UNIFIED IDEOGRAPH-464D..CJK UNIFIED IDEOGRAPH-4660 +4662..4722 ; Uncommon_Use # 3.0 [193] CJK UNIFIED IDEOGRAPH-4662..CJK UNIFIED IDEOGRAPH-4722 +4724..4728 ; Uncommon_Use # 3.0 [5] CJK UNIFIED IDEOGRAPH-4724..CJK UNIFIED IDEOGRAPH-4728 +472A..477B ; Uncommon_Use # 3.0 [82] CJK UNIFIED IDEOGRAPH-472A..CJK UNIFIED IDEOGRAPH-477B +477D..478C ; Uncommon_Use # 3.0 [16] CJK UNIFIED IDEOGRAPH-477D..CJK UNIFIED IDEOGRAPH-478C +478E..47F3 ; Uncommon_Use # 3.0 [102] CJK UNIFIED IDEOGRAPH-478E..CJK UNIFIED IDEOGRAPH-47F3 +47F5..4881 ; Uncommon_Use # 3.0 [141] CJK UNIFIED IDEOGRAPH-47F5..CJK UNIFIED IDEOGRAPH-4881 +4883..4946 ; Uncommon_Use # 3.0 [196] CJK UNIFIED IDEOGRAPH-4883..CJK UNIFIED IDEOGRAPH-4946 +4948..4979 ; Uncommon_Use # 3.0 [50] CJK UNIFIED IDEOGRAPH-4948..CJK UNIFIED IDEOGRAPH-4979 +497B..497C ; Uncommon_Use # 3.0 [2] CJK UNIFIED IDEOGRAPH-497B..CJK UNIFIED IDEOGRAPH-497C +497E..4981 ; Uncommon_Use # 3.0 [4] CJK UNIFIED IDEOGRAPH-497E..CJK UNIFIED IDEOGRAPH-4981 +4984 ; Uncommon_Use # 3.0 CJK UNIFIED IDEOGRAPH-4984 +4987..499A ; Uncommon_Use # 3.0 [20] CJK UNIFIED IDEOGRAPH-4987..CJK UNIFIED IDEOGRAPH-499A +499C..499E ; Uncommon_Use # 3.0 [3] CJK UNIFIED IDEOGRAPH-499C..CJK UNIFIED IDEOGRAPH-499E +49A0..49B5 ; Uncommon_Use # 3.0 [22] CJK UNIFIED IDEOGRAPH-49A0..CJK UNIFIED IDEOGRAPH-49B5 +49B8..4A11 ; Uncommon_Use # 3.0 [90] CJK UNIFIED IDEOGRAPH-49B8..CJK UNIFIED IDEOGRAPH-4A11 +4A13..4AB7 ; Uncommon_Use # 3.0 [165] CJK UNIFIED IDEOGRAPH-4A13..CJK UNIFIED IDEOGRAPH-4AB7 +4AB9..4C76 ; Uncommon_Use # 3.0 [446] CJK UNIFIED IDEOGRAPH-4AB9..CJK UNIFIED IDEOGRAPH-4C76 +4C78..4C7C ; Uncommon_Use # 3.0 [5] CJK UNIFIED IDEOGRAPH-4C78..CJK UNIFIED IDEOGRAPH-4C7C +4C7E..4C80 ; Uncommon_Use # 3.0 [3] CJK UNIFIED IDEOGRAPH-4C7E..CJK UNIFIED IDEOGRAPH-4C80 +4C82..4C84 ; Uncommon_Use # 3.0 [3] CJK UNIFIED IDEOGRAPH-4C82..CJK UNIFIED IDEOGRAPH-4C84 +4C86..4C9C ; Uncommon_Use # 3.0 [23] CJK UNIFIED IDEOGRAPH-4C86..CJK UNIFIED IDEOGRAPH-4C9C +4CA4..4D12 ; Uncommon_Use # 3.0 [111] CJK UNIFIED IDEOGRAPH-4CA4..CJK UNIFIED IDEOGRAPH-4D12 +4D1A..4DAD ; Uncommon_Use # 3.0 [148] CJK UNIFIED IDEOGRAPH-4D1A..CJK UNIFIED IDEOGRAPH-4DAD +4DAF..4DB5 ; Uncommon_Use # 3.0 [7] CJK UNIFIED IDEOGRAPH-4DAF..CJK UNIFIED IDEOGRAPH-4DB5 +4DB6..4DBF ; Uncommon_Use # 13.0 [10] CJK UNIFIED IDEOGRAPH-4DB6..CJK UNIFIED IDEOGRAPH-4DBF +4E12 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-4E12 +4E29 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-4E29 +4E68 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-4E68 +4E79 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-4E79 +4E96 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-4E96 +4EA3 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-4EA3 +4EBC ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-4EBC +4ECC ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-4ECC +4EE7 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-4EE7 +4EF8..4EFA ; Uncommon_Use # 1.1 [3] CJK UNIFIED IDEOGRAPH-4EF8..CJK UNIFIED IDEOGRAPH-4EFA +4EFC ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-4EFC +4EFE ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-4EFE +4F07 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-4F07 +4F16 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-4F16 +4F28 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-4F28 +4F31 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-4F31 +4F35 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-4F35 +4F37 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-4F37 +4F40 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-4F40 +4F44 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-4F44 +4F71 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-4F71 +4F8C ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-4F8C +4F8E ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-4F8E +4FA2 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-4FA2 +4FBD ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-4FBD +4FC6 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-4FC6 +4FC8 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-4FC8 +4FCC ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-4FCC +4FE2 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-4FE2 +4FFC..4FFD ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-4FFC..CJK UNIFIED IDEOGRAPH-4FFD +5010 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5010 +5034 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5034 +5038 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5038 +503D ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-503D +5042 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5042 +5052 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5052 +5058 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5058 +507C ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-507C +5081 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5081 +5093 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5093 +5097 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5097 +509F..50A1 ; Uncommon_Use # 1.1 [3] CJK UNIFIED IDEOGRAPH-509F..CJK UNIFIED IDEOGRAPH-50A1 +50B9 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-50B9 +50C3 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-50C3 +50D8 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-50D8 +50DF ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-50DF +50E1..50E2 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-50E1..CJK UNIFIED IDEOGRAPH-50E2 +50EB ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-50EB +50F4 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-50F4 +50F7 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-50F7 +511B ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-511B +5128 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5128 +512B ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-512B +5142 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5142 +514A ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-514A +514F ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-514F +5153 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5153 +5158 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5158 +5160 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5160 +5164 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5164 +5172 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5172 +517E ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-517E +5183..5184 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-5183..CJK UNIFIED IDEOGRAPH-5184 +518E ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-518E +51A1 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-51A1 +51A3 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-51A3 +51AD ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-51AD +51B8 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-51B8 +51BA ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-51BA +51C2 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-51C2 +51D2..51D3 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-51D2..CJK UNIFIED IDEOGRAPH-51D3 +51DF ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-51DF +51EC ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-51EC +51EE ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-51EE +51F2 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-51F2 +5253 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5253 +5266 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5266 +5279 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5279 +5285 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5285 +528E ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-528E +52C4 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-52C4 +52C8 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-52C8 +52CC ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-52CC +52CE ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-52CE +52D1 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-52D1 +52D4 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-52D4 +52E1 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-52E1 +52E5 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-52E5 +52EE ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-52EE +5303..5304 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-5303..CJK UNIFIED IDEOGRAPH-5304 +5318 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5318 +531B ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-531B +531E ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-531E +5327 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5327 +5329 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5329 +5332 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5332 +5335..5336 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-5335..CJK UNIFIED IDEOGRAPH-5336 +5342 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5342 +535B ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-535B +535D ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-535D +536A ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-536A +536D ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-536D +5380 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5380 +53A1 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-53A1 +53AA..53AB ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-53AA..CJK UNIFIED IDEOGRAPH-53AB +53AF ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-53AF +53BA ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-53BA +53C5 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-53C5 +53CF ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-53CF +53DD..53DE ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-53DD..CJK UNIFIED IDEOGRAPH-53DE +53E7 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-53E7 +53FF..5400 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-53FF..CJK UNIFIED IDEOGRAPH-5400 +541A ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-541A +5422 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5422 +544C ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-544C +545D ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-545D +5469 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5469 +548A ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-548A +54B5 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-54B5 +54F6 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-54F6 +5515 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5515 +5518..5519 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-5518..CJK UNIFIED IDEOGRAPH-5519 +5547 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5547 +5560 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5560 +557A ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-557A +55E0 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-55E0 +55F8 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-55F8 +560A..560B ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-560A..CJK UNIFIED IDEOGRAPH-560B +5620 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5620 +562B ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-562B +5637 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5637 +563C ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-563C +5644 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5644 +564B ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-564B +5651 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5651 +5656 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5656 +565F ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-565F +5661 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5661 +5675 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5675 +567D ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-567D +5688 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5688 +568B ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-568B +5696 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5696 +569E ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-569E +56BA ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-56BA +56CF ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-56CF +56D9 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-56D9 +56E6 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-56E6 +56F6 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-56F6 +56F8 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-56F8 +56FB..56FC ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-56FB..CJK UNIFIED IDEOGRAPH-56FC +5705 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5705 +5711 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5711 +5717 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5717 +5721 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5721 +5724 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5724 +573D ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-573D +5743 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5743 +5748 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5748 +5755..5756 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-5755..CJK UNIFIED IDEOGRAPH-5756 +5758 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5758 +5763 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5763 +5778 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5778 +5781 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5781 +5787 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5787 +5796 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5796 +57A8 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-57A8 +57CA ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-57CA +57D1 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-57D1 +57DB ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-57DB +5817..5818 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-5817..CJK UNIFIED IDEOGRAPH-5818 +5850 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5850 +5856 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5856 +5860 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5860 +5866..5867 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-5866..CJK UNIFIED IDEOGRAPH-5867 +5877 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5877 +5895 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5895 +58AA ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-58AA +58B6 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-58B6 +58C0 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-58C0 +58C3..58C4 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-58C3..CJK UNIFIED IDEOGRAPH-58C4 +58CD ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-58CD +58D0 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-58D0 +58E1 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-58E1 +58E6 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-58E6 +58F5 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-58F5 +5901 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5901 +5905 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5905 +5908 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5908 +5911 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5911 +5913 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5913 +5923 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5923 +5933 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5933 +5936 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5936 +5959 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5959 +595B ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-595B +59B7 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-59B7 +59E7 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-59E7 +5A24 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5A24 +5A26 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5A26 +5A2C ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5A2C +5A30 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5A30 +5A54 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5A54 +5A59 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5A59 +5A6F ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5A6F +5A71 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5A71 +5A87 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5A87 +5A8D ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5A8D +5AAB ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5AAB +5AD3 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5AD3 +5AEF..5AF0 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-5AEF..CJK UNIFIED IDEOGRAPH-5AF0 +5B0A ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5B0A +5B0D ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5B0D +5B39 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5B39 +5B46 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5B46 +5B4F ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5B4F +5B52 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5B52 +5B60..5B61 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-5B60..CJK UNIFIED IDEOGRAPH-5B61 +5B6F ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5B6F +5B79 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5B79 +5B7E ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5B7E +5B86 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5B86 +5B90 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5B90 +5BA9 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5BA9 +5BB2 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5BB2 +5BB7 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5BB7 +5BBC ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5BBC +5BC8 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5BC8 +5BDA ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5BDA +5C00 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5C00 +5C1B ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5C1B +5C23 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5C23 +5C26 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5C26 +5C29 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5C29 +5C36 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5C36 +5C5A ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5C5A +5C85 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5C85 +5CB4 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5CB4 +5CB9 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5CB9 +5CD5 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5CD5 +5CDD ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5CDD +5CF5 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5CF5 +5D2B ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5D2B +5D2F ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5D2F +5D3B ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5D3B +5D53 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5D53 +5D57 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5D57 +5D60 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5D60 +5D83 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5D83 +5D96 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5D96 +5DA3..5DA4 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-5DA3..CJK UNIFIED IDEOGRAPH-5DA4 +5DAB ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5DAB +5DB3 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5DB3 +5DB9 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5DB9 +5DC4 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5DC4 +5DD7 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5DD7 +5DDA ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5DDA +5DDC ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5DDC +5DF6 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5DF6 +5E12 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5E12 +5E48 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5E48 +5E51 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5E51 +5E92 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5E92 +5EBA ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5EBA +5EC0 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5EC0 +5EEB ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5EEB +5EF9 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5EF9 +5F0E ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5F0E +5F3B ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5F3B +5F3D ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5F3D +5F8F ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5F8F +5F9A ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5F9A +5FA3..5FA4 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-5FA3..CJK UNIFIED IDEOGRAPH-5FA4 +5FB0 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5FB0 +5FC2 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5FC2 +5FCE ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5FCE +5FDB ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5FDB +5FE2 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5FE2 +5FEC ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5FEC +5FFC ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-5FFC +6023 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6023 +6056 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6056 +6061 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6061 +6071 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6071 +6074 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6074 +6091 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6091 +6093 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6093 +60A5 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-60A5 +60D2 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-60D2 +60D6 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-60D6 +60DE ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-60DE +60E5 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-60E5 +60FD ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-60FD +6102 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6102 +6107 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6107 +6111 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6111 +611E ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-611E +6131 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6131 +6133 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6133 +6135 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6135 +6138..6139 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-6138..CJK UNIFIED IDEOGRAPH-6139 +6160 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6160 +617B ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-617B +617F ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-617F +6186 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6186 +6197 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6197 +619C ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-619C +61B9 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-61B9 +61BB ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-61BB +61D3 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-61D3 +61D5 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-61D5 +61EC ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-61EC +61EF ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-61EF +6205 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6205 +6235 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6235 +6239 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6239 +6257 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6257 +628D ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-628D +629D ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-629D +62DE ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-62DE +62EA ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-62EA +630A ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-630A +6317 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6317 +6331 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6331 +6337 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6337 +635B ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-635B +638B ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-638B +6393 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6393 +63D1 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-63D1 +643B..643C ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-643B..CJK UNIFIED IDEOGRAPH-643C +6449 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6449 +645A ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-645A +647E ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-647E +6486 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6486 +64A1 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-64A1 +64AF ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-64AF +64B6 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-64B6 +64C8 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-64C8 +64D5 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-64D5 +64EE ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-64EE +64F5 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-64F5 +64F9 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-64F9 +6502 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6502 +650A ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-650A +651F ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-651F +6528 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6528 +6540 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6540 +6542 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6542 +655A ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-655A +655F ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-655F +657D ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-657D +658A ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-658A +659A ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-659A +65B5 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-65B5 +65BE ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-65BE +65C8..65C9 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-65C8..CJK UNIFIED IDEOGRAPH-65C9 +65D1 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-65D1 +65D8 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-65D8 +65DC ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-65DC +65E4 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-65E4 +65EA ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-65EA +65F9 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-65F9 +65FE ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-65FE +6617 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6617 +662C ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-662C +6637..6638 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-6637..CJK UNIFIED IDEOGRAPH-6638 +6648 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6648 +664D ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-664D +6660 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6660 +6663 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6663 +6692 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6692 +669C ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-669C +669E ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-669E +66AC..66AD ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-66AC..CJK UNIFIED IDEOGRAPH-66AD +66D0 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-66D0 +66D3 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-66D3 +66D7 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-66D7 +66DF ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-66DF +66EF ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-66EF +6702 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6702 +6707 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6707 +6719 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6719 +6724 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6724 +6729 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6729 +6767 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6767 +6788 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6788 +6796 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6796 +67BD ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-67BD +67BF ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-67BF +67D5 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-67D5 +67D7 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-67D7 +67F9 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-67F9 +6801 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6801 +6815 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6815 +6827 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6827 +6830 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6830 +6858 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6858 +685A ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-685A +685E ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-685E +687A ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-687A +6895 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6895 +6899 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6899 +68A5 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-68A5 +68B8 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-68B8 +68C3 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-68C3 +68D9 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-68D9 +68E2 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-68E2 +68E5 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-68E5 +6909 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6909 +693E ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-693E +694D ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-694D +699F ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-699F +69A2 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-69A2 +69C0 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-69C0 +69D1..69D2 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-69D1..CJK UNIFIED IDEOGRAPH-69D2 +69D5..69D7 ; Uncommon_Use # 1.1 [3] CJK UNIFIED IDEOGRAPH-69D5..CJK UNIFIED IDEOGRAPH-69D7 +6A03 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6A03 +6A1C ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6A1C +6A24 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6A24 +6A37 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6A37 +6A4A ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6A4A +6A5C ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6A5C +6A6E ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6A6E +6A70 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6A70 +6A86 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6A86 +6A8A ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6A8A +6A8F ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6A8F +6A99 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6A99 +6A9D ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6A9D +6AB1 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6AB1 +6ABE ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6ABE +6AC0 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6AC0 +6AC4 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6AC4 +6AC9 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6AC9 +6AD8 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6AD8 +6AE9 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6AE9 +6B0E ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6B0E +6B1B ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6B1B +6B2E ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6B2E +6B35 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6B35 +6B40 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6B40 +6B57..6B58 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-6B57..CJK UNIFIED IDEOGRAPH-6B58 +6B5D ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6B5D +6B68 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6B68 +6B6C ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6B6C +6B6E ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6B6E +6B71 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6B71 +6B75 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6B75 +6B7D ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6B7D +6BB8 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6BB8 +6BE9 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6BE9 +6BF1 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6BF1 +6BF4 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6BF4 +6BFA ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6BFA +6C0A ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6C0A +6C1C ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6C1C +6C2D ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6C2D +6C3C ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6C3C +6C45 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6C45 +6C6C ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6C6C +6C6E ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6C6E +6CA0 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6CA0 +6CD8 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6CD8 +6CF4 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6CF4 +6D02 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6D02 +6D1C ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6D1C +6D24 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6D24 +6D71 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6D71 +6D81 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6D81 +6D96 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6D96 +6DB0..6DB1 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-6DB0..CJK UNIFIED IDEOGRAPH-6DB1 +6DB6 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6DB6 +6DFE..6DFF ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-6DFE..CJK UNIFIED IDEOGRAPH-6DFF +6E01..6E02 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-6E01..CJK UNIFIED IDEOGRAPH-6E02 +6E06 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6E06 +6E12 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6E12 +6E18 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6E18 +6E2A ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6E2A +6E4C ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6E4C +6E6C ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6E6C +6E7B..6E7D ; Uncommon_Use # 1.1 [3] CJK UNIFIED IDEOGRAPH-6E7B..CJK UNIFIED IDEOGRAPH-6E7D +6E8B ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6E8B +6E95 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6E95 +6EDB ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6EDB +6EE3 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6EE3 +6F04 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6F04 +6F0B ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6F0B +6F42 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6F42 +6F48 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6F48 +6F4A ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6F4A +6F79 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6F79 +6F98 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6F98 +6F9A ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6F9A +6F9F ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6F9F +6FB7 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6FB7 +6FC5 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6FC5 +6FD0 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6FD0 +6FD3 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6FD3 +6FF5 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6FF5 +6FFD ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-6FFD +7010 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7010 +7013 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7013 +7047 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7047 +704B ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-704B +704E ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-704E +7072..7073 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-7072..CJK UNIFIED IDEOGRAPH-7073 +707B ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-707B +7081 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7081 +708D ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-708D +7097 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7097 +709B ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-709B +70AA ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-70AA +70B2 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-70B2 +70B6 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-70B6 +70D5 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-70D5 +70FE ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-70FE +7108 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7108 +7124 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7124 +7133..7134 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-7133..CJK UNIFIED IDEOGRAPH-7134 +7157 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7157 +716B ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-716B +716D ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-716D +718D ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-718D +7196 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7196 +71A6 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-71A6 +71AB ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-71AB +71B6 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-71B6 +71CC ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-71CC +71D3 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-71D3 +71F3 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-71F3 +71FA ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-71FA +720B ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-720B +7211 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7211 +7215 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7215 +7217 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7217 +7220 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7220 +7224..7225 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-7224..CJK UNIFIED IDEOGRAPH-7225 +722F ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-722F +7234 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7234 +7245 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7245 +724E ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-724E +7250 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7250 +7255 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7255 +72AB ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-72AB +72BE ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-72BE +7302 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7302 +7310 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7310 +7328 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7328 +7353 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7353 +739C ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-739C +73C1 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-73C1 +73F3 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-73F3 +73FB ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-73FB +7418 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7418 +7439 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7439 +743E ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-743E +7447 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7447 +7449 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7449 +7458 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7458 +747B ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-747B +7484 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7484 +7496 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7496 +749D ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-749D +74C7 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-74C7 +74C9 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-74C9 +74CC ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-74CC +74EB ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-74EB +7520 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7520 +7541 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7541 +7552 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7552 +7555 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7555 +755E ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-755E +7561 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7561 +7571 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7571 +757B ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-757B +7585 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7585 +75A9 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-75A9 +75B7 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-75B7 +75DC ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-75DC +75EE ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-75EE +762C ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-762C +7644..7645 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-7644..CJK UNIFIED IDEOGRAPH-7645 +7651 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7651 +7655 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7655 +7673 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7673 +768D ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-768D +76A1..76A2 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-76A1..CJK UNIFIED IDEOGRAPH-76A2 +76A5 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-76A5 +76A8 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-76A8 +76B3 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-76B3 +76B6 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-76B6 +76C1 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-76C1 +76CB ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-76CB +76D9 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-76D9 +76EB ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-76EB +7700 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7700 +7702 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7702 +770E ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-770E +7721 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7721 +772B ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-772B +773F ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-773F +7742 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7742 +7764 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7764 +7796 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7796 +77A4 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-77A4 +77BE ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-77BE +77C1 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-77C1 +77D2 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-77D2 +77DD ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-77DD +77E4 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-77E4 +77E6 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-77E6 +77F4..77F5 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-77F4..CJK UNIFIED IDEOGRAPH-77F5 +7824 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7824 +7836 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7836 +7842 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7842 +7846 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7846 +784B ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-784B +7876 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7876 +7888 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7888 +78C2 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-78C2 +78C7 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-78C7 +78D2 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-78D2 +78F0 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-78F0 +78F8 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-78F8 +7900 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7900 +7908 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7908 +790D ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-790D +7915 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7915 +791F..7920 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-791F..CJK UNIFIED IDEOGRAPH-7920 +7932 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7932 +7936 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7936 +7959 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7959 +796C ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-796C +796E ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-796E +7975..7976 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-7975..CJK UNIFIED IDEOGRAPH-7976 +7986..7987 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-7986..CJK UNIFIED IDEOGRAPH-7987 +799E ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-799E +79A9 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-79A9 +79BC ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-79BC +79C4 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-79C4 +79C7 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-79C7 +79CC ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-79CC +79D4 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-79D4 +79D7 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-79D7 +7A01 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7A01 +7A07 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7A07 +7A09 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7A09 +7A2C ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7A2C +7A38 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7A38 +7A3A ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7A3A +7A64 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7A64 +7A6A ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7A6A +7A6F ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7A6F +7A82 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7A82 +7A9A..7A9B ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-7A9A..CJK UNIFIED IDEOGRAPH-7A9B +7AB9 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7AB9 +7ABB..7ABD ; Uncommon_Use # 1.1 [3] CJK UNIFIED IDEOGRAPH-7ABB..CJK UNIFIED IDEOGRAPH-7ABD +7AC2 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7AC2 +7AC6 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7AC6 +7AE9 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7AE9 +7AF5 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7AF5 +7AFC ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7AFC +7B07 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7B07 +7B1F ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7B1F +7B27 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7B27 +7B29 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7B29 +7B42 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7B42 +7B53 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7B53 +7BA3 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7BA3 +7BA5 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7BA5 +7BB0 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7BB0 +7BB2 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7BB2 +7BFA ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7BFA +7C1B ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7C1B +7C2E..7C2F ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-7C2E..CJK UNIFIED IDEOGRAPH-7C2F +7C52 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7C52 +7C55 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7C55 +7C5D ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7C5D +7C76 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7C76 +7C87 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7C87 +7C93 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7C93 +7C9A ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7C9A +7CAC ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7CAC +7CD3 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7CD3 +7CDA..7CDB ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-7CDA..CJK UNIFIED IDEOGRAPH-7CDB +7CE1 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7CE1 +7CE3 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7CE3 +7CE5..7CE6 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-7CE5..CJK UNIFIED IDEOGRAPH-7CE6 +7CFC ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7CFC +7CFF ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7CFF +7D23 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7D23 +7D2A ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7D2A +7D2D ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7D2D +7D48 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7D48 +7D4D ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7D4D +7D5A ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7D5A +7D64 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7D64 +7D78 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7D78 +7D82 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7D82 +7D95 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7D95 +7D98 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7D98 +7DA4 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7DA4 +7DA8 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7DA8 +7DCD ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7DCD +7DD3 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7DD3 +7DE5 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7DE5 +7DEB ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7DEB +7DFD..7DFF ; Uncommon_Use # 1.1 [3] CJK UNIFIED IDEOGRAPH-7DFD..CJK UNIFIED IDEOGRAPH-7DFF +7E18 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7E18 +7E5B ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7E5B +7E64 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7E64 +7E9D ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7E9D +7F3B..7F3C ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-7F3B..CJK UNIFIED IDEOGRAPH-7F3C +7F41 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7F41 +7F46 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7F46 +7F59 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7F59 +7F84 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7F84 +7F90 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7F90 +7F97 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7F97 +7F99 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7F99 +7FB4 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7FB4 +7FD6 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7FD6 +7FDD ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7FDD +7FE4 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-7FE4 +800A ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-800A +802F ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-802F +803C ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-803C +8040 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8040 +8066 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8066 +8088 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8088 +808E ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-808E +8094 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8094 +80A6..80A8 ; Uncommon_Use # 1.1 [3] CJK UNIFIED IDEOGRAPH-80A6..CJK UNIFIED IDEOGRAPH-80A8 +80B3 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-80B3 +80B9 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-80B9 +80DF ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-80DF +8103..8104 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-8103..CJK UNIFIED IDEOGRAPH-8104 +8134..8135 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-8134..CJK UNIFIED IDEOGRAPH-8135 +8184 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8184 +8190 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8190 +8196 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8196 +81CB ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-81CB +81E4 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-81E4 +81EF..81F0 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-81EF..CJK UNIFIED IDEOGRAPH-81F0 +8213 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8213 +8224 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8224 +8241 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8241 +8265 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8265 +828C ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-828C +82B2 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-82B2 +82E2 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-82E2 +82FC ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-82FC +830A ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-830A +8310 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8310 +8330 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8330 +8355 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8355 +83BE ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-83BE +83E6 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-83E6 +83ED ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-83ED +8414 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8414 +8416..8417 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-8416..CJK UNIFIED IDEOGRAPH-8417 +841F ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-841F +8458 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8458 +8483 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8483 +8495 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8495 +84B7 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-84B7 +84C3 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-84C3 +84ED ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-84ED +8505 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8505 +8510 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8510 +8532..8533 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-8532..CJK UNIFIED IDEOGRAPH-8533 +854C ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-854C +8550 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8550 +857F ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-857F +8593 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8593 +85B2 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-85B2 +85BB ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-85BB +85CC ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-85CC +85EE ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-85EE +85F3 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-85F3 +85FC ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-85FC +8603 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8603 +860D..860E ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-860D..CJK UNIFIED IDEOGRAPH-860E +8610 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8610 +8615 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8615 +861D ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-861D +8637 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8637 +8657 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8657 +8675 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8675 +8689 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8689 +8692 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8692 +86A0 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-86A0 +86A6 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-86A6 +86D5 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-86D5 +86E0 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-86E0 +86E7 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-86E7 +86FD ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-86FD +871D ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-871D +872F ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-872F +873D ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-873D +8745 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8745 +8771 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8771 +878E ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-878E +8799 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8799 +87DA ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-87DA +87F0 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-87F0 +8807 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8807 +8812 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8812 +882D ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-882D +883A ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-883A +8847 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8847 +8858 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8858 +885C ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-885C +885F ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-885F +887A ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-887A +88E6 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-88E6 +88E9 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-88E9 +88ED ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-88ED +8903 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8903 +890F ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-890F +8924 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8924 +8965 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8965 +8975 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8975 +897D ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-897D +898D ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-898D +8990 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8990 +8994 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8994 +8999 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8999 +89B0 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-89B0 +89B4 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-89B4 +89BB..89BC ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-89BB..CJK UNIFIED IDEOGRAPH-89BC +89EE ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-89EE +89F5 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-89F5 +89F9 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-89F9 +89FD ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-89FD +8A05..8A06 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-8A05..CJK UNIFIED IDEOGRAPH-8A06 +8A14 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8A14 +8A19 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8A19 +8A20..8A21 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-8A20..CJK UNIFIED IDEOGRAPH-8A21 +8A2B ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8A2B +8A3D ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8A3D +8A4B ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8A4B +8A64 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8A64 +8A78 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8A78 +8A7D ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8A7D +8A88 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8A88 +8A9F ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8A9F +8AAF ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8AAF +8AB7 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8AB7 +8AD0 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8AD0 +8AEC ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8AEC +8B29 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8B29 +8B32 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8B32 +8B38 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8B38 +8B3F ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8B3F +8B61..8B62 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-8B61..CJK UNIFIED IDEOGRAPH-8B62 +8B69 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8B69 +8B75 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8B75 +8B7C ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8B7C +8B81 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8B81 +8B87 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8B87 +8B8D ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8B8D +8B8F ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8B8F +8B9B ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8B9B +8C38 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8C38 +8C40 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8C40 +8C44 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8C44 +8C51..8C53 ; Uncommon_Use # 1.1 [3] CJK UNIFIED IDEOGRAPH-8C51..CJK UNIFIED IDEOGRAPH-8C53 +8C58 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8C58 +8C74 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8C74 +8C7F ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8C7F +8C83 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8C83 +8C87 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8C87 +8C8B ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8C8B +8C9B ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8C9B +8CA6 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8CA6 +8CCB ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8CCB +8CD6 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8CD6 +8CD8 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8CD8 +8CE9 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8CE9 +8CF7 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8CF7 +8D01 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8D01 +8D11..8D12 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-8D11..CJK UNIFIED IDEOGRAPH-8D12 +8D7C ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8D7C +8DA6 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8DA6 +8DC0 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8DC0 +8DE5 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8DE5 +8E01 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8E01 +8E0B ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8E0B +8E32 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8E32 +8E46 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8E46 +8E4F ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8E4F +8E6E ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8E6E +8E75 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8E75 +8E77 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8E77 +8E79 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8E79 +8E9B ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8E9B +8EA2 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8EA2 +8EB3 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8EB3 +8EB6..8EB7 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-8EB6..CJK UNIFIED IDEOGRAPH-8EB7 +8EC1 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8EC1 +8EC4 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8EC4 +8ED9 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8ED9 +8EF0 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8EF0 +8F0F ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8F0F +8F2D ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8F2D +8F3A ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8F3A +8F41 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8F41 +8F9D ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8F9D +8FA4 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8FA4 +8FB3 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8FB3 +8FC3 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8FC3 +8FCA ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8FCA +8FE7 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-8FE7 +902A ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-902A +902C ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-902C +9037 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9037 +9040 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9040 +9046 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9046 +90AB ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-90AB +90CC..90CD ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-90CC..CJK UNIFIED IDEOGRAPH-90CD +90D2 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-90D2 +90F6 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-90F6 +910A ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-910A +913C..913D ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-913C..CJK UNIFIED IDEOGRAPH-913D +9159 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9159 +917B ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-917B +9195 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9195 +9198 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9198 +91A9 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-91A9 +91BF ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-91BF +91C4 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-91C4 +91E0 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-91E0 +91EF ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-91EF +9213 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9213 +921F ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-921F +9222 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9222 +9243 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9243 +9269..926A ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-9269..CJK UNIFIED IDEOGRAPH-926A +9281 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9281 +9284 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9284 +929E ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-929E +92BD ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-92BD +92D4 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-92D4 +92DB ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-92DB +92E2 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-92E2 +931C ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-931C +9330..9331 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-9330..CJK UNIFIED IDEOGRAPH-9331 +9362 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9362 +9368 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9368 +936B ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-936B +936F ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-936F +9373 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9373 +9378 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9378 +937F ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-937F +9381 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9381 +938B ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-938B +939C ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-939C +93A0 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-93A0 +93AB ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-93AB +93BB ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-93BB +93E0 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-93E0 +93F3 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-93F3 +9402 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9402 +9417 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9417 +941C ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-941C +941E..941F ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-941E..CJK UNIFIED IDEOGRAPH-941F +9424 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9424 +9443 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9443 +944E ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-944E +946C ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-946C +947B ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-947B +9578..9579 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-9578..CJK UNIFIED IDEOGRAPH-9579 +957E ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-957E +9585 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9585 +9597 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9597 +95B3..95B4 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-95B3..CJK UNIFIED IDEOGRAPH-95B4 +95B8 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-95B8 +95C1 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-95C1 +95D9 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-95D9 +95DD ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-95DD +9625..9626 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-9625..CJK UNIFIED IDEOGRAPH-9626 +9629 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9629 +963E ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-963E +9656..9657 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-9656..CJK UNIFIED IDEOGRAPH-9657 +9679 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9679 +967B ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-967B +967F ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-967F +9681..9682 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-9681..CJK UNIFIED IDEOGRAPH-9682 +968C ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-968C +9696 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9696 +969A ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-969A +969D ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-969D +969F ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-969F +96AB ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-96AB +96AF ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-96AF +96B5 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-96B5 +96E4 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-96E4 +96E6..96E7 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-96E6..CJK UNIFIED IDEOGRAPH-96E7 +96FC ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-96FC +9714 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9714 +9717 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9717 +971A..971B ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-971A..CJK UNIFIED IDEOGRAPH-971B +9733..9734 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-9733..CJK UNIFIED IDEOGRAPH-9734 +9737 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9737 +9740..9741 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-9740..CJK UNIFIED IDEOGRAPH-9741 +974D ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-974D +9757 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9757 +9763 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9763 +9775 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9775 +9787 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9787 +9789 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9789 +979B ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-979B +97A9 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-97A9 +97B0..97B1 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-97B0..CJK UNIFIED IDEOGRAPH-97B1 +97B5 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-97B5 +97BE ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-97BE +97C0 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-97C0 +97D2 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-97D2 +97FC ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-97FC +981F ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-981F +9825 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9825 +982A ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-982A +9833 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9833 +983A ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-983A +983E ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-983E +9842 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9842 +9847 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9847 +9856 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9856 +9866 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9866 +9868 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9868 +98B7 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-98B7 +98CA ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-98CA +98E4 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-98E4 +98EC ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-98EC +98F1 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-98F1 +98F8 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-98F8 +98FB ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-98FB +9919 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9919 +993B ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-993B +9944 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9944 +995A ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-995A +995D ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-995D +99BF ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-99BF +99E0 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-99E0 +99E6 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-99E6 +99EB ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-99EB +99F5 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-99F5 +9A10 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9A10 +9A17..9A18 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-9A17..CJK UNIFIED IDEOGRAPH-9A18 +9A3B ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9A3B +9A51 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9A51 +9A58 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9A58 +9A5D ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9A5D +9A63 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9A63 +9AA9 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9AA9 +9ABD ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9ABD +9AC8 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9AC8 +9AD7 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9AD7 +9AE0 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9AE0 +9AE4 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9AE4 +9AE8..9AE9 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-9AE8..CJK UNIFIED IDEOGRAPH-9AE9 +9AF0 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9AF0 +9B00 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9B00 +9B02 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9B02 +9B09 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9B09 +9B14 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9B14 +9B1B ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9B1B +9B34 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9B34 +9B3D ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9B3D +9B40 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9B40 +9B50 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9B50 +9B57 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9B57 +9B62 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9B62 +9B72 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9B72 +9B89 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9B89 +9B8C ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9B8C +9B99 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9B99 +9BC2 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9BC2 +9BF6 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9BF6 +9C00..9C01 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-9C00..CJK UNIFIED IDEOGRAPH-9C01 +9C03 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9C03 +9C42 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9C42 +9C4F ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9C4F +9C51 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9C51 +9C61 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9C61 +9C64 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9C64 +9C7B ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9C7B +9D0C..9D0D ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-9D0C..CJK UNIFIED IDEOGRAPH-9D0D +9D11 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9D11 +9D27 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9D27 +9D35 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9D35 +9D3C ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9D3C +9D6D ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9D6D +9D95 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9D95 +9DAE ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9DAE +9DBD ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9DBD +9DC0 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9DC0 +9DEA ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9DEA +9DFC ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9DFC +9E0E ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9E0E +9E16 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9E16 +9E1C ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9E1C +9E7B ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9E7B +9E8F..9E90 ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-9E8F..CJK UNIFIED IDEOGRAPH-9E90 +9E98 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9E98 +9E9E ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9E9E +9EA2 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9EA2 +9EAB..9EAC ; Uncommon_Use # 1.1 [2] CJK UNIFIED IDEOGRAPH-9EAB..CJK UNIFIED IDEOGRAPH-9EAC +9EB1 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9EB1 +9EEC ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9EEC +9EF1 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9EF1 +9F03 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9F03 +9F11 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9F11 +9F14 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9F14 +9F26 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9F26 +9F45 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9F45 +9F53 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9F53 +9F6D ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9F6D +9FA1 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9FA1 +9FA3 ; Uncommon_Use # 1.1 CJK UNIFIED IDEOGRAPH-9FA3 +9FA6..9FBB ; Uncommon_Use # 4.1 [22] CJK UNIFIED IDEOGRAPH-9FA6..CJK UNIFIED IDEOGRAPH-9FBB +9FBC..9FC3 ; Uncommon_Use # 5.1 [8] CJK UNIFIED IDEOGRAPH-9FBC..CJK UNIFIED IDEOGRAPH-9FC3 +9FC4..9FCB ; Uncommon_Use # 5.2 [8] CJK UNIFIED IDEOGRAPH-9FC4..CJK UNIFIED IDEOGRAPH-9FCB +9FCC ; Uncommon_Use # 6.1 CJK UNIFIED IDEOGRAPH-9FCC +9FCD..9FD5 ; Uncommon_Use # 8.0 [9] CJK UNIFIED IDEOGRAPH-9FCD..CJK UNIFIED IDEOGRAPH-9FD5 +9FD6..9FEA ; Uncommon_Use # 10.0 [21] CJK UNIFIED IDEOGRAPH-9FD6..CJK UNIFIED IDEOGRAPH-9FEA +9FEB..9FEF ; Uncommon_Use # 11.0 [5] CJK UNIFIED IDEOGRAPH-9FEB..CJK UNIFIED IDEOGRAPH-9FEF +9FF0..9FFC ; Uncommon_Use # 13.0 [13] CJK UNIFIED IDEOGRAPH-9FF0..CJK UNIFIED IDEOGRAPH-9FFC +9FFD..9FFF ; Uncommon_Use # 14.0 [3] CJK UNIFIED IDEOGRAPH-9FFD..CJK UNIFIED IDEOGRAPH-9FFF +A66F ; Uncommon_Use # 5.1 COMBINING CYRILLIC VZMET +A67C..A67D ; Uncommon_Use # 5.1 [2] COMBINING CYRILLIC KAVYKA..COMBINING CYRILLIC PAYEROK +A78B..A78C ; Uncommon_Use # 5.1 [2] LATIN CAPITAL LETTER SALTILLO..LATIN SMALL LETTER SALTILLO +A78F ; Uncommon_Use # 8.0 LATIN LETTER SINOLOGICAL DOT +A792..A793 ; Uncommon_Use # 6.1 [2] LATIN CAPITAL LETTER C WITH BAR..LATIN SMALL LETTER C WITH BAR +A7B2..A7B7 ; Uncommon_Use # 8.0 [6] LATIN CAPITAL LETTER J WITH CROSSED-TAIL..LATIN SMALL LETTER OMEGA +A7B8..A7B9 ; Uncommon_Use # 11.0 [2] LATIN CAPITAL LETTER U WITH STROKE..LATIN SMALL LETTER U WITH STROKE +A7C2..A7C3 ; Uncommon_Use # 12.0 [2] LATIN CAPITAL LETTER ANGLICANA W..LATIN SMALL LETTER ANGLICANA W +A7CB..A7CD ; Uncommon_Use # 16.0 [3] LATIN CAPITAL LETTER RAMS HORN..LATIN SMALL LETTER S WITH DIAGONAL STROKE +A7CE..A7CF ; Uncommon_Use # 17.0 [2] LATIN CAPITAL LETTER PHARYNGEAL VOICED FRICATIVE..LATIN SMALL LETTER PHARYNGEAL VOICED FRICATIVE +A7DA..A7DC ; Uncommon_Use # 16.0 [3] LATIN CAPITAL LETTER LAMBDA..LATIN CAPITAL LETTER LAMBDA WITH STROKE +A9E7..A9FE ; Uncommon_Use # 7.0 [24] MYANMAR LETTER TAI LAING NYA..MYANMAR LETTER TAI LAING BHA +AA60..AA76 ; Uncommon_Use # 5.2 [23] MYANMAR LETTER KHAMTI GA..MYANMAR LOGOGRAM KHAMTI HM +AA7A ; Uncommon_Use # 5.2 MYANMAR LETTER AITON RA +AA7C..AA7F ; Uncommon_Use # 7.0 [4] MYANMAR SIGN TAI LAING TONE-2..MYANMAR LETTER SHWE PALAUNG SHA +AB01..AB06 ; Uncommon_Use # 6.0 [6] ETHIOPIC SYLLABLE TTHU..ETHIOPIC SYLLABLE TTHO +AB09..AB0E ; Uncommon_Use # 6.0 [6] ETHIOPIC SYLLABLE DDHU..ETHIOPIC SYLLABLE DDHO +AB11..AB16 ; Uncommon_Use # 6.0 [6] ETHIOPIC SYLLABLE DZU..ETHIOPIC SYLLABLE DZO +AB20..AB26 ; Uncommon_Use # 6.0 [7] ETHIOPIC SYLLABLE CCHHA..ETHIOPIC SYLLABLE CCHHO +AB28..AB2E ; Uncommon_Use # 6.0 [7] ETHIOPIC SYLLABLE BBA..ETHIOPIC SYLLABLE BBO +AB60..AB63 ; Uncommon_Use # 8.0 [4] LATIN SMALL LETTER SAKHA YAT..LATIN SMALL LETTER UO +AB66..AB67 ; Uncommon_Use # 12.0 [2] LATIN SMALL LETTER DZ DIGRAPH WITH RETROFLEX HOOK..LATIN SMALL LETTER TS DIGRAPH WITH RETROFLEX HOOK +FA0E..FA0F ; Uncommon_Use # 1.1 [2] CJK COMPATIBILITY IDEOGRAPH-FA0E..CJK COMPATIBILITY IDEOGRAPH-FA0F +FA11 ; Uncommon_Use # 1.1 CJK COMPATIBILITY IDEOGRAPH-FA11 +FA13..FA14 ; Uncommon_Use # 1.1 [2] CJK COMPATIBILITY IDEOGRAPH-FA13..CJK COMPATIBILITY IDEOGRAPH-FA14 +FA1F ; Uncommon_Use # 1.1 CJK COMPATIBILITY IDEOGRAPH-FA1F +FA21 ; Uncommon_Use # 1.1 CJK COMPATIBILITY IDEOGRAPH-FA21 +FA23..FA24 ; Uncommon_Use # 1.1 [2] CJK COMPATIBILITY IDEOGRAPH-FA23..CJK COMPATIBILITY IDEOGRAPH-FA24 +FA27..FA29 ; Uncommon_Use # 1.1 [3] CJK COMPATIBILITY IDEOGRAPH-FA27..CJK COMPATIBILITY IDEOGRAPH-FA29 +10780 ; Uncommon_Use # 14.0 MODIFIER LETTER SMALL CAPITAL AA +10EC2..10EC4 ; Uncommon_Use # 16.0 [3] ARABIC LETTER DAL WITH TWO DOTS VERTICALLY BELOW..ARABIC LETTER KAF WITH TWO DOTS VERTICALLY BELOW +10EC7 ; Uncommon_Use # 17.0 ARABIC LETTER YEH WITH FOUR DOTS BELOW +10EFA ; Uncommon_Use # 17.0 ARABIC DOUBLE VERTICAL BAR BELOW +10EFC ; Uncommon_Use # 16.0 ARABIC COMBINING ALEF OVERLAY +10EFD..10EFF ; Uncommon_Use # 15.0 [3] ARABIC SMALL LOW WORD SAKTA..ARABIC SMALL LOW WORD MADDA +1133B ; Uncommon_Use # 11.0 COMBINING BINDU BELOW +116D0..116E3 ; Uncommon_Use # 16.0 [20] MYANMAR PAO DIGIT ZERO..MYANMAR EASTERN PWO KAREN DIGIT NINE +1AFF0..1AFF3 ; Uncommon_Use # 14.0 [4] KATAKANA LETTER MINNAN TONE-2..KATAKANA LETTER MINNAN TONE-5 +1AFF5..1AFFB ; Uncommon_Use # 14.0 [7] KATAKANA LETTER MINNAN TONE-7..KATAKANA LETTER MINNAN NASALIZED TONE-5 +1AFFD..1AFFE ; Uncommon_Use # 14.0 [2] KATAKANA LETTER MINNAN NASALIZED TONE-7..KATAKANA LETTER MINNAN NASALIZED TONE-8 +20000..2070D ; Uncommon_Use # 3.1 [1806] CJK UNIFIED IDEOGRAPH-20000..CJK UNIFIED IDEOGRAPH-2070D +2070F..20730 ; Uncommon_Use # 3.1 [34] CJK UNIFIED IDEOGRAPH-2070F..CJK UNIFIED IDEOGRAPH-20730 +20732..20778 ; Uncommon_Use # 3.1 [71] CJK UNIFIED IDEOGRAPH-20732..CJK UNIFIED IDEOGRAPH-20778 +2077A..20C52 ; Uncommon_Use # 3.1 [1241] CJK UNIFIED IDEOGRAPH-2077A..CJK UNIFIED IDEOGRAPH-20C52 +20C54..20C77 ; Uncommon_Use # 3.1 [36] CJK UNIFIED IDEOGRAPH-20C54..CJK UNIFIED IDEOGRAPH-20C77 +20C79..20C95 ; Uncommon_Use # 3.1 [29] CJK UNIFIED IDEOGRAPH-20C79..CJK UNIFIED IDEOGRAPH-20C95 +20C97..20CCE ; Uncommon_Use # 3.1 [56] CJK UNIFIED IDEOGRAPH-20C97..CJK UNIFIED IDEOGRAPH-20CCE +20CD0..20CD4 ; Uncommon_Use # 3.1 [5] CJK UNIFIED IDEOGRAPH-20CD0..CJK UNIFIED IDEOGRAPH-20CD4 +20CD6..20D14 ; Uncommon_Use # 3.1 [63] CJK UNIFIED IDEOGRAPH-20CD6..CJK UNIFIED IDEOGRAPH-20D14 +20D16..20D7B ; Uncommon_Use # 3.1 [102] CJK UNIFIED IDEOGRAPH-20D16..CJK UNIFIED IDEOGRAPH-20D7B +20D7D..20D7E ; Uncommon_Use # 3.1 [2] CJK UNIFIED IDEOGRAPH-20D7D..CJK UNIFIED IDEOGRAPH-20D7E +20D80..20E0D ; Uncommon_Use # 3.1 [142] CJK UNIFIED IDEOGRAPH-20D80..CJK UNIFIED IDEOGRAPH-20E0D +20E10..20E76 ; Uncommon_Use # 3.1 [103] CJK UNIFIED IDEOGRAPH-20E10..CJK UNIFIED IDEOGRAPH-20E76 +20E78..20E9C ; Uncommon_Use # 3.1 [37] CJK UNIFIED IDEOGRAPH-20E78..CJK UNIFIED IDEOGRAPH-20E9C +20E9E..20EA1 ; Uncommon_Use # 3.1 [4] CJK UNIFIED IDEOGRAPH-20E9E..CJK UNIFIED IDEOGRAPH-20EA1 +20EA3..20ED6 ; Uncommon_Use # 3.1 [52] CJK UNIFIED IDEOGRAPH-20EA3..CJK UNIFIED IDEOGRAPH-20ED6 +20ED8..20EF8 ; Uncommon_Use # 3.1 [33] CJK UNIFIED IDEOGRAPH-20ED8..CJK UNIFIED IDEOGRAPH-20EF8 +20EFB..20F2C ; Uncommon_Use # 3.1 [50] CJK UNIFIED IDEOGRAPH-20EFB..CJK UNIFIED IDEOGRAPH-20F2C +20F2F..20F4B ; Uncommon_Use # 3.1 [29] CJK UNIFIED IDEOGRAPH-20F2F..CJK UNIFIED IDEOGRAPH-20F4B +20F4D..20FB3 ; Uncommon_Use # 3.1 [103] CJK UNIFIED IDEOGRAPH-20F4D..CJK UNIFIED IDEOGRAPH-20FB3 +20FB5..20FBB ; Uncommon_Use # 3.1 [7] CJK UNIFIED IDEOGRAPH-20FB5..CJK UNIFIED IDEOGRAPH-20FBB +20FBD..20FE9 ; Uncommon_Use # 3.1 [45] CJK UNIFIED IDEOGRAPH-20FBD..CJK UNIFIED IDEOGRAPH-20FE9 +20FEB..2105B ; Uncommon_Use # 3.1 [113] CJK UNIFIED IDEOGRAPH-20FEB..CJK UNIFIED IDEOGRAPH-2105B +2105D..2106E ; Uncommon_Use # 3.1 [18] CJK UNIFIED IDEOGRAPH-2105D..CJK UNIFIED IDEOGRAPH-2106E +21070..21074 ; Uncommon_Use # 3.1 [5] CJK UNIFIED IDEOGRAPH-21070..CJK UNIFIED IDEOGRAPH-21074 +21077..2107A ; Uncommon_Use # 3.1 [4] CJK UNIFIED IDEOGRAPH-21077..CJK UNIFIED IDEOGRAPH-2107A +2107C..210C0 ; Uncommon_Use # 3.1 [69] CJK UNIFIED IDEOGRAPH-2107C..CJK UNIFIED IDEOGRAPH-210C0 +210C2..210C8 ; Uncommon_Use # 3.1 [7] CJK UNIFIED IDEOGRAPH-210C2..CJK UNIFIED IDEOGRAPH-210C8 +210CA..211D8 ; Uncommon_Use # 3.1 [271] CJK UNIFIED IDEOGRAPH-210CA..CJK UNIFIED IDEOGRAPH-211D8 +211DA..220C6 ; Uncommon_Use # 3.1 [3821] CJK UNIFIED IDEOGRAPH-211DA..CJK UNIFIED IDEOGRAPH-220C6 +220C8..227B4 ; Uncommon_Use # 3.1 [1773] CJK UNIFIED IDEOGRAPH-220C8..CJK UNIFIED IDEOGRAPH-227B4 +227B6..22AD4 ; Uncommon_Use # 3.1 [799] CJK UNIFIED IDEOGRAPH-227B6..CJK UNIFIED IDEOGRAPH-22AD4 +22AD6..22B42 ; Uncommon_Use # 3.1 [109] CJK UNIFIED IDEOGRAPH-22AD6..CJK UNIFIED IDEOGRAPH-22B42 +22B44..22BC9 ; Uncommon_Use # 3.1 [134] CJK UNIFIED IDEOGRAPH-22B44..CJK UNIFIED IDEOGRAPH-22BC9 +22BCB..22C50 ; Uncommon_Use # 3.1 [134] CJK UNIFIED IDEOGRAPH-22BCB..CJK UNIFIED IDEOGRAPH-22C50 +22C52..22C54 ; Uncommon_Use # 3.1 [3] CJK UNIFIED IDEOGRAPH-22C52..CJK UNIFIED IDEOGRAPH-22C54 +22C56..22CC1 ; Uncommon_Use # 3.1 [108] CJK UNIFIED IDEOGRAPH-22C56..CJK UNIFIED IDEOGRAPH-22CC1 +22CC3..22D07 ; Uncommon_Use # 3.1 [69] CJK UNIFIED IDEOGRAPH-22CC3..CJK UNIFIED IDEOGRAPH-22D07 +22D09..22D4B ; Uncommon_Use # 3.1 [67] CJK UNIFIED IDEOGRAPH-22D09..CJK UNIFIED IDEOGRAPH-22D4B +22D4D..22D66 ; Uncommon_Use # 3.1 [26] CJK UNIFIED IDEOGRAPH-22D4D..CJK UNIFIED IDEOGRAPH-22D66 +22D68..22EB2 ; Uncommon_Use # 3.1 [331] CJK UNIFIED IDEOGRAPH-22D68..CJK UNIFIED IDEOGRAPH-22EB2 +22EB4..23CB6 ; Uncommon_Use # 3.1 [3587] CJK UNIFIED IDEOGRAPH-22EB4..CJK UNIFIED IDEOGRAPH-23CB6 +23CB8..244D2 ; Uncommon_Use # 3.1 [2075] CJK UNIFIED IDEOGRAPH-23CB8..CJK UNIFIED IDEOGRAPH-244D2 +244D4..24DB7 ; Uncommon_Use # 3.1 [2276] CJK UNIFIED IDEOGRAPH-244D4..CJK UNIFIED IDEOGRAPH-24DB7 +24DB9..24DE9 ; Uncommon_Use # 3.1 [49] CJK UNIFIED IDEOGRAPH-24DB9..CJK UNIFIED IDEOGRAPH-24DE9 +24DEB..2512A ; Uncommon_Use # 3.1 [832] CJK UNIFIED IDEOGRAPH-24DEB..CJK UNIFIED IDEOGRAPH-2512A +2512C..26257 ; Uncommon_Use # 3.1 [4396] CJK UNIFIED IDEOGRAPH-2512C..CJK UNIFIED IDEOGRAPH-26257 +26259..267CB ; Uncommon_Use # 3.1 [1395] CJK UNIFIED IDEOGRAPH-26259..CJK UNIFIED IDEOGRAPH-267CB +267CD..269F1 ; Uncommon_Use # 3.1 [549] CJK UNIFIED IDEOGRAPH-267CD..CJK UNIFIED IDEOGRAPH-269F1 +269F3..269F9 ; Uncommon_Use # 3.1 [7] CJK UNIFIED IDEOGRAPH-269F3..CJK UNIFIED IDEOGRAPH-269F9 +269FB..27A3D ; Uncommon_Use # 3.1 [4163] CJK UNIFIED IDEOGRAPH-269FB..CJK UNIFIED IDEOGRAPH-27A3D +27A3F..2815C ; Uncommon_Use # 3.1 [1822] CJK UNIFIED IDEOGRAPH-27A3F..CJK UNIFIED IDEOGRAPH-2815C +2815E..28206 ; Uncommon_Use # 3.1 [169] CJK UNIFIED IDEOGRAPH-2815E..CJK UNIFIED IDEOGRAPH-28206 +28208..282E1 ; Uncommon_Use # 3.1 [218] CJK UNIFIED IDEOGRAPH-28208..CJK UNIFIED IDEOGRAPH-282E1 +282E3..28CC9 ; Uncommon_Use # 3.1 [2535] CJK UNIFIED IDEOGRAPH-282E3..CJK UNIFIED IDEOGRAPH-28CC9 +28CCB..28CCC ; Uncommon_Use # 3.1 [2] CJK UNIFIED IDEOGRAPH-28CCB..CJK UNIFIED IDEOGRAPH-28CCC +28CCE..28CD1 ; Uncommon_Use # 3.1 [4] CJK UNIFIED IDEOGRAPH-28CCE..CJK UNIFIED IDEOGRAPH-28CD1 +28CD3..29D97 ; Uncommon_Use # 3.1 [4293] CJK UNIFIED IDEOGRAPH-28CD3..CJK UNIFIED IDEOGRAPH-29D97 +29D99..2A6D6 ; Uncommon_Use # 3.1 [2366] CJK UNIFIED IDEOGRAPH-29D99..CJK UNIFIED IDEOGRAPH-2A6D6 +2A6D7..2A6DD ; Uncommon_Use # 13.0 [7] CJK UNIFIED IDEOGRAPH-2A6D7..CJK UNIFIED IDEOGRAPH-2A6DD +2A6DE..2A6DF ; Uncommon_Use # 14.0 [2] CJK UNIFIED IDEOGRAPH-2A6DE..CJK UNIFIED IDEOGRAPH-2A6DF +2A700..2B734 ; Uncommon_Use # 5.2 [4149] CJK UNIFIED IDEOGRAPH-2A700..CJK UNIFIED IDEOGRAPH-2B734 +2B735..2B738 ; Uncommon_Use # 14.0 [4] CJK UNIFIED IDEOGRAPH-2B735..CJK UNIFIED IDEOGRAPH-2B738 +2B739 ; Uncommon_Use # 15.0 CJK UNIFIED IDEOGRAPH-2B739 +2B73A..2B73F ; Uncommon_Use # 17.0 [6] CJK UNIFIED IDEOGRAPH-2B73A..CJK UNIFIED IDEOGRAPH-2B73F +2B740..2B81D ; Uncommon_Use # 6.0 [222] CJK UNIFIED IDEOGRAPH-2B740..CJK UNIFIED IDEOGRAPH-2B81D +2B820..2CEA1 ; Uncommon_Use # 8.0 [5762] CJK UNIFIED IDEOGRAPH-2B820..CJK UNIFIED IDEOGRAPH-2CEA1 +2CEA2..2CEAD ; Uncommon_Use # 17.0 [12] CJK UNIFIED IDEOGRAPH-2CEA2..CJK UNIFIED IDEOGRAPH-2CEAD +2CEB0..2EBE0 ; Uncommon_Use # 10.0 [7473] CJK UNIFIED IDEOGRAPH-2CEB0..CJK UNIFIED IDEOGRAPH-2EBE0 +2EBF0..2EE5D ; Uncommon_Use # 15.1 [622] CJK UNIFIED IDEOGRAPH-2EBF0..CJK UNIFIED IDEOGRAPH-2EE5D +30000..3134A ; Uncommon_Use # 13.0 [4939] CJK UNIFIED IDEOGRAPH-30000..CJK UNIFIED IDEOGRAPH-3134A +31350..323AF ; Uncommon_Use # 15.0 [4192] CJK UNIFIED IDEOGRAPH-31350..CJK UNIFIED IDEOGRAPH-323AF +323B0..33479 ; Uncommon_Use # 17.0 [4298] CJK UNIFIED IDEOGRAPH-323B0..CJK UNIFIED IDEOGRAPH-33479 + +# Total code points: 83130 + +# Identifier_Type: Uncommon_Use Technical + +05C7 ; Uncommon_Use Technical # 4.1 HEBREW POINT QAMATS QATAN +0653 ; Uncommon_Use Technical # 3.0 ARABIC MADDAH ABOVE +0D8F..0D90 ; Uncommon_Use Technical # 3.0 [2] SINHALA LETTER ILUYANNA..SINHALA LETTER ILUUYANNA +0DDF ; Uncommon_Use Technical # 3.0 SINHALA VOWEL SIGN GAYANUKITTA +0DF3 ; Uncommon_Use Technical # 3.0 SINHALA VOWEL SIGN DIGA GAYANUKITTA +0FC6 ; Uncommon_Use Technical # 3.0 TIBETAN SYMBOL PADMA GDAN +10F9..10FA ; Uncommon_Use Technical # 4.1 [2] GEORGIAN LETTER TURNED GAN..GEORGIAN LETTER AIN +FB1E ; Uncommon_Use Technical # 1.1 HEBREW POINT JUDEO-SPANISH VARIKA +FE2E..FE2F ; Uncommon_Use Technical # 8.0 [2] COMBINING CYRILLIC TITLO LEFT HALF..COMBINING CYRILLIC TITLO RIGHT HALF + +# Total code points: 12 + +# Identifier_Type: Uncommon_Use Technical Not_XID + +1D1DE..1D1E8 ; Uncommon_Use Technical Not_XID # 8.0 [11] MUSICAL SYMBOL KIEVAN C CLEF..MUSICAL SYMBOL KIEVAN FLAT SIGN + +# Total code points: 11 + +# Identifier_Type: Uncommon_Use Exclusion + +18A9 ; Uncommon_Use Exclusion # 3.0 MONGOLIAN LETTER ALI GALI DAGALGA +16A40..16A5E ; Uncommon_Use Exclusion # 7.0 [31] MRO LETTER TA..MRO LETTER TEK +16A60..16A69 ; Uncommon_Use Exclusion # 7.0 [10] MRO DIGIT ZERO..MRO DIGIT NINE + +# Total code points: 42 + +# Identifier_Type: Uncommon_Use Obsolete + +05A2 ; Uncommon_Use Obsolete # 4.1 HEBREW ACCENT ATNAH HAFUKH +05C5 ; Uncommon_Use Obsolete # 4.1 HEBREW MARK LOWER DOT +0F6A ; Uncommon_Use Obsolete # 3.0 TIBETAN LETTER FIXED-FORM RA +0F82..0F83 ; Uncommon_Use Obsolete # 2.0 [2] TIBETAN SIGN NYI ZLA NAA DA..TIBETAN SIGN SNA LDAN +1050..1059 ; Uncommon_Use Obsolete # 3.0 [10] MYANMAR LETTER SHA..MYANMAR VOWEL SIGN VOCALIC LL +A69E ; Uncommon_Use Obsolete # 8.0 COMBINING CYRILLIC LETTER EF +A8FD ; Uncommon_Use Obsolete # 8.0 DEVANAGARI JAIN OM + +# Total code points: 17 + +# Identifier_Type: Uncommon_Use Obsolete Not_XID + +A8FC ; Uncommon_Use Obsolete Not_XID # 8.0 DEVANAGARI SIGN SIDDHAM + +# Total code points: 1 + +# Identifier_Type: Uncommon_Use Not_XID + +218A..218B ; Uncommon_Use Not_XID # 8.0 [2] TURNED DIGIT TWO..TURNED DIGIT THREE +2BEC..2BEF ; Uncommon_Use Not_XID # 8.0 [4] LEFTWARDS TWO-HEADED ARROW WITH TRIANGLE ARROWHEADS..DOWNWARDS TWO-HEADED ARROW WITH TRIANGLE ARROWHEADS +1F54F ; Uncommon_Use Not_XID # 8.0 BOWL OF HYGIEIA + +# Total code points: 7 + +# Identifier_Type: Technical + +0180 ; Technical # 1.1 LATIN SMALL LETTER B WITH STROKE +01C0..01C3 ; Technical # 1.1 [4] LATIN LETTER DENTAL CLICK..LATIN LETTER RETROFLEX CLICK +0200..0217 ; Technical # 1.1 [24] LATIN CAPITAL LETTER A WITH DOUBLE GRAVE..LATIN SMALL LETTER U WITH INVERTED BREVE +0234..0236 ; Technical # 4.0 [3] LATIN SMALL LETTER L WITH CURL..LATIN SMALL LETTER T WITH CURL +0250..0252 ; Technical # 1.1 [3] LATIN SMALL LETTER TURNED A..LATIN SMALL LETTER TURNED ALPHA +0255 ; Technical # 1.1 LATIN SMALL LETTER C WITH CURL +0258 ; Technical # 1.1 LATIN SMALL LETTER REVERSED E +025A ; Technical # 1.1 LATIN SMALL LETTER SCHWA WITH HOOK +025C..0262 ; Technical # 1.1 [7] LATIN SMALL LETTER REVERSED OPEN E..LATIN LETTER SMALL CAPITAL G +0264..0267 ; Technical # 1.1 [4] LATIN SMALL LETTER RAMS HORN..LATIN SMALL LETTER HENG WITH HOOK +026A..0271 ; Technical # 1.1 [8] LATIN LETTER SMALL CAPITAL I..LATIN SMALL LETTER M WITH HOOK +0273..0276 ; Technical # 1.1 [4] LATIN SMALL LETTER N WITH RETROFLEX HOOK..LATIN LETTER SMALL CAPITAL OE +0278..027B ; Technical # 1.1 [4] LATIN SMALL LETTER PHI..LATIN SMALL LETTER TURNED R WITH HOOK +027D..0288 ; Technical # 1.1 [12] LATIN SMALL LETTER R WITH TAIL..LATIN SMALL LETTER T WITH RETROFLEX HOOK +028A ; Technical # 1.1 LATIN SMALL LETTER UPSILON +028C..0291 ; Technical # 1.1 [6] LATIN SMALL LETTER TURNED V..LATIN SMALL LETTER Z WITH CURL +0293..029D ; Technical # 1.1 [11] LATIN SMALL LETTER EZH WITH CURL..LATIN SMALL LETTER J WITH CROSSED-TAIL +029F..02A8 ; Technical # 1.1 [10] LATIN LETTER SMALL CAPITAL L..LATIN SMALL LETTER TC DIGRAPH WITH CURL +02A9..02AD ; Technical # 3.0 [5] LATIN SMALL LETTER FENG DIGRAPH..LATIN LETTER BIDENTAL PERCUSSIVE +02AE..02AF ; Technical # 4.0 [2] LATIN SMALL LETTER TURNED H WITH FISHHOOK..LATIN SMALL LETTER TURNED H WITH FISHHOOK AND TAIL +02B9..02BA ; Technical # 1.1 [2] MODIFIER LETTER PRIME..MODIFIER LETTER DOUBLE PRIME +02BD..02C1 ; Technical # 1.1 [5] MODIFIER LETTER REVERSED COMMA..MODIFIER LETTER REVERSED GLOTTAL STOP +02C6..02D1 ; Technical # 1.1 [12] MODIFIER LETTER CIRCUMFLEX ACCENT..MODIFIER LETTER HALF TRIANGULAR COLON +02EC ; Technical # 3.0 MODIFIER LETTER VOICING +02EE ; Technical # 3.0 MODIFIER LETTER DOUBLE APOSTROPHE +030E..0315 ; Technical # 1.1 [8] COMBINING DOUBLE VERTICAL LINE ABOVE..COMBINING COMMA ABOVE RIGHT +0317..031A ; Technical # 1.1 [4] COMBINING ACUTE ACCENT BELOW..COMBINING LEFT ANGLE ABOVE +031C..0320 ; Technical # 1.1 [5] COMBINING LEFT HALF RING BELOW..COMBINING MINUS SIGN BELOW +0324..0325 ; Technical # 1.1 [2] COMBINING DIAERESIS BELOW..COMBINING RING BELOW +0329..0330 ; Technical # 1.1 [8] COMBINING VERTICAL LINE BELOW..COMBINING TILDE BELOW +0333 ; Technical # 1.1 COMBINING DOUBLE LOW LINE +0335 ; Technical # 1.1 COMBINING SHORT STROKE OVERLAY +0337..033F ; Technical # 1.1 [9] COMBINING SHORT SOLIDUS OVERLAY..COMBINING DOUBLE OVERLINE +0342 ; Technical # 1.1 COMBINING GREEK PERISPOMENI +0346..034E ; Technical # 3.0 [9] COMBINING BRIDGE ABOVE..COMBINING UPWARDS ARROW BELOW +0350..0357 ; Technical # 4.0 [8] COMBINING RIGHT ARROWHEAD ABOVE..COMBINING RIGHT HALF RING ABOVE +0359..035C ; Technical # 4.1 [4] COMBINING ASTERISK BELOW..COMBINING DOUBLE BREVE BELOW +035D..035F ; Technical # 4.0 [3] COMBINING DOUBLE BREVE..COMBINING DOUBLE MACRON BELOW +0360..0361 ; Technical # 1.1 [2] COMBINING DOUBLE TILDE..COMBINING DOUBLE INVERTED BREVE +0362 ; Technical # 3.0 COMBINING DOUBLE RIGHTWARDS ARROW BELOW +03CF ; Technical # 5.1 GREEK CAPITAL KAI SYMBOL +03D7 ; Technical # 3.0 GREEK KAI SYMBOL +0559 ; Technical # 1.1 ARMENIAN MODIFIER LETTER LEFT HALF RING +0560 ; Technical # 11.0 ARMENIAN SMALL LETTER TURNED AYB +0588 ; Technical # 11.0 ARMENIAN SMALL LETTER YI WITH STROKE +0671 ; Technical # 1.1 ARABIC LETTER ALEF WASLA +06E5..06E6 ; Technical # 1.1 [2] ARABIC SMALL WAW..ARABIC SMALL YEH +0870..0887 ; Technical # 14.0 [24] ARABIC LETTER ALEF WITH ATTACHED FATHA..ARABIC BASELINE ROUND DOT +08C9 ; Technical # 14.0 ARABIC SMALL FARSI YEH +0950 ; Technical # 1.1 DEVANAGARI OM +0953..0954 ; Technical # 1.1 [2] DEVANAGARI GRAVE ACCENT..DEVANAGARI ACUTE ACCENT +097D ; Technical # 4.1 DEVANAGARI LETTER GLOTTAL STOP +0A74 ; Technical # 1.1 GURMUKHI EK ONKAR +0AD0 ; Technical # 1.1 GUJARATI OM +0B82 ; Technical # 1.1 TAMIL SIGN ANUSVARA +0BD0 ; Technical # 5.1 TAMIL OM +0D81 ; Technical # 13.0 SINHALA SIGN CANDRABINDU +0EAF ; Technical # 1.1 LAO ELLIPSIS +0F00 ; Technical # 2.0 TIBETAN SYLLABLE OM +0F18..0F19 ; Technical # 2.0 [2] TIBETAN ASTROLOGICAL SIGN -KHYUD PA..TIBETAN ASTROLOGICAL SIGN SDONG TSHUGS +0F35 ; Technical # 2.0 TIBETAN MARK NGAS BZUNG NYI ZLA +0F37 ; Technical # 2.0 TIBETAN MARK NGAS BZUNG SGOR RTAGS +0F3E..0F3F ; Technical # 2.0 [2] TIBETAN SIGN YAR TSHES..TIBETAN SIGN MAR TSHES +17CE..17CF ; Technical # 3.0 [2] KHMER SIGN KAKABAT..KHMER SIGN AHSDA +1ABF..1AC0 ; Technical # 13.0 [2] COMBINING LATIN SMALL LETTER W BELOW..COMBINING LATIN SMALL LETTER TURNED W BELOW +1ACF..1ADD ; Technical # 17.0 [15] COMBINING DOUBLE CARON..COMBINING DOT-AND-RING BELOW +1AE0..1AEB ; Technical # 17.0 [12] COMBINING LEFT TACK ABOVE..COMBINING DOUBLE RIGHTWARDS ARROW ABOVE +1D00..1D2B ; Technical # 4.0 [44] LATIN LETTER SMALL CAPITAL A..CYRILLIC LETTER SMALL CAPITAL EL +1D2F ; Technical # 4.0 MODIFIER LETTER CAPITAL BARRED B +1D3B ; Technical # 4.0 MODIFIER LETTER CAPITAL REVERSED N +1D4E ; Technical # 4.0 MODIFIER LETTER SMALL TURNED I +1D6B ; Technical # 4.0 LATIN SMALL LETTER UE +1D6C..1D77 ; Technical # 4.1 [12] LATIN SMALL LETTER B WITH MIDDLE TILDE..LATIN SMALL LETTER TURNED G +1D79..1D9A ; Technical # 4.1 [34] LATIN SMALL LETTER INSULAR G..LATIN SMALL LETTER EZH WITH RETROFLEX HOOK +1DC4..1DCA ; Technical # 5.0 [7] COMBINING MACRON-ACUTE..COMBINING LATIN SMALL LETTER R BELOW +1DCB..1DCD ; Technical # 5.1 [3] COMBINING BREVE-MACRON..COMBINING DOUBLE CIRCUMFLEX ABOVE +1DCF..1DD0 ; Technical # 5.1 [2] COMBINING ZIGZAG BELOW..COMBINING IS BELOW +1DE7..1DF5 ; Technical # 7.0 [15] COMBINING LATIN SMALL LETTER ALPHA..COMBINING UP TACK ABOVE +1DF6..1DF9 ; Technical # 10.0 [4] COMBINING KAVYKA ABOVE RIGHT..COMBINING WIDE INVERTED BRIDGE BELOW +1DFB ; Technical # 9.0 COMBINING DELETION MARK +1DFC ; Technical # 6.0 COMBINING DOUBLE INVERTED BREVE BELOW +1DFD ; Technical # 5.2 COMBINING ALMOST EQUAL TO BELOW +1DFE..1DFF ; Technical # 5.0 [2] COMBINING LEFT ARROWHEAD ABOVE..COMBINING RIGHT ARROWHEAD AND DOWN ARROWHEAD BELOW +1E00..1E01 ; Technical # 1.1 [2] LATIN CAPITAL LETTER A WITH RING BELOW..LATIN SMALL LETTER A WITH RING BELOW +1E18..1E1B ; Technical # 1.1 [4] LATIN CAPITAL LETTER E WITH CIRCUMFLEX BELOW..LATIN SMALL LETTER E WITH TILDE BELOW +1E2A..1E2D ; Technical # 1.1 [4] LATIN CAPITAL LETTER H WITH BREVE BELOW..LATIN SMALL LETTER I WITH TILDE BELOW +1E72..1E77 ; Technical # 1.1 [6] LATIN CAPITAL LETTER U WITH DIAERESIS BELOW..LATIN SMALL LETTER U WITH CIRCUMFLEX BELOW +1E9C..1E9D ; Technical # 5.1 [2] LATIN SMALL LETTER LONG S WITH DIAGONAL STROKE..LATIN SMALL LETTER LONG S WITH HIGH STROKE +1E9F ; Technical # 5.1 LATIN SMALL LETTER DELTA +1EFA..1EFF ; Technical # 5.1 [6] LATIN CAPITAL LETTER MIDDLE-WELSH LL..LATIN SMALL LETTER Y WITH LOOP +203F..2040 ; Technical # 1.1 [2] UNDERTIE..CHARACTER TIE +20D0..20DC ; Technical # 1.1 [13] COMBINING LEFT HARPOON ABOVE..COMBINING FOUR DOTS ABOVE +20E1 ; Technical # 1.1 COMBINING LEFT RIGHT ARROW ABOVE +20E5..20EA ; Technical # 3.2 [6] COMBINING REVERSE SOLIDUS OVERLAY..COMBINING LEFTWARDS ARROW OVERLAY +20EB ; Technical # 4.1 COMBINING LONG DOUBLE SOLIDUS OVERLAY +20EC..20EF ; Technical # 5.0 [4] COMBINING RIGHTWARDS HARPOON WITH BARB DOWNWARDS..COMBINING RIGHT ARROW BELOW +20F0 ; Technical # 5.1 COMBINING ASTERISK ABOVE +2118 ; Technical # 1.1 SCRIPT CAPITAL P +212E ; Technical # 1.1 ESTIMATED SYMBOL +2C60..2C67 ; Technical # 5.0 [8] LATIN CAPITAL LETTER L WITH DOUBLE BAR..LATIN CAPITAL LETTER H WITH DESCENDER +2C77 ; Technical # 5.0 LATIN SMALL LETTER TAILLESS PHI +2C78..2C7B ; Technical # 5.1 [4] LATIN SMALL LETTER E WITH NOTCH..LATIN LETTER SMALL CAPITAL TURNED E +2D27 ; Technical # 6.1 GEORGIAN SMALL LETTER YN +2D2D ; Technical # 6.1 GEORGIAN SMALL LETTER AEN +3021..302D ; Technical # 1.1 [13] HANGZHOU NUMERAL ONE..IDEOGRAPHIC ENTERING TONE MARK +3031..3035 ; Technical # 1.1 [5] VERTICAL KANA REPEAT MARK..VERTICAL KANA REPEAT MARK LOWER HALF +303B..303C ; Technical # 3.2 [2] VERTICAL IDEOGRAPHIC ITERATION MARK..MASU MARK +A717..A71A ; Technical # 5.0 [4] MODIFIER LETTER DOT VERTICAL BAR..MODIFIER LETTER LOWER RIGHT CORNER ANGLE +A71B..A71F ; Technical # 5.1 [5] MODIFIER LETTER RAISED UP ARROW..MODIFIER LETTER LOW INVERTED EXCLAMATION MARK +A788 ; Technical # 5.1 MODIFIER LETTER LOW CIRCUMFLEX ACCENT +A78E ; Technical # 6.0 LATIN SMALL LETTER L WITH RETROFLEX HOOK AND BELT +A7AE ; Technical # 9.0 LATIN CAPITAL LETTER SMALL CAPITAL I +A7AF ; Technical # 11.0 LATIN LETTER SMALL CAPITAL Q +A7BA..A7BF ; Technical # 12.0 [6] LATIN CAPITAL LETTER GLOTTAL A..LATIN SMALL LETTER GLOTTAL U +A7C5..A7C6 ; Technical # 12.0 [2] LATIN CAPITAL LETTER S WITH HOOK..LATIN CAPITAL LETTER Z WITH PALATAL HOOK +A7FA ; Technical # 6.0 LATIN LETTER SMALL CAPITAL TURNED M +AB68 ; Technical # 13.0 LATIN SMALL LETTER TURNED R WITH MIDDLE TILDE +FE20..FE23 ; Technical # 1.1 [4] COMBINING LIGATURE LEFT HALF..COMBINING DOUBLE TILDE RIGHT HALF +FE24..FE26 ; Technical # 5.1 [3] COMBINING MACRON LEFT HALF..COMBINING CONJOINING MACRON +FE27..FE2D ; Technical # 7.0 [7] COMBINING LIGATURE LEFT HALF BELOW..COMBINING CONJOINING MACRON BELOW +FE73 ; Technical # 3.2 ARABIC TAIL FRAGMENT +10EC5..10EC6 ; Technical # 17.0 [2] ARABIC SMALL YEH BARREE WITH TWO DOTS BELOW..ARABIC LETTER THIN NOON +10EFB ; Technical # 17.0 ARABIC SMALL LOW NOON +16FF2..16FF6 ; Technical # 17.0 [5] CHINESE SMALL SIMPLIFIED ER..YANGQIN SIGN SLOW TWO BEATS +1CF00..1CF2D ; Technical # 14.0 [46] ZNAMENNY COMBINING MARK GORAZDO NIZKO S KRYZHEM ON LEFT..ZNAMENNY COMBINING MARK KRYZH ON LEFT +1CF30..1CF46 ; Technical # 14.0 [23] ZNAMENNY COMBINING TONAL RANGE MARK MRACHNO..ZNAMENNY PRIZNAK MODIFIER ROG +1D165..1D169 ; Technical # 3.1 [5] MUSICAL SYMBOL COMBINING STEM..MUSICAL SYMBOL COMBINING TREMOLO-3 +1D16D..1D172 ; Technical # 3.1 [6] MUSICAL SYMBOL COMBINING AUGMENTATION DOT..MUSICAL SYMBOL COMBINING FLAG-5 +1D17B..1D182 ; Technical # 3.1 [8] MUSICAL SYMBOL COMBINING ACCENT..MUSICAL SYMBOL COMBINING LOURE +1D185..1D18B ; Technical # 3.1 [7] MUSICAL SYMBOL COMBINING DOIT..MUSICAL SYMBOL COMBINING TRIPLE TONGUE +1D1AA..1D1AD ; Technical # 3.1 [4] MUSICAL SYMBOL COMBINING DOWN BOW..MUSICAL SYMBOL COMBINING SNAP PIZZICATO +1DF00..1DF1E ; Technical # 14.0 [31] LATIN SMALL LETTER FENG DIGRAPH WITH TRILL..LATIN SMALL LETTER S WITH CURL +1DF25..1DF2A ; Technical # 15.0 [6] LATIN SMALL LETTER D WITH MID-HEIGHT LEFT HOOK..LATIN SMALL LETTER T WITH MID-HEIGHT LEFT HOOK + +# Total code points: 682 + +# Identifier_Type: Technical Exclusion + +2CF0..2CF1 ; Technical Exclusion # 5.2 [2] COPTIC COMBINING SPIRITUS ASPER..COPTIC COMBINING SPIRITUS LENIS + +# Total code points: 2 + +# Identifier_Type: Technical Obsolete + +018D ; Technical Obsolete # 1.1 LATIN SMALL LETTER TURNED DELTA +01AA..01AB ; Technical Obsolete # 1.1 [2] LATIN LETTER REVERSED ESH LOOP..LATIN SMALL LETTER T WITH PALATAL HOOK +01BA..01BB ; Technical Obsolete # 1.1 [2] LATIN SMALL LETTER EZH WITH TAIL..LATIN LETTER TWO WITH STROKE +01BE ; Technical Obsolete # 1.1 LATIN LETTER INVERTED GLOTTAL STOP WITH STROKE +0277 ; Technical Obsolete # 1.1 LATIN SMALL LETTER CLOSED OMEGA +027C ; Technical Obsolete # 1.1 LATIN SMALL LETTER R WITH LONG LEG +029E ; Technical Obsolete # 1.1 LATIN SMALL LETTER TURNED K +03F3 ; Technical Obsolete # 1.1 GREEK LETTER YOT +03FC ; Technical Obsolete # 4.1 GREEK RHO WITH STROKE SYMBOL +0484..0486 ; Technical Obsolete # 1.1 [3] COMBINING CYRILLIC PALATALIZATION..COMBINING CYRILLIC PSILI PNEUMATA +0487 ; Technical Obsolete # 5.1 COMBINING CYRILLIC POKRYTIE +0D04 ; Technical Obsolete # 13.0 MALAYALAM LETTER VEDIC ANUSVARA +17D1 ; Technical Obsolete # 3.0 KHMER SIGN VIRIAM +17DD ; Technical Obsolete # 4.0 KHMER SIGN ATTHACAN +1DC0..1DC3 ; Technical Obsolete # 4.1 [4] COMBINING DOTTED GRAVE ACCENT..COMBINING SUSPENSION MARK +1DCE ; Technical Obsolete # 5.1 COMBINING OGONEK ABOVE +1DD1..1DE6 ; Technical Obsolete # 5.1 [22] COMBINING UR ABOVE..COMBINING LATIN SMALL LETTER Z +1FB0..1FB1 ; Technical Obsolete # 1.1 [2] GREEK SMALL LETTER ALPHA WITH VRACHY..GREEK SMALL LETTER ALPHA WITH MACRON +2180..2182 ; Technical Obsolete # 1.1 [3] ROMAN NUMERAL ONE THOUSAND C D..ROMAN NUMERAL TEN THOUSAND +2183 ; Technical Obsolete # 3.0 ROMAN NUMERAL REVERSED ONE HUNDRED +302E..302F ; Technical Obsolete # 1.1 [2] HANGUL SINGLE DOT TONE MARK..HANGUL DOUBLE DOT TONE MARK +A722..A72F ; Technical Obsolete # 5.1 [14] LATIN CAPITAL LETTER EGYPTOLOGICAL ALEF..LATIN SMALL LETTER CUATRILLO WITH COMMA +1D242..1D244 ; Technical Obsolete # 4.1 [3] COMBINING GREEK MUSICAL TRISEME..COMBINING GREEK MUSICAL PENTASEME + +# Total code points: 70 + +# Identifier_Type: Technical Obsolete Not_XID + +2E00..2E0D ; Technical Obsolete Not_XID # 4.1 [14] RIGHT ANGLE SUBSTITUTION MARKER..RIGHT RAISED OMISSION BRACKET + +# Total code points: 14 + +# Identifier_Type: Technical Not_XID + +0375 ; Technical Not_XID # 1.1 GREEK LOWER NUMERAL SIGN +0888 ; Technical Not_XID # 14.0 ARABIC RAISED ROUND DOT +20DD..20E0 ; Technical Not_XID # 1.1 [4] COMBINING ENCLOSING CIRCLE..COMBINING ENCLOSING CIRCLE BACKSLASH +20E2..20E3 ; Technical Not_XID # 3.0 [2] COMBINING ENCLOSING SCREEN..COMBINING ENCLOSING KEYCAP +20E4 ; Technical Not_XID # 3.2 COMBINING ENCLOSING UPWARD POINTING TRIANGLE +24EB..24FE ; Technical Not_XID # 3.2 [20] NEGATIVE CIRCLED NUMBER ELEVEN..DOUBLE CIRCLED NUMBER TEN +24FF ; Technical Not_XID # 4.0 NEGATIVE CIRCLED DIGIT ZERO +2800..28FF ; Technical Not_XID # 3.0 [256] BRAILLE PATTERN BLANK..BRAILLE PATTERN DOTS-12345678 +327F ; Technical Not_XID # 1.1 KOREAN STANDARD SYMBOL +4DC0..4DFF ; Technical Not_XID # 4.0 [64] HEXAGRAM FOR THE CREATIVE HEAVEN..HEXAGRAM FOR BEFORE COMPLETION +A708..A716 ; Technical Not_XID # 4.1 [15] MODIFIER LETTER EXTRA-HIGH DOTTED TONE BAR..MODIFIER LETTER EXTRA-LOW LEFT-STEM TONE BAR +FBB2..FBC1 ; Technical Not_XID # 6.0 [16] ARABIC SYMBOL DOT ABOVE..ARABIC SYMBOL SMALL TAH BELOW +FBC2 ; Technical Not_XID # 14.0 ARABIC SYMBOL WASLA ABOVE +FBC3..FBD2 ; Technical Not_XID # 17.0 [16] ARABIC LIGATURE JALLA WA-ALAA..ARABIC LIGATURE ALAYHI AR-RAHMAH +FD3E..FD3F ; Technical Not_XID # 1.1 [2] ORNATE LEFT PARENTHESIS..ORNATE RIGHT PARENTHESIS +FD40..FD4F ; Technical Not_XID # 14.0 [16] ARABIC LIGATURE RAHIMAHU ALLAAH..ARABIC LIGATURE RAHIMAHUM ALLAAH +FD90..FD91 ; Technical Not_XID # 17.0 [2] ARABIC LIGATURE RAHMATU ALLAAHI ALAYH..ARABIC LIGATURE RAHMATU ALLAAHI ALAYHAA +FDC8..FDCE ; Technical Not_XID # 17.0 [7] ARABIC LIGATURE RAHIMAHU ALLAAH TAAALAA..ARABIC LIGATURE KARRAMA ALLAAHU WAJHAH +FDCF ; Technical Not_XID # 14.0 ARABIC LIGATURE SALAAMUHU ALAYNAA +FDFD ; Technical Not_XID # 4.0 ARABIC LIGATURE BISMILLAH AR-RAHMAN AR-RAHEEM +FDFE..FDFF ; Technical Not_XID # 14.0 [2] ARABIC LIGATURE SUBHAANAHU WA TAAALAA..ARABIC LIGATURE AZZA WA JALL +FE45..FE46 ; Technical Not_XID # 3.2 [2] SESAME DOT..WHITE SESAME DOT +1CF50..1CFC3 ; Technical Not_XID # 14.0 [116] ZNAMENNY NEUME KRYUK..ZNAMENNY NEUME PAUK +1D000..1D0F5 ; Technical Not_XID # 3.1 [246] BYZANTINE MUSICAL SYMBOL PSILI..BYZANTINE MUSICAL SYMBOL GORGON NEO KATO +1D100..1D126 ; Technical Not_XID # 3.1 [39] MUSICAL SYMBOL SINGLE BARLINE..MUSICAL SYMBOL DRUM CLEF-2 +1D129 ; Technical Not_XID # 5.1 MUSICAL SYMBOL MULTIPLE MEASURE REST +1D12A..1D15D ; Technical Not_XID # 3.1 [52] MUSICAL SYMBOL DOUBLE SHARP..MUSICAL SYMBOL WHOLE NOTE +1D16A..1D16C ; Technical Not_XID # 3.1 [3] MUSICAL SYMBOL FINGERED TREMOLO-1..MUSICAL SYMBOL FINGERED TREMOLO-3 +1D183..1D184 ; Technical Not_XID # 3.1 [2] MUSICAL SYMBOL ARPEGGIATO UP..MUSICAL SYMBOL ARPEGGIATO DOWN +1D18C..1D1A9 ; Technical Not_XID # 3.1 [30] MUSICAL SYMBOL RINFORZANDO..MUSICAL SYMBOL DEGREE SLASH +1D1AE..1D1BA ; Technical Not_XID # 3.1 [13] MUSICAL SYMBOL PEDAL MARK..MUSICAL SYMBOL SEMIBREVIS BLACK +1D1C1..1D1DD ; Technical Not_XID # 3.1 [29] MUSICAL SYMBOL LONGA PERFECTA REST..MUSICAL SYMBOL PES SUBPUNCTIS +1D1E9..1D1EA ; Technical Not_XID # 14.0 [2] MUSICAL SYMBOL SORI..MUSICAL SYMBOL KORON +1D300..1D356 ; Technical Not_XID # 4.0 [87] MONOGRAM FOR EARTH..TETRAGRAM FOR FOSTERING + +# Total code points: 1052 + +# Identifier_Type: Exclusion + +03E2..03EF ; Exclusion # 1.1 [14] COPTIC CAPITAL LETTER SHEI..COPTIC SMALL LETTER DEI +0800..082D ; Exclusion # 5.2 [46] SAMARITAN LETTER ALAF..SAMARITAN MARK NEQUDAA +1681..169A ; Exclusion # 3.0 [26] OGHAM LETTER BEITH..OGHAM LETTER PEITH +16A0..16EA ; Exclusion # 3.0 [75] RUNIC LETTER FEHU FEOH FE F..RUNIC LETTER X +16EE..16F0 ; Exclusion # 3.0 [3] RUNIC ARLAUG SYMBOL..RUNIC BELGTHOR SYMBOL +16F1..16F8 ; Exclusion # 7.0 [8] RUNIC LETTER K..RUNIC LETTER FRANKS CASKET AESC +1700..170C ; Exclusion # 3.2 [13] TAGALOG LETTER A..TAGALOG LETTER YA +170D ; Exclusion # 14.0 TAGALOG LETTER RA +170E..1714 ; Exclusion # 3.2 [7] TAGALOG LETTER LA..TAGALOG SIGN VIRAMA +1715 ; Exclusion # 14.0 TAGALOG SIGN PAMUDPOD +171F ; Exclusion # 14.0 TAGALOG LETTER ARCHAIC RA +1720..1734 ; Exclusion # 3.2 [21] HANUNOO LETTER A..HANUNOO SIGN PAMUDPOD +1740..1753 ; Exclusion # 3.2 [20] BUHID LETTER A..BUHID VOWEL SIGN U +1760..176C ; Exclusion # 3.2 [13] TAGBANWA LETTER A..TAGBANWA LETTER YA +176E..1770 ; Exclusion # 3.2 [3] TAGBANWA LETTER LA..TAGBANWA LETTER SA +1772..1773 ; Exclusion # 3.2 [2] TAGBANWA VOWEL SIGN I..TAGBANWA VOWEL SIGN U +1810..1819 ; Exclusion # 3.0 [10] MONGOLIAN DIGIT ZERO..MONGOLIAN DIGIT NINE +1820..1877 ; Exclusion # 3.0 [88] MONGOLIAN LETTER A..MONGOLIAN LETTER MANCHU ZHA +1878 ; Exclusion # 11.0 MONGOLIAN LETTER CHA WITH TWO DOTS +1880..18A8 ; Exclusion # 3.0 [41] MONGOLIAN LETTER ALI GALI ANUSVARA ONE..MONGOLIAN LETTER MANCHU ALI GALI BHA +18AA ; Exclusion # 5.1 MONGOLIAN LETTER MANCHU ALI GALI LHA +1A00..1A1B ; Exclusion # 4.1 [28] BUGINESE LETTER KA..BUGINESE VOWEL SIGN AE +1CFA ; Exclusion # 12.0 VEDIC SIGN DOUBLE ANUSVARA ANTARGOMUKHA +2C00..2C2E ; Exclusion # 4.1 [47] GLAGOLITIC CAPITAL LETTER AZU..GLAGOLITIC CAPITAL LETTER LATINATE MYSLITE +2C2F ; Exclusion # 14.0 GLAGOLITIC CAPITAL LETTER CAUDATE CHRIVI +2C30..2C5E ; Exclusion # 4.1 [47] GLAGOLITIC SMALL LETTER AZU..GLAGOLITIC SMALL LETTER LATINATE MYSLITE +2C5F ; Exclusion # 14.0 GLAGOLITIC SMALL LETTER CAUDATE CHRIVI +2C80..2CE4 ; Exclusion # 4.1 [101] COPTIC CAPITAL LETTER ALFA..COPTIC SYMBOL KAI +2CEB..2CEF ; Exclusion # 5.2 [5] COPTIC CAPITAL LETTER CRYPTOGRAMMIC SHEI..COPTIC COMBINING NI ABOVE +2CF2..2CF3 ; Exclusion # 6.1 [2] COPTIC CAPITAL LETTER BOHAIRIC KHEI..COPTIC SMALL LETTER BOHAIRIC KHEI +A840..A873 ; Exclusion # 5.0 [52] PHAGS-PA LETTER KA..PHAGS-PA LETTER CANDRABINDU +A930..A953 ; Exclusion # 5.1 [36] REJANG LETTER KA..REJANG VIRAMA +10000..1000B ; Exclusion # 4.0 [12] LINEAR B SYLLABLE B008 A..LINEAR B SYLLABLE B046 JE +1000D..10026 ; Exclusion # 4.0 [26] LINEAR B SYLLABLE B036 JO..LINEAR B SYLLABLE B032 QO +10028..1003A ; Exclusion # 4.0 [19] LINEAR B SYLLABLE B060 RA..LINEAR B SYLLABLE B042 WO +1003C..1003D ; Exclusion # 4.0 [2] LINEAR B SYLLABLE B017 ZA..LINEAR B SYLLABLE B074 ZE +1003F..1004D ; Exclusion # 4.0 [15] LINEAR B SYLLABLE B020 ZO..LINEAR B SYLLABLE B091 TWO +10050..1005D ; Exclusion # 4.0 [14] LINEAR B SYMBOL B018..LINEAR B SYMBOL B089 +10080..100FA ; Exclusion # 4.0 [123] LINEAR B IDEOGRAM B100 MAN..LINEAR B IDEOGRAM VESSEL B305 +10280..1029C ; Exclusion # 5.1 [29] LYCIAN LETTER A..LYCIAN LETTER X +102A0..102D0 ; Exclusion # 5.1 [49] CARIAN LETTER A..CARIAN LETTER UUU3 +10300..1031E ; Exclusion # 3.1 [31] OLD ITALIC LETTER A..OLD ITALIC LETTER UU +1031F ; Exclusion # 7.0 OLD ITALIC LETTER ESS +1032D..1032F ; Exclusion # 10.0 [3] OLD ITALIC LETTER YE..OLD ITALIC LETTER SOUTHERN TSE +10330..1034A ; Exclusion # 3.1 [27] GOTHIC LETTER AHSA..GOTHIC LETTER NINE HUNDRED +10350..1037A ; Exclusion # 7.0 [43] OLD PERMIC LETTER AN..COMBINING OLD PERMIC LETTER SII +10380..1039D ; Exclusion # 4.0 [30] UGARITIC LETTER ALPA..UGARITIC LETTER SSU +103A0..103C3 ; Exclusion # 4.1 [36] OLD PERSIAN SIGN A..OLD PERSIAN SIGN HA +103C8..103CF ; Exclusion # 4.1 [8] OLD PERSIAN SIGN AURAMAZDAA..OLD PERSIAN SIGN BUUMISH +103D1..103D5 ; Exclusion # 4.1 [5] OLD PERSIAN NUMBER ONE..OLD PERSIAN NUMBER HUNDRED +10400..10425 ; Exclusion # 3.1 [38] DESERET CAPITAL LETTER LONG I..DESERET CAPITAL LETTER ENG +10426..10427 ; Exclusion # 4.0 [2] DESERET CAPITAL LETTER OI..DESERET CAPITAL LETTER EW +10428..1044D ; Exclusion # 3.1 [38] DESERET SMALL LETTER LONG I..DESERET SMALL LETTER ENG +1044E..1049D ; Exclusion # 4.0 [80] DESERET SMALL LETTER OI..OSMANYA LETTER OO +104A0..104A9 ; Exclusion # 4.0 [10] OSMANYA DIGIT ZERO..OSMANYA DIGIT NINE +10500..10527 ; Exclusion # 7.0 [40] ELBASAN LETTER A..ELBASAN LETTER KHE +10530..10563 ; Exclusion # 7.0 [52] CAUCASIAN ALBANIAN LETTER ALT..CAUCASIAN ALBANIAN LETTER KIW +10570..1057A ; Exclusion # 14.0 [11] VITHKUQI CAPITAL LETTER A..VITHKUQI CAPITAL LETTER GA +1057C..1058A ; Exclusion # 14.0 [15] VITHKUQI CAPITAL LETTER HA..VITHKUQI CAPITAL LETTER RE +1058C..10592 ; Exclusion # 14.0 [7] VITHKUQI CAPITAL LETTER SE..VITHKUQI CAPITAL LETTER XE +10594..10595 ; Exclusion # 14.0 [2] VITHKUQI CAPITAL LETTER Y..VITHKUQI CAPITAL LETTER ZE +10597..105A1 ; Exclusion # 14.0 [11] VITHKUQI SMALL LETTER A..VITHKUQI SMALL LETTER GA +105A3..105B1 ; Exclusion # 14.0 [15] VITHKUQI SMALL LETTER HA..VITHKUQI SMALL LETTER RE +105B3..105B9 ; Exclusion # 14.0 [7] VITHKUQI SMALL LETTER SE..VITHKUQI SMALL LETTER XE +105BB..105BC ; Exclusion # 14.0 [2] VITHKUQI SMALL LETTER Y..VITHKUQI SMALL LETTER ZE +105C0..105F3 ; Exclusion # 16.0 [52] TODHRI LETTER A..TODHRI LETTER OO +10600..10736 ; Exclusion # 7.0 [311] LINEAR A SIGN AB001..LINEAR A SIGN A664 +10740..10755 ; Exclusion # 7.0 [22] LINEAR A SIGN A701 A..LINEAR A SIGN A732 JE +10760..10767 ; Exclusion # 7.0 [8] LINEAR A SIGN A800..LINEAR A SIGN A807 +10800..10805 ; Exclusion # 4.0 [6] CYPRIOT SYLLABLE A..CYPRIOT SYLLABLE JA +10808 ; Exclusion # 4.0 CYPRIOT SYLLABLE JO +1080A..10835 ; Exclusion # 4.0 [44] CYPRIOT SYLLABLE KA..CYPRIOT SYLLABLE WO +10837..10838 ; Exclusion # 4.0 [2] CYPRIOT SYLLABLE XA..CYPRIOT SYLLABLE XE +1083C ; Exclusion # 4.0 CYPRIOT SYLLABLE ZA +1083F ; Exclusion # 4.0 CYPRIOT SYLLABLE ZO +10840..10855 ; Exclusion # 5.2 [22] IMPERIAL ARAMAIC LETTER ALEPH..IMPERIAL ARAMAIC LETTER TAW +10860..10876 ; Exclusion # 7.0 [23] PALMYRENE LETTER ALEPH..PALMYRENE LETTER TAW +10880..1089E ; Exclusion # 7.0 [31] NABATAEAN LETTER FINAL ALEPH..NABATAEAN LETTER TAW +108E0..108F2 ; Exclusion # 8.0 [19] HATRAN LETTER ALEPH..HATRAN LETTER QOPH +108F4..108F5 ; Exclusion # 8.0 [2] HATRAN LETTER SHIN..HATRAN LETTER TAW +10900..10915 ; Exclusion # 5.0 [22] PHOENICIAN LETTER ALF..PHOENICIAN LETTER TAU +10920..10939 ; Exclusion # 5.1 [26] LYDIAN LETTER A..LYDIAN LETTER C +10940..10959 ; Exclusion # 17.0 [26] SIDETIC LETTER N01..SIDETIC LETTER N26 +10980..109B7 ; Exclusion # 6.1 [56] MEROITIC HIEROGLYPHIC LETTER A..MEROITIC CURSIVE LETTER DA +109BE..109BF ; Exclusion # 6.1 [2] MEROITIC CURSIVE LOGOGRAM RMT..MEROITIC CURSIVE LOGOGRAM IMN +10A00..10A03 ; Exclusion # 4.1 [4] KHAROSHTHI LETTER A..KHAROSHTHI VOWEL SIGN VOCALIC R +10A05..10A06 ; Exclusion # 4.1 [2] KHAROSHTHI VOWEL SIGN E..KHAROSHTHI VOWEL SIGN O +10A0C..10A13 ; Exclusion # 4.1 [8] KHAROSHTHI VOWEL LENGTH MARK..KHAROSHTHI LETTER GHA +10A15..10A17 ; Exclusion # 4.1 [3] KHAROSHTHI LETTER CA..KHAROSHTHI LETTER JA +10A19..10A33 ; Exclusion # 4.1 [27] KHAROSHTHI LETTER NYA..KHAROSHTHI LETTER TTTHA +10A34..10A35 ; Exclusion # 11.0 [2] KHAROSHTHI LETTER TTTA..KHAROSHTHI LETTER VHA +10A38..10A3A ; Exclusion # 4.1 [3] KHAROSHTHI SIGN BAR ABOVE..KHAROSHTHI SIGN DOT BELOW +10A3F ; Exclusion # 4.1 KHAROSHTHI VIRAMA +10A60..10A7C ; Exclusion # 5.2 [29] OLD SOUTH ARABIAN LETTER HE..OLD SOUTH ARABIAN LETTER THETH +10A80..10A9C ; Exclusion # 7.0 [29] OLD NORTH ARABIAN LETTER HEH..OLD NORTH ARABIAN LETTER ZAH +10AC0..10AC7 ; Exclusion # 7.0 [8] MANICHAEAN LETTER ALEPH..MANICHAEAN LETTER WAW +10AC9..10AE6 ; Exclusion # 7.0 [30] MANICHAEAN LETTER ZAYIN..MANICHAEAN ABBREVIATION MARK BELOW +10B00..10B35 ; Exclusion # 5.2 [54] AVESTAN LETTER A..AVESTAN LETTER HE +10B40..10B55 ; Exclusion # 5.2 [22] INSCRIPTIONAL PARTHIAN LETTER ALEPH..INSCRIPTIONAL PARTHIAN LETTER TAW +10B60..10B72 ; Exclusion # 5.2 [19] INSCRIPTIONAL PAHLAVI LETTER ALEPH..INSCRIPTIONAL PAHLAVI LETTER TAW +10B80..10B91 ; Exclusion # 7.0 [18] PSALTER PAHLAVI LETTER ALEPH..PSALTER PAHLAVI LETTER TAW +10C00..10C48 ; Exclusion # 5.2 [73] OLD TURKIC LETTER ORKHON A..OLD TURKIC LETTER ORKHON BASH +10C80..10CB2 ; Exclusion # 8.0 [51] OLD HUNGARIAN CAPITAL LETTER A..OLD HUNGARIAN CAPITAL LETTER US +10CC0..10CF2 ; Exclusion # 8.0 [51] OLD HUNGARIAN SMALL LETTER A..OLD HUNGARIAN SMALL LETTER US +10D40..10D65 ; Exclusion # 16.0 [38] GARAY DIGIT ZERO..GARAY CAPITAL LETTER OLD NA +10D69..10D6D ; Exclusion # 16.0 [5] GARAY VOWEL SIGN E..GARAY CONSONANT NASALIZATION MARK +10D6F..10D85 ; Exclusion # 16.0 [23] GARAY REDUPLICATION MARK..GARAY SMALL LETTER OLD NA +10E80..10EA9 ; Exclusion # 13.0 [42] YEZIDI LETTER ELIF..YEZIDI LETTER ET +10EAB..10EAC ; Exclusion # 13.0 [2] YEZIDI COMBINING HAMZA MARK..YEZIDI COMBINING MADDA MARK +10EB0..10EB1 ; Exclusion # 13.0 [2] YEZIDI LETTER LAM WITH DOT ABOVE..YEZIDI LETTER YOT WITH CIRCUMFLEX ABOVE +10F00..10F1C ; Exclusion # 11.0 [29] OLD SOGDIAN LETTER ALEPH..OLD SOGDIAN LETTER FINAL TAW WITH VERTICAL TAIL +10F27 ; Exclusion # 11.0 OLD SOGDIAN LIGATURE AYIN-DALETH +10F30..10F50 ; Exclusion # 11.0 [33] SOGDIAN LETTER ALEPH..SOGDIAN COMBINING STROKE BELOW +10F70..10F85 ; Exclusion # 14.0 [22] OLD UYGHUR LETTER ALEPH..OLD UYGHUR COMBINING TWO DOTS BELOW +10FB0..10FC4 ; Exclusion # 13.0 [21] CHORASMIAN LETTER ALEPH..CHORASMIAN LETTER TAW +10FE0..10FF6 ; Exclusion # 12.0 [23] ELYMAIC LETTER ALEPH..ELYMAIC LIGATURE ZAYIN-YODH +11000..11046 ; Exclusion # 6.0 [71] BRAHMI SIGN CANDRABINDU..BRAHMI VIRAMA +11066..1106F ; Exclusion # 6.0 [10] BRAHMI DIGIT ZERO..BRAHMI DIGIT NINE +11070..11075 ; Exclusion # 14.0 [6] BRAHMI SIGN OLD TAMIL VIRAMA..BRAHMI LETTER OLD TAMIL LLA +1107F ; Exclusion # 7.0 BRAHMI NUMBER JOINER +11080..110BA ; Exclusion # 5.2 [59] KAITHI SIGN CANDRABINDU..KAITHI SIGN NUKTA +110C2 ; Exclusion # 14.0 KAITHI VOWEL SIGN VOCALIC R +110D0..110E8 ; Exclusion # 6.1 [25] SORA SOMPENG LETTER SAH..SORA SOMPENG LETTER MAE +110F0..110F9 ; Exclusion # 6.1 [10] SORA SOMPENG DIGIT ZERO..SORA SOMPENG DIGIT NINE +11150..11173 ; Exclusion # 7.0 [36] MAHAJANI LETTER A..MAHAJANI SIGN NUKTA +11176 ; Exclusion # 7.0 MAHAJANI LIGATURE SHRI +11180..111C4 ; Exclusion # 6.1 [69] SHARADA SIGN CANDRABINDU..SHARADA OM +111C9..111CC ; Exclusion # 8.0 [4] SHARADA SANDHI MARK..SHARADA EXTRA SHORT VOWEL MARK +111CE..111CF ; Exclusion # 13.0 [2] SHARADA VOWEL SIGN PRISHTHAMATRA E..SHARADA SIGN INVERTED CANDRABINDU +111D0..111D9 ; Exclusion # 6.1 [10] SHARADA DIGIT ZERO..SHARADA DIGIT NINE +111DA ; Exclusion # 7.0 SHARADA EKAM +111DC ; Exclusion # 8.0 SHARADA HEADSTROKE +11200..11211 ; Exclusion # 7.0 [18] KHOJKI LETTER A..KHOJKI LETTER JJA +11213..11237 ; Exclusion # 7.0 [37] KHOJKI LETTER NYA..KHOJKI SIGN SHADDA +1123E ; Exclusion # 9.0 KHOJKI SIGN SUKUN +1123F..11241 ; Exclusion # 15.0 [3] KHOJKI LETTER QA..KHOJKI VOWEL SIGN VOCALIC R +11280..11286 ; Exclusion # 8.0 [7] MULTANI LETTER A..MULTANI LETTER GA +11288 ; Exclusion # 8.0 MULTANI LETTER GHA +1128A..1128D ; Exclusion # 8.0 [4] MULTANI LETTER CA..MULTANI LETTER JJA +1128F..1129D ; Exclusion # 8.0 [15] MULTANI LETTER NYA..MULTANI LETTER BA +1129F..112A8 ; Exclusion # 8.0 [10] MULTANI LETTER BHA..MULTANI LETTER RHA +112B0..112EA ; Exclusion # 7.0 [59] KHUDAWADI LETTER A..KHUDAWADI SIGN VIRAMA +112F0..112F9 ; Exclusion # 7.0 [10] KHUDAWADI DIGIT ZERO..KHUDAWADI DIGIT NINE +11300 ; Exclusion # 8.0 GRANTHA SIGN COMBINING ANUSVARA ABOVE +11302 ; Exclusion # 7.0 GRANTHA SIGN ANUSVARA +11305..1130C ; Exclusion # 7.0 [8] GRANTHA LETTER A..GRANTHA LETTER VOCALIC L +1130F..11310 ; Exclusion # 7.0 [2] GRANTHA LETTER EE..GRANTHA LETTER AI +11313..11328 ; Exclusion # 7.0 [22] GRANTHA LETTER OO..GRANTHA LETTER NA +1132A..11330 ; Exclusion # 7.0 [7] GRANTHA LETTER PA..GRANTHA LETTER RA +11332..11333 ; Exclusion # 7.0 [2] GRANTHA LETTER LA..GRANTHA LETTER LLA +11335..11339 ; Exclusion # 7.0 [5] GRANTHA LETTER VA..GRANTHA LETTER HA +1133D..11344 ; Exclusion # 7.0 [8] GRANTHA SIGN AVAGRAHA..GRANTHA VOWEL SIGN VOCALIC RR +11347..11348 ; Exclusion # 7.0 [2] GRANTHA VOWEL SIGN EE..GRANTHA VOWEL SIGN AI +1134B..1134D ; Exclusion # 7.0 [3] GRANTHA VOWEL SIGN OO..GRANTHA SIGN VIRAMA +11350 ; Exclusion # 8.0 GRANTHA OM +11357 ; Exclusion # 7.0 GRANTHA AU LENGTH MARK +1135D..11363 ; Exclusion # 7.0 [7] GRANTHA SIGN PLUTA..GRANTHA VOWEL SIGN VOCALIC LL +11366..1136C ; Exclusion # 7.0 [7] COMBINING GRANTHA DIGIT ZERO..COMBINING GRANTHA DIGIT SIX +11370..11374 ; Exclusion # 7.0 [5] COMBINING GRANTHA LETTER A..COMBINING GRANTHA LETTER PA +11380..11389 ; Exclusion # 16.0 [10] TULU-TIGALARI LETTER A..TULU-TIGALARI LETTER VOCALIC LL +1138B ; Exclusion # 16.0 TULU-TIGALARI LETTER EE +1138E ; Exclusion # 16.0 TULU-TIGALARI LETTER AI +11390..113B5 ; Exclusion # 16.0 [38] TULU-TIGALARI LETTER OO..TULU-TIGALARI LETTER LLLA +113B7..113C0 ; Exclusion # 16.0 [10] TULU-TIGALARI SIGN AVAGRAHA..TULU-TIGALARI VOWEL SIGN VOCALIC LL +113C2 ; Exclusion # 16.0 TULU-TIGALARI VOWEL SIGN EE +113C5 ; Exclusion # 16.0 TULU-TIGALARI VOWEL SIGN AI +113C7..113CA ; Exclusion # 16.0 [4] TULU-TIGALARI VOWEL SIGN OO..TULU-TIGALARI SIGN CANDRA ANUNASIKA +113CC..113D3 ; Exclusion # 16.0 [8] TULU-TIGALARI SIGN ANUSVARA..TULU-TIGALARI SIGN PLUTA +113E1..113E2 ; Exclusion # 16.0 [2] TULU-TIGALARI VEDIC TONE SVARITA..TULU-TIGALARI VEDIC TONE ANUDATTA +11480..114C5 ; Exclusion # 7.0 [70] TIRHUTA ANJI..TIRHUTA GVANG +114C7 ; Exclusion # 7.0 TIRHUTA OM +114D0..114D9 ; Exclusion # 7.0 [10] TIRHUTA DIGIT ZERO..TIRHUTA DIGIT NINE +11580..115B5 ; Exclusion # 7.0 [54] SIDDHAM LETTER A..SIDDHAM VOWEL SIGN VOCALIC RR +115B8..115C0 ; Exclusion # 7.0 [9] SIDDHAM VOWEL SIGN E..SIDDHAM SIGN NUKTA +115D8..115DD ; Exclusion # 8.0 [6] SIDDHAM LETTER THREE-CIRCLE ALTERNATE I..SIDDHAM VOWEL SIGN ALTERNATE UU +11600..11640 ; Exclusion # 7.0 [65] MODI LETTER A..MODI SIGN ARDHACANDRA +11644 ; Exclusion # 7.0 MODI SIGN HUVA +11650..11659 ; Exclusion # 7.0 [10] MODI DIGIT ZERO..MODI DIGIT NINE +11680..116B7 ; Exclusion # 6.1 [56] TAKRI LETTER A..TAKRI SIGN NUKTA +116B8 ; Exclusion # 12.0 TAKRI LETTER ARCHAIC KHA +116C0..116C9 ; Exclusion # 6.1 [10] TAKRI DIGIT ZERO..TAKRI DIGIT NINE +11700..11719 ; Exclusion # 8.0 [26] AHOM LETTER KA..AHOM LETTER JHA +1171A ; Exclusion # 11.0 AHOM LETTER ALTERNATE BA +1171D..1172B ; Exclusion # 8.0 [15] AHOM CONSONANT SIGN MEDIAL LA..AHOM SIGN KILLER +11730..11739 ; Exclusion # 8.0 [10] AHOM DIGIT ZERO..AHOM DIGIT NINE +11740..11746 ; Exclusion # 14.0 [7] AHOM LETTER CA..AHOM LETTER LLA +11800..1183A ; Exclusion # 11.0 [59] DOGRA LETTER A..DOGRA SIGN NUKTA +118A0..118E9 ; Exclusion # 7.0 [74] WARANG CITI CAPITAL LETTER NGAA..WARANG CITI DIGIT NINE +118FF ; Exclusion # 7.0 WARANG CITI OM +11900..11906 ; Exclusion # 13.0 [7] DIVES AKURU LETTER A..DIVES AKURU LETTER E +11909 ; Exclusion # 13.0 DIVES AKURU LETTER O +1190C..11913 ; Exclusion # 13.0 [8] DIVES AKURU LETTER KA..DIVES AKURU LETTER JA +11915..11916 ; Exclusion # 13.0 [2] DIVES AKURU LETTER NYA..DIVES AKURU LETTER TTA +11918..11935 ; Exclusion # 13.0 [30] DIVES AKURU LETTER DDA..DIVES AKURU VOWEL SIGN E +11937..11938 ; Exclusion # 13.0 [2] DIVES AKURU VOWEL SIGN AI..DIVES AKURU VOWEL SIGN O +1193B..11943 ; Exclusion # 13.0 [9] DIVES AKURU SIGN ANUSVARA..DIVES AKURU SIGN NUKTA +11950..11959 ; Exclusion # 13.0 [10] DIVES AKURU DIGIT ZERO..DIVES AKURU DIGIT NINE +119A0..119A7 ; Exclusion # 12.0 [8] NANDINAGARI LETTER A..NANDINAGARI LETTER VOCALIC RR +119AA..119D7 ; Exclusion # 12.0 [46] NANDINAGARI LETTER E..NANDINAGARI VOWEL SIGN VOCALIC RR +119DA..119E1 ; Exclusion # 12.0 [8] NANDINAGARI VOWEL SIGN E..NANDINAGARI SIGN AVAGRAHA +119E3..119E4 ; Exclusion # 12.0 [2] NANDINAGARI HEADSTROKE..NANDINAGARI VOWEL SIGN PRISHTHAMATRA E +11A00..11A3E ; Exclusion # 10.0 [63] ZANABAZAR SQUARE LETTER A..ZANABAZAR SQUARE CLUSTER-FINAL LETTER VA +11A47 ; Exclusion # 10.0 ZANABAZAR SQUARE SUBJOINER +11A50..11A83 ; Exclusion # 10.0 [52] SOYOMBO LETTER A..SOYOMBO LETTER KSSA +11A84..11A85 ; Exclusion # 12.0 [2] SOYOMBO SIGN JIHVAMULIYA..SOYOMBO SIGN UPADHMANIYA +11A86..11A99 ; Exclusion # 10.0 [20] SOYOMBO CLUSTER-INITIAL LETTER RA..SOYOMBO SUBJOINER +11A9D ; Exclusion # 11.0 SOYOMBO MARK PLUTA +11AC0..11AF8 ; Exclusion # 7.0 [57] PAU CIN HAU LETTER PA..PAU CIN HAU GLOTTAL STOP FINAL +11B60..11B67 ; Exclusion # 17.0 [8] SHARADA VOWEL SIGN OE..SHARADA VOWEL SIGN CANDRA O +11BC0..11BE0 ; Exclusion # 16.0 [33] SUNUWAR LETTER DEVI..SUNUWAR LETTER KLOKO +11BF0..11BF9 ; Exclusion # 16.0 [10] SUNUWAR DIGIT ZERO..SUNUWAR DIGIT NINE +11C00..11C08 ; Exclusion # 9.0 [9] BHAIKSUKI LETTER A..BHAIKSUKI LETTER VOCALIC L +11C0A..11C36 ; Exclusion # 9.0 [45] BHAIKSUKI LETTER E..BHAIKSUKI VOWEL SIGN VOCALIC L +11C38..11C40 ; Exclusion # 9.0 [9] BHAIKSUKI VOWEL SIGN E..BHAIKSUKI SIGN AVAGRAHA +11C50..11C59 ; Exclusion # 9.0 [10] BHAIKSUKI DIGIT ZERO..BHAIKSUKI DIGIT NINE +11C72..11C8F ; Exclusion # 9.0 [30] MARCHEN LETTER KA..MARCHEN LETTER A +11C92..11CA7 ; Exclusion # 9.0 [22] MARCHEN SUBJOINED LETTER KA..MARCHEN SUBJOINED LETTER ZA +11CA9..11CB6 ; Exclusion # 9.0 [14] MARCHEN SUBJOINED LETTER YA..MARCHEN SIGN CANDRABINDU +11D00..11D06 ; Exclusion # 10.0 [7] MASARAM GONDI LETTER A..MASARAM GONDI LETTER E +11D08..11D09 ; Exclusion # 10.0 [2] MASARAM GONDI LETTER AI..MASARAM GONDI LETTER O +11D0B..11D36 ; Exclusion # 10.0 [44] MASARAM GONDI LETTER AU..MASARAM GONDI VOWEL SIGN VOCALIC R +11D3A ; Exclusion # 10.0 MASARAM GONDI VOWEL SIGN E +11D3C..11D3D ; Exclusion # 10.0 [2] MASARAM GONDI VOWEL SIGN AI..MASARAM GONDI VOWEL SIGN O +11D3F..11D47 ; Exclusion # 10.0 [9] MASARAM GONDI VOWEL SIGN AU..MASARAM GONDI RA-KARA +11D50..11D59 ; Exclusion # 10.0 [10] MASARAM GONDI DIGIT ZERO..MASARAM GONDI DIGIT NINE +11D60..11D65 ; Exclusion # 11.0 [6] GUNJALA GONDI LETTER A..GUNJALA GONDI LETTER UU +11D67..11D68 ; Exclusion # 11.0 [2] GUNJALA GONDI LETTER EE..GUNJALA GONDI LETTER AI +11D6A..11D8E ; Exclusion # 11.0 [37] GUNJALA GONDI LETTER OO..GUNJALA GONDI VOWEL SIGN UU +11D90..11D91 ; Exclusion # 11.0 [2] GUNJALA GONDI VOWEL SIGN EE..GUNJALA GONDI VOWEL SIGN AI +11D93..11D98 ; Exclusion # 11.0 [6] GUNJALA GONDI VOWEL SIGN OO..GUNJALA GONDI OM +11DA0..11DA9 ; Exclusion # 11.0 [10] GUNJALA GONDI DIGIT ZERO..GUNJALA GONDI DIGIT NINE +11DB0..11DDB ; Exclusion # 17.0 [44] TOLONG SIKI LETTER I..TOLONG SIKI UNGGA +11DE0..11DE9 ; Exclusion # 17.0 [10] TOLONG SIKI DIGIT ZERO..TOLONG SIKI DIGIT NINE +11EE0..11EF6 ; Exclusion # 11.0 [23] MAKASAR LETTER KA..MAKASAR VOWEL SIGN O +11F00..11F10 ; Exclusion # 15.0 [17] KAWI SIGN CANDRABINDU..KAWI LETTER O +11F12..11F3A ; Exclusion # 15.0 [41] KAWI LETTER KA..KAWI VOWEL SIGN VOCALIC R +11F3E..11F42 ; Exclusion # 15.0 [5] KAWI VOWEL SIGN E..KAWI CONJOINER +11F50..11F59 ; Exclusion # 15.0 [10] KAWI DIGIT ZERO..KAWI DIGIT NINE +11F5A ; Exclusion # 16.0 KAWI SIGN NUKTA +12000..1236E ; Exclusion # 5.0 [879] CUNEIFORM SIGN A..CUNEIFORM SIGN ZUM +1236F..12398 ; Exclusion # 7.0 [42] CUNEIFORM SIGN KAP ELAMITE..CUNEIFORM SIGN UM TIMES ME +12399 ; Exclusion # 8.0 CUNEIFORM SIGN U U +12400..12462 ; Exclusion # 5.0 [99] CUNEIFORM NUMERIC SIGN TWO ASH..CUNEIFORM NUMERIC SIGN OLD ASSYRIAN ONE QUARTER +12463..1246E ; Exclusion # 7.0 [12] CUNEIFORM NUMERIC SIGN ONE QUARTER GUR..CUNEIFORM NUMERIC SIGN NINE U VARIANT FORM +12480..12543 ; Exclusion # 8.0 [196] CUNEIFORM SIGN AB TIMES NUN TENU..CUNEIFORM SIGN ZU5 TIMES THREE DISH TENU +12F90..12FF0 ; Exclusion # 14.0 [97] CYPRO-MINOAN SIGN CM001..CYPRO-MINOAN SIGN CM114 +13000..1342E ; Exclusion # 5.2 [1071] EGYPTIAN HIEROGLYPH A001..EGYPTIAN HIEROGLYPH AA032 +1342F ; Exclusion # 15.0 EGYPTIAN HIEROGLYPH V011D +13440..13455 ; Exclusion # 15.0 [22] EGYPTIAN HIEROGLYPH MIRROR HORIZONTALLY..EGYPTIAN HIEROGLYPH MODIFIER DAMAGED +13460..143FA ; Exclusion # 16.0 [3995] EGYPTIAN HIEROGLYPH-13460..EGYPTIAN HIEROGLYPH-143FA +14400..14646 ; Exclusion # 8.0 [583] ANATOLIAN HIEROGLYPH A001..ANATOLIAN HIEROGLYPH A530 +16100..16139 ; Exclusion # 16.0 [58] GURUNG KHEMA LETTER A..GURUNG KHEMA DIGIT NINE +16A70..16ABE ; Exclusion # 14.0 [79] TANGSA LETTER OZ..TANGSA LETTER ZA +16AC0..16AC9 ; Exclusion # 14.0 [10] TANGSA DIGIT ZERO..TANGSA DIGIT NINE +16AD0..16AED ; Exclusion # 7.0 [30] BASSA VAH LETTER ENNI..BASSA VAH LETTER I +16AF0..16AF4 ; Exclusion # 7.0 [5] BASSA VAH COMBINING HIGH TONE..BASSA VAH COMBINING HIGH-LOW TONE +16B00..16B36 ; Exclusion # 7.0 [55] PAHAWH HMONG VOWEL KEEB..PAHAWH HMONG MARK CIM TAUM +16B40..16B43 ; Exclusion # 7.0 [4] PAHAWH HMONG SIGN VOS SEEV..PAHAWH HMONG SIGN IB YAM +16B50..16B59 ; Exclusion # 7.0 [10] PAHAWH HMONG DIGIT ZERO..PAHAWH HMONG DIGIT NINE +16B63..16B77 ; Exclusion # 7.0 [21] PAHAWH HMONG SIGN VOS LUB..PAHAWH HMONG SIGN CIM NRES TOS +16B7D..16B8F ; Exclusion # 7.0 [19] PAHAWH HMONG CLAN SIGN TSHEEJ..PAHAWH HMONG CLAN SIGN VWJ +16D40..16D6C ; Exclusion # 16.0 [45] KIRAT RAI SIGN ANUSVARA..KIRAT RAI SIGN SAAT +16D70..16D79 ; Exclusion # 16.0 [10] KIRAT RAI DIGIT ZERO..KIRAT RAI DIGIT NINE +16E40..16E7F ; Exclusion # 11.0 [64] MEDEFAIDRIN CAPITAL LETTER M..MEDEFAIDRIN SMALL LETTER Y +16EA0..16EB8 ; Exclusion # 17.0 [25] BERIA ERFE CAPITAL LETTER ARKAB..BERIA ERFE CAPITAL LETTER AY +16EBB..16ED3 ; Exclusion # 17.0 [25] BERIA ERFE SMALL LETTER ARKAB..BERIA ERFE SMALL LETTER AY +16FE0 ; Exclusion # 9.0 TANGUT ITERATION MARK +16FE1 ; Exclusion # 10.0 NUSHU ITERATION MARK +16FE4 ; Exclusion # 13.0 KHITAN SMALL SCRIPT FILLER +17000..187EC ; Exclusion # 9.0 [6125] TANGUT IDEOGRAPH-17000..TANGUT IDEOGRAPH-187EC +187ED..187F1 ; Exclusion # 11.0 [5] TANGUT IDEOGRAPH-187ED..TANGUT IDEOGRAPH-187F1 +187F2..187F7 ; Exclusion # 12.0 [6] TANGUT IDEOGRAPH-187F2..TANGUT IDEOGRAPH-187F7 +187F8..187FF ; Exclusion # 17.0 [8] TANGUT IDEOGRAPH-187F8..TANGUT IDEOGRAPH-187FF +18800..18AF2 ; Exclusion # 9.0 [755] TANGUT COMPONENT-001..TANGUT COMPONENT-755 +18AF3..18CD5 ; Exclusion # 13.0 [483] TANGUT COMPONENT-756..KHITAN SMALL SCRIPT CHARACTER-18CD5 +18CFF ; Exclusion # 16.0 KHITAN SMALL SCRIPT CHARACTER-18CFF +18D00..18D08 ; Exclusion # 13.0 [9] TANGUT IDEOGRAPH-18D00..TANGUT IDEOGRAPH-18D08 +18D09..18D1E ; Exclusion # 17.0 [22] TANGUT IDEOGRAPH-18D09..TANGUT IDEOGRAPH-18D1E +18D80..18DF2 ; Exclusion # 17.0 [115] TANGUT COMPONENT-769..TANGUT COMPONENT-883 +1B170..1B2FB ; Exclusion # 10.0 [396] NUSHU CHARACTER-1B170..NUSHU CHARACTER-1B2FB +1BC00..1BC6A ; Exclusion # 7.0 [107] DUPLOYAN LETTER H..DUPLOYAN LETTER VOCALIC M +1BC70..1BC7C ; Exclusion # 7.0 [13] DUPLOYAN AFFIX LEFT HORIZONTAL SECANT..DUPLOYAN AFFIX ATTACHED TANGENT HOOK +1BC80..1BC88 ; Exclusion # 7.0 [9] DUPLOYAN AFFIX HIGH ACUTE..DUPLOYAN AFFIX HIGH VERTICAL +1BC90..1BC99 ; Exclusion # 7.0 [10] DUPLOYAN AFFIX LOW ACUTE..DUPLOYAN AFFIX LOW ARROW +1BC9D..1BC9E ; Exclusion # 7.0 [2] DUPLOYAN THICK LETTER SELECTOR..DUPLOYAN DOUBLE MARK +1DA00..1DA36 ; Exclusion # 8.0 [55] SIGNWRITING HEAD RIM..SIGNWRITING AIR SUCKING IN +1DA3B..1DA6C ; Exclusion # 8.0 [50] SIGNWRITING MOUTH CLOSED NEUTRAL..SIGNWRITING EXCITEMENT +1DA75 ; Exclusion # 8.0 SIGNWRITING UPPER BODY TILTING FROM HIP JOINTS +1DA84 ; Exclusion # 8.0 SIGNWRITING LOCATION HEAD NECK +1DA9B..1DA9F ; Exclusion # 8.0 [5] SIGNWRITING FILL MODIFIER-2..SIGNWRITING FILL MODIFIER-6 +1DAA1..1DAAF ; Exclusion # 8.0 [15] SIGNWRITING ROTATION MODIFIER-2..SIGNWRITING ROTATION MODIFIER-16 +1E000..1E006 ; Exclusion # 9.0 [7] COMBINING GLAGOLITIC LETTER AZU..COMBINING GLAGOLITIC LETTER ZHIVETE +1E008..1E018 ; Exclusion # 9.0 [17] COMBINING GLAGOLITIC LETTER ZEMLJA..COMBINING GLAGOLITIC LETTER HERU +1E01B..1E021 ; Exclusion # 9.0 [7] COMBINING GLAGOLITIC LETTER SHTA..COMBINING GLAGOLITIC LETTER YATI +1E023..1E024 ; Exclusion # 9.0 [2] COMBINING GLAGOLITIC LETTER YU..COMBINING GLAGOLITIC LETTER SMALL YUS +1E026..1E02A ; Exclusion # 9.0 [5] COMBINING GLAGOLITIC LETTER YO..COMBINING GLAGOLITIC LETTER FITA +1E290..1E2AE ; Exclusion # 14.0 [31] TOTO LETTER PA..TOTO SIGN RISING TONE +1E4D0..1E4F9 ; Exclusion # 15.0 [42] NAG MUNDARI LETTER O..NAG MUNDARI DIGIT NINE +1E5D0..1E5FA ; Exclusion # 16.0 [43] OL ONAL LETTER O..OL ONAL DIGIT NINE +1E6C0..1E6DE ; Exclusion # 17.0 [31] TAI YO LETTER LOW KO..TAI YO LETTER HIGH KVO +1E6E0..1E6F5 ; Exclusion # 17.0 [22] TAI YO LETTER AA..TAI YO SIGN OM +1E6FE..1E6FF ; Exclusion # 17.0 [2] TAI YO SYMBOL MUEANG..TAI YO XAM LAI +1E800..1E8C4 ; Exclusion # 7.0 [197] MENDE KIKAKUI SYLLABLE M001 KI..MENDE KIKAKUI SYLLABLE M060 NYON +1E8D0..1E8D6 ; Exclusion # 7.0 [7] MENDE KIKAKUI COMBINING NUMBER TEENS..MENDE KIKAKUI COMBINING NUMBER MILLIONS + +# Total code points: 20862 + +# Identifier_Type: Exclusion Not_XID + +0830..083E ; Exclusion Not_XID # 5.2 [15] SAMARITAN PUNCTUATION NEQUDAA..SAMARITAN PUNCTUATION ANNAAU +1680 ; Exclusion Not_XID # 3.0 OGHAM SPACE MARK +169B..169C ; Exclusion Not_XID # 3.0 [2] OGHAM FEATHER MARK..OGHAM REVERSED FEATHER MARK +16EB..16ED ; Exclusion Not_XID # 3.0 [3] RUNIC SINGLE PUNCTUATION..RUNIC CROSS PUNCTUATION +1735..1736 ; Exclusion Not_XID # 3.2 [2] PHILIPPINE SINGLE PUNCTUATION..PHILIPPINE DOUBLE PUNCTUATION +1800..180A ; Exclusion Not_XID # 3.0 [11] MONGOLIAN BIRGA..MONGOLIAN NIRUGU +1A1E..1A1F ; Exclusion Not_XID # 4.1 [2] BUGINESE PALLAWA..BUGINESE END OF SECTION +2CE5..2CEA ; Exclusion Not_XID # 4.1 [6] COPTIC SYMBOL MI RO..COPTIC SYMBOL SHIMA SIMA +2CF9..2CFF ; Exclusion Not_XID # 4.1 [7] COPTIC OLD NUBIAN FULL STOP..COPTIC MORPHOLOGICAL DIVIDER +2E30 ; Exclusion Not_XID # 5.1 RING POINT +2E3C ; Exclusion Not_XID # 7.0 STENOGRAPHIC FULL STOP +A874..A877 ; Exclusion Not_XID # 5.0 [4] PHAGS-PA SINGLE HEAD MARK..PHAGS-PA MARK DOUBLE SHAD +A95F ; Exclusion Not_XID # 5.1 REJANG SECTION MARK +10100..10102 ; Exclusion Not_XID # 4.0 [3] AEGEAN WORD SEPARATOR LINE..AEGEAN CHECK MARK +10107..10133 ; Exclusion Not_XID # 4.0 [45] AEGEAN NUMBER ONE..AEGEAN NUMBER NINETY THOUSAND +10137..1013F ; Exclusion Not_XID # 4.0 [9] AEGEAN WEIGHT BASE UNIT..AEGEAN MEASURE THIRD SUBUNIT +10320..10323 ; Exclusion Not_XID # 3.1 [4] OLD ITALIC NUMERAL ONE..OLD ITALIC NUMERAL FIFTY +1039F ; Exclusion Not_XID # 4.0 UGARITIC WORD DIVIDER +103D0 ; Exclusion Not_XID # 4.1 OLD PERSIAN WORD DIVIDER +1056F ; Exclusion Not_XID # 7.0 CAUCASIAN ALBANIAN CITATION MARK +10857..1085F ; Exclusion Not_XID # 5.2 [9] IMPERIAL ARAMAIC SECTION SIGN..IMPERIAL ARAMAIC NUMBER TEN THOUSAND +10877..1087F ; Exclusion Not_XID # 7.0 [9] PALMYRENE LEFT-POINTING FLEURON..PALMYRENE NUMBER TWENTY +108A7..108AF ; Exclusion Not_XID # 7.0 [9] NABATAEAN NUMBER ONE..NABATAEAN NUMBER ONE HUNDRED +108FB..108FF ; Exclusion Not_XID # 8.0 [5] HATRAN NUMBER ONE..HATRAN NUMBER ONE HUNDRED +10916..10919 ; Exclusion Not_XID # 5.0 [4] PHOENICIAN NUMBER ONE..PHOENICIAN NUMBER ONE HUNDRED +1091A..1091B ; Exclusion Not_XID # 5.2 [2] PHOENICIAN NUMBER TWO..PHOENICIAN NUMBER THREE +1091F ; Exclusion Not_XID # 5.0 PHOENICIAN WORD SEPARATOR +1093F ; Exclusion Not_XID # 5.1 LYDIAN TRIANGULAR MARK +109BC..109BD ; Exclusion Not_XID # 8.0 [2] MEROITIC CURSIVE FRACTION ELEVEN TWELFTHS..MEROITIC CURSIVE FRACTION ONE HALF +109C0..109CF ; Exclusion Not_XID # 8.0 [16] MEROITIC CURSIVE NUMBER ONE..MEROITIC CURSIVE NUMBER SEVENTY +109D2..109FF ; Exclusion Not_XID # 8.0 [46] MEROITIC CURSIVE NUMBER ONE HUNDRED..MEROITIC CURSIVE FRACTION TEN TWELFTHS +10A40..10A47 ; Exclusion Not_XID # 4.1 [8] KHAROSHTHI DIGIT ONE..KHAROSHTHI NUMBER ONE THOUSAND +10A48 ; Exclusion Not_XID # 11.0 KHAROSHTHI FRACTION ONE HALF +10A50..10A58 ; Exclusion Not_XID # 4.1 [9] KHAROSHTHI PUNCTUATION DOT..KHAROSHTHI PUNCTUATION LINES +10A7D..10A7F ; Exclusion Not_XID # 5.2 [3] OLD SOUTH ARABIAN NUMBER ONE..OLD SOUTH ARABIAN NUMERIC INDICATOR +10A9D..10A9F ; Exclusion Not_XID # 7.0 [3] OLD NORTH ARABIAN NUMBER ONE..OLD NORTH ARABIAN NUMBER TWENTY +10AC8 ; Exclusion Not_XID # 7.0 MANICHAEAN SIGN UD +10AEB..10AF6 ; Exclusion Not_XID # 7.0 [12] MANICHAEAN NUMBER ONE..MANICHAEAN PUNCTUATION LINE FILLER +10B39..10B3F ; Exclusion Not_XID # 5.2 [7] AVESTAN ABBREVIATION MARK..LARGE ONE RING OVER TWO RINGS PUNCTUATION +10B58..10B5F ; Exclusion Not_XID # 5.2 [8] INSCRIPTIONAL PARTHIAN NUMBER ONE..INSCRIPTIONAL PARTHIAN NUMBER ONE THOUSAND +10B78..10B7F ; Exclusion Not_XID # 5.2 [8] INSCRIPTIONAL PAHLAVI NUMBER ONE..INSCRIPTIONAL PAHLAVI NUMBER ONE THOUSAND +10B99..10B9C ; Exclusion Not_XID # 7.0 [4] PSALTER PAHLAVI SECTION MARK..PSALTER PAHLAVI FOUR DOTS WITH DOT +10BA9..10BAF ; Exclusion Not_XID # 7.0 [7] PSALTER PAHLAVI NUMBER ONE..PSALTER PAHLAVI NUMBER ONE HUNDRED +10CFA..10CFF ; Exclusion Not_XID # 8.0 [6] OLD HUNGARIAN NUMBER ONE..OLD HUNGARIAN NUMBER ONE THOUSAND +10D6E ; Exclusion Not_XID # 16.0 GARAY HYPHEN +10D8E..10D8F ; Exclusion Not_XID # 16.0 [2] GARAY PLUS SIGN..GARAY MINUS SIGN +10EAD ; Exclusion Not_XID # 13.0 YEZIDI HYPHENATION MARK +10F1D..10F26 ; Exclusion Not_XID # 11.0 [10] OLD SOGDIAN NUMBER ONE..OLD SOGDIAN FRACTION ONE HALF +10F51..10F59 ; Exclusion Not_XID # 11.0 [9] SOGDIAN NUMBER ONE..SOGDIAN PUNCTUATION HALF CIRCLE WITH DOT +10F86..10F89 ; Exclusion Not_XID # 14.0 [4] OLD UYGHUR PUNCTUATION BAR..OLD UYGHUR PUNCTUATION FOUR DOTS +10FC5..10FCB ; Exclusion Not_XID # 13.0 [7] CHORASMIAN NUMBER ONE..CHORASMIAN NUMBER ONE HUNDRED +11047..1104D ; Exclusion Not_XID # 6.0 [7] BRAHMI DANDA..BRAHMI PUNCTUATION LOTUS +11052..11065 ; Exclusion Not_XID # 6.0 [20] BRAHMI NUMBER ONE..BRAHMI NUMBER ONE THOUSAND +110BB..110BC ; Exclusion Not_XID # 5.2 [2] KAITHI ABBREVIATION SIGN..KAITHI ENUMERATION SIGN +110BD ; Exclusion Not_XID # 5.2 KAITHI NUMBER SIGN +110BE..110C1 ; Exclusion Not_XID # 5.2 [4] KAITHI SECTION MARK..KAITHI DOUBLE DANDA +110CD ; Exclusion Not_XID # 11.0 KAITHI NUMBER SIGN ABOVE +11174..11175 ; Exclusion Not_XID # 7.0 [2] MAHAJANI ABBREVIATION SIGN..MAHAJANI SECTION MARK +111C5..111C8 ; Exclusion Not_XID # 6.1 [4] SHARADA DANDA..SHARADA SEPARATOR +111CD ; Exclusion Not_XID # 7.0 SHARADA SUTRA MARK +111DB ; Exclusion Not_XID # 8.0 SHARADA SIGN SIDDHAM +111DD..111DF ; Exclusion Not_XID # 8.0 [3] SHARADA CONTINUATION SIGN..SHARADA SECTION MARK-2 +11238..1123D ; Exclusion Not_XID # 7.0 [6] KHOJKI DANDA..KHOJKI ABBREVIATION SIGN +112A9 ; Exclusion Not_XID # 8.0 MULTANI SECTION MARK +113D4..113D5 ; Exclusion Not_XID # 16.0 [2] TULU-TIGALARI DANDA..TULU-TIGALARI DOUBLE DANDA +113D7..113D8 ; Exclusion Not_XID # 16.0 [2] TULU-TIGALARI SIGN OM PUSHPIKA..TULU-TIGALARI SIGN SHRII PUSHPIKA +114C6 ; Exclusion Not_XID # 7.0 TIRHUTA ABBREVIATION SIGN +115C1..115C9 ; Exclusion Not_XID # 7.0 [9] SIDDHAM SIGN SIDDHAM..SIDDHAM END OF TEXT MARK +115CA..115D7 ; Exclusion Not_XID # 8.0 [14] SIDDHAM SECTION MARK WITH TRIDENT AND U-SHAPED ORNAMENTS..SIDDHAM SECTION MARK WITH CIRCLES AND FOUR ENCLOSURES +11641..11643 ; Exclusion Not_XID # 7.0 [3] MODI DANDA..MODI ABBREVIATION SIGN +11660..1166C ; Exclusion Not_XID # 9.0 [13] MONGOLIAN BIRGA WITH ORNAMENT..MONGOLIAN TURNED SWIRL BIRGA WITH DOUBLE ORNAMENT +116B9 ; Exclusion Not_XID # 14.0 TAKRI ABBREVIATION SIGN +1173A..1173F ; Exclusion Not_XID # 8.0 [6] AHOM NUMBER TEN..AHOM SYMBOL VI +1183B ; Exclusion Not_XID # 11.0 DOGRA ABBREVIATION SIGN +118EA..118F2 ; Exclusion Not_XID # 7.0 [9] WARANG CITI NUMBER TEN..WARANG CITI NUMBER NINETY +11944..11946 ; Exclusion Not_XID # 13.0 [3] DIVES AKURU DOUBLE DANDA..DIVES AKURU END OF TEXT MARK +119E2 ; Exclusion Not_XID # 12.0 NANDINAGARI SIGN SIDDHAM +11A3F..11A46 ; Exclusion Not_XID # 10.0 [8] ZANABAZAR SQUARE INITIAL HEAD MARK..ZANABAZAR SQUARE CLOSING DOUBLE-LINED HEAD MARK +11A9A..11A9C ; Exclusion Not_XID # 10.0 [3] SOYOMBO MARK TSHEG..SOYOMBO MARK DOUBLE SHAD +11A9E..11AA2 ; Exclusion Not_XID # 10.0 [5] SOYOMBO HEAD MARK WITH MOON AND SUN AND TRIPLE FLAME..SOYOMBO TERMINAL MARK-2 +11BE1 ; Exclusion Not_XID # 16.0 SUNUWAR SIGN PVO +11C41..11C45 ; Exclusion Not_XID # 9.0 [5] BHAIKSUKI DANDA..BHAIKSUKI GAP FILLER-2 +11C5A..11C6C ; Exclusion Not_XID # 9.0 [19] BHAIKSUKI NUMBER ONE..BHAIKSUKI HUNDREDS UNIT MARK +11C70..11C71 ; Exclusion Not_XID # 9.0 [2] MARCHEN HEAD MARK..MARCHEN MARK SHAD +11EF7..11EF8 ; Exclusion Not_XID # 11.0 [2] MAKASAR PASSIMBANG..MAKASAR END OF SECTION +11F43..11F4F ; Exclusion Not_XID # 15.0 [13] KAWI DANDA..KAWI PUNCTUATION CLOSING SPIRAL +12470..12473 ; Exclusion Not_XID # 5.0 [4] CUNEIFORM PUNCTUATION SIGN OLD ASSYRIAN WORD DIVIDER..CUNEIFORM PUNCTUATION SIGN DIAGONAL TRICOLON +12474 ; Exclusion Not_XID # 7.0 CUNEIFORM PUNCTUATION SIGN DIAGONAL QUADCOLON +12FF1..12FF2 ; Exclusion Not_XID # 14.0 [2] CYPRO-MINOAN SIGN CM301..CYPRO-MINOAN SIGN CM302 +13430..13438 ; Exclusion Not_XID # 12.0 [9] EGYPTIAN HIEROGLYPH VERTICAL JOINER..EGYPTIAN HIEROGLYPH END SEGMENT +13439..1343F ; Exclusion Not_XID # 15.0 [7] EGYPTIAN HIEROGLYPH INSERT AT MIDDLE..EGYPTIAN HIEROGLYPH END WALLED ENCLOSURE +16A6E..16A6F ; Exclusion Not_XID # 7.0 [2] MRO DANDA..MRO DOUBLE DANDA +16AF5 ; Exclusion Not_XID # 7.0 BASSA VAH FULL STOP +16B37..16B3F ; Exclusion Not_XID # 7.0 [9] PAHAWH HMONG SIGN VOS THOM..PAHAWH HMONG SIGN XYEEM FAIB +16B44..16B45 ; Exclusion Not_XID # 7.0 [2] PAHAWH HMONG SIGN XAUS..PAHAWH HMONG SIGN CIM TSOV ROG +16B5B..16B61 ; Exclusion Not_XID # 7.0 [7] PAHAWH HMONG NUMBER TENS..PAHAWH HMONG NUMBER TRILLIONS +16D6D..16D6F ; Exclusion Not_XID # 16.0 [3] KIRAT RAI SIGN YUPI..KIRAT RAI DOUBLE DANDA +16E80..16E9A ; Exclusion Not_XID # 11.0 [27] MEDEFAIDRIN DIGIT ZERO..MEDEFAIDRIN EXCLAMATION OH +1BC9C ; Exclusion Not_XID # 7.0 DUPLOYAN SIGN O WITH CROSS +1BC9F ; Exclusion Not_XID # 7.0 DUPLOYAN PUNCTUATION CHINOOK FULL STOP +1D800..1D9FF ; Exclusion Not_XID # 8.0 [512] SIGNWRITING HAND-FIST INDEX..SIGNWRITING HEAD +1DA37..1DA3A ; Exclusion Not_XID # 8.0 [4] SIGNWRITING AIR BLOW SMALL ROTATIONS..SIGNWRITING BREATH EXHALE +1DA6D..1DA74 ; Exclusion Not_XID # 8.0 [8] SIGNWRITING SHOULDER HIP SPINE..SIGNWRITING TORSO-FLOORPLANE TWISTING +1DA76..1DA83 ; Exclusion Not_XID # 8.0 [14] SIGNWRITING LIMB COMBINATION..SIGNWRITING LOCATION DEPTH +1DA85..1DA8B ; Exclusion Not_XID # 8.0 [7] SIGNWRITING LOCATION TORSO..SIGNWRITING PARENTHESIS +1E5FF ; Exclusion Not_XID # 16.0 OL ONAL ABBREVIATION SIGN +1E8C7..1E8CF ; Exclusion Not_XID # 7.0 [9] MENDE KIKAKUI DIGIT ONE..MENDE KIKAKUI DIGIT NINE + +# Total code points: 1142 + +# Identifier_Type: Obsolete + +0138 ; Obsolete # 1.1 LATIN SMALL LETTER KRA +01B9 ; Obsolete # 1.1 LATIN SMALL LETTER EZH REVERSED +01BF ; Obsolete # 1.1 LATIN LETTER WYNN +01F6..01F7 ; Obsolete # 3.0 [2] LATIN CAPITAL LETTER HWAIR..LATIN CAPITAL LETTER WYNN +021C..021D ; Obsolete # 3.0 [2] LATIN CAPITAL LETTER YOGH..LATIN SMALL LETTER YOGH +0345 ; Obsolete # 1.1 COMBINING GREEK YPOGEGRAMMENI +0363..036F ; Obsolete # 3.2 [13] COMBINING LATIN SMALL LETTER A..COMBINING LATIN SMALL LETTER X +0370..0373 ; Obsolete # 5.1 [4] GREEK CAPITAL LETTER HETA..GREEK SMALL LETTER ARCHAIC SAMPI +0376..0377 ; Obsolete # 5.1 [2] GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA..GREEK SMALL LETTER PAMPHYLIAN DIGAMMA +037B..037D ; Obsolete # 5.0 [3] GREEK SMALL REVERSED LUNATE SIGMA SYMBOL..GREEK SMALL REVERSED DOTTED LUNATE SIGMA SYMBOL +037F ; Obsolete # 7.0 GREEK CAPITAL LETTER YOT +03D8..03D9 ; Obsolete # 3.2 [2] GREEK LETTER ARCHAIC KOPPA..GREEK SMALL LETTER ARCHAIC KOPPA +03DA ; Obsolete # 1.1 GREEK LETTER STIGMA +03DB ; Obsolete # 3.0 GREEK SMALL LETTER STIGMA +03DC ; Obsolete # 1.1 GREEK LETTER DIGAMMA +03DD ; Obsolete # 3.0 GREEK SMALL LETTER DIGAMMA +03DE ; Obsolete # 1.1 GREEK LETTER KOPPA +03DF ; Obsolete # 3.0 GREEK SMALL LETTER KOPPA +03E0 ; Obsolete # 1.1 GREEK LETTER SAMPI +03E1 ; Obsolete # 3.0 GREEK SMALL LETTER SAMPI +03F7..03F8 ; Obsolete # 4.0 [2] GREEK CAPITAL LETTER SHO..GREEK SMALL LETTER SHO +03FA..03FB ; Obsolete # 4.0 [2] GREEK CAPITAL LETTER SAN..GREEK SMALL LETTER SAN +03FD..03FF ; Obsolete # 4.1 [3] GREEK CAPITAL REVERSED LUNATE SIGMA SYMBOL..GREEK CAPITAL REVERSED DOTTED LUNATE SIGMA SYMBOL +0460..0481 ; Obsolete # 1.1 [34] CYRILLIC CAPITAL LETTER OMEGA..CYRILLIC SMALL LETTER KOPPA +0483 ; Obsolete # 1.1 COMBINING CYRILLIC TITLO +049C..049D ; Obsolete # 1.1 [2] CYRILLIC CAPITAL LETTER KA WITH VERTICAL STROKE..CYRILLIC SMALL LETTER KA WITH VERTICAL STROKE +04A6..04A7 ; Obsolete # 1.1 [2] CYRILLIC CAPITAL LETTER PE WITH MIDDLE HOOK..CYRILLIC SMALL LETTER PE WITH MIDDLE HOOK +04B8..04B9 ; Obsolete # 1.1 [2] CYRILLIC CAPITAL LETTER CHE WITH VERTICAL STROKE..CYRILLIC SMALL LETTER CHE WITH VERTICAL STROKE +0500..050F ; Obsolete # 3.2 [16] CYRILLIC CAPITAL LETTER KOMI DE..CYRILLIC SMALL LETTER KOMI TJE +0514..0523 ; Obsolete # 5.1 [16] CYRILLIC CAPITAL LETTER LHA..CYRILLIC SMALL LETTER EN WITH MIDDLE HOOK +0526..0527 ; Obsolete # 6.0 [2] CYRILLIC CAPITAL LETTER SHHA WITH DESCENDER..CYRILLIC SMALL LETTER SHHA WITH DESCENDER +0528..052F ; Obsolete # 7.0 [8] CYRILLIC CAPITAL LETTER EN WITH LEFT HOOK..CYRILLIC SMALL LETTER EL WITH DESCENDER +063B..063C ; Obsolete # 5.1 [2] ARABIC LETTER KEHEH WITH TWO DOTS ABOVE..ARABIC LETTER KEHEH WITH THREE DOTS BELOW +063E..063F ; Obsolete # 5.1 [2] ARABIC LETTER FARSI YEH WITH TWO DOTS ABOVE..ARABIC LETTER FARSI YEH WITH THREE DOTS ABOVE +0640 ; Obsolete # 1.1 ARABIC TATWEEL +066E..066F ; Obsolete # 3.2 [2] ARABIC LETTER DOTLESS BEH..ARABIC LETTER DOTLESS QAF +0690 ; Obsolete # 1.1 ARABIC LETTER DAL WITH FOUR DOTS ABOVE +06AC ; Obsolete # 1.1 ARABIC LETTER KAF WITH DOT ABOVE +077E..077F ; Obsolete # 5.1 [2] ARABIC LETTER SEEN WITH INVERTED V..ARABIC LETTER KAF WITH TWO DOTS ABOVE +088E ; Obsolete # 14.0 ARABIC VERTICAL TAIL +08AD..08B1 ; Obsolete # 7.0 [5] ARABIC LETTER LOW ALEF..ARABIC LETTER STRAIGHT WAW +08B5 ; Obsolete # 14.0 ARABIC LETTER QAF WITH DOT BELOW AND NO DOTS ABOVE +090C ; Obsolete # 1.1 DEVANAGARI LETTER VOCALIC L +093D ; Obsolete # 1.1 DEVANAGARI SIGN AVAGRAHA +094E ; Obsolete # 5.2 DEVANAGARI VOWEL SIGN PRISHTHAMATRA E +0951..0952 ; Obsolete # 1.1 [2] DEVANAGARI STRESS SIGN UDATTA..DEVANAGARI STRESS SIGN ANUDATTA +0960..0963 ; Obsolete # 1.1 [4] DEVANAGARI LETTER VOCALIC RR..DEVANAGARI VOWEL SIGN VOCALIC LL +0971 ; Obsolete # 5.1 DEVANAGARI SIGN HIGH SPACING DOT +0978 ; Obsolete # 7.0 DEVANAGARI LETTER MARWARI DDA +0980 ; Obsolete # 7.0 BENGALI ANJI +09BD ; Obsolete # 4.0 BENGALI SIGN AVAGRAHA +09E0..09E3 ; Obsolete # 1.1 [4] BENGALI LETTER VOCALIC RR..BENGALI VOWEL SIGN VOCALIC LL +09FC ; Obsolete # 10.0 BENGALI LETTER VEDIC ANUSVARA +0ABD ; Obsolete # 1.1 GUJARATI SIGN AVAGRAHA +0AE0 ; Obsolete # 1.1 GUJARATI LETTER VOCALIC RR +0AE1..0AE3 ; Obsolete # 4.0 [3] GUJARATI LETTER VOCALIC LL..GUJARATI VOWEL SIGN VOCALIC LL +0B3D ; Obsolete # 1.1 ORIYA SIGN AVAGRAHA +0B60..0B61 ; Obsolete # 1.1 [2] ORIYA LETTER VOCALIC RR..ORIYA LETTER VOCALIC LL +0C00 ; Obsolete # 7.0 TELUGU SIGN COMBINING CANDRABINDU ABOVE +0C34 ; Obsolete # 7.0 TELUGU LETTER LLLA +0C3D ; Obsolete # 5.1 TELUGU SIGN AVAGRAHA +0C58..0C59 ; Obsolete # 5.1 [2] TELUGU LETTER TSA..TELUGU LETTER DZA +0C5C ; Obsolete # 17.0 TELUGU ARCHAIC SHRII +0C60..0C61 ; Obsolete # 1.1 [2] TELUGU LETTER VOCALIC RR..TELUGU LETTER VOCALIC LL +0C81 ; Obsolete # 7.0 KANNADA SIGN CANDRABINDU +0C8C ; Obsolete # 1.1 KANNADA LETTER VOCALIC L +0CB1 ; Obsolete # 1.1 KANNADA LETTER RRA +0CBD ; Obsolete # 4.0 KANNADA SIGN AVAGRAHA +0CDC ; Obsolete # 17.0 KANNADA ARCHAIC SHRII +0CDE ; Obsolete # 1.1 KANNADA LETTER FA +0CE0..0CE1 ; Obsolete # 1.1 [2] KANNADA LETTER VOCALIC RR..KANNADA LETTER VOCALIC LL +0CE2..0CE3 ; Obsolete # 5.0 [2] KANNADA VOWEL SIGN VOCALIC L..KANNADA VOWEL SIGN VOCALIC LL +0CF1..0CF2 ; Obsolete # 5.0 [2] KANNADA SIGN JIHVAMULIYA..KANNADA SIGN UPADHMANIYA +0D01 ; Obsolete # 7.0 MALAYALAM SIGN CANDRABINDU +0D3A ; Obsolete # 6.0 MALAYALAM LETTER TTTA +0D3B..0D3C ; Obsolete # 10.0 [2] MALAYALAM SIGN VERTICAL BAR VIRAMA..MALAYALAM SIGN CIRCULAR VIRAMA +0D3D ; Obsolete # 5.1 MALAYALAM SIGN AVAGRAHA +0D4C ; Obsolete # 1.1 MALAYALAM VOWEL SIGN AU +0D4E ; Obsolete # 6.0 MALAYALAM LETTER DOT REPH +0D5F ; Obsolete # 8.0 MALAYALAM LETTER ARCHAIC II +0D60..0D61 ; Obsolete # 1.1 [2] MALAYALAM LETTER VOCALIC RR..MALAYALAM LETTER VOCALIC LL +0D9E ; Obsolete # 3.0 SINHALA LETTER KANTAJA NAASIKYAYA +0F86..0F8B ; Obsolete # 2.0 [6] TIBETAN SIGN LCI RTAGS..TIBETAN SIGN GRU MED RGYINGS +0F8C..0F8F ; Obsolete # 6.0 [4] TIBETAN SIGN INVERTED MCHU CAN..TIBETAN SUBJOINED SIGN INVERTED MCHU CAN +10A0..10C5 ; Obsolete # 1.1 [38] GEORGIAN CAPITAL LETTER AN..GEORGIAN CAPITAL LETTER HOE +10F1..10F6 ; Obsolete # 1.1 [6] GEORGIAN LETTER HE..GEORGIAN LETTER FI +1100..1159 ; Obsolete # 1.1 [90] HANGUL CHOSEONG KIYEOK..HANGUL CHOSEONG YEORINHIEUH +115A..115E ; Obsolete # 5.2 [5] HANGUL CHOSEONG KIYEOK-TIKEUT..HANGUL CHOSEONG TIKEUT-RIEUL +1161..11A2 ; Obsolete # 1.1 [66] HANGUL JUNGSEONG A..HANGUL JUNGSEONG SSANGARAEA +11A3..11A7 ; Obsolete # 5.2 [5] HANGUL JUNGSEONG A-EU..HANGUL JUNGSEONG O-YAE +11A8..11F9 ; Obsolete # 1.1 [82] HANGUL JONGSEONG KIYEOK..HANGUL JONGSEONG YEORINHIEUH +11FA..11FF ; Obsolete # 5.2 [6] HANGUL JONGSEONG KIYEOK-NIEUN..HANGUL JONGSEONG SSANGNIEUN +1369..1371 ; Obsolete # 3.0 [9] ETHIOPIC DIGIT ONE..ETHIOPIC DIGIT NINE +17A8 ; Obsolete # 3.0 KHMER INDEPENDENT VOWEL QUK +17D3 ; Obsolete # 3.0 KHMER SIGN BATHAMASAT +17DC ; Obsolete # 3.0 KHMER SIGN AVAKRAHASANYA +1AB0..1ABD ; Obsolete # 7.0 [14] COMBINING DOUBLED CIRCUMFLEX ACCENT..COMBINING PARENTHESES BELOW +1C80..1C88 ; Obsolete # 9.0 [9] CYRILLIC SMALL LETTER ROUNDED VE..CYRILLIC SMALL LETTER UNBLENDED UK +1CD0..1CD2 ; Obsolete # 5.2 [3] VEDIC TONE KARSHANA..VEDIC TONE PRENKHA +1CD4..1CF2 ; Obsolete # 5.2 [31] VEDIC SIGN YAJURVEDIC MIDLINE SVARITA..VEDIC SIGN ARDHAVISARGA +1CF3..1CF6 ; Obsolete # 6.1 [4] VEDIC SIGN ROTATED ARDHAVISARGA..VEDIC SIGN UPADHMANIYA +1CF7 ; Obsolete # 10.0 VEDIC SIGN ATIKRAMA +1CF8..1CF9 ; Obsolete # 7.0 [2] VEDIC TONE RING ABOVE..VEDIC TONE DOUBLE RING ABOVE +1F00..1F15 ; Obsolete # 1.1 [22] GREEK SMALL LETTER ALPHA WITH PSILI..GREEK SMALL LETTER EPSILON WITH DASIA AND OXIA +1F18..1F1D ; Obsolete # 1.1 [6] GREEK CAPITAL LETTER EPSILON WITH PSILI..GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA +1F20..1F45 ; Obsolete # 1.1 [38] GREEK SMALL LETTER ETA WITH PSILI..GREEK SMALL LETTER OMICRON WITH DASIA AND OXIA +1F48..1F4D ; Obsolete # 1.1 [6] GREEK CAPITAL LETTER OMICRON WITH PSILI..GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA +1F50..1F57 ; Obsolete # 1.1 [8] GREEK SMALL LETTER UPSILON WITH PSILI..GREEK SMALL LETTER UPSILON WITH DASIA AND PERISPOMENI +1F59 ; Obsolete # 1.1 GREEK CAPITAL LETTER UPSILON WITH DASIA +1F5B ; Obsolete # 1.1 GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA +1F5D ; Obsolete # 1.1 GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA +1F5F..1F70 ; Obsolete # 1.1 [18] GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI..GREEK SMALL LETTER ALPHA WITH VARIA +1F72 ; Obsolete # 1.1 GREEK SMALL LETTER EPSILON WITH VARIA +1F74 ; Obsolete # 1.1 GREEK SMALL LETTER ETA WITH VARIA +1F76 ; Obsolete # 1.1 GREEK SMALL LETTER IOTA WITH VARIA +1F78 ; Obsolete # 1.1 GREEK SMALL LETTER OMICRON WITH VARIA +1F7A ; Obsolete # 1.1 GREEK SMALL LETTER UPSILON WITH VARIA +1F7C ; Obsolete # 1.1 GREEK SMALL LETTER OMEGA WITH VARIA +1F80..1F9F ; Obsolete # 1.1 [32] GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI..GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1FB6..1FBA ; Obsolete # 1.1 [5] GREEK SMALL LETTER ALPHA WITH PERISPOMENI..GREEK CAPITAL LETTER ALPHA WITH VARIA +1FBC ; Obsolete # 1.1 GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI +1FC2..1FC4 ; Obsolete # 1.1 [3] GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI +1FC6..1FC8 ; Obsolete # 1.1 [3] GREEK SMALL LETTER ETA WITH PERISPOMENI..GREEK CAPITAL LETTER EPSILON WITH VARIA +1FCA ; Obsolete # 1.1 GREEK CAPITAL LETTER ETA WITH VARIA +1FCC ; Obsolete # 1.1 GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI +1FD0..1FD2 ; Obsolete # 1.1 [3] GREEK SMALL LETTER IOTA WITH VRACHY..GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA +1FD6..1FDA ; Obsolete # 1.1 [5] GREEK SMALL LETTER IOTA WITH PERISPOMENI..GREEK CAPITAL LETTER IOTA WITH VARIA +1FE0..1FE2 ; Obsolete # 1.1 [3] GREEK SMALL LETTER UPSILON WITH VRACHY..GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA +1FE4..1FEA ; Obsolete # 1.1 [7] GREEK SMALL LETTER RHO WITH PSILI..GREEK CAPITAL LETTER UPSILON WITH VARIA +1FF2..1FF4 ; Obsolete # 1.1 [3] GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI +1FF6..1FF8 ; Obsolete # 1.1 [3] GREEK SMALL LETTER OMEGA WITH PERISPOMENI..GREEK CAPITAL LETTER OMICRON WITH VARIA +1FFA ; Obsolete # 1.1 GREEK CAPITAL LETTER OMEGA WITH VARIA +1FFC ; Obsolete # 1.1 GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI +2132 ; Obsolete # 1.1 TURNED CAPITAL F +214E ; Obsolete # 5.0 TURNED SMALL F +2184 ; Obsolete # 5.0 LATIN SMALL LETTER REVERSED C +2185..2188 ; Obsolete # 5.1 [4] ROMAN NUMERAL SIX LATE FORM..ROMAN NUMERAL ONE HUNDRED THOUSAND +2C6D..2C6F ; Obsolete # 5.1 [3] LATIN CAPITAL LETTER ALPHA..LATIN CAPITAL LETTER TURNED A +2C70 ; Obsolete # 5.2 LATIN CAPITAL LETTER TURNED ALPHA +2C71..2C73 ; Obsolete # 5.1 [3] LATIN SMALL LETTER V WITH RIGHT HOOK..LATIN SMALL LETTER W WITH HOOK +2C74..2C76 ; Obsolete # 5.0 [3] LATIN SMALL LETTER V WITH CURL..LATIN SMALL LETTER HALF H +2C7E..2C7F ; Obsolete # 5.2 [2] LATIN CAPITAL LETTER S WITH SWASH TAIL..LATIN CAPITAL LETTER Z WITH SWASH TAIL +2D00..2D25 ; Obsolete # 4.1 [38] GEORGIAN SMALL LETTER AN..GEORGIAN SMALL LETTER HOE +2DE0..2DFF ; Obsolete # 5.1 [32] COMBINING CYRILLIC LETTER BE..COMBINING CYRILLIC LETTER IOTIFIED BIG YUS +31F0..31FF ; Obsolete # 3.2 [16] KATAKANA LETTER SMALL KU..KATAKANA LETTER SMALL RO +A640..A65F ; Obsolete # 5.1 [32] CYRILLIC CAPITAL LETTER ZEMLYA..CYRILLIC SMALL LETTER YN +A660..A661 ; Obsolete # 6.0 [2] CYRILLIC CAPITAL LETTER REVERSED TSE..CYRILLIC SMALL LETTER REVERSED TSE +A662..A66E ; Obsolete # 5.1 [13] CYRILLIC CAPITAL LETTER SOFT DE..CYRILLIC LETTER MULTIOCULAR O +A674..A67B ; Obsolete # 6.1 [8] COMBINING CYRILLIC LETTER UKRAINIAN IE..COMBINING CYRILLIC LETTER OMEGA +A67F..A697 ; Obsolete # 5.1 [25] CYRILLIC PAYEROK..CYRILLIC SMALL LETTER SHWE +A698..A69B ; Obsolete # 7.0 [4] CYRILLIC CAPITAL LETTER DOUBLE O..CYRILLIC SMALL LETTER CROSSED O +A69F ; Obsolete # 6.1 COMBINING CYRILLIC LETTER IOTIFIED E +A730..A76F ; Obsolete # 5.1 [64] LATIN LETTER SMALL CAPITAL F..LATIN SMALL LETTER CON +A771..A787 ; Obsolete # 5.1 [23] LATIN SMALL LETTER DUM..LATIN SMALL LETTER INSULAR T +A790..A791 ; Obsolete # 6.0 [2] LATIN CAPITAL LETTER N WITH DESCENDER..LATIN SMALL LETTER N WITH DESCENDER +A794..A79F ; Obsolete # 7.0 [12] LATIN SMALL LETTER C WITH PALATAL HOOK..LATIN SMALL LETTER VOLAPUK UE +A7A0..A7A9 ; Obsolete # 6.0 [10] LATIN CAPITAL LETTER G WITH OBLIQUE STROKE..LATIN SMALL LETTER S WITH OBLIQUE STROKE +A7AB..A7AD ; Obsolete # 7.0 [3] LATIN CAPITAL LETTER REVERSED OPEN E..LATIN CAPITAL LETTER L WITH BELT +A7B0..A7B1 ; Obsolete # 7.0 [2] LATIN CAPITAL LETTER TURNED K..LATIN CAPITAL LETTER TURNED T +A7C0..A7C1 ; Obsolete # 14.0 [2] LATIN CAPITAL LETTER OLD POLISH O..LATIN SMALL LETTER OLD POLISH O +A7C4 ; Obsolete # 12.0 LATIN CAPITAL LETTER C WITH PALATAL HOOK +A7C7..A7CA ; Obsolete # 13.0 [4] LATIN CAPITAL LETTER D WITH SHORT STROKE OVERLAY..LATIN SMALL LETTER S WITH SHORT STROKE OVERLAY +A7D0..A7D1 ; Obsolete # 14.0 [2] LATIN CAPITAL LETTER CLOSED INSULAR G..LATIN SMALL LETTER CLOSED INSULAR G +A7D2 ; Obsolete # 17.0 LATIN CAPITAL LETTER DOUBLE THORN +A7D3 ; Obsolete # 14.0 LATIN SMALL LETTER DOUBLE THORN +A7D4 ; Obsolete # 17.0 LATIN CAPITAL LETTER DOUBLE WYNN +A7D5..A7D9 ; Obsolete # 14.0 [5] LATIN SMALL LETTER DOUBLE WYNN..LATIN SMALL LETTER SIGMOID S +A7F5..A7F6 ; Obsolete # 13.0 [2] LATIN CAPITAL LETTER REVERSED HALF H..LATIN SMALL LETTER REVERSED HALF H +A7F7 ; Obsolete # 7.0 LATIN EPIGRAPHIC LETTER SIDEWAYS I +A7FB..A7FF ; Obsolete # 5.1 [5] LATIN EPIGRAPHIC LETTER REVERSED F..LATIN EPIGRAPHIC LETTER ARCHAIC M +A8E0..A8F7 ; Obsolete # 5.2 [24] COMBINING DEVANAGARI DIGIT ZERO..DEVANAGARI SIGN CANDRABINDU AVAGRAHA +A8FB ; Obsolete # 5.2 DEVANAGARI HEADSTROKE +A8FE..A8FF ; Obsolete # 11.0 [2] DEVANAGARI LETTER AY..DEVANAGARI VOWEL SIGN AY +A960..A97C ; Obsolete # 5.2 [29] HANGUL CHOSEONG TIKEUT-MIEUM..HANGUL CHOSEONG SSANGYEORINHIEUH +A9E0..A9E6 ; Obsolete # 7.0 [7] MYANMAR LETTER SHAN GHA..MYANMAR MODIFIER LETTER SHAN REDUPLICATION +AB30..AB5A ; Obsolete # 7.0 [43] LATIN SMALL LETTER BARRED ALPHA..LATIN SMALL LETTER Y WITH SHORT RIGHT LEG +AB64..AB65 ; Obsolete # 7.0 [2] LATIN SMALL LETTER INVERTED ALPHA..GREEK LETTER SMALL CAPITAL OMEGA +D7B0..D7C6 ; Obsolete # 5.2 [23] HANGUL JUNGSEONG O-YEO..HANGUL JUNGSEONG ARAEA-E +D7CB..D7FB ; Obsolete # 5.2 [49] HANGUL JONGSEONG NIEUN-RIEUL..HANGUL JONGSEONG PHIEUPH-THIEUTH +10140..10174 ; Obsolete # 4.1 [53] GREEK ACROPHONIC ATTIC ONE QUARTER..GREEK ACROPHONIC STRATIAN FIFTY MNAS +101FD ; Obsolete # 5.1 PHAISTOS DISC SIGN COMBINING OBLIQUE STROKE +102E0 ; Obsolete # 7.0 COPTIC EPACT THOUSANDS MARK +16FE3 ; Obsolete # 12.0 OLD CHINESE ITERATION MARK +16FF0..16FF1 ; Obsolete # 13.0 [2] VIETNAMESE ALTERNATE READING MARK CA..VIETNAMESE ALTERNATE READING MARK NHAY +1B000..1B001 ; Obsolete # 6.0 [2] KATAKANA LETTER ARCHAIC E..HIRAGANA LETTER ARCHAIC YE +1B002..1B11E ; Obsolete # 10.0 [285] HENTAIGANA LETTER A-1..HENTAIGANA LETTER N-MU-MO-2 +1B11F..1B122 ; Obsolete # 14.0 [4] HIRAGANA LETTER ARCHAIC WU..KATAKANA LETTER ARCHAIC WU +1B132 ; Obsolete # 15.0 HIRAGANA LETTER SMALL KO +1B150..1B152 ; Obsolete # 12.0 [3] HIRAGANA LETTER SMALL WI..HIRAGANA LETTER SMALL WO +1B155 ; Obsolete # 15.0 KATAKANA LETTER SMALL KO +1B164..1B167 ; Obsolete # 12.0 [4] KATAKANA LETTER SMALL WI..KATAKANA LETTER SMALL N +1E08F ; Obsolete # 15.0 COMBINING CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I + +# Total code points: 1639 + +# Identifier_Type: Obsolete Not_XID + +0482 ; Obsolete Not_XID # 1.1 CYRILLIC THOUSANDS SIGN +0488..0489 ; Obsolete Not_XID # 3.0 [2] COMBINING CYRILLIC HUNDRED THOUSANDS SIGN..COMBINING CYRILLIC MILLIONS SIGN +05C6 ; Obsolete Not_XID # 4.1 HEBREW PUNCTUATION NUN HAFUKHA +17D8 ; Obsolete Not_XID # 3.0 KHMER SIGN BEYYAL +1CD3 ; Obsolete Not_XID # 5.2 VEDIC SIGN NIHSHVASA +2056 ; Obsolete Not_XID # 4.1 THREE DOT PUNCTUATION +2058..205E ; Obsolete Not_XID # 4.1 [7] FOUR DOT PUNCTUATION..VERTICAL FOUR DOTS +2127 ; Obsolete Not_XID # 1.1 INVERTED OHM SIGN +214F ; Obsolete Not_XID # 5.1 SYMBOL FOR SAMARITAN SOURCE +2E0E..2E16 ; Obsolete Not_XID # 4.1 [9] EDITORIAL CORONIS..DOTTED RIGHT-POINTING ANGLE +2E2A..2E2F ; Obsolete Not_XID # 5.1 [6] TWO DOTS OVER ONE DOT PUNCTUATION..VERTICAL TILDE +2E31 ; Obsolete Not_XID # 5.2 WORD SEPARATOR MIDDLE DOT +2E32 ; Obsolete Not_XID # 6.1 TURNED COMMA +2E35 ; Obsolete Not_XID # 6.1 TURNED SEMICOLON +2E39 ; Obsolete Not_XID # 6.1 TOP HALF SECTION SIGN +301E ; Obsolete Not_XID # 1.1 DOUBLE PRIME QUOTATION MARK +A670..A673 ; Obsolete Not_XID # 5.1 [4] COMBINING CYRILLIC TEN MILLIONS SIGN..SLAVONIC ASTERISK +A700..A707 ; Obsolete Not_XID # 4.1 [8] MODIFIER LETTER CHINESE TONE YIN PING..MODIFIER LETTER CHINESE TONE YANG RU +A8F8..A8FA ; Obsolete Not_XID # 5.2 [3] DEVANAGARI SIGN PUSHPIKA..DEVANAGARI CARET +101D0..101FC ; Obsolete Not_XID # 5.1 [45] PHAISTOS DISC SIGN PEDESTRIAN..PHAISTOS DISC SIGN WAVY BAND +102E1..102FB ; Obsolete Not_XID # 7.0 [27] COPTIC EPACT DIGIT ONE..COPTIC EPACT NUMBER NINE HUNDRED +1D200..1D241 ; Obsolete Not_XID # 4.1 [66] GREEK VOCAL NOTATION SYMBOL-1..GREEK INSTRUMENTAL NOTATION SYMBOL-54 +1D245 ; Obsolete Not_XID # 4.1 GREEK MUSICAL LEIMMA + +# Total code points: 190 + +# Identifier_Type: Not_XID + +0009..000D ; Not_XID # 1.1 [5] .. +0020..0026 ; Not_XID # 1.1 [7] SPACE..AMPERSAND +0028..002C ; Not_XID # 1.1 [5] LEFT PARENTHESIS..COMMA +002F ; Not_XID # 1.1 SOLIDUS +003B..0040 ; Not_XID # 1.1 [6] SEMICOLON..COMMERCIAL AT +005B..005E ; Not_XID # 1.1 [4] LEFT SQUARE BRACKET..CIRCUMFLEX ACCENT +0060 ; Not_XID # 1.1 GRAVE ACCENT +007B..007E ; Not_XID # 1.1 [4] LEFT CURLY BRACKET..TILDE +0085 ; Not_XID # 1.1 +00A1..00A7 ; Not_XID # 1.1 [7] INVERTED EXCLAMATION MARK..SECTION SIGN +00A9 ; Not_XID # 1.1 COPYRIGHT SIGN +00AB..00AC ; Not_XID # 1.1 [2] LEFT-POINTING DOUBLE ANGLE QUOTATION MARK..NOT SIGN +00AE ; Not_XID # 1.1 REGISTERED SIGN +00B0..00B1 ; Not_XID # 1.1 [2] DEGREE SIGN..PLUS-MINUS SIGN +00B6 ; Not_XID # 1.1 PILCROW SIGN +00BB ; Not_XID # 1.1 RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK +00BF ; Not_XID # 1.1 INVERTED QUESTION MARK +00D7 ; Not_XID # 1.1 MULTIPLICATION SIGN +00F7 ; Not_XID # 1.1 DIVISION SIGN +02C2..02C5 ; Not_XID # 1.1 [4] MODIFIER LETTER LEFT ARROWHEAD..MODIFIER LETTER DOWN ARROWHEAD +02D2..02D7 ; Not_XID # 1.1 [6] MODIFIER LETTER CENTRED RIGHT HALF RING..MODIFIER LETTER MINUS SIGN +02DE ; Not_XID # 1.1 MODIFIER LETTER RHOTIC HOOK +02DF ; Not_XID # 3.0 MODIFIER LETTER CROSS ACCENT +02E5..02E9 ; Not_XID # 1.1 [5] MODIFIER LETTER EXTRA-HIGH TONE BAR..MODIFIER LETTER EXTRA-LOW TONE BAR +02ED ; Not_XID # 3.0 MODIFIER LETTER UNASPIRATED +02EF..02FF ; Not_XID # 4.0 [17] MODIFIER LETTER LOW DOWN ARROWHEAD..MODIFIER LETTER LOW LEFT ARROW +03F6 ; Not_XID # 3.2 GREEK REVERSED LUNATE EPSILON SYMBOL +055A..055F ; Not_XID # 1.1 [6] ARMENIAN APOSTROPHE..ARMENIAN ABBREVIATION MARK +0589 ; Not_XID # 1.1 ARMENIAN FULL STOP +058D..058E ; Not_XID # 7.0 [2] RIGHT-FACING ARMENIAN ETERNITY SIGN..LEFT-FACING ARMENIAN ETERNITY SIGN +058F ; Not_XID # 6.1 ARMENIAN DRAM SIGN +05BE ; Not_XID # 1.1 HEBREW PUNCTUATION MAQAF +05C0 ; Not_XID # 1.1 HEBREW PUNCTUATION PASEQ +05C3 ; Not_XID # 1.1 HEBREW PUNCTUATION SOF PASUQ +0600..0603 ; Not_XID # 4.0 [4] ARABIC NUMBER SIGN..ARABIC SIGN SAFHA +0604 ; Not_XID # 6.1 ARABIC SIGN SAMVAT +0605 ; Not_XID # 7.0 ARABIC NUMBER MARK ABOVE +0606..060A ; Not_XID # 5.1 [5] ARABIC-INDIC CUBE ROOT..ARABIC-INDIC PER TEN THOUSAND SIGN +060B ; Not_XID # 4.1 AFGHANI SIGN +060C ; Not_XID # 1.1 ARABIC COMMA +060D..060F ; Not_XID # 4.0 [3] ARABIC DATE SEPARATOR..ARABIC SIGN MISRA +061B ; Not_XID # 1.1 ARABIC SEMICOLON +061D ; Not_XID # 14.0 ARABIC END OF TEXT MARK +061E ; Not_XID # 4.1 ARABIC TRIPLE DOT PUNCTUATION MARK +061F ; Not_XID # 1.1 ARABIC QUESTION MARK +066A..066D ; Not_XID # 1.1 [4] ARABIC PERCENT SIGN..ARABIC FIVE POINTED STAR +06D4 ; Not_XID # 1.1 ARABIC FULL STOP +06DD ; Not_XID # 1.1 ARABIC END OF AYAH +06DE ; Not_XID # 1.1 ARABIC START OF RUB EL HIZB +06E9 ; Not_XID # 1.1 ARABIC PLACE OF SAJDAH +0890..0891 ; Not_XID # 14.0 [2] ARABIC POUND MARK ABOVE..ARABIC PIASTRE MARK ABOVE +08E2 ; Not_XID # 9.0 ARABIC DISPUTED END OF AYAH +0964..0965 ; Not_XID # 1.1 [2] DEVANAGARI DANDA..DEVANAGARI DOUBLE DANDA +0970 ; Not_XID # 1.1 DEVANAGARI ABBREVIATION SIGN +09F2..09FA ; Not_XID # 1.1 [9] BENGALI RUPEE MARK..BENGALI ISSHAR +09FB ; Not_XID # 5.2 BENGALI GANDA MARK +09FD ; Not_XID # 10.0 BENGALI ABBREVIATION SIGN +0A76 ; Not_XID # 11.0 GURMUKHI ABBREVIATION SIGN +0AF0 ; Not_XID # 6.1 GUJARATI ABBREVIATION SIGN +0AF1 ; Not_XID # 4.0 GUJARATI RUPEE SIGN +0B70 ; Not_XID # 1.1 ORIYA ISSHAR +0B72..0B77 ; Not_XID # 6.0 [6] ORIYA FRACTION ONE QUARTER..ORIYA FRACTION THREE SIXTEENTHS +0BF0..0BF2 ; Not_XID # 1.1 [3] TAMIL NUMBER TEN..TAMIL NUMBER ONE THOUSAND +0BF3..0BFA ; Not_XID # 4.0 [8] TAMIL DAY SIGN..TAMIL NUMBER SIGN +0C77 ; Not_XID # 12.0 TELUGU SIGN SIDDHAM +0C78..0C7F ; Not_XID # 5.1 [8] TELUGU FRACTION DIGIT ZERO FOR ODD POWERS OF FOUR..TELUGU SIGN TUUMU +0C84 ; Not_XID # 11.0 KANNADA SIGN SIDDHAM +0D4F ; Not_XID # 9.0 MALAYALAM SIGN PARA +0D58..0D5E ; Not_XID # 9.0 [7] MALAYALAM FRACTION ONE ONE-HUNDRED-AND-SIXTIETH..MALAYALAM FRACTION ONE FIFTH +0D70..0D75 ; Not_XID # 5.1 [6] MALAYALAM NUMBER TEN..MALAYALAM FRACTION THREE QUARTERS +0D76..0D78 ; Not_XID # 9.0 [3] MALAYALAM FRACTION ONE SIXTEENTH..MALAYALAM FRACTION THREE SIXTEENTHS +0D79 ; Not_XID # 5.1 MALAYALAM DATE MARK +0DF4 ; Not_XID # 3.0 SINHALA PUNCTUATION KUNDDALIYA +0E3F ; Not_XID # 1.1 THAI CURRENCY SYMBOL BAHT +0E4F ; Not_XID # 1.1 THAI CHARACTER FONGMAN +0E5A..0E5B ; Not_XID # 1.1 [2] THAI CHARACTER ANGKHANKHU..THAI CHARACTER KHOMUT +0F01..0F0A ; Not_XID # 2.0 [10] TIBETAN MARK GTER YIG MGO TRUNCATED A..TIBETAN MARK BKA- SHOG YIG MGO +0F0D..0F17 ; Not_XID # 2.0 [11] TIBETAN MARK SHAD..TIBETAN ASTROLOGICAL SIGN SGRA GCAN -CHAR RTAGS +0F1A..0F1F ; Not_XID # 2.0 [6] TIBETAN SIGN RDEL DKAR GCIG..TIBETAN SIGN RDEL DKAR RDEL NAG +0F2A..0F34 ; Not_XID # 2.0 [11] TIBETAN DIGIT HALF ONE..TIBETAN MARK BSDUS RTAGS +0F36 ; Not_XID # 2.0 TIBETAN MARK CARET -DZUD RTAGS BZHI MIG CAN +0F38 ; Not_XID # 2.0 TIBETAN MARK CHE MGO +0F3A..0F3D ; Not_XID # 2.0 [4] TIBETAN MARK GUG RTAGS GYON..TIBETAN MARK ANG KHANG GYAS +0F85 ; Not_XID # 2.0 TIBETAN MARK PALUTA +0FBE..0FC5 ; Not_XID # 3.0 [8] TIBETAN KU RU KHA..TIBETAN SYMBOL RDO RJE +0FC7..0FCC ; Not_XID # 3.0 [6] TIBETAN SYMBOL RDO RJE RGYA GRAM..TIBETAN SYMBOL NOR BU BZHI -KHYIL +0FCE ; Not_XID # 5.1 TIBETAN SIGN RDEL NAG RDEL DKAR +0FCF ; Not_XID # 3.0 TIBETAN SIGN RDEL NAG GSUM +0FD0..0FD1 ; Not_XID # 4.1 [2] TIBETAN MARK BSKA- SHOG GI MGO RGYAN..TIBETAN MARK MNYAM YIG GI MGO RGYAN +0FD2..0FD4 ; Not_XID # 5.1 [3] TIBETAN MARK NYIS TSHEG..TIBETAN MARK CLOSING BRDA RNYING YIG MGO SGAB MA +0FD5..0FD8 ; Not_XID # 5.2 [4] RIGHT-FACING SVASTI SIGN..LEFT-FACING SVASTI SIGN WITH DOTS +0FD9..0FDA ; Not_XID # 6.0 [2] TIBETAN MARK LEADING MCHAN RTAGS..TIBETAN MARK TRAILING MCHAN RTAGS +104A..104F ; Not_XID # 3.0 [6] MYANMAR SIGN LITTLE SECTION..MYANMAR SYMBOL GENITIVE +109E..109F ; Not_XID # 5.1 [2] MYANMAR SYMBOL SHAN ONE..MYANMAR SYMBOL SHAN EXCLAMATION +10FB ; Not_XID # 1.1 GEORGIAN PARAGRAPH SEPARATOR +1360 ; Not_XID # 4.1 ETHIOPIC SECTION MARK +1361..1368 ; Not_XID # 3.0 [8] ETHIOPIC WORDSPACE..ETHIOPIC PARAGRAPH SEPARATOR +1372..137C ; Not_XID # 3.0 [11] ETHIOPIC NUMBER TEN..ETHIOPIC NUMBER TEN THOUSAND +1390..1399 ; Not_XID # 4.1 [10] ETHIOPIC TONAL MARK YIZET..ETHIOPIC TONAL MARK KURT +17D4..17D6 ; Not_XID # 3.0 [3] KHMER SIGN KHAN..KHMER SIGN CAMNUC PII KUUH +17D9..17DB ; Not_XID # 3.0 [3] KHMER SIGN PHNAEK MUAN..KHMER CURRENCY SYMBOL RIEL +17F0..17F9 ; Not_XID # 4.0 [10] KHMER SYMBOL LEK ATTAK SON..KHMER SYMBOL LEK ATTAK PRAM-BUON +19E0..19FF ; Not_XID # 4.0 [32] KHMER SYMBOL PATHAMASAT..KHMER SYMBOL DAP-PRAM ROC +1ABE ; Not_XID # 7.0 COMBINING PARENTHESES OVERLAY +2012..2016 ; Not_XID # 1.1 [5] FIGURE DASH..DOUBLE VERTICAL LINE +2018 ; Not_XID # 1.1 LEFT SINGLE QUOTATION MARK +201A..2023 ; Not_XID # 1.1 [10] SINGLE LOW-9 QUOTATION MARK..TRIANGULAR BULLET +2028..2029 ; Not_XID # 1.1 [2] LINE SEPARATOR..PARAGRAPH SEPARATOR +2030..2032 ; Not_XID # 1.1 [3] PER MILLE SIGN..PRIME +2035 ; Not_XID # 1.1 REVERSED PRIME +2038..203B ; Not_XID # 1.1 [4] CARET..REFERENCE MARK +203D ; Not_XID # 1.1 INTERROBANG +2041..2046 ; Not_XID # 1.1 [6] CARET INSERTION POINT..RIGHT SQUARE BRACKET WITH QUILL +204A..204D ; Not_XID # 3.0 [4] TIRONIAN SIGN ET..BLACK RIGHTWARDS BULLET +204E..2052 ; Not_XID # 3.2 [5] LOW ASTERISK..COMMERCIAL MINUS SIGN +2053 ; Not_XID # 4.0 SWUNG DASH +2055 ; Not_XID # 4.1 FLOWER PUNCTUATION MARK +20A0..20A7 ; Not_XID # 1.1 [8] EURO-CURRENCY SIGN..PESETA SIGN +20A9..20AA ; Not_XID # 1.1 [2] WON SIGN..NEW SHEQEL SIGN +20AB ; Not_XID # 2.0 DONG SIGN +20AC ; Not_XID # 2.1 EURO SIGN +20AD..20AF ; Not_XID # 3.0 [3] KIP SIGN..DRACHMA SIGN +20B0..20B1 ; Not_XID # 3.2 [2] GERMAN PENNY SIGN..PESO SIGN +20B2..20B5 ; Not_XID # 4.1 [4] GUARANI SIGN..CEDI SIGN +20B6..20B8 ; Not_XID # 5.2 [3] LIVRE TOURNOIS SIGN..TENGE SIGN +20B9 ; Not_XID # 6.0 INDIAN RUPEE SIGN +20BA ; Not_XID # 6.2 TURKISH LIRA SIGN +20BB..20BD ; Not_XID # 7.0 [3] NORDIC MARK SIGN..RUBLE SIGN +20BE ; Not_XID # 8.0 LARI SIGN +20BF ; Not_XID # 10.0 BITCOIN SIGN +20C0 ; Not_XID # 14.0 SOM SIGN +20C1 ; Not_XID # 17.0 SAUDI RIYAL SIGN +2104 ; Not_XID # 1.1 CENTRE LINE SYMBOL +2108 ; Not_XID # 1.1 SCRUPLE +2114 ; Not_XID # 1.1 L B BAR SYMBOL +2117 ; Not_XID # 1.1 SOUND RECORDING COPYRIGHT +211E..211F ; Not_XID # 1.1 [2] PRESCRIPTION TAKE..RESPONSE +2123 ; Not_XID # 1.1 VERSICLE +2125 ; Not_XID # 1.1 OUNCE SIGN +2129 ; Not_XID # 1.1 TURNED GREEK SMALL LETTER IOTA +213A ; Not_XID # 3.0 ROTATED CAPITAL Q +2141..2144 ; Not_XID # 3.2 [4] TURNED SANS-SERIF CAPITAL G..TURNED SANS-SERIF CAPITAL Y +214A..214B ; Not_XID # 3.2 [2] PROPERTY LINE..TURNED AMPERSAND +214C ; Not_XID # 4.1 PER SIGN +214D ; Not_XID # 5.0 AKTIESELSKAB +2190..21EA ; Not_XID # 1.1 [91] LEFTWARDS ARROW..UPWARDS WHITE ARROW FROM BAR +21EB..21F3 ; Not_XID # 3.0 [9] UPWARDS WHITE ARROW ON PEDESTAL..UP DOWN WHITE ARROW +21F4..21FF ; Not_XID # 3.2 [12] RIGHT ARROW WITH SMALL CIRCLE..LEFT RIGHT OPEN-HEADED ARROW +2200..222B ; Not_XID # 1.1 [44] FOR ALL..INTEGRAL +222E ; Not_XID # 1.1 CONTOUR INTEGRAL +2231..22F1 ; Not_XID # 1.1 [193] CLOCKWISE INTEGRAL..DOWN RIGHT DIAGONAL ELLIPSIS +22F2..22FF ; Not_XID # 3.2 [14] ELEMENT OF WITH LONG HORIZONTAL STROKE..Z NOTATION BAG MEMBERSHIP +2300 ; Not_XID # 1.1 DIAMETER SIGN +2301 ; Not_XID # 3.0 ELECTRIC ARROW +2302..2328 ; Not_XID # 1.1 [39] HOUSE..KEYBOARD +232B..237A ; Not_XID # 1.1 [80] ERASE TO THE LEFT..APL FUNCTIONAL SYMBOL ALPHA +237B ; Not_XID # 3.0 NOT CHECK MARK +237C ; Not_XID # 3.2 RIGHT ANGLE WITH DOWNWARDS ZIGZAG ARROW +237D..239A ; Not_XID # 3.0 [30] SHOULDERED OPEN BOX..CLEAR SCREEN SYMBOL +239B..23CE ; Not_XID # 3.2 [52] LEFT PARENTHESIS UPPER HOOK..RETURN SYMBOL +23CF..23D0 ; Not_XID # 4.0 [2] EJECT SYMBOL..VERTICAL LINE EXTENSION +23D1..23DB ; Not_XID # 4.1 [11] METRICAL BREVE..FUSE +23DC..23E7 ; Not_XID # 5.0 [12] TOP PARENTHESIS..ELECTRICAL INTERSECTION +23E8 ; Not_XID # 5.2 DECIMAL EXPONENT SYMBOL +23E9..23F3 ; Not_XID # 6.0 [11] BLACK RIGHT-POINTING DOUBLE TRIANGLE..HOURGLASS WITH FLOWING SAND +23F4..23FA ; Not_XID # 7.0 [7] BLACK MEDIUM LEFT-POINTING TRIANGLE..BLACK CIRCLE FOR RECORD +23FB..23FE ; Not_XID # 9.0 [4] POWER SYMBOL..POWER SLEEP SYMBOL +23FF ; Not_XID # 10.0 OBSERVER EYE SYMBOL +2400..2424 ; Not_XID # 1.1 [37] SYMBOL FOR NULL..SYMBOL FOR NEWLINE +2425..2426 ; Not_XID # 3.0 [2] SYMBOL FOR DELETE FORM TWO..SYMBOL FOR SUBSTITUTE FORM TWO +2427..2429 ; Not_XID # 16.0 [3] SYMBOL FOR DELETE SQUARE CHECKER BOARD FORM..SYMBOL FOR DELETE MEDIUM SHADE FORM +2440..244A ; Not_XID # 1.1 [11] OCR HOOK..OCR DOUBLE BACKSLASH +2500..2595 ; Not_XID # 1.1 [150] BOX DRAWINGS LIGHT HORIZONTAL..RIGHT ONE EIGHTH BLOCK +2596..259F ; Not_XID # 3.2 [10] QUADRANT LOWER LEFT..QUADRANT UPPER RIGHT AND LOWER LEFT AND LOWER RIGHT +25A0..25EF ; Not_XID # 1.1 [80] BLACK SQUARE..LARGE CIRCLE +25F0..25F7 ; Not_XID # 3.0 [8] WHITE SQUARE WITH UPPER LEFT QUADRANT..WHITE CIRCLE WITH UPPER RIGHT QUADRANT +25F8..25FF ; Not_XID # 3.2 [8] UPPER LEFT TRIANGLE..LOWER RIGHT TRIANGLE +2600..2613 ; Not_XID # 1.1 [20] BLACK SUN WITH RAYS..SALTIRE +2614..2615 ; Not_XID # 4.0 [2] UMBRELLA WITH RAIN DROPS..HOT BEVERAGE +2616..2617 ; Not_XID # 3.2 [2] WHITE SHOGI PIECE..BLACK SHOGI PIECE +2618 ; Not_XID # 4.1 SHAMROCK +2619 ; Not_XID # 3.0 REVERSED ROTATED FLORAL HEART BULLET +261A..266F ; Not_XID # 1.1 [86] BLACK LEFT POINTING INDEX..MUSIC SHARP SIGN +2670..2671 ; Not_XID # 3.0 [2] WEST SYRIAC CROSS..EAST SYRIAC CROSS +2672..267D ; Not_XID # 3.2 [12] UNIVERSAL RECYCLING SYMBOL..PARTIALLY-RECYCLED PAPER SYMBOL +267E..267F ; Not_XID # 4.1 [2] PERMANENT PAPER SIGN..WHEELCHAIR SYMBOL +2680..2689 ; Not_XID # 3.2 [10] DIE FACE-1..BLACK CIRCLE WITH TWO WHITE DOTS +268A..2691 ; Not_XID # 4.0 [8] MONOGRAM FOR YANG..BLACK FLAG +2692..269C ; Not_XID # 4.1 [11] HAMMER AND PICK..FLEUR-DE-LIS +269D ; Not_XID # 5.1 OUTLINED WHITE STAR +269E..269F ; Not_XID # 5.2 [2] THREE LINES CONVERGING RIGHT..THREE LINES CONVERGING LEFT +26A0..26A1 ; Not_XID # 4.0 [2] WARNING SIGN..HIGH VOLTAGE SIGN +26A2..26B1 ; Not_XID # 4.1 [16] DOUBLED FEMALE SIGN..FUNERAL URN +26B2 ; Not_XID # 5.0 NEUTER +26B3..26BC ; Not_XID # 5.1 [10] CERES..SESQUIQUADRATE +26BD..26BF ; Not_XID # 5.2 [3] SOCCER BALL..SQUARED KEY +26C0..26C3 ; Not_XID # 5.1 [4] WHITE DRAUGHTS MAN..BLACK DRAUGHTS KING +26C4..26CD ; Not_XID # 5.2 [10] SNOWMAN WITHOUT SNOW..DISABLED CAR +26CE ; Not_XID # 6.0 OPHIUCHUS +26CF..26E1 ; Not_XID # 5.2 [19] PICK..RESTRICTED LEFT ENTRY-2 +26E2 ; Not_XID # 6.0 ASTRONOMICAL SYMBOL FOR URANUS +26E3 ; Not_XID # 5.2 HEAVY CIRCLE WITH STROKE AND TWO DOTS ABOVE +26E4..26E7 ; Not_XID # 6.0 [4] PENTAGRAM..INVERTED PENTAGRAM +26E8..26FF ; Not_XID # 5.2 [24] BLACK CROSS ON SHIELD..WHITE FLAG WITH HORIZONTAL MIDDLE BLACK STRIPE +2700 ; Not_XID # 7.0 BLACK SAFETY SCISSORS +2701..2704 ; Not_XID # 1.1 [4] UPPER BLADE SCISSORS..WHITE SCISSORS +2705 ; Not_XID # 6.0 WHITE HEAVY CHECK MARK +2706..2709 ; Not_XID # 1.1 [4] TELEPHONE LOCATION SIGN..ENVELOPE +270A..270B ; Not_XID # 6.0 [2] RAISED FIST..RAISED HAND +270C..2727 ; Not_XID # 1.1 [28] VICTORY HAND..WHITE FOUR POINTED STAR +2728 ; Not_XID # 6.0 SPARKLES +2729..274B ; Not_XID # 1.1 [35] STRESS OUTLINED WHITE STAR..HEAVY EIGHT TEARDROP-SPOKED PROPELLER ASTERISK +274C ; Not_XID # 6.0 CROSS MARK +274D ; Not_XID # 1.1 SHADOWED WHITE CIRCLE +274E ; Not_XID # 6.0 NEGATIVE SQUARED CROSS MARK +274F..2752 ; Not_XID # 1.1 [4] LOWER RIGHT DROP-SHADOWED WHITE SQUARE..UPPER RIGHT SHADOWED WHITE SQUARE +2753..2755 ; Not_XID # 6.0 [3] BLACK QUESTION MARK ORNAMENT..WHITE EXCLAMATION MARK ORNAMENT +2756 ; Not_XID # 1.1 BLACK DIAMOND MINUS WHITE X +2757 ; Not_XID # 5.2 HEAVY EXCLAMATION MARK SYMBOL +2758..275E ; Not_XID # 1.1 [7] LIGHT VERTICAL BAR..HEAVY DOUBLE COMMA QUOTATION MARK ORNAMENT +275F..2760 ; Not_XID # 6.0 [2] HEAVY LOW SINGLE COMMA QUOTATION MARK ORNAMENT..HEAVY LOW DOUBLE COMMA QUOTATION MARK ORNAMENT +2761..2767 ; Not_XID # 1.1 [7] CURVED STEM PARAGRAPH SIGN ORNAMENT..ROTATED FLORAL HEART BULLET +2768..2775 ; Not_XID # 3.2 [14] MEDIUM LEFT PARENTHESIS ORNAMENT..MEDIUM RIGHT CURLY BRACKET ORNAMENT +2776..2794 ; Not_XID # 1.1 [31] DINGBAT NEGATIVE CIRCLED DIGIT ONE..HEAVY WIDE-HEADED RIGHTWARDS ARROW +2795..2797 ; Not_XID # 6.0 [3] HEAVY PLUS SIGN..HEAVY DIVISION SIGN +2798..27AF ; Not_XID # 1.1 [24] HEAVY SOUTH EAST ARROW..NOTCHED LOWER RIGHT-SHADOWED WHITE RIGHTWARDS ARROW +27B0 ; Not_XID # 6.0 CURLY LOOP +27B1..27BE ; Not_XID # 1.1 [14] NOTCHED UPPER RIGHT-SHADOWED WHITE RIGHTWARDS ARROW..OPEN-OUTLINED RIGHTWARDS ARROW +27BF ; Not_XID # 6.0 DOUBLE CURLY LOOP +27C0..27C6 ; Not_XID # 4.1 [7] THREE DIMENSIONAL ANGLE..RIGHT S-SHAPED BAG DELIMITER +27C7..27CA ; Not_XID # 5.0 [4] OR WITH DOT INSIDE..VERTICAL BAR WITH HORIZONTAL STROKE +27CB ; Not_XID # 6.1 MATHEMATICAL RISING DIAGONAL +27CC ; Not_XID # 5.1 LONG DIVISION +27CD ; Not_XID # 6.1 MATHEMATICAL FALLING DIAGONAL +27CE..27CF ; Not_XID # 6.0 [2] SQUARED LOGICAL AND..SQUARED LOGICAL OR +27D0..27EB ; Not_XID # 3.2 [28] WHITE DIAMOND WITH CENTRED DOT..MATHEMATICAL RIGHT DOUBLE ANGLE BRACKET +27EC..27EF ; Not_XID # 5.1 [4] MATHEMATICAL LEFT WHITE TORTOISE SHELL BRACKET..MATHEMATICAL RIGHT FLATTENED PARENTHESIS +27F0..27FF ; Not_XID # 3.2 [16] UPWARDS QUADRUPLE ARROW..LONG RIGHTWARDS SQUIGGLE ARROW +2900..2A0B ; Not_XID # 3.2 [268] RIGHTWARDS TWO-HEADED ARROW WITH VERTICAL STROKE..SUMMATION WITH INTEGRAL +2A0D..2A73 ; Not_XID # 3.2 [103] FINITE PART INTEGRAL..EQUALS SIGN ABOVE TILDE OPERATOR +2A77..2ADB ; Not_XID # 3.2 [101] EQUALS SIGN WITH TWO DOTS ABOVE AND TWO DOTS BELOW..TRANSVERSAL INTERSECTION +2ADD..2AFF ; Not_XID # 3.2 [35] NONFORKING..N-ARY WHITE VERTICAL BAR +2B00..2B0D ; Not_XID # 4.0 [14] NORTH EAST WHITE ARROW..UP DOWN BLACK ARROW +2B0E..2B13 ; Not_XID # 4.1 [6] RIGHTWARDS ARROW WITH TIP DOWNWARDS..SQUARE WITH BOTTOM HALF BLACK +2B14..2B1A ; Not_XID # 5.0 [7] SQUARE WITH UPPER RIGHT DIAGONAL HALF BLACK..DOTTED SQUARE +2B1B..2B1F ; Not_XID # 5.1 [5] BLACK LARGE SQUARE..BLACK PENTAGON +2B20..2B23 ; Not_XID # 5.0 [4] WHITE PENTAGON..HORIZONTAL BLACK HEXAGON +2B24..2B4C ; Not_XID # 5.1 [41] BLACK LARGE CIRCLE..RIGHTWARDS ARROW ABOVE REVERSE TILDE OPERATOR +2B4D..2B4F ; Not_XID # 7.0 [3] DOWNWARDS TRIANGLE-HEADED ZIGZAG ARROW..SHORT BACKSLANTED SOUTH ARROW +2B50..2B54 ; Not_XID # 5.1 [5] WHITE MEDIUM STAR..WHITE RIGHT-POINTING PENTAGON +2B55..2B59 ; Not_XID # 5.2 [5] HEAVY LARGE CIRCLE..HEAVY CIRCLED SALTIRE +2B5A..2B73 ; Not_XID # 7.0 [26] SLANTED NORTH ARROW WITH HOOKED HEAD..DOWNWARDS TRIANGLE-HEADED ARROW TO BAR +2B76..2B95 ; Not_XID # 7.0 [32] NORTH WEST TRIANGLE-HEADED ARROW TO BAR..RIGHTWARDS BLACK ARROW +2B96 ; Not_XID # 17.0 EQUALS SIGN WITH INFINITY ABOVE +2B97 ; Not_XID # 13.0 SYMBOL FOR TYPE A ELECTRONICS +2B98..2BB9 ; Not_XID # 7.0 [34] THREE-D TOP-LIGHTED LEFTWARDS EQUILATERAL ARROWHEAD..UP ARROWHEAD IN A RECTANGLE BOX +2BBA..2BBC ; Not_XID # 11.0 [3] OVERLAPPING WHITE SQUARES..OVERLAPPING BLACK SQUARES +2BBD..2BC8 ; Not_XID # 7.0 [12] BALLOT BOX WITH LIGHT X..BLACK MEDIUM RIGHT-POINTING TRIANGLE CENTRED +2BC9 ; Not_XID # 12.0 NEPTUNE FORM TWO +2BCA..2BD1 ; Not_XID # 7.0 [8] TOP HALF BLACK CIRCLE..UNCERTAINTY SIGN +2BD2 ; Not_XID # 10.0 GROUP MARK +2BD3..2BEB ; Not_XID # 11.0 [25] PLUTO FORM TWO..STAR WITH RIGHT HALF BLACK +2BF0..2BFE ; Not_XID # 11.0 [15] ERIS FORM ONE..REVERSED RIGHT ANGLE +2BFF ; Not_XID # 12.0 HELLSCHREIBER PAUSE SYMBOL +2E17 ; Not_XID # 4.1 DOUBLE OBLIQUE HYPHEN +2E18..2E1B ; Not_XID # 5.1 [4] INVERTED INTERROBANG..TILDE WITH RING ABOVE +2E1C..2E1D ; Not_XID # 4.1 [2] LEFT LOW PARAPHRASE BRACKET..RIGHT LOW PARAPHRASE BRACKET +2E1E..2E29 ; Not_XID # 5.1 [12] TILDE WITH DOT ABOVE..RIGHT DOUBLE PARENTHESIS +2E33..2E34 ; Not_XID # 6.1 [2] RAISED DOT..RAISED COMMA +2E36..2E38 ; Not_XID # 6.1 [3] DAGGER WITH LEFT GUARD..TURNED DAGGER +2E3A..2E3B ; Not_XID # 6.1 [2] TWO-EM DASH..THREE-EM DASH +2E3D..2E42 ; Not_XID # 7.0 [6] VERTICAL SIX DOTS..DOUBLE LOW-REVERSED-9 QUOTATION MARK +2E43..2E44 ; Not_XID # 9.0 [2] DASH WITH LEFT UPTURN..DOUBLE SUSPENSION MARK +2E45..2E49 ; Not_XID # 10.0 [5] INVERTED LOW KAVYKA..DOUBLE STACKED COMMA +2E4A..2E4E ; Not_XID # 11.0 [5] DOTTED SOLIDUS..PUNCTUS ELEVATUS MARK +2E4F ; Not_XID # 12.0 CORNISH VERSE DIVIDER +2E50..2E52 ; Not_XID # 13.0 [3] CROSS PATTY WITH RIGHT CROSSBAR..TIRONIAN SIGN CAPITAL ET +2E53..2E5D ; Not_XID # 14.0 [11] MEDIEVAL EXCLAMATION MARK..OBLIQUE HYPHEN +2E80..2E99 ; Not_XID # 3.0 [26] CJK RADICAL REPEAT..CJK RADICAL RAP +2E9B..2E9E ; Not_XID # 3.0 [4] CJK RADICAL CHOKE..CJK RADICAL DEATH +2EA0..2EF2 ; Not_XID # 3.0 [83] CJK RADICAL CIVILIAN..CJK RADICAL J-SIMPLIFIED TURTLE +2FF0..2FFB ; Not_XID # 3.0 [12] IDEOGRAPHIC DESCRIPTION CHARACTER LEFT TO RIGHT..IDEOGRAPHIC DESCRIPTION CHARACTER OVERLAID +2FFC..2FFF ; Not_XID # 15.1 [4] IDEOGRAPHIC DESCRIPTION CHARACTER SURROUND FROM RIGHT..IDEOGRAPHIC DESCRIPTION CHARACTER ROTATION +3001..3004 ; Not_XID # 1.1 [4] IDEOGRAPHIC COMMA..JAPANESE INDUSTRIAL STANDARD SYMBOL +3008..301D ; Not_XID # 1.1 [22] LEFT ANGLE BRACKET..REVERSED DOUBLE PRIME QUOTATION MARK +301F..3020 ; Not_XID # 1.1 [2] LOW DOUBLE PRIME QUOTATION MARK..POSTAL MARK FACE +3030 ; Not_XID # 1.1 WAVY DASH +3037 ; Not_XID # 1.1 IDEOGRAPHIC TELEGRAPH LINE FEED SEPARATOR SYMBOL +303D ; Not_XID # 3.2 PART ALTERNATION MARK +303E ; Not_XID # 3.0 IDEOGRAPHIC VARIATION INDICATOR +303F ; Not_XID # 1.1 IDEOGRAPHIC HALF FILL SPACE +3190..3191 ; Not_XID # 1.1 [2] IDEOGRAPHIC ANNOTATION LINKING MARK..IDEOGRAPHIC ANNOTATION REVERSE MARK +31C0..31CF ; Not_XID # 4.1 [16] CJK STROKE T..CJK STROKE N +31D0..31E3 ; Not_XID # 5.1 [20] CJK STROKE H..CJK STROKE Q +31E4..31E5 ; Not_XID # 16.0 [2] CJK STROKE HXG..CJK STROKE SZP +31EF ; Not_XID # 15.1 IDEOGRAPHIC DESCRIPTION CHARACTER SUBTRACTION +3248..324F ; Not_XID # 5.2 [8] CIRCLED NUMBER TEN ON BLACK SQUARE..CIRCLED NUMBER EIGHTY ON BLACK SQUARE +A67E ; Not_XID # 5.1 CYRILLIC KAVYKA +A720..A721 ; Not_XID # 5.0 [2] MODIFIER LETTER STRESS AND HIGH TONE..MODIFIER LETTER STRESS AND LOW TONE +A789..A78A ; Not_XID # 5.1 [2] MODIFIER LETTER COLON..MODIFIER LETTER SHORT EQUALS SIGN +A830..A839 ; Not_XID # 5.2 [10] NORTH INDIC FRACTION ONE QUARTER..NORTH INDIC QUANTITY MARK +A92E ; Not_XID # 5.1 KAYAH LI SIGN CWI +AA77..AA79 ; Not_XID # 5.2 [3] MYANMAR SYMBOL AITON EXCLAMATION..MYANMAR SYMBOL AITON TWO +AB5B ; Not_XID # 7.0 MODIFIER BREVE WITH INVERTED BREVE +AB6A..AB6B ; Not_XID # 13.0 [2] MODIFIER LETTER LEFT TACK..MODIFIER LETTER RIGHT TACK +FFF9..FFFB ; Not_XID # 3.0 [3] INTERLINEAR ANNOTATION ANCHOR..INTERLINEAR ANNOTATION TERMINATOR +FFFC ; Not_XID # 2.1 OBJECT REPLACEMENT CHARACTER +FFFD ; Not_XID # 1.1 REPLACEMENT CHARACTER +10175..1018A ; Not_XID # 4.1 [22] GREEK ONE HALF SIGN..GREEK ZERO SIGN +1018B..1018C ; Not_XID # 7.0 [2] GREEK ONE QUARTER SIGN..GREEK SINUSOID SIGN +1018D..1018E ; Not_XID # 9.0 [2] GREEK INDICTION SIGN..NOMISMA SIGN +10190..1019B ; Not_XID # 5.1 [12] ROMAN SEXTANS SIGN..ROMAN CENTURIAL SIGN +1019C ; Not_XID # 13.0 ASCIA SYMBOL +101A0 ; Not_XID # 7.0 GREEK SYMBOL TAU RHO +10E60..10E7E ; Not_XID # 5.2 [31] RUMI DIGIT ONE..RUMI FRACTION TWO THIRDS +10ED0..10ED8 ; Not_XID # 17.0 [9] ARABIC BIBLICAL END OF VERSE..ARABIC LIGATURE NAWWARA ALLAAHU MARQADAH +111E1..111F4 ; Not_XID # 7.0 [20] SINHALA ARCHAIC DIGIT ONE..SINHALA ARCHAIC NUMBER ONE THOUSAND +11B00..11B09 ; Not_XID # 15.0 [10] DEVANAGARI HEAD MARK..DEVANAGARI SIGN MINDU +11FC0..11FF1 ; Not_XID # 12.0 [50] TAMIL FRACTION ONE THREE-HUNDRED-AND-TWENTIETH..TAMIL SIGN VAKAIYARAA +11FFF ; Not_XID # 12.0 TAMIL PUNCTUATION END OF TEXT +16FE2 ; Not_XID # 12.0 OLD CHINESE HOOK MARK +1CC00..1CCD5 ; Not_XID # 16.0 [214] UP-POINTING GO-KART..LOWER RIGHT QUADRANT STANDING KNIGHT +1CCFA..1CCFC ; Not_XID # 17.0 [3] SNAKE SYMBOL..NOSE SYMBOL +1CD00..1CEB3 ; Not_XID # 16.0 [436] BLOCK OCTANT-3..BLACK RIGHT TRIANGLE CARET +1CEBA..1CED0 ; Not_XID # 17.0 [23] FRAGILE SYMBOL..LEUKOTHEA +1CEE0..1CEF0 ; Not_XID # 17.0 [17] GEOMANTIC FIGURE POPULUS..MEDIUM SMALL WHITE CIRCLE WITH HORIZONTAL BAR +1D2C0..1D2D3 ; Not_XID # 15.0 [20] KAKTOVIK NUMERAL ZERO..KAKTOVIK NUMERAL NINETEEN +1D2E0..1D2F3 ; Not_XID # 11.0 [20] MAYAN NUMERAL ZERO..MAYAN NUMERAL NINETEEN +1D360..1D371 ; Not_XID # 5.0 [18] COUNTING ROD UNIT DIGIT ONE..COUNTING ROD TENS DIGIT NINE +1D372..1D378 ; Not_XID # 11.0 [7] IDEOGRAPHIC TALLY MARK ONE..TALLY MARK FIVE +1EC71..1ECB4 ; Not_XID # 11.0 [68] INDIC SIYAQ NUMBER ONE..INDIC SIYAQ ALTERNATE LAKH MARK +1ED01..1ED3D ; Not_XID # 12.0 [61] OTTOMAN SIYAQ NUMBER ONE..OTTOMAN SIYAQ FRACTION ONE SIXTH +1EEF0..1EEF1 ; Not_XID # 6.1 [2] ARABIC MATHEMATICAL OPERATOR MEEM WITH HAH WITH TATWEEL..ARABIC MATHEMATICAL OPERATOR HAH WITH DAL +1F000..1F02B ; Not_XID # 5.1 [44] MAHJONG TILE EAST WIND..MAHJONG TILE BACK +1F030..1F093 ; Not_XID # 5.1 [100] DOMINO TILE HORIZONTAL BACK..DOMINO TILE VERTICAL-06-06 +1F0A0..1F0AE ; Not_XID # 6.0 [15] PLAYING CARD BACK..PLAYING CARD KING OF SPADES +1F0B1..1F0BE ; Not_XID # 6.0 [14] PLAYING CARD ACE OF HEARTS..PLAYING CARD KING OF HEARTS +1F0BF ; Not_XID # 7.0 PLAYING CARD RED JOKER +1F0C1..1F0CF ; Not_XID # 6.0 [15] PLAYING CARD ACE OF DIAMONDS..PLAYING CARD BLACK JOKER +1F0D1..1F0DF ; Not_XID # 6.0 [15] PLAYING CARD ACE OF CLUBS..PLAYING CARD WHITE JOKER +1F0E0..1F0F5 ; Not_XID # 7.0 [22] PLAYING CARD FOOL..PLAYING CARD TRUMP-21 +1F10B..1F10C ; Not_XID # 7.0 [2] DINGBAT CIRCLED SANS-SERIF DIGIT ZERO..DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT ZERO +1F10D..1F10F ; Not_XID # 13.0 [3] CIRCLED ZERO WITH SLASH..CIRCLED DOLLAR SIGN WITH OVERLAID BACKSLASH +1F12F ; Not_XID # 11.0 COPYLEFT SYMBOL +1F150..1F156 ; Not_XID # 6.0 [7] NEGATIVE CIRCLED LATIN CAPITAL LETTER A..NEGATIVE CIRCLED LATIN CAPITAL LETTER G +1F157 ; Not_XID # 5.2 NEGATIVE CIRCLED LATIN CAPITAL LETTER H +1F158..1F15E ; Not_XID # 6.0 [7] NEGATIVE CIRCLED LATIN CAPITAL LETTER I..NEGATIVE CIRCLED LATIN CAPITAL LETTER O +1F15F ; Not_XID # 5.2 NEGATIVE CIRCLED LATIN CAPITAL LETTER P +1F160..1F169 ; Not_XID # 6.0 [10] NEGATIVE CIRCLED LATIN CAPITAL LETTER Q..NEGATIVE CIRCLED LATIN CAPITAL LETTER Z +1F16D..1F16F ; Not_XID # 13.0 [3] CIRCLED CC..CIRCLED HUMAN FIGURE +1F170..1F178 ; Not_XID # 6.0 [9] NEGATIVE SQUARED LATIN CAPITAL LETTER A..NEGATIVE SQUARED LATIN CAPITAL LETTER I +1F179 ; Not_XID # 5.2 NEGATIVE SQUARED LATIN CAPITAL LETTER J +1F17A ; Not_XID # 6.0 NEGATIVE SQUARED LATIN CAPITAL LETTER K +1F17B..1F17C ; Not_XID # 5.2 [2] NEGATIVE SQUARED LATIN CAPITAL LETTER L..NEGATIVE SQUARED LATIN CAPITAL LETTER M +1F17D..1F17E ; Not_XID # 6.0 [2] NEGATIVE SQUARED LATIN CAPITAL LETTER N..NEGATIVE SQUARED LATIN CAPITAL LETTER O +1F17F ; Not_XID # 5.2 NEGATIVE SQUARED LATIN CAPITAL LETTER P +1F180..1F189 ; Not_XID # 6.0 [10] NEGATIVE SQUARED LATIN CAPITAL LETTER Q..NEGATIVE SQUARED LATIN CAPITAL LETTER Z +1F18A..1F18D ; Not_XID # 5.2 [4] CROSSED NEGATIVE SQUARED LATIN CAPITAL LETTER P..NEGATIVE SQUARED SA +1F18E..1F18F ; Not_XID # 6.0 [2] NEGATIVE SQUARED AB..NEGATIVE SQUARED WC +1F191..1F19A ; Not_XID # 6.0 [10] SQUARED CL..SQUARED VS +1F19B..1F1AC ; Not_XID # 9.0 [18] SQUARED THREE D..SQUARED VOD +1F1AD ; Not_XID # 13.0 MASK WORK SYMBOL +1F1E6..1F1FF ; Not_XID # 6.0 [26] REGIONAL INDICATOR SYMBOL LETTER A..REGIONAL INDICATOR SYMBOL LETTER Z +1F260..1F265 ; Not_XID # 10.0 [6] ROUNDED SYMBOL FOR FU..ROUNDED SYMBOL FOR CAI +1F300..1F320 ; Not_XID # 6.0 [33] CYCLONE..SHOOTING STAR +1F321..1F32C ; Not_XID # 7.0 [12] THERMOMETER..WIND BLOWING FACE +1F32D..1F32F ; Not_XID # 8.0 [3] HOT DOG..BURRITO +1F330..1F335 ; Not_XID # 6.0 [6] CHESTNUT..CACTUS +1F336 ; Not_XID # 7.0 HOT PEPPER +1F337..1F37C ; Not_XID # 6.0 [70] TULIP..BABY BOTTLE +1F37D ; Not_XID # 7.0 FORK AND KNIFE WITH PLATE +1F37E..1F37F ; Not_XID # 8.0 [2] BOTTLE WITH POPPING CORK..POPCORN +1F380..1F393 ; Not_XID # 6.0 [20] RIBBON..GRADUATION CAP +1F394..1F39F ; Not_XID # 7.0 [12] HEART WITH TIP ON THE LEFT..ADMISSION TICKETS +1F3A0..1F3C4 ; Not_XID # 6.0 [37] CAROUSEL HORSE..SURFER +1F3C5 ; Not_XID # 7.0 SPORTS MEDAL +1F3C6..1F3CA ; Not_XID # 6.0 [5] TROPHY..SWIMMER +1F3CB..1F3CE ; Not_XID # 7.0 [4] WEIGHT LIFTER..RACING CAR +1F3CF..1F3D3 ; Not_XID # 8.0 [5] CRICKET BAT AND BALL..TABLE TENNIS PADDLE AND BALL +1F3D4..1F3DF ; Not_XID # 7.0 [12] SNOW CAPPED MOUNTAIN..STADIUM +1F3E0..1F3F0 ; Not_XID # 6.0 [17] HOUSE BUILDING..EUROPEAN CASTLE +1F3F1..1F3F7 ; Not_XID # 7.0 [7] WHITE PENNANT..LABEL +1F3F8..1F3FF ; Not_XID # 8.0 [8] BADMINTON RACQUET AND SHUTTLECOCK..EMOJI MODIFIER FITZPATRICK TYPE-6 +1F400..1F43E ; Not_XID # 6.0 [63] RAT..PAW PRINTS +1F43F ; Not_XID # 7.0 CHIPMUNK +1F440 ; Not_XID # 6.0 EYES +1F441 ; Not_XID # 7.0 EYE +1F442..1F4F7 ; Not_XID # 6.0 [182] EAR..CAMERA +1F4F8 ; Not_XID # 7.0 CAMERA WITH FLASH +1F4F9..1F4FC ; Not_XID # 6.0 [4] VIDEO CAMERA..VIDEOCASSETTE +1F4FD..1F4FE ; Not_XID # 7.0 [2] FILM PROJECTOR..PORTABLE STEREO +1F4FF ; Not_XID # 8.0 PRAYER BEADS +1F500..1F53D ; Not_XID # 6.0 [62] TWISTED RIGHTWARDS ARROWS..DOWN-POINTING SMALL RED TRIANGLE +1F53E..1F53F ; Not_XID # 7.0 [2] LOWER RIGHT SHADOWED WHITE CIRCLE..UPPER RIGHT SHADOWED WHITE CIRCLE +1F540..1F543 ; Not_XID # 6.1 [4] CIRCLED CROSS POMMEE..NOTCHED LEFT SEMICIRCLE WITH THREE DOTS +1F544..1F54A ; Not_XID # 7.0 [7] NOTCHED RIGHT SEMICIRCLE WITH THREE DOTS..DOVE OF PEACE +1F54B..1F54E ; Not_XID # 8.0 [4] KAABA..MENORAH WITH NINE BRANCHES +1F550..1F567 ; Not_XID # 6.0 [24] CLOCK FACE ONE OCLOCK..CLOCK FACE TWELVE-THIRTY +1F568..1F579 ; Not_XID # 7.0 [18] RIGHT SPEAKER..JOYSTICK +1F57A ; Not_XID # 9.0 MAN DANCING +1F57B..1F5A3 ; Not_XID # 7.0 [41] LEFT HAND TELEPHONE RECEIVER..BLACK DOWN POINTING BACKHAND INDEX +1F5A4 ; Not_XID # 9.0 BLACK HEART +1F5A5..1F5FA ; Not_XID # 7.0 [86] DESKTOP COMPUTER..WORLD MAP +1F5FB..1F5FF ; Not_XID # 6.0 [5] MOUNT FUJI..MOYAI +1F600 ; Not_XID # 6.1 GRINNING FACE +1F601..1F610 ; Not_XID # 6.0 [16] GRINNING FACE WITH SMILING EYES..NEUTRAL FACE +1F611 ; Not_XID # 6.1 EXPRESSIONLESS FACE +1F612..1F614 ; Not_XID # 6.0 [3] UNAMUSED FACE..PENSIVE FACE +1F615 ; Not_XID # 6.1 CONFUSED FACE +1F616 ; Not_XID # 6.0 CONFOUNDED FACE +1F617 ; Not_XID # 6.1 KISSING FACE +1F618 ; Not_XID # 6.0 FACE THROWING A KISS +1F619 ; Not_XID # 6.1 KISSING FACE WITH SMILING EYES +1F61A ; Not_XID # 6.0 KISSING FACE WITH CLOSED EYES +1F61B ; Not_XID # 6.1 FACE WITH STUCK-OUT TONGUE +1F61C..1F61E ; Not_XID # 6.0 [3] FACE WITH STUCK-OUT TONGUE AND WINKING EYE..DISAPPOINTED FACE +1F61F ; Not_XID # 6.1 WORRIED FACE +1F620..1F625 ; Not_XID # 6.0 [6] ANGRY FACE..DISAPPOINTED BUT RELIEVED FACE +1F626..1F627 ; Not_XID # 6.1 [2] FROWNING FACE WITH OPEN MOUTH..ANGUISHED FACE +1F628..1F62B ; Not_XID # 6.0 [4] FEARFUL FACE..TIRED FACE +1F62C ; Not_XID # 6.1 GRIMACING FACE +1F62D ; Not_XID # 6.0 LOUDLY CRYING FACE +1F62E..1F62F ; Not_XID # 6.1 [2] FACE WITH OPEN MOUTH..HUSHED FACE +1F630..1F633 ; Not_XID # 6.0 [4] FACE WITH OPEN MOUTH AND COLD SWEAT..FLUSHED FACE +1F634 ; Not_XID # 6.1 SLEEPING FACE +1F635..1F640 ; Not_XID # 6.0 [12] DIZZY FACE..WEARY CAT FACE +1F641..1F642 ; Not_XID # 7.0 [2] SLIGHTLY FROWNING FACE..SLIGHTLY SMILING FACE +1F643..1F644 ; Not_XID # 8.0 [2] UPSIDE-DOWN FACE..FACE WITH ROLLING EYES +1F645..1F64F ; Not_XID # 6.0 [11] FACE WITH NO GOOD GESTURE..PERSON WITH FOLDED HANDS +1F650..1F67F ; Not_XID # 7.0 [48] NORTH WEST POINTING LEAF..REVERSE CHECKER BOARD +1F680..1F6C5 ; Not_XID # 6.0 [70] ROCKET..LEFT LUGGAGE +1F6C6..1F6CF ; Not_XID # 7.0 [10] TRIANGLE WITH ROUNDED CORNERS..BED +1F6D0 ; Not_XID # 8.0 PLACE OF WORSHIP +1F6D1..1F6D2 ; Not_XID # 9.0 [2] OCTAGONAL SIGN..SHOPPING TROLLEY +1F6D3..1F6D4 ; Not_XID # 10.0 [2] STUPA..PAGODA +1F6D5 ; Not_XID # 12.0 HINDU TEMPLE +1F6D6..1F6D7 ; Not_XID # 13.0 [2] HUT..ELEVATOR +1F6D8 ; Not_XID # 17.0 LANDSLIDE +1F6DC ; Not_XID # 15.0 WIRELESS +1F6DD..1F6DF ; Not_XID # 14.0 [3] PLAYGROUND SLIDE..RING BUOY +1F6E0..1F6EC ; Not_XID # 7.0 [13] HAMMER AND WRENCH..AIRPLANE ARRIVING +1F6F0..1F6F3 ; Not_XID # 7.0 [4] SATELLITE..PASSENGER SHIP +1F6F4..1F6F6 ; Not_XID # 9.0 [3] SCOOTER..CANOE +1F6F7..1F6F8 ; Not_XID # 10.0 [2] SLED..FLYING SAUCER +1F6F9 ; Not_XID # 11.0 SKATEBOARD +1F6FA ; Not_XID # 12.0 AUTO RICKSHAW +1F6FB..1F6FC ; Not_XID # 13.0 [2] PICKUP TRUCK..ROLLER SKATE +1F700..1F773 ; Not_XID # 6.0 [116] ALCHEMICAL SYMBOL FOR QUINTESSENCE..ALCHEMICAL SYMBOL FOR HALF OUNCE +1F774..1F776 ; Not_XID # 15.0 [3] LOT OF FORTUNE..LUNAR ECLIPSE +1F777..1F77A ; Not_XID # 17.0 [4] VESTA FORM TWO..PARTHENOPE FORM TWO +1F77B..1F77F ; Not_XID # 15.0 [5] HAUMEA..ORCUS +1F780..1F7D4 ; Not_XID # 7.0 [85] BLACK LEFT-POINTING ISOSCELES RIGHT TRIANGLE..HEAVY TWELVE POINTED PINWHEEL STAR +1F7D5..1F7D8 ; Not_XID # 11.0 [4] CIRCLED TRIANGLE..NEGATIVE CIRCLED SQUARE +1F7D9 ; Not_XID # 15.0 NINE POINTED WHITE STAR +1F7E0..1F7EB ; Not_XID # 12.0 [12] LARGE ORANGE CIRCLE..LARGE BROWN SQUARE +1F7F0 ; Not_XID # 14.0 HEAVY EQUALS SIGN +1F800..1F80B ; Not_XID # 7.0 [12] LEFTWARDS ARROW WITH SMALL TRIANGLE ARROWHEAD..DOWNWARDS ARROW WITH LARGE TRIANGLE ARROWHEAD +1F810..1F847 ; Not_XID # 7.0 [56] LEFTWARDS ARROW WITH SMALL EQUILATERAL ARROWHEAD..DOWNWARDS HEAVY ARROW +1F850..1F859 ; Not_XID # 7.0 [10] LEFTWARDS SANS-SERIF ARROW..UP DOWN SANS-SERIF ARROW +1F860..1F887 ; Not_XID # 7.0 [40] WIDE-HEADED LEFTWARDS LIGHT BARB ARROW..WIDE-HEADED SOUTH WEST VERY HEAVY BARB ARROW +1F890..1F8AD ; Not_XID # 7.0 [30] LEFTWARDS TRIANGLE ARROWHEAD..WHITE ARROW SHAFT WIDTH TWO THIRDS +1F8B0..1F8B1 ; Not_XID # 13.0 [2] ARROW POINTING UPWARDS THEN NORTH WEST..ARROW POINTING RIGHTWARDS THEN CURVING SOUTH WEST +1F8B2..1F8BB ; Not_XID # 16.0 [10] RIGHTWARDS ARROW WITH LOWER HOOK..SOUTH WEST ARROW FROM BAR +1F8C0..1F8C1 ; Not_XID # 16.0 [2] LEFTWARDS ARROW FROM DOWNWARDS ARROW..RIGHTWARDS ARROW FROM DOWNWARDS ARROW +1F8D0..1F8D8 ; Not_XID # 17.0 [9] LONG RIGHTWARDS ARROW OVER LONG LEFTWARDS ARROW..LONG LEFT RIGHT ARROW WITH DEPENDENT LOBE +1F900..1F90B ; Not_XID # 10.0 [12] CIRCLED CROSS FORMEE WITH FOUR DOTS..DOWNWARD FACING NOTCHED HOOK WITH DOT +1F90C ; Not_XID # 13.0 PINCHED FINGERS +1F90D..1F90F ; Not_XID # 12.0 [3] WHITE HEART..PINCHING HAND +1F910..1F918 ; Not_XID # 8.0 [9] ZIPPER-MOUTH FACE..SIGN OF THE HORNS +1F919..1F91E ; Not_XID # 9.0 [6] CALL ME HAND..HAND WITH INDEX AND MIDDLE FINGERS CROSSED +1F91F ; Not_XID # 10.0 I LOVE YOU HAND SIGN +1F920..1F927 ; Not_XID # 9.0 [8] FACE WITH COWBOY HAT..SNEEZING FACE +1F928..1F92F ; Not_XID # 10.0 [8] FACE WITH ONE EYEBROW RAISED..SHOCKED FACE WITH EXPLODING HEAD +1F930 ; Not_XID # 9.0 PREGNANT WOMAN +1F931..1F932 ; Not_XID # 10.0 [2] BREAST-FEEDING..PALMS UP TOGETHER +1F933..1F93E ; Not_XID # 9.0 [12] SELFIE..HANDBALL +1F93F ; Not_XID # 12.0 DIVING MASK +1F940..1F94B ; Not_XID # 9.0 [12] WILTED FLOWER..MARTIAL ARTS UNIFORM +1F94C ; Not_XID # 10.0 CURLING STONE +1F94D..1F94F ; Not_XID # 11.0 [3] LACROSSE STICK AND BALL..FLYING DISC +1F950..1F95E ; Not_XID # 9.0 [15] CROISSANT..PANCAKES +1F95F..1F96B ; Not_XID # 10.0 [13] DUMPLING..CANNED FOOD +1F96C..1F970 ; Not_XID # 11.0 [5] LEAFY GREEN..SMILING FACE WITH SMILING EYES AND THREE HEARTS +1F971 ; Not_XID # 12.0 YAWNING FACE +1F972 ; Not_XID # 13.0 SMILING FACE WITH TEAR +1F973..1F976 ; Not_XID # 11.0 [4] FACE WITH PARTY HORN AND PARTY HAT..FREEZING FACE +1F977..1F978 ; Not_XID # 13.0 [2] NINJA..DISGUISED FACE +1F979 ; Not_XID # 14.0 FACE HOLDING BACK TEARS +1F97A ; Not_XID # 11.0 FACE WITH PLEADING EYES +1F97B ; Not_XID # 12.0 SARI +1F97C..1F97F ; Not_XID # 11.0 [4] LAB COAT..FLAT SHOE +1F980..1F984 ; Not_XID # 8.0 [5] CRAB..UNICORN FACE +1F985..1F991 ; Not_XID # 9.0 [13] EAGLE..SQUID +1F992..1F997 ; Not_XID # 10.0 [6] GIRAFFE FACE..CRICKET +1F998..1F9A2 ; Not_XID # 11.0 [11] KANGAROO..SWAN +1F9A3..1F9A4 ; Not_XID # 13.0 [2] MAMMOTH..DODO +1F9A5..1F9AA ; Not_XID # 12.0 [6] SLOTH..OYSTER +1F9AB..1F9AD ; Not_XID # 13.0 [3] BEAVER..SEAL +1F9AE..1F9AF ; Not_XID # 12.0 [2] GUIDE DOG..PROBING CANE +1F9B0..1F9B9 ; Not_XID # 11.0 [10] EMOJI COMPONENT RED HAIR..SUPERVILLAIN +1F9BA..1F9BF ; Not_XID # 12.0 [6] SAFETY VEST..MECHANICAL LEG +1F9C0 ; Not_XID # 8.0 CHEESE WEDGE +1F9C1..1F9C2 ; Not_XID # 11.0 [2] CUPCAKE..SALT SHAKER +1F9C3..1F9CA ; Not_XID # 12.0 [8] BEVERAGE BOX..ICE CUBE +1F9CB ; Not_XID # 13.0 BUBBLE TEA +1F9CC ; Not_XID # 14.0 TROLL +1F9CD..1F9CF ; Not_XID # 12.0 [3] STANDING PERSON..DEAF PERSON +1F9D0..1F9E6 ; Not_XID # 10.0 [23] FACE WITH MONOCLE..SOCKS +1F9E7..1F9FF ; Not_XID # 11.0 [25] RED GIFT ENVELOPE..NAZAR AMULET +1FA00..1FA53 ; Not_XID # 12.0 [84] NEUTRAL CHESS KING..BLACK CHESS KNIGHT-BISHOP +1FA54..1FA57 ; Not_XID # 17.0 [4] WHITE CHESS FERZ..BLACK CHESS ALFIL +1FA60..1FA6D ; Not_XID # 11.0 [14] XIANGQI RED GENERAL..XIANGQI BLACK SOLDIER +1FA70..1FA73 ; Not_XID # 12.0 [4] BALLET SHOES..SHORTS +1FA74 ; Not_XID # 13.0 THONG SANDAL +1FA75..1FA77 ; Not_XID # 15.0 [3] LIGHT BLUE HEART..PINK HEART +1FA78..1FA7A ; Not_XID # 12.0 [3] DROP OF BLOOD..STETHOSCOPE +1FA7B..1FA7C ; Not_XID # 14.0 [2] X-RAY..CRUTCH +1FA80..1FA82 ; Not_XID # 12.0 [3] YO-YO..PARACHUTE +1FA83..1FA86 ; Not_XID # 13.0 [4] BOOMERANG..NESTING DOLLS +1FA87..1FA88 ; Not_XID # 15.0 [2] MARACAS..FLUTE +1FA89 ; Not_XID # 16.0 HARP +1FA8A ; Not_XID # 17.0 TROMBONE +1FA8E ; Not_XID # 17.0 TREASURE CHEST +1FA8F ; Not_XID # 16.0 SHOVEL +1FA90..1FA95 ; Not_XID # 12.0 [6] RINGED PLANET..BANJO +1FA96..1FAA8 ; Not_XID # 13.0 [19] MILITARY HELMET..ROCK +1FAA9..1FAAC ; Not_XID # 14.0 [4] MIRROR BALL..HAMSA +1FAAD..1FAAF ; Not_XID # 15.0 [3] FOLDING HAND FAN..KHANDA +1FAB0..1FAB6 ; Not_XID # 13.0 [7] FLY..FEATHER +1FAB7..1FABA ; Not_XID # 14.0 [4] LOTUS..NEST WITH EGGS +1FABB..1FABD ; Not_XID # 15.0 [3] HYACINTH..WING +1FABE ; Not_XID # 16.0 LEAFLESS TREE +1FABF ; Not_XID # 15.0 GOOSE +1FAC0..1FAC2 ; Not_XID # 13.0 [3] ANATOMICAL HEART..PEOPLE HUGGING +1FAC3..1FAC5 ; Not_XID # 14.0 [3] PREGNANT MAN..PERSON WITH CROWN +1FAC6 ; Not_XID # 16.0 FINGERPRINT +1FAC8 ; Not_XID # 17.0 HAIRY CREATURE +1FACD ; Not_XID # 17.0 ORCA +1FACE..1FACF ; Not_XID # 15.0 [2] MOOSE..DONKEY +1FAD0..1FAD6 ; Not_XID # 13.0 [7] BLUEBERRIES..TEAPOT +1FAD7..1FAD9 ; Not_XID # 14.0 [3] POURING LIQUID..JAR +1FADA..1FADB ; Not_XID # 15.0 [2] GINGER ROOT..PEA POD +1FADC ; Not_XID # 16.0 ROOT VEGETABLE +1FADF ; Not_XID # 16.0 SPLATTER +1FAE0..1FAE7 ; Not_XID # 14.0 [8] MELTING FACE..BUBBLES +1FAE8 ; Not_XID # 15.0 SHAKING FACE +1FAE9 ; Not_XID # 16.0 FACE WITH BAGS UNDER EYES +1FAEA ; Not_XID # 17.0 DISTORTED FACE +1FAEF ; Not_XID # 17.0 FIGHT CLOUD +1FAF0..1FAF6 ; Not_XID # 14.0 [7] HAND WITH INDEX FINGER AND THUMB CROSSED..HEART HANDS +1FAF7..1FAF8 ; Not_XID # 15.0 [2] LEFTWARDS PUSHING HAND..RIGHTWARDS PUSHING HAND +1FB00..1FB92 ; Not_XID # 13.0 [147] BLOCK SEXTANT-1..UPPER HALF INVERSE MEDIUM SHADE AND LOWER HALF BLOCK +1FB94..1FBCA ; Not_XID # 13.0 [55] LEFT HALF INVERSE MEDIUM SHADE AND RIGHT HALF BLOCK..WHITE UP-POINTING CHEVRON +1FBCB..1FBEF ; Not_XID # 16.0 [37] WHITE CROSS MARK..TOP LEFT JUSTIFIED LOWER RIGHT QUARTER BLACK CIRCLE +1FBFA ; Not_XID # 17.0 ALARM BELL SYMBOL + +# Total code points: 6487 + +# Identifier_Type: Not_NFKC + +00A0 ; Not_NFKC # 1.1 NO-BREAK SPACE +00A8 ; Not_NFKC # 1.1 DIAERESIS +00AA ; Not_NFKC # 1.1 FEMININE ORDINAL INDICATOR +00AF ; Not_NFKC # 1.1 MACRON +00B2..00B5 ; Not_NFKC # 1.1 [4] SUPERSCRIPT TWO..MICRO SIGN +00B8..00BA ; Not_NFKC # 1.1 [3] CEDILLA..MASCULINE ORDINAL INDICATOR +00BC..00BE ; Not_NFKC # 1.1 [3] VULGAR FRACTION ONE QUARTER..VULGAR FRACTION THREE QUARTERS +0132..0133 ; Not_NFKC # 1.1 [2] LATIN CAPITAL LIGATURE IJ..LATIN SMALL LIGATURE IJ +013F..0140 ; Not_NFKC # 1.1 [2] LATIN CAPITAL LETTER L WITH MIDDLE DOT..LATIN SMALL LETTER L WITH MIDDLE DOT +017F ; Not_NFKC # 1.1 LATIN SMALL LETTER LONG S +01C4..01CC ; Not_NFKC # 1.1 [9] LATIN CAPITAL LETTER DZ WITH CARON..LATIN SMALL LETTER NJ +01F1..01F3 ; Not_NFKC # 1.1 [3] LATIN CAPITAL LETTER DZ..LATIN SMALL LETTER DZ +02B0..02B8 ; Not_NFKC # 1.1 [9] MODIFIER LETTER SMALL H..MODIFIER LETTER SMALL Y +02D8..02DD ; Not_NFKC # 1.1 [6] BREVE..DOUBLE ACUTE ACCENT +02E0..02E4 ; Not_NFKC # 1.1 [5] MODIFIER LETTER SMALL GAMMA..MODIFIER LETTER SMALL REVERSED GLOTTAL STOP +0340..0341 ; Not_NFKC # 1.1 [2] COMBINING GRAVE TONE MARK..COMBINING ACUTE TONE MARK +0343..0344 ; Not_NFKC # 1.1 [2] COMBINING GREEK KORONIS..COMBINING GREEK DIALYTIKA TONOS +0374 ; Not_NFKC # 1.1 GREEK NUMERAL SIGN +037A ; Not_NFKC # 1.1 GREEK YPOGEGRAMMENI +037E ; Not_NFKC # 1.1 GREEK QUESTION MARK +0384..0385 ; Not_NFKC # 1.1 [2] GREEK TONOS..GREEK DIALYTIKA TONOS +0387 ; Not_NFKC # 1.1 GREEK ANO TELEIA +03D0..03D6 ; Not_NFKC # 1.1 [7] GREEK BETA SYMBOL..GREEK PI SYMBOL +03F0..03F2 ; Not_NFKC # 1.1 [3] GREEK KAPPA SYMBOL..GREEK LUNATE SIGMA SYMBOL +03F4..03F5 ; Not_NFKC # 3.1 [2] GREEK CAPITAL THETA SYMBOL..GREEK LUNATE EPSILON SYMBOL +03F9 ; Not_NFKC # 4.0 GREEK CAPITAL LUNATE SIGMA SYMBOL +0587 ; Not_NFKC # 1.1 ARMENIAN SMALL LIGATURE ECH YIWN +0675..0678 ; Not_NFKC # 1.1 [4] ARABIC LETTER HIGH HAMZA ALEF..ARABIC LETTER HIGH HAMZA YEH +0958..095F ; Not_NFKC # 1.1 [8] DEVANAGARI LETTER QA..DEVANAGARI LETTER YYA +09DC..09DD ; Not_NFKC # 1.1 [2] BENGALI LETTER RRA..BENGALI LETTER RHA +09DF ; Not_NFKC # 1.1 BENGALI LETTER YYA +0A33 ; Not_NFKC # 1.1 GURMUKHI LETTER LLA +0A36 ; Not_NFKC # 1.1 GURMUKHI LETTER SHA +0A59..0A5B ; Not_NFKC # 1.1 [3] GURMUKHI LETTER KHHA..GURMUKHI LETTER ZA +0A5E ; Not_NFKC # 1.1 GURMUKHI LETTER FA +0B5C..0B5D ; Not_NFKC # 1.1 [2] ORIYA LETTER RRA..ORIYA LETTER RHA +0E33 ; Not_NFKC # 1.1 THAI CHARACTER SARA AM +0EB3 ; Not_NFKC # 1.1 LAO VOWEL SIGN AM +0EDC..0EDD ; Not_NFKC # 1.1 [2] LAO HO NO..LAO HO MO +0F0C ; Not_NFKC # 2.0 TIBETAN MARK DELIMITER TSHEG BSTAR +0F43 ; Not_NFKC # 2.0 TIBETAN LETTER GHA +0F4D ; Not_NFKC # 2.0 TIBETAN LETTER DDHA +0F52 ; Not_NFKC # 2.0 TIBETAN LETTER DHA +0F57 ; Not_NFKC # 2.0 TIBETAN LETTER BHA +0F5C ; Not_NFKC # 2.0 TIBETAN LETTER DZHA +0F69 ; Not_NFKC # 2.0 TIBETAN LETTER KSSA +0F73 ; Not_NFKC # 2.0 TIBETAN VOWEL SIGN II +0F75..0F76 ; Not_NFKC # 2.0 [2] TIBETAN VOWEL SIGN UU..TIBETAN VOWEL SIGN VOCALIC R +0F78 ; Not_NFKC # 2.0 TIBETAN VOWEL SIGN VOCALIC L +0F81 ; Not_NFKC # 2.0 TIBETAN VOWEL SIGN REVERSED II +0F93 ; Not_NFKC # 2.0 TIBETAN SUBJOINED LETTER GHA +0F9D ; Not_NFKC # 2.0 TIBETAN SUBJOINED LETTER DDHA +0FA2 ; Not_NFKC # 2.0 TIBETAN SUBJOINED LETTER DHA +0FA7 ; Not_NFKC # 2.0 TIBETAN SUBJOINED LETTER BHA +0FAC ; Not_NFKC # 2.0 TIBETAN SUBJOINED LETTER DZHA +0FB9 ; Not_NFKC # 2.0 TIBETAN SUBJOINED LETTER KSSA +10FC ; Not_NFKC # 4.1 MODIFIER LETTER GEORGIAN NAR +1D2C..1D2E ; Not_NFKC # 4.0 [3] MODIFIER LETTER CAPITAL A..MODIFIER LETTER CAPITAL B +1D30..1D3A ; Not_NFKC # 4.0 [11] MODIFIER LETTER CAPITAL D..MODIFIER LETTER CAPITAL N +1D3C..1D4D ; Not_NFKC # 4.0 [18] MODIFIER LETTER CAPITAL O..MODIFIER LETTER SMALL G +1D4F..1D6A ; Not_NFKC # 4.0 [28] MODIFIER LETTER SMALL K..GREEK SUBSCRIPT SMALL LETTER CHI +1D78 ; Not_NFKC # 4.1 MODIFIER LETTER CYRILLIC EN +1D9B..1DBF ; Not_NFKC # 4.1 [37] MODIFIER LETTER SMALL TURNED ALPHA..MODIFIER LETTER SMALL THETA +1E9A ; Not_NFKC # 1.1 LATIN SMALL LETTER A WITH RIGHT HALF RING +1E9B ; Not_NFKC # 2.0 LATIN SMALL LETTER LONG S WITH DOT ABOVE +1F71 ; Not_NFKC # 1.1 GREEK SMALL LETTER ALPHA WITH OXIA +1F73 ; Not_NFKC # 1.1 GREEK SMALL LETTER EPSILON WITH OXIA +1F75 ; Not_NFKC # 1.1 GREEK SMALL LETTER ETA WITH OXIA +1F77 ; Not_NFKC # 1.1 GREEK SMALL LETTER IOTA WITH OXIA +1F79 ; Not_NFKC # 1.1 GREEK SMALL LETTER OMICRON WITH OXIA +1F7B ; Not_NFKC # 1.1 GREEK SMALL LETTER UPSILON WITH OXIA +1F7D ; Not_NFKC # 1.1 GREEK SMALL LETTER OMEGA WITH OXIA +1FBB ; Not_NFKC # 1.1 GREEK CAPITAL LETTER ALPHA WITH OXIA +1FBD..1FC1 ; Not_NFKC # 1.1 [5] GREEK KORONIS..GREEK DIALYTIKA AND PERISPOMENI +1FC9 ; Not_NFKC # 1.1 GREEK CAPITAL LETTER EPSILON WITH OXIA +1FCB ; Not_NFKC # 1.1 GREEK CAPITAL LETTER ETA WITH OXIA +1FCD..1FCF ; Not_NFKC # 1.1 [3] GREEK PSILI AND VARIA..GREEK PSILI AND PERISPOMENI +1FD3 ; Not_NFKC # 1.1 GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA +1FDB ; Not_NFKC # 1.1 GREEK CAPITAL LETTER IOTA WITH OXIA +1FDD..1FDF ; Not_NFKC # 1.1 [3] GREEK DASIA AND VARIA..GREEK DASIA AND PERISPOMENI +1FE3 ; Not_NFKC # 1.1 GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA +1FEB ; Not_NFKC # 1.1 GREEK CAPITAL LETTER UPSILON WITH OXIA +1FED..1FEF ; Not_NFKC # 1.1 [3] GREEK DIALYTIKA AND VARIA..GREEK VARIA +1FF9 ; Not_NFKC # 1.1 GREEK CAPITAL LETTER OMICRON WITH OXIA +1FFB ; Not_NFKC # 1.1 GREEK CAPITAL LETTER OMEGA WITH OXIA +1FFD..1FFE ; Not_NFKC # 1.1 [2] GREEK OXIA..GREEK DASIA +2000..200A ; Not_NFKC # 1.1 [11] EN QUAD..HAIR SPACE +2011 ; Not_NFKC # 1.1 NON-BREAKING HYPHEN +2017 ; Not_NFKC # 1.1 DOUBLE LOW LINE +2024..2026 ; Not_NFKC # 1.1 [3] ONE DOT LEADER..HORIZONTAL ELLIPSIS +202F ; Not_NFKC # 3.0 NARROW NO-BREAK SPACE +2033..2034 ; Not_NFKC # 1.1 [2] DOUBLE PRIME..TRIPLE PRIME +2036..2037 ; Not_NFKC # 1.1 [2] REVERSED DOUBLE PRIME..REVERSED TRIPLE PRIME +203C ; Not_NFKC # 1.1 DOUBLE EXCLAMATION MARK +203E ; Not_NFKC # 1.1 OVERLINE +2047 ; Not_NFKC # 3.2 DOUBLE QUESTION MARK +2048..2049 ; Not_NFKC # 3.0 [2] QUESTION EXCLAMATION MARK..EXCLAMATION QUESTION MARK +2057 ; Not_NFKC # 3.2 QUADRUPLE PRIME +205F ; Not_NFKC # 3.2 MEDIUM MATHEMATICAL SPACE +2070 ; Not_NFKC # 1.1 SUPERSCRIPT ZERO +2071 ; Not_NFKC # 3.2 SUPERSCRIPT LATIN SMALL LETTER I +2074..208E ; Not_NFKC # 1.1 [27] SUPERSCRIPT FOUR..SUBSCRIPT RIGHT PARENTHESIS +2090..2094 ; Not_NFKC # 4.1 [5] LATIN SUBSCRIPT SMALL LETTER A..LATIN SUBSCRIPT SMALL LETTER SCHWA +2095..209C ; Not_NFKC # 6.0 [8] LATIN SUBSCRIPT SMALL LETTER H..LATIN SUBSCRIPT SMALL LETTER T +20A8 ; Not_NFKC # 1.1 RUPEE SIGN +2100..2103 ; Not_NFKC # 1.1 [4] ACCOUNT OF..DEGREE CELSIUS +2105..2107 ; Not_NFKC # 1.1 [3] CARE OF..EULER CONSTANT +2109..2113 ; Not_NFKC # 1.1 [11] DEGREE FAHRENHEIT..SCRIPT SMALL L +2115..2116 ; Not_NFKC # 1.1 [2] DOUBLE-STRUCK CAPITAL N..NUMERO SIGN +2119..211D ; Not_NFKC # 1.1 [5] DOUBLE-STRUCK CAPITAL P..DOUBLE-STRUCK CAPITAL R +2120..2122 ; Not_NFKC # 1.1 [3] SERVICE MARK..TRADE MARK SIGN +2124 ; Not_NFKC # 1.1 DOUBLE-STRUCK CAPITAL Z +2126 ; Not_NFKC # 1.1 OHM SIGN +2128 ; Not_NFKC # 1.1 BLACK-LETTER CAPITAL Z +212A..212D ; Not_NFKC # 1.1 [4] KELVIN SIGN..BLACK-LETTER CAPITAL C +212F..2131 ; Not_NFKC # 1.1 [3] SCRIPT SMALL E..SCRIPT CAPITAL F +2133..2138 ; Not_NFKC # 1.1 [6] SCRIPT CAPITAL M..DALET SYMBOL +2139 ; Not_NFKC # 3.0 INFORMATION SOURCE +213B ; Not_NFKC # 4.0 FACSIMILE SIGN +213C ; Not_NFKC # 4.1 DOUBLE-STRUCK SMALL PI +213D..2140 ; Not_NFKC # 3.2 [4] DOUBLE-STRUCK SMALL GAMMA..DOUBLE-STRUCK N-ARY SUMMATION +2145..2149 ; Not_NFKC # 3.2 [5] DOUBLE-STRUCK ITALIC CAPITAL D..DOUBLE-STRUCK ITALIC SMALL J +2150..2152 ; Not_NFKC # 5.2 [3] VULGAR FRACTION ONE SEVENTH..VULGAR FRACTION ONE TENTH +2153..217F ; Not_NFKC # 1.1 [45] VULGAR FRACTION ONE THIRD..SMALL ROMAN NUMERAL ONE THOUSAND +2189 ; Not_NFKC # 5.2 VULGAR FRACTION ZERO THIRDS +222C..222D ; Not_NFKC # 1.1 [2] DOUBLE INTEGRAL..TRIPLE INTEGRAL +222F..2230 ; Not_NFKC # 1.1 [2] SURFACE INTEGRAL..VOLUME INTEGRAL +2460..24EA ; Not_NFKC # 1.1 [139] CIRCLED DIGIT ONE..CIRCLED DIGIT ZERO +2A0C ; Not_NFKC # 3.2 QUADRUPLE INTEGRAL OPERATOR +2A74..2A76 ; Not_NFKC # 3.2 [3] DOUBLE COLON EQUAL..THREE CONSECUTIVE EQUALS SIGNS +2ADC ; Not_NFKC # 3.2 FORKING +2C7C..2C7D ; Not_NFKC # 5.1 [2] LATIN SUBSCRIPT SMALL LETTER J..MODIFIER LETTER CAPITAL V +2D6F ; Not_NFKC # 4.1 TIFINAGH MODIFIER LETTER LABIALIZATION MARK +2E9F ; Not_NFKC # 3.0 CJK RADICAL MOTHER +2EF3 ; Not_NFKC # 3.0 CJK RADICAL C-SIMPLIFIED TURTLE +2F00..2FD5 ; Not_NFKC # 3.0 [214] KANGXI RADICAL ONE..KANGXI RADICAL FLUTE +3000 ; Not_NFKC # 1.1 IDEOGRAPHIC SPACE +3036 ; Not_NFKC # 1.1 CIRCLED POSTAL MARK +3038..303A ; Not_NFKC # 3.0 [3] HANGZHOU NUMERAL TEN..HANGZHOU NUMERAL THIRTY +309B..309C ; Not_NFKC # 1.1 [2] KATAKANA-HIRAGANA VOICED SOUND MARK..KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK +309F ; Not_NFKC # 3.2 HIRAGANA DIGRAPH YORI +30FF ; Not_NFKC # 3.2 KATAKANA DIGRAPH KOTO +3131..3163 ; Not_NFKC # 1.1 [51] HANGUL LETTER KIYEOK..HANGUL LETTER I +3165..318E ; Not_NFKC # 1.1 [42] HANGUL LETTER SSANGNIEUN..HANGUL LETTER ARAEAE +3192..319F ; Not_NFKC # 1.1 [14] IDEOGRAPHIC ANNOTATION ONE MARK..IDEOGRAPHIC ANNOTATION MAN MARK +3200..321C ; Not_NFKC # 1.1 [29] PARENTHESIZED HANGUL KIYEOK..PARENTHESIZED HANGUL CIEUC U +321D..321E ; Not_NFKC # 4.0 [2] PARENTHESIZED KOREAN CHARACTER OJEON..PARENTHESIZED KOREAN CHARACTER O HU +3220..3243 ; Not_NFKC # 1.1 [36] PARENTHESIZED IDEOGRAPH ONE..PARENTHESIZED IDEOGRAPH REACH +3244..3247 ; Not_NFKC # 5.2 [4] CIRCLED IDEOGRAPH QUESTION..CIRCLED IDEOGRAPH KOTO +3250 ; Not_NFKC # 4.0 PARTNERSHIP SIGN +3251..325F ; Not_NFKC # 3.2 [15] CIRCLED NUMBER TWENTY ONE..CIRCLED NUMBER THIRTY FIVE +3260..327B ; Not_NFKC # 1.1 [28] CIRCLED HANGUL KIYEOK..CIRCLED HANGUL HIEUH A +327C..327D ; Not_NFKC # 4.0 [2] CIRCLED KOREAN CHARACTER CHAMKO..CIRCLED KOREAN CHARACTER JUEUI +327E ; Not_NFKC # 4.1 CIRCLED HANGUL IEUNG U +3280..32B0 ; Not_NFKC # 1.1 [49] CIRCLED IDEOGRAPH ONE..CIRCLED IDEOGRAPH NIGHT +32B1..32BF ; Not_NFKC # 3.2 [15] CIRCLED NUMBER THIRTY SIX..CIRCLED NUMBER FIFTY +32C0..32CB ; Not_NFKC # 1.1 [12] IDEOGRAPHIC TELEGRAPH SYMBOL FOR JANUARY..IDEOGRAPHIC TELEGRAPH SYMBOL FOR DECEMBER +32CC..32CF ; Not_NFKC # 4.0 [4] SQUARE HG..LIMITED LIABILITY SIGN +32D0..32FE ; Not_NFKC # 1.1 [47] CIRCLED KATAKANA A..CIRCLED KATAKANA WO +32FF ; Not_NFKC # 12.1 SQUARE ERA NAME REIWA +3300..3376 ; Not_NFKC # 1.1 [119] SQUARE APAATO..SQUARE PC +3377..337A ; Not_NFKC # 4.0 [4] SQUARE DM..SQUARE IU +337B..33DD ; Not_NFKC # 1.1 [99] SQUARE ERA NAME HEISEI..SQUARE WB +33DE..33DF ; Not_NFKC # 4.0 [2] SQUARE V OVER M..SQUARE A OVER M +33E0..33FE ; Not_NFKC # 1.1 [31] IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY ONE..IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY THIRTY-ONE +33FF ; Not_NFKC # 4.0 SQUARE GAL +A69C..A69D ; Not_NFKC # 7.0 [2] MODIFIER LETTER CYRILLIC HARD SIGN..MODIFIER LETTER CYRILLIC SOFT SIGN +A770 ; Not_NFKC # 5.1 MODIFIER LETTER US +A7F1 ; Not_NFKC # 17.0 MODIFIER LETTER CAPITAL S +A7F2..A7F4 ; Not_NFKC # 14.0 [3] MODIFIER LETTER CAPITAL C..MODIFIER LETTER CAPITAL Q +A7F8..A7F9 ; Not_NFKC # 6.1 [2] MODIFIER LETTER CAPITAL H WITH STROKE..MODIFIER LETTER SMALL LIGATURE OE +AB5C..AB5F ; Not_NFKC # 7.0 [4] MODIFIER LETTER SMALL HENG..MODIFIER LETTER SMALL U WITH LEFT HOOK +AB69 ; Not_NFKC # 13.0 MODIFIER LETTER SMALL TURNED W +F900..FA0D ; Not_NFKC # 1.1 [270] CJK COMPATIBILITY IDEOGRAPH-F900..CJK COMPATIBILITY IDEOGRAPH-FA0D +FA10 ; Not_NFKC # 1.1 CJK COMPATIBILITY IDEOGRAPH-FA10 +FA12 ; Not_NFKC # 1.1 CJK COMPATIBILITY IDEOGRAPH-FA12 +FA15..FA1E ; Not_NFKC # 1.1 [10] CJK COMPATIBILITY IDEOGRAPH-FA15..CJK COMPATIBILITY IDEOGRAPH-FA1E +FA20 ; Not_NFKC # 1.1 CJK COMPATIBILITY IDEOGRAPH-FA20 +FA22 ; Not_NFKC # 1.1 CJK COMPATIBILITY IDEOGRAPH-FA22 +FA25..FA26 ; Not_NFKC # 1.1 [2] CJK COMPATIBILITY IDEOGRAPH-FA25..CJK COMPATIBILITY IDEOGRAPH-FA26 +FA2A..FA2D ; Not_NFKC # 1.1 [4] CJK COMPATIBILITY IDEOGRAPH-FA2A..CJK COMPATIBILITY IDEOGRAPH-FA2D +FA2E..FA2F ; Not_NFKC # 6.1 [2] CJK COMPATIBILITY IDEOGRAPH-FA2E..CJK COMPATIBILITY IDEOGRAPH-FA2F +FA30..FA6A ; Not_NFKC # 3.2 [59] CJK COMPATIBILITY IDEOGRAPH-FA30..CJK COMPATIBILITY IDEOGRAPH-FA6A +FA6B..FA6D ; Not_NFKC # 5.2 [3] CJK COMPATIBILITY IDEOGRAPH-FA6B..CJK COMPATIBILITY IDEOGRAPH-FA6D +FA70..FAD9 ; Not_NFKC # 4.1 [106] CJK COMPATIBILITY IDEOGRAPH-FA70..CJK COMPATIBILITY IDEOGRAPH-FAD9 +FB00..FB06 ; Not_NFKC # 1.1 [7] LATIN SMALL LIGATURE FF..LATIN SMALL LIGATURE ST +FB13..FB17 ; Not_NFKC # 1.1 [5] ARMENIAN SMALL LIGATURE MEN NOW..ARMENIAN SMALL LIGATURE MEN XEH +FB1D ; Not_NFKC # 3.0 HEBREW LETTER YOD WITH HIRIQ +FB1F..FB36 ; Not_NFKC # 1.1 [24] HEBREW LIGATURE YIDDISH YOD YOD PATAH..HEBREW LETTER ZAYIN WITH DAGESH +FB38..FB3C ; Not_NFKC # 1.1 [5] HEBREW LETTER TET WITH DAGESH..HEBREW LETTER LAMED WITH DAGESH +FB3E ; Not_NFKC # 1.1 HEBREW LETTER MEM WITH DAGESH +FB40..FB41 ; Not_NFKC # 1.1 [2] HEBREW LETTER NUN WITH DAGESH..HEBREW LETTER SAMEKH WITH DAGESH +FB43..FB44 ; Not_NFKC # 1.1 [2] HEBREW LETTER FINAL PE WITH DAGESH..HEBREW LETTER PE WITH DAGESH +FB46..FBB1 ; Not_NFKC # 1.1 [108] HEBREW LETTER TSADI WITH DAGESH..ARABIC LETTER YEH BARREE WITH HAMZA ABOVE FINAL FORM +FBD3..FD3D ; Not_NFKC # 1.1 [363] ARABIC LETTER NG ISOLATED FORM..ARABIC LIGATURE ALEF WITH FATHATAN ISOLATED FORM +FD50..FD8F ; Not_NFKC # 1.1 [64] ARABIC LIGATURE TEH WITH JEEM WITH MEEM INITIAL FORM..ARABIC LIGATURE MEEM WITH KHAH WITH MEEM INITIAL FORM +FD92..FDC7 ; Not_NFKC # 1.1 [54] ARABIC LIGATURE MEEM WITH JEEM WITH KHAH INITIAL FORM..ARABIC LIGATURE NOON WITH JEEM WITH YEH FINAL FORM +FDF0..FDFB ; Not_NFKC # 1.1 [12] ARABIC LIGATURE SALLA USED AS KORANIC STOP SIGN ISOLATED FORM..ARABIC LIGATURE JALLAJALALOUHOU +FDFC ; Not_NFKC # 3.2 RIAL SIGN +FE10..FE19 ; Not_NFKC # 4.1 [10] PRESENTATION FORM FOR VERTICAL COMMA..PRESENTATION FORM FOR VERTICAL HORIZONTAL ELLIPSIS +FE30..FE44 ; Not_NFKC # 1.1 [21] PRESENTATION FORM FOR VERTICAL TWO DOT LEADER..PRESENTATION FORM FOR VERTICAL RIGHT WHITE CORNER BRACKET +FE47..FE48 ; Not_NFKC # 4.0 [2] PRESENTATION FORM FOR VERTICAL LEFT SQUARE BRACKET..PRESENTATION FORM FOR VERTICAL RIGHT SQUARE BRACKET +FE49..FE52 ; Not_NFKC # 1.1 [10] DASHED OVERLINE..SMALL FULL STOP +FE54..FE66 ; Not_NFKC # 1.1 [19] SMALL SEMICOLON..SMALL EQUALS SIGN +FE68..FE6B ; Not_NFKC # 1.1 [4] SMALL REVERSE SOLIDUS..SMALL COMMERCIAL AT +FE70..FE72 ; Not_NFKC # 1.1 [3] ARABIC FATHATAN ISOLATED FORM..ARABIC DAMMATAN ISOLATED FORM +FE74 ; Not_NFKC # 1.1 ARABIC KASRATAN ISOLATED FORM +FE76..FEFC ; Not_NFKC # 1.1 [135] ARABIC FATHA ISOLATED FORM..ARABIC LIGATURE LAM WITH ALEF FINAL FORM +FF01..FF5E ; Not_NFKC # 1.1 [94] FULLWIDTH EXCLAMATION MARK..FULLWIDTH TILDE +FF5F..FF60 ; Not_NFKC # 3.2 [2] FULLWIDTH LEFT WHITE PARENTHESIS..FULLWIDTH RIGHT WHITE PARENTHESIS +FF61..FF9F ; Not_NFKC # 1.1 [63] HALFWIDTH IDEOGRAPHIC FULL STOP..HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK +FFA1..FFBE ; Not_NFKC # 1.1 [30] HALFWIDTH HANGUL LETTER KIYEOK..HALFWIDTH HANGUL LETTER HIEUH +FFC2..FFC7 ; Not_NFKC # 1.1 [6] HALFWIDTH HANGUL LETTER A..HALFWIDTH HANGUL LETTER E +FFCA..FFCF ; Not_NFKC # 1.1 [6] HALFWIDTH HANGUL LETTER YEO..HALFWIDTH HANGUL LETTER OE +FFD2..FFD7 ; Not_NFKC # 1.1 [6] HALFWIDTH HANGUL LETTER YO..HALFWIDTH HANGUL LETTER YU +FFDA..FFDC ; Not_NFKC # 1.1 [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGUL LETTER I +FFE0..FFE6 ; Not_NFKC # 1.1 [7] FULLWIDTH CENT SIGN..FULLWIDTH WON SIGN +FFE8..FFEE ; Not_NFKC # 1.1 [7] HALFWIDTH FORMS LIGHT VERTICAL..HALFWIDTH WHITE CIRCLE +10781..10785 ; Not_NFKC # 14.0 [5] MODIFIER LETTER SUPERSCRIPT TRIANGULAR COLON..MODIFIER LETTER SMALL B WITH HOOK +10787..107B0 ; Not_NFKC # 14.0 [42] MODIFIER LETTER SMALL DZ DIGRAPH..MODIFIER LETTER SMALL V WITH RIGHT HOOK +107B2..107BA ; Not_NFKC # 14.0 [9] MODIFIER LETTER SMALL CAPITAL Y..MODIFIER LETTER SMALL S WITH CURL +1CCD6..1CCF9 ; Not_NFKC # 16.0 [36] OUTLINED LATIN CAPITAL LETTER A..OUTLINED DIGIT NINE +1D15E..1D164 ; Not_NFKC # 3.1 [7] MUSICAL SYMBOL HALF NOTE..MUSICAL SYMBOL ONE HUNDRED TWENTY-EIGHTH NOTE +1D1BB..1D1C0 ; Not_NFKC # 3.1 [6] MUSICAL SYMBOL MINIMA..MUSICAL SYMBOL FUSA BLACK +1D400..1D454 ; Not_NFKC # 3.1 [85] MATHEMATICAL BOLD CAPITAL A..MATHEMATICAL ITALIC SMALL G +1D456..1D49C ; Not_NFKC # 3.1 [71] MATHEMATICAL ITALIC SMALL I..MATHEMATICAL SCRIPT CAPITAL A +1D49E..1D49F ; Not_NFKC # 3.1 [2] MATHEMATICAL SCRIPT CAPITAL C..MATHEMATICAL SCRIPT CAPITAL D +1D4A2 ; Not_NFKC # 3.1 MATHEMATICAL SCRIPT CAPITAL G +1D4A5..1D4A6 ; Not_NFKC # 3.1 [2] MATHEMATICAL SCRIPT CAPITAL J..MATHEMATICAL SCRIPT CAPITAL K +1D4A9..1D4AC ; Not_NFKC # 3.1 [4] MATHEMATICAL SCRIPT CAPITAL N..MATHEMATICAL SCRIPT CAPITAL Q +1D4AE..1D4B9 ; Not_NFKC # 3.1 [12] MATHEMATICAL SCRIPT CAPITAL S..MATHEMATICAL SCRIPT SMALL D +1D4BB ; Not_NFKC # 3.1 MATHEMATICAL SCRIPT SMALL F +1D4BD..1D4C0 ; Not_NFKC # 3.1 [4] MATHEMATICAL SCRIPT SMALL H..MATHEMATICAL SCRIPT SMALL K +1D4C1 ; Not_NFKC # 4.0 MATHEMATICAL SCRIPT SMALL L +1D4C2..1D4C3 ; Not_NFKC # 3.1 [2] MATHEMATICAL SCRIPT SMALL M..MATHEMATICAL SCRIPT SMALL N +1D4C5..1D505 ; Not_NFKC # 3.1 [65] MATHEMATICAL SCRIPT SMALL P..MATHEMATICAL FRAKTUR CAPITAL B +1D507..1D50A ; Not_NFKC # 3.1 [4] MATHEMATICAL FRAKTUR CAPITAL D..MATHEMATICAL FRAKTUR CAPITAL G +1D50D..1D514 ; Not_NFKC # 3.1 [8] MATHEMATICAL FRAKTUR CAPITAL J..MATHEMATICAL FRAKTUR CAPITAL Q +1D516..1D51C ; Not_NFKC # 3.1 [7] MATHEMATICAL FRAKTUR CAPITAL S..MATHEMATICAL FRAKTUR CAPITAL Y +1D51E..1D539 ; Not_NFKC # 3.1 [28] MATHEMATICAL FRAKTUR SMALL A..MATHEMATICAL DOUBLE-STRUCK CAPITAL B +1D53B..1D53E ; Not_NFKC # 3.1 [4] MATHEMATICAL DOUBLE-STRUCK CAPITAL D..MATHEMATICAL DOUBLE-STRUCK CAPITAL G +1D540..1D544 ; Not_NFKC # 3.1 [5] MATHEMATICAL DOUBLE-STRUCK CAPITAL I..MATHEMATICAL DOUBLE-STRUCK CAPITAL M +1D546 ; Not_NFKC # 3.1 MATHEMATICAL DOUBLE-STRUCK CAPITAL O +1D54A..1D550 ; Not_NFKC # 3.1 [7] MATHEMATICAL DOUBLE-STRUCK CAPITAL S..MATHEMATICAL DOUBLE-STRUCK CAPITAL Y +1D552..1D6A3 ; Not_NFKC # 3.1 [338] MATHEMATICAL DOUBLE-STRUCK SMALL A..MATHEMATICAL MONOSPACE SMALL Z +1D6A4..1D6A5 ; Not_NFKC # 4.1 [2] MATHEMATICAL ITALIC SMALL DOTLESS I..MATHEMATICAL ITALIC SMALL DOTLESS J +1D6A8..1D7C9 ; Not_NFKC # 3.1 [290] MATHEMATICAL BOLD CAPITAL ALPHA..MATHEMATICAL SANS-SERIF BOLD ITALIC PI SYMBOL +1D7CA..1D7CB ; Not_NFKC # 5.0 [2] MATHEMATICAL BOLD CAPITAL DIGAMMA..MATHEMATICAL BOLD SMALL DIGAMMA +1D7CE..1D7FF ; Not_NFKC # 3.1 [50] MATHEMATICAL BOLD DIGIT ZERO..MATHEMATICAL MONOSPACE DIGIT NINE +1E030..1E06D ; Not_NFKC # 15.0 [62] MODIFIER LETTER CYRILLIC SMALL A..MODIFIER LETTER CYRILLIC SMALL STRAIGHT U WITH STROKE +1EE00..1EE03 ; Not_NFKC # 6.1 [4] ARABIC MATHEMATICAL ALEF..ARABIC MATHEMATICAL DAL +1EE05..1EE1F ; Not_NFKC # 6.1 [27] ARABIC MATHEMATICAL WAW..ARABIC MATHEMATICAL DOTLESS QAF +1EE21..1EE22 ; Not_NFKC # 6.1 [2] ARABIC MATHEMATICAL INITIAL BEH..ARABIC MATHEMATICAL INITIAL JEEM +1EE24 ; Not_NFKC # 6.1 ARABIC MATHEMATICAL INITIAL HEH +1EE27 ; Not_NFKC # 6.1 ARABIC MATHEMATICAL INITIAL HAH +1EE29..1EE32 ; Not_NFKC # 6.1 [10] ARABIC MATHEMATICAL INITIAL YEH..ARABIC MATHEMATICAL INITIAL QAF +1EE34..1EE37 ; Not_NFKC # 6.1 [4] ARABIC MATHEMATICAL INITIAL SHEEN..ARABIC MATHEMATICAL INITIAL KHAH +1EE39 ; Not_NFKC # 6.1 ARABIC MATHEMATICAL INITIAL DAD +1EE3B ; Not_NFKC # 6.1 ARABIC MATHEMATICAL INITIAL GHAIN +1EE42 ; Not_NFKC # 6.1 ARABIC MATHEMATICAL TAILED JEEM +1EE47 ; Not_NFKC # 6.1 ARABIC MATHEMATICAL TAILED HAH +1EE49 ; Not_NFKC # 6.1 ARABIC MATHEMATICAL TAILED YEH +1EE4B ; Not_NFKC # 6.1 ARABIC MATHEMATICAL TAILED LAM +1EE4D..1EE4F ; Not_NFKC # 6.1 [3] ARABIC MATHEMATICAL TAILED NOON..ARABIC MATHEMATICAL TAILED AIN +1EE51..1EE52 ; Not_NFKC # 6.1 [2] ARABIC MATHEMATICAL TAILED SAD..ARABIC MATHEMATICAL TAILED QAF +1EE54 ; Not_NFKC # 6.1 ARABIC MATHEMATICAL TAILED SHEEN +1EE57 ; Not_NFKC # 6.1 ARABIC MATHEMATICAL TAILED KHAH +1EE59 ; Not_NFKC # 6.1 ARABIC MATHEMATICAL TAILED DAD +1EE5B ; Not_NFKC # 6.1 ARABIC MATHEMATICAL TAILED GHAIN +1EE5D ; Not_NFKC # 6.1 ARABIC MATHEMATICAL TAILED DOTLESS NOON +1EE5F ; Not_NFKC # 6.1 ARABIC MATHEMATICAL TAILED DOTLESS QAF +1EE61..1EE62 ; Not_NFKC # 6.1 [2] ARABIC MATHEMATICAL STRETCHED BEH..ARABIC MATHEMATICAL STRETCHED JEEM +1EE64 ; Not_NFKC # 6.1 ARABIC MATHEMATICAL STRETCHED HEH +1EE67..1EE6A ; Not_NFKC # 6.1 [4] ARABIC MATHEMATICAL STRETCHED HAH..ARABIC MATHEMATICAL STRETCHED KAF +1EE6C..1EE72 ; Not_NFKC # 6.1 [7] ARABIC MATHEMATICAL STRETCHED MEEM..ARABIC MATHEMATICAL STRETCHED QAF +1EE74..1EE77 ; Not_NFKC # 6.1 [4] ARABIC MATHEMATICAL STRETCHED SHEEN..ARABIC MATHEMATICAL STRETCHED KHAH +1EE79..1EE7C ; Not_NFKC # 6.1 [4] ARABIC MATHEMATICAL STRETCHED DAD..ARABIC MATHEMATICAL STRETCHED DOTLESS BEH +1EE7E ; Not_NFKC # 6.1 ARABIC MATHEMATICAL STRETCHED DOTLESS FEH +1EE80..1EE89 ; Not_NFKC # 6.1 [10] ARABIC MATHEMATICAL LOOPED ALEF..ARABIC MATHEMATICAL LOOPED YEH +1EE8B..1EE9B ; Not_NFKC # 6.1 [17] ARABIC MATHEMATICAL LOOPED LAM..ARABIC MATHEMATICAL LOOPED GHAIN +1EEA1..1EEA3 ; Not_NFKC # 6.1 [3] ARABIC MATHEMATICAL DOUBLE-STRUCK BEH..ARABIC MATHEMATICAL DOUBLE-STRUCK DAL +1EEA5..1EEA9 ; Not_NFKC # 6.1 [5] ARABIC MATHEMATICAL DOUBLE-STRUCK WAW..ARABIC MATHEMATICAL DOUBLE-STRUCK YEH +1EEAB..1EEBB ; Not_NFKC # 6.1 [17] ARABIC MATHEMATICAL DOUBLE-STRUCK LAM..ARABIC MATHEMATICAL DOUBLE-STRUCK GHAIN +1F100..1F10A ; Not_NFKC # 5.2 [11] DIGIT ZERO FULL STOP..DIGIT NINE COMMA +1F110..1F12E ; Not_NFKC # 5.2 [31] PARENTHESIZED LATIN CAPITAL LETTER A..CIRCLED WZ +1F130 ; Not_NFKC # 6.0 SQUARED LATIN CAPITAL LETTER A +1F131 ; Not_NFKC # 5.2 SQUARED LATIN CAPITAL LETTER B +1F132..1F13C ; Not_NFKC # 6.0 [11] SQUARED LATIN CAPITAL LETTER C..SQUARED LATIN CAPITAL LETTER M +1F13D ; Not_NFKC # 5.2 SQUARED LATIN CAPITAL LETTER N +1F13E ; Not_NFKC # 6.0 SQUARED LATIN CAPITAL LETTER O +1F13F ; Not_NFKC # 5.2 SQUARED LATIN CAPITAL LETTER P +1F140..1F141 ; Not_NFKC # 6.0 [2] SQUARED LATIN CAPITAL LETTER Q..SQUARED LATIN CAPITAL LETTER R +1F142 ; Not_NFKC # 5.2 SQUARED LATIN CAPITAL LETTER S +1F143..1F145 ; Not_NFKC # 6.0 [3] SQUARED LATIN CAPITAL LETTER T..SQUARED LATIN CAPITAL LETTER V +1F146 ; Not_NFKC # 5.2 SQUARED LATIN CAPITAL LETTER W +1F147..1F149 ; Not_NFKC # 6.0 [3] SQUARED LATIN CAPITAL LETTER X..SQUARED LATIN CAPITAL LETTER Z +1F14A..1F14E ; Not_NFKC # 5.2 [5] SQUARED HV..SQUARED PPV +1F14F ; Not_NFKC # 6.0 SQUARED WC +1F16A..1F16B ; Not_NFKC # 6.1 [2] RAISED MC SIGN..RAISED MD SIGN +1F16C ; Not_NFKC # 12.0 RAISED MR SIGN +1F190 ; Not_NFKC # 5.2 SQUARE DJ +1F200 ; Not_NFKC # 5.2 SQUARE HIRAGANA HOKA +1F201..1F202 ; Not_NFKC # 6.0 [2] SQUARED KATAKANA KOKO..SQUARED KATAKANA SA +1F210..1F231 ; Not_NFKC # 5.2 [34] SQUARED CJK UNIFIED IDEOGRAPH-624B..SQUARED CJK UNIFIED IDEOGRAPH-6253 +1F232..1F23A ; Not_NFKC # 6.0 [9] SQUARED CJK UNIFIED IDEOGRAPH-7981..SQUARED CJK UNIFIED IDEOGRAPH-55B6 +1F23B ; Not_NFKC # 9.0 SQUARED CJK UNIFIED IDEOGRAPH-914D +1F240..1F248 ; Not_NFKC # 5.2 [9] TORTOISE SHELL BRACKETED CJK UNIFIED IDEOGRAPH-672C..TORTOISE SHELL BRACKETED CJK UNIFIED IDEOGRAPH-6557 +1F250..1F251 ; Not_NFKC # 6.0 [2] CIRCLED IDEOGRAPH ADVANTAGE..CIRCLED IDEOGRAPH ACCEPT +1FBF0..1FBF9 ; Not_NFKC # 13.0 [10] SEGMENTED DIGIT ZERO..SEGMENTED DIGIT NINE +2F800..2FA1D ; Not_NFKC # 3.1 [542] CJK COMPATIBILITY IDEOGRAPH-2F800..CJK COMPATIBILITY IDEOGRAPH-2FA1D + +# Total code points: 4958 + +# Identifier_Type: Default_Ignorable + +00AD ; Default_Ignorable # 1.1 SOFT HYPHEN +034F ; Default_Ignorable # 3.2 COMBINING GRAPHEME JOINER +061C ; Default_Ignorable # 6.3 ARABIC LETTER MARK +115F..1160 ; Default_Ignorable # 1.1 [2] HANGUL CHOSEONG FILLER..HANGUL JUNGSEONG FILLER +17B4..17B5 ; Default_Ignorable # 3.0 [2] KHMER VOWEL INHERENT AQ..KHMER VOWEL INHERENT AA +180B..180D ; Default_Ignorable # 3.0 [3] MONGOLIAN FREE VARIATION SELECTOR ONE..MONGOLIAN FREE VARIATION SELECTOR THREE +180E ; Default_Ignorable # 3.0 MONGOLIAN VOWEL SEPARATOR +180F ; Default_Ignorable # 14.0 MONGOLIAN FREE VARIATION SELECTOR FOUR +200B..200F ; Default_Ignorable # 1.1 [5] ZERO WIDTH SPACE..RIGHT-TO-LEFT MARK +202A..202E ; Default_Ignorable # 1.1 [5] LEFT-TO-RIGHT EMBEDDING..RIGHT-TO-LEFT OVERRIDE +2060..2063 ; Default_Ignorable # 3.2 [4] WORD JOINER..INVISIBLE SEPARATOR +2064 ; Default_Ignorable # 5.1 INVISIBLE PLUS +2066..2069 ; Default_Ignorable # 6.3 [4] LEFT-TO-RIGHT ISOLATE..POP DIRECTIONAL ISOLATE +3164 ; Default_Ignorable # 1.1 HANGUL FILLER +FE00..FE0F ; Default_Ignorable # 3.2 [16] VARIATION SELECTOR-1..VARIATION SELECTOR-16 +FEFF ; Default_Ignorable # 1.1 ZERO WIDTH NO-BREAK SPACE +FFA0 ; Default_Ignorable # 1.1 HALFWIDTH HANGUL FILLER +1BCA0..1BCA3 ; Default_Ignorable # 7.0 [4] SHORTHAND FORMAT LETTER OVERLAP..SHORTHAND FORMAT UP STEP +1D173..1D17A ; Default_Ignorable # 3.1 [8] MUSICAL SYMBOL BEGIN BEAM..MUSICAL SYMBOL END PHRASE +E0020..E007F ; Default_Ignorable # 3.1 [96] TAG SPACE..CANCEL TAG +E0100..E01EF ; Default_Ignorable # 4.0 [240] VARIATION SELECTOR-17..VARIATION SELECTOR-256 + +# Total code points: 398 + +# Identifier_Type: Deprecated + +0149 ; Deprecated # 1.1 LATIN SMALL LETTER N PRECEDED BY APOSTROPHE +0673 ; Deprecated # 1.1 ARABIC LETTER ALEF WITH WAVY HAMZA BELOW +0F77 ; Deprecated # 2.0 TIBETAN VOWEL SIGN VOCALIC RR +0F79 ; Deprecated # 2.0 TIBETAN VOWEL SIGN VOCALIC LL +17A3..17A4 ; Deprecated # 3.0 [2] KHMER INDEPENDENT VOWEL QAQ..KHMER INDEPENDENT VOWEL QAA +206A..206F ; Deprecated # 1.1 [6] INHIBIT SYMMETRIC SWAPPING..NOMINAL DIGIT SHAPES +2329..232A ; Deprecated # 1.1 [2] LEFT-POINTING ANGLE BRACKET..RIGHT-POINTING ANGLE BRACKET +E0001 ; Deprecated # 3.1 LANGUAGE TAG + +# Total code points: 15 diff --git a/lib/elixir/unicode/PropList.txt b/lib/elixir/unicode/PropList.txt new file mode 100644 index 00000000000..e64b4224d72 --- /dev/null +++ b/lib/elixir/unicode/PropList.txt @@ -0,0 +1,1944 @@ +# PropList-17.0.0.txt +# Date: 2025-06-30, 06:19:01 GMT +# © 2025 Unicode®, Inc. +# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. +# For terms of use and license, see https://www.unicode.org/terms_of_use.html +# +# Unicode Character Database +# For documentation, see https://www.unicode.org/reports/tr44/ + +# ================================================ + +0009..000D ; White_Space # Cc [5] .. +0020 ; White_Space # Zs SPACE +0085 ; White_Space # Cc +00A0 ; White_Space # Zs NO-BREAK SPACE +1680 ; White_Space # Zs OGHAM SPACE MARK +2000..200A ; White_Space # Zs [11] EN QUAD..HAIR SPACE +2028 ; White_Space # Zl LINE SEPARATOR +2029 ; White_Space # Zp PARAGRAPH SEPARATOR +202F ; White_Space # Zs NARROW NO-BREAK SPACE +205F ; White_Space # Zs MEDIUM MATHEMATICAL SPACE +3000 ; White_Space # Zs IDEOGRAPHIC SPACE + +# Total code points: 25 + +# ================================================ + +061C ; Bidi_Control # Cf ARABIC LETTER MARK +200E..200F ; Bidi_Control # Cf [2] LEFT-TO-RIGHT MARK..RIGHT-TO-LEFT MARK +202A..202E ; Bidi_Control # Cf [5] LEFT-TO-RIGHT EMBEDDING..RIGHT-TO-LEFT OVERRIDE +2066..2069 ; Bidi_Control # Cf [4] LEFT-TO-RIGHT ISOLATE..POP DIRECTIONAL ISOLATE + +# Total code points: 12 + +# ================================================ + +200C..200D ; Join_Control # Cf [2] ZERO WIDTH NON-JOINER..ZERO WIDTH JOINER + +# Total code points: 2 + +# ================================================ + +002D ; Dash # Pd HYPHEN-MINUS +058A ; Dash # Pd ARMENIAN HYPHEN +05BE ; Dash # Pd HEBREW PUNCTUATION MAQAF +1400 ; Dash # Pd CANADIAN SYLLABICS HYPHEN +1806 ; Dash # Pd MONGOLIAN TODO SOFT HYPHEN +2010..2015 ; Dash # Pd [6] HYPHEN..HORIZONTAL BAR +2053 ; Dash # Po SWUNG DASH +207B ; Dash # Sm SUPERSCRIPT MINUS +208B ; Dash # Sm SUBSCRIPT MINUS +2212 ; Dash # Sm MINUS SIGN +2E17 ; Dash # Pd DOUBLE OBLIQUE HYPHEN +2E1A ; Dash # Pd HYPHEN WITH DIAERESIS +2E3A..2E3B ; Dash # Pd [2] TWO-EM DASH..THREE-EM DASH +2E40 ; Dash # Pd DOUBLE HYPHEN +2E5D ; Dash # Pd OBLIQUE HYPHEN +301C ; Dash # Pd WAVE DASH +3030 ; Dash # Pd WAVY DASH +30A0 ; Dash # Pd KATAKANA-HIRAGANA DOUBLE HYPHEN +FE31..FE32 ; Dash # Pd [2] PRESENTATION FORM FOR VERTICAL EM DASH..PRESENTATION FORM FOR VERTICAL EN DASH +FE58 ; Dash # Pd SMALL EM DASH +FE63 ; Dash # Pd SMALL HYPHEN-MINUS +FF0D ; Dash # Pd FULLWIDTH HYPHEN-MINUS +10D6E ; Dash # Pd GARAY HYPHEN +10EAD ; Dash # Pd YEZIDI HYPHENATION MARK + +# Total code points: 31 + +# ================================================ + +002D ; Hyphen # Pd HYPHEN-MINUS +00AD ; Hyphen # Cf SOFT HYPHEN +058A ; Hyphen # Pd ARMENIAN HYPHEN +1806 ; Hyphen # Pd MONGOLIAN TODO SOFT HYPHEN +2010..2011 ; Hyphen # Pd [2] HYPHEN..NON-BREAKING HYPHEN +2E17 ; Hyphen # Pd DOUBLE OBLIQUE HYPHEN +30FB ; Hyphen # Po KATAKANA MIDDLE DOT +FE63 ; Hyphen # Pd SMALL HYPHEN-MINUS +FF0D ; Hyphen # Pd FULLWIDTH HYPHEN-MINUS +FF65 ; Hyphen # Po HALFWIDTH KATAKANA MIDDLE DOT + +# Total code points: 11 + +# ================================================ + +0022 ; Quotation_Mark # Po QUOTATION MARK +0027 ; Quotation_Mark # Po APOSTROPHE +00AB ; Quotation_Mark # Pi LEFT-POINTING DOUBLE ANGLE QUOTATION MARK +00BB ; Quotation_Mark # Pf RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK +2018 ; Quotation_Mark # Pi LEFT SINGLE QUOTATION MARK +2019 ; Quotation_Mark # Pf RIGHT SINGLE QUOTATION MARK +201A ; Quotation_Mark # Ps SINGLE LOW-9 QUOTATION MARK +201B..201C ; Quotation_Mark # Pi [2] SINGLE HIGH-REVERSED-9 QUOTATION MARK..LEFT DOUBLE QUOTATION MARK +201D ; Quotation_Mark # Pf RIGHT DOUBLE QUOTATION MARK +201E ; Quotation_Mark # Ps DOUBLE LOW-9 QUOTATION MARK +201F ; Quotation_Mark # Pi DOUBLE HIGH-REVERSED-9 QUOTATION MARK +2039 ; Quotation_Mark # Pi SINGLE LEFT-POINTING ANGLE QUOTATION MARK +203A ; Quotation_Mark # Pf SINGLE RIGHT-POINTING ANGLE QUOTATION MARK +2E42 ; Quotation_Mark # Ps DOUBLE LOW-REVERSED-9 QUOTATION MARK +300C ; Quotation_Mark # Ps LEFT CORNER BRACKET +300D ; Quotation_Mark # Pe RIGHT CORNER BRACKET +300E ; Quotation_Mark # Ps LEFT WHITE CORNER BRACKET +300F ; Quotation_Mark # Pe RIGHT WHITE CORNER BRACKET +301D ; Quotation_Mark # Ps REVERSED DOUBLE PRIME QUOTATION MARK +301E..301F ; Quotation_Mark # Pe [2] DOUBLE PRIME QUOTATION MARK..LOW DOUBLE PRIME QUOTATION MARK +FE41 ; Quotation_Mark # Ps PRESENTATION FORM FOR VERTICAL LEFT CORNER BRACKET +FE42 ; Quotation_Mark # Pe PRESENTATION FORM FOR VERTICAL RIGHT CORNER BRACKET +FE43 ; Quotation_Mark # Ps PRESENTATION FORM FOR VERTICAL LEFT WHITE CORNER BRACKET +FE44 ; Quotation_Mark # Pe PRESENTATION FORM FOR VERTICAL RIGHT WHITE CORNER BRACKET +FF02 ; Quotation_Mark # Po FULLWIDTH QUOTATION MARK +FF07 ; Quotation_Mark # Po FULLWIDTH APOSTROPHE +FF62 ; Quotation_Mark # Ps HALFWIDTH LEFT CORNER BRACKET +FF63 ; Quotation_Mark # Pe HALFWIDTH RIGHT CORNER BRACKET + +# Total code points: 30 + +# ================================================ + +0021 ; Terminal_Punctuation # Po EXCLAMATION MARK +002C ; Terminal_Punctuation # Po COMMA +002E ; Terminal_Punctuation # Po FULL STOP +003A..003B ; Terminal_Punctuation # Po [2] COLON..SEMICOLON +003F ; Terminal_Punctuation # Po QUESTION MARK +037E ; Terminal_Punctuation # Po GREEK QUESTION MARK +0387 ; Terminal_Punctuation # Po GREEK ANO TELEIA +0589 ; Terminal_Punctuation # Po ARMENIAN FULL STOP +05C3 ; Terminal_Punctuation # Po HEBREW PUNCTUATION SOF PASUQ +060C ; Terminal_Punctuation # Po ARABIC COMMA +061B ; Terminal_Punctuation # Po ARABIC SEMICOLON +061D..061F ; Terminal_Punctuation # Po [3] ARABIC END OF TEXT MARK..ARABIC QUESTION MARK +06D4 ; Terminal_Punctuation # Po ARABIC FULL STOP +0700..070A ; Terminal_Punctuation # Po [11] SYRIAC END OF PARAGRAPH..SYRIAC CONTRACTION +070C ; Terminal_Punctuation # Po SYRIAC HARKLEAN METOBELUS +07F8..07F9 ; Terminal_Punctuation # Po [2] NKO COMMA..NKO EXCLAMATION MARK +0830..0835 ; Terminal_Punctuation # Po [6] SAMARITAN PUNCTUATION NEQUDAA..SAMARITAN PUNCTUATION SHIYYAALAA +0837..083E ; Terminal_Punctuation # Po [8] SAMARITAN PUNCTUATION MELODIC QITSA..SAMARITAN PUNCTUATION ANNAAU +085E ; Terminal_Punctuation # Po MANDAIC PUNCTUATION +0964..0965 ; Terminal_Punctuation # Po [2] DEVANAGARI DANDA..DEVANAGARI DOUBLE DANDA +0E5A..0E5B ; Terminal_Punctuation # Po [2] THAI CHARACTER ANGKHANKHU..THAI CHARACTER KHOMUT +0F08 ; Terminal_Punctuation # Po TIBETAN MARK SBRUL SHAD +0F0D..0F12 ; Terminal_Punctuation # Po [6] TIBETAN MARK SHAD..TIBETAN MARK RGYA GRAM SHAD +104A..104B ; Terminal_Punctuation # Po [2] MYANMAR SIGN LITTLE SECTION..MYANMAR SIGN SECTION +1361..1368 ; Terminal_Punctuation # Po [8] ETHIOPIC WORDSPACE..ETHIOPIC PARAGRAPH SEPARATOR +166E ; Terminal_Punctuation # Po CANADIAN SYLLABICS FULL STOP +16EB..16ED ; Terminal_Punctuation # Po [3] RUNIC SINGLE PUNCTUATION..RUNIC CROSS PUNCTUATION +1735..1736 ; Terminal_Punctuation # Po [2] PHILIPPINE SINGLE PUNCTUATION..PHILIPPINE DOUBLE PUNCTUATION +17D4..17D6 ; Terminal_Punctuation # Po [3] KHMER SIGN KHAN..KHMER SIGN CAMNUC PII KUUH +17DA ; Terminal_Punctuation # Po KHMER SIGN KOOMUUT +1802..1805 ; Terminal_Punctuation # Po [4] MONGOLIAN COMMA..MONGOLIAN FOUR DOTS +1808..1809 ; Terminal_Punctuation # Po [2] MONGOLIAN MANCHU COMMA..MONGOLIAN MANCHU FULL STOP +1944..1945 ; Terminal_Punctuation # Po [2] LIMBU EXCLAMATION MARK..LIMBU QUESTION MARK +1AA8..1AAB ; Terminal_Punctuation # Po [4] TAI THAM SIGN KAAN..TAI THAM SIGN SATKAANKUU +1B4E..1B4F ; Terminal_Punctuation # Po [2] BALINESE INVERTED CARIK SIKI..BALINESE INVERTED CARIK PAREREN +1B5A..1B5B ; Terminal_Punctuation # Po [2] BALINESE PANTI..BALINESE PAMADA +1B5D..1B5F ; Terminal_Punctuation # Po [3] BALINESE CARIK PAMUNGKAH..BALINESE CARIK PAREREN +1B7D..1B7F ; Terminal_Punctuation # Po [3] BALINESE PANTI LANTANG..BALINESE PANTI BAWAK +1C3B..1C3F ; Terminal_Punctuation # Po [5] LEPCHA PUNCTUATION TA-ROL..LEPCHA PUNCTUATION TSHOOK +1C7E..1C7F ; Terminal_Punctuation # Po [2] OL CHIKI PUNCTUATION MUCAAD..OL CHIKI PUNCTUATION DOUBLE MUCAAD +2024 ; Terminal_Punctuation # Po ONE DOT LEADER +203C..203D ; Terminal_Punctuation # Po [2] DOUBLE EXCLAMATION MARK..INTERROBANG +2047..2049 ; Terminal_Punctuation # Po [3] DOUBLE QUESTION MARK..EXCLAMATION QUESTION MARK +2CF9..2CFB ; Terminal_Punctuation # Po [3] COPTIC OLD NUBIAN FULL STOP..COPTIC OLD NUBIAN INDIRECT QUESTION MARK +2E2E ; Terminal_Punctuation # Po REVERSED QUESTION MARK +2E3C ; Terminal_Punctuation # Po STENOGRAPHIC FULL STOP +2E41 ; Terminal_Punctuation # Po REVERSED COMMA +2E4C ; Terminal_Punctuation # Po MEDIEVAL COMMA +2E4E..2E4F ; Terminal_Punctuation # Po [2] PUNCTUS ELEVATUS MARK..CORNISH VERSE DIVIDER +2E53..2E54 ; Terminal_Punctuation # Po [2] MEDIEVAL EXCLAMATION MARK..MEDIEVAL QUESTION MARK +3001..3002 ; Terminal_Punctuation # Po [2] IDEOGRAPHIC COMMA..IDEOGRAPHIC FULL STOP +A4FE..A4FF ; Terminal_Punctuation # Po [2] LISU PUNCTUATION COMMA..LISU PUNCTUATION FULL STOP +A60D..A60F ; Terminal_Punctuation # Po [3] VAI COMMA..VAI QUESTION MARK +A6F3..A6F7 ; Terminal_Punctuation # Po [5] BAMUM FULL STOP..BAMUM QUESTION MARK +A876..A877 ; Terminal_Punctuation # Po [2] PHAGS-PA MARK SHAD..PHAGS-PA MARK DOUBLE SHAD +A8CE..A8CF ; Terminal_Punctuation # Po [2] SAURASHTRA DANDA..SAURASHTRA DOUBLE DANDA +A92F ; Terminal_Punctuation # Po KAYAH LI SIGN SHYA +A9C7..A9C9 ; Terminal_Punctuation # Po [3] JAVANESE PADA PANGKAT..JAVANESE PADA LUNGSI +AA5D..AA5F ; Terminal_Punctuation # Po [3] CHAM PUNCTUATION DANDA..CHAM PUNCTUATION TRIPLE DANDA +AADF ; Terminal_Punctuation # Po TAI VIET SYMBOL KOI KOI +AAF0..AAF1 ; Terminal_Punctuation # Po [2] MEETEI MAYEK CHEIKHAN..MEETEI MAYEK AHANG KHUDAM +ABEB ; Terminal_Punctuation # Po MEETEI MAYEK CHEIKHEI +FE12 ; Terminal_Punctuation # Po PRESENTATION FORM FOR VERTICAL IDEOGRAPHIC FULL STOP +FE15..FE16 ; Terminal_Punctuation # Po [2] PRESENTATION FORM FOR VERTICAL EXCLAMATION MARK..PRESENTATION FORM FOR VERTICAL QUESTION MARK +FE50..FE52 ; Terminal_Punctuation # Po [3] SMALL COMMA..SMALL FULL STOP +FE54..FE57 ; Terminal_Punctuation # Po [4] SMALL SEMICOLON..SMALL EXCLAMATION MARK +FF01 ; Terminal_Punctuation # Po FULLWIDTH EXCLAMATION MARK +FF0C ; Terminal_Punctuation # Po FULLWIDTH COMMA +FF0E ; Terminal_Punctuation # Po FULLWIDTH FULL STOP +FF1A..FF1B ; Terminal_Punctuation # Po [2] FULLWIDTH COLON..FULLWIDTH SEMICOLON +FF1F ; Terminal_Punctuation # Po FULLWIDTH QUESTION MARK +FF61 ; Terminal_Punctuation # Po HALFWIDTH IDEOGRAPHIC FULL STOP +FF64 ; Terminal_Punctuation # Po HALFWIDTH IDEOGRAPHIC COMMA +1039F ; Terminal_Punctuation # Po UGARITIC WORD DIVIDER +103D0 ; Terminal_Punctuation # Po OLD PERSIAN WORD DIVIDER +10857 ; Terminal_Punctuation # Po IMPERIAL ARAMAIC SECTION SIGN +1091F ; Terminal_Punctuation # Po PHOENICIAN WORD SEPARATOR +10A56..10A57 ; Terminal_Punctuation # Po [2] KHAROSHTHI PUNCTUATION DANDA..KHAROSHTHI PUNCTUATION DOUBLE DANDA +10AF0..10AF5 ; Terminal_Punctuation # Po [6] MANICHAEAN PUNCTUATION STAR..MANICHAEAN PUNCTUATION TWO DOTS +10B3A..10B3F ; Terminal_Punctuation # Po [6] TINY TWO DOTS OVER ONE DOT PUNCTUATION..LARGE ONE RING OVER TWO RINGS PUNCTUATION +10B99..10B9C ; Terminal_Punctuation # Po [4] PSALTER PAHLAVI SECTION MARK..PSALTER PAHLAVI FOUR DOTS WITH DOT +10F55..10F59 ; Terminal_Punctuation # Po [5] SOGDIAN PUNCTUATION TWO VERTICAL BARS..SOGDIAN PUNCTUATION HALF CIRCLE WITH DOT +10F86..10F89 ; Terminal_Punctuation # Po [4] OLD UYGHUR PUNCTUATION BAR..OLD UYGHUR PUNCTUATION FOUR DOTS +11047..1104D ; Terminal_Punctuation # Po [7] BRAHMI DANDA..BRAHMI PUNCTUATION LOTUS +110BE..110C1 ; Terminal_Punctuation # Po [4] KAITHI SECTION MARK..KAITHI DOUBLE DANDA +11141..11143 ; Terminal_Punctuation # Po [3] CHAKMA DANDA..CHAKMA QUESTION MARK +111C5..111C6 ; Terminal_Punctuation # Po [2] SHARADA DANDA..SHARADA DOUBLE DANDA +111CD ; Terminal_Punctuation # Po SHARADA SUTRA MARK +111DE..111DF ; Terminal_Punctuation # Po [2] SHARADA SECTION MARK-1..SHARADA SECTION MARK-2 +11238..1123C ; Terminal_Punctuation # Po [5] KHOJKI DANDA..KHOJKI DOUBLE SECTION MARK +112A9 ; Terminal_Punctuation # Po MULTANI SECTION MARK +113D4..113D5 ; Terminal_Punctuation # Po [2] TULU-TIGALARI DANDA..TULU-TIGALARI DOUBLE DANDA +1144B..1144D ; Terminal_Punctuation # Po [3] NEWA DANDA..NEWA COMMA +1145A..1145B ; Terminal_Punctuation # Po [2] NEWA DOUBLE COMMA..NEWA PLACEHOLDER MARK +115C2..115C5 ; Terminal_Punctuation # Po [4] SIDDHAM DANDA..SIDDHAM SEPARATOR BAR +115C9..115D7 ; Terminal_Punctuation # Po [15] SIDDHAM END OF TEXT MARK..SIDDHAM SECTION MARK WITH CIRCLES AND FOUR ENCLOSURES +11641..11642 ; Terminal_Punctuation # Po [2] MODI DANDA..MODI DOUBLE DANDA +1173C..1173E ; Terminal_Punctuation # Po [3] AHOM SIGN SMALL SECTION..AHOM SIGN RULAI +11944 ; Terminal_Punctuation # Po DIVES AKURU DOUBLE DANDA +11946 ; Terminal_Punctuation # Po DIVES AKURU END OF TEXT MARK +11A42..11A43 ; Terminal_Punctuation # Po [2] ZANABAZAR SQUARE MARK SHAD..ZANABAZAR SQUARE MARK DOUBLE SHAD +11A9B..11A9C ; Terminal_Punctuation # Po [2] SOYOMBO MARK SHAD..SOYOMBO MARK DOUBLE SHAD +11AA1..11AA2 ; Terminal_Punctuation # Po [2] SOYOMBO TERMINAL MARK-1..SOYOMBO TERMINAL MARK-2 +11C41..11C43 ; Terminal_Punctuation # Po [3] BHAIKSUKI DANDA..BHAIKSUKI WORD SEPARATOR +11C71 ; Terminal_Punctuation # Po MARCHEN MARK SHAD +11EF7..11EF8 ; Terminal_Punctuation # Po [2] MAKASAR PASSIMBANG..MAKASAR END OF SECTION +11F43..11F44 ; Terminal_Punctuation # Po [2] KAWI DANDA..KAWI DOUBLE DANDA +12470..12474 ; Terminal_Punctuation # Po [5] CUNEIFORM PUNCTUATION SIGN OLD ASSYRIAN WORD DIVIDER..CUNEIFORM PUNCTUATION SIGN DIAGONAL QUADCOLON +16A6E..16A6F ; Terminal_Punctuation # Po [2] MRO DANDA..MRO DOUBLE DANDA +16AF5 ; Terminal_Punctuation # Po BASSA VAH FULL STOP +16B37..16B39 ; Terminal_Punctuation # Po [3] PAHAWH HMONG SIGN VOS THOM..PAHAWH HMONG SIGN CIM CHEEM +16B44 ; Terminal_Punctuation # Po PAHAWH HMONG SIGN XAUS +16D6E..16D6F ; Terminal_Punctuation # Po [2] KIRAT RAI DANDA..KIRAT RAI DOUBLE DANDA +16E97..16E98 ; Terminal_Punctuation # Po [2] MEDEFAIDRIN COMMA..MEDEFAIDRIN FULL STOP +1BC9F ; Terminal_Punctuation # Po DUPLOYAN PUNCTUATION CHINOOK FULL STOP +1DA87..1DA8A ; Terminal_Punctuation # Po [4] SIGNWRITING COMMA..SIGNWRITING COLON + +# Total code points: 291 + +# ================================================ + +005E ; Other_Math # Sk CIRCUMFLEX ACCENT +03D0..03D2 ; Other_Math # L& [3] GREEK BETA SYMBOL..GREEK UPSILON WITH HOOK SYMBOL +03D5 ; Other_Math # L& GREEK PHI SYMBOL +03F0..03F1 ; Other_Math # L& [2] GREEK KAPPA SYMBOL..GREEK RHO SYMBOL +03F4..03F5 ; Other_Math # L& [2] GREEK CAPITAL THETA SYMBOL..GREEK LUNATE EPSILON SYMBOL +2016 ; Other_Math # Po DOUBLE VERTICAL LINE +2032..2034 ; Other_Math # Po [3] PRIME..TRIPLE PRIME +2040 ; Other_Math # Pc CHARACTER TIE +2061..2064 ; Other_Math # Cf [4] FUNCTION APPLICATION..INVISIBLE PLUS +207D ; Other_Math # Ps SUPERSCRIPT LEFT PARENTHESIS +207E ; Other_Math # Pe SUPERSCRIPT RIGHT PARENTHESIS +208D ; Other_Math # Ps SUBSCRIPT LEFT PARENTHESIS +208E ; Other_Math # Pe SUBSCRIPT RIGHT PARENTHESIS +20D0..20DC ; Other_Math # Mn [13] COMBINING LEFT HARPOON ABOVE..COMBINING FOUR DOTS ABOVE +20E1 ; Other_Math # Mn COMBINING LEFT RIGHT ARROW ABOVE +20E5..20E6 ; Other_Math # Mn [2] COMBINING REVERSE SOLIDUS OVERLAY..COMBINING DOUBLE VERTICAL STROKE OVERLAY +20EB..20EF ; Other_Math # Mn [5] COMBINING LONG DOUBLE SOLIDUS OVERLAY..COMBINING RIGHT ARROW BELOW +2102 ; Other_Math # L& DOUBLE-STRUCK CAPITAL C +2107 ; Other_Math # L& EULER CONSTANT +210A..2113 ; Other_Math # L& [10] SCRIPT SMALL G..SCRIPT SMALL L +2115 ; Other_Math # L& DOUBLE-STRUCK CAPITAL N +2119..211D ; Other_Math # L& [5] DOUBLE-STRUCK CAPITAL P..DOUBLE-STRUCK CAPITAL R +2124 ; Other_Math # L& DOUBLE-STRUCK CAPITAL Z +2128 ; Other_Math # L& BLACK-LETTER CAPITAL Z +2129 ; Other_Math # So TURNED GREEK SMALL LETTER IOTA +212C..212D ; Other_Math # L& [2] SCRIPT CAPITAL B..BLACK-LETTER CAPITAL C +212F..2131 ; Other_Math # L& [3] SCRIPT SMALL E..SCRIPT CAPITAL F +2133..2134 ; Other_Math # L& [2] SCRIPT CAPITAL M..SCRIPT SMALL O +2135..2138 ; Other_Math # Lo [4] ALEF SYMBOL..DALET SYMBOL +213C..213F ; Other_Math # L& [4] DOUBLE-STRUCK SMALL PI..DOUBLE-STRUCK CAPITAL PI +2145..2149 ; Other_Math # L& [5] DOUBLE-STRUCK ITALIC CAPITAL D..DOUBLE-STRUCK ITALIC SMALL J +2195..2199 ; Other_Math # So [5] UP DOWN ARROW..SOUTH WEST ARROW +219C..219F ; Other_Math # So [4] LEFTWARDS WAVE ARROW..UPWARDS TWO HEADED ARROW +21A1..21A2 ; Other_Math # So [2] DOWNWARDS TWO HEADED ARROW..LEFTWARDS ARROW WITH TAIL +21A4..21A5 ; Other_Math # So [2] LEFTWARDS ARROW FROM BAR..UPWARDS ARROW FROM BAR +21A7 ; Other_Math # So DOWNWARDS ARROW FROM BAR +21A9..21AD ; Other_Math # So [5] LEFTWARDS ARROW WITH HOOK..LEFT RIGHT WAVE ARROW +21B0..21B1 ; Other_Math # So [2] UPWARDS ARROW WITH TIP LEFTWARDS..UPWARDS ARROW WITH TIP RIGHTWARDS +21B6..21B7 ; Other_Math # So [2] ANTICLOCKWISE TOP SEMICIRCLE ARROW..CLOCKWISE TOP SEMICIRCLE ARROW +21BC..21CD ; Other_Math # So [18] LEFTWARDS HARPOON WITH BARB UPWARDS..LEFTWARDS DOUBLE ARROW WITH STROKE +21D0..21D1 ; Other_Math # So [2] LEFTWARDS DOUBLE ARROW..UPWARDS DOUBLE ARROW +21D3 ; Other_Math # So DOWNWARDS DOUBLE ARROW +21D5..21DB ; Other_Math # So [7] UP DOWN DOUBLE ARROW..RIGHTWARDS TRIPLE ARROW +21DD ; Other_Math # So RIGHTWARDS SQUIGGLE ARROW +21E4..21E5 ; Other_Math # So [2] LEFTWARDS ARROW TO BAR..RIGHTWARDS ARROW TO BAR +2308 ; Other_Math # Ps LEFT CEILING +2309 ; Other_Math # Pe RIGHT CEILING +230A ; Other_Math # Ps LEFT FLOOR +230B ; Other_Math # Pe RIGHT FLOOR +23B4..23B5 ; Other_Math # So [2] TOP SQUARE BRACKET..BOTTOM SQUARE BRACKET +23B7 ; Other_Math # So RADICAL SYMBOL BOTTOM +23D0 ; Other_Math # So VERTICAL LINE EXTENSION +23E2 ; Other_Math # So WHITE TRAPEZIUM +25A0..25A1 ; Other_Math # So [2] BLACK SQUARE..WHITE SQUARE +25AE..25B6 ; Other_Math # So [9] BLACK VERTICAL RECTANGLE..BLACK RIGHT-POINTING TRIANGLE +25BC..25C0 ; Other_Math # So [5] BLACK DOWN-POINTING TRIANGLE..BLACK LEFT-POINTING TRIANGLE +25C6..25C7 ; Other_Math # So [2] BLACK DIAMOND..WHITE DIAMOND +25CA..25CB ; Other_Math # So [2] LOZENGE..WHITE CIRCLE +25CF..25D3 ; Other_Math # So [5] BLACK CIRCLE..CIRCLE WITH UPPER HALF BLACK +25E2 ; Other_Math # So BLACK LOWER RIGHT TRIANGLE +25E4 ; Other_Math # So BLACK UPPER LEFT TRIANGLE +25E7..25EC ; Other_Math # So [6] SQUARE WITH LEFT HALF BLACK..WHITE UP-POINTING TRIANGLE WITH DOT +2605..2606 ; Other_Math # So [2] BLACK STAR..WHITE STAR +2640 ; Other_Math # So FEMALE SIGN +2642 ; Other_Math # So MALE SIGN +2660..2663 ; Other_Math # So [4] BLACK SPADE SUIT..BLACK CLUB SUIT +266D..266E ; Other_Math # So [2] MUSIC FLAT SIGN..MUSIC NATURAL SIGN +27C5 ; Other_Math # Ps LEFT S-SHAPED BAG DELIMITER +27C6 ; Other_Math # Pe RIGHT S-SHAPED BAG DELIMITER +27E6 ; Other_Math # Ps MATHEMATICAL LEFT WHITE SQUARE BRACKET +27E7 ; Other_Math # Pe MATHEMATICAL RIGHT WHITE SQUARE BRACKET +27E8 ; Other_Math # Ps MATHEMATICAL LEFT ANGLE BRACKET +27E9 ; Other_Math # Pe MATHEMATICAL RIGHT ANGLE BRACKET +27EA ; Other_Math # Ps MATHEMATICAL LEFT DOUBLE ANGLE BRACKET +27EB ; Other_Math # Pe MATHEMATICAL RIGHT DOUBLE ANGLE BRACKET +27EC ; Other_Math # Ps MATHEMATICAL LEFT WHITE TORTOISE SHELL BRACKET +27ED ; Other_Math # Pe MATHEMATICAL RIGHT WHITE TORTOISE SHELL BRACKET +27EE ; Other_Math # Ps MATHEMATICAL LEFT FLATTENED PARENTHESIS +27EF ; Other_Math # Pe MATHEMATICAL RIGHT FLATTENED PARENTHESIS +2983 ; Other_Math # Ps LEFT WHITE CURLY BRACKET +2984 ; Other_Math # Pe RIGHT WHITE CURLY BRACKET +2985 ; Other_Math # Ps LEFT WHITE PARENTHESIS +2986 ; Other_Math # Pe RIGHT WHITE PARENTHESIS +2987 ; Other_Math # Ps Z NOTATION LEFT IMAGE BRACKET +2988 ; Other_Math # Pe Z NOTATION RIGHT IMAGE BRACKET +2989 ; Other_Math # Ps Z NOTATION LEFT BINDING BRACKET +298A ; Other_Math # Pe Z NOTATION RIGHT BINDING BRACKET +298B ; Other_Math # Ps LEFT SQUARE BRACKET WITH UNDERBAR +298C ; Other_Math # Pe RIGHT SQUARE BRACKET WITH UNDERBAR +298D ; Other_Math # Ps LEFT SQUARE BRACKET WITH TICK IN TOP CORNER +298E ; Other_Math # Pe RIGHT SQUARE BRACKET WITH TICK IN BOTTOM CORNER +298F ; Other_Math # Ps LEFT SQUARE BRACKET WITH TICK IN BOTTOM CORNER +2990 ; Other_Math # Pe RIGHT SQUARE BRACKET WITH TICK IN TOP CORNER +2991 ; Other_Math # Ps LEFT ANGLE BRACKET WITH DOT +2992 ; Other_Math # Pe RIGHT ANGLE BRACKET WITH DOT +2993 ; Other_Math # Ps LEFT ARC LESS-THAN BRACKET +2994 ; Other_Math # Pe RIGHT ARC GREATER-THAN BRACKET +2995 ; Other_Math # Ps DOUBLE LEFT ARC GREATER-THAN BRACKET +2996 ; Other_Math # Pe DOUBLE RIGHT ARC LESS-THAN BRACKET +2997 ; Other_Math # Ps LEFT BLACK TORTOISE SHELL BRACKET +2998 ; Other_Math # Pe RIGHT BLACK TORTOISE SHELL BRACKET +29D8 ; Other_Math # Ps LEFT WIGGLY FENCE +29D9 ; Other_Math # Pe RIGHT WIGGLY FENCE +29DA ; Other_Math # Ps LEFT DOUBLE WIGGLY FENCE +29DB ; Other_Math # Pe RIGHT DOUBLE WIGGLY FENCE +29FC ; Other_Math # Ps LEFT-POINTING CURVED ANGLE BRACKET +29FD ; Other_Math # Pe RIGHT-POINTING CURVED ANGLE BRACKET +FE61 ; Other_Math # Po SMALL ASTERISK +FE63 ; Other_Math # Pd SMALL HYPHEN-MINUS +FE68 ; Other_Math # Po SMALL REVERSE SOLIDUS +FF3C ; Other_Math # Po FULLWIDTH REVERSE SOLIDUS +FF3E ; Other_Math # Sk FULLWIDTH CIRCUMFLEX ACCENT +1D400..1D454 ; Other_Math # L& [85] MATHEMATICAL BOLD CAPITAL A..MATHEMATICAL ITALIC SMALL G +1D456..1D49C ; Other_Math # L& [71] MATHEMATICAL ITALIC SMALL I..MATHEMATICAL SCRIPT CAPITAL A +1D49E..1D49F ; Other_Math # L& [2] MATHEMATICAL SCRIPT CAPITAL C..MATHEMATICAL SCRIPT CAPITAL D +1D4A2 ; Other_Math # L& MATHEMATICAL SCRIPT CAPITAL G +1D4A5..1D4A6 ; Other_Math # L& [2] MATHEMATICAL SCRIPT CAPITAL J..MATHEMATICAL SCRIPT CAPITAL K +1D4A9..1D4AC ; Other_Math # L& [4] MATHEMATICAL SCRIPT CAPITAL N..MATHEMATICAL SCRIPT CAPITAL Q +1D4AE..1D4B9 ; Other_Math # L& [12] MATHEMATICAL SCRIPT CAPITAL S..MATHEMATICAL SCRIPT SMALL D +1D4BB ; Other_Math # L& MATHEMATICAL SCRIPT SMALL F +1D4BD..1D4C3 ; Other_Math # L& [7] MATHEMATICAL SCRIPT SMALL H..MATHEMATICAL SCRIPT SMALL N +1D4C5..1D505 ; Other_Math # L& [65] MATHEMATICAL SCRIPT SMALL P..MATHEMATICAL FRAKTUR CAPITAL B +1D507..1D50A ; Other_Math # L& [4] MATHEMATICAL FRAKTUR CAPITAL D..MATHEMATICAL FRAKTUR CAPITAL G +1D50D..1D514 ; Other_Math # L& [8] MATHEMATICAL FRAKTUR CAPITAL J..MATHEMATICAL FRAKTUR CAPITAL Q +1D516..1D51C ; Other_Math # L& [7] MATHEMATICAL FRAKTUR CAPITAL S..MATHEMATICAL FRAKTUR CAPITAL Y +1D51E..1D539 ; Other_Math # L& [28] MATHEMATICAL FRAKTUR SMALL A..MATHEMATICAL DOUBLE-STRUCK CAPITAL B +1D53B..1D53E ; Other_Math # L& [4] MATHEMATICAL DOUBLE-STRUCK CAPITAL D..MATHEMATICAL DOUBLE-STRUCK CAPITAL G +1D540..1D544 ; Other_Math # L& [5] MATHEMATICAL DOUBLE-STRUCK CAPITAL I..MATHEMATICAL DOUBLE-STRUCK CAPITAL M +1D546 ; Other_Math # L& MATHEMATICAL DOUBLE-STRUCK CAPITAL O +1D54A..1D550 ; Other_Math # L& [7] MATHEMATICAL DOUBLE-STRUCK CAPITAL S..MATHEMATICAL DOUBLE-STRUCK CAPITAL Y +1D552..1D6A5 ; Other_Math # L& [340] MATHEMATICAL DOUBLE-STRUCK SMALL A..MATHEMATICAL ITALIC SMALL DOTLESS J +1D6A8..1D6C0 ; Other_Math # L& [25] MATHEMATICAL BOLD CAPITAL ALPHA..MATHEMATICAL BOLD CAPITAL OMEGA +1D6C2..1D6DA ; Other_Math # L& [25] MATHEMATICAL BOLD SMALL ALPHA..MATHEMATICAL BOLD SMALL OMEGA +1D6DC..1D6FA ; Other_Math # L& [31] MATHEMATICAL BOLD EPSILON SYMBOL..MATHEMATICAL ITALIC CAPITAL OMEGA +1D6FC..1D714 ; Other_Math # L& [25] MATHEMATICAL ITALIC SMALL ALPHA..MATHEMATICAL ITALIC SMALL OMEGA +1D716..1D734 ; Other_Math # L& [31] MATHEMATICAL ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD ITALIC CAPITAL OMEGA +1D736..1D74E ; Other_Math # L& [25] MATHEMATICAL BOLD ITALIC SMALL ALPHA..MATHEMATICAL BOLD ITALIC SMALL OMEGA +1D750..1D76E ; Other_Math # L& [31] MATHEMATICAL BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD CAPITAL OMEGA +1D770..1D788 ; Other_Math # L& [25] MATHEMATICAL SANS-SERIF BOLD SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD SMALL OMEGA +1D78A..1D7A8 ; Other_Math # L& [31] MATHEMATICAL SANS-SERIF BOLD EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL OMEGA +1D7AA..1D7C2 ; Other_Math # L& [25] MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL OMEGA +1D7C4..1D7CB ; Other_Math # L& [8] MATHEMATICAL SANS-SERIF BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD SMALL DIGAMMA +1D7CE..1D7FF ; Other_Math # Nd [50] MATHEMATICAL BOLD DIGIT ZERO..MATHEMATICAL MONOSPACE DIGIT NINE +1EE00..1EE03 ; Other_Math # Lo [4] ARABIC MATHEMATICAL ALEF..ARABIC MATHEMATICAL DAL +1EE05..1EE1F ; Other_Math # Lo [27] ARABIC MATHEMATICAL WAW..ARABIC MATHEMATICAL DOTLESS QAF +1EE21..1EE22 ; Other_Math # Lo [2] ARABIC MATHEMATICAL INITIAL BEH..ARABIC MATHEMATICAL INITIAL JEEM +1EE24 ; Other_Math # Lo ARABIC MATHEMATICAL INITIAL HEH +1EE27 ; Other_Math # Lo ARABIC MATHEMATICAL INITIAL HAH +1EE29..1EE32 ; Other_Math # Lo [10] ARABIC MATHEMATICAL INITIAL YEH..ARABIC MATHEMATICAL INITIAL QAF +1EE34..1EE37 ; Other_Math # Lo [4] ARABIC MATHEMATICAL INITIAL SHEEN..ARABIC MATHEMATICAL INITIAL KHAH +1EE39 ; Other_Math # Lo ARABIC MATHEMATICAL INITIAL DAD +1EE3B ; Other_Math # Lo ARABIC MATHEMATICAL INITIAL GHAIN +1EE42 ; Other_Math # Lo ARABIC MATHEMATICAL TAILED JEEM +1EE47 ; Other_Math # Lo ARABIC MATHEMATICAL TAILED HAH +1EE49 ; Other_Math # Lo ARABIC MATHEMATICAL TAILED YEH +1EE4B ; Other_Math # Lo ARABIC MATHEMATICAL TAILED LAM +1EE4D..1EE4F ; Other_Math # Lo [3] ARABIC MATHEMATICAL TAILED NOON..ARABIC MATHEMATICAL TAILED AIN +1EE51..1EE52 ; Other_Math # Lo [2] ARABIC MATHEMATICAL TAILED SAD..ARABIC MATHEMATICAL TAILED QAF +1EE54 ; Other_Math # Lo ARABIC MATHEMATICAL TAILED SHEEN +1EE57 ; Other_Math # Lo ARABIC MATHEMATICAL TAILED KHAH +1EE59 ; Other_Math # Lo ARABIC MATHEMATICAL TAILED DAD +1EE5B ; Other_Math # Lo ARABIC MATHEMATICAL TAILED GHAIN +1EE5D ; Other_Math # Lo ARABIC MATHEMATICAL TAILED DOTLESS NOON +1EE5F ; Other_Math # Lo ARABIC MATHEMATICAL TAILED DOTLESS QAF +1EE61..1EE62 ; Other_Math # Lo [2] ARABIC MATHEMATICAL STRETCHED BEH..ARABIC MATHEMATICAL STRETCHED JEEM +1EE64 ; Other_Math # Lo ARABIC MATHEMATICAL STRETCHED HEH +1EE67..1EE6A ; Other_Math # Lo [4] ARABIC MATHEMATICAL STRETCHED HAH..ARABIC MATHEMATICAL STRETCHED KAF +1EE6C..1EE72 ; Other_Math # Lo [7] ARABIC MATHEMATICAL STRETCHED MEEM..ARABIC MATHEMATICAL STRETCHED QAF +1EE74..1EE77 ; Other_Math # Lo [4] ARABIC MATHEMATICAL STRETCHED SHEEN..ARABIC MATHEMATICAL STRETCHED KHAH +1EE79..1EE7C ; Other_Math # Lo [4] ARABIC MATHEMATICAL STRETCHED DAD..ARABIC MATHEMATICAL STRETCHED DOTLESS BEH +1EE7E ; Other_Math # Lo ARABIC MATHEMATICAL STRETCHED DOTLESS FEH +1EE80..1EE89 ; Other_Math # Lo [10] ARABIC MATHEMATICAL LOOPED ALEF..ARABIC MATHEMATICAL LOOPED YEH +1EE8B..1EE9B ; Other_Math # Lo [17] ARABIC MATHEMATICAL LOOPED LAM..ARABIC MATHEMATICAL LOOPED GHAIN +1EEA1..1EEA3 ; Other_Math # Lo [3] ARABIC MATHEMATICAL DOUBLE-STRUCK BEH..ARABIC MATHEMATICAL DOUBLE-STRUCK DAL +1EEA5..1EEA9 ; Other_Math # Lo [5] ARABIC MATHEMATICAL DOUBLE-STRUCK WAW..ARABIC MATHEMATICAL DOUBLE-STRUCK YEH +1EEAB..1EEBB ; Other_Math # Lo [17] ARABIC MATHEMATICAL DOUBLE-STRUCK LAM..ARABIC MATHEMATICAL DOUBLE-STRUCK GHAIN + +# Total code points: 1362 + +# ================================================ + +0030..0039 ; Hex_Digit # Nd [10] DIGIT ZERO..DIGIT NINE +0041..0046 ; Hex_Digit # L& [6] LATIN CAPITAL LETTER A..LATIN CAPITAL LETTER F +0061..0066 ; Hex_Digit # L& [6] LATIN SMALL LETTER A..LATIN SMALL LETTER F +FF10..FF19 ; Hex_Digit # Nd [10] FULLWIDTH DIGIT ZERO..FULLWIDTH DIGIT NINE +FF21..FF26 ; Hex_Digit # L& [6] FULLWIDTH LATIN CAPITAL LETTER A..FULLWIDTH LATIN CAPITAL LETTER F +FF41..FF46 ; Hex_Digit # L& [6] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH LATIN SMALL LETTER F + +# Total code points: 44 + +# ================================================ + +0030..0039 ; ASCII_Hex_Digit # Nd [10] DIGIT ZERO..DIGIT NINE +0041..0046 ; ASCII_Hex_Digit # L& [6] LATIN CAPITAL LETTER A..LATIN CAPITAL LETTER F +0061..0066 ; ASCII_Hex_Digit # L& [6] LATIN SMALL LETTER A..LATIN SMALL LETTER F + +# Total code points: 22 + +# ================================================ + +0345 ; Other_Alphabetic # Mn COMBINING GREEK YPOGEGRAMMENI +0363..036F ; Other_Alphabetic # Mn [13] COMBINING LATIN SMALL LETTER A..COMBINING LATIN SMALL LETTER X +05B0..05BD ; Other_Alphabetic # Mn [14] HEBREW POINT SHEVA..HEBREW POINT METEG +05BF ; Other_Alphabetic # Mn HEBREW POINT RAFE +05C1..05C2 ; Other_Alphabetic # Mn [2] HEBREW POINT SHIN DOT..HEBREW POINT SIN DOT +05C4..05C5 ; Other_Alphabetic # Mn [2] HEBREW MARK UPPER DOT..HEBREW MARK LOWER DOT +05C7 ; Other_Alphabetic # Mn HEBREW POINT QAMATS QATAN +0610..061A ; Other_Alphabetic # Mn [11] ARABIC SIGN SALLALLAHOU ALAYHE WASSALLAM..ARABIC SMALL KASRA +064B..0657 ; Other_Alphabetic # Mn [13] ARABIC FATHATAN..ARABIC INVERTED DAMMA +0659..065F ; Other_Alphabetic # Mn [7] ARABIC ZWARAKAY..ARABIC WAVY HAMZA BELOW +0670 ; Other_Alphabetic # Mn ARABIC LETTER SUPERSCRIPT ALEF +06D6..06DC ; Other_Alphabetic # Mn [7] ARABIC SMALL HIGH LIGATURE SAD WITH LAM WITH ALEF MAKSURA..ARABIC SMALL HIGH SEEN +06E1..06E4 ; Other_Alphabetic # Mn [4] ARABIC SMALL HIGH DOTLESS HEAD OF KHAH..ARABIC SMALL HIGH MADDA +06E7..06E8 ; Other_Alphabetic # Mn [2] ARABIC SMALL HIGH YEH..ARABIC SMALL HIGH NOON +06ED ; Other_Alphabetic # Mn ARABIC SMALL LOW MEEM +0711 ; Other_Alphabetic # Mn SYRIAC LETTER SUPERSCRIPT ALAPH +0730..073F ; Other_Alphabetic # Mn [16] SYRIAC PTHAHA ABOVE..SYRIAC RWAHA +07A6..07B0 ; Other_Alphabetic # Mn [11] THAANA ABAFILI..THAANA SUKUN +0816..0817 ; Other_Alphabetic # Mn [2] SAMARITAN MARK IN..SAMARITAN MARK IN-ALAF +081B..0823 ; Other_Alphabetic # Mn [9] SAMARITAN MARK EPENTHETIC YUT..SAMARITAN VOWEL SIGN A +0825..0827 ; Other_Alphabetic # Mn [3] SAMARITAN VOWEL SIGN SHORT A..SAMARITAN VOWEL SIGN U +0829..082C ; Other_Alphabetic # Mn [4] SAMARITAN VOWEL SIGN LONG I..SAMARITAN VOWEL SIGN SUKUN +0897 ; Other_Alphabetic # Mn ARABIC PEPET +08D4..08DF ; Other_Alphabetic # Mn [12] ARABIC SMALL HIGH WORD AR-RUB..ARABIC SMALL HIGH WORD WAQFA +08E3..08E9 ; Other_Alphabetic # Mn [7] ARABIC TURNED DAMMA BELOW..ARABIC CURLY KASRATAN +08F0..0902 ; Other_Alphabetic # Mn [19] ARABIC OPEN FATHATAN..DEVANAGARI SIGN ANUSVARA +0903 ; Other_Alphabetic # Mc DEVANAGARI SIGN VISARGA +093A ; Other_Alphabetic # Mn DEVANAGARI VOWEL SIGN OE +093B ; Other_Alphabetic # Mc DEVANAGARI VOWEL SIGN OOE +093E..0940 ; Other_Alphabetic # Mc [3] DEVANAGARI VOWEL SIGN AA..DEVANAGARI VOWEL SIGN II +0941..0948 ; Other_Alphabetic # Mn [8] DEVANAGARI VOWEL SIGN U..DEVANAGARI VOWEL SIGN AI +0949..094C ; Other_Alphabetic # Mc [4] DEVANAGARI VOWEL SIGN CANDRA O..DEVANAGARI VOWEL SIGN AU +094E..094F ; Other_Alphabetic # Mc [2] DEVANAGARI VOWEL SIGN PRISHTHAMATRA E..DEVANAGARI VOWEL SIGN AW +0955..0957 ; Other_Alphabetic # Mn [3] DEVANAGARI VOWEL SIGN CANDRA LONG E..DEVANAGARI VOWEL SIGN UUE +0962..0963 ; Other_Alphabetic # Mn [2] DEVANAGARI VOWEL SIGN VOCALIC L..DEVANAGARI VOWEL SIGN VOCALIC LL +0981 ; Other_Alphabetic # Mn BENGALI SIGN CANDRABINDU +0982..0983 ; Other_Alphabetic # Mc [2] BENGALI SIGN ANUSVARA..BENGALI SIGN VISARGA +09BE..09C0 ; Other_Alphabetic # Mc [3] BENGALI VOWEL SIGN AA..BENGALI VOWEL SIGN II +09C1..09C4 ; Other_Alphabetic # Mn [4] BENGALI VOWEL SIGN U..BENGALI VOWEL SIGN VOCALIC RR +09C7..09C8 ; Other_Alphabetic # Mc [2] BENGALI VOWEL SIGN E..BENGALI VOWEL SIGN AI +09CB..09CC ; Other_Alphabetic # Mc [2] BENGALI VOWEL SIGN O..BENGALI VOWEL SIGN AU +09D7 ; Other_Alphabetic # Mc BENGALI AU LENGTH MARK +09E2..09E3 ; Other_Alphabetic # Mn [2] BENGALI VOWEL SIGN VOCALIC L..BENGALI VOWEL SIGN VOCALIC LL +0A01..0A02 ; Other_Alphabetic # Mn [2] GURMUKHI SIGN ADAK BINDI..GURMUKHI SIGN BINDI +0A03 ; Other_Alphabetic # Mc GURMUKHI SIGN VISARGA +0A3E..0A40 ; Other_Alphabetic # Mc [3] GURMUKHI VOWEL SIGN AA..GURMUKHI VOWEL SIGN II +0A41..0A42 ; Other_Alphabetic # Mn [2] GURMUKHI VOWEL SIGN U..GURMUKHI VOWEL SIGN UU +0A47..0A48 ; Other_Alphabetic # Mn [2] GURMUKHI VOWEL SIGN EE..GURMUKHI VOWEL SIGN AI +0A4B..0A4C ; Other_Alphabetic # Mn [2] GURMUKHI VOWEL SIGN OO..GURMUKHI VOWEL SIGN AU +0A51 ; Other_Alphabetic # Mn GURMUKHI SIGN UDAAT +0A70..0A71 ; Other_Alphabetic # Mn [2] GURMUKHI TIPPI..GURMUKHI ADDAK +0A75 ; Other_Alphabetic # Mn GURMUKHI SIGN YAKASH +0A81..0A82 ; Other_Alphabetic # Mn [2] GUJARATI SIGN CANDRABINDU..GUJARATI SIGN ANUSVARA +0A83 ; Other_Alphabetic # Mc GUJARATI SIGN VISARGA +0ABE..0AC0 ; Other_Alphabetic # Mc [3] GUJARATI VOWEL SIGN AA..GUJARATI VOWEL SIGN II +0AC1..0AC5 ; Other_Alphabetic # Mn [5] GUJARATI VOWEL SIGN U..GUJARATI VOWEL SIGN CANDRA E +0AC7..0AC8 ; Other_Alphabetic # Mn [2] GUJARATI VOWEL SIGN E..GUJARATI VOWEL SIGN AI +0AC9 ; Other_Alphabetic # Mc GUJARATI VOWEL SIGN CANDRA O +0ACB..0ACC ; Other_Alphabetic # Mc [2] GUJARATI VOWEL SIGN O..GUJARATI VOWEL SIGN AU +0AE2..0AE3 ; Other_Alphabetic # Mn [2] GUJARATI VOWEL SIGN VOCALIC L..GUJARATI VOWEL SIGN VOCALIC LL +0AFA..0AFC ; Other_Alphabetic # Mn [3] GUJARATI SIGN SUKUN..GUJARATI SIGN MADDAH +0B01 ; Other_Alphabetic # Mn ORIYA SIGN CANDRABINDU +0B02..0B03 ; Other_Alphabetic # Mc [2] ORIYA SIGN ANUSVARA..ORIYA SIGN VISARGA +0B3E ; Other_Alphabetic # Mc ORIYA VOWEL SIGN AA +0B3F ; Other_Alphabetic # Mn ORIYA VOWEL SIGN I +0B40 ; Other_Alphabetic # Mc ORIYA VOWEL SIGN II +0B41..0B44 ; Other_Alphabetic # Mn [4] ORIYA VOWEL SIGN U..ORIYA VOWEL SIGN VOCALIC RR +0B47..0B48 ; Other_Alphabetic # Mc [2] ORIYA VOWEL SIGN E..ORIYA VOWEL SIGN AI +0B4B..0B4C ; Other_Alphabetic # Mc [2] ORIYA VOWEL SIGN O..ORIYA VOWEL SIGN AU +0B56 ; Other_Alphabetic # Mn ORIYA AI LENGTH MARK +0B57 ; Other_Alphabetic # Mc ORIYA AU LENGTH MARK +0B62..0B63 ; Other_Alphabetic # Mn [2] ORIYA VOWEL SIGN VOCALIC L..ORIYA VOWEL SIGN VOCALIC LL +0B82 ; Other_Alphabetic # Mn TAMIL SIGN ANUSVARA +0BBE..0BBF ; Other_Alphabetic # Mc [2] TAMIL VOWEL SIGN AA..TAMIL VOWEL SIGN I +0BC0 ; Other_Alphabetic # Mn TAMIL VOWEL SIGN II +0BC1..0BC2 ; Other_Alphabetic # Mc [2] TAMIL VOWEL SIGN U..TAMIL VOWEL SIGN UU +0BC6..0BC8 ; Other_Alphabetic # Mc [3] TAMIL VOWEL SIGN E..TAMIL VOWEL SIGN AI +0BCA..0BCC ; Other_Alphabetic # Mc [3] TAMIL VOWEL SIGN O..TAMIL VOWEL SIGN AU +0BD7 ; Other_Alphabetic # Mc TAMIL AU LENGTH MARK +0C00 ; Other_Alphabetic # Mn TELUGU SIGN COMBINING CANDRABINDU ABOVE +0C01..0C03 ; Other_Alphabetic # Mc [3] TELUGU SIGN CANDRABINDU..TELUGU SIGN VISARGA +0C04 ; Other_Alphabetic # Mn TELUGU SIGN COMBINING ANUSVARA ABOVE +0C3E..0C40 ; Other_Alphabetic # Mn [3] TELUGU VOWEL SIGN AA..TELUGU VOWEL SIGN II +0C41..0C44 ; Other_Alphabetic # Mc [4] TELUGU VOWEL SIGN U..TELUGU VOWEL SIGN VOCALIC RR +0C46..0C48 ; Other_Alphabetic # Mn [3] TELUGU VOWEL SIGN E..TELUGU VOWEL SIGN AI +0C4A..0C4C ; Other_Alphabetic # Mn [3] TELUGU VOWEL SIGN O..TELUGU VOWEL SIGN AU +0C55..0C56 ; Other_Alphabetic # Mn [2] TELUGU LENGTH MARK..TELUGU AI LENGTH MARK +0C62..0C63 ; Other_Alphabetic # Mn [2] TELUGU VOWEL SIGN VOCALIC L..TELUGU VOWEL SIGN VOCALIC LL +0C81 ; Other_Alphabetic # Mn KANNADA SIGN CANDRABINDU +0C82..0C83 ; Other_Alphabetic # Mc [2] KANNADA SIGN ANUSVARA..KANNADA SIGN VISARGA +0CBE ; Other_Alphabetic # Mc KANNADA VOWEL SIGN AA +0CBF ; Other_Alphabetic # Mn KANNADA VOWEL SIGN I +0CC0..0CC4 ; Other_Alphabetic # Mc [5] KANNADA VOWEL SIGN II..KANNADA VOWEL SIGN VOCALIC RR +0CC6 ; Other_Alphabetic # Mn KANNADA VOWEL SIGN E +0CC7..0CC8 ; Other_Alphabetic # Mc [2] KANNADA VOWEL SIGN EE..KANNADA VOWEL SIGN AI +0CCA..0CCB ; Other_Alphabetic # Mc [2] KANNADA VOWEL SIGN O..KANNADA VOWEL SIGN OO +0CCC ; Other_Alphabetic # Mn KANNADA VOWEL SIGN AU +0CD5..0CD6 ; Other_Alphabetic # Mc [2] KANNADA LENGTH MARK..KANNADA AI LENGTH MARK +0CE2..0CE3 ; Other_Alphabetic # Mn [2] KANNADA VOWEL SIGN VOCALIC L..KANNADA VOWEL SIGN VOCALIC LL +0CF3 ; Other_Alphabetic # Mc KANNADA SIGN COMBINING ANUSVARA ABOVE RIGHT +0D00..0D01 ; Other_Alphabetic # Mn [2] MALAYALAM SIGN COMBINING ANUSVARA ABOVE..MALAYALAM SIGN CANDRABINDU +0D02..0D03 ; Other_Alphabetic # Mc [2] MALAYALAM SIGN ANUSVARA..MALAYALAM SIGN VISARGA +0D3E..0D40 ; Other_Alphabetic # Mc [3] MALAYALAM VOWEL SIGN AA..MALAYALAM VOWEL SIGN II +0D41..0D44 ; Other_Alphabetic # Mn [4] MALAYALAM VOWEL SIGN U..MALAYALAM VOWEL SIGN VOCALIC RR +0D46..0D48 ; Other_Alphabetic # Mc [3] MALAYALAM VOWEL SIGN E..MALAYALAM VOWEL SIGN AI +0D4A..0D4C ; Other_Alphabetic # Mc [3] MALAYALAM VOWEL SIGN O..MALAYALAM VOWEL SIGN AU +0D57 ; Other_Alphabetic # Mc MALAYALAM AU LENGTH MARK +0D62..0D63 ; Other_Alphabetic # Mn [2] MALAYALAM VOWEL SIGN VOCALIC L..MALAYALAM VOWEL SIGN VOCALIC LL +0D81 ; Other_Alphabetic # Mn SINHALA SIGN CANDRABINDU +0D82..0D83 ; Other_Alphabetic # Mc [2] SINHALA SIGN ANUSVARAYA..SINHALA SIGN VISARGAYA +0DCF..0DD1 ; Other_Alphabetic # Mc [3] SINHALA VOWEL SIGN AELA-PILLA..SINHALA VOWEL SIGN DIGA AEDA-PILLA +0DD2..0DD4 ; Other_Alphabetic # Mn [3] SINHALA VOWEL SIGN KETTI IS-PILLA..SINHALA VOWEL SIGN KETTI PAA-PILLA +0DD6 ; Other_Alphabetic # Mn SINHALA VOWEL SIGN DIGA PAA-PILLA +0DD8..0DDF ; Other_Alphabetic # Mc [8] SINHALA VOWEL SIGN GAETTA-PILLA..SINHALA VOWEL SIGN GAYANUKITTA +0DF2..0DF3 ; Other_Alphabetic # Mc [2] SINHALA VOWEL SIGN DIGA GAETTA-PILLA..SINHALA VOWEL SIGN DIGA GAYANUKITTA +0E31 ; Other_Alphabetic # Mn THAI CHARACTER MAI HAN-AKAT +0E34..0E3A ; Other_Alphabetic # Mn [7] THAI CHARACTER SARA I..THAI CHARACTER PHINTHU +0E4D ; Other_Alphabetic # Mn THAI CHARACTER NIKHAHIT +0EB1 ; Other_Alphabetic # Mn LAO VOWEL SIGN MAI KAN +0EB4..0EB9 ; Other_Alphabetic # Mn [6] LAO VOWEL SIGN I..LAO VOWEL SIGN UU +0EBB..0EBC ; Other_Alphabetic # Mn [2] LAO VOWEL SIGN MAI KON..LAO SEMIVOWEL SIGN LO +0ECD ; Other_Alphabetic # Mn LAO NIGGAHITA +0F71..0F7E ; Other_Alphabetic # Mn [14] TIBETAN VOWEL SIGN AA..TIBETAN SIGN RJES SU NGA RO +0F7F ; Other_Alphabetic # Mc TIBETAN SIGN RNAM BCAD +0F80..0F83 ; Other_Alphabetic # Mn [4] TIBETAN VOWEL SIGN REVERSED I..TIBETAN SIGN SNA LDAN +0F8D..0F97 ; Other_Alphabetic # Mn [11] TIBETAN SUBJOINED SIGN LCE TSA CAN..TIBETAN SUBJOINED LETTER JA +0F99..0FBC ; Other_Alphabetic # Mn [36] TIBETAN SUBJOINED LETTER NYA..TIBETAN SUBJOINED LETTER FIXED-FORM RA +102B..102C ; Other_Alphabetic # Mc [2] MYANMAR VOWEL SIGN TALL AA..MYANMAR VOWEL SIGN AA +102D..1030 ; Other_Alphabetic # Mn [4] MYANMAR VOWEL SIGN I..MYANMAR VOWEL SIGN UU +1031 ; Other_Alphabetic # Mc MYANMAR VOWEL SIGN E +1032..1036 ; Other_Alphabetic # Mn [5] MYANMAR VOWEL SIGN AI..MYANMAR SIGN ANUSVARA +1038 ; Other_Alphabetic # Mc MYANMAR SIGN VISARGA +103B..103C ; Other_Alphabetic # Mc [2] MYANMAR CONSONANT SIGN MEDIAL YA..MYANMAR CONSONANT SIGN MEDIAL RA +103D..103E ; Other_Alphabetic # Mn [2] MYANMAR CONSONANT SIGN MEDIAL WA..MYANMAR CONSONANT SIGN MEDIAL HA +1056..1057 ; Other_Alphabetic # Mc [2] MYANMAR VOWEL SIGN VOCALIC R..MYANMAR VOWEL SIGN VOCALIC RR +1058..1059 ; Other_Alphabetic # Mn [2] MYANMAR VOWEL SIGN VOCALIC L..MYANMAR VOWEL SIGN VOCALIC LL +105E..1060 ; Other_Alphabetic # Mn [3] MYANMAR CONSONANT SIGN MON MEDIAL NA..MYANMAR CONSONANT SIGN MON MEDIAL LA +1062..1064 ; Other_Alphabetic # Mc [3] MYANMAR VOWEL SIGN SGAW KAREN EU..MYANMAR TONE MARK SGAW KAREN KE PHO +1067..106D ; Other_Alphabetic # Mc [7] MYANMAR VOWEL SIGN WESTERN PWO KAREN EU..MYANMAR SIGN WESTERN PWO KAREN TONE-5 +1071..1074 ; Other_Alphabetic # Mn [4] MYANMAR VOWEL SIGN GEBA KAREN I..MYANMAR VOWEL SIGN KAYAH EE +1082 ; Other_Alphabetic # Mn MYANMAR CONSONANT SIGN SHAN MEDIAL WA +1083..1084 ; Other_Alphabetic # Mc [2] MYANMAR VOWEL SIGN SHAN AA..MYANMAR VOWEL SIGN SHAN E +1085..1086 ; Other_Alphabetic # Mn [2] MYANMAR VOWEL SIGN SHAN E ABOVE..MYANMAR VOWEL SIGN SHAN FINAL Y +1087..108C ; Other_Alphabetic # Mc [6] MYANMAR SIGN SHAN TONE-2..MYANMAR SIGN SHAN COUNCIL TONE-3 +108D ; Other_Alphabetic # Mn MYANMAR SIGN SHAN COUNCIL EMPHATIC TONE +108F ; Other_Alphabetic # Mc MYANMAR SIGN RUMAI PALAUNG TONE-5 +109A..109C ; Other_Alphabetic # Mc [3] MYANMAR SIGN KHAMTI TONE-1..MYANMAR VOWEL SIGN AITON A +109D ; Other_Alphabetic # Mn MYANMAR VOWEL SIGN AITON AI +1712..1713 ; Other_Alphabetic # Mn [2] TAGALOG VOWEL SIGN I..TAGALOG VOWEL SIGN U +1732..1733 ; Other_Alphabetic # Mn [2] HANUNOO VOWEL SIGN I..HANUNOO VOWEL SIGN U +1752..1753 ; Other_Alphabetic # Mn [2] BUHID VOWEL SIGN I..BUHID VOWEL SIGN U +1772..1773 ; Other_Alphabetic # Mn [2] TAGBANWA VOWEL SIGN I..TAGBANWA VOWEL SIGN U +17B6 ; Other_Alphabetic # Mc KHMER VOWEL SIGN AA +17B7..17BD ; Other_Alphabetic # Mn [7] KHMER VOWEL SIGN I..KHMER VOWEL SIGN UA +17BE..17C5 ; Other_Alphabetic # Mc [8] KHMER VOWEL SIGN OE..KHMER VOWEL SIGN AU +17C6 ; Other_Alphabetic # Mn KHMER SIGN NIKAHIT +17C7..17C8 ; Other_Alphabetic # Mc [2] KHMER SIGN REAHMUK..KHMER SIGN YUUKALEAPINTU +1885..1886 ; Other_Alphabetic # Mn [2] MONGOLIAN LETTER ALI GALI BALUDA..MONGOLIAN LETTER ALI GALI THREE BALUDA +18A9 ; Other_Alphabetic # Mn MONGOLIAN LETTER ALI GALI DAGALGA +1920..1922 ; Other_Alphabetic # Mn [3] LIMBU VOWEL SIGN A..LIMBU VOWEL SIGN U +1923..1926 ; Other_Alphabetic # Mc [4] LIMBU VOWEL SIGN EE..LIMBU VOWEL SIGN AU +1927..1928 ; Other_Alphabetic # Mn [2] LIMBU VOWEL SIGN E..LIMBU VOWEL SIGN O +1929..192B ; Other_Alphabetic # Mc [3] LIMBU SUBJOINED LETTER YA..LIMBU SUBJOINED LETTER WA +1930..1931 ; Other_Alphabetic # Mc [2] LIMBU SMALL LETTER KA..LIMBU SMALL LETTER NGA +1932 ; Other_Alphabetic # Mn LIMBU SMALL LETTER ANUSVARA +1933..1938 ; Other_Alphabetic # Mc [6] LIMBU SMALL LETTER TA..LIMBU SMALL LETTER LA +1A17..1A18 ; Other_Alphabetic # Mn [2] BUGINESE VOWEL SIGN I..BUGINESE VOWEL SIGN U +1A19..1A1A ; Other_Alphabetic # Mc [2] BUGINESE VOWEL SIGN E..BUGINESE VOWEL SIGN O +1A1B ; Other_Alphabetic # Mn BUGINESE VOWEL SIGN AE +1A55 ; Other_Alphabetic # Mc TAI THAM CONSONANT SIGN MEDIAL RA +1A56 ; Other_Alphabetic # Mn TAI THAM CONSONANT SIGN MEDIAL LA +1A57 ; Other_Alphabetic # Mc TAI THAM CONSONANT SIGN LA TANG LAI +1A58..1A5E ; Other_Alphabetic # Mn [7] TAI THAM SIGN MAI KANG LAI..TAI THAM CONSONANT SIGN SA +1A61 ; Other_Alphabetic # Mc TAI THAM VOWEL SIGN A +1A62 ; Other_Alphabetic # Mn TAI THAM VOWEL SIGN MAI SAT +1A63..1A64 ; Other_Alphabetic # Mc [2] TAI THAM VOWEL SIGN AA..TAI THAM VOWEL SIGN TALL AA +1A65..1A6C ; Other_Alphabetic # Mn [8] TAI THAM VOWEL SIGN I..TAI THAM VOWEL SIGN OA BELOW +1A6D..1A72 ; Other_Alphabetic # Mc [6] TAI THAM VOWEL SIGN OY..TAI THAM VOWEL SIGN THAM AI +1A73..1A74 ; Other_Alphabetic # Mn [2] TAI THAM VOWEL SIGN OA ABOVE..TAI THAM SIGN MAI KANG +1ABF..1AC0 ; Other_Alphabetic # Mn [2] COMBINING LATIN SMALL LETTER W BELOW..COMBINING LATIN SMALL LETTER TURNED W BELOW +1ACC..1ACE ; Other_Alphabetic # Mn [3] COMBINING LATIN SMALL LETTER INSULAR G..COMBINING LATIN SMALL LETTER INSULAR T +1B00..1B03 ; Other_Alphabetic # Mn [4] BALINESE SIGN ULU RICEM..BALINESE SIGN SURANG +1B04 ; Other_Alphabetic # Mc BALINESE SIGN BISAH +1B35 ; Other_Alphabetic # Mc BALINESE VOWEL SIGN TEDUNG +1B36..1B3A ; Other_Alphabetic # Mn [5] BALINESE VOWEL SIGN ULU..BALINESE VOWEL SIGN RA REPA +1B3B ; Other_Alphabetic # Mc BALINESE VOWEL SIGN RA REPA TEDUNG +1B3C ; Other_Alphabetic # Mn BALINESE VOWEL SIGN LA LENGA +1B3D..1B41 ; Other_Alphabetic # Mc [5] BALINESE VOWEL SIGN LA LENGA TEDUNG..BALINESE VOWEL SIGN TALING REPA TEDUNG +1B42 ; Other_Alphabetic # Mn BALINESE VOWEL SIGN PEPET +1B43 ; Other_Alphabetic # Mc BALINESE VOWEL SIGN PEPET TEDUNG +1B80..1B81 ; Other_Alphabetic # Mn [2] SUNDANESE SIGN PANYECEK..SUNDANESE SIGN PANGLAYAR +1B82 ; Other_Alphabetic # Mc SUNDANESE SIGN PANGWISAD +1BA1 ; Other_Alphabetic # Mc SUNDANESE CONSONANT SIGN PAMINGKAL +1BA2..1BA5 ; Other_Alphabetic # Mn [4] SUNDANESE CONSONANT SIGN PANYAKRA..SUNDANESE VOWEL SIGN PANYUKU +1BA6..1BA7 ; Other_Alphabetic # Mc [2] SUNDANESE VOWEL SIGN PANAELAENG..SUNDANESE VOWEL SIGN PANOLONG +1BA8..1BA9 ; Other_Alphabetic # Mn [2] SUNDANESE VOWEL SIGN PAMEPET..SUNDANESE VOWEL SIGN PANEULEUNG +1BAC..1BAD ; Other_Alphabetic # Mn [2] SUNDANESE CONSONANT SIGN PASANGAN MA..SUNDANESE CONSONANT SIGN PASANGAN WA +1BE7 ; Other_Alphabetic # Mc BATAK VOWEL SIGN E +1BE8..1BE9 ; Other_Alphabetic # Mn [2] BATAK VOWEL SIGN PAKPAK E..BATAK VOWEL SIGN EE +1BEA..1BEC ; Other_Alphabetic # Mc [3] BATAK VOWEL SIGN I..BATAK VOWEL SIGN O +1BED ; Other_Alphabetic # Mn BATAK VOWEL SIGN KARO O +1BEE ; Other_Alphabetic # Mc BATAK VOWEL SIGN U +1BEF..1BF1 ; Other_Alphabetic # Mn [3] BATAK VOWEL SIGN U FOR SIMALUNGUN SA..BATAK CONSONANT SIGN H +1C24..1C2B ; Other_Alphabetic # Mc [8] LEPCHA SUBJOINED LETTER YA..LEPCHA VOWEL SIGN UU +1C2C..1C33 ; Other_Alphabetic # Mn [8] LEPCHA VOWEL SIGN E..LEPCHA CONSONANT SIGN T +1C34..1C35 ; Other_Alphabetic # Mc [2] LEPCHA CONSONANT SIGN NYIN-DO..LEPCHA CONSONANT SIGN KANG +1C36 ; Other_Alphabetic # Mn LEPCHA SIGN RAN +1DD3..1DF4 ; Other_Alphabetic # Mn [34] COMBINING LATIN SMALL LETTER FLATTENED OPEN A ABOVE..COMBINING LATIN SMALL LETTER U WITH DIAERESIS +24B6..24E9 ; Other_Alphabetic # So [52] CIRCLED LATIN CAPITAL LETTER A..CIRCLED LATIN SMALL LETTER Z +2DE0..2DFF ; Other_Alphabetic # Mn [32] COMBINING CYRILLIC LETTER BE..COMBINING CYRILLIC LETTER IOTIFIED BIG YUS +A674..A67B ; Other_Alphabetic # Mn [8] COMBINING CYRILLIC LETTER UKRAINIAN IE..COMBINING CYRILLIC LETTER OMEGA +A69E..A69F ; Other_Alphabetic # Mn [2] COMBINING CYRILLIC LETTER EF..COMBINING CYRILLIC LETTER IOTIFIED E +A802 ; Other_Alphabetic # Mn SYLOTI NAGRI SIGN DVISVARA +A80B ; Other_Alphabetic # Mn SYLOTI NAGRI SIGN ANUSVARA +A823..A824 ; Other_Alphabetic # Mc [2] SYLOTI NAGRI VOWEL SIGN A..SYLOTI NAGRI VOWEL SIGN I +A825..A826 ; Other_Alphabetic # Mn [2] SYLOTI NAGRI VOWEL SIGN U..SYLOTI NAGRI VOWEL SIGN E +A827 ; Other_Alphabetic # Mc SYLOTI NAGRI VOWEL SIGN OO +A880..A881 ; Other_Alphabetic # Mc [2] SAURASHTRA SIGN ANUSVARA..SAURASHTRA SIGN VISARGA +A8B4..A8C3 ; Other_Alphabetic # Mc [16] SAURASHTRA CONSONANT SIGN HAARU..SAURASHTRA VOWEL SIGN AU +A8C5 ; Other_Alphabetic # Mn SAURASHTRA SIGN CANDRABINDU +A8FF ; Other_Alphabetic # Mn DEVANAGARI VOWEL SIGN AY +A926..A92A ; Other_Alphabetic # Mn [5] KAYAH LI VOWEL UE..KAYAH LI VOWEL O +A947..A951 ; Other_Alphabetic # Mn [11] REJANG VOWEL SIGN I..REJANG CONSONANT SIGN R +A952 ; Other_Alphabetic # Mc REJANG CONSONANT SIGN H +A980..A982 ; Other_Alphabetic # Mn [3] JAVANESE SIGN PANYANGGA..JAVANESE SIGN LAYAR +A983 ; Other_Alphabetic # Mc JAVANESE SIGN WIGNYAN +A9B4..A9B5 ; Other_Alphabetic # Mc [2] JAVANESE VOWEL SIGN TARUNG..JAVANESE VOWEL SIGN TOLONG +A9B6..A9B9 ; Other_Alphabetic # Mn [4] JAVANESE VOWEL SIGN WULU..JAVANESE VOWEL SIGN SUKU MENDUT +A9BA..A9BB ; Other_Alphabetic # Mc [2] JAVANESE VOWEL SIGN TALING..JAVANESE VOWEL SIGN DIRGA MURE +A9BC..A9BD ; Other_Alphabetic # Mn [2] JAVANESE VOWEL SIGN PEPET..JAVANESE CONSONANT SIGN KERET +A9BE..A9BF ; Other_Alphabetic # Mc [2] JAVANESE CONSONANT SIGN PENGKAL..JAVANESE CONSONANT SIGN CAKRA +A9E5 ; Other_Alphabetic # Mn MYANMAR SIGN SHAN SAW +AA29..AA2E ; Other_Alphabetic # Mn [6] CHAM VOWEL SIGN AA..CHAM VOWEL SIGN OE +AA2F..AA30 ; Other_Alphabetic # Mc [2] CHAM VOWEL SIGN O..CHAM VOWEL SIGN AI +AA31..AA32 ; Other_Alphabetic # Mn [2] CHAM VOWEL SIGN AU..CHAM VOWEL SIGN UE +AA33..AA34 ; Other_Alphabetic # Mc [2] CHAM CONSONANT SIGN YA..CHAM CONSONANT SIGN RA +AA35..AA36 ; Other_Alphabetic # Mn [2] CHAM CONSONANT SIGN LA..CHAM CONSONANT SIGN WA +AA43 ; Other_Alphabetic # Mn CHAM CONSONANT SIGN FINAL NG +AA4C ; Other_Alphabetic # Mn CHAM CONSONANT SIGN FINAL M +AA4D ; Other_Alphabetic # Mc CHAM CONSONANT SIGN FINAL H +AA7B ; Other_Alphabetic # Mc MYANMAR SIGN PAO KAREN TONE +AA7C ; Other_Alphabetic # Mn MYANMAR SIGN TAI LAING TONE-2 +AA7D ; Other_Alphabetic # Mc MYANMAR SIGN TAI LAING TONE-5 +AAB0 ; Other_Alphabetic # Mn TAI VIET MAI KANG +AAB2..AAB4 ; Other_Alphabetic # Mn [3] TAI VIET VOWEL I..TAI VIET VOWEL U +AAB7..AAB8 ; Other_Alphabetic # Mn [2] TAI VIET MAI KHIT..TAI VIET VOWEL IA +AABE ; Other_Alphabetic # Mn TAI VIET VOWEL AM +AAEB ; Other_Alphabetic # Mc MEETEI MAYEK VOWEL SIGN II +AAEC..AAED ; Other_Alphabetic # Mn [2] MEETEI MAYEK VOWEL SIGN UU..MEETEI MAYEK VOWEL SIGN AAI +AAEE..AAEF ; Other_Alphabetic # Mc [2] MEETEI MAYEK VOWEL SIGN AU..MEETEI MAYEK VOWEL SIGN AAU +AAF5 ; Other_Alphabetic # Mc MEETEI MAYEK VOWEL SIGN VISARGA +ABE3..ABE4 ; Other_Alphabetic # Mc [2] MEETEI MAYEK VOWEL SIGN ONAP..MEETEI MAYEK VOWEL SIGN INAP +ABE5 ; Other_Alphabetic # Mn MEETEI MAYEK VOWEL SIGN ANAP +ABE6..ABE7 ; Other_Alphabetic # Mc [2] MEETEI MAYEK VOWEL SIGN YENAP..MEETEI MAYEK VOWEL SIGN SOUNAP +ABE8 ; Other_Alphabetic # Mn MEETEI MAYEK VOWEL SIGN UNAP +ABE9..ABEA ; Other_Alphabetic # Mc [2] MEETEI MAYEK VOWEL SIGN CHEINAP..MEETEI MAYEK VOWEL SIGN NUNG +FB1E ; Other_Alphabetic # Mn HEBREW POINT JUDEO-SPANISH VARIKA +10376..1037A ; Other_Alphabetic # Mn [5] COMBINING OLD PERMIC LETTER AN..COMBINING OLD PERMIC LETTER SII +10A01..10A03 ; Other_Alphabetic # Mn [3] KHAROSHTHI VOWEL SIGN I..KHAROSHTHI VOWEL SIGN VOCALIC R +10A05..10A06 ; Other_Alphabetic # Mn [2] KHAROSHTHI VOWEL SIGN E..KHAROSHTHI VOWEL SIGN O +10A0C..10A0F ; Other_Alphabetic # Mn [4] KHAROSHTHI VOWEL LENGTH MARK..KHAROSHTHI SIGN VISARGA +10D24..10D27 ; Other_Alphabetic # Mn [4] HANIFI ROHINGYA SIGN HARBAHAY..HANIFI ROHINGYA SIGN TASSI +10D69 ; Other_Alphabetic # Mn GARAY VOWEL SIGN E +10EAB..10EAC ; Other_Alphabetic # Mn [2] YEZIDI COMBINING HAMZA MARK..YEZIDI COMBINING MADDA MARK +10EFA..10EFC ; Other_Alphabetic # Mn [3] ARABIC DOUBLE VERTICAL BAR BELOW..ARABIC COMBINING ALEF OVERLAY +11000 ; Other_Alphabetic # Mc BRAHMI SIGN CANDRABINDU +11001 ; Other_Alphabetic # Mn BRAHMI SIGN ANUSVARA +11002 ; Other_Alphabetic # Mc BRAHMI SIGN VISARGA +11038..11045 ; Other_Alphabetic # Mn [14] BRAHMI VOWEL SIGN AA..BRAHMI VOWEL SIGN AU +11073..11074 ; Other_Alphabetic # Mn [2] BRAHMI VOWEL SIGN OLD TAMIL SHORT E..BRAHMI VOWEL SIGN OLD TAMIL SHORT O +11080..11081 ; Other_Alphabetic # Mn [2] KAITHI SIGN CANDRABINDU..KAITHI SIGN ANUSVARA +11082 ; Other_Alphabetic # Mc KAITHI SIGN VISARGA +110B0..110B2 ; Other_Alphabetic # Mc [3] KAITHI VOWEL SIGN AA..KAITHI VOWEL SIGN II +110B3..110B6 ; Other_Alphabetic # Mn [4] KAITHI VOWEL SIGN U..KAITHI VOWEL SIGN AI +110B7..110B8 ; Other_Alphabetic # Mc [2] KAITHI VOWEL SIGN O..KAITHI VOWEL SIGN AU +110C2 ; Other_Alphabetic # Mn KAITHI VOWEL SIGN VOCALIC R +11100..11102 ; Other_Alphabetic # Mn [3] CHAKMA SIGN CANDRABINDU..CHAKMA SIGN VISARGA +11127..1112B ; Other_Alphabetic # Mn [5] CHAKMA VOWEL SIGN A..CHAKMA VOWEL SIGN UU +1112C ; Other_Alphabetic # Mc CHAKMA VOWEL SIGN E +1112D..11132 ; Other_Alphabetic # Mn [6] CHAKMA VOWEL SIGN AI..CHAKMA AU MARK +11145..11146 ; Other_Alphabetic # Mc [2] CHAKMA VOWEL SIGN AA..CHAKMA VOWEL SIGN EI +11180..11181 ; Other_Alphabetic # Mn [2] SHARADA SIGN CANDRABINDU..SHARADA SIGN ANUSVARA +11182 ; Other_Alphabetic # Mc SHARADA SIGN VISARGA +111B3..111B5 ; Other_Alphabetic # Mc [3] SHARADA VOWEL SIGN AA..SHARADA VOWEL SIGN II +111B6..111BE ; Other_Alphabetic # Mn [9] SHARADA VOWEL SIGN U..SHARADA VOWEL SIGN O +111BF ; Other_Alphabetic # Mc SHARADA VOWEL SIGN AU +111CE ; Other_Alphabetic # Mc SHARADA VOWEL SIGN PRISHTHAMATRA E +111CF ; Other_Alphabetic # Mn SHARADA SIGN INVERTED CANDRABINDU +1122C..1122E ; Other_Alphabetic # Mc [3] KHOJKI VOWEL SIGN AA..KHOJKI VOWEL SIGN II +1122F..11231 ; Other_Alphabetic # Mn [3] KHOJKI VOWEL SIGN U..KHOJKI VOWEL SIGN AI +11232..11233 ; Other_Alphabetic # Mc [2] KHOJKI VOWEL SIGN O..KHOJKI VOWEL SIGN AU +11234 ; Other_Alphabetic # Mn KHOJKI SIGN ANUSVARA +11237 ; Other_Alphabetic # Mn KHOJKI SIGN SHADDA +1123E ; Other_Alphabetic # Mn KHOJKI SIGN SUKUN +11241 ; Other_Alphabetic # Mn KHOJKI VOWEL SIGN VOCALIC R +112DF ; Other_Alphabetic # Mn KHUDAWADI SIGN ANUSVARA +112E0..112E2 ; Other_Alphabetic # Mc [3] KHUDAWADI VOWEL SIGN AA..KHUDAWADI VOWEL SIGN II +112E3..112E8 ; Other_Alphabetic # Mn [6] KHUDAWADI VOWEL SIGN U..KHUDAWADI VOWEL SIGN AU +11300..11301 ; Other_Alphabetic # Mn [2] GRANTHA SIGN COMBINING ANUSVARA ABOVE..GRANTHA SIGN CANDRABINDU +11302..11303 ; Other_Alphabetic # Mc [2] GRANTHA SIGN ANUSVARA..GRANTHA SIGN VISARGA +1133E..1133F ; Other_Alphabetic # Mc [2] GRANTHA VOWEL SIGN AA..GRANTHA VOWEL SIGN I +11340 ; Other_Alphabetic # Mn GRANTHA VOWEL SIGN II +11341..11344 ; Other_Alphabetic # Mc [4] GRANTHA VOWEL SIGN U..GRANTHA VOWEL SIGN VOCALIC RR +11347..11348 ; Other_Alphabetic # Mc [2] GRANTHA VOWEL SIGN EE..GRANTHA VOWEL SIGN AI +1134B..1134C ; Other_Alphabetic # Mc [2] GRANTHA VOWEL SIGN OO..GRANTHA VOWEL SIGN AU +11357 ; Other_Alphabetic # Mc GRANTHA AU LENGTH MARK +11362..11363 ; Other_Alphabetic # Mc [2] GRANTHA VOWEL SIGN VOCALIC L..GRANTHA VOWEL SIGN VOCALIC LL +113B8..113BA ; Other_Alphabetic # Mc [3] TULU-TIGALARI VOWEL SIGN AA..TULU-TIGALARI VOWEL SIGN II +113BB..113C0 ; Other_Alphabetic # Mn [6] TULU-TIGALARI VOWEL SIGN U..TULU-TIGALARI VOWEL SIGN VOCALIC LL +113C2 ; Other_Alphabetic # Mc TULU-TIGALARI VOWEL SIGN EE +113C5 ; Other_Alphabetic # Mc TULU-TIGALARI VOWEL SIGN AI +113C7..113CA ; Other_Alphabetic # Mc [4] TULU-TIGALARI VOWEL SIGN OO..TULU-TIGALARI SIGN CANDRA ANUNASIKA +113CC..113CD ; Other_Alphabetic # Mc [2] TULU-TIGALARI SIGN ANUSVARA..TULU-TIGALARI SIGN VISARGA +11435..11437 ; Other_Alphabetic # Mc [3] NEWA VOWEL SIGN AA..NEWA VOWEL SIGN II +11438..1143F ; Other_Alphabetic # Mn [8] NEWA VOWEL SIGN U..NEWA VOWEL SIGN AI +11440..11441 ; Other_Alphabetic # Mc [2] NEWA VOWEL SIGN O..NEWA VOWEL SIGN AU +11443..11444 ; Other_Alphabetic # Mn [2] NEWA SIGN CANDRABINDU..NEWA SIGN ANUSVARA +11445 ; Other_Alphabetic # Mc NEWA SIGN VISARGA +114B0..114B2 ; Other_Alphabetic # Mc [3] TIRHUTA VOWEL SIGN AA..TIRHUTA VOWEL SIGN II +114B3..114B8 ; Other_Alphabetic # Mn [6] TIRHUTA VOWEL SIGN U..TIRHUTA VOWEL SIGN VOCALIC LL +114B9 ; Other_Alphabetic # Mc TIRHUTA VOWEL SIGN E +114BA ; Other_Alphabetic # Mn TIRHUTA VOWEL SIGN SHORT E +114BB..114BE ; Other_Alphabetic # Mc [4] TIRHUTA VOWEL SIGN AI..TIRHUTA VOWEL SIGN AU +114BF..114C0 ; Other_Alphabetic # Mn [2] TIRHUTA SIGN CANDRABINDU..TIRHUTA SIGN ANUSVARA +114C1 ; Other_Alphabetic # Mc TIRHUTA SIGN VISARGA +115AF..115B1 ; Other_Alphabetic # Mc [3] SIDDHAM VOWEL SIGN AA..SIDDHAM VOWEL SIGN II +115B2..115B5 ; Other_Alphabetic # Mn [4] SIDDHAM VOWEL SIGN U..SIDDHAM VOWEL SIGN VOCALIC RR +115B8..115BB ; Other_Alphabetic # Mc [4] SIDDHAM VOWEL SIGN E..SIDDHAM VOWEL SIGN AU +115BC..115BD ; Other_Alphabetic # Mn [2] SIDDHAM SIGN CANDRABINDU..SIDDHAM SIGN ANUSVARA +115BE ; Other_Alphabetic # Mc SIDDHAM SIGN VISARGA +115DC..115DD ; Other_Alphabetic # Mn [2] SIDDHAM VOWEL SIGN ALTERNATE U..SIDDHAM VOWEL SIGN ALTERNATE UU +11630..11632 ; Other_Alphabetic # Mc [3] MODI VOWEL SIGN AA..MODI VOWEL SIGN II +11633..1163A ; Other_Alphabetic # Mn [8] MODI VOWEL SIGN U..MODI VOWEL SIGN AI +1163B..1163C ; Other_Alphabetic # Mc [2] MODI VOWEL SIGN O..MODI VOWEL SIGN AU +1163D ; Other_Alphabetic # Mn MODI SIGN ANUSVARA +1163E ; Other_Alphabetic # Mc MODI SIGN VISARGA +11640 ; Other_Alphabetic # Mn MODI SIGN ARDHACANDRA +116AB ; Other_Alphabetic # Mn TAKRI SIGN ANUSVARA +116AC ; Other_Alphabetic # Mc TAKRI SIGN VISARGA +116AD ; Other_Alphabetic # Mn TAKRI VOWEL SIGN AA +116AE..116AF ; Other_Alphabetic # Mc [2] TAKRI VOWEL SIGN I..TAKRI VOWEL SIGN II +116B0..116B5 ; Other_Alphabetic # Mn [6] TAKRI VOWEL SIGN U..TAKRI VOWEL SIGN AU +1171D ; Other_Alphabetic # Mn AHOM CONSONANT SIGN MEDIAL LA +1171E ; Other_Alphabetic # Mc AHOM CONSONANT SIGN MEDIAL RA +1171F ; Other_Alphabetic # Mn AHOM CONSONANT SIGN MEDIAL LIGATING RA +11720..11721 ; Other_Alphabetic # Mc [2] AHOM VOWEL SIGN A..AHOM VOWEL SIGN AA +11722..11725 ; Other_Alphabetic # Mn [4] AHOM VOWEL SIGN I..AHOM VOWEL SIGN UU +11726 ; Other_Alphabetic # Mc AHOM VOWEL SIGN E +11727..1172A ; Other_Alphabetic # Mn [4] AHOM VOWEL SIGN AW..AHOM VOWEL SIGN AM +1182C..1182E ; Other_Alphabetic # Mc [3] DOGRA VOWEL SIGN AA..DOGRA VOWEL SIGN II +1182F..11837 ; Other_Alphabetic # Mn [9] DOGRA VOWEL SIGN U..DOGRA SIGN ANUSVARA +11838 ; Other_Alphabetic # Mc DOGRA SIGN VISARGA +11930..11935 ; Other_Alphabetic # Mc [6] DIVES AKURU VOWEL SIGN AA..DIVES AKURU VOWEL SIGN E +11937..11938 ; Other_Alphabetic # Mc [2] DIVES AKURU VOWEL SIGN AI..DIVES AKURU VOWEL SIGN O +1193B..1193C ; Other_Alphabetic # Mn [2] DIVES AKURU SIGN ANUSVARA..DIVES AKURU SIGN CANDRABINDU +11940 ; Other_Alphabetic # Mc DIVES AKURU MEDIAL YA +11942 ; Other_Alphabetic # Mc DIVES AKURU MEDIAL RA +119D1..119D3 ; Other_Alphabetic # Mc [3] NANDINAGARI VOWEL SIGN AA..NANDINAGARI VOWEL SIGN II +119D4..119D7 ; Other_Alphabetic # Mn [4] NANDINAGARI VOWEL SIGN U..NANDINAGARI VOWEL SIGN VOCALIC RR +119DA..119DB ; Other_Alphabetic # Mn [2] NANDINAGARI VOWEL SIGN E..NANDINAGARI VOWEL SIGN AI +119DC..119DF ; Other_Alphabetic # Mc [4] NANDINAGARI VOWEL SIGN O..NANDINAGARI SIGN VISARGA +119E4 ; Other_Alphabetic # Mc NANDINAGARI VOWEL SIGN PRISHTHAMATRA E +11A01..11A0A ; Other_Alphabetic # Mn [10] ZANABAZAR SQUARE VOWEL SIGN I..ZANABAZAR SQUARE VOWEL LENGTH MARK +11A35..11A38 ; Other_Alphabetic # Mn [4] ZANABAZAR SQUARE SIGN CANDRABINDU..ZANABAZAR SQUARE SIGN ANUSVARA +11A39 ; Other_Alphabetic # Mc ZANABAZAR SQUARE SIGN VISARGA +11A3B..11A3E ; Other_Alphabetic # Mn [4] ZANABAZAR SQUARE CLUSTER-FINAL LETTER YA..ZANABAZAR SQUARE CLUSTER-FINAL LETTER VA +11A51..11A56 ; Other_Alphabetic # Mn [6] SOYOMBO VOWEL SIGN I..SOYOMBO VOWEL SIGN OE +11A57..11A58 ; Other_Alphabetic # Mc [2] SOYOMBO VOWEL SIGN AI..SOYOMBO VOWEL SIGN AU +11A59..11A5B ; Other_Alphabetic # Mn [3] SOYOMBO VOWEL SIGN VOCALIC R..SOYOMBO VOWEL LENGTH MARK +11A8A..11A96 ; Other_Alphabetic # Mn [13] SOYOMBO FINAL CONSONANT SIGN G..SOYOMBO SIGN ANUSVARA +11A97 ; Other_Alphabetic # Mc SOYOMBO SIGN VISARGA +11B60 ; Other_Alphabetic # Mn SHARADA VOWEL SIGN OE +11B61 ; Other_Alphabetic # Mc SHARADA VOWEL SIGN OOE +11B62..11B64 ; Other_Alphabetic # Mn [3] SHARADA VOWEL SIGN UE..SHARADA VOWEL SIGN SHORT E +11B65 ; Other_Alphabetic # Mc SHARADA VOWEL SIGN SHORT O +11B66 ; Other_Alphabetic # Mn SHARADA VOWEL SIGN CANDRA E +11B67 ; Other_Alphabetic # Mc SHARADA VOWEL SIGN CANDRA O +11C2F ; Other_Alphabetic # Mc BHAIKSUKI VOWEL SIGN AA +11C30..11C36 ; Other_Alphabetic # Mn [7] BHAIKSUKI VOWEL SIGN I..BHAIKSUKI VOWEL SIGN VOCALIC L +11C38..11C3D ; Other_Alphabetic # Mn [6] BHAIKSUKI VOWEL SIGN E..BHAIKSUKI SIGN ANUSVARA +11C3E ; Other_Alphabetic # Mc BHAIKSUKI SIGN VISARGA +11C92..11CA7 ; Other_Alphabetic # Mn [22] MARCHEN SUBJOINED LETTER KA..MARCHEN SUBJOINED LETTER ZA +11CA9 ; Other_Alphabetic # Mc MARCHEN SUBJOINED LETTER YA +11CAA..11CB0 ; Other_Alphabetic # Mn [7] MARCHEN SUBJOINED LETTER RA..MARCHEN VOWEL SIGN AA +11CB1 ; Other_Alphabetic # Mc MARCHEN VOWEL SIGN I +11CB2..11CB3 ; Other_Alphabetic # Mn [2] MARCHEN VOWEL SIGN U..MARCHEN VOWEL SIGN E +11CB4 ; Other_Alphabetic # Mc MARCHEN VOWEL SIGN O +11CB5..11CB6 ; Other_Alphabetic # Mn [2] MARCHEN SIGN ANUSVARA..MARCHEN SIGN CANDRABINDU +11D31..11D36 ; Other_Alphabetic # Mn [6] MASARAM GONDI VOWEL SIGN AA..MASARAM GONDI VOWEL SIGN VOCALIC R +11D3A ; Other_Alphabetic # Mn MASARAM GONDI VOWEL SIGN E +11D3C..11D3D ; Other_Alphabetic # Mn [2] MASARAM GONDI VOWEL SIGN AI..MASARAM GONDI VOWEL SIGN O +11D3F..11D41 ; Other_Alphabetic # Mn [3] MASARAM GONDI VOWEL SIGN AU..MASARAM GONDI SIGN VISARGA +11D43 ; Other_Alphabetic # Mn MASARAM GONDI SIGN CANDRA +11D47 ; Other_Alphabetic # Mn MASARAM GONDI RA-KARA +11D8A..11D8E ; Other_Alphabetic # Mc [5] GUNJALA GONDI VOWEL SIGN AA..GUNJALA GONDI VOWEL SIGN UU +11D90..11D91 ; Other_Alphabetic # Mn [2] GUNJALA GONDI VOWEL SIGN EE..GUNJALA GONDI VOWEL SIGN AI +11D93..11D94 ; Other_Alphabetic # Mc [2] GUNJALA GONDI VOWEL SIGN OO..GUNJALA GONDI VOWEL SIGN AU +11D95 ; Other_Alphabetic # Mn GUNJALA GONDI SIGN ANUSVARA +11D96 ; Other_Alphabetic # Mc GUNJALA GONDI SIGN VISARGA +11EF3..11EF4 ; Other_Alphabetic # Mn [2] MAKASAR VOWEL SIGN I..MAKASAR VOWEL SIGN U +11EF5..11EF6 ; Other_Alphabetic # Mc [2] MAKASAR VOWEL SIGN E..MAKASAR VOWEL SIGN O +11F00..11F01 ; Other_Alphabetic # Mn [2] KAWI SIGN CANDRABINDU..KAWI SIGN ANUSVARA +11F03 ; Other_Alphabetic # Mc KAWI SIGN VISARGA +11F34..11F35 ; Other_Alphabetic # Mc [2] KAWI VOWEL SIGN AA..KAWI VOWEL SIGN ALTERNATE AA +11F36..11F3A ; Other_Alphabetic # Mn [5] KAWI VOWEL SIGN I..KAWI VOWEL SIGN VOCALIC R +11F3E..11F3F ; Other_Alphabetic # Mc [2] KAWI VOWEL SIGN E..KAWI VOWEL SIGN AI +11F40 ; Other_Alphabetic # Mn KAWI VOWEL SIGN EU +1611E..16129 ; Other_Alphabetic # Mn [12] GURUNG KHEMA VOWEL SIGN AA..GURUNG KHEMA VOWEL LENGTH MARK +1612A..1612C ; Other_Alphabetic # Mc [3] GURUNG KHEMA CONSONANT SIGN MEDIAL YA..GURUNG KHEMA CONSONANT SIGN MEDIAL HA +1612D..1612E ; Other_Alphabetic # Mn [2] GURUNG KHEMA SIGN ANUSVARA..GURUNG KHEMA CONSONANT SIGN MEDIAL RA +16F4F ; Other_Alphabetic # Mn MIAO SIGN CONSONANT MODIFIER BAR +16F51..16F87 ; Other_Alphabetic # Mc [55] MIAO SIGN ASPIRATION..MIAO VOWEL SIGN UI +16F8F..16F92 ; Other_Alphabetic # Mn [4] MIAO TONE RIGHT..MIAO TONE BELOW +16FF0..16FF1 ; Other_Alphabetic # Mc [2] VIETNAMESE ALTERNATE READING MARK CA..VIETNAMESE ALTERNATE READING MARK NHAY +1BC9E ; Other_Alphabetic # Mn DUPLOYAN DOUBLE MARK +1E000..1E006 ; Other_Alphabetic # Mn [7] COMBINING GLAGOLITIC LETTER AZU..COMBINING GLAGOLITIC LETTER ZHIVETE +1E008..1E018 ; Other_Alphabetic # Mn [17] COMBINING GLAGOLITIC LETTER ZEMLJA..COMBINING GLAGOLITIC LETTER HERU +1E01B..1E021 ; Other_Alphabetic # Mn [7] COMBINING GLAGOLITIC LETTER SHTA..COMBINING GLAGOLITIC LETTER YATI +1E023..1E024 ; Other_Alphabetic # Mn [2] COMBINING GLAGOLITIC LETTER YU..COMBINING GLAGOLITIC LETTER SMALL YUS +1E026..1E02A ; Other_Alphabetic # Mn [5] COMBINING GLAGOLITIC LETTER YO..COMBINING GLAGOLITIC LETTER FITA +1E08F ; Other_Alphabetic # Mn COMBINING CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I +1E6E3 ; Other_Alphabetic # Mn TAI YO SIGN UE +1E6E6 ; Other_Alphabetic # Mn TAI YO SIGN AU +1E6EE..1E6EF ; Other_Alphabetic # Mn [2] TAI YO SIGN AY..TAI YO SIGN ANG +1E6F5 ; Other_Alphabetic # Mn TAI YO SIGN OM +1E947 ; Other_Alphabetic # Mn ADLAM HAMZA +1F130..1F149 ; Other_Alphabetic # So [26] SQUARED LATIN CAPITAL LETTER A..SQUARED LATIN CAPITAL LETTER Z +1F150..1F169 ; Other_Alphabetic # So [26] NEGATIVE CIRCLED LATIN CAPITAL LETTER A..NEGATIVE CIRCLED LATIN CAPITAL LETTER Z +1F170..1F189 ; Other_Alphabetic # So [26] NEGATIVE SQUARED LATIN CAPITAL LETTER A..NEGATIVE SQUARED LATIN CAPITAL LETTER Z + +# Total code points: 1510 + +# ================================================ + +3006 ; Ideographic # Lo IDEOGRAPHIC CLOSING MARK +3007 ; Ideographic # Nl IDEOGRAPHIC NUMBER ZERO +3021..3029 ; Ideographic # Nl [9] HANGZHOU NUMERAL ONE..HANGZHOU NUMERAL NINE +3038..303A ; Ideographic # Nl [3] HANGZHOU NUMERAL TEN..HANGZHOU NUMERAL THIRTY +3400..4DBF ; Ideographic # Lo [6592] CJK UNIFIED IDEOGRAPH-3400..CJK UNIFIED IDEOGRAPH-4DBF +4E00..9FFF ; Ideographic # Lo [20992] CJK UNIFIED IDEOGRAPH-4E00..CJK UNIFIED IDEOGRAPH-9FFF +F900..FA6D ; Ideographic # Lo [366] CJK COMPATIBILITY IDEOGRAPH-F900..CJK COMPATIBILITY IDEOGRAPH-FA6D +FA70..FAD9 ; Ideographic # Lo [106] CJK COMPATIBILITY IDEOGRAPH-FA70..CJK COMPATIBILITY IDEOGRAPH-FAD9 +16FE4 ; Ideographic # Mn KHITAN SMALL SCRIPT FILLER +16FF2..16FF3 ; Ideographic # Lm [2] CHINESE SMALL SIMPLIFIED ER..CHINESE SMALL TRADITIONAL ER +16FF4..16FF6 ; Ideographic # Nl [3] YANGQIN SIGN SLOW ONE BEAT..YANGQIN SIGN SLOW TWO BEATS +17000..18CD5 ; Ideographic # Lo [7382] TANGUT IDEOGRAPH-17000..KHITAN SMALL SCRIPT CHARACTER-18CD5 +18CFF..18D1E ; Ideographic # Lo [32] KHITAN SMALL SCRIPT CHARACTER-18CFF..TANGUT IDEOGRAPH-18D1E +18D80..18DF2 ; Ideographic # Lo [115] TANGUT COMPONENT-769..TANGUT COMPONENT-883 +1B170..1B2FB ; Ideographic # Lo [396] NUSHU CHARACTER-1B170..NUSHU CHARACTER-1B2FB +20000..2A6DF ; Ideographic # Lo [42720] CJK UNIFIED IDEOGRAPH-20000..CJK UNIFIED IDEOGRAPH-2A6DF +2A700..2B81D ; Ideographic # Lo [4382] CJK UNIFIED IDEOGRAPH-2A700..CJK UNIFIED IDEOGRAPH-2B81D +2B820..2CEAD ; Ideographic # Lo [5774] CJK UNIFIED IDEOGRAPH-2B820..CJK UNIFIED IDEOGRAPH-2CEAD +2CEB0..2EBE0 ; Ideographic # Lo [7473] CJK UNIFIED IDEOGRAPH-2CEB0..CJK UNIFIED IDEOGRAPH-2EBE0 +2EBF0..2EE5D ; Ideographic # Lo [622] CJK UNIFIED IDEOGRAPH-2EBF0..CJK UNIFIED IDEOGRAPH-2EE5D +2F800..2FA1D ; Ideographic # Lo [542] CJK COMPATIBILITY IDEOGRAPH-2F800..CJK COMPATIBILITY IDEOGRAPH-2FA1D +30000..3134A ; Ideographic # Lo [4939] CJK UNIFIED IDEOGRAPH-30000..CJK UNIFIED IDEOGRAPH-3134A +31350..33479 ; Ideographic # Lo [8490] CJK UNIFIED IDEOGRAPH-31350..CJK UNIFIED IDEOGRAPH-33479 + +# Total code points: 110943 + +# ================================================ + +005E ; Diacritic # Sk CIRCUMFLEX ACCENT +0060 ; Diacritic # Sk GRAVE ACCENT +00A8 ; Diacritic # Sk DIAERESIS +00AF ; Diacritic # Sk MACRON +00B4 ; Diacritic # Sk ACUTE ACCENT +00B7 ; Diacritic # Po MIDDLE DOT +00B8 ; Diacritic # Sk CEDILLA +02B0..02C1 ; Diacritic # Lm [18] MODIFIER LETTER SMALL H..MODIFIER LETTER REVERSED GLOTTAL STOP +02C2..02C5 ; Diacritic # Sk [4] MODIFIER LETTER LEFT ARROWHEAD..MODIFIER LETTER DOWN ARROWHEAD +02C6..02D1 ; Diacritic # Lm [12] MODIFIER LETTER CIRCUMFLEX ACCENT..MODIFIER LETTER HALF TRIANGULAR COLON +02D2..02DF ; Diacritic # Sk [14] MODIFIER LETTER CENTRED RIGHT HALF RING..MODIFIER LETTER CROSS ACCENT +02E0..02E4 ; Diacritic # Lm [5] MODIFIER LETTER SMALL GAMMA..MODIFIER LETTER SMALL REVERSED GLOTTAL STOP +02E5..02EB ; Diacritic # Sk [7] MODIFIER LETTER EXTRA-HIGH TONE BAR..MODIFIER LETTER YANG DEPARTING TONE MARK +02EC ; Diacritic # Lm MODIFIER LETTER VOICING +02ED ; Diacritic # Sk MODIFIER LETTER UNASPIRATED +02EE ; Diacritic # Lm MODIFIER LETTER DOUBLE APOSTROPHE +02EF..02FF ; Diacritic # Sk [17] MODIFIER LETTER LOW DOWN ARROWHEAD..MODIFIER LETTER LOW LEFT ARROW +0300..034E ; Diacritic # Mn [79] COMBINING GRAVE ACCENT..COMBINING UPWARDS ARROW BELOW +0350..0357 ; Diacritic # Mn [8] COMBINING RIGHT ARROWHEAD ABOVE..COMBINING RIGHT HALF RING ABOVE +035D..0362 ; Diacritic # Mn [6] COMBINING DOUBLE BREVE..COMBINING DOUBLE RIGHTWARDS ARROW BELOW +0374 ; Diacritic # Lm GREEK NUMERAL SIGN +0375 ; Diacritic # Sk GREEK LOWER NUMERAL SIGN +037A ; Diacritic # Lm GREEK YPOGEGRAMMENI +0384..0385 ; Diacritic # Sk [2] GREEK TONOS..GREEK DIALYTIKA TONOS +0483..0487 ; Diacritic # Mn [5] COMBINING CYRILLIC TITLO..COMBINING CYRILLIC POKRYTIE +0559 ; Diacritic # Lm ARMENIAN MODIFIER LETTER LEFT HALF RING +0591..05BD ; Diacritic # Mn [45] HEBREW ACCENT ETNAHTA..HEBREW POINT METEG +05BF ; Diacritic # Mn HEBREW POINT RAFE +05C1..05C2 ; Diacritic # Mn [2] HEBREW POINT SHIN DOT..HEBREW POINT SIN DOT +05C4..05C5 ; Diacritic # Mn [2] HEBREW MARK UPPER DOT..HEBREW MARK LOWER DOT +05C7 ; Diacritic # Mn HEBREW POINT QAMATS QATAN +064B..0652 ; Diacritic # Mn [8] ARABIC FATHATAN..ARABIC SUKUN +0657..0658 ; Diacritic # Mn [2] ARABIC INVERTED DAMMA..ARABIC MARK NOON GHUNNA +06DF..06E0 ; Diacritic # Mn [2] ARABIC SMALL HIGH ROUNDED ZERO..ARABIC SMALL HIGH UPRIGHT RECTANGULAR ZERO +06E5..06E6 ; Diacritic # Lm [2] ARABIC SMALL WAW..ARABIC SMALL YEH +06EA..06EC ; Diacritic # Mn [3] ARABIC EMPTY CENTRE LOW STOP..ARABIC ROUNDED HIGH STOP WITH FILLED CENTRE +0730..074A ; Diacritic # Mn [27] SYRIAC PTHAHA ABOVE..SYRIAC BARREKH +07A6..07B0 ; Diacritic # Mn [11] THAANA ABAFILI..THAANA SUKUN +07EB..07F3 ; Diacritic # Mn [9] NKO COMBINING SHORT HIGH TONE..NKO COMBINING DOUBLE DOT ABOVE +07F4..07F5 ; Diacritic # Lm [2] NKO HIGH TONE APOSTROPHE..NKO LOW TONE APOSTROPHE +0818..0819 ; Diacritic # Mn [2] SAMARITAN MARK OCCLUSION..SAMARITAN MARK DAGESH +0898..089F ; Diacritic # Mn [8] ARABIC SMALL HIGH WORD AL-JUZ..ARABIC HALF MADDA OVER MADDA +08C9 ; Diacritic # Lm ARABIC SMALL FARSI YEH +08CA..08D2 ; Diacritic # Mn [9] ARABIC SMALL HIGH FARSI YEH..ARABIC LARGE ROUND DOT INSIDE CIRCLE BELOW +08E3..08FE ; Diacritic # Mn [28] ARABIC TURNED DAMMA BELOW..ARABIC DAMMA WITH DOT +093C ; Diacritic # Mn DEVANAGARI SIGN NUKTA +094D ; Diacritic # Mn DEVANAGARI SIGN VIRAMA +0951..0954 ; Diacritic # Mn [4] DEVANAGARI STRESS SIGN UDATTA..DEVANAGARI ACUTE ACCENT +0971 ; Diacritic # Lm DEVANAGARI SIGN HIGH SPACING DOT +09BC ; Diacritic # Mn BENGALI SIGN NUKTA +09CD ; Diacritic # Mn BENGALI SIGN VIRAMA +0A3C ; Diacritic # Mn GURMUKHI SIGN NUKTA +0A4D ; Diacritic # Mn GURMUKHI SIGN VIRAMA +0ABC ; Diacritic # Mn GUJARATI SIGN NUKTA +0ACD ; Diacritic # Mn GUJARATI SIGN VIRAMA +0AFD..0AFF ; Diacritic # Mn [3] GUJARATI SIGN THREE-DOT NUKTA ABOVE..GUJARATI SIGN TWO-CIRCLE NUKTA ABOVE +0B3C ; Diacritic # Mn ORIYA SIGN NUKTA +0B4D ; Diacritic # Mn ORIYA SIGN VIRAMA +0B55 ; Diacritic # Mn ORIYA SIGN OVERLINE +0BCD ; Diacritic # Mn TAMIL SIGN VIRAMA +0C3C ; Diacritic # Mn TELUGU SIGN NUKTA +0C4D ; Diacritic # Mn TELUGU SIGN VIRAMA +0CBC ; Diacritic # Mn KANNADA SIGN NUKTA +0CCD ; Diacritic # Mn KANNADA SIGN VIRAMA +0D3B..0D3C ; Diacritic # Mn [2] MALAYALAM SIGN VERTICAL BAR VIRAMA..MALAYALAM SIGN CIRCULAR VIRAMA +0D4D ; Diacritic # Mn MALAYALAM SIGN VIRAMA +0DCA ; Diacritic # Mn SINHALA SIGN AL-LAKUNA +0E3A ; Diacritic # Mn THAI CHARACTER PHINTHU +0E47..0E4C ; Diacritic # Mn [6] THAI CHARACTER MAITAIKHU..THAI CHARACTER THANTHAKHAT +0E4E ; Diacritic # Mn THAI CHARACTER YAMAKKAN +0EBA ; Diacritic # Mn LAO SIGN PALI VIRAMA +0EC8..0ECC ; Diacritic # Mn [5] LAO TONE MAI EK..LAO CANCELLATION MARK +0F18..0F19 ; Diacritic # Mn [2] TIBETAN ASTROLOGICAL SIGN -KHYUD PA..TIBETAN ASTROLOGICAL SIGN SDONG TSHUGS +0F35 ; Diacritic # Mn TIBETAN MARK NGAS BZUNG NYI ZLA +0F37 ; Diacritic # Mn TIBETAN MARK NGAS BZUNG SGOR RTAGS +0F39 ; Diacritic # Mn TIBETAN MARK TSA -PHRU +0F3E..0F3F ; Diacritic # Mc [2] TIBETAN SIGN YAR TSHES..TIBETAN SIGN MAR TSHES +0F82..0F84 ; Diacritic # Mn [3] TIBETAN SIGN NYI ZLA NAA DA..TIBETAN MARK HALANTA +0F86..0F87 ; Diacritic # Mn [2] TIBETAN SIGN LCI RTAGS..TIBETAN SIGN YANG RTAGS +0FC6 ; Diacritic # Mn TIBETAN SYMBOL PADMA GDAN +1037 ; Diacritic # Mn MYANMAR SIGN DOT BELOW +1039..103A ; Diacritic # Mn [2] MYANMAR SIGN VIRAMA..MYANMAR SIGN ASAT +1063..1064 ; Diacritic # Mc [2] MYANMAR TONE MARK SGAW KAREN HATHI..MYANMAR TONE MARK SGAW KAREN KE PHO +1069..106D ; Diacritic # Mc [5] MYANMAR SIGN WESTERN PWO KAREN TONE-1..MYANMAR SIGN WESTERN PWO KAREN TONE-5 +1087..108C ; Diacritic # Mc [6] MYANMAR SIGN SHAN TONE-2..MYANMAR SIGN SHAN COUNCIL TONE-3 +108D ; Diacritic # Mn MYANMAR SIGN SHAN COUNCIL EMPHATIC TONE +108F ; Diacritic # Mc MYANMAR SIGN RUMAI PALAUNG TONE-5 +109A..109B ; Diacritic # Mc [2] MYANMAR SIGN KHAMTI TONE-1..MYANMAR SIGN KHAMTI TONE-3 +135D..135F ; Diacritic # Mn [3] ETHIOPIC COMBINING GEMINATION AND VOWEL LENGTH MARK..ETHIOPIC COMBINING GEMINATION MARK +1714 ; Diacritic # Mn TAGALOG SIGN VIRAMA +1715 ; Diacritic # Mc TAGALOG SIGN PAMUDPOD +1734 ; Diacritic # Mc HANUNOO SIGN PAMUDPOD +17C9..17D3 ; Diacritic # Mn [11] KHMER SIGN MUUSIKATOAN..KHMER SIGN BATHAMASAT +17DD ; Diacritic # Mn KHMER SIGN ATTHACAN +1939..193B ; Diacritic # Mn [3] LIMBU SIGN MUKPHRENG..LIMBU SIGN SA-I +1A60 ; Diacritic # Mn TAI THAM SIGN SAKOT +1A75..1A7C ; Diacritic # Mn [8] TAI THAM SIGN TONE-1..TAI THAM SIGN KHUEN-LUE KARAN +1A7F ; Diacritic # Mn TAI THAM COMBINING CRYPTOGRAMMIC DOT +1AB0..1ABD ; Diacritic # Mn [14] COMBINING DOUBLED CIRCUMFLEX ACCENT..COMBINING PARENTHESES BELOW +1ABE ; Diacritic # Me COMBINING PARENTHESES OVERLAY +1AC1..1ACB ; Diacritic # Mn [11] COMBINING LEFT PARENTHESIS ABOVE LEFT..COMBINING TRIPLE ACUTE ACCENT +1ACF..1ADD ; Diacritic # Mn [15] COMBINING DOUBLE CARON..COMBINING DOT-AND-RING BELOW +1AE0..1AEB ; Diacritic # Mn [12] COMBINING LEFT TACK ABOVE..COMBINING DOUBLE RIGHTWARDS ARROW ABOVE +1B34 ; Diacritic # Mn BALINESE SIGN REREKAN +1B44 ; Diacritic # Mc BALINESE ADEG ADEG +1B6B..1B73 ; Diacritic # Mn [9] BALINESE MUSICAL SYMBOL COMBINING TEGEH..BALINESE MUSICAL SYMBOL COMBINING GONG +1BAA ; Diacritic # Mc SUNDANESE SIGN PAMAAEH +1BAB ; Diacritic # Mn SUNDANESE SIGN VIRAMA +1BE6 ; Diacritic # Mn BATAK SIGN TOMPI +1BF2..1BF3 ; Diacritic # Mc [2] BATAK PANGOLAT..BATAK PANONGONAN +1C36..1C37 ; Diacritic # Mn [2] LEPCHA SIGN RAN..LEPCHA SIGN NUKTA +1C78..1C7D ; Diacritic # Lm [6] OL CHIKI MU TTUDDAG..OL CHIKI AHAD +1CD0..1CD2 ; Diacritic # Mn [3] VEDIC TONE KARSHANA..VEDIC TONE PRENKHA +1CD3 ; Diacritic # Po VEDIC SIGN NIHSHVASA +1CD4..1CE0 ; Diacritic # Mn [13] VEDIC SIGN YAJURVEDIC MIDLINE SVARITA..VEDIC TONE RIGVEDIC KASHMIRI INDEPENDENT SVARITA +1CE1 ; Diacritic # Mc VEDIC TONE ATHARVAVEDIC INDEPENDENT SVARITA +1CE2..1CE8 ; Diacritic # Mn [7] VEDIC SIGN VISARGA SVARITA..VEDIC SIGN VISARGA ANUDATTA WITH TAIL +1CED ; Diacritic # Mn VEDIC SIGN TIRYAK +1CF4 ; Diacritic # Mn VEDIC TONE CANDRA ABOVE +1CF7 ; Diacritic # Mc VEDIC SIGN ATIKRAMA +1CF8..1CF9 ; Diacritic # Mn [2] VEDIC TONE RING ABOVE..VEDIC TONE DOUBLE RING ABOVE +1D2C..1D6A ; Diacritic # Lm [63] MODIFIER LETTER CAPITAL A..GREEK SUBSCRIPT SMALL LETTER CHI +1D9B..1DBE ; Diacritic # Lm [36] MODIFIER LETTER SMALL TURNED ALPHA..MODIFIER LETTER SMALL EZH +1DC4..1DCF ; Diacritic # Mn [12] COMBINING MACRON-ACUTE..COMBINING ZIGZAG BELOW +1DF5..1DFF ; Diacritic # Mn [11] COMBINING UP TACK ABOVE..COMBINING RIGHT ARROWHEAD AND DOWN ARROWHEAD BELOW +1FBD ; Diacritic # Sk GREEK KORONIS +1FBF..1FC1 ; Diacritic # Sk [3] GREEK PSILI..GREEK DIALYTIKA AND PERISPOMENI +1FCD..1FCF ; Diacritic # Sk [3] GREEK PSILI AND VARIA..GREEK PSILI AND PERISPOMENI +1FDD..1FDF ; Diacritic # Sk [3] GREEK DASIA AND VARIA..GREEK DASIA AND PERISPOMENI +1FED..1FEF ; Diacritic # Sk [3] GREEK DIALYTIKA AND VARIA..GREEK VARIA +1FFD..1FFE ; Diacritic # Sk [2] GREEK OXIA..GREEK DASIA +2CEF..2CF1 ; Diacritic # Mn [3] COPTIC COMBINING NI ABOVE..COPTIC COMBINING SPIRITUS LENIS +2E2F ; Diacritic # Lm VERTICAL TILDE +302A..302D ; Diacritic # Mn [4] IDEOGRAPHIC LEVEL TONE MARK..IDEOGRAPHIC ENTERING TONE MARK +302E..302F ; Diacritic # Mc [2] HANGUL SINGLE DOT TONE MARK..HANGUL DOUBLE DOT TONE MARK +3099..309A ; Diacritic # Mn [2] COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK..COMBINING KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK +309B..309C ; Diacritic # Sk [2] KATAKANA-HIRAGANA VOICED SOUND MARK..KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK +30FC ; Diacritic # Lm KATAKANA-HIRAGANA PROLONGED SOUND MARK +A66F ; Diacritic # Mn COMBINING CYRILLIC VZMET +A67C..A67D ; Diacritic # Mn [2] COMBINING CYRILLIC KAVYKA..COMBINING CYRILLIC PAYEROK +A67F ; Diacritic # Lm CYRILLIC PAYEROK +A69C..A69D ; Diacritic # Lm [2] MODIFIER LETTER CYRILLIC HARD SIGN..MODIFIER LETTER CYRILLIC SOFT SIGN +A6F0..A6F1 ; Diacritic # Mn [2] BAMUM COMBINING MARK KOQNDON..BAMUM COMBINING MARK TUKWENTIS +A700..A716 ; Diacritic # Sk [23] MODIFIER LETTER CHINESE TONE YIN PING..MODIFIER LETTER EXTRA-LOW LEFT-STEM TONE BAR +A717..A71F ; Diacritic # Lm [9] MODIFIER LETTER DOT VERTICAL BAR..MODIFIER LETTER LOW INVERTED EXCLAMATION MARK +A720..A721 ; Diacritic # Sk [2] MODIFIER LETTER STRESS AND HIGH TONE..MODIFIER LETTER STRESS AND LOW TONE +A788 ; Diacritic # Lm MODIFIER LETTER LOW CIRCUMFLEX ACCENT +A789..A78A ; Diacritic # Sk [2] MODIFIER LETTER COLON..MODIFIER LETTER SHORT EQUALS SIGN +A7F1 ; Diacritic # Lm MODIFIER LETTER CAPITAL S +A7F8..A7F9 ; Diacritic # Lm [2] MODIFIER LETTER CAPITAL H WITH STROKE..MODIFIER LETTER SMALL LIGATURE OE +A806 ; Diacritic # Mn SYLOTI NAGRI SIGN HASANTA +A82C ; Diacritic # Mn SYLOTI NAGRI SIGN ALTERNATE HASANTA +A8C4 ; Diacritic # Mn SAURASHTRA SIGN VIRAMA +A8E0..A8F1 ; Diacritic # Mn [18] COMBINING DEVANAGARI DIGIT ZERO..COMBINING DEVANAGARI SIGN AVAGRAHA +A92B..A92D ; Diacritic # Mn [3] KAYAH LI TONE PLOPHU..KAYAH LI TONE CALYA PLOPHU +A92E ; Diacritic # Po KAYAH LI SIGN CWI +A953 ; Diacritic # Mc REJANG VIRAMA +A9B3 ; Diacritic # Mn JAVANESE SIGN CECAK TELU +A9C0 ; Diacritic # Mc JAVANESE PANGKON +A9E5 ; Diacritic # Mn MYANMAR SIGN SHAN SAW +AA7B ; Diacritic # Mc MYANMAR SIGN PAO KAREN TONE +AA7C ; Diacritic # Mn MYANMAR SIGN TAI LAING TONE-2 +AA7D ; Diacritic # Mc MYANMAR SIGN TAI LAING TONE-5 +AABF ; Diacritic # Mn TAI VIET TONE MAI EK +AAC0 ; Diacritic # Lo TAI VIET TONE MAI NUENG +AAC1 ; Diacritic # Mn TAI VIET TONE MAI THO +AAC2 ; Diacritic # Lo TAI VIET TONE MAI SONG +AAF6 ; Diacritic # Mn MEETEI MAYEK VIRAMA +AB5B ; Diacritic # Sk MODIFIER BREVE WITH INVERTED BREVE +AB5C..AB5F ; Diacritic # Lm [4] MODIFIER LETTER SMALL HENG..MODIFIER LETTER SMALL U WITH LEFT HOOK +AB69 ; Diacritic # Lm MODIFIER LETTER SMALL TURNED W +AB6A..AB6B ; Diacritic # Sk [2] MODIFIER LETTER LEFT TACK..MODIFIER LETTER RIGHT TACK +ABEC ; Diacritic # Mc MEETEI MAYEK LUM IYEK +ABED ; Diacritic # Mn MEETEI MAYEK APUN IYEK +FB1E ; Diacritic # Mn HEBREW POINT JUDEO-SPANISH VARIKA +FE20..FE2F ; Diacritic # Mn [16] COMBINING LIGATURE LEFT HALF..COMBINING CYRILLIC TITLO RIGHT HALF +FF3E ; Diacritic # Sk FULLWIDTH CIRCUMFLEX ACCENT +FF40 ; Diacritic # Sk FULLWIDTH GRAVE ACCENT +FF70 ; Diacritic # Lm HALFWIDTH KATAKANA-HIRAGANA PROLONGED SOUND MARK +FF9E..FF9F ; Diacritic # Lm [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK +FFE3 ; Diacritic # Sk FULLWIDTH MACRON +102E0 ; Diacritic # Mn COPTIC EPACT THOUSANDS MARK +10780..10785 ; Diacritic # Lm [6] MODIFIER LETTER SMALL CAPITAL AA..MODIFIER LETTER SMALL B WITH HOOK +10787..107B0 ; Diacritic # Lm [42] MODIFIER LETTER SMALL DZ DIGRAPH..MODIFIER LETTER SMALL V WITH RIGHT HOOK +107B2..107BA ; Diacritic # Lm [9] MODIFIER LETTER SMALL CAPITAL Y..MODIFIER LETTER SMALL S WITH CURL +10A38..10A3A ; Diacritic # Mn [3] KHAROSHTHI SIGN BAR ABOVE..KHAROSHTHI SIGN DOT BELOW +10A3F ; Diacritic # Mn KHAROSHTHI VIRAMA +10AE5..10AE6 ; Diacritic # Mn [2] MANICHAEAN ABBREVIATION MARK ABOVE..MANICHAEAN ABBREVIATION MARK BELOW +10D22..10D23 ; Diacritic # Lo [2] HANIFI ROHINGYA MARK SAKIN..HANIFI ROHINGYA MARK NA KHONNA +10D24..10D27 ; Diacritic # Mn [4] HANIFI ROHINGYA SIGN HARBAHAY..HANIFI ROHINGYA SIGN TASSI +10D4E ; Diacritic # Lm GARAY VOWEL LENGTH MARK +10D69..10D6D ; Diacritic # Mn [5] GARAY VOWEL SIGN E..GARAY CONSONANT NASALIZATION MARK +10EFA ; Diacritic # Mn ARABIC DOUBLE VERTICAL BAR BELOW +10EFD..10EFF ; Diacritic # Mn [3] ARABIC SMALL LOW WORD SAKTA..ARABIC SMALL LOW WORD MADDA +10F46..10F50 ; Diacritic # Mn [11] SOGDIAN COMBINING DOT BELOW..SOGDIAN COMBINING STROKE BELOW +10F82..10F85 ; Diacritic # Mn [4] OLD UYGHUR COMBINING DOT ABOVE..OLD UYGHUR COMBINING TWO DOTS BELOW +11046 ; Diacritic # Mn BRAHMI VIRAMA +11070 ; Diacritic # Mn BRAHMI SIGN OLD TAMIL VIRAMA +110B9..110BA ; Diacritic # Mn [2] KAITHI SIGN VIRAMA..KAITHI SIGN NUKTA +11133..11134 ; Diacritic # Mn [2] CHAKMA VIRAMA..CHAKMA MAAYYAA +11173 ; Diacritic # Mn MAHAJANI SIGN NUKTA +111C0 ; Diacritic # Mc SHARADA SIGN VIRAMA +111CA..111CC ; Diacritic # Mn [3] SHARADA SIGN NUKTA..SHARADA EXTRA SHORT VOWEL MARK +11235 ; Diacritic # Mc KHOJKI SIGN VIRAMA +11236 ; Diacritic # Mn KHOJKI SIGN NUKTA +112E9..112EA ; Diacritic # Mn [2] KHUDAWADI SIGN NUKTA..KHUDAWADI SIGN VIRAMA +1133B..1133C ; Diacritic # Mn [2] COMBINING BINDU BELOW..GRANTHA SIGN NUKTA +1134D ; Diacritic # Mc GRANTHA SIGN VIRAMA +11366..1136C ; Diacritic # Mn [7] COMBINING GRANTHA DIGIT ZERO..COMBINING GRANTHA DIGIT SIX +11370..11374 ; Diacritic # Mn [5] COMBINING GRANTHA LETTER A..COMBINING GRANTHA LETTER PA +113CE ; Diacritic # Mn TULU-TIGALARI SIGN VIRAMA +113CF ; Diacritic # Mc TULU-TIGALARI SIGN LOOPED VIRAMA +113D0 ; Diacritic # Mn TULU-TIGALARI CONJOINER +113D2 ; Diacritic # Mn TULU-TIGALARI GEMINATION MARK +113D3 ; Diacritic # Lo TULU-TIGALARI SIGN PLUTA +113E1..113E2 ; Diacritic # Mn [2] TULU-TIGALARI VEDIC TONE SVARITA..TULU-TIGALARI VEDIC TONE ANUDATTA +11442 ; Diacritic # Mn NEWA SIGN VIRAMA +11446 ; Diacritic # Mn NEWA SIGN NUKTA +114C2..114C3 ; Diacritic # Mn [2] TIRHUTA SIGN VIRAMA..TIRHUTA SIGN NUKTA +115BF..115C0 ; Diacritic # Mn [2] SIDDHAM SIGN VIRAMA..SIDDHAM SIGN NUKTA +1163F ; Diacritic # Mn MODI SIGN VIRAMA +116B6 ; Diacritic # Mc TAKRI SIGN VIRAMA +116B7 ; Diacritic # Mn TAKRI SIGN NUKTA +1172B ; Diacritic # Mn AHOM SIGN KILLER +11839..1183A ; Diacritic # Mn [2] DOGRA SIGN VIRAMA..DOGRA SIGN NUKTA +1193D ; Diacritic # Mc DIVES AKURU SIGN HALANTA +1193E ; Diacritic # Mn DIVES AKURU VIRAMA +11943 ; Diacritic # Mn DIVES AKURU SIGN NUKTA +119E0 ; Diacritic # Mn NANDINAGARI SIGN VIRAMA +11A34 ; Diacritic # Mn ZANABAZAR SQUARE SIGN VIRAMA +11A47 ; Diacritic # Mn ZANABAZAR SQUARE SUBJOINER +11A99 ; Diacritic # Mn SOYOMBO SUBJOINER +11C3F ; Diacritic # Mn BHAIKSUKI SIGN VIRAMA +11D42 ; Diacritic # Mn MASARAM GONDI SIGN NUKTA +11D44..11D45 ; Diacritic # Mn [2] MASARAM GONDI SIGN HALANTA..MASARAM GONDI VIRAMA +11D97 ; Diacritic # Mn GUNJALA GONDI VIRAMA +11DD9 ; Diacritic # Lm TOLONG SIKI SIGN SELA +11F41 ; Diacritic # Mc KAWI SIGN KILLER +11F42 ; Diacritic # Mn KAWI CONJOINER +11F5A ; Diacritic # Mn KAWI SIGN NUKTA +13447..13455 ; Diacritic # Mn [15] EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP START..EGYPTIAN HIEROGLYPH MODIFIER DAMAGED +1612F ; Diacritic # Mn GURUNG KHEMA SIGN THOLHOMA +16AF0..16AF4 ; Diacritic # Mn [5] BASSA VAH COMBINING HIGH TONE..BASSA VAH COMBINING HIGH-LOW TONE +16B30..16B36 ; Diacritic # Mn [7] PAHAWH HMONG MARK CIM TUB..PAHAWH HMONG MARK CIM TAUM +16D6B..16D6C ; Diacritic # Lm [2] KIRAT RAI SIGN VIRAMA..KIRAT RAI SIGN SAAT +16F8F..16F92 ; Diacritic # Mn [4] MIAO TONE RIGHT..MIAO TONE BELOW +16F93..16F9F ; Diacritic # Lm [13] MIAO LETTER TONE-2..MIAO LETTER REFORMED TONE-8 +16FF0..16FF1 ; Diacritic # Mc [2] VIETNAMESE ALTERNATE READING MARK CA..VIETNAMESE ALTERNATE READING MARK NHAY +1AFF0..1AFF3 ; Diacritic # Lm [4] KATAKANA LETTER MINNAN TONE-2..KATAKANA LETTER MINNAN TONE-5 +1AFF5..1AFFB ; Diacritic # Lm [7] KATAKANA LETTER MINNAN TONE-7..KATAKANA LETTER MINNAN NASALIZED TONE-5 +1AFFD..1AFFE ; Diacritic # Lm [2] KATAKANA LETTER MINNAN NASALIZED TONE-7..KATAKANA LETTER MINNAN NASALIZED TONE-8 +1CF00..1CF2D ; Diacritic # Mn [46] ZNAMENNY COMBINING MARK GORAZDO NIZKO S KRYZHEM ON LEFT..ZNAMENNY COMBINING MARK KRYZH ON LEFT +1CF30..1CF46 ; Diacritic # Mn [23] ZNAMENNY COMBINING TONAL RANGE MARK MRACHNO..ZNAMENNY PRIZNAK MODIFIER ROG +1D167..1D169 ; Diacritic # Mn [3] MUSICAL SYMBOL COMBINING TREMOLO-1..MUSICAL SYMBOL COMBINING TREMOLO-3 +1D16D..1D172 ; Diacritic # Mc [6] MUSICAL SYMBOL COMBINING AUGMENTATION DOT..MUSICAL SYMBOL COMBINING FLAG-5 +1D17B..1D182 ; Diacritic # Mn [8] MUSICAL SYMBOL COMBINING ACCENT..MUSICAL SYMBOL COMBINING LOURE +1D185..1D18B ; Diacritic # Mn [7] MUSICAL SYMBOL COMBINING DOIT..MUSICAL SYMBOL COMBINING TRIPLE TONGUE +1D1AA..1D1AD ; Diacritic # Mn [4] MUSICAL SYMBOL COMBINING DOWN BOW..MUSICAL SYMBOL COMBINING SNAP PIZZICATO +1E030..1E06D ; Diacritic # Lm [62] MODIFIER LETTER CYRILLIC SMALL A..MODIFIER LETTER CYRILLIC SMALL STRAIGHT U WITH STROKE +1E130..1E136 ; Diacritic # Mn [7] NYIAKENG PUACHUE HMONG TONE-B..NYIAKENG PUACHUE HMONG TONE-D +1E2AE ; Diacritic # Mn TOTO SIGN RISING TONE +1E2EC..1E2EF ; Diacritic # Mn [4] WANCHO TONE TUP..WANCHO TONE KOINI +1E5EE..1E5EF ; Diacritic # Mn [2] OL ONAL SIGN MU..OL ONAL SIGN IKIR +1E8D0..1E8D6 ; Diacritic # Mn [7] MENDE KIKAKUI COMBINING NUMBER TEENS..MENDE KIKAKUI COMBINING NUMBER MILLIONS +1E944..1E946 ; Diacritic # Mn [3] ADLAM ALIF LENGTHENER..ADLAM GEMINATION MARK +1E948..1E94A ; Diacritic # Mn [3] ADLAM CONSONANT MODIFIER..ADLAM NUKTA + +# Total code points: 1247 + +# ================================================ + +00B7 ; Extender # Po MIDDLE DOT +02D0..02D1 ; Extender # Lm [2] MODIFIER LETTER TRIANGULAR COLON..MODIFIER LETTER HALF TRIANGULAR COLON +0640 ; Extender # Lm ARABIC TATWEEL +07FA ; Extender # Lm NKO LAJANYALAN +0A71 ; Extender # Mn GURMUKHI ADDAK +0AFB ; Extender # Mn GUJARATI SIGN SHADDA +0B55 ; Extender # Mn ORIYA SIGN OVERLINE +0E46 ; Extender # Lm THAI CHARACTER MAIYAMOK +0EC6 ; Extender # Lm LAO KO LA +180A ; Extender # Po MONGOLIAN NIRUGU +1843 ; Extender # Lm MONGOLIAN LETTER TODO LONG VOWEL SIGN +1AA7 ; Extender # Lm TAI THAM SIGN MAI YAMOK +1C36 ; Extender # Mn LEPCHA SIGN RAN +1C7B ; Extender # Lm OL CHIKI RELAA +3005 ; Extender # Lm IDEOGRAPHIC ITERATION MARK +3031..3035 ; Extender # Lm [5] VERTICAL KANA REPEAT MARK..VERTICAL KANA REPEAT MARK LOWER HALF +309D..309E ; Extender # Lm [2] HIRAGANA ITERATION MARK..HIRAGANA VOICED ITERATION MARK +30FC..30FE ; Extender # Lm [3] KATAKANA-HIRAGANA PROLONGED SOUND MARK..KATAKANA VOICED ITERATION MARK +A015 ; Extender # Lm YI SYLLABLE WU +A60C ; Extender # Lm VAI SYLLABLE LENGTHENER +A9CF ; Extender # Lm JAVANESE PANGRANGKEP +A9E6 ; Extender # Lm MYANMAR MODIFIER LETTER SHAN REDUPLICATION +AA70 ; Extender # Lm MYANMAR MODIFIER LETTER KHAMTI REDUPLICATION +AADD ; Extender # Lm TAI VIET SYMBOL SAM +AAF3..AAF4 ; Extender # Lm [2] MEETEI MAYEK SYLLABLE REPETITION MARK..MEETEI MAYEK WORD REPETITION MARK +FF70 ; Extender # Lm HALFWIDTH KATAKANA-HIRAGANA PROLONGED SOUND MARK +10781..10782 ; Extender # Lm [2] MODIFIER LETTER SUPERSCRIPT TRIANGULAR COLON..MODIFIER LETTER SUPERSCRIPT HALF TRIANGULAR COLON +10D4E ; Extender # Lm GARAY VOWEL LENGTH MARK +10D6A ; Extender # Mn GARAY CONSONANT GEMINATION MARK +10D6F ; Extender # Lm GARAY REDUPLICATION MARK +11237 ; Extender # Mn KHOJKI SIGN SHADDA +1135D ; Extender # Lo GRANTHA SIGN PLUTA +113D2 ; Extender # Mn TULU-TIGALARI GEMINATION MARK +113D3 ; Extender # Lo TULU-TIGALARI SIGN PLUTA +115C6..115C8 ; Extender # Po [3] SIDDHAM REPETITION MARK-1..SIDDHAM REPETITION MARK-3 +11A98 ; Extender # Mn SOYOMBO GEMINATION MARK +11DD9 ; Extender # Lm TOLONG SIKI SIGN SELA +16B42..16B43 ; Extender # Lm [2] PAHAWH HMONG SIGN VOS NRUA..PAHAWH HMONG SIGN IB YAM +16FE0..16FE1 ; Extender # Lm [2] TANGUT ITERATION MARK..NUSHU ITERATION MARK +16FE3 ; Extender # Lm OLD CHINESE ITERATION MARK +16FF2..16FF3 ; Extender # Lm [2] CHINESE SMALL SIMPLIFIED ER..CHINESE SMALL TRADITIONAL ER +1E13C..1E13D ; Extender # Lm [2] NYIAKENG PUACHUE HMONG SIGN XW XW..NYIAKENG PUACHUE HMONG SYLLABLE LENGTHENER +1E5EF ; Extender # Mn OL ONAL SIGN IKIR +1E944..1E946 ; Extender # Mn [3] ADLAM ALIF LENGTHENER..ADLAM GEMINATION MARK + +# Total code points: 62 + +# ================================================ + +00AA ; Other_Lowercase # Lo FEMININE ORDINAL INDICATOR +00BA ; Other_Lowercase # Lo MASCULINE ORDINAL INDICATOR +02B0..02B8 ; Other_Lowercase # Lm [9] MODIFIER LETTER SMALL H..MODIFIER LETTER SMALL Y +02C0..02C1 ; Other_Lowercase # Lm [2] MODIFIER LETTER GLOTTAL STOP..MODIFIER LETTER REVERSED GLOTTAL STOP +02E0..02E4 ; Other_Lowercase # Lm [5] MODIFIER LETTER SMALL GAMMA..MODIFIER LETTER SMALL REVERSED GLOTTAL STOP +0345 ; Other_Lowercase # Mn COMBINING GREEK YPOGEGRAMMENI +037A ; Other_Lowercase # Lm GREEK YPOGEGRAMMENI +10FC ; Other_Lowercase # Lm MODIFIER LETTER GEORGIAN NAR +1D2C..1D6A ; Other_Lowercase # Lm [63] MODIFIER LETTER CAPITAL A..GREEK SUBSCRIPT SMALL LETTER CHI +1D78 ; Other_Lowercase # Lm MODIFIER LETTER CYRILLIC EN +1D9B..1DBF ; Other_Lowercase # Lm [37] MODIFIER LETTER SMALL TURNED ALPHA..MODIFIER LETTER SMALL THETA +2071 ; Other_Lowercase # Lm SUPERSCRIPT LATIN SMALL LETTER I +207F ; Other_Lowercase # Lm SUPERSCRIPT LATIN SMALL LETTER N +2090..209C ; Other_Lowercase # Lm [13] LATIN SUBSCRIPT SMALL LETTER A..LATIN SUBSCRIPT SMALL LETTER T +2170..217F ; Other_Lowercase # Nl [16] SMALL ROMAN NUMERAL ONE..SMALL ROMAN NUMERAL ONE THOUSAND +24D0..24E9 ; Other_Lowercase # So [26] CIRCLED LATIN SMALL LETTER A..CIRCLED LATIN SMALL LETTER Z +2C7C..2C7D ; Other_Lowercase # Lm [2] LATIN SUBSCRIPT SMALL LETTER J..MODIFIER LETTER CAPITAL V +A69C..A69D ; Other_Lowercase # Lm [2] MODIFIER LETTER CYRILLIC HARD SIGN..MODIFIER LETTER CYRILLIC SOFT SIGN +A770 ; Other_Lowercase # Lm MODIFIER LETTER US +A7F1..A7F4 ; Other_Lowercase # Lm [4] MODIFIER LETTER CAPITAL S..MODIFIER LETTER CAPITAL Q +A7F8..A7F9 ; Other_Lowercase # Lm [2] MODIFIER LETTER CAPITAL H WITH STROKE..MODIFIER LETTER SMALL LIGATURE OE +AB5C..AB5F ; Other_Lowercase # Lm [4] MODIFIER LETTER SMALL HENG..MODIFIER LETTER SMALL U WITH LEFT HOOK +AB69 ; Other_Lowercase # Lm MODIFIER LETTER SMALL TURNED W +10780 ; Other_Lowercase # Lm MODIFIER LETTER SMALL CAPITAL AA +10783..10785 ; Other_Lowercase # Lm [3] MODIFIER LETTER SMALL AE..MODIFIER LETTER SMALL B WITH HOOK +10787..107B0 ; Other_Lowercase # Lm [42] MODIFIER LETTER SMALL DZ DIGRAPH..MODIFIER LETTER SMALL V WITH RIGHT HOOK +107B2..107BA ; Other_Lowercase # Lm [9] MODIFIER LETTER SMALL CAPITAL Y..MODIFIER LETTER SMALL S WITH CURL +1E030..1E06D ; Other_Lowercase # Lm [62] MODIFIER LETTER CYRILLIC SMALL A..MODIFIER LETTER CYRILLIC SMALL STRAIGHT U WITH STROKE + +# Total code points: 312 + +# ================================================ + +2160..216F ; Other_Uppercase # Nl [16] ROMAN NUMERAL ONE..ROMAN NUMERAL ONE THOUSAND +24B6..24CF ; Other_Uppercase # So [26] CIRCLED LATIN CAPITAL LETTER A..CIRCLED LATIN CAPITAL LETTER Z +1F130..1F149 ; Other_Uppercase # So [26] SQUARED LATIN CAPITAL LETTER A..SQUARED LATIN CAPITAL LETTER Z +1F150..1F169 ; Other_Uppercase # So [26] NEGATIVE CIRCLED LATIN CAPITAL LETTER A..NEGATIVE CIRCLED LATIN CAPITAL LETTER Z +1F170..1F189 ; Other_Uppercase # So [26] NEGATIVE SQUARED LATIN CAPITAL LETTER A..NEGATIVE SQUARED LATIN CAPITAL LETTER Z + +# Total code points: 120 + +# ================================================ + +FDD0..FDEF ; Noncharacter_Code_Point # Cn [32] .. +FFFE..FFFF ; Noncharacter_Code_Point # Cn [2] .. +1FFFE..1FFFF ; Noncharacter_Code_Point # Cn [2] .. +2FFFE..2FFFF ; Noncharacter_Code_Point # Cn [2] .. +3FFFE..3FFFF ; Noncharacter_Code_Point # Cn [2] .. +4FFFE..4FFFF ; Noncharacter_Code_Point # Cn [2] .. +5FFFE..5FFFF ; Noncharacter_Code_Point # Cn [2] .. +6FFFE..6FFFF ; Noncharacter_Code_Point # Cn [2] .. +7FFFE..7FFFF ; Noncharacter_Code_Point # Cn [2] .. +8FFFE..8FFFF ; Noncharacter_Code_Point # Cn [2] .. +9FFFE..9FFFF ; Noncharacter_Code_Point # Cn [2] .. +AFFFE..AFFFF ; Noncharacter_Code_Point # Cn [2] .. +BFFFE..BFFFF ; Noncharacter_Code_Point # Cn [2] .. +CFFFE..CFFFF ; Noncharacter_Code_Point # Cn [2] .. +DFFFE..DFFFF ; Noncharacter_Code_Point # Cn [2] .. +EFFFE..EFFFF ; Noncharacter_Code_Point # Cn [2] .. +FFFFE..FFFFF ; Noncharacter_Code_Point # Cn [2] .. +10FFFE..10FFFF; Noncharacter_Code_Point # Cn [2] .. + +# Total code points: 66 + +# ================================================ + +09BE ; Other_Grapheme_Extend # Mc BENGALI VOWEL SIGN AA +09D7 ; Other_Grapheme_Extend # Mc BENGALI AU LENGTH MARK +0B3E ; Other_Grapheme_Extend # Mc ORIYA VOWEL SIGN AA +0B57 ; Other_Grapheme_Extend # Mc ORIYA AU LENGTH MARK +0BBE ; Other_Grapheme_Extend # Mc TAMIL VOWEL SIGN AA +0BD7 ; Other_Grapheme_Extend # Mc TAMIL AU LENGTH MARK +0CC0 ; Other_Grapheme_Extend # Mc KANNADA VOWEL SIGN II +0CC2 ; Other_Grapheme_Extend # Mc KANNADA VOWEL SIGN UU +0CC7..0CC8 ; Other_Grapheme_Extend # Mc [2] KANNADA VOWEL SIGN EE..KANNADA VOWEL SIGN AI +0CCA..0CCB ; Other_Grapheme_Extend # Mc [2] KANNADA VOWEL SIGN O..KANNADA VOWEL SIGN OO +0CD5..0CD6 ; Other_Grapheme_Extend # Mc [2] KANNADA LENGTH MARK..KANNADA AI LENGTH MARK +0D3E ; Other_Grapheme_Extend # Mc MALAYALAM VOWEL SIGN AA +0D57 ; Other_Grapheme_Extend # Mc MALAYALAM AU LENGTH MARK +0DCF ; Other_Grapheme_Extend # Mc SINHALA VOWEL SIGN AELA-PILLA +0DDF ; Other_Grapheme_Extend # Mc SINHALA VOWEL SIGN GAYANUKITTA +1715 ; Other_Grapheme_Extend # Mc TAGALOG SIGN PAMUDPOD +1734 ; Other_Grapheme_Extend # Mc HANUNOO SIGN PAMUDPOD +1B35 ; Other_Grapheme_Extend # Mc BALINESE VOWEL SIGN TEDUNG +1B3B ; Other_Grapheme_Extend # Mc BALINESE VOWEL SIGN RA REPA TEDUNG +1B3D ; Other_Grapheme_Extend # Mc BALINESE VOWEL SIGN LA LENGA TEDUNG +1B43..1B44 ; Other_Grapheme_Extend # Mc [2] BALINESE VOWEL SIGN PEPET TEDUNG..BALINESE ADEG ADEG +1BAA ; Other_Grapheme_Extend # Mc SUNDANESE SIGN PAMAAEH +1BF2..1BF3 ; Other_Grapheme_Extend # Mc [2] BATAK PANGOLAT..BATAK PANONGONAN +200C ; Other_Grapheme_Extend # Cf ZERO WIDTH NON-JOINER +302E..302F ; Other_Grapheme_Extend # Mc [2] HANGUL SINGLE DOT TONE MARK..HANGUL DOUBLE DOT TONE MARK +A953 ; Other_Grapheme_Extend # Mc REJANG VIRAMA +A9C0 ; Other_Grapheme_Extend # Mc JAVANESE PANGKON +FF9E..FF9F ; Other_Grapheme_Extend # Lm [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK +111C0 ; Other_Grapheme_Extend # Mc SHARADA SIGN VIRAMA +11235 ; Other_Grapheme_Extend # Mc KHOJKI SIGN VIRAMA +1133E ; Other_Grapheme_Extend # Mc GRANTHA VOWEL SIGN AA +1134D ; Other_Grapheme_Extend # Mc GRANTHA SIGN VIRAMA +11357 ; Other_Grapheme_Extend # Mc GRANTHA AU LENGTH MARK +113B8 ; Other_Grapheme_Extend # Mc TULU-TIGALARI VOWEL SIGN AA +113C2 ; Other_Grapheme_Extend # Mc TULU-TIGALARI VOWEL SIGN EE +113C5 ; Other_Grapheme_Extend # Mc TULU-TIGALARI VOWEL SIGN AI +113C7..113C9 ; Other_Grapheme_Extend # Mc [3] TULU-TIGALARI VOWEL SIGN OO..TULU-TIGALARI AU LENGTH MARK +113CF ; Other_Grapheme_Extend # Mc TULU-TIGALARI SIGN LOOPED VIRAMA +114B0 ; Other_Grapheme_Extend # Mc TIRHUTA VOWEL SIGN AA +114BD ; Other_Grapheme_Extend # Mc TIRHUTA VOWEL SIGN SHORT O +115AF ; Other_Grapheme_Extend # Mc SIDDHAM VOWEL SIGN AA +116B6 ; Other_Grapheme_Extend # Mc TAKRI SIGN VIRAMA +11930 ; Other_Grapheme_Extend # Mc DIVES AKURU VOWEL SIGN AA +1193D ; Other_Grapheme_Extend # Mc DIVES AKURU SIGN HALANTA +11F41 ; Other_Grapheme_Extend # Mc KAWI SIGN KILLER +16FF0..16FF1 ; Other_Grapheme_Extend # Mc [2] VIETNAMESE ALTERNATE READING MARK CA..VIETNAMESE ALTERNATE READING MARK NHAY +1D165..1D166 ; Other_Grapheme_Extend # Mc [2] MUSICAL SYMBOL COMBINING STEM..MUSICAL SYMBOL COMBINING SPRECHGESANG STEM +1D16D..1D172 ; Other_Grapheme_Extend # Mc [6] MUSICAL SYMBOL COMBINING AUGMENTATION DOT..MUSICAL SYMBOL COMBINING FLAG-5 +E0020..E007F ; Other_Grapheme_Extend # Cf [96] TAG SPACE..CANCEL TAG + +# Total code points: 160 + +# ================================================ + +2FF0..2FF1 ; IDS_Binary_Operator # So [2] IDEOGRAPHIC DESCRIPTION CHARACTER LEFT TO RIGHT..IDEOGRAPHIC DESCRIPTION CHARACTER ABOVE TO BELOW +2FF4..2FFD ; IDS_Binary_Operator # So [10] IDEOGRAPHIC DESCRIPTION CHARACTER FULL SURROUND..IDEOGRAPHIC DESCRIPTION CHARACTER SURROUND FROM LOWER RIGHT +31EF ; IDS_Binary_Operator # So IDEOGRAPHIC DESCRIPTION CHARACTER SUBTRACTION + +# Total code points: 13 + +# ================================================ + +2FF2..2FF3 ; IDS_Trinary_Operator # So [2] IDEOGRAPHIC DESCRIPTION CHARACTER LEFT TO MIDDLE AND RIGHT..IDEOGRAPHIC DESCRIPTION CHARACTER ABOVE TO MIDDLE AND BELOW + +# Total code points: 2 + +# ================================================ + +2FFE..2FFF ; IDS_Unary_Operator # So [2] IDEOGRAPHIC DESCRIPTION CHARACTER HORIZONTAL REFLECTION..IDEOGRAPHIC DESCRIPTION CHARACTER ROTATION + +# Total code points: 2 + +# ================================================ + +2E80..2E99 ; Radical # So [26] CJK RADICAL REPEAT..CJK RADICAL RAP +2E9B..2EF3 ; Radical # So [89] CJK RADICAL CHOKE..CJK RADICAL C-SIMPLIFIED TURTLE +2F00..2FD5 ; Radical # So [214] KANGXI RADICAL ONE..KANGXI RADICAL FLUTE + +# Total code points: 329 + +# ================================================ + +3400..4DBF ; Unified_Ideograph # Lo [6592] CJK UNIFIED IDEOGRAPH-3400..CJK UNIFIED IDEOGRAPH-4DBF +4E00..9FFF ; Unified_Ideograph # Lo [20992] CJK UNIFIED IDEOGRAPH-4E00..CJK UNIFIED IDEOGRAPH-9FFF +FA0E..FA0F ; Unified_Ideograph # Lo [2] CJK COMPATIBILITY IDEOGRAPH-FA0E..CJK COMPATIBILITY IDEOGRAPH-FA0F +FA11 ; Unified_Ideograph # Lo CJK COMPATIBILITY IDEOGRAPH-FA11 +FA13..FA14 ; Unified_Ideograph # Lo [2] CJK COMPATIBILITY IDEOGRAPH-FA13..CJK COMPATIBILITY IDEOGRAPH-FA14 +FA1F ; Unified_Ideograph # Lo CJK COMPATIBILITY IDEOGRAPH-FA1F +FA21 ; Unified_Ideograph # Lo CJK COMPATIBILITY IDEOGRAPH-FA21 +FA23..FA24 ; Unified_Ideograph # Lo [2] CJK COMPATIBILITY IDEOGRAPH-FA23..CJK COMPATIBILITY IDEOGRAPH-FA24 +FA27..FA29 ; Unified_Ideograph # Lo [3] CJK COMPATIBILITY IDEOGRAPH-FA27..CJK COMPATIBILITY IDEOGRAPH-FA29 +20000..2A6DF ; Unified_Ideograph # Lo [42720] CJK UNIFIED IDEOGRAPH-20000..CJK UNIFIED IDEOGRAPH-2A6DF +2A700..2B81D ; Unified_Ideograph # Lo [4382] CJK UNIFIED IDEOGRAPH-2A700..CJK UNIFIED IDEOGRAPH-2B81D +2B820..2CEAD ; Unified_Ideograph # Lo [5774] CJK UNIFIED IDEOGRAPH-2B820..CJK UNIFIED IDEOGRAPH-2CEAD +2CEB0..2EBE0 ; Unified_Ideograph # Lo [7473] CJK UNIFIED IDEOGRAPH-2CEB0..CJK UNIFIED IDEOGRAPH-2EBE0 +2EBF0..2EE5D ; Unified_Ideograph # Lo [622] CJK UNIFIED IDEOGRAPH-2EBF0..CJK UNIFIED IDEOGRAPH-2EE5D +30000..3134A ; Unified_Ideograph # Lo [4939] CJK UNIFIED IDEOGRAPH-30000..CJK UNIFIED IDEOGRAPH-3134A +31350..33479 ; Unified_Ideograph # Lo [8490] CJK UNIFIED IDEOGRAPH-31350..CJK UNIFIED IDEOGRAPH-33479 + +# Total code points: 101996 + +# ================================================ + +034F ; Other_Default_Ignorable_Code_Point # Mn COMBINING GRAPHEME JOINER +115F..1160 ; Other_Default_Ignorable_Code_Point # Lo [2] HANGUL CHOSEONG FILLER..HANGUL JUNGSEONG FILLER +17B4..17B5 ; Other_Default_Ignorable_Code_Point # Mn [2] KHMER VOWEL INHERENT AQ..KHMER VOWEL INHERENT AA +2065 ; Other_Default_Ignorable_Code_Point # Cn +3164 ; Other_Default_Ignorable_Code_Point # Lo HANGUL FILLER +FFA0 ; Other_Default_Ignorable_Code_Point # Lo HALFWIDTH HANGUL FILLER +FFF0..FFF8 ; Other_Default_Ignorable_Code_Point # Cn [9] .. +E0000 ; Other_Default_Ignorable_Code_Point # Cn +E0002..E001F ; Other_Default_Ignorable_Code_Point # Cn [30] .. +E0080..E00FF ; Other_Default_Ignorable_Code_Point # Cn [128] .. +E01F0..E0FFF ; Other_Default_Ignorable_Code_Point # Cn [3600] .. + +# Total code points: 3776 + +# ================================================ + +0149 ; Deprecated # L& LATIN SMALL LETTER N PRECEDED BY APOSTROPHE +0673 ; Deprecated # Lo ARABIC LETTER ALEF WITH WAVY HAMZA BELOW +0F77 ; Deprecated # Mn TIBETAN VOWEL SIGN VOCALIC RR +0F79 ; Deprecated # Mn TIBETAN VOWEL SIGN VOCALIC LL +17A3..17A4 ; Deprecated # Lo [2] KHMER INDEPENDENT VOWEL QAQ..KHMER INDEPENDENT VOWEL QAA +206A..206F ; Deprecated # Cf [6] INHIBIT SYMMETRIC SWAPPING..NOMINAL DIGIT SHAPES +2329 ; Deprecated # Ps LEFT-POINTING ANGLE BRACKET +232A ; Deprecated # Pe RIGHT-POINTING ANGLE BRACKET +E0001 ; Deprecated # Cf LANGUAGE TAG + +# Total code points: 15 + +# ================================================ + +0069..006A ; Soft_Dotted # L& [2] LATIN SMALL LETTER I..LATIN SMALL LETTER J +012F ; Soft_Dotted # L& LATIN SMALL LETTER I WITH OGONEK +0249 ; Soft_Dotted # L& LATIN SMALL LETTER J WITH STROKE +0268 ; Soft_Dotted # L& LATIN SMALL LETTER I WITH STROKE +029D ; Soft_Dotted # L& LATIN SMALL LETTER J WITH CROSSED-TAIL +02B2 ; Soft_Dotted # Lm MODIFIER LETTER SMALL J +03F3 ; Soft_Dotted # L& GREEK LETTER YOT +0456 ; Soft_Dotted # L& CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I +0458 ; Soft_Dotted # L& CYRILLIC SMALL LETTER JE +1D62 ; Soft_Dotted # Lm LATIN SUBSCRIPT SMALL LETTER I +1D96 ; Soft_Dotted # L& LATIN SMALL LETTER I WITH RETROFLEX HOOK +1DA4 ; Soft_Dotted # Lm MODIFIER LETTER SMALL I WITH STROKE +1DA8 ; Soft_Dotted # Lm MODIFIER LETTER SMALL J WITH CROSSED-TAIL +1E2D ; Soft_Dotted # L& LATIN SMALL LETTER I WITH TILDE BELOW +1ECB ; Soft_Dotted # L& LATIN SMALL LETTER I WITH DOT BELOW +2071 ; Soft_Dotted # Lm SUPERSCRIPT LATIN SMALL LETTER I +2148..2149 ; Soft_Dotted # L& [2] DOUBLE-STRUCK ITALIC SMALL I..DOUBLE-STRUCK ITALIC SMALL J +2C7C ; Soft_Dotted # Lm LATIN SUBSCRIPT SMALL LETTER J +1D422..1D423 ; Soft_Dotted # L& [2] MATHEMATICAL BOLD SMALL I..MATHEMATICAL BOLD SMALL J +1D456..1D457 ; Soft_Dotted # L& [2] MATHEMATICAL ITALIC SMALL I..MATHEMATICAL ITALIC SMALL J +1D48A..1D48B ; Soft_Dotted # L& [2] MATHEMATICAL BOLD ITALIC SMALL I..MATHEMATICAL BOLD ITALIC SMALL J +1D4BE..1D4BF ; Soft_Dotted # L& [2] MATHEMATICAL SCRIPT SMALL I..MATHEMATICAL SCRIPT SMALL J +1D4F2..1D4F3 ; Soft_Dotted # L& [2] MATHEMATICAL BOLD SCRIPT SMALL I..MATHEMATICAL BOLD SCRIPT SMALL J +1D526..1D527 ; Soft_Dotted # L& [2] MATHEMATICAL FRAKTUR SMALL I..MATHEMATICAL FRAKTUR SMALL J +1D55A..1D55B ; Soft_Dotted # L& [2] MATHEMATICAL DOUBLE-STRUCK SMALL I..MATHEMATICAL DOUBLE-STRUCK SMALL J +1D58E..1D58F ; Soft_Dotted # L& [2] MATHEMATICAL BOLD FRAKTUR SMALL I..MATHEMATICAL BOLD FRAKTUR SMALL J +1D5C2..1D5C3 ; Soft_Dotted # L& [2] MATHEMATICAL SANS-SERIF SMALL I..MATHEMATICAL SANS-SERIF SMALL J +1D5F6..1D5F7 ; Soft_Dotted # L& [2] MATHEMATICAL SANS-SERIF BOLD SMALL I..MATHEMATICAL SANS-SERIF BOLD SMALL J +1D62A..1D62B ; Soft_Dotted # L& [2] MATHEMATICAL SANS-SERIF ITALIC SMALL I..MATHEMATICAL SANS-SERIF ITALIC SMALL J +1D65E..1D65F ; Soft_Dotted # L& [2] MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL I..MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL J +1D692..1D693 ; Soft_Dotted # L& [2] MATHEMATICAL MONOSPACE SMALL I..MATHEMATICAL MONOSPACE SMALL J +1DF1A ; Soft_Dotted # L& LATIN SMALL LETTER I WITH STROKE AND RETROFLEX HOOK +1E04C..1E04D ; Soft_Dotted # Lm [2] MODIFIER LETTER CYRILLIC SMALL BYELORUSSIAN-UKRAINIAN I..MODIFIER LETTER CYRILLIC SMALL JE +1E068 ; Soft_Dotted # Lm CYRILLIC SUBSCRIPT SMALL LETTER BYELORUSSIAN-UKRAINIAN I + +# Total code points: 50 + +# ================================================ + +0E40..0E44 ; Logical_Order_Exception # Lo [5] THAI CHARACTER SARA E..THAI CHARACTER SARA AI MAIMALAI +0EC0..0EC4 ; Logical_Order_Exception # Lo [5] LAO VOWEL SIGN E..LAO VOWEL SIGN AI +19B5..19B7 ; Logical_Order_Exception # Lo [3] NEW TAI LUE VOWEL SIGN E..NEW TAI LUE VOWEL SIGN O +19BA ; Logical_Order_Exception # Lo NEW TAI LUE VOWEL SIGN AY +AAB5..AAB6 ; Logical_Order_Exception # Lo [2] TAI VIET VOWEL E..TAI VIET VOWEL O +AAB9 ; Logical_Order_Exception # Lo TAI VIET VOWEL UEA +AABB..AABC ; Logical_Order_Exception # Lo [2] TAI VIET VOWEL AUE..TAI VIET VOWEL AY + +# Total code points: 19 + +# ================================================ + +1885..1886 ; Other_ID_Start # Mn [2] MONGOLIAN LETTER ALI GALI BALUDA..MONGOLIAN LETTER ALI GALI THREE BALUDA +2118 ; Other_ID_Start # Sm SCRIPT CAPITAL P +212E ; Other_ID_Start # So ESTIMATED SYMBOL +309B..309C ; Other_ID_Start # Sk [2] KATAKANA-HIRAGANA VOICED SOUND MARK..KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK + +# Total code points: 6 + +# ================================================ + +00B7 ; Other_ID_Continue # Po MIDDLE DOT +0387 ; Other_ID_Continue # Po GREEK ANO TELEIA +1369..1371 ; Other_ID_Continue # No [9] ETHIOPIC DIGIT ONE..ETHIOPIC DIGIT NINE +19DA ; Other_ID_Continue # No NEW TAI LUE THAM DIGIT ONE +200C..200D ; Other_ID_Continue # Cf [2] ZERO WIDTH NON-JOINER..ZERO WIDTH JOINER +30FB ; Other_ID_Continue # Po KATAKANA MIDDLE DOT +FF65 ; Other_ID_Continue # Po HALFWIDTH KATAKANA MIDDLE DOT + +# Total code points: 16 + +# ================================================ + +00B2..00B3 ; ID_Compat_Math_Continue # No [2] SUPERSCRIPT TWO..SUPERSCRIPT THREE +00B9 ; ID_Compat_Math_Continue # No SUPERSCRIPT ONE +2070 ; ID_Compat_Math_Continue # No SUPERSCRIPT ZERO +2074..2079 ; ID_Compat_Math_Continue # No [6] SUPERSCRIPT FOUR..SUPERSCRIPT NINE +207A..207C ; ID_Compat_Math_Continue # Sm [3] SUPERSCRIPT PLUS SIGN..SUPERSCRIPT EQUALS SIGN +207D ; ID_Compat_Math_Continue # Ps SUPERSCRIPT LEFT PARENTHESIS +207E ; ID_Compat_Math_Continue # Pe SUPERSCRIPT RIGHT PARENTHESIS +2080..2089 ; ID_Compat_Math_Continue # No [10] SUBSCRIPT ZERO..SUBSCRIPT NINE +208A..208C ; ID_Compat_Math_Continue # Sm [3] SUBSCRIPT PLUS SIGN..SUBSCRIPT EQUALS SIGN +208D ; ID_Compat_Math_Continue # Ps SUBSCRIPT LEFT PARENTHESIS +208E ; ID_Compat_Math_Continue # Pe SUBSCRIPT RIGHT PARENTHESIS +2202 ; ID_Compat_Math_Continue # Sm PARTIAL DIFFERENTIAL +2207 ; ID_Compat_Math_Continue # Sm NABLA +221E ; ID_Compat_Math_Continue # Sm INFINITY +1D6C1 ; ID_Compat_Math_Continue # Sm MATHEMATICAL BOLD NABLA +1D6DB ; ID_Compat_Math_Continue # Sm MATHEMATICAL BOLD PARTIAL DIFFERENTIAL +1D6FB ; ID_Compat_Math_Continue # Sm MATHEMATICAL ITALIC NABLA +1D715 ; ID_Compat_Math_Continue # Sm MATHEMATICAL ITALIC PARTIAL DIFFERENTIAL +1D735 ; ID_Compat_Math_Continue # Sm MATHEMATICAL BOLD ITALIC NABLA +1D74F ; ID_Compat_Math_Continue # Sm MATHEMATICAL BOLD ITALIC PARTIAL DIFFERENTIAL +1D76F ; ID_Compat_Math_Continue # Sm MATHEMATICAL SANS-SERIF BOLD NABLA +1D789 ; ID_Compat_Math_Continue # Sm MATHEMATICAL SANS-SERIF BOLD PARTIAL DIFFERENTIAL +1D7A9 ; ID_Compat_Math_Continue # Sm MATHEMATICAL SANS-SERIF BOLD ITALIC NABLA +1D7C3 ; ID_Compat_Math_Continue # Sm MATHEMATICAL SANS-SERIF BOLD ITALIC PARTIAL DIFFERENTIAL + +# Total code points: 43 + +# ================================================ + +2202 ; ID_Compat_Math_Start # Sm PARTIAL DIFFERENTIAL +2207 ; ID_Compat_Math_Start # Sm NABLA +221E ; ID_Compat_Math_Start # Sm INFINITY +1D6C1 ; ID_Compat_Math_Start # Sm MATHEMATICAL BOLD NABLA +1D6DB ; ID_Compat_Math_Start # Sm MATHEMATICAL BOLD PARTIAL DIFFERENTIAL +1D6FB ; ID_Compat_Math_Start # Sm MATHEMATICAL ITALIC NABLA +1D715 ; ID_Compat_Math_Start # Sm MATHEMATICAL ITALIC PARTIAL DIFFERENTIAL +1D735 ; ID_Compat_Math_Start # Sm MATHEMATICAL BOLD ITALIC NABLA +1D74F ; ID_Compat_Math_Start # Sm MATHEMATICAL BOLD ITALIC PARTIAL DIFFERENTIAL +1D76F ; ID_Compat_Math_Start # Sm MATHEMATICAL SANS-SERIF BOLD NABLA +1D789 ; ID_Compat_Math_Start # Sm MATHEMATICAL SANS-SERIF BOLD PARTIAL DIFFERENTIAL +1D7A9 ; ID_Compat_Math_Start # Sm MATHEMATICAL SANS-SERIF BOLD ITALIC NABLA +1D7C3 ; ID_Compat_Math_Start # Sm MATHEMATICAL SANS-SERIF BOLD ITALIC PARTIAL DIFFERENTIAL + +# Total code points: 13 + +# ================================================ + +0021 ; Sentence_Terminal # Po EXCLAMATION MARK +002E ; Sentence_Terminal # Po FULL STOP +003F ; Sentence_Terminal # Po QUESTION MARK +0589 ; Sentence_Terminal # Po ARMENIAN FULL STOP +061D..061F ; Sentence_Terminal # Po [3] ARABIC END OF TEXT MARK..ARABIC QUESTION MARK +06D4 ; Sentence_Terminal # Po ARABIC FULL STOP +0700..0702 ; Sentence_Terminal # Po [3] SYRIAC END OF PARAGRAPH..SYRIAC SUBLINEAR FULL STOP +07F9 ; Sentence_Terminal # Po NKO EXCLAMATION MARK +0837 ; Sentence_Terminal # Po SAMARITAN PUNCTUATION MELODIC QITSA +0839 ; Sentence_Terminal # Po SAMARITAN PUNCTUATION QITSA +083D..083E ; Sentence_Terminal # Po [2] SAMARITAN PUNCTUATION SOF MASHFAAT..SAMARITAN PUNCTUATION ANNAAU +0964..0965 ; Sentence_Terminal # Po [2] DEVANAGARI DANDA..DEVANAGARI DOUBLE DANDA +104A..104B ; Sentence_Terminal # Po [2] MYANMAR SIGN LITTLE SECTION..MYANMAR SIGN SECTION +1362 ; Sentence_Terminal # Po ETHIOPIC FULL STOP +1367..1368 ; Sentence_Terminal # Po [2] ETHIOPIC QUESTION MARK..ETHIOPIC PARAGRAPH SEPARATOR +166E ; Sentence_Terminal # Po CANADIAN SYLLABICS FULL STOP +1735..1736 ; Sentence_Terminal # Po [2] PHILIPPINE SINGLE PUNCTUATION..PHILIPPINE DOUBLE PUNCTUATION +17D4..17D5 ; Sentence_Terminal # Po [2] KHMER SIGN KHAN..KHMER SIGN BARIYOOSAN +1803 ; Sentence_Terminal # Po MONGOLIAN FULL STOP +1809 ; Sentence_Terminal # Po MONGOLIAN MANCHU FULL STOP +1944..1945 ; Sentence_Terminal # Po [2] LIMBU EXCLAMATION MARK..LIMBU QUESTION MARK +1AA8..1AAB ; Sentence_Terminal # Po [4] TAI THAM SIGN KAAN..TAI THAM SIGN SATKAANKUU +1B4E..1B4F ; Sentence_Terminal # Po [2] BALINESE INVERTED CARIK SIKI..BALINESE INVERTED CARIK PAREREN +1B5A..1B5B ; Sentence_Terminal # Po [2] BALINESE PANTI..BALINESE PAMADA +1B5E..1B5F ; Sentence_Terminal # Po [2] BALINESE CARIK SIKI..BALINESE CARIK PAREREN +1B7D..1B7F ; Sentence_Terminal # Po [3] BALINESE PANTI LANTANG..BALINESE PANTI BAWAK +1C3B..1C3C ; Sentence_Terminal # Po [2] LEPCHA PUNCTUATION TA-ROL..LEPCHA PUNCTUATION NYET THYOOM TA-ROL +1C7E..1C7F ; Sentence_Terminal # Po [2] OL CHIKI PUNCTUATION MUCAAD..OL CHIKI PUNCTUATION DOUBLE MUCAAD +2024 ; Sentence_Terminal # Po ONE DOT LEADER +203C..203D ; Sentence_Terminal # Po [2] DOUBLE EXCLAMATION MARK..INTERROBANG +2047..2049 ; Sentence_Terminal # Po [3] DOUBLE QUESTION MARK..EXCLAMATION QUESTION MARK +2CF9..2CFB ; Sentence_Terminal # Po [3] COPTIC OLD NUBIAN FULL STOP..COPTIC OLD NUBIAN INDIRECT QUESTION MARK +2E2E ; Sentence_Terminal # Po REVERSED QUESTION MARK +2E3C ; Sentence_Terminal # Po STENOGRAPHIC FULL STOP +2E53..2E54 ; Sentence_Terminal # Po [2] MEDIEVAL EXCLAMATION MARK..MEDIEVAL QUESTION MARK +3002 ; Sentence_Terminal # Po IDEOGRAPHIC FULL STOP +A4FF ; Sentence_Terminal # Po LISU PUNCTUATION FULL STOP +A60E..A60F ; Sentence_Terminal # Po [2] VAI FULL STOP..VAI QUESTION MARK +A6F3 ; Sentence_Terminal # Po BAMUM FULL STOP +A6F7 ; Sentence_Terminal # Po BAMUM QUESTION MARK +A876..A877 ; Sentence_Terminal # Po [2] PHAGS-PA MARK SHAD..PHAGS-PA MARK DOUBLE SHAD +A8CE..A8CF ; Sentence_Terminal # Po [2] SAURASHTRA DANDA..SAURASHTRA DOUBLE DANDA +A92F ; Sentence_Terminal # Po KAYAH LI SIGN SHYA +A9C8..A9C9 ; Sentence_Terminal # Po [2] JAVANESE PADA LINGSA..JAVANESE PADA LUNGSI +AA5D..AA5F ; Sentence_Terminal # Po [3] CHAM PUNCTUATION DANDA..CHAM PUNCTUATION TRIPLE DANDA +AAF0..AAF1 ; Sentence_Terminal # Po [2] MEETEI MAYEK CHEIKHAN..MEETEI MAYEK AHANG KHUDAM +ABEB ; Sentence_Terminal # Po MEETEI MAYEK CHEIKHEI +FE12 ; Sentence_Terminal # Po PRESENTATION FORM FOR VERTICAL IDEOGRAPHIC FULL STOP +FE15..FE16 ; Sentence_Terminal # Po [2] PRESENTATION FORM FOR VERTICAL EXCLAMATION MARK..PRESENTATION FORM FOR VERTICAL QUESTION MARK +FE52 ; Sentence_Terminal # Po SMALL FULL STOP +FE56..FE57 ; Sentence_Terminal # Po [2] SMALL QUESTION MARK..SMALL EXCLAMATION MARK +FF01 ; Sentence_Terminal # Po FULLWIDTH EXCLAMATION MARK +FF0E ; Sentence_Terminal # Po FULLWIDTH FULL STOP +FF1F ; Sentence_Terminal # Po FULLWIDTH QUESTION MARK +FF61 ; Sentence_Terminal # Po HALFWIDTH IDEOGRAPHIC FULL STOP +10A56..10A57 ; Sentence_Terminal # Po [2] KHAROSHTHI PUNCTUATION DANDA..KHAROSHTHI PUNCTUATION DOUBLE DANDA +10F55..10F59 ; Sentence_Terminal # Po [5] SOGDIAN PUNCTUATION TWO VERTICAL BARS..SOGDIAN PUNCTUATION HALF CIRCLE WITH DOT +10F86..10F89 ; Sentence_Terminal # Po [4] OLD UYGHUR PUNCTUATION BAR..OLD UYGHUR PUNCTUATION FOUR DOTS +11047..11048 ; Sentence_Terminal # Po [2] BRAHMI DANDA..BRAHMI DOUBLE DANDA +110BE..110C1 ; Sentence_Terminal # Po [4] KAITHI SECTION MARK..KAITHI DOUBLE DANDA +11141..11143 ; Sentence_Terminal # Po [3] CHAKMA DANDA..CHAKMA QUESTION MARK +111C5..111C6 ; Sentence_Terminal # Po [2] SHARADA DANDA..SHARADA DOUBLE DANDA +111CD ; Sentence_Terminal # Po SHARADA SUTRA MARK +111DE..111DF ; Sentence_Terminal # Po [2] SHARADA SECTION MARK-1..SHARADA SECTION MARK-2 +11238..11239 ; Sentence_Terminal # Po [2] KHOJKI DANDA..KHOJKI DOUBLE DANDA +1123B..1123C ; Sentence_Terminal # Po [2] KHOJKI SECTION MARK..KHOJKI DOUBLE SECTION MARK +112A9 ; Sentence_Terminal # Po MULTANI SECTION MARK +113D4..113D5 ; Sentence_Terminal # Po [2] TULU-TIGALARI DANDA..TULU-TIGALARI DOUBLE DANDA +1144B..1144C ; Sentence_Terminal # Po [2] NEWA DANDA..NEWA DOUBLE DANDA +115C2..115C3 ; Sentence_Terminal # Po [2] SIDDHAM DANDA..SIDDHAM DOUBLE DANDA +115C9..115D7 ; Sentence_Terminal # Po [15] SIDDHAM END OF TEXT MARK..SIDDHAM SECTION MARK WITH CIRCLES AND FOUR ENCLOSURES +11641..11642 ; Sentence_Terminal # Po [2] MODI DANDA..MODI DOUBLE DANDA +1173C..1173E ; Sentence_Terminal # Po [3] AHOM SIGN SMALL SECTION..AHOM SIGN RULAI +11944 ; Sentence_Terminal # Po DIVES AKURU DOUBLE DANDA +11946 ; Sentence_Terminal # Po DIVES AKURU END OF TEXT MARK +11A42..11A43 ; Sentence_Terminal # Po [2] ZANABAZAR SQUARE MARK SHAD..ZANABAZAR SQUARE MARK DOUBLE SHAD +11A9B..11A9C ; Sentence_Terminal # Po [2] SOYOMBO MARK SHAD..SOYOMBO MARK DOUBLE SHAD +11C41..11C42 ; Sentence_Terminal # Po [2] BHAIKSUKI DANDA..BHAIKSUKI DOUBLE DANDA +11EF7..11EF8 ; Sentence_Terminal # Po [2] MAKASAR PASSIMBANG..MAKASAR END OF SECTION +11F43..11F44 ; Sentence_Terminal # Po [2] KAWI DANDA..KAWI DOUBLE DANDA +16A6E..16A6F ; Sentence_Terminal # Po [2] MRO DANDA..MRO DOUBLE DANDA +16AF5 ; Sentence_Terminal # Po BASSA VAH FULL STOP +16B37..16B38 ; Sentence_Terminal # Po [2] PAHAWH HMONG SIGN VOS THOM..PAHAWH HMONG SIGN VOS TSHAB CEEB +16B44 ; Sentence_Terminal # Po PAHAWH HMONG SIGN XAUS +16D6E..16D6F ; Sentence_Terminal # Po [2] KIRAT RAI DANDA..KIRAT RAI DOUBLE DANDA +16E98 ; Sentence_Terminal # Po MEDEFAIDRIN FULL STOP +1BC9F ; Sentence_Terminal # Po DUPLOYAN PUNCTUATION CHINOOK FULL STOP +1DA88 ; Sentence_Terminal # Po SIGNWRITING FULL STOP + +# Total code points: 170 + +# ================================================ + +180B..180D ; Variation_Selector # Mn [3] MONGOLIAN FREE VARIATION SELECTOR ONE..MONGOLIAN FREE VARIATION SELECTOR THREE +180F ; Variation_Selector # Mn MONGOLIAN FREE VARIATION SELECTOR FOUR +FE00..FE0F ; Variation_Selector # Mn [16] VARIATION SELECTOR-1..VARIATION SELECTOR-16 +E0100..E01EF ; Variation_Selector # Mn [240] VARIATION SELECTOR-17..VARIATION SELECTOR-256 + +# Total code points: 260 + +# ================================================ + +0009..000D ; Pattern_White_Space # Cc [5] .. +0020 ; Pattern_White_Space # Zs SPACE +0085 ; Pattern_White_Space # Cc +200E..200F ; Pattern_White_Space # Cf [2] LEFT-TO-RIGHT MARK..RIGHT-TO-LEFT MARK +2028 ; Pattern_White_Space # Zl LINE SEPARATOR +2029 ; Pattern_White_Space # Zp PARAGRAPH SEPARATOR + +# Total code points: 11 + +# ================================================ + +0021..0023 ; Pattern_Syntax # Po [3] EXCLAMATION MARK..NUMBER SIGN +0024 ; Pattern_Syntax # Sc DOLLAR SIGN +0025..0027 ; Pattern_Syntax # Po [3] PERCENT SIGN..APOSTROPHE +0028 ; Pattern_Syntax # Ps LEFT PARENTHESIS +0029 ; Pattern_Syntax # Pe RIGHT PARENTHESIS +002A ; Pattern_Syntax # Po ASTERISK +002B ; Pattern_Syntax # Sm PLUS SIGN +002C ; Pattern_Syntax # Po COMMA +002D ; Pattern_Syntax # Pd HYPHEN-MINUS +002E..002F ; Pattern_Syntax # Po [2] FULL STOP..SOLIDUS +003A..003B ; Pattern_Syntax # Po [2] COLON..SEMICOLON +003C..003E ; Pattern_Syntax # Sm [3] LESS-THAN SIGN..GREATER-THAN SIGN +003F..0040 ; Pattern_Syntax # Po [2] QUESTION MARK..COMMERCIAL AT +005B ; Pattern_Syntax # Ps LEFT SQUARE BRACKET +005C ; Pattern_Syntax # Po REVERSE SOLIDUS +005D ; Pattern_Syntax # Pe RIGHT SQUARE BRACKET +005E ; Pattern_Syntax # Sk CIRCUMFLEX ACCENT +0060 ; Pattern_Syntax # Sk GRAVE ACCENT +007B ; Pattern_Syntax # Ps LEFT CURLY BRACKET +007C ; Pattern_Syntax # Sm VERTICAL LINE +007D ; Pattern_Syntax # Pe RIGHT CURLY BRACKET +007E ; Pattern_Syntax # Sm TILDE +00A1 ; Pattern_Syntax # Po INVERTED EXCLAMATION MARK +00A2..00A5 ; Pattern_Syntax # Sc [4] CENT SIGN..YEN SIGN +00A6 ; Pattern_Syntax # So BROKEN BAR +00A7 ; Pattern_Syntax # Po SECTION SIGN +00A9 ; Pattern_Syntax # So COPYRIGHT SIGN +00AB ; Pattern_Syntax # Pi LEFT-POINTING DOUBLE ANGLE QUOTATION MARK +00AC ; Pattern_Syntax # Sm NOT SIGN +00AE ; Pattern_Syntax # So REGISTERED SIGN +00B0 ; Pattern_Syntax # So DEGREE SIGN +00B1 ; Pattern_Syntax # Sm PLUS-MINUS SIGN +00B6 ; Pattern_Syntax # Po PILCROW SIGN +00BB ; Pattern_Syntax # Pf RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK +00BF ; Pattern_Syntax # Po INVERTED QUESTION MARK +00D7 ; Pattern_Syntax # Sm MULTIPLICATION SIGN +00F7 ; Pattern_Syntax # Sm DIVISION SIGN +2010..2015 ; Pattern_Syntax # Pd [6] HYPHEN..HORIZONTAL BAR +2016..2017 ; Pattern_Syntax # Po [2] DOUBLE VERTICAL LINE..DOUBLE LOW LINE +2018 ; Pattern_Syntax # Pi LEFT SINGLE QUOTATION MARK +2019 ; Pattern_Syntax # Pf RIGHT SINGLE QUOTATION MARK +201A ; Pattern_Syntax # Ps SINGLE LOW-9 QUOTATION MARK +201B..201C ; Pattern_Syntax # Pi [2] SINGLE HIGH-REVERSED-9 QUOTATION MARK..LEFT DOUBLE QUOTATION MARK +201D ; Pattern_Syntax # Pf RIGHT DOUBLE QUOTATION MARK +201E ; Pattern_Syntax # Ps DOUBLE LOW-9 QUOTATION MARK +201F ; Pattern_Syntax # Pi DOUBLE HIGH-REVERSED-9 QUOTATION MARK +2020..2027 ; Pattern_Syntax # Po [8] DAGGER..HYPHENATION POINT +2030..2038 ; Pattern_Syntax # Po [9] PER MILLE SIGN..CARET +2039 ; Pattern_Syntax # Pi SINGLE LEFT-POINTING ANGLE QUOTATION MARK +203A ; Pattern_Syntax # Pf SINGLE RIGHT-POINTING ANGLE QUOTATION MARK +203B..203E ; Pattern_Syntax # Po [4] REFERENCE MARK..OVERLINE +2041..2043 ; Pattern_Syntax # Po [3] CARET INSERTION POINT..HYPHEN BULLET +2044 ; Pattern_Syntax # Sm FRACTION SLASH +2045 ; Pattern_Syntax # Ps LEFT SQUARE BRACKET WITH QUILL +2046 ; Pattern_Syntax # Pe RIGHT SQUARE BRACKET WITH QUILL +2047..2051 ; Pattern_Syntax # Po [11] DOUBLE QUESTION MARK..TWO ASTERISKS ALIGNED VERTICALLY +2052 ; Pattern_Syntax # Sm COMMERCIAL MINUS SIGN +2053 ; Pattern_Syntax # Po SWUNG DASH +2055..205E ; Pattern_Syntax # Po [10] FLOWER PUNCTUATION MARK..VERTICAL FOUR DOTS +2190..2194 ; Pattern_Syntax # Sm [5] LEFTWARDS ARROW..LEFT RIGHT ARROW +2195..2199 ; Pattern_Syntax # So [5] UP DOWN ARROW..SOUTH WEST ARROW +219A..219B ; Pattern_Syntax # Sm [2] LEFTWARDS ARROW WITH STROKE..RIGHTWARDS ARROW WITH STROKE +219C..219F ; Pattern_Syntax # So [4] LEFTWARDS WAVE ARROW..UPWARDS TWO HEADED ARROW +21A0 ; Pattern_Syntax # Sm RIGHTWARDS TWO HEADED ARROW +21A1..21A2 ; Pattern_Syntax # So [2] DOWNWARDS TWO HEADED ARROW..LEFTWARDS ARROW WITH TAIL +21A3 ; Pattern_Syntax # Sm RIGHTWARDS ARROW WITH TAIL +21A4..21A5 ; Pattern_Syntax # So [2] LEFTWARDS ARROW FROM BAR..UPWARDS ARROW FROM BAR +21A6 ; Pattern_Syntax # Sm RIGHTWARDS ARROW FROM BAR +21A7..21AD ; Pattern_Syntax # So [7] DOWNWARDS ARROW FROM BAR..LEFT RIGHT WAVE ARROW +21AE ; Pattern_Syntax # Sm LEFT RIGHT ARROW WITH STROKE +21AF..21CD ; Pattern_Syntax # So [31] DOWNWARDS ZIGZAG ARROW..LEFTWARDS DOUBLE ARROW WITH STROKE +21CE..21CF ; Pattern_Syntax # Sm [2] LEFT RIGHT DOUBLE ARROW WITH STROKE..RIGHTWARDS DOUBLE ARROW WITH STROKE +21D0..21D1 ; Pattern_Syntax # So [2] LEFTWARDS DOUBLE ARROW..UPWARDS DOUBLE ARROW +21D2 ; Pattern_Syntax # Sm RIGHTWARDS DOUBLE ARROW +21D3 ; Pattern_Syntax # So DOWNWARDS DOUBLE ARROW +21D4 ; Pattern_Syntax # Sm LEFT RIGHT DOUBLE ARROW +21D5..21F3 ; Pattern_Syntax # So [31] UP DOWN DOUBLE ARROW..UP DOWN WHITE ARROW +21F4..22FF ; Pattern_Syntax # Sm [268] RIGHT ARROW WITH SMALL CIRCLE..Z NOTATION BAG MEMBERSHIP +2300..2307 ; Pattern_Syntax # So [8] DIAMETER SIGN..WAVY LINE +2308 ; Pattern_Syntax # Ps LEFT CEILING +2309 ; Pattern_Syntax # Pe RIGHT CEILING +230A ; Pattern_Syntax # Ps LEFT FLOOR +230B ; Pattern_Syntax # Pe RIGHT FLOOR +230C..231F ; Pattern_Syntax # So [20] BOTTOM RIGHT CROP..BOTTOM RIGHT CORNER +2320..2321 ; Pattern_Syntax # Sm [2] TOP HALF INTEGRAL..BOTTOM HALF INTEGRAL +2322..2328 ; Pattern_Syntax # So [7] FROWN..KEYBOARD +2329 ; Pattern_Syntax # Ps LEFT-POINTING ANGLE BRACKET +232A ; Pattern_Syntax # Pe RIGHT-POINTING ANGLE BRACKET +232B..237B ; Pattern_Syntax # So [81] ERASE TO THE LEFT..NOT CHECK MARK +237C ; Pattern_Syntax # Sm RIGHT ANGLE WITH DOWNWARDS ZIGZAG ARROW +237D..239A ; Pattern_Syntax # So [30] SHOULDERED OPEN BOX..CLEAR SCREEN SYMBOL +239B..23B3 ; Pattern_Syntax # Sm [25] LEFT PARENTHESIS UPPER HOOK..SUMMATION BOTTOM +23B4..23DB ; Pattern_Syntax # So [40] TOP SQUARE BRACKET..FUSE +23DC..23E1 ; Pattern_Syntax # Sm [6] TOP PARENTHESIS..BOTTOM TORTOISE SHELL BRACKET +23E2..2429 ; Pattern_Syntax # So [72] WHITE TRAPEZIUM..SYMBOL FOR DELETE MEDIUM SHADE FORM +242A..243F ; Pattern_Syntax # Cn [22] .. +2440..244A ; Pattern_Syntax # So [11] OCR HOOK..OCR DOUBLE BACKSLASH +244B..245F ; Pattern_Syntax # Cn [21] .. +2500..25B6 ; Pattern_Syntax # So [183] BOX DRAWINGS LIGHT HORIZONTAL..BLACK RIGHT-POINTING TRIANGLE +25B7 ; Pattern_Syntax # Sm WHITE RIGHT-POINTING TRIANGLE +25B8..25C0 ; Pattern_Syntax # So [9] BLACK RIGHT-POINTING SMALL TRIANGLE..BLACK LEFT-POINTING TRIANGLE +25C1 ; Pattern_Syntax # Sm WHITE LEFT-POINTING TRIANGLE +25C2..25F7 ; Pattern_Syntax # So [54] BLACK LEFT-POINTING SMALL TRIANGLE..WHITE CIRCLE WITH UPPER RIGHT QUADRANT +25F8..25FF ; Pattern_Syntax # Sm [8] UPPER LEFT TRIANGLE..LOWER RIGHT TRIANGLE +2600..266E ; Pattern_Syntax # So [111] BLACK SUN WITH RAYS..MUSIC NATURAL SIGN +266F ; Pattern_Syntax # Sm MUSIC SHARP SIGN +2670..2767 ; Pattern_Syntax # So [248] WEST SYRIAC CROSS..ROTATED FLORAL HEART BULLET +2768 ; Pattern_Syntax # Ps MEDIUM LEFT PARENTHESIS ORNAMENT +2769 ; Pattern_Syntax # Pe MEDIUM RIGHT PARENTHESIS ORNAMENT +276A ; Pattern_Syntax # Ps MEDIUM FLATTENED LEFT PARENTHESIS ORNAMENT +276B ; Pattern_Syntax # Pe MEDIUM FLATTENED RIGHT PARENTHESIS ORNAMENT +276C ; Pattern_Syntax # Ps MEDIUM LEFT-POINTING ANGLE BRACKET ORNAMENT +276D ; Pattern_Syntax # Pe MEDIUM RIGHT-POINTING ANGLE BRACKET ORNAMENT +276E ; Pattern_Syntax # Ps HEAVY LEFT-POINTING ANGLE QUOTATION MARK ORNAMENT +276F ; Pattern_Syntax # Pe HEAVY RIGHT-POINTING ANGLE QUOTATION MARK ORNAMENT +2770 ; Pattern_Syntax # Ps HEAVY LEFT-POINTING ANGLE BRACKET ORNAMENT +2771 ; Pattern_Syntax # Pe HEAVY RIGHT-POINTING ANGLE BRACKET ORNAMENT +2772 ; Pattern_Syntax # Ps LIGHT LEFT TORTOISE SHELL BRACKET ORNAMENT +2773 ; Pattern_Syntax # Pe LIGHT RIGHT TORTOISE SHELL BRACKET ORNAMENT +2774 ; Pattern_Syntax # Ps MEDIUM LEFT CURLY BRACKET ORNAMENT +2775 ; Pattern_Syntax # Pe MEDIUM RIGHT CURLY BRACKET ORNAMENT +2794..27BF ; Pattern_Syntax # So [44] HEAVY WIDE-HEADED RIGHTWARDS ARROW..DOUBLE CURLY LOOP +27C0..27C4 ; Pattern_Syntax # Sm [5] THREE DIMENSIONAL ANGLE..OPEN SUPERSET +27C5 ; Pattern_Syntax # Ps LEFT S-SHAPED BAG DELIMITER +27C6 ; Pattern_Syntax # Pe RIGHT S-SHAPED BAG DELIMITER +27C7..27E5 ; Pattern_Syntax # Sm [31] OR WITH DOT INSIDE..WHITE SQUARE WITH RIGHTWARDS TICK +27E6 ; Pattern_Syntax # Ps MATHEMATICAL LEFT WHITE SQUARE BRACKET +27E7 ; Pattern_Syntax # Pe MATHEMATICAL RIGHT WHITE SQUARE BRACKET +27E8 ; Pattern_Syntax # Ps MATHEMATICAL LEFT ANGLE BRACKET +27E9 ; Pattern_Syntax # Pe MATHEMATICAL RIGHT ANGLE BRACKET +27EA ; Pattern_Syntax # Ps MATHEMATICAL LEFT DOUBLE ANGLE BRACKET +27EB ; Pattern_Syntax # Pe MATHEMATICAL RIGHT DOUBLE ANGLE BRACKET +27EC ; Pattern_Syntax # Ps MATHEMATICAL LEFT WHITE TORTOISE SHELL BRACKET +27ED ; Pattern_Syntax # Pe MATHEMATICAL RIGHT WHITE TORTOISE SHELL BRACKET +27EE ; Pattern_Syntax # Ps MATHEMATICAL LEFT FLATTENED PARENTHESIS +27EF ; Pattern_Syntax # Pe MATHEMATICAL RIGHT FLATTENED PARENTHESIS +27F0..27FF ; Pattern_Syntax # Sm [16] UPWARDS QUADRUPLE ARROW..LONG RIGHTWARDS SQUIGGLE ARROW +2800..28FF ; Pattern_Syntax # So [256] BRAILLE PATTERN BLANK..BRAILLE PATTERN DOTS-12345678 +2900..2982 ; Pattern_Syntax # Sm [131] RIGHTWARDS TWO-HEADED ARROW WITH VERTICAL STROKE..Z NOTATION TYPE COLON +2983 ; Pattern_Syntax # Ps LEFT WHITE CURLY BRACKET +2984 ; Pattern_Syntax # Pe RIGHT WHITE CURLY BRACKET +2985 ; Pattern_Syntax # Ps LEFT WHITE PARENTHESIS +2986 ; Pattern_Syntax # Pe RIGHT WHITE PARENTHESIS +2987 ; Pattern_Syntax # Ps Z NOTATION LEFT IMAGE BRACKET +2988 ; Pattern_Syntax # Pe Z NOTATION RIGHT IMAGE BRACKET +2989 ; Pattern_Syntax # Ps Z NOTATION LEFT BINDING BRACKET +298A ; Pattern_Syntax # Pe Z NOTATION RIGHT BINDING BRACKET +298B ; Pattern_Syntax # Ps LEFT SQUARE BRACKET WITH UNDERBAR +298C ; Pattern_Syntax # Pe RIGHT SQUARE BRACKET WITH UNDERBAR +298D ; Pattern_Syntax # Ps LEFT SQUARE BRACKET WITH TICK IN TOP CORNER +298E ; Pattern_Syntax # Pe RIGHT SQUARE BRACKET WITH TICK IN BOTTOM CORNER +298F ; Pattern_Syntax # Ps LEFT SQUARE BRACKET WITH TICK IN BOTTOM CORNER +2990 ; Pattern_Syntax # Pe RIGHT SQUARE BRACKET WITH TICK IN TOP CORNER +2991 ; Pattern_Syntax # Ps LEFT ANGLE BRACKET WITH DOT +2992 ; Pattern_Syntax # Pe RIGHT ANGLE BRACKET WITH DOT +2993 ; Pattern_Syntax # Ps LEFT ARC LESS-THAN BRACKET +2994 ; Pattern_Syntax # Pe RIGHT ARC GREATER-THAN BRACKET +2995 ; Pattern_Syntax # Ps DOUBLE LEFT ARC GREATER-THAN BRACKET +2996 ; Pattern_Syntax # Pe DOUBLE RIGHT ARC LESS-THAN BRACKET +2997 ; Pattern_Syntax # Ps LEFT BLACK TORTOISE SHELL BRACKET +2998 ; Pattern_Syntax # Pe RIGHT BLACK TORTOISE SHELL BRACKET +2999..29D7 ; Pattern_Syntax # Sm [63] DOTTED FENCE..BLACK HOURGLASS +29D8 ; Pattern_Syntax # Ps LEFT WIGGLY FENCE +29D9 ; Pattern_Syntax # Pe RIGHT WIGGLY FENCE +29DA ; Pattern_Syntax # Ps LEFT DOUBLE WIGGLY FENCE +29DB ; Pattern_Syntax # Pe RIGHT DOUBLE WIGGLY FENCE +29DC..29FB ; Pattern_Syntax # Sm [32] INCOMPLETE INFINITY..TRIPLE PLUS +29FC ; Pattern_Syntax # Ps LEFT-POINTING CURVED ANGLE BRACKET +29FD ; Pattern_Syntax # Pe RIGHT-POINTING CURVED ANGLE BRACKET +29FE..2AFF ; Pattern_Syntax # Sm [258] TINY..N-ARY WHITE VERTICAL BAR +2B00..2B2F ; Pattern_Syntax # So [48] NORTH EAST WHITE ARROW..WHITE VERTICAL ELLIPSE +2B30..2B44 ; Pattern_Syntax # Sm [21] LEFT ARROW WITH SMALL CIRCLE..RIGHTWARDS ARROW THROUGH SUPERSET +2B45..2B46 ; Pattern_Syntax # So [2] LEFTWARDS QUADRUPLE ARROW..RIGHTWARDS QUADRUPLE ARROW +2B47..2B4C ; Pattern_Syntax # Sm [6] REVERSE TILDE OPERATOR ABOVE RIGHTWARDS ARROW..RIGHTWARDS ARROW ABOVE REVERSE TILDE OPERATOR +2B4D..2B73 ; Pattern_Syntax # So [39] DOWNWARDS TRIANGLE-HEADED ZIGZAG ARROW..DOWNWARDS TRIANGLE-HEADED ARROW TO BAR +2B74..2B75 ; Pattern_Syntax # Cn [2] .. +2B76..2BFF ; Pattern_Syntax # So [138] NORTH WEST TRIANGLE-HEADED ARROW TO BAR..HELLSCHREIBER PAUSE SYMBOL +2E00..2E01 ; Pattern_Syntax # Po [2] RIGHT ANGLE SUBSTITUTION MARKER..RIGHT ANGLE DOTTED SUBSTITUTION MARKER +2E02 ; Pattern_Syntax # Pi LEFT SUBSTITUTION BRACKET +2E03 ; Pattern_Syntax # Pf RIGHT SUBSTITUTION BRACKET +2E04 ; Pattern_Syntax # Pi LEFT DOTTED SUBSTITUTION BRACKET +2E05 ; Pattern_Syntax # Pf RIGHT DOTTED SUBSTITUTION BRACKET +2E06..2E08 ; Pattern_Syntax # Po [3] RAISED INTERPOLATION MARKER..DOTTED TRANSPOSITION MARKER +2E09 ; Pattern_Syntax # Pi LEFT TRANSPOSITION BRACKET +2E0A ; Pattern_Syntax # Pf RIGHT TRANSPOSITION BRACKET +2E0B ; Pattern_Syntax # Po RAISED SQUARE +2E0C ; Pattern_Syntax # Pi LEFT RAISED OMISSION BRACKET +2E0D ; Pattern_Syntax # Pf RIGHT RAISED OMISSION BRACKET +2E0E..2E16 ; Pattern_Syntax # Po [9] EDITORIAL CORONIS..DOTTED RIGHT-POINTING ANGLE +2E17 ; Pattern_Syntax # Pd DOUBLE OBLIQUE HYPHEN +2E18..2E19 ; Pattern_Syntax # Po [2] INVERTED INTERROBANG..PALM BRANCH +2E1A ; Pattern_Syntax # Pd HYPHEN WITH DIAERESIS +2E1B ; Pattern_Syntax # Po TILDE WITH RING ABOVE +2E1C ; Pattern_Syntax # Pi LEFT LOW PARAPHRASE BRACKET +2E1D ; Pattern_Syntax # Pf RIGHT LOW PARAPHRASE BRACKET +2E1E..2E1F ; Pattern_Syntax # Po [2] TILDE WITH DOT ABOVE..TILDE WITH DOT BELOW +2E20 ; Pattern_Syntax # Pi LEFT VERTICAL BAR WITH QUILL +2E21 ; Pattern_Syntax # Pf RIGHT VERTICAL BAR WITH QUILL +2E22 ; Pattern_Syntax # Ps TOP LEFT HALF BRACKET +2E23 ; Pattern_Syntax # Pe TOP RIGHT HALF BRACKET +2E24 ; Pattern_Syntax # Ps BOTTOM LEFT HALF BRACKET +2E25 ; Pattern_Syntax # Pe BOTTOM RIGHT HALF BRACKET +2E26 ; Pattern_Syntax # Ps LEFT SIDEWAYS U BRACKET +2E27 ; Pattern_Syntax # Pe RIGHT SIDEWAYS U BRACKET +2E28 ; Pattern_Syntax # Ps LEFT DOUBLE PARENTHESIS +2E29 ; Pattern_Syntax # Pe RIGHT DOUBLE PARENTHESIS +2E2A..2E2E ; Pattern_Syntax # Po [5] TWO DOTS OVER ONE DOT PUNCTUATION..REVERSED QUESTION MARK +2E2F ; Pattern_Syntax # Lm VERTICAL TILDE +2E30..2E39 ; Pattern_Syntax # Po [10] RING POINT..TOP HALF SECTION SIGN +2E3A..2E3B ; Pattern_Syntax # Pd [2] TWO-EM DASH..THREE-EM DASH +2E3C..2E3F ; Pattern_Syntax # Po [4] STENOGRAPHIC FULL STOP..CAPITULUM +2E40 ; Pattern_Syntax # Pd DOUBLE HYPHEN +2E41 ; Pattern_Syntax # Po REVERSED COMMA +2E42 ; Pattern_Syntax # Ps DOUBLE LOW-REVERSED-9 QUOTATION MARK +2E43..2E4F ; Pattern_Syntax # Po [13] DASH WITH LEFT UPTURN..CORNISH VERSE DIVIDER +2E50..2E51 ; Pattern_Syntax # So [2] CROSS PATTY WITH RIGHT CROSSBAR..CROSS PATTY WITH LEFT CROSSBAR +2E52..2E54 ; Pattern_Syntax # Po [3] TIRONIAN SIGN CAPITAL ET..MEDIEVAL QUESTION MARK +2E55 ; Pattern_Syntax # Ps LEFT SQUARE BRACKET WITH STROKE +2E56 ; Pattern_Syntax # Pe RIGHT SQUARE BRACKET WITH STROKE +2E57 ; Pattern_Syntax # Ps LEFT SQUARE BRACKET WITH DOUBLE STROKE +2E58 ; Pattern_Syntax # Pe RIGHT SQUARE BRACKET WITH DOUBLE STROKE +2E59 ; Pattern_Syntax # Ps TOP HALF LEFT PARENTHESIS +2E5A ; Pattern_Syntax # Pe TOP HALF RIGHT PARENTHESIS +2E5B ; Pattern_Syntax # Ps BOTTOM HALF LEFT PARENTHESIS +2E5C ; Pattern_Syntax # Pe BOTTOM HALF RIGHT PARENTHESIS +2E5D ; Pattern_Syntax # Pd OBLIQUE HYPHEN +2E5E..2E7F ; Pattern_Syntax # Cn [34] .. +3001..3003 ; Pattern_Syntax # Po [3] IDEOGRAPHIC COMMA..DITTO MARK +3008 ; Pattern_Syntax # Ps LEFT ANGLE BRACKET +3009 ; Pattern_Syntax # Pe RIGHT ANGLE BRACKET +300A ; Pattern_Syntax # Ps LEFT DOUBLE ANGLE BRACKET +300B ; Pattern_Syntax # Pe RIGHT DOUBLE ANGLE BRACKET +300C ; Pattern_Syntax # Ps LEFT CORNER BRACKET +300D ; Pattern_Syntax # Pe RIGHT CORNER BRACKET +300E ; Pattern_Syntax # Ps LEFT WHITE CORNER BRACKET +300F ; Pattern_Syntax # Pe RIGHT WHITE CORNER BRACKET +3010 ; Pattern_Syntax # Ps LEFT BLACK LENTICULAR BRACKET +3011 ; Pattern_Syntax # Pe RIGHT BLACK LENTICULAR BRACKET +3012..3013 ; Pattern_Syntax # So [2] POSTAL MARK..GETA MARK +3014 ; Pattern_Syntax # Ps LEFT TORTOISE SHELL BRACKET +3015 ; Pattern_Syntax # Pe RIGHT TORTOISE SHELL BRACKET +3016 ; Pattern_Syntax # Ps LEFT WHITE LENTICULAR BRACKET +3017 ; Pattern_Syntax # Pe RIGHT WHITE LENTICULAR BRACKET +3018 ; Pattern_Syntax # Ps LEFT WHITE TORTOISE SHELL BRACKET +3019 ; Pattern_Syntax # Pe RIGHT WHITE TORTOISE SHELL BRACKET +301A ; Pattern_Syntax # Ps LEFT WHITE SQUARE BRACKET +301B ; Pattern_Syntax # Pe RIGHT WHITE SQUARE BRACKET +301C ; Pattern_Syntax # Pd WAVE DASH +301D ; Pattern_Syntax # Ps REVERSED DOUBLE PRIME QUOTATION MARK +301E..301F ; Pattern_Syntax # Pe [2] DOUBLE PRIME QUOTATION MARK..LOW DOUBLE PRIME QUOTATION MARK +3020 ; Pattern_Syntax # So POSTAL MARK FACE +3030 ; Pattern_Syntax # Pd WAVY DASH +FD3E ; Pattern_Syntax # Pe ORNATE LEFT PARENTHESIS +FD3F ; Pattern_Syntax # Ps ORNATE RIGHT PARENTHESIS +FE45..FE46 ; Pattern_Syntax # Po [2] SESAME DOT..WHITE SESAME DOT + +# Total code points: 2760 + +# ================================================ + +0600..0605 ; Prepended_Concatenation_Mark # Cf [6] ARABIC NUMBER SIGN..ARABIC NUMBER MARK ABOVE +06DD ; Prepended_Concatenation_Mark # Cf ARABIC END OF AYAH +070F ; Prepended_Concatenation_Mark # Cf SYRIAC ABBREVIATION MARK +0890..0891 ; Prepended_Concatenation_Mark # Cf [2] ARABIC POUND MARK ABOVE..ARABIC PIASTRE MARK ABOVE +08E2 ; Prepended_Concatenation_Mark # Cf ARABIC DISPUTED END OF AYAH +110BD ; Prepended_Concatenation_Mark # Cf KAITHI NUMBER SIGN +110CD ; Prepended_Concatenation_Mark # Cf KAITHI NUMBER SIGN ABOVE + +# Total code points: 13 + +# ================================================ + +1F1E6..1F1FF ; Regional_Indicator # So [26] REGIONAL INDICATOR SYMBOL LETTER A..REGIONAL INDICATOR SYMBOL LETTER Z + +# Total code points: 26 + +# ================================================ + +0654..0655 ; Modifier_Combining_Mark # Mn [2] ARABIC HAMZA ABOVE..ARABIC HAMZA BELOW +0658 ; Modifier_Combining_Mark # Mn ARABIC MARK NOON GHUNNA +06DC ; Modifier_Combining_Mark # Mn ARABIC SMALL HIGH SEEN +06E3 ; Modifier_Combining_Mark # Mn ARABIC SMALL LOW SEEN +06E7..06E8 ; Modifier_Combining_Mark # Mn [2] ARABIC SMALL HIGH YEH..ARABIC SMALL HIGH NOON +08CA..08CB ; Modifier_Combining_Mark # Mn [2] ARABIC SMALL HIGH FARSI YEH..ARABIC SMALL HIGH YEH BARREE WITH TWO DOTS BELOW +08CD..08CF ; Modifier_Combining_Mark # Mn [3] ARABIC SMALL HIGH ZAH..ARABIC LARGE ROUND DOT BELOW +08D3 ; Modifier_Combining_Mark # Mn ARABIC SMALL LOW WAW +08F3 ; Modifier_Combining_Mark # Mn ARABIC SMALL HIGH WAW + +# Total code points: 14 + +# EOF diff --git a/lib/elixir/unicode/PropertyValueAliases.txt b/lib/elixir/unicode/PropertyValueAliases.txt new file mode 100644 index 00000000000..b92662eda28 --- /dev/null +++ b/lib/elixir/unicode/PropertyValueAliases.txt @@ -0,0 +1,1735 @@ +# PropertyValueAliases-17.0.0.txt +# Date: 2025-06-30, 06:16:21 GMT +# © 2025 Unicode®, Inc. +# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. +# For terms of use and license, see https://www.unicode.org/terms_of_use.html +# +# Unicode Character Database +# For documentation, see https://www.unicode.org/reports/tr44/ +# +# This file contains aliases for property values used in the UCD. +# These names can be used for XML formats of UCD data, for regular-expression +# property tests, and other programmatic textual descriptions of Unicode data. +# +# The names may be translated in appropriate environments, and additional +# aliases may be useful. +# +# FORMAT +# +# Each line describes a property value name. +# This consists of three or more fields, separated by semicolons. +# +# First Field: The first field describes the property for which that +# property value name is used. +# +# Second Field: The second field is the short name for the property value. +# It is typically an abbreviation, but in a number of cases it is simply +# a duplicate of the "long name" in the third field. +# +# Third Field: The third field is the long name for the property value, +# typically the formal name used in documentation about the property value. +# +# In the case of Canonical_Combining_Class (ccc), there are 4 fields: +# The second field is numeric, the third is the short name, and the fourth is the long name. +# +# The above are the preferred aliases. Other aliases may be listed in additional fields. +# +# Loose matching should be applied to all property names and property values, with +# the exception of String Property values. With loose matching of property names and +# values, the case distinctions, whitespace, hyphens, and '_' are ignored. +# For Numeric Property values, numeric equivalence is applied: thus "01.00" +# is equivalent to "1". +# +# NOTE: Property value names are NOT unique across properties. For example: +# +# AL means Arabic Letter for the Bidi_Class property, and +# AL means Above_Left for the Canonical_Combining_Class property, and +# AL means Alphabetic for the Line_Break property. +# +# In addition, some property names may be the same as some property value names. +# For example: +# +# sc means the Script property, and +# Sc means the General_Category property value Currency_Symbol (Sc) +# +# The combination of property value and property name is, however, unique. +# +# For more information, see UAX #44, Unicode Character Database, and +# UTS #18, Unicode Regular Expressions. +# ================================================ + + +# ASCII_Hex_Digit (AHex) + +AHex; N ; No ; F ; False +AHex; Y ; Yes ; T ; True + +# Age (age) + +age; 1.1 ; V1_1 +age; 2.0 ; V2_0 +age; 2.1 ; V2_1 +age; 3.0 ; V3_0 +age; 3.1 ; V3_1 +age; 3.2 ; V3_2 +age; 4.0 ; V4_0 +age; 4.1 ; V4_1 +age; 5.0 ; V5_0 +age; 5.1 ; V5_1 +age; 5.2 ; V5_2 +age; 6.0 ; V6_0 +age; 6.1 ; V6_1 +age; 6.2 ; V6_2 +age; 6.3 ; V6_3 +age; 7.0 ; V7_0 +age; 8.0 ; V8_0 +age; 9.0 ; V9_0 +age; 10.0 ; V10_0 +age; 11.0 ; V11_0 +age; 12.0 ; V12_0 +age; 12.1 ; V12_1 +age; 13.0 ; V13_0 +age; 14.0 ; V14_0 +age; 15.0 ; V15_0 +age; 15.1 ; V15_1 +age; 16.0 ; V16_0 +age; 17.0 ; V17_0 +age; NA ; Unassigned + +# Alphabetic (Alpha) + +Alpha; N ; No ; F ; False +Alpha; Y ; Yes ; T ; True + +# Bidi_Class (bc) + +bc ; AL ; Arabic_Letter +bc ; AN ; Arabic_Number +bc ; B ; Paragraph_Separator +bc ; BN ; Boundary_Neutral +bc ; CS ; Common_Separator +bc ; EN ; European_Number +bc ; ES ; European_Separator +bc ; ET ; European_Terminator +bc ; FSI ; First_Strong_Isolate +bc ; L ; Left_To_Right +bc ; LRE ; Left_To_Right_Embedding +bc ; LRI ; Left_To_Right_Isolate +bc ; LRO ; Left_To_Right_Override +bc ; NSM ; Nonspacing_Mark +bc ; ON ; Other_Neutral +bc ; PDF ; Pop_Directional_Format +bc ; PDI ; Pop_Directional_Isolate +bc ; R ; Right_To_Left +bc ; RLE ; Right_To_Left_Embedding +bc ; RLI ; Right_To_Left_Isolate +bc ; RLO ; Right_To_Left_Override +bc ; S ; Segment_Separator +bc ; WS ; White_Space + +# Bidi_Control (Bidi_C) + +Bidi_C; N ; No ; F ; False +Bidi_C; Y ; Yes ; T ; True + +# Bidi_Mirrored (Bidi_M) + +Bidi_M; N ; No ; F ; False +Bidi_M; Y ; Yes ; T ; True + +# Bidi_Mirroring_Glyph (bmg) + + +# Bidi_Paired_Bracket (bpb) + +# @missing: 0000..10FFFF; Bidi_Paired_Bracket; + +# Bidi_Paired_Bracket_Type (bpt) + +bpt; c ; Close +bpt; n ; None +bpt; o ; Open +# @missing: 0000..10FFFF; Bidi_Paired_Bracket_Type; n + +# Block (blk) + +blk; Adlam ; Adlam +blk; Aegean_Numbers ; Aegean_Numbers +blk; Ahom ; Ahom +blk; Alchemical ; Alchemical_Symbols +blk; Alphabetic_PF ; Alphabetic_Presentation_Forms +blk; Anatolian_Hieroglyphs ; Anatolian_Hieroglyphs +blk; Ancient_Greek_Music ; Ancient_Greek_Musical_Notation +blk; Ancient_Greek_Numbers ; Ancient_Greek_Numbers +blk; Ancient_Symbols ; Ancient_Symbols +blk; Arabic ; Arabic +blk; Arabic_Ext_A ; Arabic_Extended_A +blk; Arabic_Ext_B ; Arabic_Extended_B +blk; Arabic_Ext_C ; Arabic_Extended_C +blk; Arabic_Math ; Arabic_Mathematical_Alphabetic_Symbols +blk; Arabic_PF_A ; Arabic_Presentation_Forms_A ; Arabic_Presentation_Forms-A +blk; Arabic_PF_B ; Arabic_Presentation_Forms_B +blk; Arabic_Sup ; Arabic_Supplement +blk; Armenian ; Armenian +blk; Arrows ; Arrows +blk; ASCII ; Basic_Latin +blk; Avestan ; Avestan +blk; Balinese ; Balinese +blk; Bamum ; Bamum +blk; Bamum_Sup ; Bamum_Supplement +blk; Bassa_Vah ; Bassa_Vah +blk; Batak ; Batak +blk; Bengali ; Bengali +blk; Beria_Erfe ; Beria_Erfe +blk; Bhaiksuki ; Bhaiksuki +blk; Block_Elements ; Block_Elements +blk; Bopomofo ; Bopomofo +blk; Bopomofo_Ext ; Bopomofo_Extended +blk; Box_Drawing ; Box_Drawing +blk; Brahmi ; Brahmi +blk; Braille ; Braille_Patterns +blk; Buginese ; Buginese +blk; Buhid ; Buhid +blk; Byzantine_Music ; Byzantine_Musical_Symbols +blk; Carian ; Carian +blk; Caucasian_Albanian ; Caucasian_Albanian +blk; Chakma ; Chakma +blk; Cham ; Cham +blk; Cherokee ; Cherokee +blk; Cherokee_Sup ; Cherokee_Supplement +blk; Chess_Symbols ; Chess_Symbols +blk; Chorasmian ; Chorasmian +blk; CJK ; CJK_Unified_Ideographs +blk; CJK_Compat ; CJK_Compatibility +blk; CJK_Compat_Forms ; CJK_Compatibility_Forms +blk; CJK_Compat_Ideographs ; CJK_Compatibility_Ideographs +blk; CJK_Compat_Ideographs_Sup ; CJK_Compatibility_Ideographs_Supplement +blk; CJK_Ext_A ; CJK_Unified_Ideographs_Extension_A +blk; CJK_Ext_B ; CJK_Unified_Ideographs_Extension_B +blk; CJK_Ext_C ; CJK_Unified_Ideographs_Extension_C +blk; CJK_Ext_D ; CJK_Unified_Ideographs_Extension_D +blk; CJK_Ext_E ; CJK_Unified_Ideographs_Extension_E +blk; CJK_Ext_F ; CJK_Unified_Ideographs_Extension_F +blk; CJK_Ext_G ; CJK_Unified_Ideographs_Extension_G +blk; CJK_Ext_H ; CJK_Unified_Ideographs_Extension_H +blk; CJK_Ext_I ; CJK_Unified_Ideographs_Extension_I +blk; CJK_Ext_J ; CJK_Unified_Ideographs_Extension_J +blk; CJK_Radicals_Sup ; CJK_Radicals_Supplement +blk; CJK_Strokes ; CJK_Strokes +blk; CJK_Symbols ; CJK_Symbols_And_Punctuation +blk; Compat_Jamo ; Hangul_Compatibility_Jamo +blk; Control_Pictures ; Control_Pictures +blk; Coptic ; Coptic +blk; Coptic_Epact_Numbers ; Coptic_Epact_Numbers +blk; Counting_Rod ; Counting_Rod_Numerals +blk; Cuneiform ; Cuneiform +blk; Cuneiform_Numbers ; Cuneiform_Numbers_And_Punctuation +blk; Currency_Symbols ; Currency_Symbols +blk; Cypriot_Syllabary ; Cypriot_Syllabary +blk; Cypro_Minoan ; Cypro_Minoan +blk; Cyrillic ; Cyrillic +blk; Cyrillic_Ext_A ; Cyrillic_Extended_A +blk; Cyrillic_Ext_B ; Cyrillic_Extended_B +blk; Cyrillic_Ext_C ; Cyrillic_Extended_C +blk; Cyrillic_Ext_D ; Cyrillic_Extended_D +blk; Cyrillic_Sup ; Cyrillic_Supplement ; Cyrillic_Supplementary +blk; Deseret ; Deseret +blk; Devanagari ; Devanagari +blk; Devanagari_Ext ; Devanagari_Extended +blk; Devanagari_Ext_A ; Devanagari_Extended_A +blk; Diacriticals ; Combining_Diacritical_Marks +blk; Diacriticals_Ext ; Combining_Diacritical_Marks_Extended +blk; Diacriticals_For_Symbols ; Combining_Diacritical_Marks_For_Symbols; Combining_Marks_For_Symbols +blk; Diacriticals_Sup ; Combining_Diacritical_Marks_Supplement +blk; Dingbats ; Dingbats +blk; Dives_Akuru ; Dives_Akuru +blk; Dogra ; Dogra +blk; Domino ; Domino_Tiles +blk; Duployan ; Duployan +blk; Early_Dynastic_Cuneiform ; Early_Dynastic_Cuneiform +blk; Egyptian_Hieroglyph_Format_Controls; Egyptian_Hieroglyph_Format_Controls +blk; Egyptian_Hieroglyphs ; Egyptian_Hieroglyphs +blk; Egyptian_Hieroglyphs_Ext_A ; Egyptian_Hieroglyphs_Extended_A +blk; Elbasan ; Elbasan +blk; Elymaic ; Elymaic +blk; Emoticons ; Emoticons +blk; Enclosed_Alphanum ; Enclosed_Alphanumerics +blk; Enclosed_Alphanum_Sup ; Enclosed_Alphanumeric_Supplement +blk; Enclosed_CJK ; Enclosed_CJK_Letters_And_Months +blk; Enclosed_Ideographic_Sup ; Enclosed_Ideographic_Supplement +blk; Ethiopic ; Ethiopic +blk; Ethiopic_Ext ; Ethiopic_Extended +blk; Ethiopic_Ext_A ; Ethiopic_Extended_A +blk; Ethiopic_Ext_B ; Ethiopic_Extended_B +blk; Ethiopic_Sup ; Ethiopic_Supplement +blk; Garay ; Garay +blk; Geometric_Shapes ; Geometric_Shapes +blk; Geometric_Shapes_Ext ; Geometric_Shapes_Extended +blk; Georgian ; Georgian +blk; Georgian_Ext ; Georgian_Extended +blk; Georgian_Sup ; Georgian_Supplement +blk; Glagolitic ; Glagolitic +blk; Glagolitic_Sup ; Glagolitic_Supplement +blk; Gothic ; Gothic +blk; Grantha ; Grantha +blk; Greek ; Greek_And_Coptic +blk; Greek_Ext ; Greek_Extended +blk; Gujarati ; Gujarati +blk; Gunjala_Gondi ; Gunjala_Gondi +blk; Gurmukhi ; Gurmukhi +blk; Gurung_Khema ; Gurung_Khema +blk; Half_And_Full_Forms ; Halfwidth_And_Fullwidth_Forms +blk; Half_Marks ; Combining_Half_Marks +blk; Hangul ; Hangul_Syllables +blk; Hanifi_Rohingya ; Hanifi_Rohingya +blk; Hanunoo ; Hanunoo +blk; Hatran ; Hatran +blk; Hebrew ; Hebrew +blk; High_PU_Surrogates ; High_Private_Use_Surrogates +blk; High_Surrogates ; High_Surrogates +blk; Hiragana ; Hiragana +blk; IDC ; Ideographic_Description_Characters +blk; Ideographic_Symbols ; Ideographic_Symbols_And_Punctuation +blk; Imperial_Aramaic ; Imperial_Aramaic +blk; Indic_Number_Forms ; Common_Indic_Number_Forms +blk; Indic_Siyaq_Numbers ; Indic_Siyaq_Numbers +blk; Inscriptional_Pahlavi ; Inscriptional_Pahlavi +blk; Inscriptional_Parthian ; Inscriptional_Parthian +blk; IPA_Ext ; IPA_Extensions +blk; Jamo ; Hangul_Jamo +blk; Jamo_Ext_A ; Hangul_Jamo_Extended_A +blk; Jamo_Ext_B ; Hangul_Jamo_Extended_B +blk; Javanese ; Javanese +blk; Kaithi ; Kaithi +blk; Kaktovik_Numerals ; Kaktovik_Numerals +blk; Kana_Ext_A ; Kana_Extended_A +blk; Kana_Ext_B ; Kana_Extended_B +blk; Kana_Sup ; Kana_Supplement +blk; Kanbun ; Kanbun +blk; Kangxi ; Kangxi_Radicals +blk; Kannada ; Kannada +blk; Katakana ; Katakana +blk; Katakana_Ext ; Katakana_Phonetic_Extensions +blk; Kawi ; Kawi +blk; Kayah_Li ; Kayah_Li +blk; Kharoshthi ; Kharoshthi +blk; Khitan_Small_Script ; Khitan_Small_Script +blk; Khmer ; Khmer +blk; Khmer_Symbols ; Khmer_Symbols +blk; Khojki ; Khojki +blk; Khudawadi ; Khudawadi +blk; Kirat_Rai ; Kirat_Rai +blk; Lao ; Lao +blk; Latin_1_Sup ; Latin_1_Supplement ; Latin_1 +blk; Latin_Ext_A ; Latin_Extended_A +blk; Latin_Ext_Additional ; Latin_Extended_Additional +blk; Latin_Ext_B ; Latin_Extended_B +blk; Latin_Ext_C ; Latin_Extended_C +blk; Latin_Ext_D ; Latin_Extended_D +blk; Latin_Ext_E ; Latin_Extended_E +blk; Latin_Ext_F ; Latin_Extended_F +blk; Latin_Ext_G ; Latin_Extended_G +blk; Lepcha ; Lepcha +blk; Letterlike_Symbols ; Letterlike_Symbols +blk; Limbu ; Limbu +blk; Linear_A ; Linear_A +blk; Linear_B_Ideograms ; Linear_B_Ideograms +blk; Linear_B_Syllabary ; Linear_B_Syllabary +blk; Lisu ; Lisu +blk; Lisu_Sup ; Lisu_Supplement +blk; Low_Surrogates ; Low_Surrogates +blk; Lycian ; Lycian +blk; Lydian ; Lydian +blk; Mahajani ; Mahajani +blk; Mahjong ; Mahjong_Tiles +blk; Makasar ; Makasar +blk; Malayalam ; Malayalam +blk; Mandaic ; Mandaic +blk; Manichaean ; Manichaean +blk; Marchen ; Marchen +blk; Masaram_Gondi ; Masaram_Gondi +blk; Math_Alphanum ; Mathematical_Alphanumeric_Symbols +blk; Math_Operators ; Mathematical_Operators +blk; Mayan_Numerals ; Mayan_Numerals +blk; Medefaidrin ; Medefaidrin +blk; Meetei_Mayek ; Meetei_Mayek +blk; Meetei_Mayek_Ext ; Meetei_Mayek_Extensions +blk; Mende_Kikakui ; Mende_Kikakui +blk; Meroitic_Cursive ; Meroitic_Cursive +blk; Meroitic_Hieroglyphs ; Meroitic_Hieroglyphs +blk; Miao ; Miao +blk; Misc_Arrows ; Miscellaneous_Symbols_And_Arrows +blk; Misc_Math_Symbols_A ; Miscellaneous_Mathematical_Symbols_A +blk; Misc_Math_Symbols_B ; Miscellaneous_Mathematical_Symbols_B +blk; Misc_Pictographs ; Miscellaneous_Symbols_And_Pictographs +blk; Misc_Symbols ; Miscellaneous_Symbols +blk; Misc_Symbols_Sup ; Miscellaneous_Symbols_Supplement +blk; Misc_Technical ; Miscellaneous_Technical +blk; Modi ; Modi +blk; Modifier_Letters ; Spacing_Modifier_Letters +blk; Modifier_Tone_Letters ; Modifier_Tone_Letters +blk; Mongolian ; Mongolian +blk; Mongolian_Sup ; Mongolian_Supplement +blk; Mro ; Mro +blk; Multani ; Multani +blk; Music ; Musical_Symbols +blk; Myanmar ; Myanmar +blk; Myanmar_Ext_A ; Myanmar_Extended_A +blk; Myanmar_Ext_B ; Myanmar_Extended_B +blk; Myanmar_Ext_C ; Myanmar_Extended_C +blk; Nabataean ; Nabataean +blk; Nag_Mundari ; Nag_Mundari +blk; Nandinagari ; Nandinagari +blk; NB ; No_Block +blk; New_Tai_Lue ; New_Tai_Lue +blk; Newa ; Newa +blk; NKo ; NKo +blk; Number_Forms ; Number_Forms +blk; Nushu ; Nushu +blk; Nyiakeng_Puachue_Hmong ; Nyiakeng_Puachue_Hmong +blk; OCR ; Optical_Character_Recognition +blk; Ogham ; Ogham +blk; Ol_Chiki ; Ol_Chiki +blk; Ol_Onal ; Ol_Onal +blk; Old_Hungarian ; Old_Hungarian +blk; Old_Italic ; Old_Italic +blk; Old_North_Arabian ; Old_North_Arabian +blk; Old_Permic ; Old_Permic +blk; Old_Persian ; Old_Persian +blk; Old_Sogdian ; Old_Sogdian +blk; Old_South_Arabian ; Old_South_Arabian +blk; Old_Turkic ; Old_Turkic +blk; Old_Uyghur ; Old_Uyghur +blk; Oriya ; Oriya +blk; Ornamental_Dingbats ; Ornamental_Dingbats +blk; Osage ; Osage +blk; Osmanya ; Osmanya +blk; Ottoman_Siyaq_Numbers ; Ottoman_Siyaq_Numbers +blk; Pahawh_Hmong ; Pahawh_Hmong +blk; Palmyrene ; Palmyrene +blk; Pau_Cin_Hau ; Pau_Cin_Hau +blk; Phags_Pa ; Phags_Pa +blk; Phaistos ; Phaistos_Disc +blk; Phoenician ; Phoenician +blk; Phonetic_Ext ; Phonetic_Extensions +blk; Phonetic_Ext_Sup ; Phonetic_Extensions_Supplement +blk; Playing_Cards ; Playing_Cards +blk; Psalter_Pahlavi ; Psalter_Pahlavi +blk; PUA ; Private_Use_Area ; Private_Use +blk; Punctuation ; General_Punctuation +blk; Rejang ; Rejang +blk; Rumi ; Rumi_Numeral_Symbols +blk; Runic ; Runic +blk; Samaritan ; Samaritan +blk; Saurashtra ; Saurashtra +blk; Sharada ; Sharada +blk; Sharada_Sup ; Sharada_Supplement +blk; Shavian ; Shavian +blk; Shorthand_Format_Controls ; Shorthand_Format_Controls +blk; Siddham ; Siddham +blk; Sidetic ; Sidetic +blk; Sinhala ; Sinhala +blk; Sinhala_Archaic_Numbers ; Sinhala_Archaic_Numbers +blk; Small_Forms ; Small_Form_Variants +blk; Small_Kana_Ext ; Small_Kana_Extension +blk; Sogdian ; Sogdian +blk; Sora_Sompeng ; Sora_Sompeng +blk; Soyombo ; Soyombo +blk; Specials ; Specials +blk; Sundanese ; Sundanese +blk; Sundanese_Sup ; Sundanese_Supplement +blk; Sunuwar ; Sunuwar +blk; Sup_Arrows_A ; Supplemental_Arrows_A +blk; Sup_Arrows_B ; Supplemental_Arrows_B +blk; Sup_Arrows_C ; Supplemental_Arrows_C +blk; Sup_Math_Operators ; Supplemental_Mathematical_Operators +blk; Sup_PUA_A ; Supplementary_Private_Use_Area_A +blk; Sup_PUA_B ; Supplementary_Private_Use_Area_B +blk; Sup_Punctuation ; Supplemental_Punctuation +blk; Sup_Symbols_And_Pictographs ; Supplemental_Symbols_And_Pictographs +blk; Super_And_Sub ; Superscripts_And_Subscripts +blk; Sutton_SignWriting ; Sutton_SignWriting +blk; Syloti_Nagri ; Syloti_Nagri +blk; Symbols_And_Pictographs_Ext_A ; Symbols_And_Pictographs_Extended_A +blk; Symbols_For_Legacy_Computing ; Symbols_For_Legacy_Computing +blk; Symbols_For_Legacy_Computing_Sup ; Symbols_For_Legacy_Computing_Supplement +blk; Syriac ; Syriac +blk; Syriac_Sup ; Syriac_Supplement +blk; Tagalog ; Tagalog +blk; Tagbanwa ; Tagbanwa +blk; Tags ; Tags +blk; Tai_Le ; Tai_Le +blk; Tai_Tham ; Tai_Tham +blk; Tai_Viet ; Tai_Viet +blk; Tai_Xuan_Jing ; Tai_Xuan_Jing_Symbols +blk; Tai_Yo ; Tai_Yo +blk; Takri ; Takri +blk; Tamil ; Tamil +blk; Tamil_Sup ; Tamil_Supplement +blk; Tangsa ; Tangsa +blk; Tangut ; Tangut +blk; Tangut_Components ; Tangut_Components +blk; Tangut_Components_Sup ; Tangut_Components_Supplement +blk; Tangut_Sup ; Tangut_Supplement +blk; Telugu ; Telugu +blk; Thaana ; Thaana +blk; Thai ; Thai +blk; Tibetan ; Tibetan +blk; Tifinagh ; Tifinagh +blk; Tirhuta ; Tirhuta +blk; Todhri ; Todhri +blk; Tolong_Siki ; Tolong_Siki +blk; Toto ; Toto +blk; Transport_And_Map ; Transport_And_Map_Symbols +blk; Tulu_Tigalari ; Tulu_Tigalari +blk; UCAS ; Unified_Canadian_Aboriginal_Syllabics; Canadian_Syllabics +blk; UCAS_Ext ; Unified_Canadian_Aboriginal_Syllabics_Extended +blk; UCAS_Ext_A ; Unified_Canadian_Aboriginal_Syllabics_Extended_A +blk; Ugaritic ; Ugaritic +blk; Vai ; Vai +blk; Vedic_Ext ; Vedic_Extensions +blk; Vertical_Forms ; Vertical_Forms +blk; Vithkuqi ; Vithkuqi +blk; VS ; Variation_Selectors +blk; VS_Sup ; Variation_Selectors_Supplement +blk; Wancho ; Wancho +blk; Warang_Citi ; Warang_Citi +blk; Yezidi ; Yezidi +blk; Yi_Radicals ; Yi_Radicals +blk; Yi_Syllables ; Yi_Syllables +blk; Yijing ; Yijing_Hexagram_Symbols +blk; Zanabazar_Square ; Zanabazar_Square +blk; Znamenny_Music ; Znamenny_Musical_Notation + +# Canonical_Combining_Class (ccc) + +ccc; 0; NR ; Not_Reordered +ccc; 1; OV ; Overlay +ccc; 6; HANR ; Han_Reading +ccc; 7; NK ; Nukta +ccc; 8; KV ; Kana_Voicing +ccc; 9; VR ; Virama +ccc; 10; CCC10 ; CCC10 +ccc; 11; CCC11 ; CCC11 +ccc; 12; CCC12 ; CCC12 +ccc; 13; CCC13 ; CCC13 +ccc; 14; CCC14 ; CCC14 +ccc; 15; CCC15 ; CCC15 +ccc; 16; CCC16 ; CCC16 +ccc; 17; CCC17 ; CCC17 +ccc; 18; CCC18 ; CCC18 +ccc; 19; CCC19 ; CCC19 +ccc; 20; CCC20 ; CCC20 +ccc; 21; CCC21 ; CCC21 +ccc; 22; CCC22 ; CCC22 +ccc; 23; CCC23 ; CCC23 +ccc; 24; CCC24 ; CCC24 +ccc; 25; CCC25 ; CCC25 +ccc; 26; CCC26 ; CCC26 +ccc; 27; CCC27 ; CCC27 +ccc; 28; CCC28 ; CCC28 +ccc; 29; CCC29 ; CCC29 +ccc; 30; CCC30 ; CCC30 +ccc; 31; CCC31 ; CCC31 +ccc; 32; CCC32 ; CCC32 +ccc; 33; CCC33 ; CCC33 +ccc; 34; CCC34 ; CCC34 +ccc; 35; CCC35 ; CCC35 +ccc; 36; CCC36 ; CCC36 +ccc; 84; CCC84 ; CCC84 +ccc; 91; CCC91 ; CCC91 +ccc; 103; CCC103 ; CCC103 +ccc; 107; CCC107 ; CCC107 +ccc; 118; CCC118 ; CCC118 +ccc; 122; CCC122 ; CCC122 +ccc; 129; CCC129 ; CCC129 +ccc; 130; CCC130 ; CCC130 +ccc; 132; CCC132 ; CCC132 +ccc; 133; CCC133 ; CCC133 # RESERVED +ccc; 200; ATBL ; Attached_Below_Left +ccc; 202; ATB ; Attached_Below +ccc; 214; ATA ; Attached_Above +ccc; 216; ATAR ; Attached_Above_Right +ccc; 218; BL ; Below_Left +ccc; 220; B ; Below +ccc; 222; BR ; Below_Right +ccc; 224; L ; Left +ccc; 226; R ; Right +ccc; 228; AL ; Above_Left +ccc; 230; A ; Above +ccc; 232; AR ; Above_Right +ccc; 233; DB ; Double_Below +ccc; 234; DA ; Double_Above +ccc; 240; IS ; Iota_Subscript + +# Case_Folding (cf) + +# @missing: 0000..10FFFF; Case_Folding; + +# Case_Ignorable (CI) + +CI ; N ; No ; F ; False +CI ; Y ; Yes ; T ; True + +# Cased (Cased) + +Cased; N ; No ; F ; False +Cased; Y ; Yes ; T ; True + +# Changes_When_Casefolded (CWCF) + +CWCF; N ; No ; F ; False +CWCF; Y ; Yes ; T ; True + +# Changes_When_Casemapped (CWCM) + +CWCM; N ; No ; F ; False +CWCM; Y ; Yes ; T ; True + +# Changes_When_Lowercased (CWL) + +CWL; N ; No ; F ; False +CWL; Y ; Yes ; T ; True + +# Changes_When_NFKC_Casefolded (CWKCF) + +CWKCF; N ; No ; F ; False +CWKCF; Y ; Yes ; T ; True + +# Changes_When_Titlecased (CWT) + +CWT; N ; No ; F ; False +CWT; Y ; Yes ; T ; True + +# Changes_When_Uppercased (CWU) + +CWU; N ; No ; F ; False +CWU; Y ; Yes ; T ; True + +# Composition_Exclusion (CE) + +CE ; N ; No ; F ; False +CE ; Y ; Yes ; T ; True + +# Dash (Dash) + +Dash; N ; No ; F ; False +Dash; Y ; Yes ; T ; True + +# Decomposition_Mapping (dm) + +# @missing: 0000..10FFFF; Decomposition_Mapping; + +# Decomposition_Type (dt) + +dt ; Can ; Canonical ; can +dt ; Com ; Compat ; com +dt ; Enc ; Circle ; enc +dt ; Fin ; Final ; fin +dt ; Font ; Font ; font +dt ; Fra ; Fraction ; fra +dt ; Init ; Initial ; init +dt ; Iso ; Isolated ; iso +dt ; Med ; Medial ; med +dt ; Nar ; Narrow ; nar +dt ; Nb ; Nobreak ; nb +dt ; None ; None ; none +dt ; Sml ; Small ; sml +dt ; Sqr ; Square ; sqr +dt ; Sub ; Sub ; sub +dt ; Sup ; Super ; sup +dt ; Vert ; Vertical ; vert +dt ; Wide ; Wide ; wide + +# Default_Ignorable_Code_Point (DI) + +DI ; N ; No ; F ; False +DI ; Y ; Yes ; T ; True + +# Deprecated (Dep) + +Dep; N ; No ; F ; False +Dep; Y ; Yes ; T ; True + +# Diacritic (Dia) + +Dia; N ; No ; F ; False +Dia; Y ; Yes ; T ; True + +# East_Asian_Width (ea) + +ea ; A ; Ambiguous +ea ; F ; Fullwidth +ea ; H ; Halfwidth +ea ; N ; Neutral +ea ; Na ; Narrow +ea ; W ; Wide + +# Emoji (Emoji) + +Emoji; N ; No ; F ; False +Emoji; Y ; Yes ; T ; True + +# Emoji_Component (EComp) + +EComp; N ; No ; F ; False +EComp; Y ; Yes ; T ; True + +# Emoji_Modifier (EMod) + +EMod; N ; No ; F ; False +EMod; Y ; Yes ; T ; True + +# Emoji_Modifier_Base (EBase) + +EBase; N ; No ; F ; False +EBase; Y ; Yes ; T ; True + +# Emoji_Presentation (EPres) + +EPres; N ; No ; F ; False +EPres; Y ; Yes ; T ; True + +# Equivalent_Unified_Ideograph (EqUIdeo) + + +# Expands_On_NFC (XO_NFC) + +XO_NFC; N ; No ; F ; False +XO_NFC; Y ; Yes ; T ; True + +# Expands_On_NFD (XO_NFD) + +XO_NFD; N ; No ; F ; False +XO_NFD; Y ; Yes ; T ; True + +# Expands_On_NFKC (XO_NFKC) + +XO_NFKC; N ; No ; F ; False +XO_NFKC; Y ; Yes ; T ; True + +# Expands_On_NFKD (XO_NFKD) + +XO_NFKD; N ; No ; F ; False +XO_NFKD; Y ; Yes ; T ; True + +# Extended_Pictographic (ExtPict) + +ExtPict; N ; No ; F ; False +ExtPict; Y ; Yes ; T ; True + +# Extender (Ext) + +Ext; N ; No ; F ; False +Ext; Y ; Yes ; T ; True + +# FC_NFKC_Closure (FC_NFKC) + +# @missing: 0000..10FFFF; FC_NFKC_Closure; + +# Full_Composition_Exclusion (Comp_Ex) + +Comp_Ex; N ; No ; F ; False +Comp_Ex; Y ; Yes ; T ; True + +# General_Category (gc) + +gc ; C ; Other # Cc | Cf | Cn | Co | Cs +gc ; Cc ; Control ; cntrl +gc ; Cf ; Format +gc ; Cn ; Unassigned +gc ; Co ; Private_Use +gc ; Cs ; Surrogate +gc ; L ; Letter # Ll | Lm | Lo | Lt | Lu +gc ; LC ; Cased_Letter # Ll | Lt | Lu +gc ; Ll ; Lowercase_Letter +gc ; Lm ; Modifier_Letter +gc ; Lo ; Other_Letter +gc ; Lt ; Titlecase_Letter +gc ; Lu ; Uppercase_Letter +gc ; M ; Mark ; Combining_Mark # Mc | Me | Mn +gc ; Mc ; Spacing_Mark +gc ; Me ; Enclosing_Mark +gc ; Mn ; Nonspacing_Mark +gc ; N ; Number # Nd | Nl | No +gc ; Nd ; Decimal_Number ; digit +gc ; Nl ; Letter_Number +gc ; No ; Other_Number +gc ; P ; Punctuation ; punct # Pc | Pd | Pe | Pf | Pi | Po | Ps +gc ; Pc ; Connector_Punctuation +gc ; Pd ; Dash_Punctuation +gc ; Pe ; Close_Punctuation +gc ; Pf ; Final_Punctuation +gc ; Pi ; Initial_Punctuation +gc ; Po ; Other_Punctuation +gc ; Ps ; Open_Punctuation +gc ; S ; Symbol # Sc | Sk | Sm | So +gc ; Sc ; Currency_Symbol +gc ; Sk ; Modifier_Symbol +gc ; Sm ; Math_Symbol +gc ; So ; Other_Symbol +gc ; Z ; Separator # Zl | Zp | Zs +gc ; Zl ; Line_Separator +gc ; Zp ; Paragraph_Separator +gc ; Zs ; Space_Separator +# @missing: 0000..10FFFF; General_Category; Unassigned + +# Grapheme_Base (Gr_Base) + +Gr_Base; N ; No ; F ; False +Gr_Base; Y ; Yes ; T ; True + +# Grapheme_Cluster_Break (GCB) + +GCB; CN ; Control +GCB; CR ; CR +GCB; EB ; E_Base +GCB; EBG ; E_Base_GAZ +GCB; EM ; E_Modifier +GCB; EX ; Extend +GCB; GAZ ; Glue_After_Zwj +GCB; L ; L +GCB; LF ; LF +GCB; LV ; LV +GCB; LVT ; LVT +GCB; PP ; Prepend +GCB; RI ; Regional_Indicator +GCB; SM ; SpacingMark +GCB; T ; T +GCB; V ; V +GCB; XX ; Other +GCB; ZWJ ; ZWJ + +# Grapheme_Extend (Gr_Ext) + +Gr_Ext; N ; No ; F ; False +Gr_Ext; Y ; Yes ; T ; True + +# Grapheme_Link (Gr_Link) + +Gr_Link; N ; No ; F ; False +Gr_Link; Y ; Yes ; T ; True + +# Hangul_Syllable_Type (hst) + +hst; L ; Leading_Jamo +hst; LV ; LV_Syllable +hst; LVT ; LVT_Syllable +hst; NA ; Not_Applicable +hst; T ; Trailing_Jamo +hst; V ; Vowel_Jamo + +# Hex_Digit (Hex) + +Hex; N ; No ; F ; False +Hex; Y ; Yes ; T ; True + +# Hyphen (Hyphen) + +Hyphen; N ; No ; F ; False +Hyphen; Y ; Yes ; T ; True + +# IDS_Binary_Operator (IDSB) + +IDSB; N ; No ; F ; False +IDSB; Y ; Yes ; T ; True + +# IDS_Trinary_Operator (IDST) + +IDST; N ; No ; F ; False +IDST; Y ; Yes ; T ; True + +# IDS_Unary_Operator (IDSU) + +IDSU; N ; No ; F ; False +IDSU; Y ; Yes ; T ; True + +# ID_Compat_Math_Continue (ID_Compat_Math_Continue) + +ID_Compat_Math_Continue; N ; No ; F ; False +ID_Compat_Math_Continue; Y ; Yes ; T ; True + +# ID_Compat_Math_Start (ID_Compat_Math_Start) + +ID_Compat_Math_Start; N ; No ; F ; False +ID_Compat_Math_Start; Y ; Yes ; T ; True + +# ID_Continue (IDC) + +IDC; N ; No ; F ; False +IDC; Y ; Yes ; T ; True + +# ID_Start (IDS) + +IDS; N ; No ; F ; False +IDS; Y ; Yes ; T ; True + +# ISO_Comment (isc) + +# @missing: 0000..10FFFF; ISO_Comment; + +# Ideographic (Ideo) + +Ideo; N ; No ; F ; False +Ideo; Y ; Yes ; T ; True + +# Indic_Conjunct_Break (InCB) + +InCB; Consonant ; Consonant +InCB; Extend ; Extend +InCB; Linker ; Linker +InCB; None ; None + +# Indic_Positional_Category (InPC) + +InPC; Bottom ; Bottom +InPC; Bottom_And_Left ; Bottom_And_Left +InPC; Bottom_And_Right ; Bottom_And_Right +InPC; Left ; Left +InPC; Left_And_Right ; Left_And_Right +InPC; NA ; Not_Applicable +InPC; Overstruck ; Overstruck +InPC; Right ; Right +InPC; Top ; Top +InPC; Top_And_Bottom ; Top_And_Bottom +InPC; Top_And_Bottom_And_Left ; Top_And_Bottom_And_Left +InPC; Top_And_Bottom_And_Right ; Top_And_Bottom_And_Right +InPC; Top_And_Left ; Top_And_Left +InPC; Top_And_Left_And_Right ; Top_And_Left_And_Right +InPC; Top_And_Right ; Top_And_Right +InPC; Visual_Order_Left ; Visual_Order_Left + +# Indic_Syllabic_Category (InSC) + +InSC; Avagraha ; Avagraha +InSC; Bindu ; Bindu +InSC; Brahmi_Joining_Number ; Brahmi_Joining_Number +InSC; Cantillation_Mark ; Cantillation_Mark +InSC; Consonant ; Consonant +InSC; Consonant_Dead ; Consonant_Dead +InSC; Consonant_Final ; Consonant_Final +InSC; Consonant_Head_Letter ; Consonant_Head_Letter +InSC; Consonant_Initial_Postfixed ; Consonant_Initial_Postfixed +InSC; Consonant_Killer ; Consonant_Killer +InSC; Consonant_Medial ; Consonant_Medial +InSC; Consonant_Placeholder ; Consonant_Placeholder +InSC; Consonant_Preceding_Repha ; Consonant_Preceding_Repha +InSC; Consonant_Prefixed ; Consonant_Prefixed +InSC; Consonant_Subjoined ; Consonant_Subjoined +InSC; Consonant_Succeeding_Repha ; Consonant_Succeeding_Repha +InSC; Consonant_With_Stacker ; Consonant_With_Stacker +InSC; Gemination_Mark ; Gemination_Mark +InSC; Invisible_Stacker ; Invisible_Stacker +InSC; Joiner ; Joiner +InSC; Modifying_Letter ; Modifying_Letter +InSC; Non_Joiner ; Non_Joiner +InSC; Nukta ; Nukta +InSC; Number ; Number +InSC; Number_Joiner ; Number_Joiner +InSC; Other ; Other +InSC; Pure_Killer ; Pure_Killer +InSC; Register_Shifter ; Register_Shifter +InSC; Reordering_Killer ; Reordering_Killer +InSC; Syllable_Modifier ; Syllable_Modifier +InSC; Tone_Letter ; Tone_Letter +InSC; Tone_Mark ; Tone_Mark +InSC; Virama ; Virama +InSC; Visarga ; Visarga +InSC; Vowel ; Vowel +InSC; Vowel_Dependent ; Vowel_Dependent +InSC; Vowel_Independent ; Vowel_Independent + +# Jamo_Short_Name (JSN) + +JSN; A ; A +JSN; AE ; AE +JSN; B ; B +JSN; BB ; BB +JSN; BS ; BS +JSN; C ; C +JSN; D ; D +JSN; DD ; DD +JSN; E ; E +JSN; EO ; EO +JSN; EU ; EU +JSN; G ; G +JSN; GG ; GG +JSN; GS ; GS +JSN; H ; H +JSN; I ; I +JSN; J ; J +JSN; JJ ; JJ +JSN; K ; K +JSN; L ; L +JSN; LB ; LB +JSN; LG ; LG +JSN; LH ; LH +JSN; LM ; LM +JSN; LP ; LP +JSN; LS ; LS +JSN; LT ; LT +JSN; M ; M +JSN; N ; N +JSN; NG ; NG +JSN; NH ; NH +JSN; NJ ; NJ +JSN; O ; O +JSN; OE ; OE +JSN; P ; P +JSN; R ; R +JSN; S ; S +JSN; SS ; SS +JSN; T ; T +JSN; U ; U +JSN; WA ; WA +JSN; WAE ; WAE +JSN; WE ; WE +JSN; WEO ; WEO +JSN; WI ; WI +JSN; YA ; YA +JSN; YAE ; YAE +JSN; YE ; YE +JSN; YEO ; YEO +JSN; YI ; YI +JSN; YO ; YO +JSN; YU ; YU +# @missing: 0000..10FFFF; Jamo_Short_Name; + +# Join_Control (Join_C) + +Join_C; N ; No ; F ; False +Join_C; Y ; Yes ; T ; True + +# Joining_Group (jg) + +jg ; African_Feh ; African_Feh +jg ; African_Noon ; African_Noon +jg ; African_Qaf ; African_Qaf +jg ; Ain ; Ain +jg ; Alaph ; Alaph +jg ; Alef ; Alef +jg ; Beh ; Beh +jg ; Beth ; Beth +jg ; Burushaski_Yeh_Barree ; Burushaski_Yeh_Barree +jg ; Dal ; Dal +jg ; Dalath_Rish ; Dalath_Rish +jg ; E ; E +jg ; Farsi_Yeh ; Farsi_Yeh +jg ; Fe ; Fe +jg ; Feh ; Feh +jg ; Final_Semkath ; Final_Semkath +jg ; Gaf ; Gaf +jg ; Gamal ; Gamal +jg ; Hah ; Hah +jg ; Hanifi_Rohingya_Kinna_Ya ; Hanifi_Rohingya_Kinna_Ya +jg ; Hanifi_Rohingya_Pa ; Hanifi_Rohingya_Pa +jg ; He ; He +jg ; Heh ; Heh +jg ; Heh_Goal ; Heh_Goal +jg ; Heth ; Heth +jg ; Kaf ; Kaf +jg ; Kaph ; Kaph +jg ; Kashmiri_Yeh ; Kashmiri_Yeh +jg ; Khaph ; Khaph +jg ; Knotted_Heh ; Knotted_Heh +jg ; Lam ; Lam +jg ; Lamadh ; Lamadh +jg ; Malayalam_Bha ; Malayalam_Bha +jg ; Malayalam_Ja ; Malayalam_Ja +jg ; Malayalam_Lla ; Malayalam_Lla +jg ; Malayalam_Llla ; Malayalam_Llla +jg ; Malayalam_Nga ; Malayalam_Nga +jg ; Malayalam_Nna ; Malayalam_Nna +jg ; Malayalam_Nnna ; Malayalam_Nnna +jg ; Malayalam_Nya ; Malayalam_Nya +jg ; Malayalam_Ra ; Malayalam_Ra +jg ; Malayalam_Ssa ; Malayalam_Ssa +jg ; Malayalam_Tta ; Malayalam_Tta +jg ; Manichaean_Aleph ; Manichaean_Aleph +jg ; Manichaean_Ayin ; Manichaean_Ayin +jg ; Manichaean_Beth ; Manichaean_Beth +jg ; Manichaean_Daleth ; Manichaean_Daleth +jg ; Manichaean_Dhamedh ; Manichaean_Dhamedh +jg ; Manichaean_Five ; Manichaean_Five +jg ; Manichaean_Gimel ; Manichaean_Gimel +jg ; Manichaean_Heth ; Manichaean_Heth +jg ; Manichaean_Hundred ; Manichaean_Hundred +jg ; Manichaean_Kaph ; Manichaean_Kaph +jg ; Manichaean_Lamedh ; Manichaean_Lamedh +jg ; Manichaean_Mem ; Manichaean_Mem +jg ; Manichaean_Nun ; Manichaean_Nun +jg ; Manichaean_One ; Manichaean_One +jg ; Manichaean_Pe ; Manichaean_Pe +jg ; Manichaean_Qoph ; Manichaean_Qoph +jg ; Manichaean_Resh ; Manichaean_Resh +jg ; Manichaean_Sadhe ; Manichaean_Sadhe +jg ; Manichaean_Samekh ; Manichaean_Samekh +jg ; Manichaean_Taw ; Manichaean_Taw +jg ; Manichaean_Ten ; Manichaean_Ten +jg ; Manichaean_Teth ; Manichaean_Teth +jg ; Manichaean_Thamedh ; Manichaean_Thamedh +jg ; Manichaean_Twenty ; Manichaean_Twenty +jg ; Manichaean_Waw ; Manichaean_Waw +jg ; Manichaean_Yodh ; Manichaean_Yodh +jg ; Manichaean_Zayin ; Manichaean_Zayin +jg ; Meem ; Meem +jg ; Mim ; Mim +jg ; No_Joining_Group ; No_Joining_Group +jg ; Noon ; Noon +jg ; Nun ; Nun +jg ; Nya ; Nya +jg ; Pe ; Pe +jg ; Qaf ; Qaf +jg ; Qaph ; Qaph +jg ; Reh ; Reh +jg ; Reversed_Pe ; Reversed_Pe +jg ; Rohingya_Yeh ; Rohingya_Yeh +jg ; Sad ; Sad +jg ; Sadhe ; Sadhe +jg ; Seen ; Seen +jg ; Semkath ; Semkath +jg ; Shin ; Shin +jg ; Straight_Waw ; Straight_Waw +jg ; Swash_Kaf ; Swash_Kaf +jg ; Syriac_Waw ; Syriac_Waw +jg ; Tah ; Tah +jg ; Taw ; Taw +jg ; Teh_Marbuta ; Teh_Marbuta +jg ; Teh_Marbuta_Goal ; Teh_Marbuta_Goal ; Hamza_On_Heh_Goal +jg ; Teth ; Teth +jg ; Thin_Noon ; Thin_Noon +jg ; Thin_Yeh ; Thin_Yeh +jg ; Vertical_Tail ; Vertical_Tail +jg ; Waw ; Waw +jg ; Yeh ; Yeh +jg ; Yeh_Barree ; Yeh_Barree +jg ; Yeh_With_Tail ; Yeh_With_Tail +jg ; Yudh ; Yudh +jg ; Yudh_He ; Yudh_He +jg ; Zain ; Zain +jg ; Zhain ; Zhain + +# Joining_Type (jt) + +jt ; C ; Join_Causing +jt ; D ; Dual_Joining +jt ; L ; Left_Joining +jt ; R ; Right_Joining +jt ; T ; Transparent +jt ; U ; Non_Joining + +# Line_Break (lb) + +lb ; AI ; Ambiguous +lb ; AK ; Aksara +lb ; AL ; Alphabetic +lb ; AP ; Aksara_Prebase +lb ; AS ; Aksara_Start +lb ; B2 ; Break_Both +lb ; BA ; Break_After +lb ; BB ; Break_Before +lb ; BK ; Mandatory_Break +lb ; CB ; Contingent_Break +lb ; CJ ; Conditional_Japanese_Starter +lb ; CL ; Close_Punctuation +lb ; CM ; Combining_Mark +lb ; CP ; Close_Parenthesis +lb ; CR ; Carriage_Return +lb ; EB ; E_Base +lb ; EM ; E_Modifier +lb ; EX ; Exclamation +lb ; GL ; Glue +lb ; H2 ; H2 +lb ; H3 ; H3 +lb ; HH ; Unambiguous_Hyphen +lb ; HL ; Hebrew_Letter +lb ; HY ; Hyphen +lb ; ID ; Ideographic +lb ; IN ; Inseparable ; Inseperable +lb ; IS ; Infix_Numeric +lb ; JL ; JL +lb ; JT ; JT +lb ; JV ; JV +lb ; LF ; Line_Feed +lb ; NL ; Next_Line +lb ; NS ; Nonstarter +lb ; NU ; Numeric +lb ; OP ; Open_Punctuation +lb ; PO ; Postfix_Numeric +lb ; PR ; Prefix_Numeric +lb ; QU ; Quotation +lb ; RI ; Regional_Indicator +lb ; SA ; Complex_Context +lb ; SG ; Surrogate +lb ; SP ; Space +lb ; SY ; Break_Symbols +lb ; VF ; Virama_Final +lb ; VI ; Virama +lb ; WJ ; Word_Joiner +lb ; XX ; Unknown +lb ; ZW ; ZWSpace +lb ; ZWJ ; ZWJ + +# Logical_Order_Exception (LOE) + +LOE; N ; No ; F ; False +LOE; Y ; Yes ; T ; True + +# Lowercase (Lower) + +Lower; N ; No ; F ; False +Lower; Y ; Yes ; T ; True + +# Lowercase_Mapping (lc) + +# @missing: 0000..10FFFF; Lowercase_Mapping; + +# Math (Math) + +Math; N ; No ; F ; False +Math; Y ; Yes ; T ; True + +# Modifier_Combining_Mark (MCM) + +MCM; N ; No ; F ; False +MCM; Y ; Yes ; T ; True + +# NFC_Quick_Check (NFC_QC) + +NFC_QC; M ; Maybe +NFC_QC; N ; No +NFC_QC; Y ; Yes + +# NFD_Quick_Check (NFD_QC) + +NFD_QC; N ; No +NFD_QC; Y ; Yes + +# NFKC_Casefold (NFKC_CF) + + +# NFKC_Quick_Check (NFKC_QC) + +NFKC_QC; M ; Maybe +NFKC_QC; N ; No +NFKC_QC; Y ; Yes + +# NFKC_Simple_Casefold (NFKC_SCF) + + +# NFKD_Quick_Check (NFKD_QC) + +NFKD_QC; N ; No +NFKD_QC; Y ; Yes + +# Name (na) + +# @missing: 0000..10FFFF; Name; + +# Name_Alias (Name_Alias) + +# @missing: 0000..10FFFF; Name_Alias; + +# Noncharacter_Code_Point (NChar) + +NChar; N ; No ; F ; False +NChar; Y ; Yes ; T ; True + +# Numeric_Type (nt) + +nt ; De ; Decimal +nt ; Di ; Digit +nt ; None ; None +nt ; Nu ; Numeric + +# Numeric_Value (nv) + +# @missing: 0000..10FFFF; Numeric_Value; NaN + +# Other_Alphabetic (OAlpha) + +OAlpha; N ; No ; F ; False +OAlpha; Y ; Yes ; T ; True + +# Other_Default_Ignorable_Code_Point (ODI) + +ODI; N ; No ; F ; False +ODI; Y ; Yes ; T ; True + +# Other_Grapheme_Extend (OGr_Ext) + +OGr_Ext; N ; No ; F ; False +OGr_Ext; Y ; Yes ; T ; True + +# Other_ID_Continue (OIDC) + +OIDC; N ; No ; F ; False +OIDC; Y ; Yes ; T ; True + +# Other_ID_Start (OIDS) + +OIDS; N ; No ; F ; False +OIDS; Y ; Yes ; T ; True + +# Other_Lowercase (OLower) + +OLower; N ; No ; F ; False +OLower; Y ; Yes ; T ; True + +# Other_Math (OMath) + +OMath; N ; No ; F ; False +OMath; Y ; Yes ; T ; True + +# Other_Uppercase (OUpper) + +OUpper; N ; No ; F ; False +OUpper; Y ; Yes ; T ; True + +# Pattern_Syntax (Pat_Syn) + +Pat_Syn; N ; No ; F ; False +Pat_Syn; Y ; Yes ; T ; True + +# Pattern_White_Space (Pat_WS) + +Pat_WS; N ; No ; F ; False +Pat_WS; Y ; Yes ; T ; True + +# Prepended_Concatenation_Mark (PCM) + +PCM; N ; No ; F ; False +PCM; Y ; Yes ; T ; True + +# Quotation_Mark (QMark) + +QMark; N ; No ; F ; False +QMark; Y ; Yes ; T ; True + +# Radical (Radical) + +Radical; N ; No ; F ; False +Radical; Y ; Yes ; T ; True + +# Regional_Indicator (RI) + +RI ; N ; No ; F ; False +RI ; Y ; Yes ; T ; True + +# Script (sc) + +sc ; Adlm ; Adlam +sc ; Aghb ; Caucasian_Albanian +sc ; Ahom ; Ahom +sc ; Arab ; Arabic +sc ; Armi ; Imperial_Aramaic +sc ; Armn ; Armenian +sc ; Avst ; Avestan +sc ; Bali ; Balinese +sc ; Bamu ; Bamum +sc ; Bass ; Bassa_Vah +sc ; Batk ; Batak +sc ; Beng ; Bengali +sc ; Berf ; Beria_Erfe +sc ; Bhks ; Bhaiksuki +sc ; Bopo ; Bopomofo +sc ; Brah ; Brahmi +sc ; Brai ; Braille +sc ; Bugi ; Buginese +sc ; Buhd ; Buhid +sc ; Cakm ; Chakma +sc ; Cans ; Canadian_Aboriginal +sc ; Cari ; Carian +sc ; Cham ; Cham +sc ; Cher ; Cherokee +sc ; Chrs ; Chorasmian +sc ; Copt ; Coptic ; Qaac +sc ; Cpmn ; Cypro_Minoan +sc ; Cprt ; Cypriot +sc ; Cyrl ; Cyrillic +sc ; Deva ; Devanagari +sc ; Diak ; Dives_Akuru +sc ; Dogr ; Dogra +sc ; Dsrt ; Deseret +sc ; Dupl ; Duployan +sc ; Egyp ; Egyptian_Hieroglyphs +sc ; Elba ; Elbasan +sc ; Elym ; Elymaic +sc ; Ethi ; Ethiopic +sc ; Gara ; Garay +sc ; Geor ; Georgian +sc ; Glag ; Glagolitic +sc ; Gong ; Gunjala_Gondi +sc ; Gonm ; Masaram_Gondi +sc ; Goth ; Gothic +sc ; Gran ; Grantha +sc ; Grek ; Greek +sc ; Gujr ; Gujarati +sc ; Gukh ; Gurung_Khema +sc ; Guru ; Gurmukhi +sc ; Hang ; Hangul +sc ; Hani ; Han +sc ; Hano ; Hanunoo +sc ; Hatr ; Hatran +sc ; Hebr ; Hebrew +sc ; Hira ; Hiragana +sc ; Hluw ; Anatolian_Hieroglyphs +sc ; Hmng ; Pahawh_Hmong +sc ; Hmnp ; Nyiakeng_Puachue_Hmong +sc ; Hrkt ; Katakana_Or_Hiragana +sc ; Hung ; Old_Hungarian +sc ; Ital ; Old_Italic +sc ; Java ; Javanese +sc ; Kali ; Kayah_Li +sc ; Kana ; Katakana +sc ; Kawi ; Kawi +sc ; Khar ; Kharoshthi +sc ; Khmr ; Khmer +sc ; Khoj ; Khojki +sc ; Kits ; Khitan_Small_Script +sc ; Knda ; Kannada +sc ; Krai ; Kirat_Rai +sc ; Kthi ; Kaithi +sc ; Lana ; Tai_Tham +sc ; Laoo ; Lao +sc ; Latn ; Latin +sc ; Lepc ; Lepcha +sc ; Limb ; Limbu +sc ; Lina ; Linear_A +sc ; Linb ; Linear_B +sc ; Lisu ; Lisu +sc ; Lyci ; Lycian +sc ; Lydi ; Lydian +sc ; Mahj ; Mahajani +sc ; Maka ; Makasar +sc ; Mand ; Mandaic +sc ; Mani ; Manichaean +sc ; Marc ; Marchen +sc ; Medf ; Medefaidrin +sc ; Mend ; Mende_Kikakui +sc ; Merc ; Meroitic_Cursive +sc ; Mero ; Meroitic_Hieroglyphs +sc ; Mlym ; Malayalam +sc ; Modi ; Modi +sc ; Mong ; Mongolian +sc ; Mroo ; Mro +sc ; Mtei ; Meetei_Mayek +sc ; Mult ; Multani +sc ; Mymr ; Myanmar +sc ; Nagm ; Nag_Mundari +sc ; Nand ; Nandinagari +sc ; Narb ; Old_North_Arabian +sc ; Nbat ; Nabataean +sc ; Newa ; Newa +sc ; Nkoo ; Nko +sc ; Nshu ; Nushu +sc ; Ogam ; Ogham +sc ; Olck ; Ol_Chiki +sc ; Onao ; Ol_Onal +sc ; Orkh ; Old_Turkic +sc ; Orya ; Oriya +sc ; Osge ; Osage +sc ; Osma ; Osmanya +sc ; Ougr ; Old_Uyghur +sc ; Palm ; Palmyrene +sc ; Pauc ; Pau_Cin_Hau +sc ; Perm ; Old_Permic +sc ; Phag ; Phags_Pa +sc ; Phli ; Inscriptional_Pahlavi +sc ; Phlp ; Psalter_Pahlavi +sc ; Phnx ; Phoenician +sc ; Plrd ; Miao +sc ; Prti ; Inscriptional_Parthian +sc ; Rjng ; Rejang +sc ; Rohg ; Hanifi_Rohingya +sc ; Runr ; Runic +sc ; Samr ; Samaritan +sc ; Sarb ; Old_South_Arabian +sc ; Saur ; Saurashtra +sc ; Sgnw ; SignWriting +sc ; Shaw ; Shavian +sc ; Shrd ; Sharada +sc ; Sidd ; Siddham +sc ; Sidt ; Sidetic +sc ; Sind ; Khudawadi +sc ; Sinh ; Sinhala +sc ; Sogd ; Sogdian +sc ; Sogo ; Old_Sogdian +sc ; Sora ; Sora_Sompeng +sc ; Soyo ; Soyombo +sc ; Sund ; Sundanese +sc ; Sunu ; Sunuwar +sc ; Sylo ; Syloti_Nagri +sc ; Syrc ; Syriac +sc ; Tagb ; Tagbanwa +sc ; Takr ; Takri +sc ; Tale ; Tai_Le +sc ; Talu ; New_Tai_Lue +sc ; Taml ; Tamil +sc ; Tang ; Tangut +sc ; Tavt ; Tai_Viet +sc ; Tayo ; Tai_Yo +sc ; Telu ; Telugu +sc ; Tfng ; Tifinagh +sc ; Tglg ; Tagalog +sc ; Thaa ; Thaana +sc ; Thai ; Thai +sc ; Tibt ; Tibetan +sc ; Tirh ; Tirhuta +sc ; Tnsa ; Tangsa +sc ; Todr ; Todhri +sc ; Tols ; Tolong_Siki +sc ; Toto ; Toto +sc ; Tutg ; Tulu_Tigalari +sc ; Ugar ; Ugaritic +sc ; Vaii ; Vai +sc ; Vith ; Vithkuqi +sc ; Wara ; Warang_Citi +sc ; Wcho ; Wancho +sc ; Xpeo ; Old_Persian +sc ; Xsux ; Cuneiform +sc ; Yezi ; Yezidi +sc ; Yiii ; Yi +sc ; Zanb ; Zanabazar_Square +sc ; Zinh ; Inherited ; Qaai +sc ; Zyyy ; Common +sc ; Zzzz ; Unknown + +# Script_Extensions (scx) + + +# Sentence_Break (SB) + +SB ; AT ; ATerm +SB ; CL ; Close +SB ; CR ; CR +SB ; EX ; Extend +SB ; FO ; Format +SB ; LE ; OLetter +SB ; LF ; LF +SB ; LO ; Lower +SB ; NU ; Numeric +SB ; SC ; SContinue +SB ; SE ; Sep +SB ; SP ; Sp +SB ; ST ; STerm +SB ; UP ; Upper +SB ; XX ; Other + +# Sentence_Terminal (STerm) + +STerm; N ; No ; F ; False +STerm; Y ; Yes ; T ; True + +# Simple_Case_Folding (scf) + +# @missing: 0000..10FFFF; Simple_Case_Folding; + +# Simple_Lowercase_Mapping (slc) + +# @missing: 0000..10FFFF; Simple_Lowercase_Mapping; + +# Simple_Titlecase_Mapping (stc) + +# @missing: 0000..10FFFF; Simple_Titlecase_Mapping; + +# Simple_Uppercase_Mapping (suc) + +# @missing: 0000..10FFFF; Simple_Uppercase_Mapping; + +# Soft_Dotted (SD) + +SD ; N ; No ; F ; False +SD ; Y ; Yes ; T ; True + +# Terminal_Punctuation (Term) + +Term; N ; No ; F ; False +Term; Y ; Yes ; T ; True + +# Titlecase_Mapping (tc) + +# @missing: 0000..10FFFF; Titlecase_Mapping; + +# Unicode_1_Name (na1) + +# @missing: 0000..10FFFF; Unicode_1_Name; + +# Unified_Ideograph (UIdeo) + +UIdeo; N ; No ; F ; False +UIdeo; Y ; Yes ; T ; True + +# Uppercase (Upper) + +Upper; N ; No ; F ; False +Upper; Y ; Yes ; T ; True + +# Uppercase_Mapping (uc) + +# @missing: 0000..10FFFF; Uppercase_Mapping; + +# Variation_Selector (VS) + +VS ; N ; No ; F ; False +VS ; Y ; Yes ; T ; True + +# Vertical_Orientation (vo) + +vo ; R ; Rotated +vo ; Tr ; Transformed_Rotated +vo ; Tu ; Transformed_Upright +vo ; U ; Upright + +# White_Space (WSpace) + +WSpace; N ; No ; F ; False +WSpace; Y ; Yes ; T ; True + +# Word_Break (WB) + +WB ; CR ; CR +WB ; DQ ; Double_Quote +WB ; EB ; E_Base +WB ; EBG ; E_Base_GAZ +WB ; EM ; E_Modifier +WB ; EX ; ExtendNumLet +WB ; Extend ; Extend +WB ; FO ; Format +WB ; GAZ ; Glue_After_Zwj +WB ; HL ; Hebrew_Letter +WB ; KA ; Katakana +WB ; LE ; ALetter +WB ; LF ; LF +WB ; MB ; MidNumLet +WB ; ML ; MidLetter +WB ; MN ; MidNum +WB ; NL ; Newline +WB ; NU ; Numeric +WB ; RI ; Regional_Indicator +WB ; SQ ; Single_Quote +WB ; WSegSpace ; WSegSpace +WB ; XX ; Other +WB ; ZWJ ; ZWJ + +# XID_Continue (XIDC) + +XIDC; N ; No ; F ; False +XIDC; Y ; Yes ; T ; True + +# XID_Start (XIDS) + +XIDS; N ; No ; F ; False +XIDS; Y ; Yes ; T ; True + +# cjkAccountingNumeric (cjkAccountingNumeric) + +# @missing: 0000..10FFFF; cjkAccountingNumeric; NaN + +# cjkCompatibilityVariant (cjkCompatibilityVariant) + +# @missing: 0000..10FFFF; cjkCompatibilityVariant; + +# cjkIICore (cjkIICore) + +# @missing: 0000..10FFFF; cjkIICore; + +# cjkIRG_GSource (cjkIRG_GSource) + +# @missing: 0000..10FFFF; cjkIRG_GSource; + +# cjkIRG_HSource (cjkIRG_HSource) + +# @missing: 0000..10FFFF; cjkIRG_HSource; + +# cjkIRG_JSource (cjkIRG_JSource) + +# @missing: 0000..10FFFF; cjkIRG_JSource; + +# cjkIRG_KPSource (cjkIRG_KPSource) + +# @missing: 0000..10FFFF; cjkIRG_KPSource; + +# cjkIRG_KSource (cjkIRG_KSource) + +# @missing: 0000..10FFFF; cjkIRG_KSource; + +# cjkIRG_MSource (cjkIRG_MSource) + +# @missing: 0000..10FFFF; cjkIRG_MSource; + +# cjkIRG_SSource (cjkIRG_SSource) + +# @missing: 0000..10FFFF; cjkIRG_SSource; + +# cjkIRG_TSource (cjkIRG_TSource) + +# @missing: 0000..10FFFF; cjkIRG_TSource; + +# cjkIRG_UKSource (cjkIRG_UKSource) + +# @missing: 0000..10FFFF; cjkIRG_UKSource; + +# cjkIRG_USource (cjkIRG_USource) + +# @missing: 0000..10FFFF; cjkIRG_USource; + +# cjkIRG_VSource (cjkIRG_VSource) + +# @missing: 0000..10FFFF; cjkIRG_VSource; + +# cjkOtherNumeric (cjkOtherNumeric) + +# @missing: 0000..10FFFF; cjkOtherNumeric; NaN + +# cjkPrimaryNumeric (cjkPrimaryNumeric) + +# @missing: 0000..10FFFF; cjkPrimaryNumeric; NaN + +# cjkRSUnicode (cjkRSUnicode) + +# @missing: 0000..10FFFF; cjkRSUnicode; + +# kEH_Cat (kEH_Cat) + +# @missing: 0000..10FFFF; kEH_Cat; + +# kEH_Desc (kEH_Desc) + +# @missing: 0000..10FFFF; kEH_Desc; + +# kEH_HG (kEH_HG) + +# @missing: 0000..10FFFF; kEH_HG; + +# kEH_IFAO (kEH_IFAO) + +# @missing: 0000..10FFFF; kEH_IFAO; + +# kEH_JSesh (kEH_JSesh) + +# @missing: 0000..10FFFF; kEH_JSesh; + +# kEH_NoMirror (kEH_NoMirror) + +kEH_NoMirror; N ; No ; F ; False +kEH_NoMirror; Y ; Yes ; T ; True + +# kEH_NoRotate (kEH_NoRotate) + +kEH_NoRotate; N ; No ; F ; False +kEH_NoRotate; Y ; Yes ; T ; True + +# kMandarin (cjkMandarin) + +# @missing: 0000..10FFFF; kMandarin; + +# kTotalStrokes (cjkTotalStrokes) + +# @missing: 0000..10FFFF; kTotalStrokes; + +# kUnihanCore2020 (cjkUnihanCore2020) + +# @missing: 0000..10FFFF; kUnihanCore2020; + +# EOF diff --git a/lib/elixir/unicode/ScriptExtensions.txt b/lib/elixir/unicode/ScriptExtensions.txt new file mode 100644 index 00000000000..98b8d0fb06e --- /dev/null +++ b/lib/elixir/unicode/ScriptExtensions.txt @@ -0,0 +1,235 @@ +# ScriptExtensions-17.0.0.txt +# Date: 2025-08-01, 21:42:00 GMT +# © 2025 Unicode®, Inc. +# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. +# For terms of use and license, see https://www.unicode.org/terms_of_use.html +# +# Unicode Character Database +# For documentation, see https://www.unicode.org/reports/tr44/ +# +# The Script_Extensions property indicates which characters are commonly used +# with more than one script, but with a limited number of scripts. +# For each code point, there is one or more property values. Each such value is a Script property value. +# For more information, see: +# UAX #24, Unicode Script Property: https://www.unicode.org/reports/tr24/ +# Especially the sections: +# https://www.unicode.org/reports/tr24/#Assignment_Script_Values +# https://www.unicode.org/reports/tr24/#Assignment_ScriptX_Values +# +# Each Script_Extensions value in this file consists of a set +# of one or more abbreviated Script property values. The ordering of the +# values in that set is not material, but for stability in presentation +# it is given here as alphabetical. +# +# All code points not explicitly listed for Script_Extensions +# have as their value the corresponding Script property value. +# +# @missing: 0000..10FFFF;