# Copyright 2010-2021, Google Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

# Description:
#   The implementation of dictionaries used in Mozc.

load("@bazel_skylib//rules:run_binary.bzl", "run_binary")
load(
    "//:build_defs.bzl",
    "mozc_cc_binary",
    "mozc_cc_library",
    "mozc_cc_test",
    "mozc_py_binary",
    "mozc_py_library",
    "mozc_select",
)

package(default_visibility = ["//visibility:private"])

mozc_cc_library(
    name = "dictionary_interface",
    hdrs = ["dictionary_interface.h"],
    visibility = [
        # For //converter:converter_impl.
        "//converter:__pkg__",
        # For //dictionary/system:system_dictionary.
        "//dictionary/system:__pkg__",
        # For //engine:small_immutable_converter.
        "//engine:__pkg__",
        # For //prediction:dictionary_predictor.
        "//prediction:__pkg__",
        # For //rewriter:language_aware_rewriter.
        "//rewriter:__pkg__",
    ],
    deps = [
        ":dictionary_token",
        "//protocol:user_dictionary_storage_cc_proto",
        "//request:conversion_request",
        "@com_google_absl//absl/strings",
    ],
)

mozc_cc_test(
    name = "dictionary_impl_test",
    size = "small",
    srcs = ["dictionary_impl_test.cc"],
    deps = [
        ":dictionary_impl",
        ":dictionary_interface",
        ":dictionary_token",
        ":pos_matcher",
        ":user_dictionary",
        ":user_pos",
        "//base:util",
        "//config:config_handler",
        "//data_manager/testing:mock_data_manager",
        "//dictionary/system:system_dictionary",
        "//dictionary/system:value_dictionary",
        "//protocol:config_cc_proto",
        "//request:conversion_request",
        "//testing:gunit_main",
        "@com_google_absl//absl/strings",
    ],
)

mozc_cc_library(
    name = "dictionary_impl",
    srcs = [
        "dictionary_impl.cc",
    ],
    hdrs = [
        "dictionary_impl.h",
    ],
    visibility = ["//engine:__pkg__"],
    deps = [
        ":dictionary_interface",
        ":dictionary_token",
        ":pos_matcher",
        "//base:util",
        "//protocol:config_cc_proto",
        "//request:conversion_request",
        "@com_google_absl//absl/algorithm:container",
        "@com_google_absl//absl/log:check",
        "@com_google_absl//absl/strings",
    ],
)

mozc_cc_library(
    name = "suffix_dictionary",
    srcs = ["suffix_dictionary.cc"],
    hdrs = ["suffix_dictionary.h"],
    visibility = ["//engine:__pkg__"],
    deps = [
        ":dictionary_interface",
        ":dictionary_token",
        "//base/container:serialized_string_array",
        "//request:conversion_request",
        "@com_google_absl//absl/log",
        "@com_google_absl//absl/log:check",
        "@com_google_absl//absl/strings",
        "@com_google_absl//absl/types:span",
    ],
)

mozc_cc_library(
    name = "user_dictionary",
    srcs = [
        "user_dictionary.cc",
    ],
    hdrs = [
        "user_dictionary.h",
    ],
    visibility = [
        "//:__subpackages__",
        "//converter:__pkg__",
    ],
    deps = [
        ":dictionary_interface",
        ":dictionary_token",
        ":pos_matcher",
        ":user_dictionary_importer",
        ":user_dictionary_storage",
        ":user_dictionary_util",
        ":user_pos",
        "//base:file_util",
        "//base:hash",
        "//base:thread",
        "//base:vlog",
        "//base/strings:assign",
        "//base/strings:japanese",
        "//base/strings:unicode",
        "//protocol:config_cc_proto",
        "//protocol:user_dictionary_storage_cc_proto",
        "//request:conversion_request",
        "@com_google_absl//absl/base:core_headers",
        "@com_google_absl//absl/container:flat_hash_set",
        "@com_google_absl//absl/hash",
        "@com_google_absl//absl/log",
        "@com_google_absl//absl/log:check",
        "@com_google_absl//absl/status",
        "@com_google_absl//absl/status:statusor",
        "@com_google_absl//absl/strings",
        "@com_google_absl//absl/synchronization",
    ],
)

mozc_cc_test(
    name = "user_dictionary_test",
    # The size is "large" because AsyncImportTest takes long time.
    # https://github.com/google/mozc/issues/1418
    size = "large",
    srcs = [
        "user_dictionary_test.cc",
    ],
    deps = [
        ":dictionary_interface",
        ":dictionary_mock",
        ":dictionary_test_util",
        ":dictionary_token",
        ":pos_matcher",
        ":user_dictionary",
        ":user_dictionary_storage",
        ":user_pos",
        "//base:file_util",
        "//base:random",
        "//base:system_util",
        "//base:thread",
        "//base/file:temp_dir",
        "//config:config_handler",
        "//data_manager/testing:mock_data_manager",
        "//protocol:config_cc_proto",
        "//protocol:user_dictionary_storage_cc_proto",
        "//request:conversion_request",
        "//testing:gunit_main",
        "//testing:mozctest",
        "@com_google_absl//absl/log:check",
        "@com_google_absl//absl/strings",
        "@com_google_absl//absl/strings:str_format",
    ],
)

mozc_cc_library(
    name = "user_dictionary_stub",
    hdrs = ["user_dictionary_stub.h"],
    visibility = [
        "//:__subpackages__",
        "//converter:__pkg__",
    ],
    deps = [
        ":dictionary_interface",
        "//protocol:user_dictionary_storage_cc_proto",
        "//request:conversion_request",
        "@com_google_absl//absl/strings",
    ],
)

# TODO(team): move this rule into dictionary/system.
mozc_cc_library(
    name = "dictionary_token",
    hdrs = ["dictionary_token.h"],
    visibility = [
        "//converter:__subpackages__",
        "//dictionary/system:__subpackages__",
        # For //prediction:dictionary_predictor.
        "//prediction:__pkg__",
        # For //rewriter:date_rewriter
        "//rewriter:__pkg__",
    ],
    deps = ["@com_google_absl//absl/strings"],
)

mozc_py_binary(
    name = "gen_aux_dictionary",
    srcs = ["gen_aux_dictionary.py"],
    visibility = [
        "//:__subpackages__",
        "//data/dictionary:__subpackages__",
        "//data/dictionary_oss:__subpackages__",
        "//dictionary:__subpackages__",
    ],
)

mozc_py_binary(
    name = "gen_filtered_dictionary",
    srcs = ["gen_filtered_dictionary.py"],
    visibility = [
        "//:__subpackages__",
        "//data/dictionary:__subpackages__",
        "//data/dictionary_oss:__subpackages__",
        "//dictionary:__subpackages__",
    ],
)

mozc_cc_library(
    name = "pos_group",
    hdrs = ["pos_group.h"],
    visibility = [
        "//:__subpackages__",
        "//rewriter:__pkg__",
    ],
    deps = ["@com_google_absl//absl/types:span"],
)

mozc_py_binary(
    name = "gen_pos_map",
    srcs = ["gen_pos_map.py"],
    visibility = ["//data_manager:__subpackages__"],
    deps = [
        "//build_tools:code_generator_util",
    ],
)

run_binary(
    name = "pos_map",
    srcs = [
        "//data/rules:third_party_pos_map.def",
        "//data/rules:user_pos.def",
    ],
    outs = ["pos_map.inc"],
    args = [
        "--user_pos_file=" +
        "$(location //data/rules:user_pos.def)",
        "--third_party_pos_map_file=" +
        "$(location" +
        " //data/rules:third_party_pos_map.def)",
        "--output=$(location pos_map.inc)",
    ],
    tool = ":gen_pos_map",
)

mozc_cc_library(
    name = "user_dictionary_storage",
    srcs = ["user_dictionary_storage.cc"],
    hdrs = ["user_dictionary_storage.h"],
    visibility = [
        "//:__subpackages__",
        "//rewriter:__pkg__",
    ],
    deps = [
        ":user_dictionary_util",
        "//base:config_file_stream",
        "//base:file_stream",
        "//base:file_util",
        "//base:process_mutex",
        "//base/protobuf:coded_stream",
        "//base/protobuf:zero_copy_stream_impl",
        "//protocol:user_dictionary_storage_cc_proto",
        "@com_google_absl//absl/log",
        "@com_google_absl//absl/log:check",
        "@com_google_absl//absl/random",
        "@com_google_absl//absl/status",
        "@com_google_absl//absl/status:statusor",
        "@com_google_absl//absl/strings",
        "@com_google_absl//absl/strings:str_format",
    ],
)

mozc_cc_test(
    name = "user_dictionary_storage_test",
    size = "small",
    srcs = ["user_dictionary_storage_test.cc"],
    deps = [
        ":user_dictionary_importer",
        ":user_dictionary_storage",
        ":user_dictionary_util",
        "//base:file_stream",
        "//base:file_util",
        "//base:mmap",
        "//base:random",
        "//base:system_util",
        "//base/file:temp_dir",
        "//protocol:user_dictionary_storage_cc_proto",
        "//testing:gunit_main",
        "//testing:mozctest",
        "//testing:testing_util",
        "@com_google_absl//absl/log:check",
        "@com_google_absl//absl/random",
        "@com_google_absl//absl/status:statusor",
        "@com_google_absl//absl/strings",
    ],
)

mozc_cc_library(
    name = "user_dictionary_util",
    srcs = ["user_dictionary_util.cc"],
    hdrs = ["user_dictionary_util.h"],
    visibility = [
        "//:__subpackages__",
    ],
    deps = [
        ":user_pos",
        "//base:vlog",
        "//base/strings:japanese",
        "//base/strings:unicode",
        "//protocol:user_dictionary_storage_cc_proto",
        "@com_google_absl//absl/algorithm:container",
        "@com_google_absl//absl/log",
        "@com_google_absl//absl/status",
        "@com_google_absl//absl/strings",
    ],
)

mozc_cc_test(
    name = "user_dictionary_util_test",
    size = "small",
    srcs = ["user_dictionary_util_test.cc"],
    deps = [
        ":user_dictionary_util",
        "//protocol:user_dictionary_storage_cc_proto",
        "//testing:gunit_main",
        "//testing:testing_util",
        "@com_google_absl//absl/strings",
    ],
)

mozc_cc_library(
    name = "user_dictionary_importer",
    srcs = ["user_dictionary_importer.cc"],
    hdrs = ["user_dictionary_importer.h"],
    textual_hdrs = [":pos_map"],
    visibility = [
        "//gui:__subpackages__",
    ],
    deps = [
        ":user_dictionary_storage",
        ":user_dictionary_util",
        "//base:hash",
        "//base:japanese_util",
        "//base:mmap",
        "//base:number_util",
        "//base:util",
        "//base:vlog",
        "//base/strings:assign",
        "//base/strings:unicode",
        "//protocol:user_dictionary_storage_cc_proto",
        "@com_google_absl//absl/algorithm:container",
        "@com_google_absl//absl/base:core_headers",
        "@com_google_absl//absl/container:flat_hash_set",
        "@com_google_absl//absl/hash",
        "@com_google_absl//absl/log",
        "@com_google_absl//absl/log:check",
        "@com_google_absl//absl/status",
        "@com_google_absl//absl/status:statusor",
        "@com_google_absl//absl/strings",
        "@com_google_absl//absl/types:span",
    ],
)

mozc_cc_test(
    name = "user_dictionary_importer_test",
    size = "small",
    srcs = ["user_dictionary_importer_test.cc"],
    deps = [
        ":user_dictionary_importer",
        ":user_dictionary_storage",
        ":user_dictionary_util",
        "//protocol:user_dictionary_storage_cc_proto",
        "//testing:gunit_main",
        "@com_google_absl//absl/strings",
        "@com_google_absl//absl/strings:string_view",
    ],
)

# TODO(team): move this rule into dictionary/system.
mozc_cc_library(
    name = "text_dictionary_loader",
    srcs = [
        "text_dictionary_loader.cc",
    ],
    hdrs = [
        "text_dictionary_loader.h",
    ],
    visibility = ["//dictionary:__subpackages__"],
    deps = [
        ":dictionary_token",
        ":pos_matcher",
        "//base:japanese_util",
        "//base:multifile",
        "//base:util",
        "//base:vlog",
        "@com_google_absl//absl/base:core_headers",
        "@com_google_absl//absl/flags:flag",
        "@com_google_absl//absl/log",
        "@com_google_absl//absl/log:check",
        "@com_google_absl//absl/strings",
        "@com_google_absl//absl/types:span",
    ],
)

mozc_cc_test(
    name = "text_dictionary_loader_test",
    size = "small",
    srcs = [
        "text_dictionary_loader_test.cc",
    ],
    deps = [
        ":dictionary_token",
        ":pos_matcher",
        ":text_dictionary_loader",
        "//base:file_util",
        "//base/file:temp_dir",
        "//data_manager/testing:mock_data_manager",
        "//testing:gunit_main",
        "//testing:mozctest",
        "//testing:test_peer",
        "@com_google_absl//absl/types:span",
    ],
)

mozc_cc_library(
    name = "user_pos",
    srcs = ["user_pos.cc"],
    hdrs = ["user_pos.h"],
    visibility = [
        "//converter:__pkg__",
        "//engine:__pkg__",
        "//gui:__subpackages__",
        "//rewriter:__pkg__",
    ],
    deps = [
        "//base:bits",
        "//base/container:serialized_string_array",
        "//base/strings:assign",
        "//data_manager",
        "@com_google_absl//absl/container:flat_hash_set",
        "@com_google_absl//absl/log:check",
        "@com_google_absl//absl/strings",
    ],
)

mozc_cc_test(
    name = "user_pos_test",
    size = "small",
    srcs = ["user_pos_test.cc"],
    deps = [
        ":user_pos",
        "//data_manager/testing:mock_data_manager",
        "//testing:gunit_main",
        "@com_google_absl//absl/log:check",
        "@com_google_absl//absl/strings",
    ],
)

mozc_py_library(
    name = "pos_util",
    srcs = ["pos_util.py"],
    deps = [
        "//build_tools:code_generator_util",
    ],
)

mozc_py_binary(
    name = "gen_user_pos_data",
    srcs = ["gen_user_pos_data.py"],
    visibility = [
        "//:__subpackages__",
        "//data_manager:__subpackages__",
    ],
    deps = [
        ":pos_util",
        "//build_tools:serialized_string_array_builder",
    ],
)

mozc_py_binary(
    name = "gen_pos_matcher_code",
    srcs = ["gen_pos_matcher_code.py"],
    visibility = [
        "//:__subpackages__",
        "//data_manager:__subpackages__",
    ],
    deps = [":pos_util"],
)

run_binary(
    name = "pos_matcher_impl",
    srcs = ["//data/rules:pos_matcher_rule.def"],
    outs = ["pos_matcher_impl.inc"],
    args = [
        "--pos_matcher_rule_file=" +
        "$(location //data/rules:pos_matcher_rule.def)",
        "--output_pos_matcher_h=$(location pos_matcher_impl.inc)",
    ],
    tool = ":gen_pos_matcher_code",
)

mozc_cc_library(
    name = "pos_matcher",
    hdrs = ["pos_matcher.h"],
    textual_hdrs = [":pos_matcher_impl"],
    visibility = ["//:__subpackages__"],
    deps = [
        "@com_google_absl//absl/base:core_headers",
        "@com_google_absl//absl/types:span",
    ],
)

mozc_py_binary(
    name = "gen_pos_rewrite_rule",
    srcs = ["gen_pos_rewrite_rule.py"],
    visibility = [
        "//:__subpackages__",
        "//data_manager:__subpackages__",
    ],
)

mozc_cc_binary(
    name = "gen_system_dictionary_data_main",
    srcs = ["gen_system_dictionary_data_main.cc"],
    visibility = [
        "//:__subpackages__",
        "//data_manager:__subpackages__",
    ],
    deps = [
        ":pos_matcher",
        ":text_dictionary_loader",
        "//base:file_stream",
        "//base:init_mozc_buildtool",
        "//data_manager",
        "//dictionary/system:system_dictionary_builder",
        "@com_google_absl//absl/flags:flag",
        "@com_google_absl//absl/log:check",
        "@com_google_absl//absl/strings",
    ],
)

mozc_cc_library(
    name = "dictionary_mock",
    testonly = True,
    hdrs = ["dictionary_mock.h"],
    visibility = [
        "//converter:__pkg__",
        "//dictionary:__subpackages__",
        "//engine:__pkg__",
        "//prediction:__pkg__",
        "//rewriter:__pkg__",
    ],
    deps = [
        ":dictionary_interface",
        ":dictionary_token",
        "//protocol:user_dictionary_storage_cc_proto",
        "//request:conversion_request",
        "//testing:gunit",
        "@com_google_absl//absl/strings:string_view",
    ],
)

mozc_py_binary(
    name = "gen_suffix_data",
    srcs = ["gen_suffix_data.py"],
    visibility = [
        "//:__subpackages__",
        "//data_manager:__subpackages__",
    ],
    deps = [
        "//build_tools:serialized_string_array_builder",
    ],
)

mozc_cc_test(
    name = "suffix_dictionary_test",
    size = "small",
    srcs = ["suffix_dictionary_test.cc"],
    deps = [
        ":dictionary_interface",
        ":dictionary_mock",
        ":dictionary_test_util",
        ":dictionary_token",
        ":suffix_dictionary",
        "//data_manager/testing:mock_data_manager",
        "//request:conversion_request",
        "//testing:gunit_main",
        "@com_google_absl//absl/strings",
        "@com_google_absl//absl/types:span",
    ],
)

mozc_cc_library(
    name = "dictionary_test_util",
    testonly = True,
    srcs = ["dictionary_test_util.cc"],
    hdrs = ["dictionary_test_util.h"],
    visibility = ["//dictionary:__subpackages__"],
    deps = [
        ":dictionary_interface",
        ":dictionary_token",
        "//testing",
        "@com_google_absl//absl/container:btree",
        "@com_google_absl//absl/container:flat_hash_map",
        "@com_google_absl//absl/strings",
        "@com_google_absl//absl/strings:str_format",
        "@com_google_absl//absl/types:span",
    ],
)

mozc_cc_library(
    name = "single_kanji_dictionary",
    srcs = ["single_kanji_dictionary.cc"],
    hdrs = ["single_kanji_dictionary.h"],
    visibility = [
        "//:__subpackages__",
        "//prediction:__pkg__",
    ],
    deps = [
        "//base:text_normalizer",
        "//base:util",
        "//base/container:serialized_string_array",
        "//data_manager",
        "//data_manager:serialized_dictionary",
        "@com_google_absl//absl/log:check",
        "@com_google_absl//absl/strings",
    ],
)

mozc_cc_test(
    name = "single_kanji_dictionary_test",
    size = "small",
    srcs = ["single_kanji_dictionary_test.cc"],
    deps = [
        ":single_kanji_dictionary",
        "//data_manager/testing:mock_data_manager",
        "//testing:gunit_main",
        "//testing:mozctest",
        "@com_google_absl//absl/strings:string_view",
    ],
)

mozc_py_binary(
    name = "gen_zip_code_seed",
    srcs = [
        "gen_zip_code_seed.py",
        "zip_code_util.py",
    ],
    visibility = ["//train/zip_code:__subpackages__"],
)

run_binary(
    name = "zip_code_data",
    srcs = mozc_select(
        default = [],
        oss = [
            "@zip_code_ken_all//:KEN_ALL.CSV",
            "@zip_code_jigyosyo//:JIGYOSYO.CSV",
        ],
        prod = [
        ],
    ),
    outs = ["zip_code.tsv"],
    args = [
        "--output",
        "$(location zip_code.tsv)",
    ] + mozc_select(
        oss = [
            "--zip_code",
            "$(location @zip_code_ken_all//:KEN_ALL.CSV)",
            "--jigyosyo",
            "$(location @zip_code_jigyosyo//:JIGYOSYO.CSV)",
        ],
        prod = [],
    ),
    tool = ":gen_zip_code_seed",
    visibility = [
        "//data:__subpackages__",
        "//dictionary:__subpackages__",
    ],
)
