/** @file
 * @brief Auxiliary program of omindextest
 */
/* Copyright (C) 2019 Bruno Baruffaldi
 * Copyright (C) 2020 Parth Kapadia
 * Copyright (C) 2021,2022,2023,2025,2026 Olly Betts
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License as
 * published by the Free Software Foundation; either version 2 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, see
 * <https://www.gnu.org/licenses/>.
 */

#include <config.h>

#include <xapian.h>

#include <algorithm>
#include <iomanip>
#include <iostream>
#include <string>
#include <unordered_map>
#include <vector>

#include "values.h"

using namespace std;

enum test_result { PASS, FAIL };

// Macro to mark optional terms.
//
// If there are optional terms in a testcase, either all or none must be
// present.  This is useful with libextractor where the required plugin
// may not be installed, but libextractor will still return generic metadata.
//
// This works by appending '\xff', which we remove before comparing.  This
// means terms ending with this byte can't be used in testcases, but it's
// invalid in UTF-8 so probably not a problematic limitation.
#define OPT(T) (T "\xff")

// Macro to mark terms which should NOT be indexed.
//
// This works by appending '\xfe', which we remove before comparing.  This
// means terms ending with this byte can't be used in testcases, but it's
// invalid in UTF-8 so probably not a problematic limitation.
#define NOT(T) (T "\xfe")

struct testcase {
    vector<string> terms;

    vector<pair<Xapian::valueno, string>> values;

    testcase(vector<string> v)
        : terms(std::move(v)) {}

    testcase(vector<string> v, vector<pair<Xapian::valueno, string>> v2)
        : terms(std::move(v)), values(std::move(v2)) {}
};


static unordered_map<string, testcase> tests;

static void
index_test()
{
    tests.insert({"plaintext/iso88591.txt",
                  {{"à", "d'après", "françois", "idée", "réalisation"}}});
    tests.insert({"plaintext/utf16be-bom.txt",
                  {{"Zjoey", "Zfood", "Zедой", "Z喬伊不分享食物"}}});
    tests.insert({"plaintext/utf16le-bom.txt",
                  {{"Zjoey", "Zfood", "Zедой", "Z喬伊不分享食物"}}});
    tests.insert({"plaintext/utf8.txt",
                  {{"Zjoey", "Zfood", "Zедой", "Z喬伊不分享食物"}}});
    tests.insert({"plaintext/utf8-bom.txt",
                  {{"Zjoey", "Zfood", "Zедой", "Z喬伊不分享食物"}}});
    tests.insert({"test-csv.csv",
                  {{"ZFcsv", "Zbreak", "Zwere"}}});
    tests.insert({"test-html.html",
                  {{"Ajeroen", "ZAoom", "ZSworld", "Shello", "Zchapter"}}});
    tests.insert({"svg/dctitle.svg",
                  {{"Sreal", "Stitle", NOT("Signored"),
                    "Timage/svg+xml", "Zstart"}}});
    tests.insert({"svg/diagram.svg",
                  {{"Sdiagram", "Timage/svg+xml", "Zstart"}}});
    tests.insert({"svg/diagram.svgz",
                  {{"Sdiagram", "Timage/svg+xml-compressed", "Zstart"}}});
#ifdef HAVE_GMIME
    tests.insert({"email/html.eml",
                  {{"Aexample", "Ame", "Aorg", "Auser", "Shtml",
                    "Tmessage/rfc822",
                    "XMID:E1p1II7-008OVw-1w@example.org",
                    "XTOada", "XTOexample", "XTOorg", "XTOuser",
                    "html", "message", "test"},
                  {{{VALUE_CREATED, int_to_binary_string(1670034611)},
                    {VALUE_SIZE, Xapian::sortable_serialise(450)},
                    {VALUE_MD5, "y.<0RW\xb0\xf4\xd2+\xa8\x09\xde\xff|\x0d"}
                   }}}});
    tests.insert({"email/text.eml",
                  {{"Aexample", "Ame", "Aorg", "Auser", "Stext", "Tmessage/rfc822",
                    "XMID:E1p1II7-008OVw-1v@example.org",
                    "XTOexample", "XTOorg", "XTOuser",
                    "comment1", "comment2", "keyword1", "keyword2",
                    "message", "plain", "text"},
                  {{{VALUE_CREATED, int_to_binary_string(1670034611)},
                    {VALUE_SIZE, Xapian::sortable_serialise(477)},
                    {VALUE_MD5,
                     "C\x7f\x17;;\x87\x91\x5c\x05?\x83\x14\xec\xaa\xad\x94"}
                   }}}});
#endif
#if defined HAVE_POPPLER
    tests.insert({"pdf/poppler.pdf",
                  {{"ZFpoppler", "Zsub", "Ztext", "Ztitl", "Zpie"}}});
# if defined HAVE_LIBSPECTRE
    // Same testcase works for external tools ps2pdf+pdftotext and the
    // libspectre+poppler worker, but we don't have a good way to handle ps2pdf
    // not being available so we currently only enable this testcase for the
    // worker.
    tests.insert({"postscript/small.ps",
                  {{"ZAolli", "ZFsmall", "ZSdocument", "ZStitl",
                    "Zdocument", "Zpostscript"}}});
# endif
#endif
#if defined HAVE_LIBEBOOK
    tests.insert({"fb2/hello.fb2",
                  {{"Ajeroen", "Aooms", "Zbeauti", "Zchapter", "Zdocument",
                    "Zooms3", "Zsubsect", "Ztoday", "ZFhello", "Zoutlin"}}});
    tests.insert({"fb2/lang-name.fb2",
                  {{"Adavid", "Atardon", "ZAdavid", "ZFlang", "Zeel",
                    "Zhovercraft", "Zlanguag"}}});
    tests.insert({"fb2/lang2.fb2",
                  {{"Adavid", "Atardon", "ZAtardon", "Zúhořů", "Zhovercraft",
                    "Zlanguag", "Zmój", "Zof", "Zpełen", "Zwęgorzi",
                    "Zpoduszkowiec", "Zvznášedlo"}}});
    tests.insert({"lrf/hello.lrf",
                  {{"ZFhello", "Zhello", "Zworld"}}});
    tests.insert({"pdb/PalmDOC-hello.pdb",
                  {{"Shello", "Sworld", "ZFpalmdoc", "Zworld"}}});
    tests.insert({"pdb/PeanutPress-hello.pdb",
                  {{"Fpeanutpress", "ZFhello", "Zhello"}}});
    tests.insert({"pdb/test.pdb",
                  {{"Sdemodemo", "ZFtest", "Zwherearew"}}});
#endif
#if defined HAVE_LIBETONYEK
    tests.insert({"iwork/test-keynote.key",
                  {{"ZFkeynot", "Zbold", "Znow", "Zsubtitl"}}});
    tests.insert({"iwork/test-pages.pages",
                  {{"ZFpage", "Zfull", "Zhovercraft", "Zwęgorzi"}}});
#endif
#if defined HAVE_TESSERACT
    tests.insert({"image/Test1.gif",
                  {{"Znoisyimag", "Zocr", "Ztesseract"}}});
    tests.insert({"image/Test2.pgm",
                  {{"ZFtest2", "Znoisyimag", "Ztesseract"}}});
    tests.insert({"image/Test3.ppm",
                  {{"ZFtest3", "Zocr", "Ztest"}}});
    tests.insert({"image/Test4.tiff",
                  {{"Znoisyimag", "Ztesseract", "Zto"}}});
    tests.insert({"image/Test5.webp",
                  {{"Znoisyimag", "Ztesseract", "Ztest"}}});
    tests.insert({"image/poster-2.jpg",
                  {{"ZFposter", "Zby", "Zproperti", "Zsurveil", "Zvideo"}}});
    tests.insert({"image/poster.jpg",
                  {{"Zbicycl", "Zride", "Zroller", "Zskateboard"}}});
    tests.insert({"image/scan-page.png",
                  {{"Zannual", "Zfed", "Zreturn", "Zwhile"}}});
#endif
#define OFFICE_TESTCASES(PREFIX) \
    /* blank file */ \
    /* pass the test if no terms are found */ \
    tests.insert({PREFIX "opendoc/blank.odt", \
                  {{}}}); \
    /* corrupted file (ODP) */ \
    /* tests.insert({PREFIX "corrupt_file.odp", {"ZSnatur"}}); */ \
    \
    /* ODF */ \
    tests.insert({PREFIX "opendoc/test.odt", \
                  {{"Zедой", "Z喬伊不分享食物"}}}); \
    tests.insert({PREFIX "opendoc/text.odt", \
                  {{"Stesttitle", NOTLO("Aolly",) "Zsampl", "Zhead", "Ztext", \
                    "Zhello", "Zworld"}}}); \
    tests.insert({PREFIX "opendoc/text_template.ott", \
                  {{"Zjane", "Zdoe", "Zstructur"}}}); \
    NOTLO(\
    tests.insert({PREFIX "opendoc/presentation.odp", \
                  {{"Zfascin", "Zfact", "Zpustak", "Zmahal", "Zmillion", \
                    "Zpeopl", "Zbirthday", "501"}}}); \
    )\
    tests.insert({PREFIX "opendoc/presentation_template.otp", \
                  {{"ZSalizarin", "Zhead", "Zworld", "Ztext"}}}); \
    tests.insert({PREFIX "opendoc/spreadsheet.ods", \
                  {{"Zhello", "Zworld", "Zsampl", "2"}}}); \
    tests.insert({PREFIX "opendoc/spreadsheet_template.ots", \
                  {{"Zfood", "Zpasta", "Zpercentag", "40"}}}); \
    tests.insert({PREFIX "opendoc/draw.odg", \
                  {{"Zparth", "Zkapadia"}}}); \
    \
    /* Apache OpenOffice */ \
    tests.insert({PREFIX "staroffice/calc.sxc", \
                  {{"Ztoy", "Zproduct", "Zcost", "Zquantiti", "Zcardboard"}}}); \
    tests.insert({PREFIX "staroffice/calc_template.stc", \
                  {{NOTLO("ZSpurchas", "ZStemplat",) "Zproduct", "Zquantiti", \
                    "Zsampl"}}}); \
    tests.insert({PREFIX "staroffice/text.sxw", \
                  {{"Zhello", "Zsampl", "Zopenoffic", "Zwriter"}}}); \
    tests.insert({PREFIX "staroffice/text_template.stw", \
                  {{"Zhello", "Zworld", "Zsampl", "Zhead", \
                    NOTLO("ZStemplat", "ZStext")}}}); \
    tests.insert({PREFIX "staroffice/presentation.sxi", \
                  {{"Zhead", "Zhello", "Zopenoffic", "Zimpress"}}}); \
    tests.insert({PREFIX "staroffice/presentation_template.sti", \
                  {{NOTLO("ZSproject", "ZSresearch",) "Zhead", "Ztext"}}}); \
    \
    /* Microsoft XML formats */ \
    tests.insert({PREFIX "msxml/Book.xlsx", \
                  {{"Zmodi", "Zgood", "Zemploye"}}}); \
    tests.insert({PREFIX "msxml/2sheets.xlsx", \
                  {{NOTLO("0.123456",) LO("0.12346",) \
                    "123.456", "15", "2021", \
                    NOTLO("3.14159265358979",) LO("3.14159",) \
                    "43", "55", "Aolly", "Ssheet", "Stitle", "xmas"}}}); \
    tests.insert({PREFIX "msxml/Doc.docx", \
                  {{"Zедой", "Z喬伊不分享食物", "ZSbakeri"}}}); \
    tests.insert({PREFIX "msxml/Nature.pptx", \
                  {{"ZSnatur", "Zbeauti", "Zsampl"}}}); \
    tests.insert({PREFIX "msxml/vnd.ms-xpsdocument_xpstest.xps", \
                 {{"second", "header", "footer"}}});
#if defined HAVE_LIBARCHIVE
# define LO(...)
# define NOTLO(...) __VA_ARGS__
    OFFICE_TESTCASES("")
# undef NOTLO
# undef LO
#endif
#if defined HAVE_LIBREOFFICEKIT_LIBREOFFICEKIT_HXX
# define LO(...) __VA_ARGS__
# define NOTLO(...)
    OFFICE_TESTCASES("lok-")
# undef NOTLO
# undef LO
#endif
    tests.insert({"abw/test.abw",
                  {{"Sabiword", "Stitle", "ZAparth",
                    "Zabiword", "Zsampl", "Zdocument"}
#ifndef HAVE_LIBABW // libabw doesn't currently extract created date.
                  , {{{VALUE_CREATED, int_to_binary_string(1595794074)}}}
#endif
                 }});
    tests.insert({"abw/macbeth.zabw",
                  {{"Ashakespeare", "Awilliam", "Smacbeth",
                    "ambition", "macduff", "shall"}
#ifndef HAVE_LIBABW // libabw doesn't currently extract created date.
                  , {{{VALUE_CREATED, int_to_binary_string(1641662439)}}}
#endif
                 }});
    tests.insert({"abw/test1.abw",
                  {{"Zедой", "Z喬伊不分享食物",
                    NOT("aaauiwaalimbeku")}}});
    tests.insert({"abw/Friendly-Letter.awt",
                  {{"address", "addressee", "body", "dear", "sincerely"}}});
    tests.insert({"gnumeric/invoice.gnumeric",
                  {{"100", "21", "3.5", "Sgnumeric", "Stitle",
                    "emboldened", "invoice", "sheetname", "test"},
                  {{{VALUE_CREATED, int_to_binary_string(1743446406)}}}
                 }});
#if defined HAVE_LIBCDR
    // .cdr versions >= 16 are not included in the tests as they will work
    // correctly only with libcdr >= 0.1.6
    tests.insert({"cdr/test1.cdr",
                  {{"Zalgerian", "Zcalibri"}}});
    tests.insert({"cdr/test2.cdr",
                  {{"Zедой", "Z喬伊不分享食物", "Zdocument"}}});
#endif
#if defined HAVE_LIBEXTRACTOR
    // Testcase for libextractor need to allow for the required plugin not
    // being installed as libextractor still returns generic metadata.
    tests.insert({"video/file_example_OGG_480_1_7mg.ogg",
                  {{"Eogg", "Tvideo/ogg",
                    OPT("Searth"), OPT("Splanet")}}});
    tests.insert({"video/file_example_AVI_480_750kB.avi",
                  {{"Eavi", "Tvideo/x-msvideo",
                    OPT("Zcodec"), OPT("Zh264"),
                    OPT("480x270"), OPT("msvideo"), OPT("30"), OPT("fps")}}});
    tests.insert({"audio/file_example_OOG_1MG.ogg",
                  {{"Eogg", "Taudio/ogg",
                    OPT("Akevin"), OPT("Amacleod"),
                    OPT("Simpact"), OPT("ZSmoderato")}}});
    tests.insert({"audio/file_example_WAV_1MG.wav",
                  {{"Ewav", "Taudio/x-wav",
                    OPT("Zstereo"), OPT("wav"), OPT("Zms")}}});
#endif
#if defined HAVE_LIBGEPUB
    tests.insert({"epub/epub2test.epub",
                  {{"Eepub", "Tapplication/epub+zip", "Aolly",
                    "book", "chapter", "welcome"}}});
    tests.insert({"epub/epub3test.epub",
                  {{"Eepub", "Tapplication/epub+zip", "Aolly",
                    "book", "chapter", "welcome"}}});
#endif
#if defined HAVE_LIBMWAW
    tests.insert({"apple_works/test_word.cwk",
                  {{"Aparth", "Sword", "Zhello", "Zdocument"}}});
    tests.insert({"apple_works/test_spreadsheet.cwk",
                  {{"Aparth", "Sspreadsheet", "Zpizza", "220"}}});
    tests.insert({"apple_works/test_draw.cwk",
                  {{"Zdraw", "Zsampl", "Zgraphic"}}});
#endif
}

static void
escape(const string& s, std::ostream& stream)
{
    for (unsigned char ch : s) {
        if (ch >= 0x20 && ch < 127 && ch != '\\') {
            stream << ch;
        } else {
            stream << "\\x"
                   << std::hex << std::setfill('0') << std::setw(2) << int(ch);
        }
    }
}

static test_result
compare_test(testcase& test, const Xapian::Document& doc, const string& file)
{
    sort(test.terms.begin(), test.terms.end());
    Xapian::TermIterator term_iterator = doc.termlist_begin();
    bool all_required_terms_exist = true;
    string missing_optional;
    bool no_optional = true;
    for (auto& i : test.terms) {
        if (i.back() == '\xff') {
            string t(i, 0, i.size() - 1);
            term_iterator.skip_to(t);
            // Optional term.
            if (term_iterator == doc.termlist_end() || *term_iterator != t) {
                missing_optional += ' ';
                missing_optional += t;
            } else {
                no_optional = false;
            }
        } else if (i.back() == '\xfe') {
            string t(i, 0, i.size() - 1);
            term_iterator.skip_to(t);
            // Term which should not be indexed.
            if (term_iterator != doc.termlist_end() && *term_iterator == t) {
                cerr << file << ": error: Term " << t
                     << " indexes this file but shouldn't\n";
                all_required_terms_exist = false;
            }
        } else {
            auto t = i;
            term_iterator.skip_to(t);
            if (term_iterator == doc.termlist_end() || *term_iterator != t) {
                cerr << file << ": error: Term " << t
                     << " should index this file but doesn't\n";
                all_required_terms_exist = false;
            }
        }
    }

    bool values_ok = true;
    for (auto& i : test.values) {
        const string& v = doc.get_value(i.first);
        if (v != i.second) {
            cerr << file << ": error: Value slot " << i.first << " should be ";
            escape(i.second, cerr);
            cerr << " not ";
            escape(v, cerr);
            cerr << '\n';
            values_ok = false;
        }
    }
    if (!values_ok) {
        return FAIL;
    }
    if (!missing_optional.empty() && !no_optional) {
        cerr << file << ": error: Only some of the optional terms index this "
                        "file, missing:" << missing_optional << '\n';
    } else if (all_required_terms_exist) {
        // All terms found (including degenerate case where no terms are listed
        // to check for).
        return PASS;
    }
    cerr << "Expected at least these terms:";
    for (auto& t : test.terms) {
        if (t.back() == '\xff') {
            // Optional term.
            cerr << " OPT(" << t.substr(0, t.size() - 1) << ')';
        } else if (t.back() == '\xfe') {
            // Optional term.
            cerr << " NOT(" << t.substr(0, t.size() - 1) << ')';
        } else {
            cerr << ' ' << t;
        }
    }
    cerr << "\nFull list of terms actually present:";
    for (term_iterator = doc.termlist_begin();
         term_iterator != doc.termlist_end();
         ++term_iterator) {
        cerr << ' ' << *term_iterator;
    }
    cerr << '\n';
    return FAIL;
}

int
main(int argc, char** argv)
{
    test_result result = PASS;
    if (argc <= 1)
        return 1;

    Xapian::Database db(argv[1]);

    index_test();
    for (auto t = db.allterms_begin("U"); t != db.allterms_end("U"); ++t) {
        const string& term = *t;
        string url(term, 2);
        Xapian::PostingIterator p = db.postlist_begin(term);
        if (p == db.postlist_end(term)) {
            // This shouldn't be possible.
            cerr << "Term " << term << " doesn't index anything?!\n";
            result = FAIL;
            continue;
        }
        Xapian::docid did = *p;
        Xapian::Document doc = db.get_document(did);
        auto iter = tests.find(url);
        if (iter != tests.end()) {
            test_result individual_result = compare_test(iter->second, doc,
                                                         url);
            if (individual_result == FAIL)
                result = FAIL;
            tests.erase(iter);
        }
        if (++p != db.postlist_end(term)) {
            cerr << "URL term " << term << " indexes more than one document\n";
            result = FAIL;
        }
    }

    for (auto t : tests) {
        cerr << "Testcase for URL " << t.first << " wasn't exercised\n";
        result = FAIL;
    }

    return result == FAIL ? 1 : 0;
}
