/** @file
 * @brief Main module for omega (example CGI frontend for Xapian)
 */
/* Copyright 1999,2000,2001 BrightStation PLC
 * Copyright 2001 James Aylett
 * Copyright 2001,2002 Ananova Ltd
 * Copyright 2002-2024 Olly Betts
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License as
 * published by the Free Software Foundation; either version 2 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, see
 * <https://www.gnu.org/licenses/>.
 */

#include <config.h>

#include <cerrno>
#include <cstdio>
#include <ctime>

#include <algorithm>
#include <cstring>
#include <iostream>
#include <set>

#include "safefcntl.h"
#include "safeunistd.h"

#include "omega.h"
#include "utils.h"
#include "cgiparam.h"
#include "query.h"
#include "str.h"
#include "stringutils.h"
#include "expand.h"
#include "parseint.h"

using namespace std;

static const char DEFAULT_STEM_LANGUAGE[] = "english";

// A character which doesn't require URL encoding, and isn't likely to appear
// in filter values.
const char filter_sep = '~';

Xapian::Enquire * enquire;
Xapian::Database db;

map<string, string> option;

bool set_content_type = false;

bool suppress_http_headers = false;

string dbname;
string fmtname;
string filters, old_filters;

Xapian::docid hits_per_page = 0;
Xapian::docid min_hits = 0;

// percentage cut-off
int threshold = 0;

Xapian::MultiValueKeyMaker* sort_keymaker = NULL;
Xapian::valueno sort_key = Xapian::BAD_VALUENO; // Don't sort.
bool reverse_sort = true;
bool sort_after = false;
Xapian::Enquire::docid_order docid_order = Xapian::Enquire::ASCENDING;

Xapian::valueno collapse_key = 0;
bool collapse = false;

static string
map_dbname_to_dir(const string &database_name)
{
    return database_dir + database_name;
}

static void
add_database(const string& this_dbname)
{
    if (!dbname.empty()) dbname += '/';
    dbname += this_dbname;

    Xapian::Database this_db(map_dbname_to_dir(this_dbname));
    db.add_database(this_db);

    size_t this_db_size = this_db.size();
    size_t db_size = db.size();
    size_t i = 0;
    while (subdbs.size() != db_size) {
        subdbs.emplace_back(this_dbname, i++, this_db_size);
    }
}

// Get database(s) to search.
template<typename IT>
void
parse_db_params(const pair<IT, IT>& dbs)
{
    dbname.resize(0);
    // Only add a repeated db once.
    set<string> seen;
    for (auto i = dbs.first; i != dbs.second; ++i) {
        const string& v = i->second;
        if (v.empty()) continue;
        size_t p = 0, q;
        while (true) {
            q = v.find('/', p);
            string s(v, p, q - p);
            if (!s.empty() && seen.find(s) == seen.end()) {
                add_database(s);
                seen.insert(s);
            }
            if (q == string::npos) break;
            p = q + 1;
        }
    }
}

#define FILTER_CODE \
    "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_-"

template<typename T>
static void
filters_encode_uint(T v)
{
    do {
        if (v >= 64)
            filters += ' ';
        filters += FILTER_CODE[v & 63];
        v >>= 6;
    } while (v);
}

static void
filters_append(const string& bterm, const string* prev)
{
    auto reuse = prev ? common_prefix_length(*prev, bterm) : 0u;
    if (prev)
        filters_encode_uint(reuse);
    filters_encode_uint(bterm.size() - reuse);
    filters.append(bterm, reuse);

    auto e = bterm.find(filter_sep);
    if (usual(e == string::npos)) {
        old_filters += bterm;
    } else {
        // For old_filters, we don't try to reuse part of the previous term,
        // and if a filter contains filter_sep then we double it to escape.
        // Each filter must start with an alnum (checked before we get called)
        // and the value after the last filter is the default op, which is
        // encoded as a non-alnum so filter_sep followed by something other
        // than filter_sep must be separating filters.
        string::size_type b = 0;
        while (e != string::npos) {
            old_filters.append(bterm, b, e + 1 - b);
            b = e;
            e = bterm.find(filter_sep, b + 1);
        }
        old_filters.append(bterm, b, string::npos);
    }
    old_filters += filter_sep;
}

int main(int argc, char *argv[])
try {
    {
        // Check for SERVER_PROTOCOL=INCLUDED, which is set when we're being
        // included in a page via a server-side include directive.  In this
        // case we suppress sending a Content-Type: header.
        const char* p = getenv("SERVER_PROTOCOL");
        if (p && strcmp(p, "INCLUDED") == 0) {
            suppress_http_headers = true;
        }
    }

    read_config_file();

    option["flag_default"] = "true";

    // set default thousands and decimal separators: e.g. "16,729 hits" "1.4K"
    option["decimal"] = ".";
    option["thousand"] = ",";

    // set the default stemming language
    option["stemmer"] = DEFAULT_STEM_LANGUAGE;

    // FIXME: set cout to linebuffered not stdout.  Or just flush regularly...
    // setvbuf(stdout, NULL, _IOLBF, 0);

    const char * method = getenv("REQUEST_METHOD");
    if (method == NULL) {
        if (argc > 1 && (argv[1][0] != '-' || strchr(argv[1], '='))) {
            // omega 'P=information retrieval' DB=papers
            // check for a leading '-' on the first arg so "omega --version",
            // "omega --help", and similar take the next branch
            decode_argv(argv + 1);
        } else {
            // Seems we're running from the command line so give version
            // and allow a query to be entered for testing
            cout << PROGRAM_NAME " - " PACKAGE " " VERSION "\n";
            if (argc > 1) exit(0);
            cout << "Enter NAME=VALUE lines, end with blank line\n";
            decode_test();
        }
    } else {
        if (*method == 'P')
            decode_post();
        else
            decode_get();
    }

    try {
        parse_db_params(cgi_params.equal_range("DB"));
        if (dbname.empty()) {
            add_database(default_db);
        }
        enquire = new Xapian::Enquire(db);
    } catch (const Xapian::Error &) {
        enquire = NULL;
        db = Xapian::Database();
    }

    hits_per_page = 0;
    auto val = cgi_params.find("HITSPERPAGE");
    if (val != cgi_params.end()) {
        if (!parse_unsigned(val->second.c_str(), hits_per_page)) {
            throw "HITSPERPAGE parameter must be >= 0";
        }
    }
    if (hits_per_page == 0) {
        hits_per_page = 10;
    } else if (hits_per_page > 1000) {
        hits_per_page = 1000;
    }

    val = cgi_params.find("DEFAULTOP");
    if (val != cgi_params.end()) {
        const string & v = val->second;
        if (v == "OR" || v == "or")
            default_op = Xapian::Query::OP_OR;
    }

    val = cgi_params.find("FMT");
    if (val != cgi_params.end()) {
        const string & v = val->second;
        if (!v.empty()) fmtname = v;
    }
    if (fmtname.empty())
        fmtname = default_template;

    auto ml = cgi_params.equal_range("MORELIKE");
    if (enquire && ml.first != ml.second) {
        Xapian::RSet tmprset;
        for (auto i = ml.first; i != ml.second; ++i) {
            const string& v = i->second;
            Xapian::docid docid = atol(v.c_str());
            if (docid == 0) {
                // Assume it's MORELIKE=Quid1138 and that Quid1138 is a UID
                // from an external source - we just find the correspond docid.
                Xapian::PostingIterator p = db.postlist_begin(v);
                if (p != db.postlist_end(v)) docid = *p;
            }
            if (docid != 0) {
                tmprset.add_document(docid);
            }
        }

        if (!tmprset.empty()) {
            OmegaExpandDecider decider(db);
            set_expansion_scheme(*enquire, option);
            Xapian::ESet eset(enquire->get_eset(40, tmprset, &decider));
            string morelike_query;
            for (auto&& term : eset) {
                if (!morelike_query.empty()) {
                    if (default_op == Xapian::Query::OP_OR) {
                        morelike_query += ' ';
                    } else {
                        morelike_query += " OR ";
                    }
                }
                morelike_query += pretty_term(term);
            }
            add_query_string(string(), morelike_query);
        }
    } else {
        // add expand/topterms terms if appropriate
        string expand_terms;
        if (cgi_params.find("ADD") != cgi_params.end()) {
            auto g = cgi_params.equal_range("X");
            for (auto i = g.first; i != g.second; ++i) {
                const string & v = i->second;
                if (!v.empty()) {
                    if (!expand_terms.empty())
                        expand_terms += ' ';
                    expand_terms += v;
                }
            }
        }

        // collect the unprefixed prob fields
        auto g = cgi_params.equal_range("P");
        for (auto i = g.first; i != g.second; ++i) {
            const string & v = i->second;
            if (!v.empty()) {
                // If there are expand terms, append them to the first
                // non-empty P parameter.
                if (!expand_terms.empty()) {
                    string q = v;
                    q += ' ';
                    q += expand_terms;
                    add_query_string(string(), q);
                    expand_terms = string();
                } else {
                    add_query_string(string(), v);
                }
            }
        }

        if (!expand_terms.empty()) {
            add_query_string(string(), expand_terms);
        }
    }

    auto begin = cgi_params.lower_bound("P.");
    auto end = cgi_params.lower_bound("P/"); // '/' is '.' + 1.
    for (auto i = begin; i != end; ++i) {
        const string & v = i->second;
        if (!v.empty()) {
            string pfx(i->first, 2, string::npos);
            add_query_string(pfx, v);
        }
    }

    // set any boolean filters
    auto g = cgi_params.equal_range("B");
    if (g.first != g.second) {
        vector<string> filter_v;
        for (auto i = g.first; i != g.second; ++i) {
            const string & v = i->second;
            // we'll definitely get empty B fields from "-ALL-" options
            if (!v.empty() && C_isalnum(v[0])) {
                add_bterm(v);
                filter_v.push_back(v);
            }
        }
        sort(filter_v.begin(), filter_v.end());
        const string* prev = NULL;
        for (const string& bterm : filter_v) {
            filters_append(bterm, prev);
            prev = &bterm;
        }
    }

    // Current filters format:
    //
    // [<encoded length><boolean filter term>]*
    // ['!'[<encoded length><negated boolean filter term>]*]?
    // ['.'<collapse key>]?
    // ['$'<encoded date range slot (omitted for term-based)>?
    //     ['!'<date start>]?
    //     ['.'<date end>]?
    //     ['~'<date span>]?
    // [['-'?<sort key>[['-'|'+']<sort key>]+]|<sort key>|]?
    // <encoded integer of default_op, docid_order, sort_after, sort_reverse>
    //
    // (filter terms in ascending byte sorted order, and with second and
    // subsequent actually stored as <reuse character><tail>)
    //
    // old_filters format:
    //
    // [<boolean filter term with any '~' escaped to '~~'>'~']*
    // ['!'<negated boolean filter term with any '~' escaped to '~~'>'~']*
    // ['$'<date range slot>'$'<date start>'$'<date end>'$'<date span>]*
    // ['.'|'-']   ; default_op AND vs OR
    // <date start>'~'<date end>'~'<date span>['~'<date value slot>]?
    // ['~'<collapse key>]?   ; present if <collapse key> non-empty or
    //                        ; previous element present
    // ['D'|'X']? ; 'D' for docid_order DESCENDING; 'X' for DONT_CARE.
    // [['-'?<sort key>[['-'|'+']<sort key>]+]|<sort key>]? ['R'|'F'|'f']?
    //
    // (filter terms in ascending byte sorted order)

    // set any negated boolean filters
    g = cgi_params.equal_range("N");
    if (g.first != g.second) {
        vector<string> filter_v;
        for (auto i = g.first; i != g.second; ++i) {
            const string & v = i->second;
            // we'll definitely get empty N fields from "-ALL-" options
            if (!v.empty() && C_isalnum(v[0])) {
                add_nterm(v);
                filter_v.push_back(v);
            }
        }
        if (!filter_v.empty()) {
            filters += '!';
            sort(filter_v.begin(), filter_v.end());
            const string* prev = NULL;
            for (const string& nterm : filter_v) {
                old_filters += '!';
                filters_append(nterm, prev);
                prev = &nterm;
            }
        }
    }

    // collapsing
    val = cgi_params.find("COLLAPSE");
    if (val != cgi_params.end()) {
        const string& v = val->second;
        if (!v.empty()) {
            if (!parse_unsigned(val->second.c_str(), collapse_key)) {
                throw "COLLAPSE parameter must be >= 0";
            }
            collapse = true;
            filters += '.';
            filters_encode_uint(collapse_key);
        }
    }

    // date range filters
    struct date_range {
        string start, end, span;
    };
    map<Xapian::valueno, date_range> date_ranges;
    begin = cgi_params.lower_bound("START.");
    end = cgi_params.lower_bound("START/"); // '/' is '.' + 1.
    for (auto i = begin; i != end; ++i) {
        const string & v = i->second;
        if (!v.empty()) {
            Xapian::valueno slot;
            if (!parse_unsigned(i->first.c_str() +
                                CONST_STRLEN("START."), slot)) {
                throw "START slot value must be >= 0";
            }
            date_ranges[slot].start = v;
        }
    }
    begin = cgi_params.lower_bound("END.");
    end = cgi_params.lower_bound("END/"); // '/' is '.' + 1.
    for (auto i = begin; i != end; ++i) {
        const string & v = i->second;
        if (!v.empty()) {
            Xapian::valueno slot;
            if (!parse_unsigned(i->first.c_str() +
                                CONST_STRLEN("END."), slot)) {
                throw "END slot value must be >= 0";
            }
            date_ranges[slot].end = v;
        }
    }
    begin = cgi_params.lower_bound("SPAN.");
    end = cgi_params.lower_bound("SPAN/"); // '/' is '.' + 1.
    for (auto i = begin; i != end; ++i) {
        const string & v = i->second;
        if (!v.empty()) {
            Xapian::valueno slot;
            if (!parse_unsigned(i->first.c_str() +
                                CONST_STRLEN("SPAN."), slot)) {
                throw "SPAN slot value must be >= 0";
            }
            date_ranges[slot].span = v;
        }
    }

    string date_start, date_end, date_span;
    val = cgi_params.find("START");
    if (val != cgi_params.end()) {
        date_start = val->second;
    }
    val = cgi_params.find("END");
    if (val != cgi_params.end()) {
        date_end = val->second;
    }
    val = cgi_params.find("SPAN");
    if (val != cgi_params.end()) {
        date_span = val->second;
    }
    val = cgi_params.find("DATEVALUE");
    Xapian::valueno date_value_slot = Xapian::BAD_VALUENO;
    if (val != cgi_params.end() &&
        !parse_unsigned(val->second.c_str(), date_value_slot)) {
        throw "DATEVALUE slot must be >= 0";
    }
    if (date_value_slot != Xapian::BAD_VALUENO ||
        !date_start.empty() ||
        !date_end.empty() ||
        !date_span.empty()) {
        // Process DATEVALUE=n and associated values unless we saw START.n=...
        // or END.n=... or SPAN.n=...
        date_ranges.emplace(date_value_slot,
                            date_range{date_start, date_end, date_span});
    }
    for (auto i : date_ranges) {
        auto slot = i.first;
        auto r = i.second;
        add_date_filter(r.start, r.end, r.span, slot);
        filters += '$';
        if (slot != Xapian::BAD_VALUENO) {
            filters_encode_uint(slot);
            if (slot != date_value_slot) {
                old_filters += '$';
                old_filters += str(slot);
                old_filters += '$';
                old_filters += r.start;
                old_filters += '$';
                old_filters += r.end;
                old_filters += '$';
                old_filters += r.span;
            }
        }
        if (!r.start.empty()) {
            filters += '!';
            filters += r.start;
        }
        if (!r.end.empty()) {
            filters += '.';
            filters += r.end;
        }
        if (!r.span.empty()) {
            filters += '~';
            filters += r.span;
        }
    }

    old_filters += (default_op == Xapian::Query::OP_AND ? '.' : '-');
    old_filters += date_start;
    old_filters += filter_sep;
    old_filters += date_end;
    old_filters += filter_sep;
    old_filters += date_span;
    if (date_value_slot != Xapian::BAD_VALUENO) {
        // This means we'll force the first page when reloading or changing
        // page starting from existing URLs upon upgrade to 1.4.1, but the
        // exact same existing URL could be for a search without the date
        // filter where we want to force the first page, so there's an inherent
        // ambiguity there.  Forcing first page in this case seems the least
        // problematic side-effect.
        old_filters += filter_sep;
        old_filters += str(date_value_slot);
    }

    // Percentage relevance cut-off
    val = cgi_params.find("THRESHOLD");
    if (val != cgi_params.end()) {
        unsigned int temp;
        if (val->second[0] == '-') {
            if (!parse_unsigned(val->second.c_str() + 1, temp)) {
                throw "THRESHOLD parameter must be an integer";
            }
            threshold = 0;
        } else if (!parse_unsigned(val->second.c_str(), temp)) {
            throw "THRESHOLD parameter must be an integer";
        }
        if (temp > 100) {
            threshold = 100;
        } else {
            threshold = temp;
        }
    }

    if (collapse) {
        old_filters += filter_sep;
        old_filters += str(collapse_key);
    } else if (date_value_slot != Xapian::BAD_VALUENO) {
        // We need to either omit filter_sep for both or neither, or else the
        // encoding is ambiguous.
        old_filters += filter_sep;
    }

    // docid order
    val = cgi_params.find("DOCIDORDER");
    if (val != cgi_params.end()) {
        const string & v = val->second;
        if (!v.empty()) {
            char ch = v[0];
            if (ch == 'D') {
                docid_order = Xapian::Enquire::DESCENDING;
                old_filters += 'D';
            } else if (ch != 'A') {
                docid_order = Xapian::Enquire::DONT_CARE;
            } else {
                // This was a bug in the 1.4.x filter encoding (we should have
                // added nothing here and 'X' in the `ch != 'A'` case), but the
                // current "DONT_CARE" implementation actually always results
                // in ascending docid order so it wasn't worth breaking
                // compatibility in a stable release series to fix.
                old_filters += 'X';
            }
        }
    }

    // sorting
    val = cgi_params.find("SORT");
    if (val != cgi_params.end()) {
        const char * base = val->second.c_str();
        const char * p = base;
        do {
            bool rev = (*p != '+');
            if (*p == '-' || *p == '+') {
                ++p;
            }
            if (!C_isdigit(*p)) {
                // Invalid.
                break;
            }
            errno = 0;
            char * q;
            Xapian::valueno slot = strtoul(p, &q, 10);
            p = q;
            if (errno != 0) {
                // Invalid.
                break;
            }

            if (sort_key != Xapian::BAD_VALUENO) {
                // Multiple sort keys specified, so we need a KeyMaker.

                // Omit leading '+'.
                if (reverse_sort) {
                    filters += '-';
                    old_filters += '-';
                }
                filters_encode_uint(sort_key);
                old_filters += str(sort_key);

                sort_keymaker = new Xapian::MultiValueKeyMaker;
                sort_keymaker->add_value(sort_key, !reverse_sort);
                sort_key = Xapian::BAD_VALUENO;
                reverse_sort = true;
            }

            if (sort_keymaker) {
                filters += (rev ? '-' : '+');
                old_filters += (rev ? '-' : '+');
                filters_encode_uint(slot);
                old_filters += str(slot);
                sort_keymaker->add_value(slot, !rev);
            } else {
                sort_key = slot;
                reverse_sort = rev;
            }
            while (C_isspace(*p) || *p == ',') ++p;
        } while (*p);

        val = cgi_params.find("SORTREVERSE");
        if (val != cgi_params.end()) {
            unsigned int temp;
            if (!parse_unsigned(val->second.c_str(), temp)) {
                throw "SORTREVERSE parameter must be >= 0";
            }
            if (temp != 0) {
                reverse_sort = !reverse_sort;
            }
        }
        val = cgi_params.find("SORTAFTER");
        if (val != cgi_params.end()) {
            unsigned int temp;
            if (!parse_unsigned(val->second.c_str(), temp)) {
                throw "SORTAFTER parameter must be >= 0";
            }
            sort_after = bool(temp);
        }

        // Add the sorting related options to filters too.
        if (!sort_keymaker) {
            filters_encode_uint(sort_key);
            old_filters += str(sort_key);
        }
        if (sort_after) {
            if (reverse_sort) {
                old_filters += 'R';
            } else {
                old_filters += 'F';
            }
        } else {
            if (!reverse_sort) {
                old_filters += 'f';
            }
        }
    }

    {
        // Encode default_op, docid_order, reverse_sort and sort_after together
        // in a single character.
        unsigned v = 0;
        switch (default_op) {
          case Xapian::Query::OP_AND:
            break;
          case Xapian::Query::OP_OR:
            v = 0x01 * 12;
            break;
          default:
            // Additional supported value should encode as:
            // 0x02 * 12
            // 0x03 * 12
            // ...
            break;
        }
        v |= 0x04 * static_cast<unsigned>(docid_order);
        if (reverse_sort) v |= 0x01;
        if (sort_after) v |= 0x02;
        filters_encode_uint(v);
    }

    // min_hits (fill mset past topdoc+(hits_per_page+1) to
    // topdoc+max(hits_per_page+1,min_hits)
    val = cgi_params.find("MINHITS");
    if (val != cgi_params.end()) {
        if (!parse_unsigned(val->second.c_str(), min_hits)) {
            throw "MINHITS parameter must be >= 0";
        }
    }

    parse_omegascript();
} catch (const Xapian::Error &e) {
    if (!set_content_type && !suppress_http_headers)
        cout << "Content-Type: text/html\n\n";
    cout << "Exception: " << html_escape(e.get_description()) << endl;
} catch (const std::exception &e) {
    if (!set_content_type && !suppress_http_headers)
        cout << "Content-Type: text/html\n\n";
    cout << "Exception: std::exception " << html_escape(e.what()) << endl;
} catch (const string &s) {
    if (!set_content_type && !suppress_http_headers)
        cout << "Content-Type: text/html\n\n";
    cout << "Exception: " << html_escape(s) << endl;
} catch (const char *s) {
    if (!set_content_type && !suppress_http_headers)
        cout << "Content-Type: text/html\n\n";
    cout << "Exception: " << html_escape(s) << endl;
} catch (...) {
    if (!set_content_type && !suppress_http_headers)
        cout << "Content-Type: text/html\n\n";
    cout << "Caught unknown exception" << endl;
}
