Skip to content

Revamp QueryParser #2310

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jhy merged 2 commits into from
Apr 24, 2025
Merged

Revamp QueryParser #2310

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3da797c
Revamp QueryParser
jhy Apr 24, 2025
24db204
Implement `:has()` and `:is()` with direct descent
jhy Apr 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions src/main/java/org/jsoup/parser/TokenQueue.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,11 @@ public boolean matches(String seq) {
return reader.matchesIgnoreCase(seq);
}

/** Tests if the next character on the queue matches the character, case-sensitively. */
public boolean matches(char c) {
return reader.matches(c);
}

/**
@deprecated will be removed in 1.21.1.
*/
Expand Down Expand Up @@ -98,6 +103,15 @@ public boolean matchChomp(String seq) {
return reader.matchConsumeIgnoreCase(seq);
}

/** If the queue matches the supplied (case-sensitive) character, consume it off the queue. */
public boolean matchChomp(char c) {
if (reader.matches(c)) {
consume();
return true;
}
return false;
}

/**
Tests if queue starts with a whitespace character.
@return if starts with whitespace
Expand Down
19 changes: 5 additions & 14 deletions src/main/java/org/jsoup/select/CombiningEvaluator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,11 @@ public abstract class CombiningEvaluator extends Evaluator {
updateEvaluators();
}

public void add(Evaluator e) {
evaluators.add(e);
updateEvaluators();
}

@Override protected void reset() {
for (Evaluator evaluator : evaluators) {
evaluator.reset();
Expand All @@ -42,15 +47,6 @@ public abstract class CombiningEvaluator extends Evaluator {
return cost;
}

@Nullable Evaluator rightMostEvaluator() {
return num > 0 ? evaluators.get(num - 1) : null;
}

void replaceRightMostEvaluator(Evaluator replacement) {
evaluators.set(num - 1, replacement);
updateEvaluators();
}

void updateEvaluators() {
// used so we don't need to bash on size() for every match test
num = evaluators.size();
Expand Down Expand Up @@ -110,11 +106,6 @@ public Or(Collection<Evaluator> evaluators) {
super();
}

public void add(Evaluator e) {
evaluators.add(e);
updateEvaluators();
}

@Override
public boolean matches(Element root, Element node) {
for (int i = 0; i < num; i++) {
Expand Down
238 changes: 118 additions & 120 deletions src/main/java/org/jsoup/select/QueryParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@
import org.jsoup.internal.StringUtil;
import org.jsoup.helper.Validate;
import org.jsoup.parser.TokenQueue;
import org.jspecify.annotations.Nullable;

import java.util.ArrayList;
import java.util.List;
import java.util.function.Function;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

Expand All @@ -16,12 +16,12 @@
* Parses a CSS selector into an Evaluator tree.
*/
public class QueryParser {
private final static char[] Combinators = {',', '>', '+', '~', ' '};
private final static char[] Combinators = {'>', '+', '~'}; // ' ' is also a combinator, but found implicitly
private final static String[] AttributeEvals = new String[]{"=", "!=", "^=", "$=", "*=", "~="};
private final static char[] SequenceEnders = {',', ')'};

private final TokenQueue tq;
private final String query;
private final List<Evaluator> evals = new ArrayList<>();

/**
* Create a new QueryParser.
Expand All @@ -35,11 +35,12 @@ private QueryParser(String query) {
}

/**
* Parse a CSS query into an Evaluator. If you are evaluating the same query repeatedly, it may be more efficient to
* parse it once and reuse the Evaluator.
* @param query CSS query
* @return Evaluator
* @see Selector selector query syntax
Parse a CSS query into an Evaluator. If you are evaluating the same query repeatedly, it may be more efficient to
parse it once and reuse the Evaluator.

@param query CSS query
@return Evaluator
@see Selector selector query syntax
*/
public static Evaluator parse(String query) {
try {
Expand All @@ -51,137 +52,131 @@ public static Evaluator parse(String query) {
}

/**
* Parse the query
* @return Evaluator
Parse the query. We use this simplified expression of the grammar:
<pre>
SelectorGroup ::= Selector (',' Selector)*
Selector ::= SimpleSequence ( Combinator SimpleSequence )*
SimpleSequence ::= [ TypeSelector ] ( ID | Class | Attribute | Pseudo )*
Pseudo ::= ':' Name [ '(' SelectorGroup ')' ]
Combinator ::= S+ // descendant (whitespace)
| '>' // child
| '+' // adjacent sibling
| '~' // general sibling
</pre>

See <a href="https://www.w3.org/TR/selectors-4/#grammar">selectors-4</a> for the real thing
*/
Evaluator parse() {
Evaluator eval = parseSelectorGroup();
tq.consumeWhitespace();
if (!tq.isEmpty())
throw new Selector.SelectorParseException("Could not parse query '%s': unexpected token at '%s'", query, tq.remainder());
return eval;
}

if (tq.matchesAny(Combinators)) { // if starts with a combinator, use root as elements
evals.add(new StructuralEvaluator.Root());
combinator(tq.consume());
} else {
evals.add(consumeEvaluator());
Evaluator parseSelectorGroup() {
// SelectorGroup. Into an Or if > 1 Selector
Evaluator left = parseSelector();
while (tq.matchChomp(',')) {
Evaluator right = parseSelector();
left = or(left, right);
}
return left;
}

while (!tq.isEmpty()) {
// hierarchy and extras
boolean seenWhite = tq.consumeWhitespace();
Evaluator parseSelector() {
// SimpleSequence ( Combinator SimpleSequence )*
Evaluator left = parseSimpleSequence();

while (true) {
char combinator = 0;
if (tq.consumeWhitespace())
combinator = ' '; // maybe descendant?
if (tq.matchesAny(Combinators)) // no, explicit
combinator = tq.consume();
else if (tq.matchesAny(SequenceEnders)) // , - space after simple like "foo , bar"; ) - close of :has()
break;

if (tq.matchesAny(Combinators)) {
combinator(tq.consume());
} else if (seenWhite) {
combinator(' ');
} else { // E.class, E#id, E[attr] etc. AND
evals.add(consumeEvaluator()); // take next el, #. etc off queue
if (combinator != 0) {
Evaluator right = parseSimpleSequence();
left = combinator(left, combinator, right);
} else {
break;
}
}

if (evals.size() == 1)
return evals.get(0);

return new CombiningEvaluator.And(evals);
return left;
}

private void combinator(char combinator) {
Evaluator parseSimpleSequence() {
// SimpleSequence ::= TypeSelector? ( Hash | Class | Pseudo )*
Evaluator left = null;
tq.consumeWhitespace();
String subQuery = consumeSubQuery(); // support multi > childs

Evaluator rootEval; // the new topmost evaluator
Evaluator currentEval; // the evaluator the new eval will be combined to. could be root, or rightmost or.
Evaluator newEval = parse(subQuery); // the evaluator to add into target evaluator
boolean replaceRightMost = false;

if (evals.size() == 1) {
rootEval = currentEval = evals.get(0);
// make sure OR (,) has precedence:
if (rootEval instanceof CombiningEvaluator.Or && combinator != ',') {
currentEval = ((CombiningEvaluator.Or) currentEval).rightMostEvaluator();
assert currentEval != null; // rightMost signature can return null (if none set), but always will have one by this point
replaceRightMost = true;
}
}
else {
rootEval = currentEval = new CombiningEvaluator.And(evals);

// one optional type selector
if (tq.matchesWord() || tq.matches("*|"))
left = byTag();
else if (tq.matchChomp('*'))
left = new Evaluator.AllElements();
else if (tq.matchesAny(Combinators)) // e.g. query is "> div"; type is root element
left = new StructuralEvaluator.Root();

// zero or more subclasses (#, ., [)
while(true) {
Evaluator right = parseSubclass();
if (right != null)
left = and(left, right);
else break; // no more simple tokens
}
evals.clear();

// for most combinators: change the current eval into an AND of the current eval and the new eval
if (left == null)
throw new Selector.SelectorParseException("Could not parse query '%s': unexpected token at '%s'", query, tq.remainder());
return left;
}

static Evaluator combinator(Evaluator left, char combinator, Evaluator right) {
switch (combinator) {
case '>':
ImmediateParentRun run = currentEval instanceof ImmediateParentRun ?
(ImmediateParentRun) currentEval : new ImmediateParentRun(currentEval);
run.add(newEval);
currentEval = run;
break;
ImmediateParentRun run = left instanceof ImmediateParentRun ?
(ImmediateParentRun) left : new ImmediateParentRun(left);
run.add(right);
return run;
case ' ':
currentEval = new CombiningEvaluator.And(new StructuralEvaluator.Ancestor(currentEval), newEval);
break;
return and(new StructuralEvaluator.Ancestor(left), right);
case '+':
currentEval = new CombiningEvaluator.And(new StructuralEvaluator.ImmediatePreviousSibling(currentEval), newEval);
break;
return and(new StructuralEvaluator.ImmediatePreviousSibling(left), right);
case '~':
currentEval = new CombiningEvaluator.And(new StructuralEvaluator.PreviousSibling(currentEval), newEval);
break;
case ',':
CombiningEvaluator.Or or;
if (currentEval instanceof CombiningEvaluator.Or) {
or = (CombiningEvaluator.Or) currentEval;
} else {
or = new CombiningEvaluator.Or();
or.add(currentEval);
}
or.add(newEval);
currentEval = or;
break;
return and(new StructuralEvaluator.PreviousSibling(left), right);
default:
throw new Selector.SelectorParseException("Unknown combinator '%s'", combinator);
}
}

if (replaceRightMost)
((CombiningEvaluator.Or) rootEval).replaceRightMostEvaluator(currentEval);
else rootEval = currentEval;
evals.add(rootEval);
@Nullable Evaluator parseSubclass() {
// Subclass: ID | Class | Attribute | Pseudo
if (tq.matchChomp('#')) return byId();
else if (tq.matchChomp('.')) return byClass();
else if (tq.matches('[')) return byAttribute();
else if (tq.matchChomp(':')) return parsePseudoSelector();
else return null;
}

private String consumeSubQuery() {
StringBuilder sq = StringUtil.borrowBuilder();
boolean seenClause = false; // eat until we hit a combinator after eating something else
while (!tq.isEmpty()) {
if (tq.matchesAny(Combinators)) {
if (seenClause)
break;
sq.append(tq.consume());
continue;
}
seenClause = true;
if (tq.matches("("))
sq.append("(").append(tq.chompBalanced('(', ')')).append(")");
else if (tq.matches("["))
sq.append("[").append(tq.chompBalanced('[', ']')).append("]");
else if (tq.matches("\\")) { // bounce over escapes
sq.append(TokenQueue.escapeCssIdentifier(tq.consumeCssIdentifier()));
} else
sq.append(tq.consume());
/** Merge two evals into an Or. */
static Evaluator or(Evaluator left, Evaluator right) {
if (left instanceof CombiningEvaluator.Or) {
((CombiningEvaluator.Or) left).add(right);
return left;
}
return StringUtil.releaseBuilder(sq);
return new CombiningEvaluator.Or(left, right);
}

private Evaluator consumeEvaluator() {
if (tq.matchChomp("#"))
return byId();
else if (tq.matchChomp("."))
return byClass();
else if (tq.matchesWord() || tq.matches("*|"))
return byTag();
else if (tq.matches("["))
return byAttribute();
else if (tq.matchChomp("*"))
return new Evaluator.AllElements();
else if (tq.matchChomp(":"))
return parsePseudoSelector();
else // unhandled
throw new Selector.SelectorParseException("Could not parse query '%s': unexpected token at '%s'", query, tq.remainder());
/** Merge two evals into an And. */
static Evaluator and(@Nullable Evaluator left, Evaluator right) {
if (left == null) return right;
if (left instanceof CombiningEvaluator.And) {
((CombiningEvaluator.And) left).add(right);
return left;
}
return new CombiningEvaluator.And(left, right);
}

private Evaluator parsePseudoSelector() {
Expand Down Expand Up @@ -299,7 +294,7 @@ else if (key.equals("*")) // any attribute
else
eval = new Evaluator.Attribute(key);
} else {
if (cq.matchChomp("="))
if (cq.matchChomp('='))
eval = new Evaluator.AttributeWithValue(key, cq.remainder());
else if (cq.matchChomp("!="))
eval = new Evaluator.AttributeWithValueNot(key, cq.remainder());
Expand Down Expand Up @@ -364,16 +359,19 @@ private int consumeIndex() {

// pseudo selector :has(el)
private Evaluator has() {
String subQuery = consumeParens();
Validate.notEmpty(subQuery, ":has(selector) sub-select must not be empty");
return new StructuralEvaluator.Has(parse(subQuery));
return parseNested(StructuralEvaluator.Has::new, ":has() must have a selector");
}

// psuedo selector :is()
// pseudo selector :is()
private Evaluator is() {
String subQuery = consumeParens();
Validate.notEmpty(subQuery, ":is(selector) sub-select must not be empty");
return new StructuralEvaluator.Is(parse(subQuery));
return parseNested(StructuralEvaluator.Is::new, ":is() must have a selector");
}

private Evaluator parseNested(Function<Evaluator, Evaluator> func, String err) {
Validate.isTrue(tq.matchChomp('('), err);
Evaluator eval = parseSelectorGroup();
Validate.isTrue(tq.matchChomp(')'), err);
return func.apply(eval);
}

// pseudo selector :contains(text), containsOwn(text)
Expand Down
Loading