Skip to content

Commit

Permalink
Track doc comments and render them in :doc
Browse files Browse the repository at this point in the history
  • Loading branch information
roberth committed Jul 9, 2024
1 parent 07cf330 commit c1cf997
Show file tree
Hide file tree
Showing 17 changed files with 343 additions and 10 deletions.
61 changes: 61 additions & 0 deletions src/libexpr/eval.cc
Original file line number Diff line number Diff line change
Expand Up @@ -557,6 +557,67 @@ std::optional<EvalState::Doc> EvalState::getDoc(Value & v)
.doc = doc,
};
}
if (v.isLambda()) {
auto exprLambda = v.payload.lambda.fun;

std::stringstream s(std::ios_base::out);
std::string name;
auto pos = positions[exprLambda->getPos()];
std::string docStr;

if (exprLambda->name) {
name = symbols[exprLambda->name];
}

if (exprLambda->docComment) {
auto begin = positions[exprLambda->docComment.begin];
auto end = positions[exprLambda->docComment.end];
auto docCommentStr = begin.getSnippetUpTo(end);

// Strip "/**" and "*/"
constexpr size_t prefixLen = 3;
constexpr size_t suffixLen = 2;
docStr = docCommentStr.substr(prefixLen, docCommentStr.size() - prefixLen - suffixLen);
if (docStr.empty())
return {};
// Turn the now missing "/**" into indentation
docStr = " " + docStr;
// Strip indentation (for the whole, potentially multi-line string)
docStr = stripIndentation(docStr);
}

if (name.empty()) {
s << "Function ";
}
else {
s << "Function **" << name << "**";
if (pos)
s << "\\\n" ;
else
s << "\\\n";
}
if (pos) {
s << "defined at " << pos;
}
if (!docStr.empty()) {
s << "\n\n";
}

s << docStr;

s << '\0'; // for making a c string below
std::string ss = s.str();

return Doc {
.pos = pos,
.name = name,
.arity = 0, // FIXME: figure out how deep by syntax only? It's not semantically useful though...
.args = {},
.doc =
// FIXME: this leaks; make the field std::string?
strdup(ss.data()),
};
}
return {};
}

Expand Down
46 changes: 40 additions & 6 deletions src/libexpr/lexer.l
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
%option stack
%option nodefault
%option nounput noyy_top_state

%option extra-type="::nix::LexerState *"

%s DEFAULT
%x STRING
Expand All @@ -23,6 +23,12 @@
#include "nixexpr.hh"
#include "parser-tab.hh"

// !!! FIXME !!!
#define YY_EXTRA_TYPE ::nix::LexerState *
int yylex_init_extra ( YY_EXTRA_TYPE user_defined, yyscan_t* scanner);
YY_EXTRA_TYPE yyget_extra ( yyscan_t yyscanner );
#undef YY_EXTRA_TYPE

using namespace nix;

namespace nix {
Expand All @@ -35,10 +41,24 @@ static void initLoc(YYLTYPE * loc)
loc->first_column = loc->last_column = 0;
}

static void adjustLoc(YYLTYPE * loc, const char * s, size_t len)
static void adjustLoc(yyscan_t yyscanner, YYLTYPE * loc, const char * s, size_t len)
{
loc->stash();

LexerState & lexerState = *yyget_extra(yyscanner);

if (lexerState.docCommentDistance == 1) {
// Preceding token was a doc comment.
ParserLocation doc;
doc.first_column = lexerState.lastDocCommentLoc.first_column;
ParserLocation docEnd;
docEnd.first_column = lexerState.lastDocCommentLoc.last_column;
DocComment docComment{lexerState.at(doc), lexerState.at(docEnd)};
PosIdx locPos = lexerState.at(*loc);
lexerState.positionToDocComment.emplace(locPos, docComment);
}
lexerState.docCommentDistance++;

loc->first_column = loc->last_column;
loc->last_column += len;
}
Expand Down Expand Up @@ -79,7 +99,7 @@ static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length)
#pragma GCC diagnostic ignored "-Wimplicit-fallthrough"

#define YY_USER_INIT initLoc(yylloc)
#define YY_USER_ACTION adjustLoc(yylloc, yytext, yyleng);
#define YY_USER_ACTION adjustLoc(yyscanner, yylloc, yytext, yyleng);

#define PUSH_STATE(state) yy_push_state(state, yyscanner)
#define POP_STATE() yy_pop_state(yyscanner)
Expand Down Expand Up @@ -279,9 +299,23 @@ or { return OR_KW; }
{SPATH} { yylval->path = {yytext, (size_t) yyleng}; return SPATH; }
{URI} { yylval->uri = {yytext, (size_t) yyleng}; return URI; }

[ \t\r\n]+ /* eat up whitespace */
\#[^\r\n]* /* single-line comments */
\/\*([^*]|\*+[^*/])*\*+\/ /* long comments */
\/\*\*([^*/]|\*+[^*/])*\*+\/ /* doc comments */ {
LexerState & lexerState = *yyget_extra(yyscanner);
lexerState.docCommentDistance = 0;
lexerState.lastDocCommentLoc.first_line = yylloc->first_line;
lexerState.lastDocCommentLoc.first_column = yylloc->first_column;
lexerState.lastDocCommentLoc.last_column = yylloc->last_column;
}


%{
// The following rules have docCommentDistance--
// This compensates for the docCommentDistance++ which happens by default to
// make all the other rules invalidate the doc comment.
%}
[ \t\r\n]+ /* eat up whitespace */ { yyget_extra(yyscanner)->docCommentDistance--; }
\#[^\r\n]* /* single-line comments */ { yyget_extra(yyscanner)->docCommentDistance--; }
\/\*([^*]|\*+[^*/])*\*+\/ /* long comments */ { yyget_extra(yyscanner)->docCommentDistance--; }

{ANY} {
/* Don't return a negative number, as this will cause
Expand Down
14 changes: 14 additions & 0 deletions src/libexpr/nixexpr.cc
Original file line number Diff line number Diff line change
Expand Up @@ -582,6 +582,20 @@ std::string ExprLambda::showNamePos(const EvalState & state) const
return fmt("%1% at %2%", id, state.positions[pos]);
}

void ExprLambda::setDocComment(DocComment docComment) {
if (!this->docComment) {
this->docComment = docComment;

// Curried functions are defined by putting a function directly
// in the body of another function. To render docs for those, we
// need to propagate the doc comment to the innermost function.
//
// If we have our own comment, we've already propagated it, so this
// belongs in the same conditional.
body->setDocComment(docComment);
}
};



/* Position table. */
Expand Down
63 changes: 63 additions & 0 deletions src/libexpr/nixexpr.hh
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,61 @@ class EvalState;
struct ExprWith;
struct StaticEnv;

/**
* A documentation comment, in the sense of [RFC 145](https://github.com/NixOS/rfcs/blob/master/rfcs/0145-doc-strings.md)
*
* Note that this does not implement the following:
* - argument attribute names ("formals"): TBD
* - argument names: these are internal to the function and their names may not be optimal for documentation
* - function arity (degree of currying or number of ':'s):
* - Functions returning partially applied functions have a higher arity
* than can be determined locally and without evaluation.
* We do not want to present false data.
* - Some functions should be thought of as transformations of other
* functions. For instance `overlay -> overlay -> overlay` is the simplest
* way to understand `composeExtensions`, but its implementation looks like
* `f: g: final: prev: <...>`. The parameters `final` and `prev` are part
* of the overlay concept, while distracting from the function's purpose.
*/
struct DocComment {

#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wcomment" // "nested comment start" is intentional

/**
* Start of the comment, including `/**`.
*/
PosIdx begin;

#pragma GCC diagnostic pop

/**
* Position right after the final asterisk and `/` that terminate the comment.
*/
PosIdx end;

/**
* Whether the comment is set.
*
* A `DocComment` is small enough that it makes sense to pass by value, and
* therefore baking optionality into it is also useful, to avoiding the memory
* overhead of `std::optional`.
*/
operator bool() const { return static_cast<bool>(begin); }

// /**
// * The number of times a function was applied to get from the documented lambda
// * to the current one.
// */
// size_t curriedDepth;

// /**
// * A name from a binding.
// *
// * This is a `ref` so that a sequence of `DocComments` can instantly agree on the received name during parsing.
// */
// ref<std::optional<Symbol>> name = make_ref<std::optional<Symbol>>(std::nullopt);
};

/**
* An attribute path is a sequence of attribute names.
Expand Down Expand Up @@ -57,6 +112,7 @@ struct Expr
virtual void eval(EvalState & state, Env & env, Value & v);
virtual Value * maybeThunk(EvalState & state, Env & env);
virtual void setName(Symbol name);
virtual void setDocComment(DocComment docComment) { };
virtual PosIdx getPos() const { return noPos; }
};

Expand Down Expand Up @@ -188,6 +244,10 @@ struct ExprAttrs : Expr
Kind kind;
Expr * e;
PosIdx pos;
/**
* noPos pretty often
*/
PosIdx docStartPos;
Displacement displ; // displacement
AttrDef(Expr * e, const PosIdx & pos, Kind kind = Kind::Plain)
: kind(kind), e(e), pos(pos) { };
Expand Down Expand Up @@ -281,6 +341,8 @@ struct ExprLambda : Expr
Symbol arg;
Formals * formals;
Expr * body;
DocComment docComment;

ExprLambda(PosIdx pos, Symbol arg, Formals * formals, Expr * body)
: pos(pos), arg(arg), formals(formals), body(body)
{
Expand All @@ -293,6 +355,7 @@ struct ExprLambda : Expr
std::string showNamePos(const EvalState & state) const;
inline bool hasFormals() const { return formals != nullptr; }
PosIdx getPos() const override { return pos; }
virtual void setDocComment(DocComment docComment) override;
COMMON_METHODS
};

Expand Down
38 changes: 38 additions & 0 deletions src/libexpr/parser-state.hh
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,43 @@ struct ParserLocation
first_column = stashed_first_column;
last_column = stashed_last_column;
}

/** Latest doc comment position, or 0. */
int doc_comment_first_line, doc_comment_first_column, doc_comment_last_column;
};

struct LexerState
{
/**
* Tracks the distance to the last doc comment, in terms of lexer tokens.
*
* The lexer sets this to 0 when reading a doc comment, and increments it
* for every matched rule; see `lexer-helpers.cc`.
* Whitespace and comment rules decrement the distance, so that they result
* in a net 0 change in distance.
*/
int docCommentDistance = 9999;
// We're not using this whole struct actually
ParserLocation lastDocCommentLoc;

/**
* @brief Parser location to DocComment
*
* The shared_ptr allows us to have a single instance for each comment,
* so that it can be updated with things like a name from a binding during
* parsing.
*/
std::map<PosIdx, DocComment> positionToDocComment;

PosTable & positions;
PosTable::Origin origin;

PosIdx at(const ParserLocation & loc);
};

struct ParserState
{
const LexerState & lexerState;
SymbolTable & symbols;
PosTable & positions;
Expr * result;
Expand Down Expand Up @@ -270,6 +303,11 @@ inline Expr * ParserState::stripIndentation(const PosIdx pos,
return new ExprConcatStrings(pos, true, es2);
}

inline PosIdx LexerState::at(const ParserLocation & loc)
{
return positions.add(origin, loc.first_column);
}

inline PosIdx ParserState::at(const ParserLocation & loc)
{
return positions.add(origin, loc.first_column);
Expand Down
Loading

0 comments on commit c1cf997

Please sign in to comment.