From f4c1d12c59379f64fed656e21617cc4ed932d248 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Fri, 15 Mar 2024 11:27:14 +0100 Subject: [PATCH 1/2] use md4c as markdown library --- config | 2 +- ngx_http_awesomeindex_module.c | 159 +++++++++++++++++++++++++-------- t/README.md | 17 ---- t/show_dotfiles/README.txt | 3 + 4 files changed, 125 insertions(+), 56 deletions(-) create mode 100644 t/show_dotfiles/README.txt diff --git a/config b/config index e0d7ccd..d8fa0ae 100644 --- a/config +++ b/config @@ -1,6 +1,6 @@ # vim:ft=sh: ngx_addon_name=ngx_http_awesomeindex_module -ngx_module_libs="-lmarkdown" +ngx_module_libs="-lmd4c-html" if [ "$ngx_module_link" = DYNAMIC ] ; then ngx_module_type=HTTP diff --git a/ngx_http_awesomeindex_module.c b/ngx_http_awesomeindex_module.c index c6e4385..6f15f97 100644 --- a/ngx_http_awesomeindex_module.c +++ b/ngx_http_awesomeindex_module.c @@ -22,10 +22,10 @@ #include #include #include +#include "ngx_buf.h" #include "ngx_string.h" -#include - +#include "md4c-html.h" #include "template.h" #if defined(__GNUC__) && (__GNUC__ >= 3) @@ -328,7 +328,85 @@ ngx_awesomeindex_conf_set_headerfooter(ngx_conf_t *cf, ngx_command_t *cmd, void #define ngx_has_flag(_where, _what) \ (((_where) & (_what)) == (_what)) +/********************************* + *** Simple grow-able buffer *** + *********************************/ +/* We render to a memory buffer instead of directly outputting the rendered + * documents, as this allows using this utility for evaluating performance + * of MD4C (--stat option). This allows us to measure just time of the parser, + * without the I/O. + */ + +struct membuffer { + char* data; + size_t asize; + size_t size; + u_char fail; +}; + +static u_char +membuf_init(struct membuffer* buf, MD_SIZE new_asize) +{ + buf->size = 0; + buf->asize = new_asize; + buf->data = malloc(buf->asize); + if(buf->data == NULL) { + buf->fail = 1; + return 1; + } + buf->fail = 0; + return 0; +} + +static void +membuf_free(struct membuffer* buf) +{ + if(buf && buf->data) free(buf->data); +} + +static void +membuf_grow(struct membuffer* buf, size_t new_asize) +{ + buf->data = realloc(buf->data, new_asize); + if(buf->data == NULL) { + buf->fail = 1; + } + buf->asize = new_asize; +} + +static void +membuf_append(struct membuffer* buf, const char* data, MD_SIZE size) +{ + if(buf->asize < buf->size + size) + membuf_grow(buf, buf->size + buf->size / 2 + size); + memcpy(buf->data + buf->size, data, size); + buf->size += size; +} + +static void +process_md_output(const MD_CHAR* text, MD_SIZE size, void* userdata) +{ + struct membuffer *buf = (struct membuffer*) userdata; + if (!buf->fail) membuf_append(buf, text, size); +} + +static ssize_t ngx_awesomeindex_read_file(ngx_http_request_t *r, u_char* filename, size_t file_len, u_char* dest) +{ + ngx_file_t file; + ngx_memzero(&file, sizeof(ngx_file_t)); + file.fd = ngx_open_file(filename, NGX_FILE_RDONLY, NGX_FILE_OPEN, 0); + if (file.fd == NGX_INVALID_FILE) { + ngx_log_error(NGX_LOG_ALERT, r->connection->log, ngx_errno, + "cannot open readme file \"%V\"", filename); + return NGX_ERROR; + } + file.log = r->connection->log; + + ssize_t n = ngx_read_file(&file, dest, file_len, 0); + ngx_close_file(file.fd); + return n; +} typedef struct { @@ -941,7 +1019,7 @@ make_content_buf( if (readme_path) { ngx_log_debug3(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, - "HTTP awesomeindex README: %s, MD%d, len: %d", readme_path, readme_md, readme_file_len); + "http awesomeindex README: %s, MD%d, len: %d", readme_path, readme_md, readme_file_len); } /* @@ -1019,21 +1097,46 @@ make_content_buf( if (entry[i].link) len += ngx_sizeof_ssz("-shortcut"); } - char *readme_md_content; + struct membuffer readme_md_content = {0}; if (readme_file_len) { if (readme_md) { - FILE *md_file = fopen((char *)readme_path, "r"); - MMIOT *mkd = mkd_in(md_file, MKD_FLAGS); - mkd_compile(mkd, MKD_FLAGS); - readme_file_len = mkd_document(mkd, &readme_md_content); - fclose(md_file); - ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "converted %s, len=%d", readme_path, readme_file_len); + ngx_buf_t *buf_in = ngx_create_temp_buf(r->pool, readme_file_len); + ssize_t rbts = ngx_awesomeindex_read_file(r, readme_path, readme_file_len, buf_in->last); + + if (rbts != NGX_ERROR) { + buf_in->last += rbts; + ngx_log_debug(NGX_LOG_DEBUG_HTTP, r->connection->log, ngx_errno, "http awesomeindex: read %d bytes from %s", rbts, readme_path); + + // Allocating buffer for parsing readme + if (!membuf_init(&readme_md_content, readme_file_len + readme_file_len/5)) { + ngx_log_debug(NGX_LOG_DEBUG_HTTP, r->connection->log, ngx_errno, "MEMBUF INITTED"); + int ret = md_html((MD_CHAR*) buf_in->start, readme_file_len, process_md_output, + &readme_md_content, MD_FLAG_PERMISSIVEURLAUTOLINKS | MD_FLAG_TABLES, 0); + if (ret) { + ngx_log_error(NGX_LOG_ALERT, r->connection->log, ngx_errno, "error parsing markdown %s", readme_path); + } else if (readme_md_content.fail) { + ngx_log_error(NGX_LOG_ALERT, r->connection->log, ngx_errno, "could not allocate memory for parsing readme"); + } else { + ngx_log_debug(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "parsed markdown len=%d", readme_md_content.size); + len += readme_md_content.size; + } + } else { + ngx_log_error(NGX_LOG_ALERT, r->connection->log, ngx_errno, "could not allocate memory for parsing readme"); + } + } else { + readme_md_content.fail = 1; + ngx_log_error(NGX_LOG_ALERT, r->connection->log, ngx_errno, "cannot read file \"%V\"", readme_path); + } + } else { + len += readme_file_len; } - len += footer_pre.len + readme_file_len + footer_post.len; + len += footer_pre.len + footer_post.len; } - if ((b = ngx_create_temp_buf(r->pool, len)) == NULL) + if ((b = ngx_create_temp_buf(r->pool, len)) == NULL) { + membuf_free(&readme_md_content); return NGX_HTTP_INTERNAL_SERVER_ERROR; + } /* * Determine the sorting criteria. URL arguments look like: @@ -1302,33 +1405,13 @@ make_content_buf( b->last = ngx_cpymem_str(b->last, footer_pre); if (readme_md) { - b->last = ngx_cpymem(b->last, readme_md_content, readme_file_len); + if (!readme_md_content.fail) b->last = ngx_cpymem(b->last, readme_md_content.data, readme_md_content.size); + membuf_free(&readme_md_content); } else { - ngx_file_t file; - ngx_memzero(&file, sizeof(ngx_file_t)); - file.fd = ngx_open_file(readme_path, NGX_FILE_RDONLY, NGX_FILE_OPEN, 0); - if (file.fd == NGX_INVALID_FILE) { - ngx_log_error(NGX_LOG_ALERT, r->connection->log, ngx_errno, - "cannot open readme file \"%V\"", readme_path); - return NGX_ERROR; - } - file.log = r->connection->log; - ngx_log_debug(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "opened file %d", file); - - ssize_t n = readme_file_len; - while (n > 0) { - ssize_t rbts = ngx_read_file(&file, - b->last + file.offset, - n, - file.offset); - if (rbts == NGX_ERROR) { - ngx_close_file(file.fd); - ngx_log_error(NGX_LOG_ALERT, r->connection->log, ngx_errno, - "cannot read readme file \"%V\"", readme_path); - return NGX_ERROR; - } - b->last += rbts; - n -= rbts; + ssize_t rbts = ngx_awesomeindex_read_file(r, readme_path, readme_file_len, b->last); + if (rbts != NGX_ERROR) b->last += rbts; + else { + ngx_log_error(NGX_LOG_ALERT, r->connection->log, ngx_errno, "cannot read file \"%V\"", readme_path); } } diff --git a/t/README.md b/t/README.md index d54b2fb..b5277b4 100644 --- a/t/README.md +++ b/t/README.md @@ -1,22 +1,5 @@ # Markdown: Syntax -* [Overview](#Overview) - * [Philosophy](#Philosophy) -* [Block Elements](#Block-Elements) - * [Paragraphs and Line Breaks](#Paragraphs-and-Line-Breaks) - * [Headers](#Headers) - * [Blockquotes](#Blockquotes) - * [Lists](#Lists) - * [Code Blocks](#Code-Blocks) - * [Horizontal Rules](#Horizontal-Rules) -* [Span Elements](#Span-Elements) - * [Links](#Links) - * [Emphasis](#Emphasis) - * [Code](#Code) - * [Table](#Table) - ----- - ## Overview ### Philosophy diff --git a/t/show_dotfiles/README.txt b/t/show_dotfiles/README.txt new file mode 100644 index 0000000..53191b8 --- /dev/null +++ b/t/show_dotfiles/README.txt @@ -0,0 +1,3 @@ +

Hello World

+ +Hello World, this is me ;-) From c85a9fa00b8e5e81904f19cfbe15b95a1c7b0f4f Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Fri, 15 Mar 2024 12:07:30 +0100 Subject: [PATCH 2/2] feat: render txt files correctly --- ngx_http_awesomeindex_module.c | 49 +- t/child-directory/README.txt | 7 + t/show_dotfiles/{README.txt => README.html} | 0 template.h | 5 +- template.html | 705 ++++++++------------ 5 files changed, 322 insertions(+), 444 deletions(-) create mode 100644 t/child-directory/README.txt rename t/show_dotfiles/{README.txt => README.html} (100%) diff --git a/ngx_http_awesomeindex_module.c b/ngx_http_awesomeindex_module.c index 6f15f97..5ebfad9 100644 --- a/ngx_http_awesomeindex_module.c +++ b/ngx_http_awesomeindex_module.c @@ -637,11 +637,21 @@ static const ngx_str_t img_icon_pre = ngx_string("\"\""); -static const ngx_str_t footer_pre = +static const ngx_str_t footer_md_pre = ngx_string("