Updating in-tree block documentation

#!/usr/bin/env python3

import requests
import mwparserfromhell
import yaml

import os
import sys

def update_files(*filenames):
blocklabels = {
yaml.safe_load(open(filename))['label']: filename
for filename in filenames
}

pages = requests.get(
'https://wiki.gnuradio.org/api.php',
params={
'action': 'query',
'format': 'json',
'prop': 'revisions',
'rvprop': 'content',
'titles': '|'.join((blocklabel.replace(' ', '_') for blocklabel in blocklabels))
}
).json()
pages = {
page['title']: page['revisions'][0]['*']
for page in pages['query']['pages'].values()
if 'revisions' in page
}

for blocklabel, filename in blocklabels.items():
if blocklabel not in pages:
print('Not found on wiki:', blocklabel)
continue
print(filename, blocklabel)
page = pages[blocklabel]
page = mwparserfromhell.parse(page)

sections = page.get_sections(matches='Parameters', include_lead=True)
sections = (section.strip_code() for section in sections)

doc = '\n\n'.join(sections)

block_lines = [*open(filename)]

replaced_doc = False

old_doc_start = -1 if block_lines[-1].rstrip() == 'file_format: 1' else len(block_lines)
for index, line in enumerate(block_lines):
if line.startswith('documentation:'):
old_doc_start = index
break
old_doc_end = old_doc_start
for index, line in enumerate(block_lines[old_doc_start:]):
if not line.startswith(' ') and line.strip() and not line.startswith('documentation'):
old_doc_end = index + old_doc_start
break

doc_lines = ['documentation: |-\n', *[' ' + line.strip() + '\n' for line in doc.split('\n')], '\n']
block_lines[old_doc_start:old_doc_end] = doc_lines

open(filename, 'w').write(''.join(block_lines))

if __name__ == '__main__':
if sys.argv[1:]:
update_files(*sys.argv[1:])
else:
files = []
for root, dirs, files in os.walk("."):
files = [os.path.join(root, file) for file in files if file.endswith('.block.yml')]
if files:
update_files(*files)
Hi,

I am just learning gnuradio and I find it takes a lot of time to look
up the blocks on the wiki.

The attached python script, if run in the gnuradio source tree root,
will update all the blocks' local documentation from the wiki.

It uses requests, pyyaml, and mwparserfromhell.

It makes a _lot_ of updates. A patch updating them all would be pretty large.

Here is an example:

--- a/gr-blocks/grc/blocks_add_const_vxx.block.yml
+++ b/gr-blocks/grc/blocks_add_const_vxx.block.yml
@@ -49,4 +49,14 @@ cpp_templates:
callbacks:
- set_k(${const})

+documentation: |-
+ Category:Block Docs
+ Adds a constant value to each item that passes though. output[m]
= input[m] + constant vector.
+
+ Parameters
+ (R): Run-time adjustable
+
+ Constant (R)
+ The entered value must be of the same type as the input / output
ports (i.e. complex, float, etc.).
+