Rubén Sánchez

Romancero bot: converting article text into voice with Amazon Polly and sending it to Telegram

The following Python script extracts the content of an article using Readibility (ported to Python), converts it to voice using the Amazon Polly service and finally sends the audio as a voice note to a given user in Telegram using Telethon (Telegram client for Python).

For running the script you will need to install the following Python packages:

pip install boto3
pip install awscli
pip install readability-lxml
pip install telethon

pip install boto3

pip install awscli

pip install readability-lxml

pip install telethon

Also, you will need to create a AWS account. If you already have an AWS account, make sure that you have a user created in the IAM Management Console with the following permissions:

User permissions for creating Polly jobs and accessing/writing files in a S3 bucket.

When creating this user, make sure you write down its ID access key and its secret access key. You will need them to configure the aws-cli client.

With this Amazon credentials, you can configure the AWS client by executing the following command:

aws configure

1	aws configure

In this step, you will need to fulfill the details with the user credentials you wrote down when creating it.

Now, you will need to create a Telegram API ID. For this, you can go to the Telegram section “Create an Application“. After following the steps described in the official documentation, you will obtain an API ID (it’s a number) and a API hash (it’s a string).

With these steps already completed, you can place all the needed details in the script and run it.

import boto3
import textwrap
import requests
import re
from readability import Document
from telethon import TelegramClient, events, sync
import time
import os

#--------------------------------------------------------------------
# Configuration
#--------------------------------------------------------------------

##### Article #####
# Define URL
url = "PLACE_THE_URL_HERE"

##### Telegram #####
# Telegram API credentials
api_id = PLACE_API_ID_HERE
api_hash = 'PLACE_API_HASH_HERE'
# Telegram user to send messages
telegram_user = "PLACE_TELEGRAM_USER_HERE"
# Create Telegram client
telegram_client = TelegramClient('session_name', api_id, api_hash)
telegram_client.start()

##### AWS configuration #####
# Get S3 session
session = boto3.Session(profile_name='default')
# Get polly client
polly = session.client('polly')
# Create a S3 client to retrieve the file content
s3 = session.client('s3')
# Define bucket name
bucket_name = 'PLACE_BUCKET_NAME_HERE'
#--------------------------------------------------------------------
# Get HTML from the article
#--------------------------------------------------------------------

# Get HTML
response = requests.get(url)

# Extract content with Readibility
doc = Document(response.text)

# Get article body
html_text = doc.summary()

# Regular expression to identify HTML tags, e.g.: 
html_tag_re = r"<\\?[^>]+>"

# Remove HTML tags from the article body
text_only = re.sub( html_tag_re, "", html_text,)

# Send message to user pointing to the URL that is going to be converted
telegram_client.send_message(telegram_user, "Converting: %s" % url)

#--------------------------------------------------------------------
# Convert text to voice
#--------------------------------------------------------------------

# Start Polly task to save in a Bucket
resp = polly.start_speech_synthesis_task(OutputFormat='mp3',
        OutputS3BucketName=bucket_name,
        Text=text_only,
        VoiceId='Enrique')

# Get Polly task
task = polly.get_speech_synthesis_task(TaskId=resp['SynthesisTask']['TaskId'])

# Monitor task status until it is completed
while task['SynthesisTask']['TaskStatus'] != 'completed':
    # Wait 2 seconds between server poll
    time.sleep(2)
    # Get Polly task
    task = polly.get_speech_synthesis_task(TaskId=task['SynthesisTask']['TaskId'])
    # Print the status of the task
    print("Task status: %s" % task['SynthesisTask']['TaskStatus'])

print("Task completed!")

#--------------------------------------------------------------------
# Retrieve file and send to Telegram user
#--------------------------------------------------------------------

# Regular expression to extract the key (file name) of the synthesized file
key_re = r'/([0-9A-Za-z-.]+)$'
# Search the regular expression in the OutputUri 
regex_search = re.search(key_re, task['SynthesisTask']['OutputUri'])
# Take only the first group of the re (key)
file_key = regex_search.group(1)

# Get file name to store in local
title_sanitized = doc.short_title().replace('"', '')
title_sanitized = title_sanitized.replace(':', '')
file_name = "%s.mp3" % title_sanitized

# Download file from the bucket and store it in a MP3 local file
with open(file_name, 'wb') as data:
    s3.download_fileobj(bucket_name, file_key, data)

# Delete remote bucket file
s3.delete_object(Bucket=bucket_name,Key=file_key)
# Delete local file
if os.path.isfile(file_key):
    os.remove(file_key)

# Send MP3 file as a voice note to the telegram user
telegram_client.send_file(telegram_user, file_name, voice_note=True)
# Send signature
telegram_client.send_message(telegram_user, "Message sent from Romancero bot.")

100

101

102

103

104

105

106

107

108

109

110

111

112

import boto3

import textwrap

import requests

import re

from readability import Document

from telethon import TelegramClient, events, sync

import time

import os

#--------------------------------------------------------------------

# Configuration

#--------------------------------------------------------------------

##### Article #####

# Define URL

url = "PLACE_THE_URL_HERE"

##### Telegram #####

# Telegram API credentials

api_id = PLACE_API_ID_HERE

api_hash = 'PLACE_API_HASH_HERE'

# Telegram user to send messages

telegram_user = "PLACE_TELEGRAM_USER_HERE"

# Create Telegram client

telegram_client = TelegramClient('session_name', api_id, api_hash)

telegram_client.start()

##### AWS configuration #####

# Get S3 session

session = boto3.Session(profile_name='default')

# Get polly client

polly = session.client('polly')

# Create a S3 client to retrieve the file content

s3 = session.client('s3')

# Define bucket name

bucket_name = 'PLACE_BUCKET_NAME_HERE'

#--------------------------------------------------------------------

# Get HTML from the article

#--------------------------------------------------------------------

# Get HTML

response = requests.get(url)

# Extract content with Readibility

doc = Document(response.text)

# Get article body

html_text = doc.summary()

# Regular expression to identify HTML tags, e.g.:

html_tag_re = r"<\\?[^>]+>"

# Remove HTML tags from the article body

text_only = re.sub( html_tag_re, "", html_text,)

# Send message to user pointing to the URL that is going to be converted

telegram_client.send_message(telegram_user, "Converting: %s" % url)

#--------------------------------------------------------------------

# Convert text to voice

#--------------------------------------------------------------------

# Start Polly task to save in a Bucket

resp = polly.start_speech_synthesis_task(OutputFormat='mp3',

OutputS3BucketName=bucket_name,

Text=text_only,

VoiceId='Enrique')

# Get Polly task

task = polly.get_speech_synthesis_task(TaskId=resp['SynthesisTask']['TaskId'])

# Monitor task status until it is completed

while task['SynthesisTask']['TaskStatus'] != 'completed':

# Wait 2 seconds between server poll

time.sleep(2)

# Get Polly task

task = polly.get_speech_synthesis_task(TaskId=task['SynthesisTask']['TaskId'])

# Print the status of the task

print("Task status: %s" % task['SynthesisTask']['TaskStatus'])

print("Task completed!")

#--------------------------------------------------------------------

# Retrieve file and send to Telegram user

#--------------------------------------------------------------------

# Regular expression to extract the key (file name) of the synthesized file

key_re = r'/([0-9A-Za-z-.]+)$'

# Search the regular expression in the OutputUri

regex_search = re.search(key_re, task['SynthesisTask']['OutputUri'])

# Take only the first group of the re (key)

file_key = regex_search.group(1)

# Get file name to store in local

title_sanitized = doc.short_title().replace('"', '')

title_sanitized = title_sanitized.replace(':', '')

file_name = "%s.mp3" % title_sanitized

# Download file from the bucket and store it in a MP3 local file

with open(file_name, 'wb') as data:

s3.download_fileobj(bucket_name, file_key, data)

# Delete remote bucket file

s3.delete_object(Bucket=bucket_name,Key=file_key)

# Delete local file

if os.path.isfile(file_key):

os.remove(file_key)

# Send MP3 file as a voice note to the telegram user

telegram_client.send_file(telegram_user, file_name, voice_note=True)

# Send signature

telegram_client.send_message(telegram_user, "Message sent from Romancero bot.")

The user you specified will receive a message like this:

Parser in PHP using regular expressions

You can use regular expressions in PHP by using the function preg_match ( string $pattern , string $subject [, array &$matches [, int $flags = 0 [, int $offset = 0 ]]] ) . Only the first two paremeters are mandatory and they are the regex and the string where you want to search respectively.

In case of finding a result, preg_match() returns an array where the item at index 0 is the whole match. From 1 onwards they are placed the different groups of your regular expressions (in case there is any). If no match is found, preg_match() returns null.

One of the details that must be taken into account when using regular expressions on PHP is that they must be enclosed by forward slashes (/), e.g. $multiline_meaning_re = '/^([A-za-z ,"().\';:]+)/'; . This regular expression matches any string with any set of letters, spaces, commas, double and single quotes, parenthesis points, colon and/or semicolon.

As a complete example, the following snippet opens a file, parses it to look for English idioms and uploads all of them a MySQL database.

<?php
$servername = "";
$database = "";
$username = "";
$password = "";
$conn = mysqli_connect($servername, $username, $password, $database);

// Check connection
if (!$conn) {
    die("Connection failed: " . mysqli_connect_error());
}

    class Idiom {
        var $idiom = "";
        var $meaning = "";
        var $example = [];
        function print(){
            echo($this->idiom . "<br>" . $this->meaning . "<ul>");
            print_r($this->example);
            if(count($this->example) == 0 )
              echo("ERROR!!!!. There should be at least one example");
            foreach ($this->example as $value) {
                echo("<li>" . $value . "</li>");
            }
            echo("</ul>");
        }
        function upload($conn){
          echo("Uploading...");
          $example = "<ul class='list-group'>";
          foreach ($this->example as $value) {
            if($value !== '')
              $example = $example . "<li class='list-group-item'>" . $value . '</li>';
          }
          $example = $example . "</ul>";

          $idiom = mysqli_real_escape_string($conn, $this->idiom);
          $meaning = mysqli_real_escape_string($conn, $this->meaning);
          $example = mysqli_real_escape_string($conn, $example);

          $query = 'INSERT INTO idioms (`idiom`, `meaning`, `example`) VALUES ("'. $idiom . '","'. $meaning . '","'. $example . '")';
          mysqli_query($conn, $query);
        }
    }

    $file = fopen("idioms.txt", "r") or die("Unable to open file!");
    $idiom_meaning_re     = '/^([A-za-z,. -\/()\']+):([A-za-z ,"().\';:\n]+)/';
    $multiline_meaning_re = '/^([A-za-z ,"().\';:]+)/';
    $example_re           = '/^\|--([A-Za-z0-9 \',?.-;$\n"]+[^:])/';
    $new_idiom = 0;
    $new_example = 0;
    $idiom = null;
    $example = "";
    // Output one line until end-of-file
    while(!feof($file)) {
      $line = fgets($file);
      //echo($line . "<br>");
      preg_match($idiom_meaning_re, $line, $matches);

      if($matches != null){
        if($new_example === 1){
          array_push($idiom->example, $example);
          $new_example = 0;
        }
        if($idiom != null){
          $idiom->print();
          $idiom->upload($conn);
        }
        $idiom = new Idiom;
        $idiom->example = [];
        $example = "";
        $new_idiom = 1;
        $idiom->idiom = trim($matches[1]);
        $idiom->meaning = trim($matches[2]);
      }else{
        preg_match($example_re, $line, $matches);
        if($matches != null){
           if($example !== "")
              array_push($idiom->example, $example);
           $new_idiom = 0;
           $new_example = 1;
           $example = trim($matches[1]);
        }else{
          preg_match($multiline_meaning_re, $line, $matches);
          //var_dump($matches);
          if($matches != null && $new_idiom){
            $idiom->meaning = $idiom->meaning . ' ' . trim($matches[1]);
            //echo($idiom->meaning);
          } elseif ($matches != null && $new_example) {
            $example = $example . ' ' . trim($matches[1]);
          } else {
            $new_idiom   = 0;
            $new_example = 0;
          }
        }
      }
    }
    echo("Closing file");
    fclose($file);

    mysqli_close($conn);

?>

100

101

102

<?php

$servername = "";

$database = "";

$username = "";

$password = "";

$conn = mysqli_connect($servername, $username, $password, $database);

// Check connection

if (!$conn) {

die("Connection failed: " . mysqli_connect_error());

}

class Idiom {

var $idiom = "";

var $meaning = "";

var $example = [];

function print(){

echo($this->idiom . "<br>" . $this->meaning . "<ul>");

print_r($this->example);

if(count($this->example) == 0 )

echo("ERROR!!!!. There should be at least one example");

foreach ($this->example as $value) {

echo("<li>" . $value . "</li>");

}

echo("</ul>");

}

function upload($conn){

echo("Uploading...");

$example = "<ul class='list-group'>";

foreach ($this->example as $value) {

if($value !== '')

$example = $example . "<li class='list-group-item'>" . $value . '</li>';

}

$example = $example . "</ul>";

$idiom = mysqli_real_escape_string($conn, $this->idiom);

$meaning = mysqli_real_escape_string($conn, $this->meaning);

$example = mysqli_real_escape_string($conn, $example);

$query = 'INSERT INTO idioms (`idiom`, `meaning`, `example`) VALUES ("'. $idiom . '","'. $meaning . '","'. $example . '")';

mysqli_query($conn, $query);

}

$file = fopen("idioms.txt", "r") or die("Unable to open file!");

$idiom_meaning_re = '/^([A-za-z,. -\/()\']+):([A-za-z ,"().\';:\n]+)/';

$multiline_meaning_re = '/^([A-za-z ,"().\';:]+)/';

$example_re = '/^\|--([A-Za-z0-9 \',?.-;$\n"]+[^:])/';

$new_idiom = 0;

$new_example = 0;

$idiom = null;

$example = "";

// Output one line until end-of-file

while(!feof($file)) {

$line = fgets($file);

//echo($line . "<br>");

preg_match($idiom_meaning_re, $line, $matches);

if($matches != null){

if($new_example === 1){

array_push($idiom->example, $example);

$new_example = 0;

}

if($idiom != null){

$idiom->print();

$idiom->upload($conn);

}

$idiom = new Idiom;

$idiom->example = [];

$example = "";

$new_idiom = 1;

$idiom->idiom = trim($matches[1]);

$idiom->meaning = trim($matches[2]);

}else{

preg_match($example_re, $line, $matches);

if($matches != null){

if($example !== "")

array_push($idiom->example, $example);

$new_idiom = 0;

$new_example = 1;

$example = trim($matches[1]);

}else{

preg_match($multiline_meaning_re, $line, $matches);

//var_dump($matches);

if($matches != null && $new_idiom){

$idiom->meaning = $idiom->meaning . ' ' . trim($matches[1]);

//echo($idiom->meaning);

} elseif ($matches != null && $new_example) {

$example = $example . ' ' . trim($matches[1]);

} else {

$new_idiom = 0;

$new_example = 0;

}

echo("Closing file");

fclose($file);

mysqli_close($conn);

You can find further information about the preg_match() in the PHP official documentation.

Introduction to D3 (Data-Driven Document)

One of the first questions you may ask yourself when getting introduced in D3 is: why are we using selectAll(‘html-tag-name’) method if there is no item to select of that type?

First, let’s see an example of the situation we are talking about:

<body>
  <script>
    const dataset = [12, 31, 22, 17, 25, 18, 29, 14, 9];
    
    // Add your code below this line
    d3.select('body')
      .selectAll('h2')
      .data(dataset)
      .enter()
      .append('h2')
      .text('New Title')
    // Add your code above this line
  </script>
</body>

<body>

const dataset = [12, 31, 22, 17, 25, 18, 29, 14, 9];

// Add your code below this line

d3.select('body')

.selectAll('h2')

.data(dataset)

.enter()

.append('h2')

.text('New Title')

// Add your code above this line

</script>

</body>

In the previous example, the only existing HTML tag is <body> . We select body ( d3.select('body') ) and then we perform the .selectAll('h2') . On its own, it makes no sense as the method won’t return any value since no <h2> tag exists. Nevertheless, it will make sense if we keep looking at the following code.

After the select .selectAll('h2') we attach the existing dataset to the selected items ( .data(dataset) ). Then, we use the enter() method, which gives meaning to the previous .selectAll('h2') . When using enter() , D3 looks for the number of selected items to bind them with the data. In case of having not enough items in the selection, the enter() method will create them.

Therefore, as .selectAll('h2') was empty and the dataset variable contains 9 elements, it will iterate the code 9 times. In case of having already created some <h2> elements, it will simply fulfill the HTML code the necessary iteration to cover all the dataset elements. Remember that who does this iteration is the data() method. It parses the data set, and any method that’s chained after data() is run once for each item in the data set.

You can find more information in the official documentation at the D3js.org website.

Scales

In D3 there exists the Scale function to change the value of the data set so that it can fit in the screen. Two important methods are range() and domain(). The domain method covers the set of input values whereas the range function convers the set of output values. Let’s see an example:

const scale = d3.scaleLinear();
scale.domain([50, 480]);
scale.range([10, 500]);

scale(50) // Returns 10
scale(480) // Returns 500
scale(325) // Returns 323.37
scale(750) // Returns 807.67

const scale = d3.scaleLinear();

scale.domain([50, 480]);

scale.range([10, 500]);

scale(50) // Returns 10

scale(480) // Returns 500

scale(325) // Returns 323.37

scale(750) // Returns 807.67

From freeCodeCamp:

The domain() method passes information to the scale about the raw data values for the plot. The range() method gives it information about the actual space on the web page for the visualization.

Send audio from mobile phone (Android/iOS) to Raspberry

Platform used: Raspberry Pi 3 B+
Bluetooth module: built-in
First, update firmware to make sure you have latest version:

sudo rpi-update

1	sudo rpi-update

Configure the bluetooth in the Raspberry:

pi@raspberrypi:~ $ bluetoothctl
[NEW] Controller B8:27:EB:DB:58:46 raspberrypi [default]
[NEW] Device C0:EE:FB:44:43:9A Noname
[bluetooth]# power on
Changing power on succeeded
[bluetooth]# agent on
Agent registered
[bluetooth]# default-agent
Default agent request successful
[bluetooth]# discoverable on
Changing discoverable on succeeded
[CHG] Controller B8:27:EB:DB:58:46 Discoverable: yes
[bluetooth]# pairable on
Changing pairable on succeeded

pi@raspberrypi:~ $ bluetoothctl

[NEW] Controller B8:27:EB:DB:58:46 raspberrypi [default]

[NEW] Device C0:EE:FB:44:43:9A Noname

[bluetooth]# power on

Changing power on succeeded

[bluetooth]# agent on

Agent registered

[bluetooth]# default-agent

Default agent request successful

[bluetooth]# discoverable on

Changing discoverable on succeeded

[CHG] Controller B8:27:EB:DB:58:46 Discoverable: yes

[bluetooth]# pairable on

Changing pairable on succeeded

In your mobile phone, search for the raspberry bluetooth signal and pair to it:

[Noname]# connect C0:EE:FB:44:43:9A
Attempting to connect to C0:EE:FB:44:43:9A
[CHG] Device C0:EE:FB:44:43:9A ServicesResolved: yes
Connection successful
[Noname]# trust C0:EE:FB:44:43:9A
Changing C0:EE:FB:44:43:9A trust succeeded
[CHG] Controller B8:27:EB:DB:58:46 Discoverable: no
[Noname]# exit

[Noname]# connect C0:EE:FB:44:43:9A

Attempting to connect to C0:EE:FB:44:43:9A

[CHG] Device C0:EE:FB:44:43:9A ServicesResolved: yes

Connection successful

[Noname]# trust C0:EE:FB:44:43:9A

Changing C0:EE:FB:44:43:9A trust succeeded

[CHG] Controller B8:27:EB:DB:58:46 Discoverable: no

[Noname]# exit

Now, you can try to play some audio in your mobile and it should be reproduced in the the Raspberry.
In my case, audio was driven out to the HDMI. In case you want to switch it through the Jack 3.5 mm output you can run:

sudo raspi-config

1	sudo raspi-config

Then, select Advanced Options > Audio > Force 3.5 mm (‘headphone’) jack
With this all set, you can stream any audio such as the built-in music player, Spotify or YouTube to the Raspberry and from it to the connected speakers or HDMI display.

Finally, as the device has been stored as a trusted device, every time the Raspberry is booted, you won’t need to repeat this process. It will be so easy as connecting your mobile phone to the available raspberry bluetooth signal.

Handling with several devices

If you pair several devices, only first connected device will be able to stream audio. If I connect my computer to the raspberry and then I try to connect my mobile phone (both previously paired and trusted), phone connection will fail. First, I’ll need to disconenct my computer and only then I’ll be able to successfully connect to the raspberry with my mobile phone.

Web scraper with Scrapy

#!/usr/bin/env python
# -*- coding: utf-8 -*-

import scrapy
import re

title_regex       = r'Letra de\s+([a-zA-Z0-9áéíóúñü_,!¡¿?"() ]+)\s-'
empty_lines_regex = r"^\s+$"
tabs_regex        = r"^[\n\t]+"

class ConchaPiquerSpider(scrapy.Spider):
    name = 'conchitabot'
    allowed_domain = ['http://www.coveralia.com']
    start_urls = ['http://www.coveralia.com/letras-de/concha-piquer.php']
    custom_settings = {
        'FEED_EXPORT_ENCODING': 'utf-8',
    }
    BASE_URL = 'http://www.coveralia.com'
    def parse(self, response):
        lyric_links = response.css(".lista_uno li a::attr(href)").extract()
        for link in lyric_links:
            absolute_url = self.BASE_URL + link
            yield scrapy.Request(absolute_url, callback=self.parse_lyric)
        lyric_names_raw = response.css(".lista_uno li a::text").extract()


    def parse_lyric(self,response):
        raw_titles = response.css("h1").extract()
        for raw_title in raw_titles:
            match = re.search(title_regex, raw_title.encode("utf-8"))
            if match:
                title = match.group(1)
        raw_text = response.css("#HOTWordsTxt::text").extract()
        encoded_text = []
        single_string = ""
        for item_text in raw_text:
            single_string = single_string + item_text

        lyric = self.clean_lyric(single_string)

        text_file = open("./letras/" + title + ".txt", "w")
        text_file.write(lyric)
        text_file.close()

    def clean_lyric(self,dirty_str):
        encoded = dirty_str.encode("utf-8")
        no_spaces = re.sub(r"^\s+", '', encoded)
        no_tabs = re.sub(r"[\n\t]+", '', no_spaces)
        return no_tabs

#!/usr/bin/env python

# -*- coding: utf-8 -*-

import scrapy

import re

title_regex = r'Letra de\s+([a-zA-Z0-9áéíóúñü_,!¡¿?"() ]+)\s-'

empty_lines_regex = r"^\s+$"

tabs_regex = r"^[\n\t]+"

class ConchaPiquerSpider(scrapy.Spider):

name = 'conchitabot'

allowed_domain = ['http://www.coveralia.com']

start_urls = ['http://www.coveralia.com/letras-de/concha-piquer.php']

custom_settings = {

'FEED_EXPORT_ENCODING': 'utf-8',

}

BASE_URL = 'http://www.coveralia.com'

def parse(self, response):

lyric_links = response.css(".lista_uno li a::attr(href)").extract()

for link in lyric_links:

absolute_url = self.BASE_URL + link

yield scrapy.Request(absolute_url, callback=self.parse_lyric)

lyric_names_raw = response.css(".lista_uno li a::text").extract()

def parse_lyric(self,response):

raw_titles = response.css("h1").extract()

for raw_title in raw_titles:

match = re.search(title_regex, raw_title.encode("utf-8"))

if match:

title = match.group(1)

raw_text = response.css("#HOTWordsTxt::text").extract()

encoded_text = []

single_string = ""

for item_text in raw_text:

single_string = single_string + item_text

lyric = self.clean_lyric(single_string)

text_file = open("./letras/" + title + ".txt", "w")

text_file.write(lyric)

text_file.close()

def clean_lyric(self,dirty_str):

encoded = dirty_str.encode("utf-8")

no_spaces = re.sub(r"^\s+", '', encoded)

no_tabs = re.sub(r"[\n\t]+", '', no_spaces)

return no_tabs

#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import re

def get_sorted_files(Directory):
    filenamelist = []
    for root, dirs, files in os.walk(Directory):
        for name in files:
            fullname = os.path.join(root, name)
            filenamelist.append(fullname)
    return sorted(filenamelist)

text = "<head><meta charset='utf-8'>"
folder = "./letras/"
files = get_sorted_files(folder)
for filename in files:
    filebase = re.sub(folder, "", filename)
    filebase = re.sub("\..*$", "", filebase)
    with open(filename,'r') as f:
        text = text + "<h1>" + filebase + "</h1><pre>" + f.read() + "</pre>"

unified = open("unified.html", "w")
unified.write(text)
unified.close()

#!/usr/bin/env python

# -*- coding: utf-8 -*-

import os

import re

def get_sorted_files(Directory):

filenamelist = []

for root, dirs, files in os.walk(Directory):

for name in files:

fullname = os.path.join(root, name)

filenamelist.append(fullname)

return sorted(filenamelist)

text = "<head><meta charset='utf-8'>"

folder = "./letras/"

files = get_sorted_files(folder)

for filename in files:

filebase = re.sub(folder, "", filename)

filebase = re.sub("\..*$", "", filebase)

with open(filename,'r') as f:

text = text + "<h1>" + filebase + "</h1><pre>" + f.read() + "</pre>"

unified = open("unified.html", "w")

unified.write(text)

unified.close()

Inner product in vector space

The inner product is an operation that measures the similarity between vectors. In a general way, the inner product could be defined as an operation of 2 operands, which are elements of a vector space. The result is a scalar in the set of the complex numbers:

\[ \left \langle \cdot, \cdot \right \rangle : V \times V \rightarrow \mathbb{C} \]

Formal properties

For $x, y, z \in V$ and $\alpha \in \mathbb{C}$, the inner product must fulfill the following rules:

To be distributive to vector addition:

$ \left \langle x+y, z \right \rangle = \left \langle x, z \right \rangle + \left \langle y, z \right \rangle $

Conmutative with conjugate (applies when vectors are complex):

$ \left \langle x,y \right \rangle = \left \langle y, x \right \rangle^* $

Distributive respect scalar multiplication:

$ \left \langle \alpha x, y \right \rangle = \alpha^* \left \langle x, u \right \rangle $

$ \left \langle x, \alpha y \right \rangle = \alpha \left \langle x, u \right \rangle $

The self inner product must be necessarily a real number:

$ \left \langle x, x \right \rangle \geq 0 $

The self inner product can be zero only when the element is the null element:

$ \left \langle x,x \right \rangle = 0 \Leftrightarrow x = 0 $

Inner product in $\mathbb{R}^2 $

The inner product in $ \mathbb{R}^2$ is defined as follows:

$ \left \langle x, y \right \rangle = x_0 y_0 + x_1 y_1 $

In self inner product represents the squared norm of the vector:

$ \left \langle x, x \right \rangle = x^2_0 + y^2_0 = \left \| x \right \|^2 $

Inner product in finite length signals

In this case, the inner product is defined as:

\[ \left \langle x ,y \right \rangle = \sum_{n= 0}^{N-1} x^*[n] y[n] \]

Properties of vector spaces

Vector spaces must meet the following rules:
Addition to be commutative:
$ x + y = y + x $

Addition to be distributive:
$ (x+y)+z = x + (y + z) $

Scalar multiplication to be distributive with respect to vector addition:
$ \alpha\left(x + y \right) = \alpha x + \alpha y$

Scalar multiplication to be distributive with respect to vector the addition of field scalars:
$ \left( \alpha + \beta \right) x = \alpha x + \beta y $

Scalar multiplication to be associative:
$ \alpha\left(\beta x \right) = \left(\alpha \beta \right) x $

It must exist a null element:
$ \exists 0 \in V \ \ | \ \ x + 0 = 0 + x = x $

It must exist an inverse element for every element in the vector space:
$ \forall x \in V \exists (-x)\ \ | \ \ x + (-x) = 0$

The uvm_object class

The uvm_object class is the base class for all UVM classes. From it, all the rest of classes are extended. It provides basic functionalities such as print, compare, copy and similar methods.

This class can be used when defining reusable parts of a sequence items. For example, in a packet like uvm_sequence_item, we could define a uvm_object extended object for defining the header. This would be:

class packet_header extends uvm_object;

   rand bit [2:0] len;
   rand bit [2:0] addr;

   `uvm_object_utils_begin(packet_header)
      `uvm_field_int(len, UVM_DEFAULT)
      `uvm_field_int(addr, UVM_DEFAULT)
   `uvm_object_utils_end

   function new (string name="packet_header");
      super.new(name);
   endfunction : new

endclass : packet_header

class packet_header extends uvm_object;

rand bit [2:0] len;

rand bit [2:0] addr;

`uvm_object_utils_begin(packet_header)

`uvm_field_int(len, UVM_DEFAULT)

`uvm_field_int(addr, UVM_DEFAULT)

`uvm_object_utils_end

function new (string name="packet_header");

super.new(name);

endfunction : new

endclass : packet_header

This packet_header could be included in a packet class for conforming the uvm_sequence_item (the transaction) which will compose the sequences:

class simple_packet extends uvm_sequence_item;

   rand packet_header header;
   rand bit [4:0] payload;

   `uvm_object_utils_begin(simple_packet)
      `uvm_field_object(header, UVM_DEFAULT)
      `uvm_field_int(payload, UVM_DEFAULT)
   `uvm_object_utils_end

   function new (string name = "simple_packet");
      super.new(name);
      header = packet_header::type_id::create("header");
   endfunction : new

endclass : packet

class simple_packet extends uvm_sequence_item;

rand packet_header header;

rand bit [4:0] payload;

`uvm_object_utils_begin(simple_packet)

`uvm_field_object(header, UVM_DEFAULT)

`uvm_field_int(payload, UVM_DEFAULT)

`uvm_object_utils_end

function new (string name = "simple_packet");

super.new(name);

header = packet_header::type_id::create("header");

endfunction : new

endclass : packet

Pseudo-random number generator with Fibonacci sequence

\[ s_k = (k\cdot A) \bmod B\]

$s_k$ is the pseudo-random number and $A$ and $B$ are prime numbers. $k$ is in the range $[0,B-1]$. If $k$ is greater than $B-1$, the results will be repeat as $B$ is the period of the sequence.
For example, $A = 7$ and $B = 17$. This sequence written in MATLAB could be:

A = 7;
B = 17;
n = [];

t = 0:B;
for i=0:B
    n = [n mod(i*A,B)];
end

stem(t,n);

A = 7;

B = 17;

n = [];

t = 0:B;

for i=0:B

n = [n mod(i*A,B)];

end

stem(t,n);

Cordic in MATLAB

Let’s z be a 2D point in the space as $z = x + jy$, if we want to rotate this point a given angle $\theta$, we get the following expressions:
\[e^{j\theta} \cdot z = \left(\cos{\theta} + j \sin{\theta}\right)\left(x+jy\right) \\ = x\cos{\theta}-y\sin{\theta} + j \left(y \cos{\theta} + x \sin{\theta} \right) \\ = x’ + j y’ \]

Then, for a generic point, the rotation can be expressed as an equation system, where $x’$ and $y’$ are the new coordinates, $\theta$ is the rotation angle and $x$ and $y$ are the original coordinates:
\[\begin{bmatrix}
x’\\
y’
\end{bmatrix}=
\begin{bmatrix}
\cos{\theta} & -\sin{\theta}\\
\sin{\theta} & \cos{\theta}
\end{bmatrix}\begin{bmatrix}
x\\
y
\end{bmatrix} \]

This rotation can be coded in MATLAB as:

%% Function to rotate vector
function v = rotate(P, theta)
    rot = [cos(theta) -sin(theta);
           sin(theta) cos(theta)];
    v = rot*P;
end

%% Function to rotate vector

function v = rotate(P, theta)

rot = [cos(theta) -sin(theta);

sin(theta) cos(theta)];

v = rot*P;

end

A possible implementation of the cordic algorithm could be:

%% Clear all previous values
clear all;

%% Define vectors
A = [2;-3];
O = [0;0];

%% Define accuracy
error_limit = 0.01;

%% Initialize variables
start_angle = pi/2; % 90º
current_angle = start_angle;
acc_angle = 0;

A_rotated = A;
steps = 0;

% Second quadrant (90º - 180º)
if(A_rotated(1) < 0 && A_rotated(2) > 0)
    A_rotated = rotate(A_rotated, -pi/2);
    acc_angle = pi/2;
% Third quadrant (180º - 270º)
elseif(A_rotated(1) < 0 && A_rotated(2) < 0)
    A_rotated = rotate(A_rotated, -pi);
    acc_angle = pi;
% Forth quadrant (270º - 360º)
elseif(A_rotated(1) > 0 && A_rotated(2) < 0)
    A_rotated = rotate(A_rotated, -3*pi/2);
    acc_angle = 3*pi/2;    
end


%% Compute angle
% Keep rotating while error is too high
while(abs(A_rotated(2)) > error_limit)
    % Represent current vector
    quiver(0, 0,A_rotated(1), A_rotated(2));
    % Keep previous vectors
    hold on;
    % Decrease angle rotation
    current_angle = current_angle/2;
    % (For debugging purposes)
    current_angle_deg = current_angle*180/pi;
    % Save current error
    error = A_rotated(2);
    % If y coordinate is still positive
    if(error > 0)
        % Rotate again conterclockwise
        A_rotated = rotate(A_rotated, -1*current_angle);
        % Accumulate rotated angle
        acc_angle = acc_angle + current_angle;
    % If y coordinate is negative
    else
        % Rotate vector clockwise
        A_rotated = rotate(A_rotated, current_angle);
        % Substract current angle to the accumulator because we have
        % overcome the actual angle value
        acc_angle = acc_angle - current_angle;
    end
    
    % (For debugging purposes)
    acc_angle_deg = acc_angle*180/pi;
    % Increase step counter
    steps = steps + 1;
end

% Print angle, hypotenuse length and number of steps needed to compute
fprintf('Angle = %f\nHypotenuse = %f\nNumber of steps: %d\n', acc_angle*180/pi, A_rotated(1), steps);


%% Function to rotate vector
function v = rotate(P, theta)
    rot = [cos(theta) -sin(theta);
           sin(theta) cos(theta)];
    v = rot*P;
end

%% Clear all previous values

clear all;

%% Define vectors

A = [2;-3];

O = [0;0];

%% Define accuracy

error_limit = 0.01;

%% Initialize variables

start_angle = pi/2; % 90º

current_angle = start_angle;

acc_angle = 0;

A_rotated = A;

steps = 0;

% Second quadrant (90º - 180º)

if(A_rotated(1) < 0 && A_rotated(2) > 0)

A_rotated = rotate(A_rotated, -pi/2);

acc_angle = pi/2;

% Third quadrant (180º - 270º)

elseif(A_rotated(1) < 0 && A_rotated(2) < 0)

A_rotated = rotate(A_rotated, -pi);

acc_angle = pi;

% Forth quadrant (270º - 360º)

elseif(A_rotated(1) > 0 && A_rotated(2) < 0)

A_rotated = rotate(A_rotated, -3*pi/2);

acc_angle = 3*pi/2;

end

%% Compute angle

% Keep rotating while error is too high

while(abs(A_rotated(2)) > error_limit)

% Represent current vector

quiver(0, 0,A_rotated(1), A_rotated(2));

% Keep previous vectors

hold on;

% Decrease angle rotation

current_angle = current_angle/2;

% (For debugging purposes)

current_angle_deg = current_angle*180/pi;

% Save current error

error = A_rotated(2);

% If y coordinate is still positive

if(error > 0)

% Rotate again conterclockwise

A_rotated = rotate(A_rotated, -1*current_angle);

% Accumulate rotated angle

acc_angle = acc_angle + current_angle;

% If y coordinate is negative

else

% Rotate vector clockwise

A_rotated = rotate(A_rotated, current_angle);

% Substract current angle to the accumulator because we have

% overcome the actual angle value

acc_angle = acc_angle - current_angle;

end

% (For debugging purposes)

acc_angle_deg = acc_angle*180/pi;

% Increase step counter

steps = steps + 1;

end

% Print angle, hypotenuse length and number of steps needed to compute

fprintf('Angle = %f\nHypotenuse = %f\nNumber of steps: %d\n', acc_angle*180/pi, A_rotated(1), steps);

%% Function to rotate vector

function v = rotate(P, theta)

rot = [cos(theta) -sin(theta);

sin(theta) cos(theta)];

v = rot*P;

end

I have coded an interactive applet to illustrate the algorithm. It has been done using the p5.js library. The error limit has been set to $0.5$.

Scales

Handling with several devices

Formal properties

Inner product in \(\mathbb{R}^2 \)

Inner product in finite length signals