in Functions/FunctionCode/Twitter/TweetProcessingFunction/TweetFunctionCSharp.cs [131:320]
public async Task<bool> ParseTweet(string entireTweet, TraceWriter log, int i)
{
// Convert JSON to dynamic C# object
tweetObj = JObject.Parse(entireTweet);
tweet = tweetObj;
//Connect to Azure SQL Database & bring in Twitter Handles & IDs
string twitterHandles =
ExecuteSqlQuery("select value FROM pbist_twitter.configuration where name = \'twitterHandle\'", "value");
string twitterHandleId =
ExecuteSqlQuery("select value FROM pbist_twitter.configuration where name = \'twitterHandleId\'",
"value");
if (i == 0)
{
ExecuteSqlNonQuery($"UPDATE pbist_twitter.twitter_query SET TweetId='{tweet["TweetId"]}' WHERE Id = 1");
}
// Split out all the handles & create dictionary
String[] handle = null;
String[] handleId = null;
var dictionary = new Dictionary<string, string>();
if (twitterHandles != String.Empty)
{
handle = SplitHandles(twitterHandles, ',');
handleId = SplitHandles(twitterHandleId, ',');
for (int index = 0; index < handle.Length; index++)
{
dictionary.Add(handle[index], handleId[index]);
}
}
//log.Info("********************ParseTweet************************** TweetText: " + tweet.TweetText.ToString());
//log.Info("********************ParseTweet************************** Handles: " + twitterHandles);
//log.Info("********************ParseTweet************************** Handle IDs: " + twitterHandleId);
//log.Info("********************ParseTweet************************** Tweet Language: " + tweet.TweetLanguageCode.ToString());
// Check if language of tweet is supported for sentiment analysis
originalTweets["lang"] = tweet.TweetLanguageCode.ToString();
originalTweets["sentimentPosNeg"] = "Undefined";
originalTweets["sentiment"] = null;
originalTweets["sentimentBin"] = null;
string sentiment = await MakeSentimentRequest(tweet);
if (sentiment != null)
{
sentiment = (double.Parse(sentiment) * 2 - 1).ToString(CultureInfo.InvariantCulture);
string sentimentBin = (Math.Floor(double.Parse(sentiment) * 10) / 10).ToString(CultureInfo.InvariantCulture);
string sentimentPosNeg = String.Empty;
if (double.Parse(sentimentBin) > 0.1)
{
sentimentPosNeg = "Positive";
}
else if (double.Parse(sentimentBin) < -0.1)
{
sentimentPosNeg = "Negative";
}
else
{
sentimentPosNeg = "Neutral";
}
//Save sentiment and language metadata into dictionary
originalTweets["sentiment"] = sentiment;
originalTweets["sentimentBin"] = sentimentBin;
originalTweets["sentimentPosNeg"] = sentimentPosNeg;
}
// Work out account and tweet direction for retweets
if (tweet.OriginalTweet != null)
{
processedTweets["direction"] = "Text Retweet";
originalTweets["twitterhandle"] = tweet.OriginalTweet.UserDetails.UserName;
originalTweets["userlocation"] = tweet.OriginalTweet.UserDetails.Location;
if (dictionary.Count > 0)
{
foreach (var entry in dictionary)
{
HashtagDirectionCheck(entry, " Retweet");
MessageDirectionCheck(entry, tweetObj.SelectToken("OriginalTweet.UserMentions"), " Retweet");
}
}
// Save retweets into SQL table
saveTweets(tweet.OriginalTweet);
}
// Works out the tweet direction for original tweets (not retweets)
else
{
originalTweets["twitterhandle"] = tweet.UserDetails.UserName;
originalTweets["userlocation"] = tweet.UserDetails.Location;
if (dictionary.Count > 0)
{
foreach (var entry in dictionary)
{
HashtagDirectionCheck(entry);
MessageDirectionCheck(entry, tweetObj.SelectToken("UserMentions"));
}
}
// Save original tweets into SQL Table
saveTweets(tweet);
}
processedTweets["tweetid"] = tweet.TweetId;
//Save time metadata about processed tweets
string createdat = tweet.CreatedAt.ToString();
DateTime ts = DateTime.ParseExact(createdat, "ddd MMM dd HH:mm:ss +ffff yyyy", CultureInfo.CurrentCulture);
processedTweets["dateorig"] = DateTime.Parse(ts.Year.ToString() + " " + ts.Month.ToString() + " " + ts.Day.ToString() + " " + ts.Hour.ToString() + ":" + ts.Minute.ToString() + ":" + ts.Second.ToString()).ToString(CultureInfo.InvariantCulture);
processedTweets["minuteofdate"] = DateTime.Parse(ts.Year.ToString() + " " + ts.Month.ToString() + " " + ts.Day.ToString() + " " + ts.Hour.ToString() + ":" + ts.Minute.ToString() + ":00").ToString(CultureInfo.InvariantCulture);
processedTweets["hourofdate"] = DateTime.Parse(ts.Year.ToString() + " " + ts.Month.ToString() + " " + ts.Day.ToString() + " " + ts.Hour.ToString() + ":00:00").ToString(CultureInfo.InvariantCulture);
//Save media and follower metadata about processed tweets
processedTweets["authorimage_url"] = tweet.UserDetails.ProfileImageUrl;
processedTweets["username"] = tweet.UserDetails.UserName;
processedTweets["userlocation"] = tweet.UserDetails.Location;
processedTweets["user_followers"] = tweet.UserDetails.FollowersCount;
processedTweets["user_friends"] = tweet.UserDetails.FavouritesCount;
processedTweets["user_favorites"] = tweet.UserDetails.FriendsCount;
processedTweets["user_totaltweets"] = tweet.UserDetails.StatusesCount;
string firstUrl = String.Empty;
if (tweetObj.SelectToken("MediaUrls") != null && tweetObj.SelectToken("MediaUrls").HasValues)
{
firstUrl = tweet.MediaUrls[0];
if (firstUrl != String.Empty)
{
processedTweets["image_url"] = firstUrl;
}
}
if (tweet.favorited != "true")
{
processedTweets["favorited"] = "1";
}
if (tweet.OriginalTweet != null)
{
processedTweets["retweet"] = "True";
}
//Save processed tweets into SQL
int response = 0;
response = ExecuteSqlScalar(
$"Select count(1) FROM pbist_twitter.tweets_processed WHERE tweetid = '{processedTweets["tweetid"]}'");
if (response == 0)
{
try
{
ExecuteSqlNonQuery(generateSQLQuery("pbist_twitter.tweets_processed", processedTweets));
}
catch (Exception e) { }
}
string text = tweet.TweetText.ToString();
//Populate hashtag slicer table
if (text.Contains("#"))
{
hashtagSlicer["tweetid"] = tweet.TweetId;
hashtagmentions(text, '#', "facet", "pbist_twitter.hashtag_slicer", hashtagSlicer);
}
//Populate author hashtag network table
if (text.Contains("#"))
{
authorHashtagGraph["tweetid"] = tweet.TweetId;
authorHashtagGraph["author"] = tweet.UserDetails.UserName;
hashtagmentions(text, '#', "hashtag", "pbist_twitter.authorhashtag_graph", authorHashtagGraph);
}
//Populate mention slicer table
if (text.Contains("@"))
{
mentionSlicer["tweetid"] = tweet.TweetId;
hashtagmentions(text, '@', "facet", "pbist_twitter.mention_slicer", mentionSlicer);
}
//Populate author mention network table
if (text.Contains("@"))
{
authorMentionGraph["tweetid"] = tweet.TweetId;
authorMentionGraph["author"] = tweet.UserDetails.UserName;
hashtagmentions(text, '@', "mention", "pbist_twitter.authormention_graph", authorMentionGraph);
}
return true;
}