Big Data
ajayreddy428
Rcodes.docxQuiz2
# comments start with #
# to quit q()
# two steps to install any library
#install.packages("rattle")
#library(rattle)
setwd("D:/AJITH/CUMBERLANDS/Ph.D/SEMESTER 3/Data Science & Big Data Analy (ITS-836-51)/RStudio/Week2")
getwd()
x <- 3 # x is a vector of length 1
print(x)
v1 <- c(2,4,6,8,10)
print(v1)
print(v1[3])
v <- c(1:10) #creates a vector of 10 elements numbered 1 through 10. More complicated data
print(v)
print(v[6])
# Import test data
test<-read.csv("CVEs.csv")
test1<-read.csv("CVEs.csv", sep=",")
test2<-read.table("CVEs.csv", sep=",")
write.csv(test2, file="out.csv")
# Write CSV in R
write.table(test1, file = "out1.csv",row.names=TRUE, na="",col.names=TRUE, sep=",")
head(test)
tail(test)
summary(test)
head <- head(test)
tail <- tail(test)
cor(test$X, test$index)
sd(test$index)
var(test$index)
plot(test$index)
hist(test$index)
str(test$index)
quit()
Quiz3
setwd("C:/Users/ialsmadi/Desktop/University_of_Cumberlands/Lectures/Week2/RScripts")
getwd()
# Import test data
data<-read.csv("yearly_sales.csv")
#A 5-number summary is a set of 5 descriptive statistics for summarizing a continuous univariate data set.
#It consists of the data set's: minimum, 1st quartile, median, 3rd quartile, maximum
#Find the set, L, of data below the median. The 1st quartile is the median of L.
#Find the set, U, of data above the median. The 3rd quartile is the median of U.
print(summary(data))
anscombe<-read.csv("anscombe.csv")
print(summary(anscombe))
sd(anscombe$X)
var(anscombe$X)
sd(anscombe$x1)
var(anscombe$x1)
sd(anscombe$x2)
var(anscombe$x2)
sd(anscombe$x3)
var(anscombe$x3)
sd(anscombe$x4)
var(anscombe$x4)
sd(anscombe$y1)
var(anscombe$y1)
sd(anscombe$y2)
var(anscombe$y2)
sd(anscombe$y3)
var(anscombe$y3)
##-- now some "magic" to do the 4 regressions in a loop:
ff <- y ~ x
mods <- setNames(as.list(1:4), paste0("lm", 1:4))
for(i in 1:4) {
ff[2:3] <- lapply(paste0(c("y","x"), i), as.name)
## or ff[[2]] <- as.name(paste0("y", i))
## ff[[3]] <- as.name(paste0("x", i))
mods[[i]] <- lmi <- lm(ff, data = anscombe)
print(anova(lmi))
}
## See how close they are (numerically!)
sapply(mods, coef)
lapply(mods, function(fm) coef(summary(fm)))
## Now, do what you should have done in the first place: PLOTS
op <- par(mfrow = c(2, 2), mar = 0.1+c(4,4,1,1), oma = c(0, 0, 2, 0))
for(i in 1:4) {
ff[2:3] <- lapply(paste0(c("y","x"), i), as.name)
plot(ff, data = anscombe, col = "red", pch = 21, bg = "orange", cex = 1.2,
xlim = c(3, 19), ylim = c(3, 13))
abline(mods[[i]], col = "blue")
}
mtext("Anscombe's 4 Regression data sets", outer = TRUE, cex = 1.5)
par(op)
plot(sort(data$num_of_orders))
hist(sort(data$num_of_orders))
plot(density(sort(data$num_of_orders)))
plot(sort(data$gender))
hist(sort(data$sales_total))
plot(density(sort(data$sales_total)))
library(lattice)
densityplot(data$num_of_orders)
# top plot
# bottom plot as log10 is actually
# easier to read, but this plot is in natural log
densityplot(log(data$num_of_orders))
densityplot(data$sales_total)
densityplot(log(data$sales_total))
hist(data$sales_total, breaks=100, main="Sales total",
xlab="sales", col="gray")
# draw a line for the media
abline(v = median(data$sales_total), col = "magenta", lwd = 4)
# use rug() function to see the actual datapoints
rug(data$sales_total)
#Boxplots can be created for individual variables or for variables by group.
#The format is boxplot(x, data=), where x is a formula and data= denotes the data frame providing
#the data.
boxplot(data$sales_total,data=data, main="Dis by Sales",
xlab="Sales", ylab="Total")
# Boxplot of MPG by Car Cylinders, using one of R built-in datasets
boxplot(mpg~cyl,data=mtcars, main="Car Milage Data",
xlab="Number of Cylinders", ylab="Miles Per Gallon")
#in our boxplot above, we might want to draw a horizontal line at 12 where the national standard is.
abline(h = 12)
boxplot(data$sales_total,data=data, main="Total sales Bplot",
xlab="Sales", ylab="Total")
# Dot chart of a single numeric vector
dotchart(mtcars$mpg, labels = row.names(mtcars),
cex = 0.6, xlab = "mpg")
#install.packages("ROCR")
#library(ROCR)
# Simple Scatterplot
attach(mtcars)
plot(wt, mpg, main="Scatterplot Example",
xlab="Car Weight ", ylab="Miles Per Gallon ", pch=19)
#The R function abline() can be used to add vertical, horizontal or regression lines to a graph
plot(data$sales_total, data$gender)
# Add fit lines
abline(lm(data$sales_total~ data$num_of_orders), col="red") # regression line (y~x)
lines(lowess(data$sales_total, data$num_of_orders), col="blue") # lowess line (x,y)
# Basic Scatterplot Matrix
pairs(data)
pairs(data[0:2])
# Scatterplot Matrices from the car Package
install.packages("car")
library(car)
install.packages("ggplot2")
library(ggplot2)
quit()
Quiz4
install.packages("tidyverse")
library(tidyverse) # data manipulation
install.packages("cluster")
library(cluster) # clustering algorithms
install.packages("factoextra")
library(factoextra) # clustering algorithms & visualization
setwd("C:/Users/ialsmadi/Desktop/University_of_Cumberlands/Lectures/Week3/RScripts")
getwd()
# Import test data
data<-read.csv("grades_km_input.csv")
print(summary(data))
data1 <- na.omit(data)
columns <- data[1, ]
print(summary(data))
#As we don't want the clustering algorithm to depend to an arbitrary
#variable unit, we start by scaling data using the R function scale:
data1 <- scale(data1)
head(data1)
distance <- get_dist(data1)
print(distance)
# plot cluster library
library(cluster)
# K-Means Cluster Analysis
# simplest example, just the dataset and number of clusters
fit <- kmeans(data1, 5) # 5 cluster solution
# get cluster means
aggregate(data1,by=list(fit$cluster),FUN=mean)
# append cluster assignment
mydata <- data.frame(data1, fit$cluster)
clusplot(mydata, fit$cluster, color=TRUE, shade=TRUE,
labels=2, lines=0)
fit <- kmeans(data1, 8) # 8 cluster solution
# get cluster means
aggregate(data1,by=list(fit$cluster),FUN=mean)
# append cluster assignment
mydata <- data.frame(data1, fit$cluster)
clusplot(mydata, fit$cluster, color=TRUE, shade=TRUE,
labels=2, lines=0)
# K-Means Clustering with 5 clusters
fit <- kmeans(mydata, 5)
# Determine number of clusters
wss <- (nrow(data1)-1)*sum(apply(data1,2,var))
for (i in 2:15) wss[i] <- sum(kmeans(data1,
centers=i)$withinss)
#A plot of the within groups sum of squares by number of clusters extracted can help determine the appropriate number of clusters.
#The analyst looks for a bend in the plot similar to a scree test in factor analysis
# We want (total within-cluster variation) to be the lowest
plot(1:15, wss, type="b", xlab="Number of Clusters",
ylab="Within groups sum of squares")
# Determine number of clusters
wss <- (nrow(data1)-1)*sum(apply(data1,2,var))
for (i in 2:15) wss[i] <- sum(kmeans(data1,
centers=i)$withinss)
plot(1:15, wss, type="b", xlab="Number of Clusters",
ylab="Within groups sum of squares")
# Cluster Plot against 1st 2 principal components
# vary parameters for most readable graph
library(cluster)
clusplot(mydata, fit$cluster, color=TRUE, shade=TRUE,
labels=2, lines=0)
# Centroid Plot against 1st 2 discriminant functions
library(fpc)
plotcluster(mydata, fit$cluster)
fviz_dist(distance, gradient = list(low = "#00AFBB", mid = "white", high = "#FC4E07"))
# try with 25 attempts, 2 clusters
km <- kmeans(data1, centers = 2, nstart = 25)
str(km)
#The output of kmeans is a list with several bits of information. The most important being:
# cluster: A vector of integers (from 1:k) indicating the cluster to which each point is allocated.
#centers: A matrix of cluster centers.
#totss: The total sum of squares.
#withinss: Vector of within-cluster sum of squares, one component per cluster.
#tot.withinss: Total within-cluster sum of squares, i.e. sum(withinss).
#betweenss: The between-cluster sum of squares, i.e. $totss-tot.withinss$.
#size: The number of points in each cluster.
# print the clusters
print(km)
# Plot clusters
fviz_cluster(km, data = data1)
(cl <- kmeans(data1, 8))
plot(data1, col = cl$cluster)
points(cl$centers, col = 1:3, pch = 8, cex = 2)
# sum of squares
ss <- function(x) sum(scale(x, scale = FALSE)^2)
## cluster centers "fitted" to each obs.:
fitted.data1 <- fitted(cl); head(fitted.data1)
resid.data1 <- data1 - fitted(cl)
## Equalities : ----------------------------------
cbind(cl[c("betweenss", "tot.withinss", "totss")], # the same two columns
c(ss(fitted.data1), ss(resid.data1), ss(data1)))
stopifnot(all.equal(cl$ totss, ss(data1)),
all.equal(cl$ tot.withinss, ss(resid.data1)),
## these three are the same:
all.equal(cl$ betweenss, ss(fitted.data1)),
all.equal(cl$ betweenss, cl$totss - cl$tot.withinss),
## and hence also
all.equal(ss(data1), ss(fitted.data1) + ss(resid.data1))
)
kmeans(data1,1)$withinss # trivial one-cluster, (its W.SS == ss(x))
## random starts do help here with too many clusters
## (and are often recommended anyway!):
(cl <- kmeans(x, 5, nstart = 25))
plot(x, col = cl$cluster)
points(cl$centers, col = 1:5, pch = 8)
Quiz5
install.packages("tidyverse")
library(tidyverse) # data manipulation
install.packages("cluster")
library(cluster) # clustering algorithms
install.packages("factoextra")
library(factoextra) # clustering algorithms & visualization
setwd("C:/Users/ialsmadi/Desktop/University_of_Cumberlands/Lectures/Week3/RScripts")
getwd()
# Import test data
data<-read.csv("grades_km_input.csv")
print(summary(data))
data1 <- na.omit(data)
columns <- data[1, ]
print(summary(data))
#As we don't want the clustering algorithm to depend to an arbitrary
#variable unit, we start by scaling data using the R function scale:
data1 <- scale(data1)
head(data1)
distance <- get_dist(data1)
print(distance)
# plot cluster library
library(cluster)
# K-Means Cluster Analysis
# simplest example, just the dataset and number of clusters
fit <- kmeans(data1, 5) # 5 cluster solution
# get cluster means
aggregate(data1,by=list(fit$cluster),FUN=mean)
# append cluster assignment
mydata <- data.frame(data1, fit$cluster)
clusplot(mydata, fit$cluster, color=TRUE, shade=TRUE,
labels=2, lines=0)
fit <- kmeans(data1, 8) # 8 cluster solution
# get cluster means
aggregate(data1,by=list(fit$cluster),FUN=mean)
# append cluster assignment
mydata <- data.frame(data1, fit$cluster)
clusplot(mydata, fit$cluster, color=TRUE, shade=TRUE,
labels=2, lines=0)
# K-Means Clustering with 5 clusters
fit <- kmeans(mydata, 5)
# Determine number of clusters
wss <- (nrow(data1)-1)*sum(apply(data1,2,var))
for (i in 2:15) wss[i] <- sum(kmeans(data1,
centers=i)$withinss)
#A plot of the within groups sum of squares by number of clusters extracted can help determine the appropriate number of clusters.
#The analyst looks for a bend in the plot similar to a scree test in factor analysis
# We want (total within-cluster variation) to be the lowest
plot(1:15, wss, type="b", xlab="Number of Clusters",
ylab="Within groups sum of squares")
# Determine number of clusters
wss <- (nrow(data1)-1)*sum(apply(data1,2,var))
for (i in 2:15) wss[i] <- sum(kmeans(data1,
centers=i)$withinss)
plot(1:15, wss, type="b", xlab="Number of Clusters",
ylab="Within groups sum of squares")
# Cluster Plot against 1st 2 principal components
# vary parameters for most readable graph
library(cluster)
clusplot(mydata, fit$cluster, color=TRUE, shade=TRUE,
labels=2, lines=0)
# Centroid Plot against 1st 2 discriminant functions
library(fpc)
plotcluster(mydata, fit$cluster)
fviz_dist(distance, gradient = list(low = "#00AFBB", mid = "white", high = "#FC4E07"))
# try with 25 attempts, 2 clusters
km <- kmeans(data1, centers = 2, nstart = 25)
str(km)
#The output of kmeans is a list with several bits of information. The most important being:
# cluster: A vector of integers (from 1:k) indicating the cluster to which each point is allocated.
#centers: A matrix of cluster centers.
#totss: The total sum of squares.
#withinss: Vector of within-cluster sum of squares, one component per cluster.
#tot.withinss: Total within-cluster sum of squares, i.e. sum(withinss).
#betweenss: The between-cluster sum of squares, i.e. $totss-tot.withinss$.
#size: The number of points in each cluster.
# print the clusters
print(km)
# Plot clusters
fviz_cluster(km, data = data1)
(cl <- kmeans(data1, 8))
plot(data1, col = cl$cluster)
points(cl$centers, col = 1:3, pch = 8, cex = 2)
# sum of squares
ss <- function(x) sum(scale(x, scale = FALSE)^2)
## cluster centers "fitted" to each obs.:
fitted.data1 <- fitted(cl); head(fitted.data1)
resid.data1 <- data1 - fitted(cl)
## Equalities : ----------------------------------
cbind(cl[c("betweenss", "tot.withinss", "totss")], # the same two columns
c(ss(fitted.data1), ss(resid.data1), ss(data1)))
stopifnot(all.equal(cl$ totss, ss(data1)),
all.equal(cl$ tot.withinss, ss(resid.data1)),
## these three are the same:
all.equal(cl$ betweenss, ss(fitted.data1)),
all.equal(cl$ betweenss, cl$totss - cl$tot.withinss),
## and hence also
all.equal(ss(data1), ss(fitted.data1) + ss(resid.data1))
)
kmeans(data1,1)$withinss # trivial one-cluster, (its W.SS == ss(x))
## random starts do help here with too many clusters
## (and are often recommended anyway!):
(cl <- kmeans(x, 5, nstart = 25))
plot(x, col = cl$cluster)
points(cl$centers, col = 1:5, pch = 8)
Quiz6
if(!require(arules)) install.packages("arules")
if(!require(arulesViz)) install.packages("arulesViz")
if(!require(dplyr)) install.packages("dplyr")
if(!require(lubridate)) install.packages("lubridate")
if(!require(ggplot2)) install.packages("ggplot2")
if(!require(knitr)) install.packages("knitr")
if(!require(RColorBrewer)) install.packages("RColorBrewer")
library(arules)
library(arulesViz)
library(dplyr)
library(plyr)
library(lubridate)
library(ggplot2)
library(knitr)
library(RColorBrewer)
setwd("D:/AJITH/CUMBERLANDS/Ph.D/SEMESTER 3/Data Science & Big Data Analy (ITS-836-51)/RStudio/Week6")
getwd()
# First example from : http://www.rpubs.com/thirus83/445115
#Other refs: https://rstudio-pubs-static.s3.amazonaws.com/252505_4931623e7ab14851b1002d1bc61e94d9.html
#https://towardsdatascience.com/association-rule-mining-in-r-ddf2d044ae50
#https://www.datacamp.com/community/tutorials/market-basket-analysis-r
#https://blog.exploratory.io/introduction-to-association-rules-market-basket-analysis-in-r-7a0dd900a3e0
#https://www.r-bloggers.com/association-rule-learning-and-the-apriori-algorithm/
#http://www.rdatamining.com/examples/association-rules
#https://github.com/natarajan1993/Market-Basket-Analysis-with-R
# https://github.com/Deepaknatural/Training/blob/master/MarketBasket_Latest.R
#https://rstudio-pubs-static.s3.amazonaws.com/267119_9a033b870b9641198b19134b7e61fe56.html
# First lets use the AdultUCI dataset that comes bundled with the arules package.
data()
data("Groceries")
summary(Groceries)
rules <- apriori(Groceries,parameter=list(support=0.002, confidence = 0.5))
print(summary(rules))
print(rules)
inspect(head(sort(rules, by = "lift")))
plot(rules)
head(quality(rules))
plot(rules, method = "grouped")
plot(rules,method = "scatterplot")
plot(rules,method = "graph")
# Import test data
df <- read.csv("OnlineRetailSmall.csv")
head(df)
df <- df[complete.cases(df), ] # Drop missing values
# Change Description and Country columns to factors
# Factors are the data objects which are used to categorize the data and store it as levels.
df %>% mutate(Description = as.factor(Description),
Country = as.factor(Country))
# Change InvoiceDate to Date datatype
df$Date <- as.Date(df$InvoiceDate)
df$InvoiceDate <- as.Date(df$InvoiceDate)
# Extract time from the InvoiceDate column
TransTime<- format(as.POSIXct(df$InvoiceDate),"%H:%M:%S")
# Convert InvoiceNo into numeric
InvoiceNo <- as.numeric(as.character(df$InvoiceNo))
# Add new columns to original dataframe
cbind(df, TransTime, InvoiceNo)
glimpse(df)
# Group by invoice number and combine order item strings with a comma
transactionData <- ddply(df,c("InvoiceNo","Date"),
function(df1)paste(df1$Description,collapse = ","))
transactionData$InvoiceNo <- NULL # Don't need these columns
transactionData$Date <- NULL
colnames(transactionData) <- c("items")
head(transactionData)
write.csv(transactionData,"market_basket_transactionsSmall1.csv", quote = FALSE, row.names = TRUE)
# MBA analysis
# From package arules
tr <- read.transactions('market_basket_transactionsSmall.csv', format = 'basket', sep=',')
summary(tr)
# plot the frequency of items
itemFrequencyPlot(tr)
itemFrequencyPlot(tr,topN=20,type="absolute",col=brewer.pal(8,'Pastel2'), main="Absolute Item Frequency Plot")
arules::itemFrequencyPlot(tr,
topN=20,
col=brewer.pal(8,'Pastel2'),
main='Relative Item Frequency Plot',
type="relative",
ylab="Item Frequency (Relative)")
# Generate the a priori rules
association.rules <- apriori(tr, parameter = list(supp=0.001, conf=0.8,maxlen=10))
summary(association.rules)
inspect(association.rules[1:10]) # Top 10 association rules
# Select rules which are subsets of larger rules -> Remove rows where the sums of the subsets are > 1
subset.rules <- which(colSums(is.subset(association.rules, association.rules)) > 1) # get subset rules in vector
# What did customers buy before buying "METAL"
metal.association.rules <- apriori(tr, parameter = list(supp=0.001, conf=0.8),appearance = list(default="lhs",rhs="METAL"))
inspect(head(metal.association.rules))
# What did customers buy after buying "METAL"
metal.association.rules2 <- apriori(tr, parameter = list(supp=0.001, conf=0.8),appearance = list(lhs="METAL",default="rhs"))
inspect(head(metal.association.rules2))
# Plotting
# Filter rules with confidence greater than 0.4 or 40%
subRules<-association.rules[quality(association.rules)$confidence>0.4]
#Plot SubRules
plot(subRules)
# Top 10 rules viz
top10subRules <- head(subRules, n = 10, by = "confidence")
plot(top10subRules, method = "graph", engine = "htmlwidget")
# Filter top 20 rules with highest lift
# Paralell Coordinates plot - visualize which products along with which items cause what kind of sales.
# Closer arrows re bought together
subRules2<-head(subRules, n=20, by="lift")
plot(subRules2, method="paracoord")
