About neohope

一直在努力,还没想过要放弃...

Redis分片(Jedis)

Redis的分片技术一般是通过客户端或代理来实现的

1、用jedis实现分片的时候,服务端不需要做任何配置即可

package com.djhu.redis.test;

import java.util.ArrayList;
import java.util.List;

import redis.clients.jedis.JedisShardInfo;
import redis.clients.jedis.ShardedJedis;

public class JedisShardTest
{
	public static void main(String[] args)
	{
		List<JedisShardInfo> jedisShardInfoList = new ArrayList<JedisShardInfo>();
		jedisShardInfoList.add(new JedisShardInfo("172.16.172.4", 6379));
		jedisShardInfoList.add(new JedisShardInfo("172.16.172.4", 6380));

		ShardedJedis sharded = new ShardedJedis(jedisShardInfoList);
		sharded.set("key01", "a");
		sharded.set("key02", "b");
		sharded.set("key03", "c");
		sharded.set("key04", "d");
		sharded.set("key05", "e");
		
		System.out.println(sharded.get("key03"));
	}
}

2、用Jedis连接池实现分片

package com.djhu.redis.test;

import java.util.ArrayList;
import java.util.List;

import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisShardInfo;
import redis.clients.jedis.ShardedJedis;
import redis.clients.jedis.ShardedJedisPool;
import redis.clients.jedis.JedisPoolConfig;
import redis.clients.util.Hashing;
import redis.clients.util.Sharded;

public class JedisSharedFactory
{
	// 最大可用连接数,默认值为8,如果赋值为-1则表示不限制
	private static int MAX_TOTAL = 256;
	// 最大空闲连接数,默认值为8
	private static int MAX_IDLE = 32;
	// 最小空闲连接数
	private static int MIN_IDLE = 4;
	// 最大等待连接毫秒数,默认值为-1表示永不超时
	private static int MAX_WAIT = 3000;
	// 连接redis超时时间
	private static int TIMEOUT = 3000;
	// true表示验证连接
	private static boolean TEST_ON_BORROW = true;

	//连接池
	private static ShardedJedisPool jedisPool = null;
	public static void initJedisPool()
	{
		try
		{
			JedisPoolConfig config = new JedisPoolConfig();
			config.setMaxTotal(MAX_TOTAL);
			config.setMaxIdle(MAX_IDLE);
			config.setMinIdle(MIN_IDLE);
			config.setMaxWaitMillis(MAX_WAIT);
			config.setTestOnBorrow(TEST_ON_BORROW);
			
			List<JedisShardInfo> jedisShardInfoList = new ArrayList<JedisShardInfo>();
			jedisShardInfoList.add(new JedisShardInfo("172.16.172.4", 6379));
			jedisShardInfoList.add(new JedisShardInfo("172.16.172.4", 6380));
			jedisPool = new ShardedJedisPool(config, jedisShardInfoList,Hashing.MURMUR_HASH,Sharded.DEFAULT_KEY_TAG_PATTERN);
		} 
		catch (Exception e)
		{
			e.printStackTrace();
		}
	}

	public synchronized static ShardedJedis getConnection()
	{
		try
		{
			if (jedisPool != null)
			{
				ShardedJedis resource = jedisPool.getResource();
				return resource;
			} else
			{
				return null;
			}
		}
		catch (Exception e)
		{
			e.printStackTrace();
			return null;
		}
	}

	public static void returnResource(final ShardedJedis jedis)
	{
		if (jedis != null)
		{
			jedis.close();
		}
	}
	
	public static void main(String[] args)
	{
		initJedisPool();
		ShardedJedis redis = getConnection();
		redis.set("key10", "j");
		redis.set("key11", "k");
		redis.set("key12", "l");
		redis.set("key13", "m");
		redis.set("key14", "n");
		
		System.out.print(redis.get("key12"));
		
		returnResource(redis);
	}
}

Jedis连接Redis3 Cluster

1、源码如下

package com.djhu.redis.test;

import java.util.Set;
import java.util.HashSet;

import redis.clients.jedis.HostAndPort;
import redis.clients.jedis.JedisCluster;

public class JedisClusterTest
{
	public static void main(String[] args)
	{
		Set<HostAndPort> jedisClusterNodes = new HashSet<HostAndPort>();  
        jedisClusterNodes.add(new HostAndPort("172.16.172.4", 6379));  
        jedisClusterNodes.add(new HostAndPort("172.16.172.4", 6380));  
        jedisClusterNodes.add(new HostAndPort("172.16.172.4", 6381));  
        jedisClusterNodes.add(new HostAndPort("172.16.172.4", 6382));  
        jedisClusterNodes.add(new HostAndPort("172.16.172.4", 6383));  
        jedisClusterNodes.add(new HostAndPort("172.16.172.4", 7384));  
        
		//JedisCluster cluster = new JedisCluster(jedisClusterNodes,3000,1000);
        JedisCluster cluster = new JedisCluster(jedisClusterNodes);
		cluster.set("key10", "j");
		cluster.set("key11", "k");
		cluster.set("key12", "l");
		cluster.set("key13", "m");
		cluster.set("key14", "n");
		
		System.out.println(cluster.get("key12"));
		
	}
}

2、如果遇到下面错误,主要是因为建立cluster时,ip用了127.0.0.1。用其他ip重建一下cluster,就可以解决了。

Exception in thread "main" redis.clients.jedis.exceptions.JedisClusterMaxRedirectionsException: Too many Cluster redirections?
	at redis.clients.jedis.JedisClusterCommand.runWithRetries(JedisClusterCommand.java:34)
	at redis.clients.jedis.JedisClusterCommand.runWithRetries(JedisClusterCommand.java:68)
	at redis.clients.jedis.JedisClusterCommand.runWithRetries(JedisClusterCommand.java:85)
	at redis.clients.jedis.JedisClusterCommand.runWithRetries(JedisClusterCommand.java:68)
	at redis.clients.jedis.JedisClusterCommand.runWithRetries(JedisClusterCommand.java:85)
	at redis.clients.jedis.JedisClusterCommand.runWithRetries(JedisClusterCommand.java:68)
	at redis.clients.jedis.JedisClusterCommand.run(JedisClusterCommand.java:29)
	at redis.clients.jedis.JedisCluster.set(JedisCluster.java:75)

Redis3配置Cluster

1、redis3的cluster是基于ruby的,所以要安装好ruby,然后安装redis的gem

gem install redis

2、然后配置6份redis,修改配置如下

节点
dbm6379 dbm6380 dbm6381 dbs6382 dbm6383 dbm6384
配置文件 dbm6379/conf/redis.conf dbm6380/conf/redis.conf dbm6381/conf/redis.conf dbs6382/conf/redis.conf dbm6383/conf/redis.conf dbm6384/conf/redis.conf
port 6379 6380 6381 6382 6383 6384
logfile “/home/neohope/DB/redis-3.0.4/cluster/dbm6379/logs/redis_log.log” “/home/neohope/DB/redis-3.0.4/cluster/dbm6380/logs/redis_log.log” “/home/neohope/DB/redis-3.0.4/cluster/dbm6381/logs/redis_log.log” “/home/neohope/DB/redis-3.0.4/cluster/dbm6382/logs/redis_log.log” “/home/neohope/DB/redis-3.0.4/cluster/dbm6383/logs/redis_log.log” “/home/neohope/DB/redis-3.0.4/cluster/dbm6384/logs/redis_log.log”
dir “/home/neohope/DB/redis-3.0.4/cluster/dbm6379/data” “/home/neohope/DB/redis-3.0.4/cluster/dbm6380/data” “/home/neohope/DB/redis-3.0.4/cluster/dbm6381/data” “/home/neohope/DB/redis-3.0.4/cluster/dbm6382/data” “/home/neohope/DB/redis-3.0.4/cluster/dbm6383/data” “/home/neohope/DB/redis-3.0.4/cluster/dbm6384/data”
cluster-enabled yes yes yes yes yes yes
cluster-config-file nodes-6379.conf nodes-6380.conf nodes-6381.conf nodes-6382.conf nodes-6383.conf nodes-6384.conf
cluster-node-timeout 15000 15000 15000 15000 15000 15000
cluster-migration-barrier 1 1 1 1 1 1
cluster-require-full-coverage yes yes yes yes yes yes

3、启动redis

#!/bin/sh
~/DB/redis-3.0.4/bin/redis-server ~/DB/redis-3.0.4/cluster/dbm6379/conf/redis.conf & echo $! & ~/DB/redis-3.0.4/bin/redis-server ~/DB/redis-3.0.4/cluster/dbm6380/conf/redis.conf & echo $! & ~/DB/redis-3.0.4/bin/redis-server ~/DB/redis-3.0.4/cluster/dbm6381/conf/redis.conf & echo $! & ~/DB/redis-3.0.4/bin/redis-server ~/DB/redis-3.0.4/cluster/dbs6382/conf/redis.conf & echo $! & ~/DB/redis-3.0.4/bin/redis-server ~/DB/redis-3.0.4/cluster/dbs6383/conf/redis.conf & echo $! & ~/DB/redis-3.0.4/bin/redis-server ~/DB/redis-3.0.4/cluster/dbs6384/conf/redis.conf & echo $!

4、配置cluster

#!/bin/sh
#这里最好不要用127.0.0.1做地址
~/DB/redis-3.0.4/bin/redis-trib.rb create --replicas 1 127.0.0.1:6379 127.0.0.1:6380 127.0.0.1:6381 127.0.0.1:6382 127.0.0.1:6383 127.0.0.1:6384      

5、测试redis cluster

~/DB/redis-3.0.4/bin/redis-cli -c -p 6379 
set key01 a
set key02 b
set key03 c
set key04 d
dbsize
keys *
get key03

6、关闭redis

#!/bin/sh
~/DB/redis-3.0.4/bin/redis-cli -p 6379 shutdown
~/DB/redis-3.0.4/bin/redis-cli -p 6380 shutdown
~/DB/redis-3.0.4/bin/redis-cli -p 6381 shutdown
~/DB/redis-3.0.4/bin/redis-cli -p 6382 shutdown
~/DB/redis-3.0.4/bin/redis-cli -p 6383 shutdown
~/DB/redis-3.0.4/bin/redis-cli -p 6384 shutdown

参考:
redis cluster tutorial

Debian8 Jessie修改Terminal分辨率

昨天安装了Debian8 Jessie,禁用了gdb以后,发现Terminal分辨率变的超级低。
用以前的方法调整了一下,发现搞不定了,搞了大半天,终于找到了靠谱的方案:

1、修改/etc/default/grub
找到

#GRUB_GFXMODE=XXXXXX

修改该行,并新增一行,最后效果为

GRUB_GFXMODE=1024x768x24
GRUB_GFXPAYLOAD_LINUX=1024x768x24

2、修改/etc/grub.d/00_header
找到

if [ "x${GRUB_GFXMODE}" = "x" ] ; then auto ; fi,

修改该行,并新增一行,最后效果为

if [ "x${GRUB_GFXMODE}" = "x" ] ; then GRUB_GFXMODE=1024x768x24 ; fi
if [ "x${GRUB_GFXPAYLOAD}" = "x" ] ; then GRUB_GFXPAYLOAD=1024x768x24 ; fi

3、仍然是etc/grub.d/00_header
找到

set gfxmode=${GRUB_GFXMODE}

在其后新增一行,最后效果为

set gfxmode=${GRUB_GFXMODE}
set gfxpayload=${GRUB_GFXPAYLOAD}

4、更新grub并重启,搞定

update-grub2
reboot

5、感觉Debian8 Jessie的配置改变了好多。。。

Redis主从数据库(Shell)

1、基本配置如下

数据库 master slave01 slave02
配置文件 redis.master.conf redis.slave01.conf redis.slave02.conf
ip地址 localhost localhost localhost
端口 6379 6380 6381
logfile “D:/Database/Redis2.8/mirror/master/logs/redis_log.txt” “D:/Database/Redis2.8/mirror/slave01/logs/redis_log.txt” “D:/Database/Redis2.8/mirror/slave02/logs/redis_log.txt”
dir “D:/Database/Redis2.8/mirror/master/data/” “D:/Database/Redis2.8/mirror/slave01/data/” “D:/Database/Redis2.8/mirror/slave02/data/”
slaveof localhost 6379 localhost 6379
slave-serve-stale-data yes yes
slave-read-only yes yes
slave-priority 100 100
maxheap 1073741824 1073741824
heapdir D:\Database\Redis2.8\mirror\slave01\heap D:\Database\Redis2.8\mirror\slave02\heap

2、启动

redis-server.exe D:\Database\Redis2.8\mirror\redis.master.conf
redis-server.exe D:\Database\Redis2.8\mirror\redis.slave01.conf
redis-server.exe D:\Database\Redis2.8\mirror\redis.slave02.conf

3、测试

D:\Database\Redis2.8>redis-cli
127.0.0.1:6379> dbsize
(integer) 0
127.0.0.1:6379> set key01 a
OK
127.0.0.1:6379> exit

D:\Database\Redis2.8>redis-cli -p 6380
127.0.0.1:6380> get key01
"a"
127.0.0.1:6380> set key02 b
(error) READONLY You can't write against a read only slave.
127.0.0.1:6380> exit

D:\Database\Redis2.8>redis-cli -p 6381
127.0.0.1:6381> get key01
"a"
127.0.0.1:6381> set key02 b
(error) READONLY You can't write against a read only slave.
127.0.0.1:6381> exit

4、关闭

redis-cli -p 6380 shutdown
redis-cli -p 6381 shutdown
redis-cli -p 6379 shutdown

CentOS7常见操作

1、通过CentOS光盘安装软件

#挂载光盘
mount -r /dev/sr0 /media/cdrom

#修改文件,将光盘查找路径定为/media/cdrom
vi /etc/yum.repos.d/CentOS-Media.repo

#刷新列表
yum --disablerepo=\* --enablerepo=c7-media repolist

#安装vim
yum --disablerepo=\* --enablerepo=c7-media install vim

#通过命令查找包并安装包
yum --disablerepo=\* --enablerepo=c7-media provides ifconfig
yum --disablerepo=\* --enablerepo=c7-media install net-tools

#卸载包
yum erase 包名

2、rpm安装及卸载软件

#安装
rpm -ivh rpm文件路径

#安装并不进行依赖检查
rpm -ivh --nodeps rpm文件路径

#更新
rpm -Uvh rpm文件路径

#删除
rpm -e 包名

#查看包的用途、版本等信息
rpm -qpi rpm文件路径

#查看包的文件清单
rpm -qpl rpm文件路径

#查看软件包文档位置
rpm -qpd rpm文件路径

#查看软件包的配置文件
rpm -qpc rpm文件路径

#查看软件包依赖关系
rpm -qpR rpm文件路径

3、rpm查看已安装软件包信息

#查看已经安装的软件
rpm -qa
 
#查看一个文件属于哪个软件包
rpm -qf 文件绝对路径

#查看安装软件包都安装到何处;
rpm -ql 包名

#查看包的信息
rpm  -qi 包名

#查看包配置文件
rpm -qc 包名

#查看包文档安装位置
rpm -qd 包名

#查看包的依赖
rpm -qR 包名

4、配置网络为静态地址

vim /etc/sysconfig/network-scripts/ifcfg-enp0s3
#修改或新增以下内容
BOOTPROTO=static
ONBOOT=yes
IPADDR=172.16.172.80
NETMASK=255.255.0.0
GATEWAY=172.16.172.2

#重启网络
service network restart

5、防火墙配置

#查看状态
systemctl status firewalld 

#启动
systemctl start firewalld

#关闭
systemctl stop firewalld

#开机启用
systemctl enable firewalld

#开机禁用 
systemctl disable firewalld

互联网后面10年的重点

当今互联网,已经基本解决了吃、穿、用、住、行、娱乐的问题。

下馆子吃,已经被大众点评、美团、饿了么等刮分完毕。
不下馆子吃、穿、用,各大商城都在做,被淘宝、京东、亚马逊、一号店等刮分完毕。
住、行被携程、艺龙、滴滴快的、神州等刮分完毕。

旅游类的娱乐项目,很大一部分开支为吃、住、行。除去这一部分,组团、自由行等也已经慢慢向成熟市场变化,包括马蜂窝等很多都在做。
非旅游类的娱乐项目,在大众点评、同城等也有了立足之地,更何况这铺天盖地的购票软件咯。

上述市场,一些已经局势明朗很难有大的变化,一些仍是杀得难解难开,一些还待开发。

依据互联网的长尾效应,后面大家要做的是差异化竞争,现在国内互联网巨头中,还没有特别好的、在红海中的差异化竞争案例。大家刚刚从实体店的红海杀到了互联网的蓝海,并正在把互联网蓝海杀红。我相信这方面还是有很多可以做的事情的。

另外,当一个人吃、穿、用、住、行、娱乐的问题解决了,就会有更高的追求。按以往的历史规律,医疗、健康、教育、艺术、科技、政治的需求会逐渐增强。而且,当下各大公司正在努力向医疗领域进军,互联网这10年,应该是医疗健康的十年。

Hadoop增删改查(Java)

需要的jar包在hadoop里都可以找到,下面的例子中,至少需要这些jar包:

commons-cli-1.2.jar
commons-collections-3.2.1.jar
commons-configuration-1.6.jar
commons-io-2.4.jar
commons-lang-2.6.jar
commons-logging-1.1.3.jar
guava-11.0.2.jar
hadoop-auth-2.7.1.jar
hadoop-common-2.7.1.jar
hadoop-hdfs-2.7.1.jar
htrace-core-3.1.0-incubating.jar
log4j-1.2.17.jar
protobuf-java-2.5.0.jar
servlet-api.jar
slf4j-api-1.7.10.jar
slf4j-log4j12-1.7.10.jar

代码如下:

package com.neohope.hadoop.test;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

public class HDFSTest {

	static Configuration hdfsConfig;
	static {
		hdfsConfig = new Configuration();
		hdfsConfig.addResource(new Path("etc/hadoop/core-site.xml"));
		hdfsConfig.addResource(new Path("etc/hadoop/hdfs-site.xml"));
	}

	// 创建文件夹
	public static void createDirectory(String dirPath) throws IOException {
		FileSystem fs = FileSystem.get(hdfsConfig);
		Path p = new Path(dirPath);
		try {
			fs.mkdirs(p);
		} finally {
			fs.close();
		}
	}

	// 删除文件夹
	public static void deleteDirectory(String dirPath) throws IOException {
		FileSystem fs = FileSystem.get(hdfsConfig);
		Path p = new Path(dirPath);
		try {
			fs.deleteOnExit(p);
		} finally {
			fs.close();
		}
	}

	// 重命名文件夹
	public static void renameDirectory(String oldDirPath, String newDirPath)
			throws IOException {
		renameFile(oldDirPath, newDirPath);
	}

	// 枚举文件
	public static void listFiles(String dirPath) throws IOException {
		FileSystem hdfs = FileSystem.get(hdfsConfig);
		Path listf = new Path(dirPath);
		try {
			FileStatus statuslist[] = hdfs.listStatus(listf);
			for (FileStatus status : statuslist) {
				System.out.println(status.getPath().toString());
			}
		} finally {
			hdfs.close();
		}
	}

	// 新建文件
	public static void createFile(String filePath) throws IOException {
		FileSystem fs = FileSystem.get(hdfsConfig);
		Path p = new Path(filePath);
		try {
			fs.createNewFile(p);
		} finally {
			fs.close();
		}
	}

	// 删除文件
	public static void deleteFile(String filePath) throws IOException {
		FileSystem fs = FileSystem.get(hdfsConfig);
		Path p = new Path(filePath);
		try {
			fs.deleteOnExit(p);
		} finally {
			fs.close();
		}
	}

	// 重命名文件
	public static void renameFile(String oldFilePath, String newFilePath)
			throws IOException {
		FileSystem fs = FileSystem.get(hdfsConfig);
		Path oldPath = new Path(oldFilePath);
		Path newPath = new Path(newFilePath);
		try {
			fs.rename(oldPath, newPath);
		} finally {
			fs.close();
		}
	}

	// 上传文件
	public static void putFile(String locaPath, String hdfsPath)
			throws IOException {
		FileSystem fs = FileSystem.get(hdfsConfig);
		Path src = new Path(locaPath);
		Path dst = new Path(hdfsPath);
		try {
			fs.copyFromLocalFile(src, dst);
		} finally {
			fs.close();
		}
	}

	// 取回文件
	public static void getFile(String hdfsPath, String locaPath)
			throws IOException {
		FileSystem fs = FileSystem.get(hdfsConfig);
		Path src = new Path(hdfsPath);
		Path dst = new Path(locaPath);
		try {
			fs.copyToLocalFile(false, src, dst, true);
		} finally {
			fs.close();
		}
	}

	// 读取文件
	public static void readFile(String hdfsPath) throws IOException {
		FileSystem hdfs = FileSystem.get(hdfsConfig);
		Path filePath = new Path(hdfsPath);

		InputStream in = null;
		BufferedReader buff = null;
		try {
			in = hdfs.open(filePath);
			buff = new BufferedReader(new InputStreamReader(in));
			String str = null;
			while ((str = buff.readLine()) != null) {
				System.out.println(str);
			}
		} finally {
			buff.close();
			in.close();
			hdfs.close();
		}
	}

	public static void main(String[] args) throws IOException {
		System.setProperty("HADOOP_USER_NAME", "hadoop");
		// createDirectory("hdfs://hadoop-master:9000/usr");
		// createDirectory("hdfs://hadoop-master:9000/usr/hansen");
		// createDirectory("hdfs://hadoop-master:9000/usr/hansen/test");
		// renameDirectory("hdfs://hadoop-master:9000/usr/hansen/test","hdfs://hadoop-master:9000/usr/hansen/test01");
		// createFile("hdfs://hadoop-master:9000/usr/hansen/test01/hello.txt");
		// renameFile("hdfs://hadoop-master:9000/usr/hansen/test01/hello.txt","hdfs://hadoop-master:9000/usr/hansen/test01/hello01.txt");
		// putFile("hello.txt","hdfs://hadoop-master:9000/usr/hansen/test01/hello02.txt");
		// getFile("hdfs://hadoop-master:9000/usr/hansen/test01/hello02.txt","hello02.txt");
		// readFile("hdfs://hadoop-master:9000/usr/hansen/test01/hello02.txt");
		listFiles("hdfs://hadoop-master:9000/usr/hansen/test01/");
	}

}

Debian默认命令行启动

由于Debian6和Debian8服务管理方式不同,很多旧方法不再适用于Debian8

1、Debian6之前:修改内核启动参数
1.1、修改文件/etc/default/grub

GRUB_CMDLINE_LINUX_DEFAULT="quiet"

GRUB_CMDLINE_LINUX_DEFAULT="quiet text"

1.2、然后运行

update-grub

2、Debian8之后:systemctl

#切换为命令行登录
systemctl set-default multi-user.target
#退出GUI
gnome-session-quit

#切换为GUI登录
systemctl set-default graphical

3、Grub禁用图像模式
3.1、修改文件/etc/default/grub

#GRUB_TERMINAL=console

GRUB_TERMINAL=console

3.2、然后运行

update-grub